#!/bin/sh

MACHINE_FILE=""
NUMBER_OF_WORKERS=""
PRG=""
PRG_ARGS=""
HN=$(hostname)
TMP_PATTERN="/tmp/.beegfs-copy.XXXXXXXX"
TMP_MACHINE_FILE=$(mktemp ${TMP_PATTERN})
UNIQ_MACHINE_FILE=$(mktemp ${TMP_PATTERN})
ORIG_MACHINE_FILE=""
NNODES=0
GASPI_LAUNCHER="ssh"
COPY_ARGS=""

#helper functions
remove_temp_file()
{
    rm -f $TMP_MACHINE_FILE
    rm -f $UNIQ_MACHINE_FILE
}

usage()
{
    echo
    echo "BeeGFS-copy -- v8.2.2"
    echo "Usage: beegfs-copy -m <machinefile> [INITIALIZATION OPTIONS] -t <# Threads per node> [COPYING OPTIONS] SOURCE(S) DEST_DIRECTORY"
    echo
    echo "RECOMMENDATION: Do not modify the directory trees during the copy"
    echo "process to avoid inconsistent copying of the data!"
    echo
    echo "Required parameters"
    echo "  -m <machinefile>         With the list of the names of the nodes."
    echo "  -t <# Threads per node>  Number of threads per node."
    echo
    echo "Available initialization options:"
    echo "  -n <nodes>       Start as many <nodes> from machine file."
    echo "  -h               This help."
    echo
    echo "Available copying options:"
    echo "  -a               Access time of the source file(s) remains unmodified."
    echo "                   (Note that not all file systems support this feature or might be configured to not use it.)"
    echo "  -c               Chunk size for copy operations (in MB)."
    echo "  -k               Keep the modification time of the source in the destination."
    echo "  -p               Partition copy threshold (in MB)."
    echo
    echo "Other features:"
    echo "  -l               List differences: comparing source and destination paths, checking for missing files"
    echo "                   and directories, files with different sizes or older modification times in the destination."
    echo "                   No changes made. Only one source path accepted. The destination can be either a file or a"
    echo "                   directory, if destination is a file the source must be a file as well."
    echo
    echo "  -u               Update: compare source and destination paths, updating the destination by"
    echo "                   copying only missing files and directories, files with different sizes or older"
    echo "                   modification times in the destination."
    echo "                   Only one source path accepted."
    echo "                   The destination can be either a file or a directory, if the destination is a file the"
    echo "                   source must be a file as well."
    echo
    echo "Additional options:"
    echo "  -d               Optional condition for modification time for -l and -u features: files"
    echo "                   with different modification time between source will tagged as different."
    echo "                   (default condition for modification time: only if the source is newer"
    echo "                   than the destination)."
    echo
    echo "Global options:"
    echo "  -s               Print thread statistics."
    echo "  -v <level>       Verbose mode, <level> optional [1-3]."
    echo
    remove_temp_file
}

check_env_var()
{
    var_name="$1"
    eval "value=\$$var_name"
    if [ -n "$value" ]; then
        return 0
    else
        return 1
    fi
}

cleanup()
{
    if [ $1 -eq -1 ]; then
        kill_procs
    fi
    remove_temp_file
}

print_error_exit()
{
    echo
    echo "Error: $1"
    echo
    remove_temp_file
    exit 1
}

validate_machinefile()
{
    if [ ! -s $MACHINE_FILE ]; then
        print_error_exit "Empty machine file ($1)"
    fi

    #newline at the end
    endl=`tail -c 1 $MACHINE_FILE`
    if [ "$endl" != "" ]; then
        print_error_exit "No newline at end of machine file ($1)"
    fi

    #create tmp_file
    rm -f $TMP_MACHINE_FILE

    touch $TMP_MACHINE_FILE 2>/dev/null && chmod 777 $MACHINE_FILE ||
    {
        print_error_exit "User permissions failure: $PWD is not writable"
    }

    ncount=0
    previous=""
    while read LINE;
    do
        i=$LINE

        if [ "$i" = "$previous" ]; then
            echo
            echo "Error: incorrect machine file (-m $ORIG_MACHINE_FILE) "
            echo "a hostname $i is listed twice"
            echo "you can define multiple processes per node with beegfs-copy parameter [-t])"
            echo
            cleanup 1
            exit 1
        fi

        previous=$i
        if [ -n "$i" ]; then
            ping -c 1 $i >/dev/null 2>&1 ||
            {
                print_error_exit "Host not reachable ($i)"
            }

            ncount=$((ncount + 1))
            if [ $NNODES -lt $ncount ]; then
                break
            fi
            echo $i >>$TMP_MACHINE_FILE
        fi
    done <$MACHINE_FILE

    ORIG_MACHINE_FILE=$MACHINE_FILE
    MACHINE_FILE=$TMP_MACHINE_FILE
    uniq $MACHINE_FILE >$UNIQ_MACHINE_FILE
    #number of nodes (NNODES) must fit number of hosts
    n=$(wc -l <$MACHINE_FILE)
    if [ $n -lt $NNODES ]; then
        print_error_exit "Not enough hosts ($n) for required number of nodes (-n $NNODES)"
    fi
}

kill_procs()
{
    echo "Killing procs..."
    for i in $(cat $UNIQ_MACHINE_FILE);
    do
        $GASPI_LAUNCHER $i "nohup killall -9 ${PRG} $(cat /dev/null)" >/dev/null 2>&1 &
    done
    remove_temp_file
    wait
    exit 1
}

if [ "$#" -eq 0 ]; then
    usage
    exit 1
fi

#command line parsing
while [ -n "$1" ]; do
    case $1 in
        -a)
            COPY_ARGS="${COPY_ARGS} $1"
            shift
            ;;
        -c)
            COPY_ARGS="${COPY_ARGS} $1 $2"
            if [ -z "$2" ]; then
                print_error_exit "Missing parameter for flag $1"
            fi
            shift
            shift
            ;;
        -d)
            COPY_ARGS="${COPY_ARGS} $1"
            shift
            ;;
        -h | --help)
            usage
            exit 0
            ;;
        -k)
            COPY_ARGS="${COPY_ARGS} $1"
            shift
            ;;
        -l)
            COPY_ARGS="${COPY_ARGS} $1"
            shift
            ;;
        -m | --machinefile)
            shift
            if [ -r $1 ]; then
                MACHINE_FILE=$1
            else
                print_error_exit "Cannot read $1 (-m option) (or file does not exist)"
            fi
            shift
            ;;
        -n | --nodes)
            shift
            NNODES=$1
            shift
            ;;
        -p)
            COPY_ARGS="${COPY_ARGS} $1 $2"
            if [ -z "$2" ]; then
                print_error_exit "Missing parameter for flag $1"
            fi
            shift
            shift
            ;;
        -s)
            COPY_ARGS="${COPY_ARGS} $1"
            shift
            ;;
        -t)
            COPY_ARGS="${COPY_ARGS} $1 $2"
            if [ -z "$2" ]; then
                print_error_exit "Missing parameter for flag $1"
            else
                NUMBER_OF_WORKERS=$2
            fi
            shift
            shift
            ;;
        -u)
            COPY_ARGS="${COPY_ARGS} $1"
            shift
            ;;
        -v)
            if [ -n "$2" ] && [ "$(echo "$2" | cut -c1)" != "-" ]; then
                COPY_ARGS="${COPY_ARGS} $1 $2"
                shift
            else
                #no argument provided, use default 1
                COPY_ARGS="${COPY_ARGS} $1 1"
            fi
            shift
            ;;
        -*)
            echo ""
            usage
            print_error_exit "Invalid flag ($1) option!"
            ;;
        *)
            case $1 in
                /*)
                    PATHS="${PATHS} $1"
                    ;;
                *)
                    PATHS="${PATHS} $PWD/$1"
                    ;;
            esac
            shift
    esac
done

if [ "${MACHINE_FILE}" = "" ]; then
    print_error_exit "Machine file is missing!"
fi

if [ "${NUMBER_OF_WORKERS}" = "" ]; then
    print_error_exit "Number of threads is missing!"
fi

bin_location=$(readlink -f `dirname $0`)
PRG=$bin_location/beegfs-copy.bin

PRG_ARGS="${COPY_ARGS}"
PRG_ARGS="${PRG_ARGS} ${PATHS}"

if [ ! -f "$PRG" ]; then
    print_error_exit "Could not find beegfs-copy.bin in ${bin_location}"
fi

trap kill_procs TERM INT QUIT

#use all host in machines file
if [ $NNODES -eq 0 ]; then
    NNODES=`cat $MACHINE_FILE | wc -l`
fi

validate_machinefile $MACHINE_FILE

if [ ! -x ${bin_location}/ssh.spawner ]; then
    print_error_exit "The required spawner is missing (or not executable)"
fi

master_node=`head -n 1 $MACHINE_FILE`
previous="$master_node"
machine_file_location=$(readlink -f $MACHINE_FILE)

node=1
local_rank=0

for HOST in $(tail -n +2 $MACHINE_FILE);
do
    cmd="${bin_location}/ssh.spawner ${master_node} 0 ${machine_file_location} 0"
    cmd="$cmd $node $local_rank $NNODES"
    cmd="$cmd 0" # set GPI NUMA flag to false
    cmd="$cmd ${PRG} ${PRG_ARGS}"
    if [ ! -z "$SHELL" ]; then
        if [ $SHELL = "/bin/tcsh" ]; then
            $GASPI_LAUNCHER $HOST "/bin/sh; nohup $cmd $(cat /dev/null)" &
            >/dev/stdout &
        else
            $GASPI_LAUNCHER $HOST "nohup $cmd $(cat /dev/null)"  &
        fi
    else
        $GASPI_LAUNCHER $HOST "nohup $cmd $(cat /dev/null)" &
    fi
    previous="$HOST"
    node=$((node+1))
done

#start master
if [ "$master_node" != "$HN" ] && [ "$master_node" != "localhost" ]; then
    chmod a+r $MFILE >/dev/null 2>&1
    TMP_FILE=`readlink -f $MACHINE_FILE`
    MACHINE_FILE=$TMP_FILE
    scp $MACHINE_FILE ${master_node}:${MACHINE_FILE} >/dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo "Warning: Failed to copy machinefile to remote host. Program might not start correctly."
    fi

    cmd="/bin/sh -c"
    cmd="$cmd 'export GASPI_MASTER=$master_node"
    cmd="$cmd; export GASPI_SOCKET=0"
    cmd="$cmd; export GASPI_TYPE=GASPI_MASTER"
    cmd="$cmd; export GASPI_MFILE=$MACHINE_FILE"
    cmd="$cmd; export GASPI_RANK=0"
    cmd="$cmd; export GASPI_NRANKS=$NNODES"
    if check_env_var "BEEGFS_LICENSED_FEATURE"; then
        cmd="$cmd; export BEEGFS_LICENSED_FEATURE=$BEEGFS_LICENSED_FEATURE"
    fi

    if check_env_var "BEEGFS_BINARY_NAME"; then
        cmd="$cmd; export BEEGFS_BINARY_NAME=$BEEGFS_BINARY_NAME"
    fi
    if check_env_var "BEEGFS_COPY_LICENSE_PATH"; then
        cmd="$cmd; export BEEGFS_COPY_LICENSE_PATH=$BEEGFS_COPY_LICENSE_PATH"
        echo "BEEGFS_COPY_LICENSE_PATH found"
    fi
    cmd="$cmd; $PRG $PRG_ARGS'"
    $GASPI_LAUNCHER $master_node $cmd
else
    export GASPI_MASTER=$HN
    export GASPI_SOCKET=0
    export GASPI_TYPE=GASPI_MASTER
    export GASPI_MFILE=$MACHINE_FILE
    export GASPI_RANK=0
    export GASPI_NRANKS=$NNODES
    if check_env_var "BEEGFS_LICENSED_FEATURE"; then
        export BEEGFS_LICENSED_FEATURE=$BEEGFS_LICENSED_FEATURE
    fi
    if check_env_var "BEEGFS_BINARY_NAME"; then
        export BEEGFS_BINARY_NAME=$BEEGFS_BINARY_NAME
    fi
    if check_env_var "BEEGFS_COPY_LICENSE_PATH"; then
        export BEEGFS_COPY_LICENSE_PATH=$BEEGFS_COPY_LICENSE_PATH
    fi
    $PRG $PRG_ARGS
fi

binary_return_value=$?
if [ $binary_return_value -lt 0 ]; then
    echo "Error: Failed to start $PRG on $master_node"
    cleanup $binary_return_value
    exit $binary_return_value
fi

wait

cleanup 0

exit $binary_return_value
