Commit 91abb7a3 authored by Guillaume REMBERT's avatar Guillaume REMBERT

ZFEC support as ECC engine due to issue between libcurlftpfs and par2 +...

ZFEC support as ECC engine due to issue between libcurlftpfs and par2 + parallel de/compression support
parent 5ea229a4
......@@ -9,13 +9,13 @@ EURYBOX_CORPORATE="My Corporate"
#Details level on script execution
#Can be: debug, verbose, info, warning or error
EURYBOX_LOG_LEVEL="debug"
EURYBOX_LOG_LEVEL="info"
#Parallelism depth / depends on host CPU
EURYBOX_PARALLELISM_LEVEL="8"
#Max retry on failure before stopping execution
EURYBOX_MAX_RETRY_ON_FAILURE="5"
#Network calls timeout (in seconds)
EURYBOX_NETWORK_TIMEOUT="2"
EURYBOX_NETWORK_TIMEOUT="1"
#Version
EURYBOX_VERSION="1.0"
......@@ -95,21 +95,27 @@ EURYBOX_BACKUP_TARGETS=(
#Backup archive
#FORMAT= tar/tar.gz/tar.bz2
#HASH= sha256/sha512
#ECC= par2
#ECC_LEVEL = 0/1/.../99/100
#FEC= par2/zfec
#FEC_FILE_NUM = 1/2/...
#FEC_LEVEL = 0/1/.../99/100
#FEC_FILE_NUM_MIN = 1/2/... (<= FEC_FILE_NUM)
#ENC_ALGO= openssl supported algo (aes-256-xts/aes-256-ctr/...)
EURYBOX_BACKUP_ARCHIVE=(
[NAME]="nodeXX_archive"
[OWNER]="root:root"
[MASK]="400"
[LABEL]="backup_u:object_r:backup_t:s0"
[FORMAT]="tar"
[FORMAT]="tar.bz2"
[HASH]="sha512"
[ECC]="par2"
[ECC_LEVEL]="10"
[ENCRYPT]="true"
[ENC_ALGO]="aes-256-ctr"
[PASSWORD]="MyStrongEncryptionPassword"
[FEC]="zfec"
[FEC_FILE_NUM]="10"
#FOR PAR2 ONLY
[FEC_LEVEL]="10"
#FOR ZFEC ONLY
[FEC_FILE_NUM_MIN]="5"
)
#Backup destination
......
......@@ -54,7 +54,7 @@ case $EURYBOX_BACKUP_TYPE in
#Create archive file
euryboxctrl_create_arch
#Re-Start all VMs
euryboxctrl_start_services
# euryboxctrl_start_services
#Create errors correcting codes files
euryboxctrl_create_eccf
#Create description file
......
......@@ -28,7 +28,7 @@ else
exit 1
fi
eurybox_message_display message STATUS "Checking configuration"
eurybox_check_configuration BACKUP
eurybox_check_configuration EXEC_COMMAND
#Check input parameters
eurybox_message_display message STATUS "Checking parameters"
......@@ -36,19 +36,18 @@ eurybox_check_arguments $SCRIPT_ARGUMENTS
#Check environment
eurybox_message_display message STATUS "Checking environment"
eurybox_check_environment BACKUP
eurybox_message_display message STATUS "$EURYBOX_CORPORATE exec commands script started"
eurybox_check_environment EXEC_COMMAND
eurybox_message_display message STATUS "$EURYBOX_CORPORATE execute commands script started"
#Start all VMs
euryboxctrl_start_services
#Execution commands on virtual machines / services
#Execute commands on virtual machines / services
euryboxctrl_exec_command_services
#Finished
END_TIME=$(date +%s)
DIFF_TIME=$(( $END_TIME - $START_TIME ))
eurybox_message_display message STATUS "Command exec finished"
eurybox_message_display message TIMING "Command exec took $DIFF_TIME seconds"
eurybox_message_display message STATUS "Commands execution finished"
eurybox_message_display message TIMING "Commands execution took $DIFF_TIME seconds"
......@@ -118,7 +118,7 @@ eurybox_backup_mount_target ()
if [[ !($STATUS -eq 0) ]]
then
NB_TRY_LEFT=$(( $NB_TRY_LEFT - 1 ))
eurybox_display_message warning BACKUP "Backup target SSHFS mount failed and target folder is empty - trying to mount again in 5 seconds - error $STATUS:\n$MOUNT_OUT\n$NB_TRY_LEFT left"
eurybox_display_message warning BACKUP "Backup target SSHFS mount failed - trying to mount again in 5 seconds - error $STATUS:\n$MOUNT_OUT\n$NB_TRY_LEFT left"
sleep 5
fi
fi
......@@ -139,7 +139,7 @@ eurybox_backup_mount_target ()
if [[ !($STATUS -eq 0) ]]
then
NB_TRY_LEFT=$(( $NB_TRY_LEFT - 1 ))
eurybox_display_message warning BACKUP "Backup target FTPFS mount failed and target folder is empty - trying to mount again in 5 seconds - error $STATUS:\n$MOUNT_OUT\n$NB_TRY_LEFT left"
eurybox_display_message warning BACKUP "Backup target FTPFS mount failed - trying to mount again in 5 seconds - error $STATUS:\n$MOUNT_OUT\n$NB_TRY_LEFT left"
sleep 5
fi
fi
......
......@@ -125,7 +125,7 @@ eurybox_check_configuration ()
EURYBOX_SSH_OPTIONS="-oPasswordAuthentication=no -oConnectTimeout=$EURYBOX_NETWORK_TIMEOUT -oConnectionAttempts=$EURYBOX_MAX_RETRY_ON_FAILURE"
EURYBOX_NMAP_OPTIONS="--host-timeout $EURYBOX_NETWORK_TIMEOUT --max-retries $EURYBOX_MAX_RETRY_ON_FAILURE"
EURYBOX_LS_OPTIONS="-x"
EURYBOX_MNT_NFS_OPTIONS="-o hard,retry=$EURYBOX_MAX_RETRY_ON_FAILURE,rsize=8192,wsize=8192,timeo=${EURYBOX_NETWORK_TIMEOUT}0"
EURYBOX_MNT_NFS_OPTIONS="-o hard,retry=$EURYBOX_MAX_RETRY_ON_FAILURE,timeo=${EURYBOX_NETWORK_TIMEOUT}0"
EURYBOX_MNT_SSHFS_OPTIONS="-o allow_root ${EURYBOX_SSH_OPTIONS}"
EURYBOX_MNT_FTPFS_OPTIONS="-o allow_root,connect_timeout=${EURYBOX_NETWORK_TIMEOUT}"
case $EURYBOX_LOG_LEVEL in
......@@ -147,14 +147,14 @@ eurybox_check_configuration ()
EURYBOX_MNT_NFS_OPTIONS+=" -v"
EURYBOX_MNT_SSHFS_OPTIONS+=" -oLogLevel=verbose"
EURYBOX_MNT_FTPFS_OPTIONS+=" -v"
EURYBOX_VIRSH_OPTIONS+=" -q -d 4"
EURYBOX_VIRSH_OPTIONS+=" -q -d 3"
;;
info )
EURYBOX_SSH_OPTIONS+=" -oLogLevel=info"
EURYBOX_NMAP_OPTIONS+=" -d0 -v"
EURYBOX_LS_OPTIONS+=" -l"
EURYBOX_MNT_SSHFS_OPTIONS+=" -oLogLevel=info"
EURYBOX_VIRSH_OPTIONS+=" -q -d 6"
EURYBOX_VIRSH_OPTIONS+=" -q -d 4"
;;
warning )
EURYBOX_SSH_OPTIONS+=" -oLogLevel=info"
......@@ -261,7 +261,7 @@ eurybox_check_environment ()
local BKP_SPACE_LEFT
local BKP_ACCESS_OK
#Check external sotfware used
eurybox_check_command grep sudo ssh openssl tar par2 virsh sha512sum sha256sum awk pigz df tail virt-sysprep tput nmap rpcinfo pbzip2 sshfs curlftpfs lftp cryptsetup
eurybox_check_command grep sudo ssh openssl tar par2 zfec virsh sha512sum sha256sum awk df tail virt-sysprep tput nmap rpcinfo pigz pbzip2 sshfs curlftpfs lftp cryptsetup
#Check kernel capabilities
eurybox_check_kernel
#Check distribution is coherent with configuration
......
This diff is collapsed.
......@@ -71,13 +71,13 @@ eurybox_detect_archive_content ()
eurybox_display_message warning DETECT "Archive description version search failed - unknown backup version found: $BKP_VERS"
;;
esac
BKP_ECC=`echo "${ARCH_DESC}" | grep "ECC_FORMAT" | awk -F":" '{print $NF}'`
BKP_ECC=`echo "${ARCH_DESC}" | grep "FEC_FORMAT" | awk -F":" '{print $NF}'`
case $BKP_ECC in
"" )
eurybox_display_message warning DETECT "Archive description correcting codes search failed"
;;
par2 )
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,ECC]=$BKP_ECC
par2|zfec )
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,FEC]=$BKP_ECC
eurybox_display_message debug DETECT "Archive description correcting codes search success - found: $BKP_ECC"
;;
* )
......
......@@ -54,18 +54,10 @@ euryboxctrl_check_sign ()
euryboxctrl_create_arch ()
{
#Create the archive
eurybox_display_message message EURYBOXCTRL "Creating the archive file"
case "${EURYBOX_BACKUP_ARCHIVE[FORMAT]}" in
tar|tar.gz|tar.bz2 )
eurybox_create_archive_tar
;;
* )
eurybox_display_message error EURYBOXCTRL "Archive format unknown: ${EURYBOX_BACKUP_ARCHIVE[FORMAT]}."
;;
esac
eurybox_create_archive_tar
#Flush temporary directory
eurybox_flush_directory $EURYBOX_BACKUP_TMP_FOLDER
}
......@@ -81,28 +73,14 @@ euryboxctrl_create_eccf ()
{
#Create the error correcting file
eurybox_display_message message EURYBOXCTRL "Creating the error correcting codes files"
case "${EURYBOX_BACKUP_ARCHIVE[ECC]}" in
par2 )
eurybox_create_archive_par
;;
* )
eurybox_display_message error EURYBOXCTRL "Archive ECC type unknown: ${EURYBOX_BACKUP_ARCHIVE[ECC]}."
;;
esac
eurybox_create_archive_fec
}
euryboxctrl_create_hash ()
{
#Create the associated hash file
eurybox_display_message message EURYBOXCTRL "Creating the hash file"
case "${EURYBOX_BACKUP_ARCHIVE[HASH]}" in
sha256|sha512 )
eurybox_create_archive_sha
;;
* )
eurybox_display_message error EURYBOXCTRL "Archive HASH type unknown: ${EURYBOX_BACKUP_ARCHIVE[HASH]}."
;;
esac
eurybox_create_archive_hash
}
euryboxctrl_create_sign ()
......
......@@ -115,6 +115,8 @@ eurybox_display_detected_archives ()
local COMP=${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,COMP]}
local TYPE=${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,TYPE]}
local VERS=${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,VERS]}
local FEC=${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,FEC]}
local HASH=${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,HASH]}
local YEAR=`echo ${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,TIME]} | awk -F"_" '{ print $1 }'`
local MONTH=`echo ${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,TIME]} | awk -F"_" '{ print $2 }'`
local DAY=`echo ${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,TIME]} | awk -F"_" '{ print $3 }'`
......@@ -124,7 +126,7 @@ eurybox_display_detected_archives ()
if [[ $NAME != $PREVIOUS_NAME ]]
then
echo "------------ [[ ARCHIVE: $NAME ]] ------------"
echo "YEAR \\ MONTH \\ DAY @ HOUR:MINUTE:SECOND | TYPE | VERS | ENC | COMP | ID"
echo "YEAR \\ MONTH \\ DAY @ HOUR:MINUTE:SECOND | TYPE | VERS | ENCR | COMP | FECC | HASH | ID"
PREVIOUS_YEAR=""
PREVIOUS_MONTH=""
fi
......@@ -163,13 +165,13 @@ eurybox_display_detected_archives ()
esac
case $ENC in
false )
ARCH_DESC="$ARCH_DESC | "
ARCH_DESC="$ARCH_DESC | "
;;
true )
ARCH_DESC="$ARCH_DESC | X "
ARCH_DESC="$ARCH_DESC | X "
;;
* )
ARCH_DESC="$ARCH_DESC | ? "
ARCH_DESC="$ARCH_DESC | ? "
;;
esac
case $COMP in
......@@ -184,7 +186,31 @@ eurybox_display_detected_archives ()
;;
* )
ARCH_DESC="$ARCH_DESC | ? "
echo "$COMP"
;;
esac
case $FEC in
none )
ARCH_DESC="$ARCH_DESC | "
;;
par2 )
ARCH_DESC="$ARCH_DESC | PAR2"
;;
zfec )
ARCH_DESC="$ARCH_DESC | ZFEC"
;;
* )
ARCH_DESC="$ARCH_DESC | ? "
;;
esac
case $HASH in
none )
ARCH_DESC="$ARCH_DESC | "
;;
sha512|sha256 )
ARCH_DESC="$ARCH_DESC | SHA "
;;
* )
ARCH_DESC="$ARCH_DESC | ? "
;;
esac
ARCH_DESC="$ARCH_DESC | $ARCH_NUM"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment