Commit 680bff77 authored by Guillaume REMBERT's avatar Guillaume REMBERT

Added checks and recovery to restore process + minor fixes

parent 91abb7a3
......@@ -4,7 +4,7 @@ Version: 1.0.0
Date: 2016/04/02
-------------------------------
Author(s):
Guillaume REMBERT, guillaume.rembert@euryecetelecom.com
Guillaume REMBERT
-------------------------------
License:
MIT - Please read LICENSE file for more information
......
......@@ -9,7 +9,7 @@ EURYBOX_CORPORATE="My Corporate"
#Details level on script execution
#Can be: debug, verbose, info, warning or error
EURYBOX_LOG_LEVEL="info"
EURYBOX_LOG_LEVEL="verbose"
#Parallelism depth / depends on host CPU
EURYBOX_PARALLELISM_LEVEL="8"
#Max retry on failure before stopping execution
......@@ -105,7 +105,7 @@ EURYBOX_BACKUP_ARCHIVE=(
[OWNER]="root:root"
[MASK]="400"
[LABEL]="backup_u:object_r:backup_t:s0"
[FORMAT]="tar.bz2"
[FORMAT]="tar.gz"
[HASH]="sha512"
[ENCRYPT]="true"
[ENC_ALGO]="aes-256-ctr"
......@@ -113,7 +113,7 @@ EURYBOX_BACKUP_ARCHIVE=(
[FEC]="zfec"
[FEC_FILE_NUM]="10"
#FOR PAR2 ONLY
[FEC_LEVEL]="10"
[FEC_LEVEL]="25"
#FOR ZFEC ONLY
[FEC_FILE_NUM_MIN]="5"
)
......@@ -128,8 +128,9 @@ EURYBOX_BACKUP_ARCHIVE=(
#-PART: local unmounted partition or raw device (as USB device, additional disks, ...) + local mount point provided by kernel. TODO: LUKS encrypted partition provided by kernel and cryptsetup tool
#-LOCAL: local directory
EURYBOX_BACKUP_DESTINATION=(
[PROTOCOL]="FTP"
[MOUNT]="/mnt/local_backup_folder"
[PROTOCOL]="LOCAL"
# [MOUNT]="/mnt/local_backup_folder"
[MOUNT]="/mnt/new_ftp"
#FOR SSH AND FTP
[USER]="test"
#FOR SSH(22), FTP(21) AND NFS(2049)
......
......@@ -39,7 +39,7 @@ eurybox_check_environment BACKUP
eurybox_display_message message STATUS "$EURYBOX_CORPORATE backup script started"
#Prepare environment for backup
euryboxctrl_prepare_backup
case $EURYBOX_BACKUP_TYPE in
......@@ -54,7 +54,7 @@ case $EURYBOX_BACKUP_TYPE in
#Create archive file
euryboxctrl_create_arch
#Re-Start all VMs
# euryboxctrl_start_services
euryboxctrl_start_services
#Create errors correcting codes files
euryboxctrl_create_eccf
#Create description file
......
......@@ -49,11 +49,8 @@ euryboxctrl_discover_available_archives
#Select archive to be restored
euryboxctrl_select_restore_target
#Check integrity of archive file
euryboxctrl_check_desc
euryboxctrl_check_hash
euryboxctrl_check_eccf
euryboxctrl_check_arch
#Prepare environment for restoration
euryboxctrl_prepare_restore
case $EURYBOX_RESTORE_TYPE in
cloned_vm )
......@@ -79,7 +76,7 @@ case $EURYBOX_RESTORE_TYPE in
;;
esac
euryboxctrl_terminate_restore
#Finished
......
......@@ -3,6 +3,40 @@
#TODO: Implement lockfile and check if not already running
#Desc: verify the archive access and content
#No arg required
eurybox_check_arch ()
{
local ARCH_OUT
local STATUS
local ARCH_FORMAT="${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},FORMAT]}"
local ARCH_ENC="${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},ENC]}"
local ARCH_FILE="${EURYBOX_BACKUP_DESTINATION[MOUNT]}/${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},FILENAME]}.${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},FORMAT]}"
case ${ARCH_FORMAT} in
"tar" ) TAR_OPTIONS="-Stv";;
"tar.gz" ) TAR_OPTIONS="-Stv --use-compress-program=pigz";;
"tar.bz2" ) TAR_OPTIONS="-Stv --use-compress-program=lbzip2";;
esac
if [[ ${ARCH_ENC} = "true" ]]
then
ARCH_OUT=`sudo sh -c "openssl enc -${EURYBOX_BACKUP_ARCHIVE[ENC_ALGO]} -d -in ${ARCH_FILE} -k ${EURYBOX_BACKUP_ARCHIVE[PASSWORD]} | tar ${TAR_OPTIONS}" 2>&1`
STATUS=$?
else
ARCH_OUT=`sudo sh -c "tar ${TAR_OPTIONS}f ${ARCH_FILE}" 2>&1`
STATUS=$?
fi
if [[ !($STATUS -eq 0) ]]
then
eurybox_display_message warning CHECK "Archive use check failed - error $STATUS:\n$ARCH_OUT"
EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},ARCH_STATUS]="KO"
else
EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},ARCH_STATUS]="OK"
eurybox_display_message message CHECK "Archive use check - OK"
eurybox_display_message debug CHECK "Archive command output:\n$ARCH_OUT"
fi
}
#Desc: check the input arguments
#TODO: implement arguments usage (debug mode / config file ...) and checking
eurybox_check_arguments ()
......@@ -178,6 +212,94 @@ eurybox_check_configuration ()
eurybox_display_message debug CHECK "ftpfs mount options: $EURYBOX_MNT_FTPFS_OPTIONS"
}
#Desc: verify the archive ECCF and try to repair it
#No arg required
eurybox_check_eccf ()
{
local ECC_OUT
local STATUS
local ECC_FORMAT="${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},FEC]}"
local ECC_FILE="${EURYBOX_BACKUP_DESTINATION[MOUNT]}/${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},FILENAME]}.${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},FORMAT]}"
local ECC_FILE_ROOT="${EURYBOX_BACKUP_DESTINATION[MOUNT]}/${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},FILENAME]}"
case "${ECC_FORMAT}" in
par2 )
ECC_OUT=`sudo sh -c "par2 r ${ECC_FILE_ROOT}.par2" 2>&1`
STATUS=$?
;;
zfec )
ECC_OUT=`sudo sh -c "zunfec -f -o ${ECC_FILE}.recovered ${ECC_FILE}.*.fec" 2>&1`
STATUS=$?
if [[ ($STATUS -eq 0) ]]
then
EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},ECC_STATUS]="OK"
eurybox_display_message message CHECK "Archive ECC check/recovery - recovered file: ${ECC_FILE}.recovered"
eurybox_display_message debug CHECK "ECC command output:\n$ECC_OUT"
ECC_OUT=`sudo sh -c "mv ${ECC_FILE} ${ECC_FILE}.corrupted" 2>&1`
STATUS=$?
if [[ !($STATUS -eq 0) ]]
then
eurybox_display_message warning CHECK "Archive ECC check/recovery success but cannot move corrupted archive - error $STATUS:\n$ECC_OUT"
else
ECC_OUT=`sudo sh -c "mv ${ECC_FILE}.recovered ${ECC_FILE}" 2>&1`
STATUS=$?
if [[ !($STATUS -eq 0) ]]
then
eurybox_display_message warning CHECK "Archive ECC check/recovery success but cannot move recovered archive - error $STATUS:\n$ECC_OUT"
else
eurybox_display_message message CHECK "Archive ECC check/recovery - OK"
eurybox_display_message debug CHECK "ECC command output:\n$ECC_OUT"
fi
fi
fi
;;
* )
eurybox_display_message error CHECK "Archive ECC format unknown: ${ECC_FORMAT}"
;;
esac
if [[ !($STATUS -eq 0) ]]
then
eurybox_display_message warning CHECK "Archive ECC check/recovery failed - error $STATUS:\n$ECC_OUT"
EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},ECC_STATUS]="KO"
else
EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},ECC_STATUS]="OK"
eurybox_display_message message CHECK "Archive ECC check/recovery - OK"
eurybox_display_message debug CHECK "ECC command output:\n$ECC_OUT"
fi
}
#Desc: verify the archive hash
#No arg required
eurybox_check_hash()
{
local HASH_OUT
local STATUS
local HASH_FORMAT="${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},HASH]}"
local HASH_FILE="${EURYBOX_BACKUP_DESTINATION[MOUNT]}/${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},FILENAME]}.${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},HASH]}"
case "${HASH_FORMAT}" in
sha256 )
HASH_OUT=`sudo sh -c "sha256sum -c $HASH_FILE" 2>&1`
STATUS=$?
;;
sha512 )
HASH_OUT=`sudo sh -c "sha512sum -c $HASH_FILE" 2>&1`
STATUS=$?
;;
* )
eurybox_display_message error CHECK "Archive hash format unknown: ${HASH_FORMAT}"
;;
esac
if [[ !($STATUS -eq 0) ]]
then
eurybox_display_message warning CHECK "Archive hash check failed - error $STATUS:\n$HASH_OUT"
EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},HASH_STATUS]="KO"
else
EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},HASH_STATUS]="OK"
eurybox_display_message message CHECK "Archive hash check - OK"
eurybox_display_message debug CHECK "Hash command output:\n$HASH_OUT"
fi
}
#Desc: verify the kernel is Linux
#No arg required
eurybox_check_kernel()
......
......@@ -19,7 +19,7 @@ eurybox_configure_interactive_restoration_target ()
eurybox_display_message error CONFIGURE "Selected archive unavailable: ${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]}"
else
EURYBOX_BACKUP_ARCHIVE[NAME]="${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},VERSION]}"
eurybox_display_message message CONFIGURE "Selected archive: ${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]} / ${EURYBOX_BACKUP_ARCHIVE[NAME]}"
eurybox_display_message message CONFIGURE "Selected archive: ${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]} / ${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},FILENAME]}"
fi
}
......
......@@ -20,12 +20,12 @@ eurybox_create_archive_desc ()
then
DESCRIPTION+="ENCRYPTION_ALGORITHM:${EURYBOX_BACKUP_ARCHIVE[ENC_ALGO]}\nARCHIVE_CONTENT:\n"
#FIXME: function to read archive with variable archive format required here
DESCRIPTION+=`sudo sh -c "openssl enc -${EURYBOX_BACKUP_ARCHIVE[ENC_ALGO]} -d -in ${EURYBOX_BACKUP_DESTINATION[MOUNT]}/${EURYBOX_BACKUP_ARCHIVE[NAME]}.${EURYBOX_BACKUP_ARCHIVE[FORMAT]} -k ${EURYBOX_BACKUP_ARCHIVE[PASSWORD]} | tar ${TAR_OPTIONS}"`
DESCRIPTION+=`sudo sh -c "openssl enc -${EURYBOX_BACKUP_ARCHIVE[ENC_ALGO]} -d -in ${EURYBOX_BACKUP_DESTINATION[MOUNT]}/${EURYBOX_BACKUP_ARCHIVE[NAME]}.${EURYBOX_BACKUP_ARCHIVE[FORMAT]} -k ${EURYBOX_BACKUP_ARCHIVE[PASSWORD]} | tar ${TAR_OPTIONS}" 2>&1`
STATUS=$?
else
DESCRIPTION+="ARCHIVE_CONTENT:\n"
#FIXME: function to read archive with variable archive format required here
DESCRIPTION+=`sudo sh -c "tar ${TAR_OPTIONS}f ${EURYBOX_BACKUP_DESTINATION[MOUNT]}/${EURYBOX_BACKUP_ARCHIVE[NAME]}.${EURYBOX_BACKUP_ARCHIVE[FORMAT]}"`
DESCRIPTION+=`sudo sh -c "tar ${TAR_OPTIONS}f ${EURYBOX_BACKUP_DESTINATION[MOUNT]}/${EURYBOX_BACKUP_ARCHIVE[NAME]}.${EURYBOX_BACKUP_ARCHIVE[FORMAT]}" 2>&1`
STATUS=$?
fi
if [[ !($STATUS -eq 0) ]]
......@@ -67,9 +67,9 @@ eurybox_create_archive_tar ()
TAR_OPTIONS="-"
fi
case ${EURYBOX_BACKUP_ARCHIVE[FORMAT]} in
"tar" ) TAR_OPTIONS="${TAR_OPTIONS}Sc";;
"tar.gz" ) TAR_OPTIONS="${TAR_OPTIONS}Sc --use-compress-program=pigz";;
"tar.bz2" ) TAR_OPTIONS="${TAR_OPTIONS}Sc --use-compress-program=pbzip2";;
"tar" ) TAR_OPTIONS="${TAR_OPTIONS}Sch";;
"tar.gz" ) TAR_OPTIONS="${TAR_OPTIONS}Sch --use-compress-program=pigz";;
"tar.bz2" ) TAR_OPTIONS="${TAR_OPTIONS}Sch --use-compress-program=pbzip2";;
esac
if [[ ${EURYBOX_BACKUP_ARCHIVE[ENCRYPT]} = "true" ]]
then
......
......@@ -31,18 +31,20 @@ eurybox_detect_archive_content ()
BKP_COMP=`echo "${ARCH_DESC}" | grep "ARCHIVE_FORMAT" | awk -F":" '{print $NF}'`
case $BKP_COMP in
"" )
eurybox_display_message warning DETECT "Archive description compression search failed"
eurybox_display_message warning DETECT "Archive description compression/format search failed"
;;
tar )
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,FORMAT]=$BKP_COMP
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,COMP]="none"
eurybox_display_message debug DETECT "Archive description compression search success - found: none"
eurybox_display_message debug DETECT "Archive description compression/format search success - found: none"
;;
tar.gz|tar.bz2 )
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,FORMAT]=$BKP_COMP
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,COMP]=$BKP_COMP
eurybox_display_message debug DETECT "Archive description compression search success - found: $BKP_COMP"
eurybox_display_message debug DETECT "Archive description compression/format search success - found: $BKP_COMP"
;;
* )
eurybox_display_message warning DETECT "Archive description compression search failed - unknown backup compression found: $BKP_COMP"
eurybox_display_message warning DETECT "Archive description compression/format search failed - unknown backup format found: $BKP_COMP"
;;
esac
BKP_TYPE=`echo "${ARCH_DESC}" | grep "ARCHIVE_TYPE" | awk -F":" '{print $NF}'`
......@@ -104,25 +106,27 @@ eurybox_detect_archive_content ()
#1 arg required: search_path
eurybox_detect_archives ()
{
local DESTINATION="$1/*"
local DESTINATION=$1
local BKP_LIST
local FILES_LIST=`sudo ls $EURYBOX_LS_OPTIONS $DESTINATION 2>&1`
local FILES_LIST=`sudo ls $DESTINATION 2>&1`
local STATUS=$?
if [[ !($STATUS -eq 0) ]]
then
eurybox_display_message error DETECT "Archive description listing not found: \n$FILES_LIST"
eurybox_display_message error DETECT "Archive description listing failed - error $STATUS\n$FILES_LIST"
else
#Filter by desc extension, extract filename and store in an array
BKP_LIST=( $(echo "$FILES_LIST" | awk -F"." '($NF == "desc") {print $0}' | awk -F"/" '{print $NF}') )
if [[ ${BKP_LIST[@]} = "" ]]
then
eurybox_display_message error DETECT "No archive found"
eurybox_display_message error DETECT "No archive found in following files\n$FILES_LIST"
else
EURYBOX_DETECTED_ARCHIVES[NUM]=${#BKP_LIST[@]}
eurybox_display_message message DETECT "${EURYBOX_DETECTED_ARCHIVES[NUM]} archive description(s) found"
eurybox_display_message debug DETECT "Detected:\n${BKP_LIST[@]}"
for (( ARCH_NUM=0;ARCH_NUM<${EURYBOX_DETECTED_ARCHIVES[NUM]};ARCH_NUM++ ))
do
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,FILE]=${BKP_LIST[$ARCH_NUM]}
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,FILENAME]=`echo "${BKP_LIST[$ARCH_NUM]}" | awk -F"." '{ for(i=1; i<(NF); i++) { if(i==1) { filename=$i } else { filename=filename"."$i } } ; print filename }'`
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,VERSION]=`echo "${BKP_LIST[$ARCH_NUM]}" | awk -F"." '{ for(i=1; i<(NF); i++) { if(i==1) { version=$i } else { version=version"."$i } } ; print version }'`
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,NAME]=`echo "${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,VERSION]}" | awk -F"_" '{ for(i=1; i<(NF-5); i++) { if(i==1) { name=$i } else { name=name"_"$i } } ; print name }'`
EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,TIME]=`echo "${EURYBOX_DETECTED_ARCHIVES[$ARCH_NUM,VERSION]}" | awk -F"_" '{ for(i=(NF-5); i<=NF; i++) { if(i==(NF-5)) { name=$i } else { name=name"_"$i } } ; print name }'`
......
......@@ -29,7 +29,8 @@ euryboxctrl_check_all_services_halted ()
euryboxctrl_check_arch ()
{
eurybox_display_message warning EURYBOXCTRL "Checking the archive file - NOT IMPLEMENTED YET"
eurybox_display_message message EURYBOXCTRL "Checking the archive file"
eurybox_check_arch
}
euryboxctrl_check_desc ()
......@@ -39,17 +40,20 @@ euryboxctrl_check_desc ()
euryboxctrl_check_eccf ()
{
eurybox_display_message warning EURYBOXCTRL "Checking the error correcting codes files - NOT IMPLEMENTED YET"
eurybox_display_message message EURYBOXCTRL "Checking the error correcting codes files"
eurybox_check_eccf
}
euryboxctrl_check_hash ()
{
eurybox_display_message warning EURYBOXCTRL "Checking the hash file - NOT IMPLEMENTED YET"
eurybox_display_message message EURYBOXCTRL "Checking the hash file"
eurybox_check_hash
}
euryboxctrl_check_sign ()
{
eurybox_display_message warning EURYBOXCTRL "Checking the signature file - NOT IMPLEMENTED YET"
eurybox_display_message warning EURYBOXCTRL "Checking the signature file - NOT IMPLEMENTED YET / ALWAYS OK RETURNED"
EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},SIGN_STATUS]="OK"
}
euryboxctrl_create_arch ()
......@@ -156,6 +160,61 @@ euryboxctrl_prepare_backup ()
eurybox_backup_mount_target
}
euryboxctrl_prepare_restore ()
{
#Mount backup target
eurybox_display_message message EURYBOXCTRL "Mounting backup origin"
eurybox_backup_mount_target
#Check archive and associated files integrity
euryboxctrl_check_hash
if [[ ${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},HASH_STATUS]} = "KO" ]]
then
eurybox_display_message message EURYBOXCTRL "Trying to repair archive with FEC data"
euryboxctrl_check_eccf
if [[ ${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},ECC_STATUS]} = "KO" ]]
then
#Un-mount backup target
eurybox_display_message message EURYBOXCTRL "Unmounting backup origin"
eurybox_backup_umount_target
eurybox_display_message error EURYBOXCTRL "FEC reparation of archive failed - restoration cannot continue"
else
euryboxctrl_check_hash
if [[ ${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},HASH_STATUS]} = "KO" ]]
then
eurybox_display_message message EURYBOXCTRL "Unmounting backup origin"
eurybox_backup_umount_target
eurybox_display_message error EURYBOXCTRL "FEC reparation of archive success but hash still mismatch - restoration cannot continue"
else
eurybox_display_message message EURYBOXCTRL "Unmounting backup origin"
eurybox_backup_umount_target
eurybox_display_message error EURYBOXCTRL "Selected archive was corrupted - FEC reparation of archive has been successfull and hash verification success - restoration can be tried again"
fi
fi
else
euryboxctrl_check_sign
if [[ ${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},SIGN_STATUS]} = "KO" ]]
then
#Un-mount backup target
eurybox_display_message message EURYBOXCTRL "Unmounting backup origin"
eurybox_backup_umount_target
eurybox_display_message error EURYBOXCTRL "Signature verification of archive failed - restoration cannot continue"
else
euryboxctrl_check_arch
if [[ ${EURYBOX_DETECTED_ARCHIVES[${EURYBOX_RESTORE_TARGET_ARCHIVE[ID]},ARCH_STATUS]} = "KO" ]]
then
#Un-mount backup target
eurybox_display_message message EURYBOXCTRL "Unmounting backup origin"
eurybox_backup_umount_target
eurybox_display_message error EURYBOXCTRL "Hash and signature OK but archive check failed - restoration cannot continue"
else
eurybox_display_message message EURYBOXCTRL "Archive origin checks are OK - restoration can continue"
fi
fi
fi
}
euryboxctrl_select_restore_target ()
{
if [[ $EURYBOX_RESTORE_TARGET_ACQUISITION = "interactive" ]]
......@@ -227,3 +286,10 @@ euryboxctrl_terminate_backup ()
eurybox_backup_umount_target
}
euryboxctrl_terminate_restore ()
{
#Unmount backup target
eurybox_display_message message EURYBOXCTRL "Unmounting backup destination"
eurybox_backup_umount_target
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment