forked from mirror/btrfs-sync
make faster and more robust
This commit is contained in:
nachoparker
parent
25d23bee02
commit
e9469983bc
128
btrfs-sync
128
btrfs-sync
@ -32,6 +32,8 @@
|
||||
# More at https://ownyourbits.com
|
||||
#
|
||||
|
||||
set -e -o pipefail
|
||||
|
||||
# help
|
||||
print_usage() {
|
||||
echo "Usage:
|
||||
@ -114,13 +116,14 @@ ps -p "$( cat /run/btrfs-sync.pid 2>/dev/null )" &>/dev/null && { echo "$BIN is
|
||||
echo $$ > /run/btrfs-sync.pid
|
||||
|
||||
## src checks
|
||||
while read entry; do SRCS+=( "$entry" ); done < <(
|
||||
echov "* Check source"
|
||||
while read entry; do SRCS+=( "$entry" ); done < <(
|
||||
for s in "${SRC[@]}"; do
|
||||
src="$(cd "$s" &>/dev/null && pwd)" || { echo "$s not found"; exit 1; } #abspath
|
||||
btrfs subvolume show "$src" &>/dev/null && echo "0|$src" || \
|
||||
for dir in $( ls -drt "$src"/* 2>/dev/null ); do
|
||||
btrfs subvolume show "$dir" &>/dev/null || continue
|
||||
DATE="$( btrfs su sh "$dir" | grep "Creation time:" | awk '{ print $3, $4 }' )"
|
||||
DATE="$( btrfs su sh "$dir" 2>/dev/null | grep "Creation time:" | awk '{ print $3, $4 }' )" \
|
||||
|| continue # not a subvolume
|
||||
SECS=$( date -d "$DATE" +"%s" )
|
||||
echo "$SECS|$dir"
|
||||
done
|
||||
@ -148,8 +151,7 @@ PV=( pv -F"time elapsed [%t] | rate %r | total size [%b]" )
|
||||
|
||||
# sync snapshots
|
||||
|
||||
## get dst snapshots ( DSTS, DST_UUIDS )
|
||||
get_dst_snapshots() {
|
||||
get_dst_snapshots() { # sets DSTS DST_UUIDS
|
||||
local DST="$1"
|
||||
unset DSTS DST_UUIDS
|
||||
while read entry; do
|
||||
@ -166,62 +168,75 @@ get_dst_snapshots() {
|
||||
)
|
||||
}
|
||||
|
||||
choose_seed() { # sets SEED
|
||||
local SRC="$1"
|
||||
|
||||
SEED="$SEED_NEXT"
|
||||
if [[ "$SEED" == "" ]]; then
|
||||
# try to get most recent src snapshot that exists in dst to use as a seed
|
||||
local RXID_CALCULATED=0
|
||||
declare -A PATH_RXID DATE_RXID SHOWP RXIDP DATEP
|
||||
local LIST="$( btrfs subvolume list -su "$SRC" )"
|
||||
SEED=$(
|
||||
for id in "${DST_UUIDS[@]}"; do
|
||||
# try to match by UUID
|
||||
local PATH_=$( awk "{ if ( \$14 == \"$id\" ) print \$16 }" <<<"$LIST" )
|
||||
local DATE=$( awk "{ if ( \$14 == \"$id\" ) print \$11, \$12 }" <<<"$LIST" )
|
||||
|
||||
# try to match by received UUID, only if necessary
|
||||
[[ "$PATH_" == "" ]] && {
|
||||
[[ "$RXID_CALCULATED" == "0" ]] && { # create table during the first iteration if needed
|
||||
local PATHS=( $( btrfs su list -u "$SRC" | awk '{ print $11 }' ) )
|
||||
for p in "${PATHS[@]}"; do
|
||||
SHOWP="$( btrfs su sh "$( dirname "$SRC" )/$( basename "$p" )" 2>/dev/null )"
|
||||
RXIDP="$( grep 'Received UUID' <<<"$SHOWP" | awk '{ print $3 }' )"
|
||||
DATEP="$( grep 'Creation time' <<<"$SHOWP" | awk '{ print $3, $4 }' )"
|
||||
[[ "$RXIDP" == "" ]] && continue
|
||||
PATH_RXID["$RXIDP"]="$p"
|
||||
DATE_RXID["$RXIDP"]="$DATEP"
|
||||
done
|
||||
RXID_CALCULATED=1
|
||||
}
|
||||
PATH_="${PATH_RXID["$id"]}"
|
||||
DATE="${DATE_RXID["$id"]}"
|
||||
}
|
||||
|
||||
[[ "$PATH_" == "" ]] || [[ "$PATH_" == "$( basename "$SRC" )" ]] && continue
|
||||
|
||||
local SECS=$( date -d "$DATE" +"%s" )
|
||||
echo "$SECS|$PATH_"
|
||||
done | sort -V | tail -1 | cut -f2 -d'|'
|
||||
)
|
||||
fi
|
||||
}
|
||||
|
||||
exists_at_dst() {
|
||||
local SHOW="$( btrfs subvolume show "$SRC" )"
|
||||
|
||||
local SRC_UUID="$( grep 'UUID:' <<<"$SHOW" | head -1 | awk '{ print $2 }' )"
|
||||
grep -q "$SRC_UUID" <<<"${DST_UUIDS[@]}" && return 0;
|
||||
|
||||
local SRC_RXID="$( grep 'Received UUID' <<<"$SHOW" | awk '{ print $3 }' )"
|
||||
grep -q "^-$" <<<"$SRC_RXID" && return 1;
|
||||
grep -q "$SRC_RXID" <<<"${DST_UUIDS[@]}" && return 0;
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
## sync incrementally
|
||||
sync_snapshot() {
|
||||
local SRC="$1"
|
||||
local PATH_ DATE SECS SEED SEED_PATH SEED_ARG PATH_RXID DATE_RXID SHOWP RXIDP DATEP
|
||||
local SHOW="$( btrfs subvolume show "$SRC" )"
|
||||
local LIST="$( btrfs subvolume list -su "$SRC" )"
|
||||
[[ -d "$SRC" ]] || return
|
||||
|
||||
# detect existing
|
||||
SRC_UUID="$( grep 'UUID:' <<<"$SHOW" | head -1 | awk '{ print $2 }' )"
|
||||
SRC_RXID="$( grep 'Received UUID' <<<"$SHOW" | awk '{ print $3 }' )"
|
||||
for id in "${DST_UUIDS[@]}"; do
|
||||
[[ "$SRC_UUID" == "$id" ]] || [[ "$SRC_RXID" == "$id" ]] && {
|
||||
echov "* Skip existing '$SRC'"
|
||||
return 0;
|
||||
}
|
||||
done
|
||||
exists_at_dst "$SRC" && { echov "* Skip existing '$SRC'"; return 0; }
|
||||
|
||||
# try to get most recent src snapshot that exists in dst to use as a seed
|
||||
local HAVE_RXID=0
|
||||
declare -A PATH_RXID DATE_RXID
|
||||
local PATHS=( $( btrfs su list -u "$SRC" | awk '{ print $11 }' ) )
|
||||
SEED=$(
|
||||
for id in "${DST_UUIDS[@]}"; do
|
||||
# try to match by UUID
|
||||
PATH_=$( awk "{ if ( \$14 == \"$id\" ) print \$16 }" <<<"$LIST" )
|
||||
DATE=$( awk "{ if ( \$14 == \"$id\" ) print \$11, \$12 }" <<<"$LIST" )
|
||||
|
||||
# try to match by received UUID, only if necessary
|
||||
[[ "$PATH_" == "" ]] && {
|
||||
[[ "$HAVE_RXID" == "0" ]] && {
|
||||
for p in "${PATHS[@]}"; do
|
||||
SHOWP="$( btrfs su sh "$( dirname "$SRC" )/$( basename "$p" )" 2>/dev/null )"
|
||||
RXIDP="$( grep 'Received UUID' <<<"$SHOWP" | awk '{ print $3 }' )"
|
||||
DATEP="$( grep 'Creation time' <<<"$SHOWP" | awk '{ print $3, $4 }' )"
|
||||
[[ "$RXIDP" == "" ]] && continue
|
||||
PATH_RXID["$RXIDP"]="$p"
|
||||
DATE_RXID["$RXIDP"]="$DATEP"
|
||||
done
|
||||
HAVE_RXID=1
|
||||
}
|
||||
PATH_="${PATH_RXID["$id"]}"
|
||||
DATE="${DATE_RXID["$id"]}"
|
||||
}
|
||||
|
||||
[[ "$PATH_" == "" ]] || [[ "$PATH_" == "$( basename "$SRC" )" ]] && continue
|
||||
|
||||
SECS=$( date -d "$DATE" +"%s" )
|
||||
echo "$SECS|$PATH_"
|
||||
done | sort -V | tail -1 | cut -f2 -d'|'
|
||||
)
|
||||
choose_seed "$SRC" # sets SEED
|
||||
|
||||
# incremental sync argument
|
||||
[[ "$SEED" != "" ]] && {
|
||||
SEED_PATH="$( dirname "$SRC" )/$( basename $SEED )"
|
||||
local SEED_PATH="$( dirname "$SRC" )/$( basename $SEED )"
|
||||
[[ -d "$SEED_PATH" ]] &&
|
||||
SEED_ARG=( -p "$SEED_PATH" ) || \
|
||||
local SEED_ARG=( -p "$SEED_PATH" ) || \
|
||||
echo "INFO: couldn't find $SEED_PATH. Non-incremental mode"
|
||||
}
|
||||
|
||||
@ -230,17 +245,22 @@ sync_snapshot() {
|
||||
[[ "$SEED_ARG" != "" ]] && echov -n " using seed '$SEED'"
|
||||
echo "..."
|
||||
|
||||
btrfs send -q ${SEED_ARG[@]} "$SRC" \
|
||||
btrfs send -q ${SEED_ARG[@]} "$SRC" \
|
||||
| "$ZIP" \
|
||||
| "${PV[@]}" \
|
||||
| "${DST_CMD[@]}" "${PIZ[@]} | sudo btrfs receive \"$DST\" 2>&1 |(grep -v -e'^At subvol ' -e'^At snapshot '||true)" \
|
||||
|| exit 1;
|
||||
get_dst_snapshots "$DST" # sets DSTS DST_UUIDS
|
||||
|
||||
# update DST list
|
||||
DSTS+=("$DST/$( basename "$SRC" )")
|
||||
DST_UUIDS+=("$SRC_UUID")
|
||||
SEED_NEXT="$SRC"
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
|
||||
# sync all snapshots found in src
|
||||
echov "* Check destination"
|
||||
get_dst_snapshots "$DST" # sets DSTS DST_UUIDS
|
||||
for src in "${SRCS[@]}"; do
|
||||
sync_snapshot "$src"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user