#!/bin/bash

#
# Simple script that synchronizes BTRFS snapshots locally or through SSH.
# Features compression, retention policy and automatic incremental sync
#
# Usage:
#  btrfs-sync [options] <src> [<src>...] [[user@]host:]<dir>
#
#  -k|--keep NUM     keep only last <NUM> sync'ed snapshots
#  -d|--delete       delete snapshots in <dst> that don't exist in <src>
#  -z|--xz           use xz     compression. Saves bandwidth, but uses one CPU
#  -Z|--pbzip2       use pbzip2 compression. Saves bandwidth, but uses all CPUs
#  -q|--quiet        don't display progress
#  -v|--verbose      display more information
#  -h|--help         show usage
#
# <src> can either be a single snapshot, or a folder containing snapshots
# <user> requires privileged permissions at <host> for the 'btrfs' command
#
# Cron example: daily synchronization over the internet, keep only last 50
#
# cat > /etc/cron.daily/btrfs-sync <<EOF
# #!/bin/bash
# /usr/local/sbin/btrfs-sync -q -k50 -z /home user@host:/path/to/snaps
# EOF
# chmod +x /etc/cron.daily/btrfs-sync
#
# Copyleft 2018 by Ignacio Nunez Hernanz <nacho _a_t_ ownyourbits _d_o_t_ com>
# GPL licensed (see end of file) * Use at your own risk!
#
# More at https://ownyourbits.com
#

# help
print_usage() {
  echo "Usage: 
  $BIN [options] <src> [<src>...] [[user@]host:]<dir>

  -k|--keep NUM     keep only last <NUM> sync'ed snapshots
  -d|--delete       delete snapshots in <dst> that don't exist in <src>
  -z|--xz           use xz     compression. Saves bandwidth, but uses one CPU
  -Z|--pbzip2       use pbzip2 compression. Saves bandwidth, but uses all CPUs
  -q|--quiet        don't display progress
  -v|--verbose      display more information
  -h|--help         show usage

<src> can either be a single snapshot, or a folder containing snapshots
<user> requires privileged permissions at <host> for the 'btrfs' command

Cron example: daily synchronization over the internet, keep only last 50

cat > /etc/cron.daily/btrfs-sync <<EOF
#!/bin/bash
/usr/local/sbin/btrfs-sync -q -k50 -z /home user@host:/path/to/snaps
EOF
chmod +x /etc/cron.daily/btrfs-sync
"
}

echov() { if [[ "$VERBOSE" == 1 ]]; then echo "$@"; fi }

#----------------------------------------------------------------------------------------------------------

# preliminary checks
BIN="${0##*/}"
[[ $# -lt 2      ]] && { print_usage                                ; exit 1; }
[[ ${EUID} -ne 0 ]] && { echo "Must be run as root. Try 'sudo $BIN'"; exit 1; }

# parse arguments
KEEP=0
ZIP=cat PIZ=cat
SILENT=">/dev/null"

OPTS=$( getopt -o hqzZk:dv -l quiet -l help -l xz -l pbzip2 -l keep: -l delete -l verbose -- "$@" 2>/dev/null )
[[ $? -ne 0 ]] && { echo "error parsing arguments"; exit 1; }
eval set -- "$OPTS"

while true; do
  case "$1" in
    -h|--help   ) print_usage; exit  0 ;;
    -q|--quiet  ) QUIET=1    ; shift 1 ;;
    -d|--delete ) DELETE=1   ; shift 1 ;;
    -k|--keep   ) KEEP=$2    ; shift 2 ;;
    -z|--xz     ) ZIP=xz     PIZ=( xz     -d ); shift 1 ;;
    -Z|--pbzip2 ) ZIP=pbzip2 PIZ=( pbzip2 -d ); shift 1 ;;
    -v|--verbose) SILENT=""  VERBOSE=1        ; shift 1 ;;
    --)                shift;  break   ;;
  esac
done

SRC=( "${@:1:$#-1}" )
DST="${@: -1}"

# detect remote dst argument
[[ "$DST" =~ : ]] && {
  NET="$( sed 's|:.*||' <<<"$DST" )"
  DST="$( sed 's|.*:||' <<<"$DST" )"
  SSH=( ssh -o ServerAliveInterval=5 -o ConnectTimeout=1 -o BatchMode=yes "$NET" )
}
[[ "$SSH" != "" ]] && DST_CMD=( ${SSH[@]} ) || DST_CMD=( eval )
${DST_CMD[@]} true &>/dev/null || { echo "SSH access error to $NET. Do you have passwordless login setup?"; exit 1; }

#----------------------------------------------------------------------------------------------------------

# more checks

## don't overlap
[[ "$SSH" != "" ]] && NUM_PS=0 || NUM_PS=2
${DST_CMD[@]} "[[ \$( pgrep -c \"btrfs-sync\" ) -ne $NUM_PS ]]" \
  && { echo "btrfs-sync already running at destination"; exit 1; }

ps -p "$( cat /run/btrfs-sync.pid 2>/dev/null )" &>/dev/null && { echo "$BIN is already running"; exit 1; }
echo $$ > /run/btrfs-sync.pid

## src checks
while read entry; do SRCS+=( "$entry" ); done < <( 
  for s in "${SRC[@]}"; do
    src="$(cd "$s" &>/dev/null && pwd)" || { echo "$s not found"; exit 1; } #abspath
    btrfs subvolume show "$src" &>/dev/null && echo "0|$src" || \
    for dir in $( ls -drt "$src"/* 2>/dev/null ); do
      btrfs subvolume show "$dir" &>/dev/null || continue
      DATE="$( btrfs su sh "$dir" | grep "Creation time:" | awk '{ print $3, $4 }' )"
      SECS=$( date -d "$DATE" +"%s" )
      echo "$SECS|$dir"
    done
  done | sort -V | sed 's=.*|=='
)
[[ ${#SRCS[@]} -eq 0 ]] && { echo "no BTRFS subvolumes found"; exit 1; }

## check pbzip2
[[ "$ZIP" == "pbzip2" ]] && {
                    type pbzip2 &>/dev/null && \
    "${DST_CMD[@]}" type pbzip2 &>/dev/null || {
      echo "INFO: 'pbzip2' not installed on both ends, fallback to 'xz'"
      ZIP=xz PIZ=unxz 
  }
}

## use 'pv' command if available
PV=( pv -F"time elapsed [%t] | rate %r | total size [%b]" )
[[ "$QUIET" == "1" ]] && PV=( cat ) || type pv &>/dev/null || {
  echo "INFO: install the 'pv' package in order to get a progress indicator"
  PV=( cat )
}

#----------------------------------------------------------------------------------------------------------

# sync snapshots

## get dst snapshots ( DSTS, DST_UUIDS )
get_dst_snapshots() {
  local DST="$1"
  unset DSTS DST_UUIDS
  while read entry; do
    DST_UUIDS+=( "$( sed 's=|.*==' <<<"$entry" )" )
    DSTS+=(      "$( sed 's=.*|==' <<<"$entry" )" )
  done < <( 
    "${DST_CMD[@]}" "
      DSTS=( \$( ls -d \"$DST\"/* 2>/dev/null ) )
      for dst in \${DSTS[@]}; do
        UUID=\$( sudo btrfs su sh \"\$dst\" 2>/dev/null | grep 'Received UUID' | awk '{ print \$3 }' )
        [[ \"\$UUID\" == \"-\" ]] || [[ \"\$UUID\" == \"\" ]] && continue
        echo \"\$UUID|\$dst\"
      done" 
  )
}

## sync incrementally
sync_snapshot() {
  local SRC="$1"
  local PATH_ DATE SECS SEED SEED_PATH SEED_ARG PATH_RXID DATE_RXID SHOWP RXIDP DATEP
  local SHOW="$( btrfs subvolume show     "$SRC" )"
  local LIST="$( btrfs subvolume list -su "$SRC" )"

  # detect existing
  SRC_UUID="$( grep 'UUID:' <<<"$SHOW" | head -1 | awk '{ print $2 }' )"
  SRC_RXID="$( grep 'Received UUID' <<<"$SHOW"   | awk '{ print $3 }' )"
  for id in "${DST_UUIDS[@]}"; do
    [[ "$SRC_UUID" == "$id" ]] || [[ "$SRC_RXID" == "$id" ]] && {
      echov "* Skip existing '$SRC'"
      return 0; 
    }
  done

  # try to get most recent src snapshot that exists in dst to use as a seed
  local HAVE_RXID=0
  declare -A PATH_RXID DATE_RXID
  local PATHS=( $( btrfs su list -u "$SRC" | awk '{ print $11 }' ) )
  SEED=$( 
    for id in "${DST_UUIDS[@]}"; do
      # try to match by UUID
      PATH_=$( awk "{ if ( \$14 == \"$id\" ) print \$16       }" <<<"$LIST" )
      DATE=$(  awk "{ if ( \$14 == \"$id\" ) print \$11, \$12 }" <<<"$LIST" )

      # try to match by received UUID, only if necessary
      [[ "$PATH_" == "" ]] && {
        [[ "$HAVE_RXID" == "0" ]] && {
          for p in "${PATHS[@]}"; do
            SHOWP="$( btrfs su sh "$( dirname "$SRC" )/$( basename "$p" )" 2>/dev/null )"
            RXIDP="$( grep 'Received UUID' <<<"$SHOWP" | awk '{ print $3     }' )"
            DATEP="$( grep 'Creation time' <<<"$SHOWP" | awk '{ print $3, $4 }' )"
            [[ "$RXIDP" == "" ]] && continue
            PATH_RXID["$RXIDP"]="$p"
            DATE_RXID["$RXIDP"]="$DATEP"
          done
          HAVE_RXID=1
        }
        PATH_="${PATH_RXID["$id"]}"
         DATE="${DATE_RXID["$id"]}"
      }

      [[ "$PATH_" == "" ]] || [[ "$PATH_" == "$( basename "$SRC" )" ]] && continue

      SECS=$( date -d "$DATE" +"%s" )
      echo "$SECS|$PATH_"
    done | sort -V | tail -1 | cut -f2 -d'|'
  )

  # incremental sync argument
  [[ "$SEED" != "" ]] && {
    SEED_PATH="$( dirname "$SRC" )/$( basename $SEED )"
    [[ -d "$SEED_PATH" ]] && 
      SEED_ARG=( -p "$SEED_PATH" ) || \
      echo "INFO: couldn't find $SEED_PATH. Non-incremental mode"
  }

  # do it
  echo -n "* Synchronizing '$src'"
  [[ "$SEED_ARG" != "" ]] && echov -n " using seed '$SEED'"
  echo "..."

   btrfs send -q ${SEED_ARG[@]} "$SRC" \
    | "$ZIP" \
    | "${PV[@]}" \
    | "${DST_CMD[@]}" "${PIZ[@]} | sudo btrfs receive \"$DST\" 2>&1 |(grep -v -e'^At subvol ' -e'^At snapshot '||true)" \
    || exit 1;
  get_dst_snapshots "$DST" # sets DSTS DST_UUIDS
}

#----------------------------------------------------------------------------------------------------------

# sync all snapshots found in src
get_dst_snapshots "$DST" # sets DSTS DST_UUIDS
for src in "${SRCS[@]}"; do
  sync_snapshot "$src"
done

#----------------------------------------------------------------------------------------------------------

# retention policy
[[ "$KEEP" != 0 ]] && \
  [[ ${#DSTS[@]} -gt $KEEP ]] && \
  echov "* Pruning old snapshots..." && \
  for (( i=0; i < $(( ${#DSTS[@]} - KEEP )); i++ )); do
    PRUNE_LIST+=( "${DSTS[$i]}" )
  done && \
  ${DST_CMD[@]} sudo btrfs subvolume delete "${PRUNE_LIST[@]}" $SILENT

# delete flag
[[ "$DELETE" == 1 ]] && \
  for dst in "${DSTS[@]}"; do
    FOUND=0
    for src in "${SRCS[@]}"; do
      [[ "$( basename $src )" == "$( basename $dst )" ]] && { FOUND=1; break; }
    done
    [[ "$FOUND" == 0 ]] && DEL_LIST+=( "$dst" )
  done
[[ "$DEL_LIST" != "" ]] && \
  echov "* Deleting non existent snapshots..." && \
  ${DST_CMD[@]} sudo btrfs subvolume delete "${DEL_LIST[@]}" $SILENT

# License
#
# This script is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This script is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this script; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330,