Restic scripting plus jq and minio client

# cat restic-backup.sh
#!/bin/bash
source /root/.restic-keys
resticprog=/usr/local/bin/restic-custom
#rcloneargs="serve restic --stdio --b2-hard-delete --cache-workers 64 --transfers 64 --retries 21"
region="s3_phx"
rundate=$(date +"%Y-%m-%d-%H%M")
logtop=/reports
logyear=$(date +"%Y")
logmonth=$(date +"%m")
logname=$logtop/$logyear/$logmonth/restic/$rundate-restic-backup
jsonspool=/tmp/restic-fss-jobs

## Backing up some OCI FSS (same as AWS EFS) NFS folders
FSS=(
"fs-oracle-apps|fs-oracle-apps|.snapshot"           ## backup all exclude .snapshot tree
"fs-app1|fs-app1|.snapshot"                         ## backup all exclude .snapshot tree
"fs-sw|fs-sw/oracle_sw,fs-sw/restic_pkg|.snapshot"  ## backup two folders exclude .snapshot tree
"fs-tifs|fs-tifs|.snapshot,.tif"                  ## backup all exclude .snapshot tree and *.tif files
)

## test commands especially before kicking off large backups
function verify_cmds
{
  f=$1
  restic_cmd=$2
  printf "\n$rundate and cmd: $restic_cmd\n"
}

function backup
{
 f=$1
 restic_cmd=$2

 jobstart=$(date +"%Y-%m-%d-%H%M")

 mkdir $jsonspool/$f
 jsonfile=$jsonspool/$f/$jobstart-restic-backup.json
 printf "$jobstart with cmd: $restic_cmd\n"

 mkdir /mnt/$f
 mount -o ro xx.xx.xx.xx:/$f /mnt/$f

 ## TODO: shell issue with passing exclude from variable. verify exclude .snapshot is working
 ## TODO: not passing *.tif exclude fail?  howto pass *?
 $restic_cmd > $jsonfile

 #cat $jsonfile >> $logname-$f.log
 umount /mnt/$f
 rmdir /mnt/$f

## Using rclone to copy to OCI object storage bucket.
## Note the extra level folder so rclone can simulate 
## a server/20190711-restic.log style.
## Very useful with using minio client to analyze logs.
 rclone copy $jsonspool s3_ash:restic-backup-logs

 rm $jsonfile
 rmdir $jsonspool/$f

 jobfinish=$(date +"%Y-%m-%d-%H%M")
 printf "jobfinish $jobfinish\n"
}

for fss in "${FSS[@]}"; do
 arrFSS=(${fss//|/ })

 folders=""
 f=${arrFSS[0]}
 IFS=',' read -ra folderarr <<< ${arrFSS[1]}
 for folder in ${folderarr[@]};do folders+="/mnt/${folder} "; done

 excludearg=""
 IFS=',' read -ra excludearr <<< ${arrFSS[2]}
 for exclude in ${excludearr[@]};do excludearg+=" --exclude ${exclude}"; done

 backup_cmd="$resticprog -r rclone:$region:restic-$f backup ${folders} $excludearg --json"

## play with verify_cmds first before actual backups
 verify_cmds "$f" "$backup_cmd"
 #backup "$f" "$backup_cmd"
done

# cat restic-check-logs.sh
#!/bin/bash

fss=(
 fs-oracle-apps
)

#checkdate="2019-07-11"
checkdate=$(date +"%Y-%m-%d")

for f in ${fss[@]}; do
  echo
  echo
  printf "$f:  "
  name=$(mc find s3-ash/restic-backup-logs/$f -name "*$checkdate*" | head -1)
  if [ -n "$name" ]
  then
    echo $name
    # play with sql --query later
    #mc sql --query "select * from S3Object"  --json-input .message_type=summary s3-ash/restic-backup-logs/$f/2019-07-09-1827-restic-backup.json
    mc cat $name  | jq -r 'select(.message_type=="summary")'
  else
    echo "Fail - no file found"
  fi
done

# ./restic-check-logs.sh

fs-oracle-apps:  s3-ash/restic-backup-logs/fs-oracle-apps/2019-07-12-0928-restic-backup.json
{
  "message_type": "summary",
  "files_new": 291,
  "files_changed": 1,
  "files_unmodified": 678976,
  "dirs_new": 0,
  "dirs_changed": 1,
  "dirs_unmodified": 0,
  "data_blobs": 171,
  "tree_blobs": 2,
  "data_added": 2244824,
  "total_files_processed": 679268,
  "total_bytes_processed": 38808398197,
  "total_duration": 1708.162522559,
  "snapshot_id": "f3e4dc06"
}