While I was working on a related project to use python to write to cloud object storage and the logic around purging; I jotted down some quick and dirty code here for my reference to build on. Normally I would recommend using the excellent restic program but in this case I am forced to use native API's. <\/p>\n\n\n\n
This serves as a reminder for it is only a very elementary tar plus gzip daily backup and subsequent purging of old backups. Just a test.<\/p>\n\n\n
\n#!\/usr\/bin\/python\n#\n#: Script Name : tarBak.py\n#: Author : Riaan Rossouw\n#: Date Created : March 13, 2019\n#: Date Updated : March 13, 2019\n#: Description : Python Script to manage tar backups\n#: Examples : tarBak.py -t target -f 'folder1,folder2' -c -g GOLD\n#: : tarBak.py --target <backup folder> --folders <folders> --create\n\nimport optparse, os, glob, sys, re, datetime\nimport tarfile\nimport socket\n\n__version__ = '0.9.1'\noptdesc = 'This script is used to manage tar backups of files'\n\nparser = optparse.OptionParser(description=optdesc,version=os.path.basename(__file__) + ' ' + __version__)\nparser.formatter.max_help_position = 50\nparser.add_option('-t', '--target', help='Specify Target', dest='target', action='append')\nparser.add_option('-f', '--folders', help='Specify Folders', dest='folders', action='append')\nparser.add_option('-c', '--create', help='Create a new backup', dest='create', action='store_true',default=False)\nparser.add_option('-p', '--purge', help='Purge older backups per policy', dest='purge', action='store_true',default=False)\nparser.add_option('-g', '--group', help='Policy group', dest='group', action='append')\nparser.add_option('-l', '--list', help='List backups', dest='listall', action='store_true',default=False)\nopts, args = parser.parse_args()\n\ndef make_tarfile(output_filename, source_dirs):\n with tarfile.open(output_filename, "w:gz") as tar:\n for source_dir in source_dirs:\n tar.add(source_dir, arcname=os.path.basename(source_dir))\n\ndef getBackupType(backup_time_created):\n utc,mt = str(backup_time_created).split('.')\n d = datetime.datetime.strptime(utc, '%Y-%m-%d %H:%M:%S').date()\n dt = d.strftime('%a %d %B %Y')\n\n if d.weekday() == 6:\n backup_t = 'WEEKLY'\n elif d.day == 1:\n backup_t = 'MONTHLY'\n elif ( (d.day == 1) and (d.mon == 1) ):\n backup_t = 'YEARLY'\n else:\n backup_t = 'DAILY'\n\n return (backup_t,dt)\n\ndef listBackups(target):\n print ("Listing backup files..")\n\n files = glob.glob(target + "*DAILY*")\n files.sort(key=os.path.getmtime, reverse=True)\n\n for file in files:\n print file\n \ndef purgeBackups(target, group):\n print ("Purging backup files..this needs testing and more logic for SILVER and BRONZE policies?")\n\n files = glob.glob(target + "*.tgz*")\n files.sort(key=os.path.getmtime, reverse=True)\n daily = 0\n weekly = 0\n monthly = 0\n yearly = 0\n \n for file in files:\n comment = ""\n if ( ("DAILY" in file) or ("WEEKLY" in file) or ("MONTHLY" in file) or ("YEARLY" in file) ):\n #t = file.split("-")[0]\n sub = re.search('files-(.+?)-2019', file)\n #print sub\n t = sub.group(1)\n else:\n t = "MANUAL"\n\n if t == "DAILY":\n comment = "DAILY"\n daily = daily + 1\n if daily > 7:\n comment = comment + " this one is more than 7 deleting"\n os.remove(file)\n elif t == "WEEKLY":\n comment = "Sun"\n weekly = weekly + 1\n if weekly > 4:\n comment = comment + " this one is more than 4 deleting"\n os.remove(file)\n elif t == "MONTHLY":\n comment = "01"\n monthly = monthly + 1\n if monthly > 12:\n comment = comment + " this one is more than 12 deleting"\n os.remove(file)\n elif t == "YEARLY":\n comment = "01"\n yearly = yearly + 1\n if yearly > 5:\n comment = comment + " this one is more than 5 deleting"\n os.remove(file)\n else:\n comment = " manual snapshot not purging"\n \n if "this one " in comment:\n print ('DELETE: {:25}: {:25}'.format(file, comment) )\n\ndef createBackup(target, folders, group):\n print ("creating backup of " + str(folders))\n hostname = socket.gethostname()\n creationDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.0")\n t,ds = getBackupType(creationDate)\n BackupName = target + "\/" + hostname + '-files-' + t + "-" + datetime.datetime.now().strftime("%Y%m%d-%H%MCST") + '.tgz'\n\n proceed = "SNAPSHOT NOT NEEDED AT THIS TIME PER THE POLICY"\n if ( group == "BRONZE") and ( (t == "MONTHLY") or (t == "YEARLY") ):\n proceed = "CLEAR TO SNAP" \n elif ( group == "SILVER" and (t == "WEEKLY") or (t == "MONTHLY" ) or (t == "YEARLY") ):\n proceed = "CLEAR TO SNAP" \n elif group == "GOLD":\n proceed = "CLEAR TO SNAP" \n else:\n result = proceed\n \n make_tarfile(BackupName, folders)\n\ndef main():\n if opts.target:\n target = opts.target[0]\n else:\n print ("\\n\\n must specify target folder")\n exit(0)\n\n if opts.listall:\n listBackups(target)\n else:\n if opts.create:\n if opts.folders:\n folders = opts.folders[0].split(',')\n else:\n print ("\\n\\n must specify folders")\n exit(0)\n createBackup(target, folders, opts.group[0])\n\n if opts.purge:\n purgeBackups(target, opts.group[0])\n\nif __name__ == '__main__':\n main()\n<\/pre><\/div>\n\n\nAnd running it like this:<\/p>\n\n\n
\n$ python tarBak.py -t \/tmp\/MyBackups\/ -f '\/home\/rrosso,\/var\/log\/syslog' -g GOLD -c\ncreating backup of ['\/home\/rrosso', '\/var\/log\/syslog']\n\n$ python tarBak.py -t \/tmp\/MyBackups\/ -p -g GOLD\nPurging backup files..this needs testing and more logic for SILVER and BRONZE policies?\nDELETE: \/tmp\/MyBackups\/xubuntu32-files-DAILY-20190313-1420CST.tgz: DAILY this one is more than 7 deleting\n<\/pre><\/div>\n\n\n$ crontab -l | tail -1\n0 5 * * * cd \/Src\/tarBak\/ ; python tarBak.py -t \/MyBackups\/ -f '\/home\/rrosso,\/var\/spool\/syslog' -c -p -g GOLD 2>&1\n<\/pre><\/div>","protected":false},"excerpt":{"rendered":"While I was working on a related project to use python to write to cloud object storage and the logic<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[3],"tags":[],"class_list":["post-1320","post","type-post","status-publish","format-standard","hentry","category-backups"],"_links":{"self":[{"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/posts\/1320","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/comments?post=1320"}],"version-history":[{"count":0,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/posts\/1320\/revisions"}],"wp:attachment":[{"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/media?parent=1320"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/categories?post=1320"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/tags?post=1320"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}