After a recent experiment I did to capture the output of jobs with json output from multiple server's to a central collector, I kept my notes for future reference. In short I used my restic backup json format to submit into MOngoDB via a little custom API.<\/p>\n
The notes are showing scripts I used whihc of course may not work in your environment they are for reference only.<\/p>\n
$ more setup.sh create-db.py apiserver.py \n::::::::::::::\nsetup.sh\n::::::::::::::\n#!\/bin\/bash\nsudo apt install -y gnupg python3-pip python3-flask python3-flask-mongoengine python3-virtualenv\nsudo pip install flask pymongo flask_pymongo\nsudo snap install robo3t-snap\n\nwget -qO - https:\/\/www.mongodb.org\/static\/pgp\/server-4.4.asc | sudo apt-key add -\necho \"deb [ arch=amd64,arm64 ] https:\/\/repo.mongodb.org\/apt\/ubuntu focal\/mongodb-org\/4.4 multiverse\" | sudo tee \/etc\/apt\/sources.list.d\/mongodb-org-4.4.list\nsudo apt-get update\nsudo apt-get install -y mongodb-org\nps --no-headers -o comm 1\nsudo systemctl start mongod\nsudo systemctl status mongod\nsudo systemctl enable mongod\n\ncd mongodb-api\nvirtualenv -p python3 venv\nsource venv\/bin\/activate\n\n## run in a new terminal one time\n#python3 create-db.py\n\n## run in this terminal after db created\n#python3 apiserver.py\n\n::::::::::::::\ncreate-db.py\n::::::::::::::\n# create-db.py\n\nfrom pymongo import MongoClient\n\nclient = MongoClient('mongodb:\/\/localhost:27017\/')\n\nmydb = client['restic-db']\n\nimport datetime\n## example json output from a restic backup run\n#{\"message_type\":\"summary\",\"files_new\":2,\"files_changed\":1,\"files_unmodified\":205021,\"dirs_new\":0,\"dirs_changed\":6,\"dirs_unmodified\":28000,\"data_blobs\":3,\"tree_blobs\":7,\"data_added\":17802,\"total_files_processed\":205024,\"total_bytes_processed\":6002872966,\"total_duration\":29.527921058,\"snapshot_id\":\"84de0f00\"\n\nmyrecord = {\n \"message_type\": \"summary\",\n \"files_new\":2,\n \"files_changed\":1,\n \"files_unmodified\":205021,\n \"dirs_new\":0,\n \"dirs_changed\":6,\n \"dirs_unmodified\":28000,\n \"data_blobs\":3,\n \"tree_blobs\":7,\n \"data_added\":17802,\n \"total_files_processed\":205024,\n \"total_bytes_processed\":6002872966,\n \"total_duration\":29.527921058,\n \"snapshot_id\":\"84de0f00\",\n \"tags\" : [\"prod\", \"daily\", \"weekly\"],\n \"date_inserted\" : datetime.datetime.utcnow(),\n \"hostname\" : \"desktop01\"\n }\n\nrecord_id = mydb.logs.insert_one(myrecord)\n\nprint (record_id)\nprint (mydb.list_collection_names())\n::::::::::::::\napiserver.py\n::::::::::::::\n# mongo.py\n\nfrom flask import Flask\nfrom flask import jsonify\nfrom flask import request\nfrom flask_pymongo import PyMongo\n\napp = Flask(__name__)\n\napp.config['MONGO_DBNAME'] = 'restic-db'\napp.config['MONGO_URI'] = 'mongodb:\/\/localhost:27017\/restic-db'\n\nmongo = PyMongo(app)\n\n@app.route('\/')\ndef index():\n return jsonify({'result' : 'v0.9.0'})\n\n@app.route('\/entry', methods=['GET'])\ndef get_all_entries():\n entry = mongo.db.logs\n output = []\n for s in entry.find():\n output.append({'hostname' : s['hostname'], 'message_type' : s['message_type']})\n return jsonify({'result' : output})\n\n@app.route('\/entry\/', methods=['GET'])\ndef get_one_entry(hostname):\n entry = mongo.db.logs\n s = entry.find_one({'hostname' : hostname})\n if s:\n output = {'hostname' : s['hostname'], 'message_type' : s['message_type']}\n else:\n output = \"No such hostname\"\n return jsonify({'result' : output})\n\n@app.route('\/entry', methods=['POST'])\ndef add_entry():\n entry = mongo.db.logs\n #details = request.get_json()\n #hostname = details[\"hostname\"]\n hostname = request.json['hostname']\n #hostname = \"'\" + str(request.values.get(\"hostname\")) + \"'\"\n message_type = request.json['message_type']\n #message_type = \"'\" + str(request.values.get(\"message_type\")) + \"'\"\n #message_type = details.message_type\n\n import datetime\n\n ## example json output from a restic backup run #{\"message_type\":\"summary\",\"files_new\":2,\"files_changed\":1,\"files_unmodified\":205021,\"dirs_new\":0,\"dirs_changed\":6,\"dirs_unmodified\":28000,\"data_blobs\":3,\"tree_blobs\":7,\"data_added\":17802,\"total_files_processed\":205024,\"total_bytes_processed\":6002872966,\"total_duration\":29.527921058,\"snapshot_id\":\"84de0f00\"}\n\n myrecord = {\n \"message_type\": message_type,\n \"files_new\":2,\n \"files_changed\":1,\n \"files_unmodified\":205021,\n \"dirs_new\":0,\n \"dirs_changed\":6,\n \"dirs_unmodified\":28000,\n \"data_blobs\":3,\n \"tree_blobs\":7,\n \"data_added\":17802,\n \"total_files_processed\":205024,\n \"total_bytes_processed\":6002872966,\n \"total_duration\":29.527921058,\n \"snapshot_id\":\"84de0f00\",\n \"tags\" : [\"prod\", \"daily\", \"weekly\"],\n \"date_inserted\" : datetime.datetime.utcnow(),\n \"hostname\" : hostname\n }\n\n #entry_id = entry.insert_one({'hostname': hostname, 'message_type': message_type})\n entry_id = entry.insert_one(myrecord)\n\n new_entry = entry.find_one({'_id': entry_id.inserted_id })\n output = {'hostname' : new_entry['hostname'], 'message_type' : new_entry['message_type']}\n return jsonify({'result' : output})\n\nif __name__ == '__main__':\n #app.run(debug=True)\n app.run (host = \"192.168.1.235\", port = 5000)<\/code><\/pre>\nTest API's from my desktop using Postman<\/h2>\n\n- GET to http:\/\/192.168.1.235:5000\/entry<\/a><\/li>\n
- GET to http:\/\/192.168.1.235:5000\/entry\/desktop1<\/a><\/li>\n
- POST to http:\/\/192.168.1.235:5000\/entry<\/a> (need to add raw body json data as well as header Content-Type: application\/json)<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"
Python Flask API Using MongoDB After a recent experiment I did to capture the output of jobs with json output<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[126,127,125,13],"tags":[],"class_list":["post-1715","post","type-post","status-publish","format-standard","hentry","category-api","category-flask","category-mongodb","category-python"],"_links":{"self":[{"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/posts\/1715","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/comments?post=1715"}],"version-history":[{"count":0,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/posts\/1715\/revisions"}],"wp:attachment":[{"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/media?parent=1715"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/categories?post=1715"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.ls-al.com\/wp-json\/wp\/v2\/tags?post=1715"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}