Riaan's SysAdmin Blog

My tips, howtos, gotchas, snippets and stuff. Use at your own risk!

APIFlaskMongoDBPython

Python Flask API Using MongoDB

admin

Python Flask API Using MongoDB

After a recent experiment I did to capture the output of jobs with json output from multiple server's to a central collector, I kept my notes for future reference. In short I used my restic backup json format to submit into MOngoDB via a little custom API.

The notes are showing scripts I used whihc of course may not work in your environment they are for reference only.

Setup

  • ubuntu 20.10 virtualbox host (use my snapshot called pre-setup)
  • run setup.sh from ~/mongodb-api
  • run python3 create-db.py from ~/mongodb-api
  • run python3 apiserver.py from ~/mongodb-api
  • test GET and POST with postman from my desktop

source code

$ more setup.sh create-db.py apiserver.py 
::::::::::::::
setup.sh
::::::::::::::
#!/bin/bash
sudo apt install -y gnupg python3-pip python3-flask python3-flask-mongoengine python3-virtualenv
sudo pip install flask pymongo flask_pymongo
sudo snap install robo3t-snap

wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add -
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.4.list
sudo apt-get update
sudo apt-get install -y mongodb-org
ps --no-headers -o comm 1
sudo systemctl start mongod
sudo systemctl status mongod
sudo systemctl enable mongod

cd mongodb-api
virtualenv -p python3 venv
source venv/bin/activate

## run in a new terminal one time
#python3 create-db.py

## run in this terminal after db created
#python3 apiserver.py

::::::::::::::
create-db.py
::::::::::::::
# create-db.py

from pymongo import MongoClient

client = MongoClient('mongodb://localhost:27017/')

mydb = client['restic-db']

import datetime
## example json output from a restic backup run
#{"message_type":"summary","files_new":2,"files_changed":1,"files_unmodified":205021,"dirs_new":0,"dirs_changed":6,"dirs_unmodified":28000,"data_blobs":3,"tree_blobs":7,"data_added":17802,"total_files_processed":205024,"total_bytes_processed":6002872966,"total_duration":29.527921058,"snapshot_id":"84de0f00"

myrecord = {
        "message_type": "summary",
    "files_new":2,
    "files_changed":1,
    "files_unmodified":205021,
    "dirs_new":0,
    "dirs_changed":6,
    "dirs_unmodified":28000,
    "data_blobs":3,
    "tree_blobs":7,
    "data_added":17802,
    "total_files_processed":205024,
    "total_bytes_processed":6002872966,
    "total_duration":29.527921058,
    "snapshot_id":"84de0f00",
        "tags" : ["prod", "daily", "weekly"],
        "date_inserted" : datetime.datetime.utcnow(),
        "hostname" : "desktop01"
        }

record_id = mydb.logs.insert_one(myrecord)

print (record_id)
print (mydb.list_collection_names())
::::::::::::::
apiserver.py
::::::::::::::
# mongo.py

from flask import Flask
from flask import jsonify
from flask import request
from flask_pymongo import PyMongo

app = Flask(__name__)

app.config['MONGO_DBNAME'] = 'restic-db'
app.config['MONGO_URI'] = 'mongodb://localhost:27017/restic-db'

mongo = PyMongo(app)

@app.route('/')
def index():
  return jsonify({'result' : 'v0.9.0'})

@app.route('/entry', methods=['GET'])
def get_all_entries():
  entry = mongo.db.logs
  output = []
  for s in entry.find():
    output.append({'hostname' : s['hostname'], 'message_type' : s['message_type']})
  return jsonify({'result' : output})

@app.route('/entry/', methods=['GET'])
def get_one_entry(hostname):
  entry = mongo.db.logs
  s = entry.find_one({'hostname' : hostname})
  if s:
    output = {'hostname' : s['hostname'], 'message_type' : s['message_type']}
  else:
    output = "No such hostname"
  return jsonify({'result' : output})

@app.route('/entry', methods=['POST'])
def add_entry():
  entry = mongo.db.logs
  #details = request.get_json()
  #hostname = details["hostname"]
  hostname = request.json['hostname']
  #hostname = "'" + str(request.values.get("hostname")) + "'"
  message_type = request.json['message_type']
  #message_type = "'" + str(request.values.get("message_type")) + "'"
  #message_type = details.message_type

  import datetime

 ## example json output from a restic backup run #{"message_type":"summary","files_new":2,"files_changed":1,"files_unmodified":205021,"dirs_new":0,"dirs_changed":6,"dirs_unmodified":28000,"data_blobs":3,"tree_blobs":7,"data_added":17802,"total_files_processed":205024,"total_bytes_processed":6002872966,"total_duration":29.527921058,"snapshot_id":"84de0f00"}

  myrecord = {
        "message_type": message_type,
        "files_new":2,
        "files_changed":1,
        "files_unmodified":205021,
        "dirs_new":0,
        "dirs_changed":6,
        "dirs_unmodified":28000,
        "data_blobs":3,
        "tree_blobs":7,
        "data_added":17802,
        "total_files_processed":205024,
        "total_bytes_processed":6002872966,
        "total_duration":29.527921058,
        "snapshot_id":"84de0f00",
        "tags" : ["prod", "daily", "weekly"],
        "date_inserted" : datetime.datetime.utcnow(),
        "hostname" : hostname
        }

  #entry_id = entry.insert_one({'hostname': hostname, 'message_type': message_type})
  entry_id = entry.insert_one(myrecord)

  new_entry = entry.find_one({'_id': entry_id.inserted_id })
  output = {'hostname' : new_entry['hostname'], 'message_type' : new_entry['message_type']}
  return jsonify({'result' : output})

if __name__ == '__main__':
    #app.run(debug=True)
    app.run (host = "192.168.1.235", port = 5000)

Test API's from my desktop using Postman

admin

Bio Info for Riaan