From d0946bde2220acee61e925abf09d26ba4830f158 Mon Sep 17 00:00:00 2001 From: Akashdeep Dhar Date: Mar 05 2021 05:40:10 +0000 Subject: Added passive metric storage into Redis container Signed-off-by: Akashdeep Dhar --- diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..bc73a29 --- /dev/null +++ b/__init__.py @@ -0,0 +1,22 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +__version__ = "v1.2.0-beta" diff --git a/base/back.py b/base/back.py index 8d6956a..e7712ce 100644 --- a/base/back.py +++ b/base/back.py @@ -30,6 +30,23 @@ class ProcessHandler: def __init__(self, prociden): self.prociden = prociden + def return_process_listing_info(self): + """ + Returns process listing information + """ + procstmp = psutil.process_iter(["pid", "name", "username", "memory_percent", "cpu_percent"]) + retndata = {} + for indx in procstmp: + singlist = { + "pid": indx.info["pid"], + "name": indx.info["name"], + "username": indx.info["username"], + "memory_percent": indx.info["memory_percent"], + "cpu_percent": indx.info["cpu_percent"], + } + retndata[indx.info["pid"]] = singlist + return retndata + def return_process_info(self): """ Returns process information @@ -272,23 +289,6 @@ class LiveUpdatingElements: retndata[indx] = singlist return retndata - def get_process_listing_info(self): - """ - Returns process listing information - """ - procstmp = psutil.process_iter(["pid", "name", "username", "memory_percent", "cpu_percent"]) - retndata = {} - for indx in procstmp: - singlist = { - "pid": indx.info["pid"], - "name": indx.info["name"], - "username": indx.info["username"], - "memory_percent": indx.info["memory_percent"], - "cpu_percent": indx.info["cpu_percent"], - } - retndata[indx.info["pid"]] = singlist - return retndata - def get_sensors_temperature(self): """ Returns thermal statistics @@ -356,7 +356,6 @@ class LiveUpdatingElements: "cpuclock": self.get_cpu_clock_speed(), "diousage": self.get_disk_io_usage(), "netusage": self.get_network_io_usage(), - "procinfo": self.get_process_listing_info(), "sensread": { "senstemp": self.get_sensors_temperature(), "fanspeed": self.get_sensors_fan_speed(), @@ -458,7 +457,6 @@ class DeadUpdatingElements(LiveUpdatingElements): "netaddrs": self.get_network_if_addresses(), "netstats": self.get_network_statistics(), "boottime": self.get_boot_time(), - "procinfo": self.get_process_listing_info(), "sensread": { "senstemp": self.get_sensors_temperature(), "fanspeed": self.get_sensors_fan_speed(), diff --git a/base/mtrc.py b/base/mtrc.py new file mode 100644 index 0000000..b5eeed7 --- /dev/null +++ b/base/mtrc.py @@ -0,0 +1,115 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +import json +from hashlib import sha256 +from sys import exit +from time import ctime, sleep, time + +import falcon +from base.back import LiveUpdatingElements +from click import echo +from redis import Redis + + +class MetricsRetrievingEndpoint(object): + def __init__(self, passcode, duration, recsqant): + """ + Initialize storage connection + """ + self.baseobjc = Redis(host="storaged", port=6379) + self.passcode = passcode + self.duration = duration + self.recsqant = recsqant + + def on_get(self, rqst, resp): + """ + Endpoint for retrieving metrics + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + mtrciden = rqst.get_param("mtrciden") + if passcode == self.passcode: + try: + if opername == "LIST": + mtrckeys = [indx.decode() for indx in self.baseobjc.keys()] + mtrckeys.sort() + retnjson = { + "duration": self.duration, + "recsqant": self.recsqant, + "mtrclist": mtrckeys + } + elif opername == "IDEN": + mtrciden = str(mtrciden).encode() + retnjson = json.loads(self.baseobjc.get(mtrciden)) + else: + retnjson = {"retnmesg": "deny"} + except Exception: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 + + +class GatherMetricToStorage(object): + def __init__(self, duration, recsqant): + """ + Initialize storage connection + """ + echo(" * Initializing metric fetch system...") + self.baseobjc = Redis(host="storaged", port=6379) + self.duration = duration + self.recsqant = recsqant + + def jsonify_system_live_updating_metrics(self): + """ + Convert metric data to a JSON-friendly format + """ + timestmp = str(time()).split(".")[0] + hashiden = sha256(timestmp.encode()).hexdigest() + keyvalpr = { + timestmp: json.dumps({ + "hashiden": hashiden, + "liveupdt": LiveUpdatingElements().return_live_data() + }) + } + return keyvalpr + + def continuously_store_data(self): + """ + Periodically push passive metrics to Redis store + """ + self.baseobjc.flushall() + try: + while True: + if self.baseobjc.dbsize() == self.recsqant: + self.baseobjc.keys().sort() + self.baseobjc.delete(self.baseobjc.keys()[0]) + self.baseobjc.mset(self.jsonify_system_live_updating_metrics()) + echo(" * [" + ctime() + "] Stored system metrics now...") + sleep(self.duration) + except KeyboardInterrupt as expt: + self.baseobjc.close() + echo("\n" + " * Closing storage connection...") + exit() diff --git a/falc.py b/falc.py index 3dd7eb7..250f62b 100644 --- a/falc.py +++ b/falc.py @@ -25,12 +25,13 @@ from secrets import choice import click import falcon +from __init__ import __version__ as drivvers from base.frnt import ( ProcessControllingEndpoint, ProcessHandlingEndpoint, StatisticalEndpoint, ) -from dish.term import mainterm +from base.mtrc import GatherMetricToStorage, MetricsRetrievingEndpoint from dish.frnt import ( ContainerInformationEndpoint, ImageInformationEndpoint, @@ -38,9 +39,11 @@ from dish.frnt import ( PreliminaryInformationEndpoint, VolumeInformationEndpoint, ) +from dish.term import mainterm from docker import __version__ as dockvers from falcon import __version__ as flcnvers from psutil import __version__ as psutvers +from redis import __version__ as redsvers from terminado import __version__ as termvers from werkzeug import __version__ as wkzgvers from werkzeug import serving @@ -60,6 +63,10 @@ class ConnectionExaminationEndpoint(object): self.passcode = passcode def on_get(self, rqst, resp): + """ + Endpoint for testing connection attempts + Method: GET + """ passcode = rqst.get_param("passcode") if passcode == self.passcode: retnjson = {"retnmesg": "allow"} @@ -100,6 +107,20 @@ class ConnectionExaminationEndpoint(object): help="Start the server on an IPv6 address." ) @click.option( + "-d", + "--duration", + "duration", + help="Set the timeperiod for metric storage.", + default=10 +) +@click.option( + "-q", + "--recsqant", + "recsqant", + help="Set the number of maintained records.", + default=1000 +) +@click.option( "-4", "--ipprotv4", "netprotc", @@ -107,11 +128,14 @@ class ConnectionExaminationEndpoint(object): help="Start the server on an IPv4 address." ) @click.version_option( - version="1.1.0-beta", + version=drivvers, prog_name=click.style("SuperVisor Driver Service", fg="magenta") ) -def mainfunc(portdata, sockport, netprotc, unixsock): +def mainfunc(portdata, sockport, netprotc, duration, recsqant, unixsock): try: + """ + Initial prompt display + """ click.echo( click.style( " ,---. . ,o \n" + @@ -121,7 +145,7 @@ def mainfunc(portdata, sockport, netprotc, unixsock): " |", bold=True ) ) - click.echo(" * " + click.style("Driver Service v1.1.0-beta", fg="green")) + click.echo(" * " + click.style("Driver Service " + drivvers, fg="green")) netpdata = "" passcode = ConnectionManager().passphrase_generator() if netprotc == "ipprotv6": @@ -138,6 +162,7 @@ def mainfunc(portdata, sockport, netprotc, unixsock): "/" + "\n" + " * " + click.style("Monitor service ", bold=True) + ": " + "Psutil v" + psutvers + "\n" + " * " + click.style("Container service ", bold=True) + ": " + "DockerPy v" + dockvers + "\n" + + " * " + click.style("Datastore service ", bold=True) + ": " + "RedisPy v" + redsvers + "\n" + " * " + click.style("WebSocket service ", bold=True) + ": " + "Terminado v" + termvers + "\n" + " * " + click.style("Endpoint service ", bold=True) + ": " + "Falcon v" + flcnvers + "\n" + " * " + click.style("HTTP server ", bold=True) + ": " + "Werkzeug v" + wkzgvers @@ -151,6 +176,7 @@ def mainfunc(portdata, sockport, netprotc, unixsock): dishntwk = NetworkInformationEndpoint(passcode, unixsock) dishvolm = VolumeInformationEndpoint(passcode, unixsock) testconn = ConnectionExaminationEndpoint(passcode) + mtrcrecv = MetricsRetrievingEndpoint(passcode, duration, recsqant) main.add_route("/basestat", basestat) main.add_route("/basepsin", basepsin) main.add_route("/basetool", basetool) @@ -160,8 +186,12 @@ def mainfunc(portdata, sockport, netprotc, unixsock): main.add_route("/dishntwk", dishntwk) main.add_route("/dishvolm", dishvolm) main.add_route("/testconn", testconn) + main.add_route("/mtrcrecv", mtrcrecv) + prdcgthr = GatherMetricToStorage(duration, recsqant) sockproc = Process(target=mainterm, args=(sockport,)) + ftchproc = Process(target=prdcgthr.continuously_store_data) sockproc.start() + ftchproc.start() serving.run_simple(netpdata, int(portdata), main) sockproc.terminate() except Exception as expt: