From b21d0245bac999272ad158f250e629ef885ad08c Mon Sep 17 00:00:00 2001 From: Akashdeep Dhar Date: May 12 2021 15:50:59 +0000 Subject: Merge pull request #28 from t0xic0der/disallow-virtualenv-directory-image Restructured project layout, disabled logs and corrected typos --- diff --git a/Dockerfile b/Dockerfile index f77e560..a49184f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,10 @@ FROM python:3.8-alpine +LABEL maintainer "Akashdeep Dhar " ENV PYTHONUNBUFFERED=1 -WORKDIR /supervisor-driver-service -COPY . . RUN apk add --no-cache docker gcc musl-dev linux-headers redis +COPY requirements.txt requirements.txt RUN pip install -r requirements.txt +COPY src/svdriver svdriver +WORKDIR /svdriver EXPOSE 8888 6969 -ENTRYPOINT ["python3", "falc.py"] +ENTRYPOINT ["python3", "falc.py", "-d", "10", "-q", "2160", "-p", "8888", "-s", "6969", "-4"] diff --git a/__init__.py b/__init__.py deleted file mode 100644 index bc73a29..0000000 --- a/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -########################################################################## -* -* Copyright © 2019-2021 Akashdeep Dhar -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program. If not, see . -* -########################################################################## -""" - -__version__ = "v1.2.0-beta" diff --git a/base/back.py b/base/back.py deleted file mode 100644 index e7712ce..0000000 --- a/base/back.py +++ /dev/null @@ -1,466 +0,0 @@ -""" -########################################################################## -* -* Copyright © 2019-2021 Akashdeep Dhar -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program. If not, see . -* -########################################################################## -""" - -import getpass -import os -import time - -import psutil - - -class ProcessHandler: - def __init__(self, prociden): - self.prociden = prociden - - def return_process_listing_info(self): - """ - Returns process listing information - """ - procstmp = psutil.process_iter(["pid", "name", "username", "memory_percent", "cpu_percent"]) - retndata = {} - for indx in procstmp: - singlist = { - "pid": indx.info["pid"], - "name": indx.info["name"], - "username": indx.info["username"], - "memory_percent": indx.info["memory_percent"], - "cpu_percent": indx.info["cpu_percent"], - } - retndata[indx.info["pid"]] = singlist - return retndata - - def return_process_info(self): - """ - Returns process information - """ - procstmp = psutil.Process(self.prociden).as_dict() - retndata = { - "pid": procstmp["pid"], - "username": procstmp["username"], - "uids": { - "real": procstmp["uids"].real, - "effective": procstmp["uids"].effective, - "saved": procstmp["uids"].saved - }, - "memory_percent": procstmp["memory_percent"], - "name": procstmp["name"], - "create_time": time.ctime(procstmp["create_time"]), - "num_ctx_switches": { - "voluntary": procstmp["num_ctx_switches"].voluntary, - "involuntary": procstmp["num_ctx_switches"].involuntary, - }, - "cpu_percent": procstmp["cpu_percent"], - "cpu_times": { - "user": procstmp["cpu_times"].user, - "system": procstmp["cpu_times"].system, - "children_user": procstmp["cpu_times"].children_user, - "children_system": procstmp["cpu_times"].children_system, - "iowait": procstmp["cpu_times"].iowait, - }, - "memory_info": { - "rss": procstmp["memory_info"].rss, - "vms": procstmp["memory_info"].vms, - "shared": procstmp["memory_info"].shared, - "text": procstmp["memory_info"].text, - "lib": procstmp["memory_info"].lib, - "data": procstmp["memory_info"].data, - "dirty": procstmp["memory_info"].dirty, - }, - "status": procstmp["status"], - "num_threads": procstmp["num_threads"], - "gids": { - "real": procstmp["gids"].real, - "effective": procstmp["gids"].effective, - "saved": procstmp["gids"].saved, - }, - "terminal": procstmp["terminal"] - } - return retndata - - def get_single_process(self): - """ - Returns information about a single process - """ - try: - return psutil.Process(int(self.prociden)) - except Exception as e: - return str(e) - - def process_killer(self): - """ - Kills process with the requested process ID - """ - singproc = self.get_single_process() - try: - if type(singproc) == psutil.Process: - singproc.kill() - return {"retnmesg": True} - except: - return {"retnmesg": False} - - def process_terminator(self): - """ - Terminates process with the requested process ID - """ - singproc = self.get_single_process() - try: - if type(singproc) == psutil.Process: - singproc.terminate() - return {"retnmesg": True} - except: - return {"retnmesg": False} - - def process_suspender(self): - """ - Suspends process with the requested process ID - """ - singproc = self.get_single_process() - try: - if type(singproc) == psutil.Process: - singproc.suspend() - return {"retnmesg": True} - except: - return {"retnmesg": False} - - def process_resumer(self): - """ - Resumes process with the requested process ID - """ - singproc = self.get_single_process() - try: - if type(singproc) == psutil.Process: - singproc.resume() - return {"retnmesg": True} - except: - return {"retnmesg": False} - - -class LiveUpdatingElements: - def get_virtual_memory_data(self): - """ - Returns physical memory data - """ - bruhdata = psutil.virtual_memory() - retndata = { - "total": bruhdata.total, - "available": bruhdata.available, - "percent": bruhdata.percent, - "used": bruhdata.used, - "active": bruhdata.active, - "inactive": bruhdata.inactive, - "buffers": bruhdata.buffers, - "cached": bruhdata.cached, - "shared": bruhdata.shared, - "slab": bruhdata.slab, - } - return retndata - - def get_swap_memory_info(self): - """ - Returns virtual memory data - """ - swapinfo = psutil.swap_memory() - retndata = { - "total": swapinfo.total, - "used": swapinfo.used, - "free": swapinfo.free, - "percent": swapinfo.percent, - "sin": swapinfo.sin, - "sout": swapinfo.sout, - } - return retndata - - def get_cpu_state_times(self): - """ - Returns CPU state time information - """ - timedata = psutil.cpu_times(percpu=True) - retndata = {} - for indx in range(len(timedata)): - elemobjc = { - "user": timedata[indx].user, - "nice": timedata[indx].nice, - "system": timedata[indx].system, - "idle": timedata[indx].idle, - "iowait": timedata[indx].iowait, - "irq": timedata[indx].irq, - "softirq": timedata[indx].softirq, - "steal": timedata[indx].steal, - "guest": timedata[indx].guest, - "guest_nice": timedata[indx].guest_nice, - } - retndata[indx] = elemobjc - return retndata - - def get_cpu_usage_percent(self): - """ - Returns CPU usage percentage per-core - """ - cpuprcnt = psutil.cpu_percent(percpu=True) - retndata = {} - for indx in range(len(cpuprcnt)): - retndata[indx] = cpuprcnt[indx] - return retndata - - def get_cpu_statistics(self): - """ - Returns CPU statistical information - """ - cpustats = psutil.cpu_stats() - retndata = { - "ctx_switches": cpustats.ctx_switches, - "interrupts": cpustats.interrupts, - "soft_interrupts": cpustats.soft_interrupts, - "syscalls": cpustats.syscalls, - } - return retndata - - def get_cpu_clock_speed(self): - """ - Returns CPU clock speed information per-core - """ - cpuclock = psutil.cpu_freq(percpu=True) - retndata = {} - for indx in range(len(cpuclock)): - singlist = { - "current": cpuclock[indx].current, - "min": cpuclock[indx].min, - "max": cpuclock[indx].max, - } - retndata[indx] = singlist - return retndata - - def get_disk_io_usage(self): - """ - Returns disk IO usage - """ - diousage = psutil.disk_io_counters(perdisk=True) - retndata = {} - for indx in diousage.keys(): - singlist = { - "read_count": diousage[indx].read_count, - "write_count": diousage[indx].write_count, - "read_bytes": diousage[indx].read_bytes, - "write_bytes": diousage[indx].write_bytes, - "read_time": diousage[indx].read_time, - "write_time": diousage[indx].write_time, - "read_merged_count": diousage[indx].read_merged_count, - "write_merged_count": diousage[indx].write_merged_count, - "busy_time": diousage[indx].busy_time, - } - retndata[indx] = singlist - return retndata - - def get_network_io_usage(self): - """ - Returns network IO usage - """ - netusage = psutil.net_io_counters(pernic=True) - retndata = {} - for indx in netusage.keys(): - singlist = { - "bytes_sent": netusage[indx].bytes_sent, - "bytes_recv": netusage[indx].bytes_recv, - "packets_sent": netusage[indx].packets_sent, - "packets_recv": netusage[indx].packets_recv, - "errin": netusage[indx].errin, - "errout": netusage[indx].errout, - "dropin": netusage[indx].dropin, - "dropout": netusage[indx].dropout, - } - retndata[indx] = singlist - return retndata - - def get_sensors_temperature(self): - """ - Returns thermal statistics - """ - senstemp = psutil.sensors_temperatures(fahrenheit=False) - retndata = {} - for indx in senstemp.keys(): - retndata[indx] = [] - for jndx in senstemp[indx]: - singdict = { - "label": jndx.label, - "current": str(jndx.current), - "high": str(jndx.high), - "critical": str(jndx.critical), - } - retndata[indx].append(singdict) - return retndata - - def get_sensors_fan_speed(self): - """ - Returns fan speed information - """ - senstemp = psutil.sensors_fans() - retndata = {} - for indx in senstemp.keys(): - retndata[indx] = [] - for jndx in senstemp[indx]: - singdict = { - "label": jndx.label, - "current": jndx.current - } - retndata[indx].append(singdict) - return retndata - - def get_sensors_battery_status(self): - """ - Returns battery statistics - """ - retndata = {} - try: - battstat = psutil.sensors_battery() - retndata = { - "percent": battstat.percent, - "secsleft": battstat.secsleft, - "power_plugged": battstat.power_plugged, - } - except: - retndata = { - "percent": 0, - "secsleft": 0, - "power_plugged": True, - } - return retndata - - def return_live_data(self): - """ - Returns combined information as a dictionary - """ - jsonobjc = { - "virtdata": self.get_virtual_memory_data(), - "swapinfo": self.get_swap_memory_info(), - "cpustats": self.get_cpu_statistics(), - "cputimes": self.get_cpu_state_times(), - "cpuprcnt": self.get_cpu_usage_percent(), - "cpuclock": self.get_cpu_clock_speed(), - "diousage": self.get_disk_io_usage(), - "netusage": self.get_network_io_usage(), - "sensread": { - "senstemp": self.get_sensors_temperature(), - "fanspeed": self.get_sensors_fan_speed(), - "battstat": self.get_sensors_battery_status(), - } - } - return jsonobjc - - -class DeadUpdatingElements(LiveUpdatingElements): - def get_os_uname_data(self): - """ - Returns static OS information - """ - unamdata = os.uname() - retndata = { - "System name": unamdata.sysname + " " + unamdata.release, - "Host name": unamdata.nodename + " [" + unamdata.machine + "] ", - "Version": unamdata.version, - "Username": getpass.getuser(), - } - return retndata - - def get_cpu_logical_count(self): - """ - Returns CPU core count - """ - cpuquant = psutil.cpu_count(logical=True) - return str(cpuquant) - - def get_all_disk_partitions(self): - """ - Returns disk partitions information - """ - diskpart = psutil.disk_partitions(all=True) - retndata = [] - for indx in diskpart: - singinfo = { - "device": indx.device, - "mountpoint": indx.mountpoint, - "fstype": indx.fstype, - "opts": indx.opts, - } - retndata.append(singinfo) - return retndata - - def get_network_statistics(self): - """ - Returns network statistics - """ - netstats = psutil.net_if_stats() - retndata = {} - for indx in netstats.keys(): - singinfo = { - "isup": netstats[indx].isup, - "duplex": netstats[indx].duplex, - "speed": netstats[indx].speed, - "mtu": netstats[indx].mtu, - } - retndata[indx] = singinfo - return retndata - - def get_network_if_addresses(self): - """ - Returns network addresses - """ - netaddrs = psutil.net_if_addrs() - retndata = {} - for indx in netaddrs.keys(): - retndata[indx] = {} - for jndx in netaddrs[indx]: - addrobjc = { - "address": jndx.address, - "netmask": jndx.netmask, - "broadcast": jndx.broadcast, - "ptp": jndx.ptp, - } - retndata[indx][jndx.family] = addrobjc - return retndata - - def get_boot_time(self): - """ - Returns boot time information - """ - boottime = time.ctime(psutil.boot_time()) - return boottime - - def return_dead_data(self): - """ - Returns combined information as a dictionary - """ - jsonobjc = { - "osnmdata": self.get_os_uname_data(), - "cpuquant": self.get_cpu_logical_count(), - "cpuclock": self.get_cpu_clock_speed(), - "diskpart": self.get_all_disk_partitions(), - "diousage": self.get_disk_io_usage(), - "netusage": self.get_network_io_usage(), - "netaddrs": self.get_network_if_addresses(), - "netstats": self.get_network_statistics(), - "boottime": self.get_boot_time(), - "sensread": { - "senstemp": self.get_sensors_temperature(), - "fanspeed": self.get_sensors_fan_speed(), - "battstat": self.get_sensors_battery_status() - } - } - return jsonobjc diff --git a/base/frnt.py b/base/frnt.py deleted file mode 100644 index 2a0a54b..0000000 --- a/base/frnt.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -########################################################################## -* -* Copyright © 2019-2021 Akashdeep Dhar -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program. If not, see . -* -########################################################################## -""" - -import json - -import falcon -from base.back import DeadUpdatingElements, LiveUpdatingElements, ProcessHandler - - -class StatisticalEndpoint(object): - def __init__(self, passcode): - self.passcode = passcode - - def on_get(self, rqst, resp): - """ - Endpoint for fetching host station information - Method: GET - """ - passcode = rqst.get_param("passcode") - opername = rqst.get_param("opername") - if passcode == self.passcode: - if opername == "livesync": - retnjson = LiveUpdatingElements().return_live_data() - elif opername == "deadsync": - retnjson = DeadUpdatingElements().return_dead_data() - else: - retnjson = {"retnmesg": "deny"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 - - -class ProcessHandlingEndpoint(object): - def __init__(self, passcode): - self.passcode = passcode - - def on_get(self, rqst, resp): - """ - Endpoint for fetching information about a specific process - Method: GET - """ - passcode = rqst.get_param("passcode") - opername = rqst.get_param("opername") - prociden = rqst.get_param("prociden") - if passcode == self.passcode: - try: - prociden = int(prociden) - if opername == "INFO": - retnjson = ProcessHandler(int(prociden)).return_process_info() - elif opername == "LIST": - retnjson = ProcessHandler(0).return_process_listing_info() - else: - retnjson = {"retnmesg": "deny"} - except Exception: - retnjson = {"retnmesg": "deny"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 - - -class ProcessControllingEndpoint(object): - def __init__(self, passcode): - self.passcode = passcode - - def on_get(self, rqst, resp): - """ - Endpoint for controlling specific processes - Method: GET - """ - passcode = rqst.get_param("passcode") - opername = rqst.get_param("opername") - prociden = rqst.get_param("prociden") - if passcode == self.passcode: - try: - prociden = int(prociden) - if opername == "KILL": - retnjson = ProcessHandler(prociden).process_killer() - elif opername == "TERM": - retnjson = ProcessHandler(prociden).process_terminator() - elif opername == "HANG": - retnjson = ProcessHandler(prociden).process_suspender() - elif opername == "CONT": - retnjson = ProcessHandler(prociden).process_resumer() - else: - retnjson = {"retnmesg": "deny"} - except Exception: - retnjson = {"retnmesg": "deny"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 diff --git a/base/mtrc.py b/base/mtrc.py deleted file mode 100644 index c845f68..0000000 --- a/base/mtrc.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -########################################################################## -* -* Copyright © 2019-2021 Akashdeep Dhar -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program. If not, see . -* -########################################################################## -""" - -import json -from hashlib import sha256 -from os import system -from sys import exit -from time import ctime, sleep, time - -import falcon -from base.back import LiveUpdatingElements -from click import echo -from redis import Redis - - -class RedisDatastoreServerSetup(): - def __init__(self, portnumb, protmode): - self.portnumb = str(portnumb) - if protmode: - self.protmode = "yes" - else: - self.protmode = "no" - - def execute_redis_server_process(self): - """ - Starting a local Redis datastore server at port 6379 and disabled protected mode - """ - try: - echo(" * Starting Redis datastore server...") - system("redis-server --port " + self.portnumb + " --protected-mode " + self.protmode) - except KeyboardInterrupt: - echo("\n" + " * Stopped Redis datastore server...") - exit() - - -class MetricsRetrievingEndpoint(object): - def __init__(self, passcode, duration, recsqant): - """ - Initialize storage connection - """ - self.baseobjc = Redis(host="127.0.0.1", port=6379) - self.passcode = passcode - self.duration = duration - self.recsqant = recsqant - - def on_get(self, rqst, resp): - """ - Endpoint for retrieving metrics - Method: GET - """ - passcode = rqst.get_param("passcode") - opername = rqst.get_param("opername") - mtrciden = rqst.get_param("mtrciden") - if passcode == self.passcode: - try: - if opername == "LIST": - mtrckeys = [indx.decode() for indx in self.baseobjc.keys()] - mtrckeys.sort() - retnjson = { - "duration": self.duration, - "recsqant": self.recsqant, - "mtrclist": mtrckeys - } - elif opername == "IDEN": - mtrciden = str(mtrciden).encode() - retnjson = json.loads(self.baseobjc.get(mtrciden)) - else: - retnjson = {"retnmesg": "deny"} - except Exception: - retnjson = {"retnmesg": "deny"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 - - -class GatherMetricToStorage(object): - def __init__(self, duration, recsqant): - """ - Initialize storage connection - """ - echo(" * Initializing metric fetch system...") - self.baseobjc = Redis(host="127.0.0.1", port=6379) - self.duration = duration - self.recsqant = recsqant - - def jsonify_system_live_updating_metrics(self): - """ - Convert metric data to a JSON-friendly format - """ - timestmp = str(time()).split(".")[0] - hashiden = sha256(timestmp.encode()).hexdigest() - keyvalpr = { - timestmp: json.dumps({ - "hashiden": hashiden, - "liveupdt": LiveUpdatingElements().return_live_data() - }) - } - return keyvalpr - - def continuously_store_data(self): - """ - Periodically push passive metrics to Redis store - """ - self.baseobjc.flushall() - try: - while True: - if self.baseobjc.dbsize() == self.recsqant: - self.baseobjc.keys().sort() - self.baseobjc.delete(self.baseobjc.keys()[0]) - self.baseobjc.mset(self.jsonify_system_live_updating_metrics()) - echo(" * [" + ctime() + "] Stored system metrics now...") - sleep(self.duration) - except KeyboardInterrupt as expt: - self.baseobjc.close() - echo("\n" + " * Closing storage connection...") - exit() diff --git a/dish/back.py b/dish/back.py deleted file mode 100644 index 899504d..0000000 --- a/dish/back.py +++ /dev/null @@ -1,264 +0,0 @@ -""" -########################################################################## -* -* Copyright © 2019-2021 Akashdeep Dhar -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program. If not, see . -* -########################################################################## -""" - -from docker import DockerClient - - -class DockerPreliminaryInformation: - def __init__(self, unixsock): - self.clinobjc = DockerClient(base_url=unixsock) - - def get_docker_info(self): - """ - Returns container station information - """ - return self.clinobjc.info() - - def get_docker_version(self): - """ - Returns container station versioning - """ - return self.clinobjc.version() - - -class DockerContainerInformation: - def __init__(self, unixsock): - self.clinobjc = DockerClient(base_url=unixsock) - - def get_container_list(self): - """ - Returns list of containers - """ - contlist = self.clinobjc.containers.list(all=True) - dispdict = {} - for indx in contlist: - dispdict[indx.short_id] = { - "id": indx.id, - "name": indx.name, - } - return dispdict - - def get_per_container_static_information(self, contiden): - """ - Returns preliminary information of a selected container - """ - try: - contobjc = self.clinobjc.containers.get(contiden) - try: - imejname = contobjc.image.tags[0] - except: - imejname = "UNAVAILABLE" - dispdict = { - "short_id": contobjc.short_id, - "id": contobjc.id, - "name": contobjc.name, - "attrs": contobjc.attrs, - "labels": contobjc.labels, - "ports": contobjc.ports, - "status": contobjc.status, - "image": { - "name": imejname, - "short_id": contobjc.image.short_id - } - } - except: - dispdict = { - "retnmesg": "deny" - } - return dispdict - - def get_per_container_logs_data(self, contiden): - """ - Returns logging information of a selected container - """ - try: - contobjc = self.clinobjc.containers.get(contiden) - dispdict = { - "logs": contobjc.logs(stream=False).decode() - } - except: - dispdict = { - "retnmesg": "deny" - } - return dispdict - - def get_per_container_top_data(self, contiden): - """ - Returns list of processes running in a selected container - """ - try: - contobjc = self.clinobjc.containers.get(contiden) - dispdict = { - "top": contobjc.top() - } - except: - dispdict = { - "retnmesg": "deny" - } - return dispdict - - def get_per_container_statistics(self, contiden): - """ - Returns hardware statistics of a selected container - """ - try: - contobjc = self.clinobjc.containers.get(contiden) - dispdict = { - "stats": contobjc.stats(stream=False) - } - except: - dispdict = { - "retnmesg": "deny" - } - return dispdict - - -class DockerImageInformation: - def __init__(self, unixsock): - self.clinobjc = DockerClient(base_url=unixsock) - - def get_image_list(self): - """ - Returns list of images - """ - imejlist = self.clinobjc.images.list(all=True) - dispdict = {} - for indx in imejlist: - try: - imejname = indx.tags[0] - except: - imejname = "UNAVAILABLE" - dispdict[indx.short_id] = { - "id": indx.id, - "name": imejname, - } - return dispdict - - def get_per_image_static_information(self, imejiden): - """ - Returns preliminary information of a selected image - """ - try: - imejobjc = self.clinobjc.images.get(imejiden) - try: - imejname = imejobjc.tags[0] - except: - imejname = "UNAVAILABLE" - dispdict = { - "short_id": imejobjc.short_id, - "id": imejobjc.id, - "name": imejname, - "attrs": imejobjc.attrs, - "labels": imejobjc.labels, - "tags": imejobjc.tags - } - except: - dispdict = { - "retnmesg": "deny" - } - return dispdict - - def get_per_image_revision_history(self, imejiden): - """ - Returns revision history of a selected container - """ - try: - imejobjc = self.clinobjc.images.get(imejiden) - dispdict = { - "history": imejobjc.history() - } - except: - dispdict = { - "retnmesg": "deny" - } - return dispdict - - -class DockerNetworkInformation: - def __init__(self, unixsock): - self.clinobjc = DockerClient(base_url=unixsock) - - def get_network_list(self): - """ - Returns list of networks - """ - ntwklist = self.clinobjc.networks.list() - dispdict = {} - for indx in ntwklist: - dispdict[indx.short_id] = { - "id": indx.id, - "name": indx.name, - } - return dispdict - - def get_per_network_static_information(self, ntwkiden): - """ - Returns preliminary information of a selected network - """ - try: - ntwkobjc = self.clinobjc.networks.get(ntwkiden) - dispdict = { - "short_id": ntwkobjc.short_id, - "id": ntwkobjc.id, - "name": ntwkobjc.name, - "attrs": ntwkobjc.attrs - } - except: - dispdict = { - "retnmesg": "deny" - } - return dispdict - - -class DockerVolumeInformation: - def __init__(self, unixsock): - self.clinobjc = DockerClient(base_url=unixsock) - - def get_volume_list(self): - """ - Returns list of volumes - """ - volmlist = self.clinobjc.volumes.list() - dispdict = {} - for indx in volmlist: - dispdict[indx.short_id] = { - "id": indx.id, - "name": indx.name, - } - return dispdict - - def get_per_volume_static_information(self, volmiden): - """ - Returns preliminary information of a selected volume - """ - try: - volmobjc = self.clinobjc.volumes.get(volmiden) - dispdict = { - "short_id": volmobjc.short_id, - "id": volmobjc.id, - "name": volmobjc.name, - "attrs": volmobjc.attrs - } - except: - dispdict = { - "retnmesg": "deny" - } - return dispdict diff --git a/dish/frnt.py b/dish/frnt.py deleted file mode 100644 index 935e9e7..0000000 --- a/dish/frnt.py +++ /dev/null @@ -1,177 +0,0 @@ -""" -########################################################################## -* -* Copyright © 2019-2021 Akashdeep Dhar -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program. If not, see . -* -########################################################################## -""" - -import json - -import falcon -from dish.back import ( - DockerPreliminaryInformation, - DockerContainerInformation, - DockerImageInformation, - DockerNetworkInformation, - DockerVolumeInformation -) - - -class PreliminaryInformationEndpoint(object): - def __init__(self, passcode, unixsock): - self.passcode = passcode - self.unixsock = unixsock - - def on_get(self, rqst, resp): - """ - Endpoint for fetching container station information - Method: GET - """ - passcode = rqst.get_param("passcode") - opername = rqst.get_param("opername") - if passcode == self.passcode: - if opername == "INFO": - retnjson = DockerPreliminaryInformation(self.unixsock).get_docker_info() - elif opername == "VERS": - retnjson = DockerPreliminaryInformation(self.unixsock).get_docker_version() - else: - retnjson = {"retnmesg": "deny"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 - - -class ContainerInformationEndpoint(object): - def __init__(self, passcode, unixsock): - self.passcode = passcode - self.unixsock = unixsock - - def on_get(self, rqst, resp): - """ - Endpoint for fetching container information - Method: GET - """ - passcode = rqst.get_param("passcode") - opername = rqst.get_param("opername") - if passcode == self.passcode: - if opername == "LIST": - retnjson = DockerContainerInformation(self.unixsock).get_container_list() - elif opername == "IDEN": - contiden = rqst.get_param("contiden") - retnjson = DockerContainerInformation(self.unixsock).get_per_container_static_information(contiden) - elif opername == "LOGS": - contiden = rqst.get_param("contiden") - retnjson = DockerContainerInformation(self.unixsock).get_per_container_logs_data(contiden) - elif opername == "HTOP": - contiden = rqst.get_param("contiden") - retnjson = DockerContainerInformation(self.unixsock).get_per_container_top_data(contiden) - elif opername == "STAT": - contiden = rqst.get_param("contiden") - retnjson = DockerContainerInformation(self.unixsock).get_per_container_statistics(contiden) - else: - retnjson = {"retnmesg": "deny"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 - - -class ImageInformationEndpoint(object): - def __init__(self, passcode, unixsock): - self.passcode = passcode - self.unixsock = unixsock - - def on_get(self, rqst, resp): - """ - Endpoint for fetching image information - Method: GET - """ - passcode = rqst.get_param("passcode") - opername = rqst.get_param("opername") - if passcode == self.passcode: - if opername == "LIST": - retnjson = DockerImageInformation(self.unixsock).get_image_list() - elif opername == "IDEN": - imejiden = rqst.get_param("imejiden") - retnjson = DockerImageInformation(self.unixsock).get_per_image_static_information(imejiden) - elif opername == "REVS": - imejiden = rqst.get_param("imejiden") - retnjson = DockerImageInformation(self.unixsock).get_per_image_revision_history(imejiden) - else: - retnjson = {"retnmesg": "deny"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 - - -class NetworkInformationEndpoint(object): - def __init__(self, passcode, unixsock): - self.passcode = passcode - self.unixsock = unixsock - - def on_get(self, rqst, resp): - """ - Endpoint for fetching network information - Method: GET - """ - passcode = rqst.get_param("passcode") - opername = rqst.get_param("opername") - if passcode == self.passcode: - if opername == "LIST": - retnjson = DockerNetworkInformation(self.unixsock).get_network_list() - elif opername == "IDEN": - ntwkiden = rqst.get_param("ntwkiden") - retnjson = DockerNetworkInformation(self.unixsock).get_per_network_static_information(ntwkiden) - else: - retnjson = {"retnmesg": "deny"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 - - -class VolumeInformationEndpoint(object): - def __init__(self, passcode, unixsock): - self.passcode = passcode - self.unixsock = unixsock - - def on_get(self, rqst, resp): - """ - Endpoint for fetching volume information - Method: GET - """ - passcode = rqst.get_param("passcode") - opername = rqst.get_param("opername") - if passcode == self.passcode: - if opername == "LIST": - retnjson = DockerVolumeInformation(self.unixsock).get_volume_list() - elif opername == "IDEN": - volmiden = rqst.get_param("volmiden") - retnjson = DockerVolumeInformation(self.unixsock).get_per_volume_static_information(volmiden) - else: - retnjson = {"retnmesg": "deny"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 diff --git a/dish/term.py b/dish/term.py deleted file mode 100644 index 77f87c8..0000000 --- a/dish/term.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -########################################################################## -* -* Copyright © 2019-2021 Akashdeep Dhar -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program. If not, see . -* -########################################################################## -""" - -from tornado.ioloop import IOLoop -import tornado.web -from terminado import TermSocket, UniqueTermManager -import sys -from hashlib import sha256 -from click import echo - - -class AttachmentEndpoint(tornado.web.RequestHandler): - def set_default_headers(self): - self.set_header("Access-Control-Allow-Origin", "*") - - def get(self): - try: - contiden = self.get_argument("contiden") - comdexec = self.get_argument("comdexec") - self.write(addhandr(contiden, comdexec)) - except Exception as expt: - echo(" * Console attachment failed! - " + str(expt)) - self.set_header("Access-Control-Allow-Origin", "*") - self.write({"retnmesg": "deny"}) - - -termmngr = UniqueTermManager(shell_command=["sh"]) - - -handlers = [ - ( - r"/websocket", TermSocket, - { - "term_manager": termmngr - } - ), - ( - r"/atchcons/", AttachmentEndpoint - ) -] - - -def mainterm(portqant): - try: - global mainobjc - mainobjc = tornado.web.Application(handlers) - echo(" * TermSocket started on port " + portqant) - mainobjc.listen(portqant, "0.0.0.0") - IOLoop.instance().start() - except KeyboardInterrupt: - echo("\n" + " * Shutting down on SIGINT") - finally: - sys.exit() - - -def addhandr(contiden, comdexec): - try: - echo(" * " + comdexec + " attached to " + contiden) - urlpatrn = sha256((contiden + comdexec).encode()).hexdigest() - comdexec = comdexec.split() - stndexec = ["docker", "exec", "-ti", contiden] - for indx in comdexec: - stndexec.append(indx) - mainobjc.add_handlers( - r".*", # match any host - [ - ( - r"/" + urlpatrn, TermSocket, - { - "term_manager": UniqueTermManager(shell_command=stndexec) - } - ) - ] - ) - return { - "retnmesg": "allow", - "urlpatrn": urlpatrn - } - except Exception as expt: - echo(" * Failed to attach terminal" + "\n" + str(expt)) - return { - "retnmesg": "deny" - } diff --git a/docker-compose.yml b/docker-compose.yml index b6e82dd..f45c132 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,5 +7,4 @@ services: - "8888:8888" - "6969:6969" volumes: - - ".:/supervisor-driver-service" - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/falc.py b/falc.py deleted file mode 100644 index 1acb985..0000000 --- a/falc.py +++ /dev/null @@ -1,220 +0,0 @@ -""" -########################################################################## -* -* Copyright © 2019-2021 Akashdeep Dhar -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program. If not, see . -* -########################################################################## -""" - -import json -from multiprocessing import Process -from secrets import choice -from time import sleep - -import click -import falcon -from __init__ import __version__ as drivvers -from base.frnt import ( - ProcessControllingEndpoint, - ProcessHandlingEndpoint, - StatisticalEndpoint, -) -from base.mtrc import ( - GatherMetricToStorage, - MetricsRetrievingEndpoint, - RedisDatastoreServerSetup, -) -from dish.frnt import ( - ContainerInformationEndpoint, - ImageInformationEndpoint, - NetworkInformationEndpoint, - PreliminaryInformationEndpoint, - VolumeInformationEndpoint, -) -from dish.term import mainterm -from docker import __version__ as dockvers -from falcon import __version__ as flcnvers -from psutil import __version__ as psutvers -from redis import __version__ as redsvers -from terminado import __version__ as termvers -from werkzeug import __version__ as wkzgvers -from werkzeug import serving - - -main = falcon.API() - - -class ConnectionManager: - def passphrase_generator(self, lent=16): - """ - Function to randomly generate a 16-character long hexadecimal passcode - """ - retndata = "".join(choice("ABCDEF0123456789") for indx in range(lent)) - return retndata - - -class ConnectionExaminationEndpoint(object): - def __init__(self, passcode): - self.passcode = passcode - - def on_get(self, rqst, resp): - """ - Endpoint for testing connection attempts - Method: GET - """ - passcode = rqst.get_param("passcode") - if passcode == self.passcode: - retnjson = {"retnmesg": "allow"} - else: - retnjson = {"retnmesg": "deny"} - resp.body = json.dumps(retnjson, ensure_ascii=False) - resp.set_header("Access-Control-Allow-Origin", "*") - resp.status = falcon.HTTP_200 - - -@click.command() -@click.option( - "-p", - "--portdata", - "portdata", - help="Set the port value for synchronization [0-65536].", - default="8888" -) -@click.option( - "-s", - "--sockport", - "sockport", - help="Set the port value for termsocket [0-65536].", - default="6969" -) -@click.option( - "-u", - "--unixsock", - "unixsock", - help="Set the UNIX socket for Docker.", - default="unix://var/run/docker.sock" -) -@click.option( - "-6", - "--ipprotv6", - "netprotc", - flag_value="ipprotv6", - help="Start the server on an IPv6 address." -) -@click.option( - "-d", - "--duration", - "duration", - help="Set the timeperiod for metric storage.", - default=10 -) -@click.option( - "-q", - "--recsqant", - "recsqant", - help="Set the number of maintained records.", - default=2160 -) -@click.option( - "-4", - "--ipprotv4", - "netprotc", - flag_value="ipprotv4", - help="Start the server on an IPv4 address." -) -@click.version_option( - version=drivvers, - prog_name=click.style("SuperVisor Driver Service", fg="magenta") -) -def mainfunc(portdata, sockport, netprotc, duration, recsqant, unixsock): - try: - """ - Initial prompt display - """ - click.echo( - click.style( - " ,---. . ,o \n" + - " `---.. .,---.,---.,---.| |.,---.,---.,---.\n" + - " || || ||---'| \ / |`---.| || \n" + - " `---'`---'|---'`---'` `' ``---'`---'` \n" + - " |", bold=True - ) - ) - click.echo(" * " + click.style("Driver Service " + drivvers, fg="green")) - netpdata = "" - passcode = ConnectionManager().passphrase_generator() - if netprotc == "ipprotv6": - click.echo(" * " + click.style("IP version ", fg="magenta") + ": " + "6") - netpdata = "::" - elif netprotc == "ipprotv4": - click.echo(" * " + click.style("IP version ", fg="magenta") + ": " + "4") - netpdata = "0.0.0.0" - click.echo( - " * " + click.style("Passcode ", bold=True) + ": " + passcode + "\n" + - " * " + click.style("Sync URI ", bold=True) + ": " + "http://" + netpdata + ":" + portdata + - "/" + "\n" + - " * " + click.style("TermSocket URI ", bold=True) + ": " + "http://" + netpdata + ":" + sockport + - "/" + "\n" + - " * " + click.style("Monitor service ", bold=True) + ": " + "Psutil v" + psutvers + "\n" + - " * " + click.style("Container service ", bold=True) + ": " + "DockerPy v" + dockvers + "\n" + - " * " + click.style("Datastore service ", bold=True) + ": " + "RedisPy v" + redsvers + "\n" + - " * " + click.style("WebSocket service ", bold=True) + ": " + "Terminado v" + termvers + "\n" + - " * " + click.style("Endpoint service ", bold=True) + ": " + "Falcon v" + flcnvers + "\n" + - " * " + click.style("HTTP server ", bold=True) + ": " + "Werkzeug v" + wkzgvers - ) - basestat = StatisticalEndpoint(passcode) - basepsin = ProcessHandlingEndpoint(passcode) - basetool = ProcessControllingEndpoint(passcode) - dishplim = PreliminaryInformationEndpoint(passcode, unixsock) - dishcont = ContainerInformationEndpoint(passcode, unixsock) - dishimej = ImageInformationEndpoint(passcode, unixsock) - dishntwk = NetworkInformationEndpoint(passcode, unixsock) - dishvolm = VolumeInformationEndpoint(passcode, unixsock) - testconn = ConnectionExaminationEndpoint(passcode) - mtrcrecv = MetricsRetrievingEndpoint(passcode, duration, recsqant) - main.add_route("/basestat", basestat) - main.add_route("/basepsin", basepsin) - main.add_route("/basetool", basetool) - main.add_route("/dishplim", dishplim) - main.add_route("/dishcont", dishcont) - main.add_route("/dishimej", dishimej) - main.add_route("/dishntwk", dishntwk) - main.add_route("/dishvolm", dishvolm) - main.add_route("/testconn", testconn) - main.add_route("/mtrcrecv", mtrcrecv) - # Start the Redis datastore server as a subprocess - redsobjc = RedisDatastoreServerSetup(6379, False) - rediserv = Process(target=redsobjc.execute_redis_server_process) - rediserv.start() - # Including mandatory sleep defaulted to 1 second for the Redis process to get started - # Might need some more time in slower devices - sleep(1) - # Start the termsocket server as a subprocess - sockproc = Process(target=mainterm, args=(sockport,)) - sockproc.start() - # Start the periodic storage of metrics as a subprocess - prdcgthr = GatherMetricToStorage(duration, recsqant) - ftchproc = Process(target=prdcgthr.continuously_store_data) - ftchproc.start() - # Start the JSON API server as the main process - serving.run_simple(netpdata, int(portdata), main) - sockproc.terminate() - except Exception as expt: - click.echo(" * " + click.style("Error occurred : " + str(expt), fg="red")) - - -if __name__ == "__main__": - mainfunc() diff --git a/src/svdriver/__init__.py b/src/svdriver/__init__.py new file mode 100644 index 0000000..34fef65 --- /dev/null +++ b/src/svdriver/__init__.py @@ -0,0 +1,22 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +__version__ = "v1.2.1-beta" diff --git a/src/svdriver/base/back.py b/src/svdriver/base/back.py new file mode 100644 index 0000000..e7712ce --- /dev/null +++ b/src/svdriver/base/back.py @@ -0,0 +1,466 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +import getpass +import os +import time + +import psutil + + +class ProcessHandler: + def __init__(self, prociden): + self.prociden = prociden + + def return_process_listing_info(self): + """ + Returns process listing information + """ + procstmp = psutil.process_iter(["pid", "name", "username", "memory_percent", "cpu_percent"]) + retndata = {} + for indx in procstmp: + singlist = { + "pid": indx.info["pid"], + "name": indx.info["name"], + "username": indx.info["username"], + "memory_percent": indx.info["memory_percent"], + "cpu_percent": indx.info["cpu_percent"], + } + retndata[indx.info["pid"]] = singlist + return retndata + + def return_process_info(self): + """ + Returns process information + """ + procstmp = psutil.Process(self.prociden).as_dict() + retndata = { + "pid": procstmp["pid"], + "username": procstmp["username"], + "uids": { + "real": procstmp["uids"].real, + "effective": procstmp["uids"].effective, + "saved": procstmp["uids"].saved + }, + "memory_percent": procstmp["memory_percent"], + "name": procstmp["name"], + "create_time": time.ctime(procstmp["create_time"]), + "num_ctx_switches": { + "voluntary": procstmp["num_ctx_switches"].voluntary, + "involuntary": procstmp["num_ctx_switches"].involuntary, + }, + "cpu_percent": procstmp["cpu_percent"], + "cpu_times": { + "user": procstmp["cpu_times"].user, + "system": procstmp["cpu_times"].system, + "children_user": procstmp["cpu_times"].children_user, + "children_system": procstmp["cpu_times"].children_system, + "iowait": procstmp["cpu_times"].iowait, + }, + "memory_info": { + "rss": procstmp["memory_info"].rss, + "vms": procstmp["memory_info"].vms, + "shared": procstmp["memory_info"].shared, + "text": procstmp["memory_info"].text, + "lib": procstmp["memory_info"].lib, + "data": procstmp["memory_info"].data, + "dirty": procstmp["memory_info"].dirty, + }, + "status": procstmp["status"], + "num_threads": procstmp["num_threads"], + "gids": { + "real": procstmp["gids"].real, + "effective": procstmp["gids"].effective, + "saved": procstmp["gids"].saved, + }, + "terminal": procstmp["terminal"] + } + return retndata + + def get_single_process(self): + """ + Returns information about a single process + """ + try: + return psutil.Process(int(self.prociden)) + except Exception as e: + return str(e) + + def process_killer(self): + """ + Kills process with the requested process ID + """ + singproc = self.get_single_process() + try: + if type(singproc) == psutil.Process: + singproc.kill() + return {"retnmesg": True} + except: + return {"retnmesg": False} + + def process_terminator(self): + """ + Terminates process with the requested process ID + """ + singproc = self.get_single_process() + try: + if type(singproc) == psutil.Process: + singproc.terminate() + return {"retnmesg": True} + except: + return {"retnmesg": False} + + def process_suspender(self): + """ + Suspends process with the requested process ID + """ + singproc = self.get_single_process() + try: + if type(singproc) == psutil.Process: + singproc.suspend() + return {"retnmesg": True} + except: + return {"retnmesg": False} + + def process_resumer(self): + """ + Resumes process with the requested process ID + """ + singproc = self.get_single_process() + try: + if type(singproc) == psutil.Process: + singproc.resume() + return {"retnmesg": True} + except: + return {"retnmesg": False} + + +class LiveUpdatingElements: + def get_virtual_memory_data(self): + """ + Returns physical memory data + """ + bruhdata = psutil.virtual_memory() + retndata = { + "total": bruhdata.total, + "available": bruhdata.available, + "percent": bruhdata.percent, + "used": bruhdata.used, + "active": bruhdata.active, + "inactive": bruhdata.inactive, + "buffers": bruhdata.buffers, + "cached": bruhdata.cached, + "shared": bruhdata.shared, + "slab": bruhdata.slab, + } + return retndata + + def get_swap_memory_info(self): + """ + Returns virtual memory data + """ + swapinfo = psutil.swap_memory() + retndata = { + "total": swapinfo.total, + "used": swapinfo.used, + "free": swapinfo.free, + "percent": swapinfo.percent, + "sin": swapinfo.sin, + "sout": swapinfo.sout, + } + return retndata + + def get_cpu_state_times(self): + """ + Returns CPU state time information + """ + timedata = psutil.cpu_times(percpu=True) + retndata = {} + for indx in range(len(timedata)): + elemobjc = { + "user": timedata[indx].user, + "nice": timedata[indx].nice, + "system": timedata[indx].system, + "idle": timedata[indx].idle, + "iowait": timedata[indx].iowait, + "irq": timedata[indx].irq, + "softirq": timedata[indx].softirq, + "steal": timedata[indx].steal, + "guest": timedata[indx].guest, + "guest_nice": timedata[indx].guest_nice, + } + retndata[indx] = elemobjc + return retndata + + def get_cpu_usage_percent(self): + """ + Returns CPU usage percentage per-core + """ + cpuprcnt = psutil.cpu_percent(percpu=True) + retndata = {} + for indx in range(len(cpuprcnt)): + retndata[indx] = cpuprcnt[indx] + return retndata + + def get_cpu_statistics(self): + """ + Returns CPU statistical information + """ + cpustats = psutil.cpu_stats() + retndata = { + "ctx_switches": cpustats.ctx_switches, + "interrupts": cpustats.interrupts, + "soft_interrupts": cpustats.soft_interrupts, + "syscalls": cpustats.syscalls, + } + return retndata + + def get_cpu_clock_speed(self): + """ + Returns CPU clock speed information per-core + """ + cpuclock = psutil.cpu_freq(percpu=True) + retndata = {} + for indx in range(len(cpuclock)): + singlist = { + "current": cpuclock[indx].current, + "min": cpuclock[indx].min, + "max": cpuclock[indx].max, + } + retndata[indx] = singlist + return retndata + + def get_disk_io_usage(self): + """ + Returns disk IO usage + """ + diousage = psutil.disk_io_counters(perdisk=True) + retndata = {} + for indx in diousage.keys(): + singlist = { + "read_count": diousage[indx].read_count, + "write_count": diousage[indx].write_count, + "read_bytes": diousage[indx].read_bytes, + "write_bytes": diousage[indx].write_bytes, + "read_time": diousage[indx].read_time, + "write_time": diousage[indx].write_time, + "read_merged_count": diousage[indx].read_merged_count, + "write_merged_count": diousage[indx].write_merged_count, + "busy_time": diousage[indx].busy_time, + } + retndata[indx] = singlist + return retndata + + def get_network_io_usage(self): + """ + Returns network IO usage + """ + netusage = psutil.net_io_counters(pernic=True) + retndata = {} + for indx in netusage.keys(): + singlist = { + "bytes_sent": netusage[indx].bytes_sent, + "bytes_recv": netusage[indx].bytes_recv, + "packets_sent": netusage[indx].packets_sent, + "packets_recv": netusage[indx].packets_recv, + "errin": netusage[indx].errin, + "errout": netusage[indx].errout, + "dropin": netusage[indx].dropin, + "dropout": netusage[indx].dropout, + } + retndata[indx] = singlist + return retndata + + def get_sensors_temperature(self): + """ + Returns thermal statistics + """ + senstemp = psutil.sensors_temperatures(fahrenheit=False) + retndata = {} + for indx in senstemp.keys(): + retndata[indx] = [] + for jndx in senstemp[indx]: + singdict = { + "label": jndx.label, + "current": str(jndx.current), + "high": str(jndx.high), + "critical": str(jndx.critical), + } + retndata[indx].append(singdict) + return retndata + + def get_sensors_fan_speed(self): + """ + Returns fan speed information + """ + senstemp = psutil.sensors_fans() + retndata = {} + for indx in senstemp.keys(): + retndata[indx] = [] + for jndx in senstemp[indx]: + singdict = { + "label": jndx.label, + "current": jndx.current + } + retndata[indx].append(singdict) + return retndata + + def get_sensors_battery_status(self): + """ + Returns battery statistics + """ + retndata = {} + try: + battstat = psutil.sensors_battery() + retndata = { + "percent": battstat.percent, + "secsleft": battstat.secsleft, + "power_plugged": battstat.power_plugged, + } + except: + retndata = { + "percent": 0, + "secsleft": 0, + "power_plugged": True, + } + return retndata + + def return_live_data(self): + """ + Returns combined information as a dictionary + """ + jsonobjc = { + "virtdata": self.get_virtual_memory_data(), + "swapinfo": self.get_swap_memory_info(), + "cpustats": self.get_cpu_statistics(), + "cputimes": self.get_cpu_state_times(), + "cpuprcnt": self.get_cpu_usage_percent(), + "cpuclock": self.get_cpu_clock_speed(), + "diousage": self.get_disk_io_usage(), + "netusage": self.get_network_io_usage(), + "sensread": { + "senstemp": self.get_sensors_temperature(), + "fanspeed": self.get_sensors_fan_speed(), + "battstat": self.get_sensors_battery_status(), + } + } + return jsonobjc + + +class DeadUpdatingElements(LiveUpdatingElements): + def get_os_uname_data(self): + """ + Returns static OS information + """ + unamdata = os.uname() + retndata = { + "System name": unamdata.sysname + " " + unamdata.release, + "Host name": unamdata.nodename + " [" + unamdata.machine + "] ", + "Version": unamdata.version, + "Username": getpass.getuser(), + } + return retndata + + def get_cpu_logical_count(self): + """ + Returns CPU core count + """ + cpuquant = psutil.cpu_count(logical=True) + return str(cpuquant) + + def get_all_disk_partitions(self): + """ + Returns disk partitions information + """ + diskpart = psutil.disk_partitions(all=True) + retndata = [] + for indx in diskpart: + singinfo = { + "device": indx.device, + "mountpoint": indx.mountpoint, + "fstype": indx.fstype, + "opts": indx.opts, + } + retndata.append(singinfo) + return retndata + + def get_network_statistics(self): + """ + Returns network statistics + """ + netstats = psutil.net_if_stats() + retndata = {} + for indx in netstats.keys(): + singinfo = { + "isup": netstats[indx].isup, + "duplex": netstats[indx].duplex, + "speed": netstats[indx].speed, + "mtu": netstats[indx].mtu, + } + retndata[indx] = singinfo + return retndata + + def get_network_if_addresses(self): + """ + Returns network addresses + """ + netaddrs = psutil.net_if_addrs() + retndata = {} + for indx in netaddrs.keys(): + retndata[indx] = {} + for jndx in netaddrs[indx]: + addrobjc = { + "address": jndx.address, + "netmask": jndx.netmask, + "broadcast": jndx.broadcast, + "ptp": jndx.ptp, + } + retndata[indx][jndx.family] = addrobjc + return retndata + + def get_boot_time(self): + """ + Returns boot time information + """ + boottime = time.ctime(psutil.boot_time()) + return boottime + + def return_dead_data(self): + """ + Returns combined information as a dictionary + """ + jsonobjc = { + "osnmdata": self.get_os_uname_data(), + "cpuquant": self.get_cpu_logical_count(), + "cpuclock": self.get_cpu_clock_speed(), + "diskpart": self.get_all_disk_partitions(), + "diousage": self.get_disk_io_usage(), + "netusage": self.get_network_io_usage(), + "netaddrs": self.get_network_if_addresses(), + "netstats": self.get_network_statistics(), + "boottime": self.get_boot_time(), + "sensread": { + "senstemp": self.get_sensors_temperature(), + "fanspeed": self.get_sensors_fan_speed(), + "battstat": self.get_sensors_battery_status() + } + } + return jsonobjc diff --git a/src/svdriver/base/frnt.py b/src/svdriver/base/frnt.py new file mode 100644 index 0000000..2a0a54b --- /dev/null +++ b/src/svdriver/base/frnt.py @@ -0,0 +1,114 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +import json + +import falcon +from base.back import DeadUpdatingElements, LiveUpdatingElements, ProcessHandler + + +class StatisticalEndpoint(object): + def __init__(self, passcode): + self.passcode = passcode + + def on_get(self, rqst, resp): + """ + Endpoint for fetching host station information + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + if passcode == self.passcode: + if opername == "livesync": + retnjson = LiveUpdatingElements().return_live_data() + elif opername == "deadsync": + retnjson = DeadUpdatingElements().return_dead_data() + else: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 + + +class ProcessHandlingEndpoint(object): + def __init__(self, passcode): + self.passcode = passcode + + def on_get(self, rqst, resp): + """ + Endpoint for fetching information about a specific process + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + prociden = rqst.get_param("prociden") + if passcode == self.passcode: + try: + prociden = int(prociden) + if opername == "INFO": + retnjson = ProcessHandler(int(prociden)).return_process_info() + elif opername == "LIST": + retnjson = ProcessHandler(0).return_process_listing_info() + else: + retnjson = {"retnmesg": "deny"} + except Exception: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 + + +class ProcessControllingEndpoint(object): + def __init__(self, passcode): + self.passcode = passcode + + def on_get(self, rqst, resp): + """ + Endpoint for controlling specific processes + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + prociden = rqst.get_param("prociden") + if passcode == self.passcode: + try: + prociden = int(prociden) + if opername == "KILL": + retnjson = ProcessHandler(prociden).process_killer() + elif opername == "TERM": + retnjson = ProcessHandler(prociden).process_terminator() + elif opername == "HANG": + retnjson = ProcessHandler(prociden).process_suspender() + elif opername == "CONT": + retnjson = ProcessHandler(prociden).process_resumer() + else: + retnjson = {"retnmesg": "deny"} + except Exception: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 diff --git a/src/svdriver/base/mtrc.py b/src/svdriver/base/mtrc.py new file mode 100644 index 0000000..36d942f --- /dev/null +++ b/src/svdriver/base/mtrc.py @@ -0,0 +1,136 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +import json +from hashlib import sha256 +from os import system +from sys import exit +from time import ctime, sleep, time + +import falcon +from base.back import LiveUpdatingElements +from click import echo +from redis import Redis + + +class RedisDatastoreServerSetup(): + def __init__(self, portnumb, protmode): + self.portnumb = str(portnumb) + if protmode: + self.protmode = "yes" + else: + self.protmode = "no" + + def execute_redis_server_process(self): + """ + Starting a local Redis datastore server at port 6379 and disabled protected mode + """ + try: + echo(" * Starting Redis datastore server...") + system("redis-server --port " + self.portnumb + " --protected-mode " + self.protmode) + except KeyboardInterrupt: + echo("\n" + " * Stopped Redis datastore server...") + exit() + + +class MetricsRetrievingEndpoint(object): + def __init__(self, passcode, duration, recsqant): + """ + Initialize storage connection + """ + self.baseobjc = Redis(host="127.0.0.1", port=6379) + self.passcode = passcode + self.duration = duration + self.recsqant = recsqant + + def on_get(self, rqst, resp): + """ + Endpoint for retrieving metrics + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + mtrciden = rqst.get_param("mtrciden") + if passcode == self.passcode: + try: + if opername == "LIST": + mtrckeys = [indx.decode() for indx in self.baseobjc.keys()] + mtrckeys.sort() + retnjson = { + "duration": self.duration, + "recsqant": self.recsqant, + "mtrclist": mtrckeys + } + elif opername == "IDEN": + mtrciden = str(mtrciden).encode() + retnjson = json.loads(self.baseobjc.get(mtrciden)) + else: + retnjson = {"retnmesg": "deny"} + except Exception: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 + + +class GatherMetricToStorage(object): + def __init__(self, duration, recsqant): + """ + Initialize storage connection + """ + echo(" * Initializing metric fetch system...") + self.baseobjc = Redis(host="127.0.0.1", port=6379) + self.duration = duration + self.recsqant = recsqant + + def jsonify_system_live_updating_metrics(self): + """ + Convert metric data to a JSON-friendly format + """ + timestmp = str(time()).split(".")[0] + hashiden = sha256(timestmp.encode()).hexdigest() + keyvalpr = { + timestmp: json.dumps({ + "hashiden": hashiden, + "liveupdt": LiveUpdatingElements().return_live_data() + }) + } + return keyvalpr + + def continuously_store_data(self): + """ + Periodically push passive metrics to Redis store + """ + self.baseobjc.flushall() + try: + while True: + if self.baseobjc.dbsize() == self.recsqant: + self.baseobjc.keys().sort() + self.baseobjc.delete(self.baseobjc.keys()[0]) + self.baseobjc.mset(self.jsonify_system_live_updating_metrics()) + # echo(" * [" + ctime() + "] Stored system metrics now...") + sleep(self.duration) + except KeyboardInterrupt as expt: + self.baseobjc.close() + echo("\n" + " * Closing storage connection...") + exit() diff --git a/src/svdriver/dish/back.py b/src/svdriver/dish/back.py new file mode 100644 index 0000000..899504d --- /dev/null +++ b/src/svdriver/dish/back.py @@ -0,0 +1,264 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +from docker import DockerClient + + +class DockerPreliminaryInformation: + def __init__(self, unixsock): + self.clinobjc = DockerClient(base_url=unixsock) + + def get_docker_info(self): + """ + Returns container station information + """ + return self.clinobjc.info() + + def get_docker_version(self): + """ + Returns container station versioning + """ + return self.clinobjc.version() + + +class DockerContainerInformation: + def __init__(self, unixsock): + self.clinobjc = DockerClient(base_url=unixsock) + + def get_container_list(self): + """ + Returns list of containers + """ + contlist = self.clinobjc.containers.list(all=True) + dispdict = {} + for indx in contlist: + dispdict[indx.short_id] = { + "id": indx.id, + "name": indx.name, + } + return dispdict + + def get_per_container_static_information(self, contiden): + """ + Returns preliminary information of a selected container + """ + try: + contobjc = self.clinobjc.containers.get(contiden) + try: + imejname = contobjc.image.tags[0] + except: + imejname = "UNAVAILABLE" + dispdict = { + "short_id": contobjc.short_id, + "id": contobjc.id, + "name": contobjc.name, + "attrs": contobjc.attrs, + "labels": contobjc.labels, + "ports": contobjc.ports, + "status": contobjc.status, + "image": { + "name": imejname, + "short_id": contobjc.image.short_id + } + } + except: + dispdict = { + "retnmesg": "deny" + } + return dispdict + + def get_per_container_logs_data(self, contiden): + """ + Returns logging information of a selected container + """ + try: + contobjc = self.clinobjc.containers.get(contiden) + dispdict = { + "logs": contobjc.logs(stream=False).decode() + } + except: + dispdict = { + "retnmesg": "deny" + } + return dispdict + + def get_per_container_top_data(self, contiden): + """ + Returns list of processes running in a selected container + """ + try: + contobjc = self.clinobjc.containers.get(contiden) + dispdict = { + "top": contobjc.top() + } + except: + dispdict = { + "retnmesg": "deny" + } + return dispdict + + def get_per_container_statistics(self, contiden): + """ + Returns hardware statistics of a selected container + """ + try: + contobjc = self.clinobjc.containers.get(contiden) + dispdict = { + "stats": contobjc.stats(stream=False) + } + except: + dispdict = { + "retnmesg": "deny" + } + return dispdict + + +class DockerImageInformation: + def __init__(self, unixsock): + self.clinobjc = DockerClient(base_url=unixsock) + + def get_image_list(self): + """ + Returns list of images + """ + imejlist = self.clinobjc.images.list(all=True) + dispdict = {} + for indx in imejlist: + try: + imejname = indx.tags[0] + except: + imejname = "UNAVAILABLE" + dispdict[indx.short_id] = { + "id": indx.id, + "name": imejname, + } + return dispdict + + def get_per_image_static_information(self, imejiden): + """ + Returns preliminary information of a selected image + """ + try: + imejobjc = self.clinobjc.images.get(imejiden) + try: + imejname = imejobjc.tags[0] + except: + imejname = "UNAVAILABLE" + dispdict = { + "short_id": imejobjc.short_id, + "id": imejobjc.id, + "name": imejname, + "attrs": imejobjc.attrs, + "labels": imejobjc.labels, + "tags": imejobjc.tags + } + except: + dispdict = { + "retnmesg": "deny" + } + return dispdict + + def get_per_image_revision_history(self, imejiden): + """ + Returns revision history of a selected container + """ + try: + imejobjc = self.clinobjc.images.get(imejiden) + dispdict = { + "history": imejobjc.history() + } + except: + dispdict = { + "retnmesg": "deny" + } + return dispdict + + +class DockerNetworkInformation: + def __init__(self, unixsock): + self.clinobjc = DockerClient(base_url=unixsock) + + def get_network_list(self): + """ + Returns list of networks + """ + ntwklist = self.clinobjc.networks.list() + dispdict = {} + for indx in ntwklist: + dispdict[indx.short_id] = { + "id": indx.id, + "name": indx.name, + } + return dispdict + + def get_per_network_static_information(self, ntwkiden): + """ + Returns preliminary information of a selected network + """ + try: + ntwkobjc = self.clinobjc.networks.get(ntwkiden) + dispdict = { + "short_id": ntwkobjc.short_id, + "id": ntwkobjc.id, + "name": ntwkobjc.name, + "attrs": ntwkobjc.attrs + } + except: + dispdict = { + "retnmesg": "deny" + } + return dispdict + + +class DockerVolumeInformation: + def __init__(self, unixsock): + self.clinobjc = DockerClient(base_url=unixsock) + + def get_volume_list(self): + """ + Returns list of volumes + """ + volmlist = self.clinobjc.volumes.list() + dispdict = {} + for indx in volmlist: + dispdict[indx.short_id] = { + "id": indx.id, + "name": indx.name, + } + return dispdict + + def get_per_volume_static_information(self, volmiden): + """ + Returns preliminary information of a selected volume + """ + try: + volmobjc = self.clinobjc.volumes.get(volmiden) + dispdict = { + "short_id": volmobjc.short_id, + "id": volmobjc.id, + "name": volmobjc.name, + "attrs": volmobjc.attrs + } + except: + dispdict = { + "retnmesg": "deny" + } + return dispdict diff --git a/src/svdriver/dish/frnt.py b/src/svdriver/dish/frnt.py new file mode 100644 index 0000000..935e9e7 --- /dev/null +++ b/src/svdriver/dish/frnt.py @@ -0,0 +1,177 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +import json + +import falcon +from dish.back import ( + DockerPreliminaryInformation, + DockerContainerInformation, + DockerImageInformation, + DockerNetworkInformation, + DockerVolumeInformation +) + + +class PreliminaryInformationEndpoint(object): + def __init__(self, passcode, unixsock): + self.passcode = passcode + self.unixsock = unixsock + + def on_get(self, rqst, resp): + """ + Endpoint for fetching container station information + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + if passcode == self.passcode: + if opername == "INFO": + retnjson = DockerPreliminaryInformation(self.unixsock).get_docker_info() + elif opername == "VERS": + retnjson = DockerPreliminaryInformation(self.unixsock).get_docker_version() + else: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 + + +class ContainerInformationEndpoint(object): + def __init__(self, passcode, unixsock): + self.passcode = passcode + self.unixsock = unixsock + + def on_get(self, rqst, resp): + """ + Endpoint for fetching container information + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + if passcode == self.passcode: + if opername == "LIST": + retnjson = DockerContainerInformation(self.unixsock).get_container_list() + elif opername == "IDEN": + contiden = rqst.get_param("contiden") + retnjson = DockerContainerInformation(self.unixsock).get_per_container_static_information(contiden) + elif opername == "LOGS": + contiden = rqst.get_param("contiden") + retnjson = DockerContainerInformation(self.unixsock).get_per_container_logs_data(contiden) + elif opername == "HTOP": + contiden = rqst.get_param("contiden") + retnjson = DockerContainerInformation(self.unixsock).get_per_container_top_data(contiden) + elif opername == "STAT": + contiden = rqst.get_param("contiden") + retnjson = DockerContainerInformation(self.unixsock).get_per_container_statistics(contiden) + else: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 + + +class ImageInformationEndpoint(object): + def __init__(self, passcode, unixsock): + self.passcode = passcode + self.unixsock = unixsock + + def on_get(self, rqst, resp): + """ + Endpoint for fetching image information + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + if passcode == self.passcode: + if opername == "LIST": + retnjson = DockerImageInformation(self.unixsock).get_image_list() + elif opername == "IDEN": + imejiden = rqst.get_param("imejiden") + retnjson = DockerImageInformation(self.unixsock).get_per_image_static_information(imejiden) + elif opername == "REVS": + imejiden = rqst.get_param("imejiden") + retnjson = DockerImageInformation(self.unixsock).get_per_image_revision_history(imejiden) + else: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 + + +class NetworkInformationEndpoint(object): + def __init__(self, passcode, unixsock): + self.passcode = passcode + self.unixsock = unixsock + + def on_get(self, rqst, resp): + """ + Endpoint for fetching network information + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + if passcode == self.passcode: + if opername == "LIST": + retnjson = DockerNetworkInformation(self.unixsock).get_network_list() + elif opername == "IDEN": + ntwkiden = rqst.get_param("ntwkiden") + retnjson = DockerNetworkInformation(self.unixsock).get_per_network_static_information(ntwkiden) + else: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 + + +class VolumeInformationEndpoint(object): + def __init__(self, passcode, unixsock): + self.passcode = passcode + self.unixsock = unixsock + + def on_get(self, rqst, resp): + """ + Endpoint for fetching volume information + Method: GET + """ + passcode = rqst.get_param("passcode") + opername = rqst.get_param("opername") + if passcode == self.passcode: + if opername == "LIST": + retnjson = DockerVolumeInformation(self.unixsock).get_volume_list() + elif opername == "IDEN": + volmiden = rqst.get_param("volmiden") + retnjson = DockerVolumeInformation(self.unixsock).get_per_volume_static_information(volmiden) + else: + retnjson = {"retnmesg": "deny"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 diff --git a/src/svdriver/dish/term.py b/src/svdriver/dish/term.py new file mode 100644 index 0000000..ec825b3 --- /dev/null +++ b/src/svdriver/dish/term.py @@ -0,0 +1,101 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +from tornado.ioloop import IOLoop +import tornado.web +from terminado import TermSocket, UniqueTermManager +import sys +from hashlib import sha256 +from click import echo + + +class AttachmentEndpoint(tornado.web.RequestHandler): + def set_default_headers(self): + self.set_header("Access-Control-Allow-Origin", "*") + + def get(self): + try: + contiden = self.get_argument("contiden") + comdexec = self.get_argument("comdexec") + self.write(addhandr(contiden, comdexec)) + except Exception as expt: + # echo(" * Console attachment failed! - " + str(expt)) + self.set_header("Access-Control-Allow-Origin", "*") + self.write({"retnmesg": "deny"}) + + +termmngr = UniqueTermManager(shell_command=["sh"]) + + +handlers = [ + ( + r"/websocket", TermSocket, + { + "term_manager": termmngr + } + ), + ( + r"/atchcons/", AttachmentEndpoint + ) +] + + +def mainterm(portqant): + try: + global mainobjc + mainobjc = tornado.web.Application(handlers) + echo(" * TermSocket started on port " + portqant) + mainobjc.listen(portqant, "0.0.0.0") + IOLoop.instance().start() + except KeyboardInterrupt: + echo("\n" + " * Shutting down on SIGINT") + finally: + sys.exit() + + +def addhandr(contiden, comdexec): + try: + # echo(" * " + comdexec + " attached to " + contiden) + urlpatrn = sha256((contiden + comdexec).encode()).hexdigest() + comdexec = comdexec.split() + stndexec = ["docker", "exec", "-ti", contiden] + for indx in comdexec: + stndexec.append(indx) + mainobjc.add_handlers( + r".*", # match any host + [ + ( + r"/" + urlpatrn, TermSocket, + { + "term_manager": UniqueTermManager(shell_command=stndexec) + } + ) + ] + ) + return { + "retnmesg": "allow", + "urlpatrn": urlpatrn + } + except Exception as expt: + # echo(" * Failed to attach terminal" + "\n" + str(expt)) + return { + "retnmesg": "deny" + } diff --git a/src/svdriver/falc.py b/src/svdriver/falc.py new file mode 100644 index 0000000..b2a33d5 --- /dev/null +++ b/src/svdriver/falc.py @@ -0,0 +1,223 @@ +""" +########################################################################## +* +* Copyright © 2019-2021 Akashdeep Dhar +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +########################################################################## +""" + +import json +import logging +from multiprocessing import Process +from secrets import choice +from time import sleep + +import click +import falcon +from __init__ import __version__ as drivvers +from base.frnt import ( + ProcessControllingEndpoint, + ProcessHandlingEndpoint, + StatisticalEndpoint, +) +from base.mtrc import ( + GatherMetricToStorage, + MetricsRetrievingEndpoint, + RedisDatastoreServerSetup, +) +from dish.frnt import ( + ContainerInformationEndpoint, + ImageInformationEndpoint, + NetworkInformationEndpoint, + PreliminaryInformationEndpoint, + VolumeInformationEndpoint, +) +from dish.term import mainterm +from docker import __version__ as dockvers +from falcon import __version__ as flcnvers +from psutil import __version__ as psutvers +from redis import __version__ as redsvers +from terminado import __version__ as termvers +from werkzeug import __version__ as wkzgvers +from werkzeug import serving + + +main = falcon.API() +loge = logging.getLogger("werkzeug") +loge.disabled = True + + +class ConnectionManager: + def passphrase_generator(self, lent=16): + """ + Function to randomly generate a 16-character long hexadecimal passcode + """ + retndata = "".join(choice("ABCDEF0123456789") for indx in range(lent)) + return retndata + + +class ConnectionExaminationEndpoint(object): + def __init__(self, passcode): + self.passcode = passcode + + def on_get(self, rqst, resp): + """ + Endpoint for testing connection attempts + Method: GET + """ + passcode = rqst.get_param("passcode") + if passcode == self.passcode: + retnjson = {"retnmesg": "allow"} + else: + retnjson = {"retnmesg": "deny"} + resp.body = json.dumps(retnjson, ensure_ascii=False) + resp.set_header("Access-Control-Allow-Origin", "*") + resp.status = falcon.HTTP_200 + + +@click.command() +@click.option( + "-p", + "--portdata", + "portdata", + help="Set the port value for synchronization [0-65536].", + default="8888" +) +@click.option( + "-s", + "--sockport", + "sockport", + help="Set the port value for termsocket [0-65536].", + default="6969" +) +@click.option( + "-u", + "--unixsock", + "unixsock", + help="Set the UNIX socket for Docker.", + default="unix://var/run/docker.sock" +) +@click.option( + "-6", + "--ipprotv6", + "netprotc", + flag_value="ipprotv6", + help="Start the server on an IPv6 address." +) +@click.option( + "-d", + "--duration", + "duration", + help="Set the timeperiod for metric storage.", + default=10 +) +@click.option( + "-q", + "--recsqant", + "recsqant", + help="Set the number of maintained records.", + default=2160 +) +@click.option( + "-4", + "--ipprotv4", + "netprotc", + flag_value="ipprotv4", + help="Start the server on an IPv4 address." +) +@click.version_option( + version=drivvers, + prog_name=click.style("SuperVisor Driver Service", fg="magenta") +) +def mainfunc(portdata, sockport, netprotc, duration, recsqant, unixsock): + try: + """ + Initial prompt display + """ + click.echo( + click.style( + " ,---. . ,o \n" + + " `---.. .,---.,---.,---.| |.,---.,---.,---.\n" + + " || || ||---'| \ / |`---.| || \n" + + " `---'`---'|---'`---'` `' ``---'`---'` \n" + + " |", bold=True + ) + ) + click.echo(" * " + click.style("Driver Service " + drivvers, fg="green")) + netpdata = "" + passcode = ConnectionManager().passphrase_generator() + if netprotc == "ipprotv6": + click.echo(" * " + click.style("IP version ", fg="magenta") + ": " + "6") + netpdata = "::" + elif netprotc == "ipprotv4": + click.echo(" * " + click.style("IP version ", fg="magenta") + ": " + "4") + netpdata = "0.0.0.0" + click.echo( + " * " + click.style("Passcode ", bold=True) + ": " + passcode + "\n" + + " * " + click.style("Sync URI ", bold=True) + ": " + "http://" + netpdata + ":" + portdata + + "/" + "\n" + + " * " + click.style("TermSocket URI ", bold=True) + ": " + "ws://" + netpdata + ":" + sockport + + "/" + "\n" + + " * " + click.style("Monitor service ", bold=True) + ": " + "Psutil v" + psutvers + "\n" + + " * " + click.style("Container service ", bold=True) + ": " + "DockerPy v" + dockvers + "\n" + + " * " + click.style("Datastore service ", bold=True) + ": " + "RedisPy v" + redsvers + "\n" + + " * " + click.style("WebSocket service ", bold=True) + ": " + "Terminado v" + termvers + "\n" + + " * " + click.style("Endpoint service ", bold=True) + ": " + "Falcon v" + flcnvers + "\n" + + " * " + click.style("HTTP server ", bold=True) + ": " + "Werkzeug v" + wkzgvers + ) + basestat = StatisticalEndpoint(passcode) + basepsin = ProcessHandlingEndpoint(passcode) + basetool = ProcessControllingEndpoint(passcode) + dishplim = PreliminaryInformationEndpoint(passcode, unixsock) + dishcont = ContainerInformationEndpoint(passcode, unixsock) + dishimej = ImageInformationEndpoint(passcode, unixsock) + dishntwk = NetworkInformationEndpoint(passcode, unixsock) + dishvolm = VolumeInformationEndpoint(passcode, unixsock) + testconn = ConnectionExaminationEndpoint(passcode) + mtrcrecv = MetricsRetrievingEndpoint(passcode, duration, recsqant) + main.add_route("/basestat", basestat) + main.add_route("/basepsin", basepsin) + main.add_route("/basetool", basetool) + main.add_route("/dishplim", dishplim) + main.add_route("/dishcont", dishcont) + main.add_route("/dishimej", dishimej) + main.add_route("/dishntwk", dishntwk) + main.add_route("/dishvolm", dishvolm) + main.add_route("/testconn", testconn) + main.add_route("/mtrcrecv", mtrcrecv) + # Start the Redis datastore server as a subprocess + redsobjc = RedisDatastoreServerSetup(6379, False) + rediserv = Process(target=redsobjc.execute_redis_server_process) + rediserv.start() + # Including mandatory sleep defaulted to 1 second for the Redis process to get started + # Might need some more time in slower devices + sleep(1) + # Start the termsocket server as a subprocess + sockproc = Process(target=mainterm, args=(sockport,)) + sockproc.start() + # Start the periodic storage of metrics as a subprocess + prdcgthr = GatherMetricToStorage(duration, recsqant) + ftchproc = Process(target=prdcgthr.continuously_store_data) + ftchproc.start() + # Start the JSON API server as the main process + serving.run_simple(netpdata, int(portdata), main) + sockproc.terminate() + except Exception as expt: + click.echo(" * " + click.style("Error occurred : " + str(expt), fg="red")) + + +if __name__ == "__main__": + mainfunc() diff --git a/src/svdriver/terminado/__init__.py b/src/svdriver/terminado/__init__.py new file mode 100644 index 0000000..48ec12a --- /dev/null +++ b/src/svdriver/terminado/__init__.py @@ -0,0 +1,15 @@ +"""Terminals served to xterm.js using Tornado websockets""" + +# Copyright (c) Jupyter Development Team +# Copyright (c) 2014, Ramalingam Saravanan +# Distributed under the terms of the Simplified BSD License. + +from .websocket import TermSocket +from .management import (TermManagerBase, SingleTermManager, + UniqueTermManager, NamedTermManager) + +import logging +# Prevent a warning about no attached handlers in Python 2 +logging.getLogger(__name__).addHandler(logging.NullHandler()) + +__version__ = '0.9.2' diff --git a/src/svdriver/terminado/_static/terminado.js b/src/svdriver/terminado/_static/terminado.js new file mode 100644 index 0000000..836e354 --- /dev/null +++ b/src/svdriver/terminado/_static/terminado.js @@ -0,0 +1,39 @@ +// Copyright (c) Jupyter Development Team +// Copyright (c) 2014, Ramalingam Saravanan +// Distributed under the terms of the Simplified BSD License. + +function make_terminal(element, size, ws_url) { + var ws = new WebSocket(ws_url); + var term = new Terminal({ + cols: size.cols, + rows: size.rows, + screenKeys: true, + useStyle: true + }); + ws.onopen = function(event) { + ws.send(JSON.stringify(["set_size", size.rows, size.cols, + window.innerHeight, window.innerWidth])); + term.on('data', function(data) { + ws.send(JSON.stringify(['stdin', data])); + }); + + term.on('title', function(title) { + document.title = title; + }); + + term.open(element); + + ws.onmessage = function(event) { + json_msg = JSON.parse(event.data); + switch(json_msg[0]) { + case "stdout": + term.write(json_msg[1]); + break; + case "disconnect": + term.write("\r\n\r\n[Finished... Terminado]\r\n"); + break; + } + }; + }; + return {socket: ws, term: term}; +} diff --git a/src/svdriver/terminado/management.py b/src/svdriver/terminado/management.py new file mode 100644 index 0000000..be811fe --- /dev/null +++ b/src/svdriver/terminado/management.py @@ -0,0 +1,362 @@ +"""Terminal management for exposing terminals to a web interface using Tornado. +""" +# Copyright (c) Jupyter Development Team +# Copyright (c) 2014, Ramalingam Saravanan +# Distributed under the terms of the Simplified BSD License. + +from __future__ import absolute_import, print_function + +import sys +if sys.version_info[0] < 3: + byte_code = ord +else: + def byte_code(x): return x + unicode = str + +from collections import deque +import itertools +import logging +import os +import signal + +try: + from ptyprocess import PtyProcessUnicode +except ImportError: + from winpty import PtyProcess as PtyProcessUnicode + +from tornado import gen +from tornado.ioloop import IOLoop + +ENV_PREFIX = "PYXTERM_" # Environment variable prefix + +DEFAULT_TERM_TYPE = "xterm" + + +class PtyWithClients(object): + def __init__(self, argv, env=[], cwd=None): + self.clients = [] + # If you start the process and then construct this object from it, + # output generated by the process prior to the object's creation + # is lost. Hence the change from 0.8.3. + # Buffer output until a client connects; then let the client + # drain the buffer. + # We keep the same read_buffer as before + self.read_buffer = deque([], maxlen=10) + self.preopen_buffer = deque([]) + self.ptyproc = PtyProcessUnicode.spawn(argv, env=env, cwd=cwd) + + def resize_to_smallest(self): + """Set the terminal size to that of the smallest client dimensions. + + A terminal not using the full space available is much nicer than a + terminal trying to use more than the available space, so we keep it + sized to the smallest client. + """ + minrows = mincols = 10001 + for client in self.clients: + rows, cols = client.size + if rows is not None and rows < minrows: + minrows = rows + if cols is not None and cols < mincols: + mincols = cols + + if minrows == 10001 or mincols == 10001: + return + + rows, cols = self.ptyproc.getwinsize() + if (rows, cols) != (minrows, mincols): + self.ptyproc.setwinsize(minrows, mincols) + + def kill(self, sig=signal.SIGTERM): + """Send a signal to the process in the pty""" + self.ptyproc.kill(sig) + + def killpg(self, sig=signal.SIGTERM): + """Send a signal to the process group of the process in the pty""" + if os.name == 'nt': + return self.ptyproc.kill(sig) + pgid = os.getpgid(self.ptyproc.pid) + os.killpg(pgid, sig) + + @gen.coroutine + def terminate(self, force=False): + '''This forces a child process to terminate. It starts nicely with + SIGHUP and SIGINT. If "force" is True then moves onto SIGKILL. This + returns True if the child was terminated. This returns False if the + child could not be terminated. ''' + if os.name == 'nt': + signals = [signal.SIGINT, signal.SIGTERM] + else: + signals = [signal.SIGHUP, signal.SIGCONT, signal.SIGINT, + signal.SIGTERM] + + loop = IOLoop.current() + def sleep(): return gen.sleep(self.ptyproc.delayafterterminate) + + if not self.ptyproc.isalive(): + raise gen.Return(True) + try: + for sig in signals: + self.kill(sig) + yield sleep() + if not self.ptyproc.isalive(): + raise gen.Return(True) + if force: + self.kill(signal.SIGKILL) + yield sleep() + if not self.ptyproc.isalive(): + raise gen.Return(True) + else: + raise gen.Return(False) + raise gen.Return(False) + except OSError: + # I think there are kernel timing issues that sometimes cause + # this to happen. I think isalive() reports True, but the + # process is dead to the kernel. + # Make one last attempt to see if the kernel is up to date. + yield sleep() + if not self.ptyproc.isalive(): + raise gen.Return(True) + else: + raise gen.Return(False) + + +def _update_removing(target, changes): + """Like dict.update(), but remove keys where the value is None. + """ + for k, v in changes.items(): + if v is None: + target.pop(k, None) + else: + target[k] = v + + +class TermManagerBase(object): + """Base class for a terminal manager.""" + + def __init__(self, shell_command, server_url="", term_settings={}, + extra_env=None, ioloop=None): + self.shell_command = shell_command + self.server_url = server_url + self.term_settings = term_settings + self.extra_env = extra_env + self.log = logging.getLogger(__name__) + + self.ptys_by_fd = {} + + if ioloop is not None: + self.ioloop = ioloop + else: + import tornado.ioloop + self.ioloop = tornado.ioloop.IOLoop.instance() + + def make_term_env(self, height=25, width=80, winheight=0, winwidth=0, **kwargs): + """Build the environment variables for the process in the terminal.""" + env = os.environ.copy() + env["TERM"] = self.term_settings.get("type", DEFAULT_TERM_TYPE) + dimensions = "%dx%d" % (width, height) + if winwidth and winheight: + dimensions += ";%dx%d" % (winwidth, winheight) + env[ENV_PREFIX+"DIMENSIONS"] = dimensions + env["COLUMNS"] = str(width) + env["LINES"] = str(height) + + if self.server_url: + env[ENV_PREFIX+"URL"] = self.server_url + + if self.extra_env: + _update_removing(env, self.extra_env) + + return env + + def new_terminal(self, **kwargs): + """Make a new terminal, return a :class:`PtyWithClients` instance.""" + options = self.term_settings.copy() + options['shell_command'] = self.shell_command + options.update(kwargs) + argv = options['shell_command'] + env = self.make_term_env(**options) + cwd = options.get('cwd', None) + return PtyWithClients(argv, env, cwd) + + def start_reading(self, ptywclients): + """Connect a terminal to the tornado event loop to read data from it.""" + fd = ptywclients.ptyproc.fd + self.ptys_by_fd[fd] = ptywclients + self.ioloop.add_handler(fd, self.pty_read, self.ioloop.READ) + + def on_eof(self, ptywclients): + """Called when the pty has closed. + """ + # Stop trying to read from that terminal + fd = ptywclients.ptyproc.fd + self.log.info("EOF on FD %d; stopping reading", fd) + del self.ptys_by_fd[fd] + self.ioloop.remove_handler(fd) + + # This closes the fd, and should result in the process being reaped. + ptywclients.ptyproc.close() + + def pty_read(self, fd, events=None): + """Called by the event loop when there is pty data ready to read.""" + ptywclients = self.ptys_by_fd[fd] + try: + s = ptywclients.ptyproc.read(65536) + client_list = ptywclients.clients + ptywclients.read_buffer.append(s) + if not client_list: + # No one to consume our output: buffer it. + ptywclients.preopen_buffer.append(s) + return + for client in ptywclients.clients: + client.on_pty_read(s) + except EOFError: + self.on_eof(ptywclients) + for client in ptywclients.clients: + client.on_pty_died() + + def get_terminal(self, url_component=None): + """Override in a subclass to give a terminal to a new websocket connection + + The :class:`TermSocket` handler works with zero or one URL components + (capturing groups in the URL spec regex). If it receives one, it is + passed as the ``url_component`` parameter; otherwise, this is None. + """ + raise NotImplementedError + + def client_disconnected(self, websocket): + """Override this to e.g. kill terminals on client disconnection. + """ + pass + + @gen.coroutine + def shutdown(self): + yield self.kill_all() + + @gen.coroutine + def kill_all(self): + futures = [] + for term in self.ptys_by_fd.values(): + futures.append(term.terminate(force=True)) + # wait for futures to finish + for f in futures: + yield f + + +class SingleTermManager(TermManagerBase): + """All connections to the websocket share a common terminal.""" + + def __init__(self, **kwargs): + super(SingleTermManager, self).__init__(**kwargs) + self.terminal = None + + def get_terminal(self, url_component=None): + if self.terminal is None: + self.terminal = self.new_terminal() + self.start_reading(self.terminal) + return self.terminal + + @gen.coroutine + def kill_all(self): + yield super(SingleTermManager, self).kill_all() + self.terminal = None + + +class MaxTerminalsReached(Exception): + def __init__(self, max_terminals): + self.max_terminals = max_terminals + + def __str__(self): + return "Cannot create more than %d terminals" % self.max_terminals + + +class UniqueTermManager(TermManagerBase): + """Give each websocket a unique terminal to use.""" + + def __init__(self, max_terminals=None, **kwargs): + super(UniqueTermManager, self).__init__(**kwargs) + self.max_terminals = max_terminals + + def get_terminal(self, url_component=None): + if self.max_terminals and len(self.ptys_by_fd) >= self.max_terminals: + raise MaxTerminalsReached(self.max_terminals) + + term = self.new_terminal() + self.start_reading(term) + return term + + def client_disconnected(self, websocket): + """Send terminal SIGHUP when client disconnects.""" + self.log.info("Websocket closed, sending SIGHUP to terminal.") + if websocket.terminal: + if os.name == 'nt': + websocket.terminal.kill() + # Immediately call the pty reader to process + # the eof and free up space + self.pty_read(websocket.terminal.ptyproc.fd) + return + websocket.terminal.killpg(signal.SIGHUP) + + +class NamedTermManager(TermManagerBase): + """Share terminals between websockets connected to the same endpoint. + """ + + def __init__(self, max_terminals=None, **kwargs): + super(NamedTermManager, self).__init__(**kwargs) + self.max_terminals = max_terminals + self.terminals = {} + + def get_terminal(self, term_name): + assert term_name is not None + + if term_name in self.terminals: + return self.terminals[term_name] + + if self.max_terminals and len(self.terminals) >= self.max_terminals: + raise MaxTerminalsReached(self.max_terminals) + + # Create new terminal + self.log.info("New terminal with specified name: %s", term_name) + term = self.new_terminal() + term.term_name = term_name + self.terminals[term_name] = term + self.start_reading(term) + return term + + name_template = "%d" + + def _next_available_name(self): + for n in itertools.count(start=1): + name = self.name_template % n + if name not in self.terminals: + return name + + def new_named_terminal(self, **kwargs): + name = self._next_available_name() + term = self.new_terminal(**kwargs) + self.log.info("New terminal with automatic name: %s", name) + term.term_name = name + self.terminals[name] = term + self.start_reading(term) + return name, term + + def kill(self, name, sig=signal.SIGTERM): + term = self.terminals[name] + term.kill(sig) # This should lead to an EOF + + @gen.coroutine + def terminate(self, name, force=False): + term = self.terminals[name] + yield term.terminate(force=force) + + def on_eof(self, ptywclients): + super(NamedTermManager, self).on_eof(ptywclients) + name = ptywclients.term_name + self.log.info("Terminal %s closed", name) + self.terminals.pop(name, None) + + @gen.coroutine + def kill_all(self): + yield super(NamedTermManager, self).kill_all() + self.terminals = {} diff --git a/src/svdriver/terminado/tests/basic_test.py b/src/svdriver/terminado/tests/basic_test.py new file mode 100644 index 0000000..e748e70 --- /dev/null +++ b/src/svdriver/terminado/tests/basic_test.py @@ -0,0 +1,259 @@ +# basic_tests.py -- Basic unit tests for Terminado + +# Copyright (c) Jupyter Development Team +# Copyright (c) 2014, Ramalingam Saravanan +# Distributed under the terms of the Simplified BSD License. + +from __future__ import absolute_import, print_function + +import unittest +from terminado import * +import tornado +import tornado.httpserver +from tornado.httpclient import HTTPError +from tornado.ioloop import IOLoop +import tornado.testing +import datetime +import logging +import json +import os +import re +import signal + +# We must set the policy for python >=3.8, see https://www.tornadoweb.org/en/stable/#installation +# Snippet from https://github.com/tornadoweb/tornado/issues/2608#issuecomment-619524992 +import sys, asyncio +if sys.version_info[0]==3 and sys.version_info[1] >= 8 and sys.platform.startswith('win'): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +# +# The timeout we use to assume no more messages are coming +# from the sever. +# +DONE_TIMEOUT = 1.0 +os.environ['ASYNC_TEST_TIMEOUT'] = "20" # Global test case timeout + +MAX_TERMS = 3 # Testing thresholds + +class TestTermClient(object): + """Test connection to a terminal manager""" + def __init__(self, websocket): + self.ws = websocket + self.pending_read = None + + @tornado.gen.coroutine + def read_msg(self): + + # Because the Tornado Websocket client has no way to cancel + # a pending read, we have to keep track of them... + if self.pending_read is None: + self.pending_read = self.ws.read_message() + + response = yield self.pending_read + self.pending_read = None + if response: + response = json.loads(response) + raise tornado.gen.Return(response) + + @tornado.gen.coroutine + def read_all_msg(self, timeout=DONE_TIMEOUT): + """Read messages until read times out""" + msglist = [] + delta = datetime.timedelta(seconds=timeout) + while True: + try: + mf = self.read_msg() + msg = yield tornado.gen.with_timeout(delta, mf) + except tornado.gen.TimeoutError: + raise tornado.gen.Return(msglist) + + msglist.append(msg) + + def write_msg(self, msg): + self.ws.write_message(json.dumps(msg)) + + @tornado.gen.coroutine + def read_stdout(self, timeout=DONE_TIMEOUT): + """Read standard output until timeout read reached, + return stdout and any non-stdout msgs received.""" + msglist = yield self.read_all_msg(timeout) + stdout = "".join([msg[1] for msg in msglist if msg[0] == 'stdout']) + othermsg = [msg for msg in msglist if msg[0] != 'stdout'] + raise tornado.gen.Return((stdout, othermsg)) + + def write_stdin(self, data): + """Write to terminal stdin""" + self.write_msg(['stdin', data]) + + @tornado.gen.coroutine + def get_pid(self): + """Get process ID of terminal shell process""" + yield self.read_stdout() # Clear out any pending + self.write_stdin("echo $$\r") + (stdout, extra) = yield self.read_stdout() + if os.name == 'nt': + match = re.search(r'echo \$\$\x1b\[0K\r\n(\d+)', stdout) + pid = int(match.groups()[0]) + else: + pid = int(stdout.split('\n')[1]) + raise tornado.gen.Return(pid) + + def close(self): + self.ws.close() + +class TermTestCase(tornado.testing.AsyncHTTPTestCase): + + # Factory for TestTermClient, because it has to be a Tornado co-routine. + # See: https://github.com/tornadoweb/tornado/issues/1161 + @tornado.gen.coroutine + def get_term_client(self, path): + port = self.get_http_port() + url = 'ws://127.0.0.1:%d%s' % (port, path) + request = tornado.httpclient.HTTPRequest(url, + headers={'Origin' : 'http://127.0.0.1:%d' % port}) + + ws = yield tornado.websocket.websocket_connect(request) + raise tornado.gen.Return(TestTermClient(ws)) + + @tornado.gen.coroutine + def get_term_clients(self, paths): + tms = yield [self.get_term_client(path) for path in paths] + raise tornado.gen.Return(tms) + + @tornado.gen.coroutine + def get_pids(self, tm_list): + pids = [] + for tm in tm_list: # Must be sequential, in case terms are shared + pid = yield tm.get_pid() + pids.append(pid) + + raise tornado.gen.Return(pids) + + def tearDown(self): + self.named_tm.kill_all() + self.single_tm.kill_all() + self.unique_tm.kill_all() + super().tearDown() + + def get_app(self): + self.named_tm = NamedTermManager(shell_command=['bash'], + max_terminals=MAX_TERMS, + ioloop=self.io_loop) + self.single_tm = SingleTermManager(shell_command=['bash'], + ioloop=self.io_loop) + self.unique_tm = UniqueTermManager(shell_command=['bash'], + max_terminals=MAX_TERMS, + ioloop=self.io_loop) + + named_tm = self.named_tm + class NewTerminalHandler(tornado.web.RequestHandler): + """Create a new named terminal, return redirect""" + def get(self): + name, terminal = named_tm.new_named_terminal() + self.redirect("/named/" + name, permanent=False) + + return tornado.web.Application([ + (r"/new", NewTerminalHandler), + (r"/named/(\w+)", TermSocket, {'term_manager': self.named_tm}), + (r"/single", TermSocket, {'term_manager': self.single_tm}), + (r"/unique", TermSocket, {'term_manager': self.unique_tm}) + ], debug=True) + + test_urls = ('/named/term1', '/unique', '/single') + +class CommonTests(TermTestCase): + @tornado.testing.gen_test + def test_basic(self): + for url in self.test_urls: + tm = yield self.get_term_client(url) + response = yield tm.read_msg() + self.assertEqual(response, ['setup', {}]) + + # Check for initial shell prompt + response = yield tm.read_msg() + self.assertEqual(response[0], 'stdout') + self.assertGreater(len(response[1]), 0) + tm.close() + + @tornado.testing.gen_test + def test_basic_command(self): + for url in self.test_urls: + tm = yield self.get_term_client(url) + yield tm.read_all_msg() + tm.write_stdin("whoami\n") + (stdout, other) = yield tm.read_stdout() + if os.name == 'nt': + assert 'whoami' in stdout + else: + assert stdout.startswith('who') + assert other == [] + tm.close() + +class NamedTermTests(TermTestCase): + def test_new(self): + response = self.fetch("/new", follow_redirects=False) + self.assertEqual(response.code, 302) + url = response.headers["Location"] + + # Check that the new terminal was created + name = url.split('/')[2] + self.assertIn(name, self.named_tm.terminals) + + @tornado.testing.gen_test + def test_namespace(self): + names = ["/named/1"]*2 + ["/named/2"]*2 + tms = yield self.get_term_clients(names) + pids = yield self.get_pids(tms) + + self.assertEqual(pids[0], pids[1]) + self.assertEqual(pids[2], pids[3]) + self.assertNotEqual(pids[0], pids[3]) + + @tornado.testing.gen_test + def test_max_terminals(self): + urls = ["/named/%d" % i for i in range(MAX_TERMS+1)] + tms = yield self.get_term_clients(urls[:MAX_TERMS]) + pids = yield self.get_pids(tms) + + # MAX_TERMS+1 should fail + tm = yield self.get_term_client(urls[MAX_TERMS]) + msg = yield tm.read_msg() + self.assertEqual(msg, None) # Connection closed + +class SingleTermTests(TermTestCase): + @tornado.testing.gen_test + def test_single_process(self): + tms = yield self.get_term_clients(["/single", "/single"]) + pids = yield self.get_pids(tms) + self.assertEqual(pids[0], pids[1]) + +class UniqueTermTests(TermTestCase): + @tornado.testing.gen_test + def test_unique_processes(self): + tms = yield self.get_term_clients(["/unique", "/unique"]) + pids = yield self.get_pids(tms) + self.assertNotEqual(pids[0], pids[1]) + + @tornado.testing.gen_test + def test_max_terminals(self): + tms = yield self.get_term_clients(['/unique'] * MAX_TERMS) + pids = yield self.get_pids(tms) + self.assertEqual(len(set(pids)), MAX_TERMS) # All PIDs unique + + # MAX_TERMS+1 should fail + tm = yield self.get_term_client("/unique") + msg = yield tm.read_msg() + self.assertEqual(msg, None) # Connection closed + + # Close one + tms[0].close() + msg = yield tms[0].read_msg() # Closed + self.assertEquals(msg, None) + + # Should be able to open back up to MAX_TERMS + tm = yield self.get_term_client("/unique") + msg = yield tm.read_msg() + self.assertEquals(msg[0], 'setup') + +if __name__ == '__main__': + unittest.main() diff --git a/src/svdriver/terminado/uimod_embed.js b/src/svdriver/terminado/uimod_embed.js new file mode 100644 index 0000000..c517e13 --- /dev/null +++ b/src/svdriver/terminado/uimod_embed.js @@ -0,0 +1,16 @@ +// Copyright (c) Jupyter Development Team +// Copyright (c) 2014, Ramalingam Saravanan +// Distributed under the terms of the Simplified BSD License. + +window.addEventListener('load', function () { + var containers = document.getElementsByClassName('terminado-container') + var container, rows, cols, protocol, ws_url; + for (var i = 0; i < containers.length; i++) { + container = containers[i]; + rows = parseInt(container.dataset.rows); + cols = parseInt(container.dataset.cols); + protocol = (window.location.protocol.indexOf("https") === 0) ? "wss" : "ws"; + ws_url = protocol+"://"+window.location.host+ container.dataset.wsUrl; + make_terminal(container, {rows: rows, cols: cols}, ws_url); + } +}, false); diff --git a/src/svdriver/terminado/uimodule.py b/src/svdriver/terminado/uimodule.py new file mode 100644 index 0000000..584259c --- /dev/null +++ b/src/svdriver/terminado/uimodule.py @@ -0,0 +1,27 @@ +"""A Tornado UI module for a terminal backed by terminado. + +See the Tornado docs for information on UI modules: +http://www.tornadoweb.org/en/stable/guide/templates.html#ui-modules +""" +# Copyright (c) Jupyter Development Team +# Copyright (c) 2014, Ramalingam Saravanan +# Distributed under the terms of the Simplified BSD License. + +import os.path +import tornado.web + +class Terminal(tornado.web.UIModule): + def render(self, ws_url, cols=80, rows=25): + return ('
').format( + ws_url=ws_url, rows=rows, cols=cols) + + def javascript_files(self): + # TODO: Can we calculate these dynamically? + return ['/xstatic/termjs/term.js', '/static/terminado.js'] + + def embedded_javascript(self): + file = os.path.join(os.path.dirname(__file__), 'uimod_embed.js') + with open(file) as f: + return f.read() diff --git a/src/svdriver/terminado/websocket.py b/src/svdriver/terminado/websocket.py new file mode 100644 index 0000000..8330d5f --- /dev/null +++ b/src/svdriver/terminado/websocket.py @@ -0,0 +1,115 @@ +"""Tornado websocket handler to serve a terminal interface. +""" +# Copyright (c) Jupyter Development Team +# Copyright (c) 2014, Ramalingam Saravanan +# Distributed under the terms of the Simplified BSD License. + +from __future__ import absolute_import, print_function + +# Python3-friendly imports +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + +import json +import logging + +import tornado.web +import tornado.websocket + + +def _cast_unicode(s): + if isinstance(s, bytes): + return s.decode('utf-8') + return s + + +class TermSocket(tornado.websocket.WebSocketHandler): + """Handler for a terminal websocket""" + + def initialize(self, term_manager): + self.term_manager = term_manager + self.term_name = "" + self.size = (None, None) + self.terminal = None + + self._logger = logging.getLogger(__name__) + + def check_origin(self, origin: str): + return True + + def origin_check(self, origin=None): + """Deprecated: backward-compat for terminado <= 0.5.""" + return self.check_origin(origin or self.request.headers.get('Origin')) + + def open(self, url_component=None): + """Websocket connection opened. + + Call our terminal manager to get a terminal, and connect to it as a + client. + """ + # Jupyter has a mixin to ping websockets and keep connections through + # proxies alive. Call super() to allow that to set up: + super(TermSocket, self).open(url_component) + + self._logger.info("TermSocket.open: %s", url_component) + + url_component = _cast_unicode(url_component) + self.term_name = url_component or 'tty' + self.terminal = self.term_manager.get_terminal(url_component) + self.terminal.clients.append(self) + self.send_json_message(["setup", {}]) + self._logger.info("TermSocket.open: Opened %s", self.term_name) + # Now drain the preopen buffer, if it exists. + buffered = "" + while True: + if not self.terminal.preopen_buffer: + break + s = self.terminal.preopen_buffer.popleft() + buffered += s + if buffered: + self.on_pty_read(buffered) + + def on_pty_read(self, text): + """Data read from pty; send to frontend""" + self.send_json_message(['stdout', text]) + + def send_json_message(self, content): + json_msg = json.dumps(content) + self.write_message(json_msg) + + def on_message(self, message): + """Handle incoming websocket message + + We send JSON arrays, where the first element is a string indicating + what kind of message this is. Data associated with the message follows. + """ + ##logging.info("TermSocket.on_message: %s - (%s) %s", self.term_name, type(message), len(message) if isinstance(message, bytes) else message[:250]) + command = json.loads(message) + msg_type = command[0] + + if msg_type == "stdin": + self.terminal.ptyproc.write(command[1]) + elif msg_type == "set_size": + self.size = command[1:3] + self.terminal.resize_to_smallest() + + def on_close(self): + """Handle websocket closing. + + Disconnect from our terminal, and tell the terminal manager we're + disconnecting. + """ + self._logger.info("Websocket closed") + if self.terminal: + self.terminal.clients.remove(self) + self.terminal.resize_to_smallest() + self.term_manager.client_disconnected(self) + + def on_pty_died(self): + """Terminal closed: tell the frontend, and close the socket. + """ + self.send_json_message(['disconnect', 1]) + self.close() + self.terminal = None diff --git a/terminado/__init__.py b/terminado/__init__.py deleted file mode 100644 index 48ec12a..0000000 --- a/terminado/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Terminals served to xterm.js using Tornado websockets""" - -# Copyright (c) Jupyter Development Team -# Copyright (c) 2014, Ramalingam Saravanan -# Distributed under the terms of the Simplified BSD License. - -from .websocket import TermSocket -from .management import (TermManagerBase, SingleTermManager, - UniqueTermManager, NamedTermManager) - -import logging -# Prevent a warning about no attached handlers in Python 2 -logging.getLogger(__name__).addHandler(logging.NullHandler()) - -__version__ = '0.9.2' diff --git a/terminado/_static/terminado.js b/terminado/_static/terminado.js deleted file mode 100644 index 836e354..0000000 --- a/terminado/_static/terminado.js +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Jupyter Development Team -// Copyright (c) 2014, Ramalingam Saravanan -// Distributed under the terms of the Simplified BSD License. - -function make_terminal(element, size, ws_url) { - var ws = new WebSocket(ws_url); - var term = new Terminal({ - cols: size.cols, - rows: size.rows, - screenKeys: true, - useStyle: true - }); - ws.onopen = function(event) { - ws.send(JSON.stringify(["set_size", size.rows, size.cols, - window.innerHeight, window.innerWidth])); - term.on('data', function(data) { - ws.send(JSON.stringify(['stdin', data])); - }); - - term.on('title', function(title) { - document.title = title; - }); - - term.open(element); - - ws.onmessage = function(event) { - json_msg = JSON.parse(event.data); - switch(json_msg[0]) { - case "stdout": - term.write(json_msg[1]); - break; - case "disconnect": - term.write("\r\n\r\n[Finished... Terminado]\r\n"); - break; - } - }; - }; - return {socket: ws, term: term}; -} diff --git a/terminado/management.py b/terminado/management.py deleted file mode 100644 index be811fe..0000000 --- a/terminado/management.py +++ /dev/null @@ -1,362 +0,0 @@ -"""Terminal management for exposing terminals to a web interface using Tornado. -""" -# Copyright (c) Jupyter Development Team -# Copyright (c) 2014, Ramalingam Saravanan -# Distributed under the terms of the Simplified BSD License. - -from __future__ import absolute_import, print_function - -import sys -if sys.version_info[0] < 3: - byte_code = ord -else: - def byte_code(x): return x - unicode = str - -from collections import deque -import itertools -import logging -import os -import signal - -try: - from ptyprocess import PtyProcessUnicode -except ImportError: - from winpty import PtyProcess as PtyProcessUnicode - -from tornado import gen -from tornado.ioloop import IOLoop - -ENV_PREFIX = "PYXTERM_" # Environment variable prefix - -DEFAULT_TERM_TYPE = "xterm" - - -class PtyWithClients(object): - def __init__(self, argv, env=[], cwd=None): - self.clients = [] - # If you start the process and then construct this object from it, - # output generated by the process prior to the object's creation - # is lost. Hence the change from 0.8.3. - # Buffer output until a client connects; then let the client - # drain the buffer. - # We keep the same read_buffer as before - self.read_buffer = deque([], maxlen=10) - self.preopen_buffer = deque([]) - self.ptyproc = PtyProcessUnicode.spawn(argv, env=env, cwd=cwd) - - def resize_to_smallest(self): - """Set the terminal size to that of the smallest client dimensions. - - A terminal not using the full space available is much nicer than a - terminal trying to use more than the available space, so we keep it - sized to the smallest client. - """ - minrows = mincols = 10001 - for client in self.clients: - rows, cols = client.size - if rows is not None and rows < minrows: - minrows = rows - if cols is not None and cols < mincols: - mincols = cols - - if minrows == 10001 or mincols == 10001: - return - - rows, cols = self.ptyproc.getwinsize() - if (rows, cols) != (minrows, mincols): - self.ptyproc.setwinsize(minrows, mincols) - - def kill(self, sig=signal.SIGTERM): - """Send a signal to the process in the pty""" - self.ptyproc.kill(sig) - - def killpg(self, sig=signal.SIGTERM): - """Send a signal to the process group of the process in the pty""" - if os.name == 'nt': - return self.ptyproc.kill(sig) - pgid = os.getpgid(self.ptyproc.pid) - os.killpg(pgid, sig) - - @gen.coroutine - def terminate(self, force=False): - '''This forces a child process to terminate. It starts nicely with - SIGHUP and SIGINT. If "force" is True then moves onto SIGKILL. This - returns True if the child was terminated. This returns False if the - child could not be terminated. ''' - if os.name == 'nt': - signals = [signal.SIGINT, signal.SIGTERM] - else: - signals = [signal.SIGHUP, signal.SIGCONT, signal.SIGINT, - signal.SIGTERM] - - loop = IOLoop.current() - def sleep(): return gen.sleep(self.ptyproc.delayafterterminate) - - if not self.ptyproc.isalive(): - raise gen.Return(True) - try: - for sig in signals: - self.kill(sig) - yield sleep() - if not self.ptyproc.isalive(): - raise gen.Return(True) - if force: - self.kill(signal.SIGKILL) - yield sleep() - if not self.ptyproc.isalive(): - raise gen.Return(True) - else: - raise gen.Return(False) - raise gen.Return(False) - except OSError: - # I think there are kernel timing issues that sometimes cause - # this to happen. I think isalive() reports True, but the - # process is dead to the kernel. - # Make one last attempt to see if the kernel is up to date. - yield sleep() - if not self.ptyproc.isalive(): - raise gen.Return(True) - else: - raise gen.Return(False) - - -def _update_removing(target, changes): - """Like dict.update(), but remove keys where the value is None. - """ - for k, v in changes.items(): - if v is None: - target.pop(k, None) - else: - target[k] = v - - -class TermManagerBase(object): - """Base class for a terminal manager.""" - - def __init__(self, shell_command, server_url="", term_settings={}, - extra_env=None, ioloop=None): - self.shell_command = shell_command - self.server_url = server_url - self.term_settings = term_settings - self.extra_env = extra_env - self.log = logging.getLogger(__name__) - - self.ptys_by_fd = {} - - if ioloop is not None: - self.ioloop = ioloop - else: - import tornado.ioloop - self.ioloop = tornado.ioloop.IOLoop.instance() - - def make_term_env(self, height=25, width=80, winheight=0, winwidth=0, **kwargs): - """Build the environment variables for the process in the terminal.""" - env = os.environ.copy() - env["TERM"] = self.term_settings.get("type", DEFAULT_TERM_TYPE) - dimensions = "%dx%d" % (width, height) - if winwidth and winheight: - dimensions += ";%dx%d" % (winwidth, winheight) - env[ENV_PREFIX+"DIMENSIONS"] = dimensions - env["COLUMNS"] = str(width) - env["LINES"] = str(height) - - if self.server_url: - env[ENV_PREFIX+"URL"] = self.server_url - - if self.extra_env: - _update_removing(env, self.extra_env) - - return env - - def new_terminal(self, **kwargs): - """Make a new terminal, return a :class:`PtyWithClients` instance.""" - options = self.term_settings.copy() - options['shell_command'] = self.shell_command - options.update(kwargs) - argv = options['shell_command'] - env = self.make_term_env(**options) - cwd = options.get('cwd', None) - return PtyWithClients(argv, env, cwd) - - def start_reading(self, ptywclients): - """Connect a terminal to the tornado event loop to read data from it.""" - fd = ptywclients.ptyproc.fd - self.ptys_by_fd[fd] = ptywclients - self.ioloop.add_handler(fd, self.pty_read, self.ioloop.READ) - - def on_eof(self, ptywclients): - """Called when the pty has closed. - """ - # Stop trying to read from that terminal - fd = ptywclients.ptyproc.fd - self.log.info("EOF on FD %d; stopping reading", fd) - del self.ptys_by_fd[fd] - self.ioloop.remove_handler(fd) - - # This closes the fd, and should result in the process being reaped. - ptywclients.ptyproc.close() - - def pty_read(self, fd, events=None): - """Called by the event loop when there is pty data ready to read.""" - ptywclients = self.ptys_by_fd[fd] - try: - s = ptywclients.ptyproc.read(65536) - client_list = ptywclients.clients - ptywclients.read_buffer.append(s) - if not client_list: - # No one to consume our output: buffer it. - ptywclients.preopen_buffer.append(s) - return - for client in ptywclients.clients: - client.on_pty_read(s) - except EOFError: - self.on_eof(ptywclients) - for client in ptywclients.clients: - client.on_pty_died() - - def get_terminal(self, url_component=None): - """Override in a subclass to give a terminal to a new websocket connection - - The :class:`TermSocket` handler works with zero or one URL components - (capturing groups in the URL spec regex). If it receives one, it is - passed as the ``url_component`` parameter; otherwise, this is None. - """ - raise NotImplementedError - - def client_disconnected(self, websocket): - """Override this to e.g. kill terminals on client disconnection. - """ - pass - - @gen.coroutine - def shutdown(self): - yield self.kill_all() - - @gen.coroutine - def kill_all(self): - futures = [] - for term in self.ptys_by_fd.values(): - futures.append(term.terminate(force=True)) - # wait for futures to finish - for f in futures: - yield f - - -class SingleTermManager(TermManagerBase): - """All connections to the websocket share a common terminal.""" - - def __init__(self, **kwargs): - super(SingleTermManager, self).__init__(**kwargs) - self.terminal = None - - def get_terminal(self, url_component=None): - if self.terminal is None: - self.terminal = self.new_terminal() - self.start_reading(self.terminal) - return self.terminal - - @gen.coroutine - def kill_all(self): - yield super(SingleTermManager, self).kill_all() - self.terminal = None - - -class MaxTerminalsReached(Exception): - def __init__(self, max_terminals): - self.max_terminals = max_terminals - - def __str__(self): - return "Cannot create more than %d terminals" % self.max_terminals - - -class UniqueTermManager(TermManagerBase): - """Give each websocket a unique terminal to use.""" - - def __init__(self, max_terminals=None, **kwargs): - super(UniqueTermManager, self).__init__(**kwargs) - self.max_terminals = max_terminals - - def get_terminal(self, url_component=None): - if self.max_terminals and len(self.ptys_by_fd) >= self.max_terminals: - raise MaxTerminalsReached(self.max_terminals) - - term = self.new_terminal() - self.start_reading(term) - return term - - def client_disconnected(self, websocket): - """Send terminal SIGHUP when client disconnects.""" - self.log.info("Websocket closed, sending SIGHUP to terminal.") - if websocket.terminal: - if os.name == 'nt': - websocket.terminal.kill() - # Immediately call the pty reader to process - # the eof and free up space - self.pty_read(websocket.terminal.ptyproc.fd) - return - websocket.terminal.killpg(signal.SIGHUP) - - -class NamedTermManager(TermManagerBase): - """Share terminals between websockets connected to the same endpoint. - """ - - def __init__(self, max_terminals=None, **kwargs): - super(NamedTermManager, self).__init__(**kwargs) - self.max_terminals = max_terminals - self.terminals = {} - - def get_terminal(self, term_name): - assert term_name is not None - - if term_name in self.terminals: - return self.terminals[term_name] - - if self.max_terminals and len(self.terminals) >= self.max_terminals: - raise MaxTerminalsReached(self.max_terminals) - - # Create new terminal - self.log.info("New terminal with specified name: %s", term_name) - term = self.new_terminal() - term.term_name = term_name - self.terminals[term_name] = term - self.start_reading(term) - return term - - name_template = "%d" - - def _next_available_name(self): - for n in itertools.count(start=1): - name = self.name_template % n - if name not in self.terminals: - return name - - def new_named_terminal(self, **kwargs): - name = self._next_available_name() - term = self.new_terminal(**kwargs) - self.log.info("New terminal with automatic name: %s", name) - term.term_name = name - self.terminals[name] = term - self.start_reading(term) - return name, term - - def kill(self, name, sig=signal.SIGTERM): - term = self.terminals[name] - term.kill(sig) # This should lead to an EOF - - @gen.coroutine - def terminate(self, name, force=False): - term = self.terminals[name] - yield term.terminate(force=force) - - def on_eof(self, ptywclients): - super(NamedTermManager, self).on_eof(ptywclients) - name = ptywclients.term_name - self.log.info("Terminal %s closed", name) - self.terminals.pop(name, None) - - @gen.coroutine - def kill_all(self): - yield super(NamedTermManager, self).kill_all() - self.terminals = {} diff --git a/terminado/tests/__init__.py b/terminado/tests/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/terminado/tests/__init__.py +++ /dev/null diff --git a/terminado/tests/basic_test.py b/terminado/tests/basic_test.py deleted file mode 100644 index e748e70..0000000 --- a/terminado/tests/basic_test.py +++ /dev/null @@ -1,259 +0,0 @@ -# basic_tests.py -- Basic unit tests for Terminado - -# Copyright (c) Jupyter Development Team -# Copyright (c) 2014, Ramalingam Saravanan -# Distributed under the terms of the Simplified BSD License. - -from __future__ import absolute_import, print_function - -import unittest -from terminado import * -import tornado -import tornado.httpserver -from tornado.httpclient import HTTPError -from tornado.ioloop import IOLoop -import tornado.testing -import datetime -import logging -import json -import os -import re -import signal - -# We must set the policy for python >=3.8, see https://www.tornadoweb.org/en/stable/#installation -# Snippet from https://github.com/tornadoweb/tornado/issues/2608#issuecomment-619524992 -import sys, asyncio -if sys.version_info[0]==3 and sys.version_info[1] >= 8 and sys.platform.startswith('win'): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - -# -# The timeout we use to assume no more messages are coming -# from the sever. -# -DONE_TIMEOUT = 1.0 -os.environ['ASYNC_TEST_TIMEOUT'] = "20" # Global test case timeout - -MAX_TERMS = 3 # Testing thresholds - -class TestTermClient(object): - """Test connection to a terminal manager""" - def __init__(self, websocket): - self.ws = websocket - self.pending_read = None - - @tornado.gen.coroutine - def read_msg(self): - - # Because the Tornado Websocket client has no way to cancel - # a pending read, we have to keep track of them... - if self.pending_read is None: - self.pending_read = self.ws.read_message() - - response = yield self.pending_read - self.pending_read = None - if response: - response = json.loads(response) - raise tornado.gen.Return(response) - - @tornado.gen.coroutine - def read_all_msg(self, timeout=DONE_TIMEOUT): - """Read messages until read times out""" - msglist = [] - delta = datetime.timedelta(seconds=timeout) - while True: - try: - mf = self.read_msg() - msg = yield tornado.gen.with_timeout(delta, mf) - except tornado.gen.TimeoutError: - raise tornado.gen.Return(msglist) - - msglist.append(msg) - - def write_msg(self, msg): - self.ws.write_message(json.dumps(msg)) - - @tornado.gen.coroutine - def read_stdout(self, timeout=DONE_TIMEOUT): - """Read standard output until timeout read reached, - return stdout and any non-stdout msgs received.""" - msglist = yield self.read_all_msg(timeout) - stdout = "".join([msg[1] for msg in msglist if msg[0] == 'stdout']) - othermsg = [msg for msg in msglist if msg[0] != 'stdout'] - raise tornado.gen.Return((stdout, othermsg)) - - def write_stdin(self, data): - """Write to terminal stdin""" - self.write_msg(['stdin', data]) - - @tornado.gen.coroutine - def get_pid(self): - """Get process ID of terminal shell process""" - yield self.read_stdout() # Clear out any pending - self.write_stdin("echo $$\r") - (stdout, extra) = yield self.read_stdout() - if os.name == 'nt': - match = re.search(r'echo \$\$\x1b\[0K\r\n(\d+)', stdout) - pid = int(match.groups()[0]) - else: - pid = int(stdout.split('\n')[1]) - raise tornado.gen.Return(pid) - - def close(self): - self.ws.close() - -class TermTestCase(tornado.testing.AsyncHTTPTestCase): - - # Factory for TestTermClient, because it has to be a Tornado co-routine. - # See: https://github.com/tornadoweb/tornado/issues/1161 - @tornado.gen.coroutine - def get_term_client(self, path): - port = self.get_http_port() - url = 'ws://127.0.0.1:%d%s' % (port, path) - request = tornado.httpclient.HTTPRequest(url, - headers={'Origin' : 'http://127.0.0.1:%d' % port}) - - ws = yield tornado.websocket.websocket_connect(request) - raise tornado.gen.Return(TestTermClient(ws)) - - @tornado.gen.coroutine - def get_term_clients(self, paths): - tms = yield [self.get_term_client(path) for path in paths] - raise tornado.gen.Return(tms) - - @tornado.gen.coroutine - def get_pids(self, tm_list): - pids = [] - for tm in tm_list: # Must be sequential, in case terms are shared - pid = yield tm.get_pid() - pids.append(pid) - - raise tornado.gen.Return(pids) - - def tearDown(self): - self.named_tm.kill_all() - self.single_tm.kill_all() - self.unique_tm.kill_all() - super().tearDown() - - def get_app(self): - self.named_tm = NamedTermManager(shell_command=['bash'], - max_terminals=MAX_TERMS, - ioloop=self.io_loop) - self.single_tm = SingleTermManager(shell_command=['bash'], - ioloop=self.io_loop) - self.unique_tm = UniqueTermManager(shell_command=['bash'], - max_terminals=MAX_TERMS, - ioloop=self.io_loop) - - named_tm = self.named_tm - class NewTerminalHandler(tornado.web.RequestHandler): - """Create a new named terminal, return redirect""" - def get(self): - name, terminal = named_tm.new_named_terminal() - self.redirect("/named/" + name, permanent=False) - - return tornado.web.Application([ - (r"/new", NewTerminalHandler), - (r"/named/(\w+)", TermSocket, {'term_manager': self.named_tm}), - (r"/single", TermSocket, {'term_manager': self.single_tm}), - (r"/unique", TermSocket, {'term_manager': self.unique_tm}) - ], debug=True) - - test_urls = ('/named/term1', '/unique', '/single') - -class CommonTests(TermTestCase): - @tornado.testing.gen_test - def test_basic(self): - for url in self.test_urls: - tm = yield self.get_term_client(url) - response = yield tm.read_msg() - self.assertEqual(response, ['setup', {}]) - - # Check for initial shell prompt - response = yield tm.read_msg() - self.assertEqual(response[0], 'stdout') - self.assertGreater(len(response[1]), 0) - tm.close() - - @tornado.testing.gen_test - def test_basic_command(self): - for url in self.test_urls: - tm = yield self.get_term_client(url) - yield tm.read_all_msg() - tm.write_stdin("whoami\n") - (stdout, other) = yield tm.read_stdout() - if os.name == 'nt': - assert 'whoami' in stdout - else: - assert stdout.startswith('who') - assert other == [] - tm.close() - -class NamedTermTests(TermTestCase): - def test_new(self): - response = self.fetch("/new", follow_redirects=False) - self.assertEqual(response.code, 302) - url = response.headers["Location"] - - # Check that the new terminal was created - name = url.split('/')[2] - self.assertIn(name, self.named_tm.terminals) - - @tornado.testing.gen_test - def test_namespace(self): - names = ["/named/1"]*2 + ["/named/2"]*2 - tms = yield self.get_term_clients(names) - pids = yield self.get_pids(tms) - - self.assertEqual(pids[0], pids[1]) - self.assertEqual(pids[2], pids[3]) - self.assertNotEqual(pids[0], pids[3]) - - @tornado.testing.gen_test - def test_max_terminals(self): - urls = ["/named/%d" % i for i in range(MAX_TERMS+1)] - tms = yield self.get_term_clients(urls[:MAX_TERMS]) - pids = yield self.get_pids(tms) - - # MAX_TERMS+1 should fail - tm = yield self.get_term_client(urls[MAX_TERMS]) - msg = yield tm.read_msg() - self.assertEqual(msg, None) # Connection closed - -class SingleTermTests(TermTestCase): - @tornado.testing.gen_test - def test_single_process(self): - tms = yield self.get_term_clients(["/single", "/single"]) - pids = yield self.get_pids(tms) - self.assertEqual(pids[0], pids[1]) - -class UniqueTermTests(TermTestCase): - @tornado.testing.gen_test - def test_unique_processes(self): - tms = yield self.get_term_clients(["/unique", "/unique"]) - pids = yield self.get_pids(tms) - self.assertNotEqual(pids[0], pids[1]) - - @tornado.testing.gen_test - def test_max_terminals(self): - tms = yield self.get_term_clients(['/unique'] * MAX_TERMS) - pids = yield self.get_pids(tms) - self.assertEqual(len(set(pids)), MAX_TERMS) # All PIDs unique - - # MAX_TERMS+1 should fail - tm = yield self.get_term_client("/unique") - msg = yield tm.read_msg() - self.assertEqual(msg, None) # Connection closed - - # Close one - tms[0].close() - msg = yield tms[0].read_msg() # Closed - self.assertEquals(msg, None) - - # Should be able to open back up to MAX_TERMS - tm = yield self.get_term_client("/unique") - msg = yield tm.read_msg() - self.assertEquals(msg[0], 'setup') - -if __name__ == '__main__': - unittest.main() diff --git a/terminado/uimod_embed.js b/terminado/uimod_embed.js deleted file mode 100644 index c517e13..0000000 --- a/terminado/uimod_embed.js +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Jupyter Development Team -// Copyright (c) 2014, Ramalingam Saravanan -// Distributed under the terms of the Simplified BSD License. - -window.addEventListener('load', function () { - var containers = document.getElementsByClassName('terminado-container') - var container, rows, cols, protocol, ws_url; - for (var i = 0; i < containers.length; i++) { - container = containers[i]; - rows = parseInt(container.dataset.rows); - cols = parseInt(container.dataset.cols); - protocol = (window.location.protocol.indexOf("https") === 0) ? "wss" : "ws"; - ws_url = protocol+"://"+window.location.host+ container.dataset.wsUrl; - make_terminal(container, {rows: rows, cols: cols}, ws_url); - } -}, false); diff --git a/terminado/uimodule.py b/terminado/uimodule.py deleted file mode 100644 index 584259c..0000000 --- a/terminado/uimodule.py +++ /dev/null @@ -1,27 +0,0 @@ -"""A Tornado UI module for a terminal backed by terminado. - -See the Tornado docs for information on UI modules: -http://www.tornadoweb.org/en/stable/guide/templates.html#ui-modules -""" -# Copyright (c) Jupyter Development Team -# Copyright (c) 2014, Ramalingam Saravanan -# Distributed under the terms of the Simplified BSD License. - -import os.path -import tornado.web - -class Terminal(tornado.web.UIModule): - def render(self, ws_url, cols=80, rows=25): - return ('
').format( - ws_url=ws_url, rows=rows, cols=cols) - - def javascript_files(self): - # TODO: Can we calculate these dynamically? - return ['/xstatic/termjs/term.js', '/static/terminado.js'] - - def embedded_javascript(self): - file = os.path.join(os.path.dirname(__file__), 'uimod_embed.js') - with open(file) as f: - return f.read() diff --git a/terminado/websocket.py b/terminado/websocket.py deleted file mode 100644 index 8330d5f..0000000 --- a/terminado/websocket.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Tornado websocket handler to serve a terminal interface. -""" -# Copyright (c) Jupyter Development Team -# Copyright (c) 2014, Ramalingam Saravanan -# Distributed under the terms of the Simplified BSD License. - -from __future__ import absolute_import, print_function - -# Python3-friendly imports -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - -import json -import logging - -import tornado.web -import tornado.websocket - - -def _cast_unicode(s): - if isinstance(s, bytes): - return s.decode('utf-8') - return s - - -class TermSocket(tornado.websocket.WebSocketHandler): - """Handler for a terminal websocket""" - - def initialize(self, term_manager): - self.term_manager = term_manager - self.term_name = "" - self.size = (None, None) - self.terminal = None - - self._logger = logging.getLogger(__name__) - - def check_origin(self, origin: str): - return True - - def origin_check(self, origin=None): - """Deprecated: backward-compat for terminado <= 0.5.""" - return self.check_origin(origin or self.request.headers.get('Origin')) - - def open(self, url_component=None): - """Websocket connection opened. - - Call our terminal manager to get a terminal, and connect to it as a - client. - """ - # Jupyter has a mixin to ping websockets and keep connections through - # proxies alive. Call super() to allow that to set up: - super(TermSocket, self).open(url_component) - - self._logger.info("TermSocket.open: %s", url_component) - - url_component = _cast_unicode(url_component) - self.term_name = url_component or 'tty' - self.terminal = self.term_manager.get_terminal(url_component) - self.terminal.clients.append(self) - self.send_json_message(["setup", {}]) - self._logger.info("TermSocket.open: Opened %s", self.term_name) - # Now drain the preopen buffer, if it exists. - buffered = "" - while True: - if not self.terminal.preopen_buffer: - break - s = self.terminal.preopen_buffer.popleft() - buffered += s - if buffered: - self.on_pty_read(buffered) - - def on_pty_read(self, text): - """Data read from pty; send to frontend""" - self.send_json_message(['stdout', text]) - - def send_json_message(self, content): - json_msg = json.dumps(content) - self.write_message(json_msg) - - def on_message(self, message): - """Handle incoming websocket message - - We send JSON arrays, where the first element is a string indicating - what kind of message this is. Data associated with the message follows. - """ - ##logging.info("TermSocket.on_message: %s - (%s) %s", self.term_name, type(message), len(message) if isinstance(message, bytes) else message[:250]) - command = json.loads(message) - msg_type = command[0] - - if msg_type == "stdin": - self.terminal.ptyproc.write(command[1]) - elif msg_type == "set_size": - self.size = command[1:3] - self.terminal.resize_to_smallest() - - def on_close(self): - """Handle websocket closing. - - Disconnect from our terminal, and tell the terminal manager we're - disconnecting. - """ - self._logger.info("Websocket closed") - if self.terminal: - self.terminal.clients.remove(self) - self.terminal.resize_to_smallest() - self.term_manager.client_disconnected(self) - - def on_pty_died(self): - """Terminal closed: tell the frontend, and close the socket. - """ - self.send_json_message(['disconnect', 1]) - self.close() - self.terminal = None