From 62f7a67b61f84d705ef856c3a8357878a633feee Mon Sep 17 00:00:00 2001 From: oidoming Date: Apr 22 2022 17:40:19 +0000 Subject: Add project --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3a01f83 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ +*.py[cod] +*$py.class +.env +.venv +env/ +venv/ +myvenv/ +myenv/ +ENV/ +env.bak/ +venv.bak/ diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000..54a9e6c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM quay.io/centos/centos:stream8 + +RUN dnf update -y && \ + dnf install -y \ + gcc \ + krb5-devel \ + python3-devel + +RUN dnf clean all + +ENV HOME /home/app +RUN useradd -r -d $HOME app +RUN usermod -aG wheel app +WORKDIR /home/app/package-updates + +RUN mkdir utils + +RUN chown -R app:wheel $HOME/package-updates +COPY --chown=app:wheel ./package_updates.py $HOME/package-updates +COPY --chown=app:wheel ./requirements.txt $HOME/package-updates +COPY --chown=app:wheel utils/* $HOME/package-updates/utils/ + +RUN pip3 install -r requirements.txt + +ENTRYPOINT ["python3", "./package_updates.py"] +USER app diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 8321d14..629fb74 --- a/README.md +++ b/README.md @@ -1 +1,88 @@ -# package-updates +# Package Updates | Centos package updates notification system + +## Usage + +```bash +git clone https://pagure.io/centos-sig-hyperscale/package-updates.git +cd package-updates +podman build -t package-updates . +podman run -it \ + --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/home/app/.centos-server-ca.cert,ro=true,relabel=private \ + --mount type=bind,src=$HOME/.centos.cert,dst=/home/app/.centos.cert,ro=true,relabel=private \ + -e PAGURE_API_KEY= \ + -e CAFILE=/home/app/.centos-server-ca.cert \ + -e CERT=/home/app/.centos.cert \ + -e KEY=/home/app/.centos.cert \ + package-updates +``` + +### Enviroment variables + +The script needs these enviroment variables: + +- PAGURE_API_KEY: Pagure API key, It is necessary to have an API Key with **issue_create, issue_comment, issue_change_status and issue_update** ACLs for the repo in order to allow the script to work with pagure issues. +- CAFILE: .centos-server-ca.cert file +- CERT: .centos.cert file +- KEY: .centos.cert file + +For more information on how to get the centos cert files, see: \ +Cert files needed for MQTT, see the Message Broker (MQTT) section: + +### Test enviroment + +You can use test_mqtt_pub/publish.py to run a mqtt publisher to test the script with a localhost mqtt server. + +Note: git.centos.org notifications don't appear very often so if you want to test the git.centos.org server leave the script for a long time running (hours or days). + +### MQTT payload +To recieve notifications from git.centos.org package updates the script listens to "git.centos.org/git.tag.creation" topic on mqtt, this topic is for git tag release creation. Here is an example of the payload for this topic: + +```json +{ + "repo": { + "custom_keys": [], + "name": "selinux-policy", + "parent": null, + "date_modified": "1553627665", + "access_users": { + "owner": [ + "centosrcm" + ], + "admin": [], + "ticket": [], + "commit": [] + }, + "namespace": "rpms", + "priorities": {}, + "close_status": [], + "access_groups": { + "admin": [], + "commit": [], + "ticket": [] + }, + "milestones": {}, + "user": { + "fullname": "CentOS Sources", + "name": "centosrcm" + }, + "date_created": "1553627665", + "fullname": "rpms/selinux-policy", + "url_path": "rpms/selinux-policy", + "id": 6059, + "tags": [], + "description": " SELinux policy configuration " + }, + "tag": "imports/c8s/selinux-policy-3.14.3-93.el8", + "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", + "agent": "centosrcm", + "authors": [ + { + "fullname": "CentOS Sources", + "name": "centosrcm" + } + ] +} +``` + +## Script flow +![Script flow diagram](images/FlowDiagram.png) \ No newline at end of file diff --git a/images/FlowDiagram.png b/images/FlowDiagram.png new file mode 100755 index 0000000..592eea9 Binary files /dev/null and b/images/FlowDiagram.png differ diff --git a/package_updates.py b/package_updates.py new file mode 100755 index 0000000..c751927 --- /dev/null +++ b/package_updates.py @@ -0,0 +1,159 @@ +#!/usr/bin/python + +import os +import sys +import random +import json +import time + +import paho.mqtt.client as mqtt + +from utils import helpers, db + + +H8S_MAIN_TAG_ID = 2249 # hyperscale8s-packages-main-release cbs koji tag +H8S_HOTFIXES_TAG_ID = 2305 # hyperscale8s-packages-hotfixes-release cbs koji tag +H8S_EXP_TAG_ID = 2245 # hyperscale8s-packages-experimental-release cbs koji tag + +CBS_URL = 'https://cbs.centos.org/kojihub' +C8S_GIT_BRANCH = 'c8s' + +""" See https://wiki.centos.org/Sources for more details + for the following MQTT values. +""" +MQTT_BROKER = 'mqtt.git.centos.org' +MQTT_PORT = 8883 +TOPIC = 'git.centos.org/git.tag.creation' +CAFILE = os.environ['CAFILE'] +CERT = os.environ['CERT'] +KEY = os.environ['KEY'] +client_id = f'hyperscale-sig-{random.randint(0, 1000)}' + + +def setup(conn): + """Get packages, creates database and see if a package + update is available, for first time running. + """ + + tags = [H8S_MAIN_TAG_ID, H8S_HOTFIXES_TAG_ID, H8S_EXP_TAG_ID] + + session = helpers.get_koji_session(CBS_URL) + cbs_tags = helpers.get_koji_tags(session, tags) + packages = helpers.join_tagged_packages(session, tags) + builds = helpers.get_latest_tagged_builds(session, packages) + + db.create_package_table(conn) + db.insert_packages_from_builds(builds, conn) + + db.create_tag_table(conn) + db.insert_tags(cbs_tags, conn) + + db.create_issue_table(conn) + + db_packages = db.select_packages(conn) + for package in db_packages: + package_name = package[1] + hs_version = ('%s-%s-%s' % (package[1], package[2], package[3])) + cbs_tag_id = package[5] + + centos_version, commit = helpers.get_git_pkg_version(package_name, C8S_GIT_BRANCH) + if centos_version == None: + continue + + print(f'centos version: {centos_version} | hs version: {hs_version}') + if helpers.compare_versions(centos_version, hs_version) == 1: + hs_tag_name = db.select_tag(cbs_tag_id, conn)[0] + issues = helpers.get_issues([package_name]) #gets all issues tagged with current package name + issue = helpers.issue_filter(package_name, issues) #get the issue that is tagged with package name and package version + if issue: + version_tag = issue['version_tag'] + if centos_version != version_tag: + new_issue = helpers.create_ticket(package_name, centos_version, hs_version, hs_tag_name, commit) + helpers.comment_on_issue(issue['issue_id'], new_issue['id'], centos_version) + helpers.close_issue(issue['issue_id']) + db.insert_issue(new_issue['id'], package_name, centos_version, conn) + else: + db.insert_issue(issue['issue_id'], package_name, centos_version, conn) + else: + new_issue = helpers.create_ticket(package_name, centos_version, hs_version, hs_tag_name, commit) + db.insert_issue(new_issue['id'], package_name, centos_version, conn) + + +def on_connect(client, user_data, flags, rc): + """Callback, stablish a conection with the + mqtt server and suscribes to a specific topic. + """ + + if rc == 0: + print(f'Connected to MQTT Broker') + else: + print(f'Failed to connect, return code {rc}') + + client.subscribe(TOPIC) + + +def on_message(client, user_data, msg): + """Callback, when a message from mqtt is recieved deserialize the + payload and checks if its a new package version to create an issue. + """ + + str_payload = msg.payload.decode('utf-8') + print(msg.topic + " " + str_payload) + payload = json.loads(str_payload) + + pkg_name = payload['repo']['name'] + git_tag = payload['tag'] + commit = payload['rev'] + + conn = user_data['db_conn'] + saved_hs_package = db.select_package(pkg_name, conn) + if saved_hs_package and C8S_GIT_BRANCH in git_tag: + session = helpers.get_koji_session(CBS_URL) + hs_tag_id = saved_hs_package[5] + cbs_hs_package = session.listTagged(hs_tag_id, latest=True, package=pkg_name) + hs_version = cbs_hs_package[0]['nvr'] + centos_version = helpers.filter_from_tag(git_tag, pkg_name) + + print(f'new version: {centos_version} | current version: {hs_version}') + if helpers.compare_versions(centos_version, hs_version) == 1: + issue = db.select_issue(pkg_name, conn) + hs_tag_name = db.select_tag(hs_tag_id, conn)[0] + new_issue = helpers.create_ticket(pkg_name, centos_version, hs_version, hs_tag_name, commit) + if issue: + db.update_issue_row(pkg_name, new_issue['id'], centos_version, conn) + is_open = helpers.is_issue_Open(centos_version) + if is_open: + helpers.comment_on_issue(issue[0], new_issue['id'], centos_version) + helpers.close_issue(issue[0]) + else: + db.insert_issue(new_issue['id'], pkg_name, centos_version, conn) + + +def listen_on_updates(conn): + """Set up the mqtt client, starts the loop and listen + on git.centos.org notifications for tag creation. + """ + + client = mqtt.Client() + client.on_connect = on_connect + client.on_message = on_message + client.tls_set(ca_certs=CAFILE, certfile=CERT, keyfile=KEY) + client.user_data_set({'db_conn': conn}) + client.connect(MQTT_BROKER, MQTT_PORT) + + client.loop_forever() + + +def main(): + print('running...') + try: + conn = db.get_connection() + setup(conn) + time.sleep(2) + listen_on_updates(conn) + except KeyboardInterrupt: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 0000000..883e75c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +koji>=1.27.1 +requests>=2.27.1 +paho-mqtt>=1.6.1 diff --git a/test_mqtt_pub/publish.py b/test_mqtt_pub/publish.py new file mode 100755 index 0000000..0ec4e46 --- /dev/null +++ b/test_mqtt_pub/publish.py @@ -0,0 +1,60 @@ +import time +import json +import random +from paho.mqtt import client as mqtt + + +broker = 'localhost' +port = 1883 +topic = "git.centos.org/git.tag.creation" +client_id = f'client-py-{random.randint(0, 1000)}' + + +def on_connect(client, user_data, flags, rc): + if rc == 0: + print(f'Connected to MQTT Broker') + else: + print(f'Failed to connect, return code {rc}') + + +def connect_mqtt(): + client = mqtt.Client(client_id) + client.on_connect = on_connect + client.connect(broker, port) + return client + + +def publish(client): + msg_count = 1 + msg = b'' + while True: + time.sleep(5) + if msg_count == 1: + msg = b'{"repo": {"custom_keys": [], "name": "selinux-policy", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/selinux-policy", "url_path": "rpms/selinux-policy", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/selinux-policy-3.14.3-93.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' + elif msg_count == 2: + msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-9.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' + else: + msg = b'{"repo": {"custom_keys": [], "name": "dnf", "parent": null, "date_modified": "1553627665", "access_users": {"owner": ["centosrcm"], "admin": [], "ticket": [], "commit": []}, "namespace": "rpms", "priorities": {}, "close_status": [], "access_groups": {"admin": [], "commit": [], "ticket": []}, "milestones": {}, "user": {"fullname": "CentOS Sources", "name": "centosrcm"}, "date_created": "1553627665", "fullname": "rpms/dnf", "url_path": "rpms/dnf", "id": 6059, "tags": [], "description": " SELinux policy configuration "}, "tag": "imports/c8s/dnf-4.7.0-1.el8", "rev": "56e29e64a64cb48a0889fd502c636b26dc7800e3", "agent": "centosrcm", "authors": [{"fullname": "CentOS Sources", "name": "centosrcm"}]}' + + result = client.publish(topic, msg) + status = result[0] + if status == 0: + print(f"Sent `{msg}` to topic `{topic}`") + else: + print(f"Failed to sent message to topic {topic}") + + if msg_count == 3: + msg_count = 0 + + msg_count += 1 + print('----------------------------------------------') + + +def run(): + client = connect_mqtt() + client.loop_start() + publish(client) + + +if __name__ == '__main__': + run() diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100755 index 0000000..e69de29 --- /dev/null +++ b/utils/__init__.py diff --git a/utils/db.py b/utils/db.py new file mode 100755 index 0000000..695bf8c --- /dev/null +++ b/utils/db.py @@ -0,0 +1,253 @@ +import sqlite3 + + +def get_connection() -> sqlite3.Connection: + """Gets an in-memory sqlite connection. + + Returns: + A sqlite connection. + """ + + return sqlite3.connect("file:mem1?mode=memory&cache=shared", uri=True) + + +def create_package_table(conn: sqlite3.Connection): + """Creates a sql table for hs packages. + + Args: + conn: sqlite3 connection + """ + + cursor = conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS package ( + package_id INTEGER PRIMARY KEY, + name text, + version text, + release text, + build_id INTEGER, + tag_id INTEGER) + """ + + cursor.execute(sql) + conn.commit() + + +def insert_package(package_id: int, name: str, version: str, release: str, + build_id: int, tag_id: int, conn: sqlite3.Connection): + """Inserts a hs package to package table. + + Args: + package_id: CBS package id. + name: String of package name. + version: String of package version. + release: String of package release. + build_id: CBS package build id. + tag_id: CBS package tag id. + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + cursor.execute("""INSERT INTO package( + package_id, + name, + version, + release, + build_id, + tag_id) + VALUES(?, ?, ?, ?, ?, ?)""", + (package_id, name, version, release, build_id, tag_id)) + conn.commit() + + +def select_package(name: str, conn: sqlite3.Connection) -> tuple: + """Gets a hs package from package table. + + Args: + name: String of package name. + conn: sqlite3 connection. + + Returns: + A tuple containing package_id, name, version, release, build_id, tag_id. + """ + + cursor = conn.cursor() + cursor.execute("SELECT * FROM package WHERE name=?", [name]) + + return cursor.fetchone() + + +def select_packages(conn: sqlite3.Connection) -> list: + """Gets all hs packages from database. + + Args: + conn: sqlite3 connection. + + Returns: + A list of tuples of hs packages. + """ + + cursor = conn.cursor() + cursor.execute("SELECT * FROM package") + + return cursor.fetchall() + + +def insert_packages_from_builds(packages: list, conn: sqlite3.Connection): + """Inserts a list of packages in the database. + + Args: + package: list of package builds from cbs. + conn: sqlite3 connection. + """ + + for package in packages: + package_id = package['package_id'] + name = package['package_name'] + version = package['version'] + release = package['release'] + build_id = package['build_id'] + tag_id = package['tag_id'] + + result = select_package(name, conn) + if result: + continue + + insert_package(package_id, name, version, release, build_id, tag_id, conn) + + +def create_tag_table(conn: sqlite3.Connection): + """Creates the tag (from CBS) table. + + Args: + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS tag ( + tag_id INTEGER PRIMARY KEY, + tag_name TEXT) + """ + + cursor.execute(sql) + conn.commit() + + +def insert_tag(tag_id: int, tag_name: str, conn: sqlite3.Connection): + """Inserts a CBS tag to the tag table. + + Args: + tag_id: CBS hs tag id. + tag_name: String of CBS hs tag name. + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + cursor.execute("""INSERT INTO tag(tag_id, tag_name) VALUES(?, ?)""", (tag_id, tag_name)) + + conn.commit() + + +def insert_tags(tags: list, conn: sqlite3.Connection): + """Inserts a list of tags to the tag table. + + Args: + package: List of cbs tags. + conn: sqlite3 connection. + """ + + for tag in tags: + tag_id = tag[0] + tag_name = tag[1] + + result = select_tag(tag_id, conn) + if result: + continue + + insert_tag(tag_id, tag_name, conn) + + +def select_tag(tag_id: str, conn: sqlite3.Connection) -> tuple: + """Gets a CBS hs tag from tag table. + + Args: + tag_id: CBS hs tag id. + conn: sqlite3 connection. + + Returns: + A tuple containing the tag name. + """ + + cursor = conn.cursor() + cursor.execute("SELECT tag_name FROM tag WHERE tag_id=?", [tag_id]) + + return cursor.fetchone() + + +def create_issue_table(conn: sqlite3.Connection): + """Creates a sql table for pagure issues. + + Args: + conn: sqlite3 connection + """ + + cursor = conn.cursor() + sql = """CREATE TABLE IF NOT EXISTS issue ( + issue_id INTEGER PRIMARY KEY, + package_name text, + version_tag text) + """ + + cursor.execute(sql) + conn.commit() + + +def insert_issue(issue_id: int, package_name: str, version_tag: str, conn: sqlite3.Connection): + """Inserts an issue to issue table. + + Args: + package_name: String of package name. + issue_id: New issue id. + version_tag: String of new centos package version, e.g. dnf-4.7.0-8.el8. + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + cursor.execute("""INSERT INTO issue( + issue_id, + package_name, + version_tag) + VALUES(?, ?, ?)""", + (issue_id, package_name, version_tag)) + conn.commit() + + +def update_issue_row(package_name: str, issue_id: int, version_tag: str, conn: sqlite3.Connection): + """Updates an issue for a specific package in database. + + Args: + package_name: String of package name. + issue_id: New issue id. + version_tag: String of new centos package version, e.g. dnf-4.7.0-8.el8. + conn: sqlite3 connection. + """ + + cursor = conn.cursor() + cursor.execute("UPDATE issue SET issue_id=?, version_tag=? WHERE package_name=?", (issue_id, version_tag, package_name)) + conn.commit() + + +def select_issue(package_name: str, conn: sqlite3.Connection) -> tuple: + """Gets an issue from issue table. + + Args: + package_name: String of package name. + conn: sqlite3 connection. + + Returns: + A tuple containing issue_id, package_name, version_tag. + """ + + cursor = conn.cursor() + cursor.execute("SELECT * FROM issue WHERE package_name=?", [package_name]) + + return cursor.fetchone() \ No newline at end of file diff --git a/utils/helpers.py b/utils/helpers.py new file mode 100755 index 0000000..b899505 --- /dev/null +++ b/utils/helpers.py @@ -0,0 +1,408 @@ +import os +import re +from urllib.parse import urljoin +import rpm +import hawkey +import koji +import requests + + +PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-bugs/' +PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-bugs/' +GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/' +GIT_CENTOS_URL = 'https://git.centos.org/rpms/' + +def get_koji_session(url: str) -> koji.ClientSession: + """Connects to the koji build system. + + Args: + url: String of koji url. + + Returns: + Koji client session. + """ + + return koji.ClientSession(url) + + +def get_tagged_packages(session: koji.ClientSession, tag: int) -> list: + """Requests for a list of packages available in a tag. + + Args: + session: Koji client session. + tag: Tag ID. + + Returns: + List of dictionaries with packages info. + """ + + packages = None + try: + packages = session.listPackages(tagID=tag) + if not packages: + print(f"There is no available packages for {tag}") + return + except koji.GenericError as err: + print(err) + + return packages + + +def join_tagged_packages(session: koji.ClientSession, tags: list) -> list: + """Joins the list of packages avalable in a list of koji tags. + + Args: + session: Koji client session. + tag: List of tag IDs. + + Returns: + List of dictionaries with packages info. + """ + + packages = [] + for tag in tags: + packages += get_tagged_packages(session, tag) + return packages + + +def get_latest_tagged_builds(session: koji.ClientSession, packages: list) -> list: + """Gets the latest cbs build for a list of packages. + + Args: + session: Koji client session. + packages: List of packages. + + Returns: + list of latest package builds. + """ + + builds = [] + for package in packages: + package_name = package['package_name'] + build = session.listTagged(package['tag_id'], latest=True, package=package_name) + if len(build) == 0: + continue + builds.append(build[0]) + + return builds + + +def get_koji_tags(session: koji.ClientSession, tags_id: list) -> list: + """Gets the tag id and tag name from koji cbs. + + Args: + session: Koji client session. + tag_ids: list of koji tag ids. + + Returns: + list of tuples containing tag id and tag name. + """ + + tags = [] + for tag_id in tags_id: + tag = session.getTag(tag_id) + tags.append((tag['id'], tag['name'])) + + return tags + + +def compare_versions(pkg1: str, pkg2: str) -> int: + """Compare package version between two packages. + + Args: + pkg1: String of the first pkg-version-release. E.g. dnf-4.7.0-8.el8 + pkg2: String of the second pkg-version-release. + + Returns: + 1 if pkg1 it's greater than pkg2, 0 if it's equal, -1 if it's lower. + """ + + nevra1 = hawkey.split_nevra(pkg1) + nevra2 = hawkey.split_nevra(pkg2) + + epoch1, v1, r1 = str(nevra1.epoch), nevra1.version, nevra1.release + epoch2, v2, r2 = str(nevra2.epoch), nevra2.version, nevra2.release + + return rpm.labelCompare((epoch1, v1, r1), (epoch2, v2, r2)) + + +def split_package(pkg: str): + """Split a package into name, version and release. + + Args: + pkg: String of package. E.g. dnf-4.7.0-8.el8 + + Returns: + Tuple containing package name, version and release. + """ + + subj = hawkey.Subject(pkg) + nevra_possibilities = subj.get_nevra_possibilities() + epel = 'el8' + + for nevra in nevra_possibilities: + if epel in str(nevra.release): + return (str(nevra.name), str(nevra.version), str(nevra.release)) + + return None + + +def filter_from_tag(tag: str, pkg: str) -> str: + """Filters the package name-version-release from a centos release tag. + + Args: + tag: String of the git tag. + pkg: String of the name of the package to filter. + + Retruns: + String of the package name-version-release. + """ + + index = tag.find(pkg) + return tag[index:] + + +def get_latest_version_with_commit(pkg: str, tags: list, branch: str) -> tuple: + """Filters latest git tag of a package. + + Args: + tags: List of git tags. + branch: String of git branch. + + Returns: + Tuple of latest version in a tag (name-version-release, commit). + """ + + branch_tags = {key: val for key, val in tags.items() if branch in key} + tags = list(branch_tags.items()) #list of tuples: [(tag, commit)] + + if len(tags) == 0: + return (None, None) + + newer = filter_from_tag(tags[0][0], pkg) + latest_commit = tags[0][1] + + for i in range(1, len(tags)): + curr = filter_from_tag(tags[i][0], pkg) + result = compare_versions(curr, newer) + if result == 1: + newer = curr + latest_commit = tags[i][1] + + return (newer, latest_commit) + + +def get_git_pkg_version(package: str, branch: str) -> tuple: + """Requests for git tags availables in a repo and searchs + for the latest version tag. + + Args: + package: String of package name. + branch: String of branch name. + + Returns: + Tuple that conatins string of pkg version and string of git commit. + """ + + params = { + 'with_commits': 'true' + } + url = urljoin(GIT_CENTOS_API_URL, f'{package}/git/tags') + + try: + res = requests.get(url=url, params=params) + res.raise_for_status() + except requests.exceptions.RequestException as err: + print(err) + return (None, None) + + tags = res.json()['tags'] + latest_version, commit = get_latest_version_with_commit(package, tags, branch) + + if latest_version == None: + return (None, None) + + return (latest_version, commit) + + +def create_ticket(pkg: str, upstream_version: str, current_version: str, cbs_tag_name: str, commit: str): + """Create a ticket on paguire.io repo. + + Args: + pkg: String of package name. + tag_version: String of package version. + """ + + commit_url = urljoin(GIT_CENTOS_URL, f'{pkg}/tree/{commit}') + issue = { + 'title': f'{upstream_version} is available', + 'issue_content': f"""Latest upstream release: {upstream_version} + Current version/release: {current_version} + URL: {commit_url}""", + 'tag': f'{cbs_tag_name},{upstream_version},{pkg}' + } + url = urljoin(PAGURE_REPO_API_URL, 'new_issue') + token = os.environ['PAGURE_API_KEY'] + headers = {'Authorization': f'access_token {token}'} + print('Creating ticket on pagure.io...') + + try: + res = requests.post(url=url, data=issue, headers=headers) + res.raise_for_status() + except requests.exceptions.HTTPError as errh: + if res.status_code == 401: + print(errh) + raise SystemExit(f'{errh}\nMake sure you have the correct API token') + print(errh) + return + except requests.exceptions.RequestException as err: + print(err) + return + + print('ticket created') + return res.json()['issue'] + + +def is_issue_Open(tag: str) -> bool: + """Verify if the issue for a specific is open. + + Args: + tag: String of issue tag. + + Returns: + True if the issue for that package update + has been created, false otherwise. + """ + + params = { + 'status': 'Open', + 'tags': tag + } + url = urljoin(PAGURE_REPO_API_URL, 'issues') + + try: + res = requests.get(url=url, params=params) + res.raise_for_status() + except requests.exceptions.RequestException as err: + print(err) + return True + + total_issues = res.json()['total_issues'] + if total_issues > 0: + return True + + return False + + +def comment_on_issue(prev_issue_id: int, new_issue_id: int, version_tag: str): + """Add a comment to a specific pagure.io issue. + + Args: + prev_issue_id: Issue id of the issue to add a comment, + new_issue_id: Issue Id of the new created issue, lo leave a link of the new issue. + version_tag: String of new package version. + """ + + new_issue_url = urljoin(PAGURE_REPO_URL, f'issue/{new_issue_id}') + comment = { + 'comment': f"""{version_tag} is available and this issue still open. + New issue has been created for the newer version. + URL: {new_issue_url}""" + } + url = urljoin(PAGURE_REPO_API_URL, f'issue/{prev_issue_id}/comment') + token = os.environ['PAGURE_API_KEY'] + headers = {'Authorization': f'access_token {token}'} + + try: + res = requests.post(url=url, data=comment, headers=headers) + res.raise_for_status() + except requests.exceptions.HTTPError as errh: + if res.status_code == 401: + print(errh) + raise SystemExit(f'{errh}\nMake sure you have the correct API token') + print(errh) + return + except requests.exceptions.RequestException as err: + print(err) + return + + print(f'Comment added on issue {prev_issue_id}') + + +def get_issues(tags: list) -> list: + """Requests for all issues that have a speciifc tag. + + Args: + tag: List of tags. + + Returns: + List of pagure.io issues. + """ + + params = { + 'status': 'Open', + 'tags': tags + } + url = urljoin(PAGURE_REPO_API_URL, 'issues') + + try: + res = requests.get(url=url, params=params) + res.raise_for_status() + except requests.exceptions.RequestException as err: + print(err) + return None + + issues = res.json()['issues'] + return issues + + +def close_issue(issue_id: int): + """Close a pagure.io issue. + + Args: + issue_id: Issue id. + """ + + data = { + 'status': 'Closed', + 'close_status': 'Invalid' + } + url = urljoin(PAGURE_REPO_API_URL, f'issue/{issue_id}/status') + token = os.environ['PAGURE_API_KEY'] + headers = {'Authorization': f'access_token {token}'} + + try: + res = requests.post(url=url, data=data, headers=headers) + res.raise_for_status() + except requests.exceptions.RequestException as err: + print(err) + print(res.json()) + return + print('issue closed') + + +def issue_filter(package_name: str, issues: list) -> dict: + """Filters and returns an issue if has a tag with a + package name and package version. + + Args: + package_name: String of package name. + issues: List of pagure.io issues. + + Returns: + dictionary containing the issue id, package name + and version tag, None if there is no package name + and package version. + """ + + version_tag = '' + for issue in issues: + c = 0 + for tag in issue['tags']: + if package_name in tag: + c += 1 + if tag != package_name: + version_tag = tag + if c == 2: + return {'issue_id': issue['id'], 'package_name': issue, 'version_tag': version_tag} + return None \ No newline at end of file diff --git a/utils/test_helpers.py b/utils/test_helpers.py new file mode 100755 index 0000000..b5d3dc9 --- /dev/null +++ b/utils/test_helpers.py @@ -0,0 +1,98 @@ +import unittest +from helpers import ( + get_latest_version_with_commit, + compare_versions, + filter_from_tag +) + + +class TestMain(unittest.TestCase): + + def test_compare_versions(self): + greater_list = ['dnf-4.7.0-5.el8', 'mesa-20.3.3-2.1.hs.el8'] + lower_list = ['dnf-4.7.0-4.1.hsx.el8', 'mesa-19.3.4-2.el8'] + + for greater, lower in zip(greater_list, lower_list): + self.assertEqual(compare_versions(greater, lower), 1) + + def test_filter_from_tag(self): + tags = { + 'dnf': 'imports/c8s/dnf-4.7.0-7.el8', + 'clang': 'imports/c8s-stream-rhel8/clang-13.0.1-1.module+el8.6.0+14118+d530a951', + 'dracut': 'imports/c8s/dracut-049-201.git20220131.el8', + 'libuser': 'libuser-0.62-23.2.hs.el8', + 'util-linux': 'imports/c8s/util-linux-2.32.1-34.el8', + 'createrepo_c': 'imports/c8s/createrepo_c-0.17.7-4.el8', + 'rpm': 'imports/c8s/rpm-4.14.el8' + } + + results = [ + 'dnf-4.7.0-7.el8', + 'clang-13.0.1-1.module+el8.6.0+14118+d530a951', + 'dracut-049-201.git20220131.el8', + 'libuser-0.62-23.2.hs.el8', + 'util-linux-2.32.1-34.el8', + 'createrepo_c-0.17.7-4.el8', + 'rpm-4.14.el8' + ] + + for tag, result in zip(tags, results): + self.assertEqual(filter_from_tag(tags[tag], tag), result) + + def test_get_latest_git_tag(self): + tags = { + "imports/c8/rpm-4.14.3-13.el8": "b445f2ffef77c56fb45f6e679d1894d22867d501", + "imports/c8/rpm-4.14.3-14.el8_4": "41043c8511245c3abba654656d2161a2ad68791a", + "imports/c8/rpm-4.14.3-19.el8": "00810bfb118747fbe980b81845009dbfcfcc8450", + "imports/c8/rpm-4.14.3-19.el8_5.2": "377311e10f5d8b70af6d6d59a8d2e2e532893492", + "imports/c8/rpm-4.14.3-4.el8": "b7b8f7e8f5cdbef3992c8ee853ecc444df930503", + "imports/c8s/rpm-4.14.3-14.el8_4": "69c9a1638657acea0c70decddb0b76959032bbea", + "imports/c8s/rpm-4.14.3-15.el8": "5a7695fcfeb883271f76b96178c0fc9de9c3697d", + "imports/c8s/rpm-4.14.3-17.el8": "bfc6f7541d6e6229df6d7951d063c7631850cdcb", + "imports/c8s/rpm-4.14.3-18.el8": "d8c505ff117b10aa0905dd8567c031610a00c928", + "imports/c8s/rpm-4.14.3-19.el8": "dcdfdde122434aaba3986a8ed22056b072c9b9cd", + "imports/c8s/rpm-4.14.3-20.el8": "c7b760fdfae67300dc7d06ba3ad7fce7ad9a6c20", + "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de", + "imports/c8s/rpm-4.14.3-22.el8": "63d1365d625c4a6bcf3eff7b5488f5554ad4ca44", + "imports/c8s/rpm-4.14.3-3.el8": "35cbef94d0e7d64f3dbec7c0e4d533ebefd5f780", + "imports/c8s/rpm-4.14.3-4.el8": "09d0f647d30af7497fa2e9d45b1254280071fc7a", + "imports/c9-beta/rpm-4.16.1.3-11.el9": "137a597a82550d8c9563615e057ee6679e497ce6", + "imports/c9-beta/rpm-4.16.1.3-7.el9": "813822951aaa7e26eec176925250d9ce7430d62b", + "imports/c9-beta/rpm-4.16.1.3-9.el9": "3f3ddd0c8927e60ed546707bac1303a44035f0ec" + } + + tags2 = { + "imports/c8/rpm-4.14.3-13.el8": "b445f2ffef77c56fb45f6e679d1894d22867d501", + "imports/c8/rpm-4.14.3-14.el8_4": "41043c8511245c3abba654656d2161a2ad68791a", + "imports/c8/rpm-4.14.3-19.el8": "00810bfb118747fbe980b81845009dbfcfcc8450", + "imports/c8/rpm-4.14.3-19.el8_5.2": "377311e10f5d8b70af6d6d59a8d2e2e532893492", + "imports/c8/rpm-4.14.3-4.el8": "b7b8f7e8f5cdbef3992c8ee853ecc444df930503", + "imports/c8s/rpm-4.14.3-14.el8_4": "69c9a1638657acea0c70decddb0b76959032bbea", + "imports/c8s/rpm-4.14.3-15.el8": "5a7695fcfeb883271f76b96178c0fc9de9c3697d", + "imports/c8s/rpm-4.14.3-17.el8": "bfc6f7541d6e6229df6d7951d063c7631850cdcb", + "imports/c8s/rpm-4.14.3-18.el8": "d8c505ff117b10aa0905dd8567c031610a00c928", + "imports/c8s/rpm-4.14.3-19.el8": "dcdfdde122434aaba3986a8ed22056b072c9b9cd", + "imports/c8s/rpm-4.14.3-20.el8": "c7b760fdfae67300dc7d06ba3ad7fce7ad9a6c20", + "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de", + "imports/c8s/rpm-4.14.3-22.el8": "63d1365d625c4a6bcf3eff7b5488f5554ad4ca44", + "imports/c9-beta/rpm-4.16.1.3-11.el9": "137a597a82550d8c9563615e057ee6679e497ce6", + "imports/c9-beta/rpm-4.16.1.3-7.el9": "813822951aaa7e26eec176925250d9ce7430d62b", + "imports/c9-beta/rpm-4.16.1.3-9.el9": "3f3ddd0c8927e60ed546707bac1303a44035f0ec" + } + + tags3 = { + "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de", + "imports/c8s/rpm-4.14.3-22.el8": "63d1365d625c4a6bcf3eff7b5488f5554ad4ca44" + } + + tags4 = { + "imports/c8s/rpm-4.14.3-21.el8": "a9e8d80332d37e00d3e07237cd042dcf248991de" + } + + self.assertEqual(get_latest_version_with_commit('rpm', tags, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) + self.assertEqual(get_latest_version_with_commit('rpm', tags2, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) + self.assertEqual(get_latest_version_with_commit('rpm', tags3, 'c8s'), ('rpm-4.14.3-22.el8', '63d1365d625c4a6bcf3eff7b5488f5554ad4ca44')) + self.assertEqual(get_latest_version_with_commit('rpm', tags4, 'c8s'), ('rpm-4.14.3-21.el8', 'a9e8d80332d37e00d3e07237cd042dcf248991de')) + + +