From 15e338318ccd82b50c782a9f1748bdca8bd2ea29 Mon Sep 17 00:00:00 2001
From: Oscar Dominguez <oscar.dominguez.ortiz@intel.com>
Date: Nov 10 2022 00:32:38 +0000
Subject: Added config option


---

diff --git a/README.md b/README.md
index ad54f35..e48727b 100755
--- a/README.md
+++ b/README.md
@@ -10,10 +10,12 @@ podman build -t package-updates .
 podman run -it \
   --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/home/app/.centos-server-ca.cert,ro=true,relabel=private \
   --mount type=bind,src=$HOME/.centos.cert,dst=/home/app/.centos.cert,ro=true,relabel=private \
+  --mount type=bind,source=config.json,dst=/home/app/package-updates/config.json \
   -e PAGURE_API_KEY=<pagure-api-key> \
   -e CAFILE=/home/app/.centos-server-ca.cert \
   -e CERT=/home/app/.centos.cert \
   -e KEY=/home/app/.centos.cert \
+  -e CONFIG_PATH=/home/app/package-updates/config.json \
   -e RUN_MODE=MQTT \
   package-updates
 ```
@@ -26,16 +28,53 @@ podman build -t package-updates .
 podman run -it \
 --mount type=bind,src=$HOME/.centos-server-ca.cert,dst=/home/app/.centos-server-ca.cert,ro=true,relabel=private \
 --mount type=bind,src=$HOME/.centos.cert,dst=/home/app/.centos.cert,ro=true,relabel=private \
---mount type=bind,source=fedora.toml,dst=/home/app/package-updates/fedora.toml
+--mount type=bind,source=config.json,dst=/home/app/package-updates/config.json \
 -e PAGURE_API_KEY=<pagure-api-key> \
 -e CAFILE=/home/app/.centos-server-ca.cert \
 -e CERT=/home/app/.centos.cert \
 -e KEY=/home/app/.centos.cert \
+-e CONFIG_PATH=/home/app/package-updates/config.json \
 -e FEDORA_MESSAGING_CONF=/home/app/package-updates/fedora.toml \
 -e RUN_MODE=AMQP \
 package-updates
 ```
 
+### Configuration
+
+Use config.json to specify your own project configuration values.
+Here you can specify:
+- MQTT:
+  - The MQTT broker, port, and topic.
+- Pagure:
+  - repo_url: Your desired Pagure repo URL, here is where the package updates tickets are going to be created.
+  - api_repo_url: Pagure repo API endpoint for repo_url.
+- Cbs:
+  - tags: Koji CBS tags. Add cbs tag ids if you want to get package updates from a specific list of cbs tags. 
+  - tags_keyword. If this is specified, you will get package updates from all cbs tags that contains this keyword. 
+
+Config.json example
+```json
+{
+    "mqtt": {
+        "broker": "mqtt.git.centos.org",
+        "port": 8883,
+        "topic": "git.centos.org/git.tag.creation"
+    },
+    "pagure": {
+        "repo_url": "https://pagure.io/centos-sig-hyperscale/package-updates/",
+        "api_repo_url": "https://pagure.io/api/0/centos-sig-hyperscale/package-updates/"
+    },
+    "cbs": {
+        "tags" : [],
+        "tags_keyword": "hyperscale"
+    }
+}
+```
+
+*The tags filed is prioritized over the keyword field, if you provide both you will only get package updates from the list of tags specified.*
+
+*For AMQP configuration use the fedora.toml file*
+
 ### Enviroment variables
 
 The script needs these enviroment variables:
@@ -44,6 +83,7 @@ The script needs these enviroment variables:
 - CAFILE: .centos-server-ca.cert file
 - CERT: .centos.cert file
 - KEY: .centos.cert file
+- CONFIG_PATH: path to config.json file
 - FEDORA_MESSAGING_CONF: path to fedora messaging configuration file (fedora.toml)
 
 For more information on how to get the centos cert files, see: <https://wiki.centos.org/Authentication#TLS_certificate> \
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..23094a5
--- /dev/null
+++ b/config.json
@@ -0,0 +1,15 @@
+{
+    "mqtt": {
+        "broker": "mqtt.git.centos.org",
+        "port": 8883,
+        "topic": "git.centos.org/git.tag.creation"
+    },
+    "pagure": {
+        "repo_url": "",
+        "api_repo_url": ""
+    },
+    "cbs": {
+        "tags" : [],
+        "tags_keyword": ""
+    }
+}
\ No newline at end of file
diff --git a/package_updates/main.py b/package_updates/main.py
index 43d4056..7fa0bc8 100644
--- a/package_updates/main.py
+++ b/package_updates/main.py
@@ -7,7 +7,7 @@ from urllib.parse import urljoin
 from amqp.amqp import AMQP
 from mqtt.mqtt import MQTT
 from utils import helpers, db
-from settings.constants import HS_CBS_TAGS, CBS_URL, C8S_GIT_BRANCH, RUN_MODE, PAGURE_REPO_URL
+from settings.constants import CBS_URL, CBS_TAGS, CBS_TAGS_KEYWORD, C8S_GIT_BRANCH, RUN_MODE, PAGURE_REPO_URL
 
 
 def setup(conn):
@@ -15,16 +15,19 @@ def setup(conn):
        update is available, for first time running.
     """
     
-    tags = HS_CBS_TAGS
-    
     session = helpers.get_koji_session(CBS_URL)
-    cbs_tags = helpers.get_koji_tags(session, tags)
+
+    if len(CBS_TAGS) > 0:
+        tags = helpers.get_koji_tags_info(session, CBS_TAGS)
+    else:
+        tags = helpers.get_koji_tags(session, CBS_TAGS_KEYWORD)
+    
     builds = helpers.get_latest_tagged_builds(session, tags)
-    targets = helpers.get_koji_build_targets(session, 'hyperscale') #gets all hyperscale cbs build targets
+    targets = helpers.get_koji_build_targets(session, CBS_TAGS_KEYWORD)
     
     tag_db = db.TagDB(conn)
     tag_db.create_tag_table()
-    tag_db.insert_tags(cbs_tags)
+    tag_db.insert_tags(tags)
 
     package_db = db.PackageDB(conn)
     package_db.create_package_table()
diff --git a/package_updates/settings/constants.py b/package_updates/settings/constants.py
index b31f541..ce1e731 100644
--- a/package_updates/settings/constants.py
+++ b/package_updates/settings/constants.py
@@ -1,15 +1,20 @@
+import json
 import os
 import random
 
 
+CONFIG_PATH = os.getenv('CONFIG_PATH') #path to config.json
+with open(CONFIG_PATH) as f:
+    config = json.load(f)
+
 # URLS
 CBS_URL = 'https://cbs.centos.org/kojihub'
 CBS_BUILD_URL = 'https://cbs.centos.org/koji/buildinfo?buildID='
 KOJIHUB_URL = 'https://kojihub.stream.centos.org/kojihub'
-PAGURE_REPO_API_URL = 'https://pagure.io/api/0/centos-sig-hyperscale/package-updates/'
-PAGURE_REPO_URL = 'https://pagure.io/centos-sig-hyperscale/package-updates/'
 GIT_CENTOS_API_URL = 'https://git.centos.org/api/0/rpms/'
 GIT_CENTOS_URL = 'https://git.centos.org/rpms/'
+PAGURE_REPO_URL = config['pagure']['repo_url'] or 'https://localhost/foo/bar/'
+PAGURE_REPO_API_URL = config['pagure']['api_repo_url'] or 'https://localhost/api/0/foo/bar/'
 
 
 # PAGURE API
@@ -24,9 +29,9 @@ C8S_GIT_BRANCH = 'c8s'
 """ See https://wiki.centos.org/Sources for more details 
     for the following MQTT values.
 """
-MQTT_BROKER = 'mqtt.git.centos.org'
-MQTT_PORT = 8883
-MQTT_TOPIC = 'git.centos.org/git.tag.creation'
+MQTT_BROKER = config['mqtt']['broker'] or 'mqtt.git.centos.org'
+MQTT_PORT = config['mqtt']['port'] or 8883
+MQTT_TOPIC = config['mqtt']['topic'] or 'git.centos.org/git.tag.creation'
 CAFILE = os.environ['CAFILE'] 
 CERT = os.environ['CERT'] 
 KEY = os.environ['KEY'] 
@@ -38,14 +43,8 @@ CBS_BUILD_TOPIC = 'org.centos.prod.cbs.buildsys.build.state.change'
 
 
 # CBS tags
-H8S_MAIN_TAG_ID = 2249              # hyperscale8s-packages-main-release cbs koji tag
-H8S_HOTFIXES_TAG_ID = 2305          # hyperscale8s-packages-hotfixes-release cbs koji tag
-H8S_EXP_TAG_ID = 2245               # hyperscale8s-packages-experimental-release cbs koji tag
-H8S_INTEL_CANDIDATE_TAG_ID = 2614   # hyperscale8s-packages-intel-candidate
-H8S_INTEL_TESTING_TAG_ID = 2615     # hyperscale8s-packages-intel-testing
-H8S_INTEL_RELEASE_TAG_ID = 2616     # hyperscale8s-packages-intel-release
-HS_CBS_TAGS = [H8S_MAIN_TAG_ID, H8S_HOTFIXES_TAG_ID, H8S_EXP_TAG_ID, 
-            H8S_INTEL_CANDIDATE_TAG_ID, H8S_INTEL_TESTING_TAG_ID, H8S_INTEL_RELEASE_TAG_ID]
+CBS_TAGS = config['cbs']['tags']
+CBS_TAGS_KEYWORD = config['cbs']['tags_keyword']
 
 
 # Run mode
diff --git a/package_updates/utils/db.py b/package_updates/utils/db.py
index 43974e4..fbff923 100755
--- a/package_updates/utils/db.py
+++ b/package_updates/utils/db.py
@@ -215,8 +215,8 @@ class TagDB:
         """
 
         for tag in tags:
-            tag_id = tag[0]
-            tag_name = tag[1]
+            tag_id = tag['id']
+            tag_name = tag['name']
 
             result = self.select_tag(tag_id)
             if result:
diff --git a/package_updates/utils/helpers.py b/package_updates/utils/helpers.py
index 21ddf28..ccdcba2 100755
--- a/package_updates/utils/helpers.py
+++ b/package_updates/utils/helpers.py
@@ -74,11 +74,11 @@ def get_latest_tagged_builds(session: koji.ClientSession, tags:list) -> list:
 
     builds = []
     for tag in tags:
-        builds += session.getLatestBuilds(tag)
+        builds += session.getLatestBuilds(tag['id'])
     return builds
 
 
-def get_koji_tags(session: koji.ClientSession, tags_id: list) -> list:
+def get_koji_tags_info(session: koji.ClientSession, tag_ids: list) -> list:
     """Gets the tag id and tag name from koji cbs.
 
        Args:
@@ -90,13 +90,34 @@ def get_koji_tags(session: koji.ClientSession, tags_id: list) -> list:
     """
     
     tags = []
-    for tag_id in tags_id:
+    for tag_id in tag_ids:
         tag = session.getTag(tag_id)
-        tags.append((tag['id'], tag['name']))
+        tags.append(tag)
     
     return tags
 
 
+def get_koji_tags(session: koji.ClientSession, keyword: str = None) -> list:
+    """Gets a list of dictionary tags from koji.
+ 
+       Args:
+           session: Koji client session.
+           keywords: String of filter for tags. E.g. If the word "hyperscale" 
+                     is provided it will return all tags containg this word.
+ 
+       Returns:
+           list of dictionaries containg koji tags.
+    """
+ 
+    koji_tags = session.listTags()
+ 
+    if keyword:
+        tags = [tag for tag in koji_tags if keyword in tag['name']]
+        return tags
+ 
+    return koji_tags
+
+
 def get_koji_build_targets(session: koji.ClientSession, filter: str) -> list:
     """Gets build targets from koji/cbs.
 

