From 983ddd86db6be6743a5fd6140db1c805cb9b151d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:44:01 +0000 Subject: [PATCH 1/20] Few changes to the build In this commit we're making a few changes to the build module: - rely on werkzeug to ensure that we create a secure filename on disk we don't want to "accidentally" leak data or wipe our hard-drive. - Add werkzeug to the requirements.txt since we're using it now - Use a context manager to create the temporary folder in which we're building the documentation - Raise the error when we fail to create, move or delete files and folders. We need to be aware of these and actually fix them. - Add a method to comment on a PR that we've started building the doc so people are aware that we're processing things - Add a dedicated method to comment to pagure's API to reduce code duplication Signed-off-by: Pierre-Yves Chibon --- diff --git a/build-scripts/build.py b/build-scripts/build.py index 270d65d..5b45b6b 100644 --- a/build-scripts/build.py +++ b/build-scripts/build.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import os import argparse from git import Repo @@ -9,6 +7,8 @@ import requests import shutil from dotenv import load_dotenv +import werkzeug.utils + load_dotenv() @@ -43,44 +43,36 @@ def get_docs_builder(pr_data): Args: A dict object with information about a pull request """ + + target_dir_name = f"{0}-pr{pr_data['id']}".format( + werkzeug.utils.secure_filename(pr_data['project']['fullname']) + ) + + # Delete it early so people get a 404 while we're building it + if os.path.exists(f"/var/www/html/{target_dir_name}"): + shutil.rmtree(f"/var/www/html/{target_dir_name}") + # Temporary directory to store the docs builder for preview in /tmp folder - temp_dir = tempfile.TemporaryDirectory(prefix="docs-ci-%s-" % pr_data['id'], - dir='/tmp') - - # Use git library to clone docs-fp-o and branch into a temporary - Repo.clone_from(url=DOCS_BUILDER_URL, to_path=f'{temp_dir.name}', - branch=DOCS_BUILDER_BRANCH) - - # Change directory into the folder. - # Only way for the script to recognise site.yml - os.chdir(f'{temp_dir.name}') - - # Build the docs site with the data from the PR. Returns the playbook - # data from site.yml This is used when writing to Apache config - playbook_data = build_docs(pr_data) - - # Check if a folder with the name intended for the PR build already exist. - # delete if it does, else move built doc to the folder where apache will serve it - if os.path.exists(f"/var/www/html/{pr_data['project']['name']}-pr{pr_data['id']}"): - shutil.rmtree(f"/var/www/html/{pr_data['project']['name']}-pr{pr_data['id']}") - try: - shutil.move(f"{temp_dir.name + playbook_data['output']['dir'][1:]}", - f"/var/www/html/{pr_data['project']['name']}-pr{pr_data['id']}") - except PermissionError: - print("Operation not permitted.") - # For other errors - except shutil.Error as error: - print(error) - - # Make sure folder is deleted after being cloned - try: - shutil.rmtree(temp_dir.name) - except FileNotFoundError: - print("File not found or already deleted") - except PermissionError: - print("Operation not permitted.") - else: - print("Temporary directory has been deleted") + with tempfile.TemporaryDirectory(prefix="docs-ci-%s-" % pr_data['id']) as temp_dir: + + # Use git library to clone docs-fp-o and branch into a temporary + Repo.clone_from(url=DOCS_BUILDER_URL, to_path=f'{temp_dir.name}', + branch=DOCS_BUILDER_BRANCH) + + # Change directory into the folder. + # Only way for the script to recognise site.yml + os.chdir(f'{temp_dir.name}') + + # Build the docs site with the data from the PR. Returns the playbook + # data from site.yml This is used when writing to Apache config + site_yml = build_docs(pr_data) + + shutil.move( + f"{temp_dir.name + site_yml['output']['dir'].strip('.)}", + f"/var/www/html/{target_dir_name}" + ) + + return target_dir_name def build_docs(pr_data): @@ -89,9 +81,9 @@ def build_docs(pr_data): docs_repo to the list of sites to be built """ with open('site.yml') as f: - playbook_data = yaml.load(f, Loader=yaml.SafeLoader) + site_yml = yaml.load(f, Loader=yaml.SafeLoader) - sources = playbook_data['content']['sources'] + sources = site_yml['content']['sources'] # Iterate through the sources and replace the docs upstream with the fork for i in range(len(sources)): if sources[i]['url'] == pr_data['project']['full_url']+'.git': @@ -107,49 +99,45 @@ def build_docs(pr_data): sources[i]['branches'] = pr_data['branch_from'] with open('site.yml', 'w') as f: - yaml.dump(playbook_data, f) + yaml.dump(site_yml, f) os.system("./build.sh") - return playbook_data + return site_yml -def post_comment(pr_data, comment): +def announce_build_start(pr_data): """ - Posts a comment under the PR with the link to the build + Posts a comment to the PR announcing that we've started building the + docs. Args: A dict object with information about a pull request Comment that should be posted """ - token = os.environ.get("api-key") - API_KEY = token - API_ENDPOINT = f"https://pagure.io/api/0/{pr_data['full_url'][18:]}/comment" + url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" - data = { - 'comment': comment - } + comment = f"Thank you for contributing to the documentation, we have " \ + "integrated, we will soon let you know where you can see it live!" + "started to build the full documentation website with these changes " \ + comment_on_pagure(comment, url) - headers = {'Authorization': f'token {API_KEY}'} - requests.post(url=API_ENDPOINT, data=data, headers=headers) - - -def post_successful_build_comment(pr_data): +def announce_build_result(pr_data, target_dir_name): """ - Posts a comment under the PR when the build is successful + Posts a comment under the PR with the link to the build - Args: - A dict object with information about a pull request - Comment that should be posted + Args: A dict object with information about a pull request """ + url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" + base_url = "https://to-be-figured-out" - comment = f"Thank you for your contribution. Use the following link to see a \ -preview of your contribution.\nDNS/{pr_data['project']['name']}-pr{pr_data['id']}. \ -Do keep in mind that the build gets deleted if there is no update for more than a \ -period of 2 weeks." + comment = "Thank you for your contribution. Use the following link to see a" \ + f"preview of your contribution.\{base_url}/{target_dir_name}. "\ + "Do keep in mind that the build gets deleted if there is no update for more than a "\ + "period of 2 weeks." - post_comment(pr_data, comment) + comment_on_pagure(comment, url) def post_unsuccessful_build_comment(pr_data): @@ -158,16 +146,23 @@ def post_unsuccessful_build_comment(pr_data): Args: A dict object with information about a pull request """ + url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" - comment = f"Thank you for your contribution. Unfortunately your PR did not build \ -for some reason. Keep pushing updates to your PR and use the following link to see a \ -preview of your contribution if it builds succesfully.\nDNS/{pr_data['project']['name']}-pr{pr_data['id']}. \ -Do keep in mind that the build gets deleted if there is no update for more than a \ -period of 2 weeks." + comment = f"Thank you for your contribution. Unfortunately your PR did "\ + "not build for some reason. Does it build locally for you?" - post_comment(pr_data, comment) + comment_on_pagure(pr_data, comment) -if __name__ == "__main__": - pr_data = get_data() - get_docs_builder(pr_data) +def comment_on_pagure(comment, url): + """ + Posts a specified comment to the specified url which is assumed to be + for a pagure instance. + Args: A dict object with information about a pull request + """ + token = os.environ.get("api-key") + data = { + 'comment': comment + } + headers = {'Authorization': f'token {token}'} + requests.post(url=url, data=data, headers=headers) diff --git a/requirements.txt b/requirements.txt index cbcd4df..13140be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ python-dotenv python-crontab dnf pycurl +werkzeug From 8df01f9cb10e501cab9a0de4100d110b825732cd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:46:45 +0000 Subject: [PATCH 2/20] Rework the consumer a little bit, drop the sites.py file - Add some more documentation - Cache the list of projects of interest instead of relying on a hard-coded list stored in sites.py - Drop the site.py file as it is no longer useful - Add dogpile.cache to the dependency list as that's what we use to cache the projects list to avoid querying it all the time Signed-off-by: Pierre-Yves Chibon --- diff --git a/build-scripts/consumer.py b/build-scripts/consumer.py index e055307..e6d240b 100644 --- a/build-scripts/consumer.py +++ b/build-scripts/consumer.py @@ -1,51 +1,87 @@ -#!/usr/bin/env python3 +""" +This file contains a fedora-messaging consumers that is called by +fedora-messaging every time a message that matches the expected topic is +received from the message bus. + +It will then trigger the build of the documentation and let people know how +it did by commenting on the pull-request. +""" + +import logging +import requests +import time +import yaml + +from dogpile.cache import make_region from fedora_messaging.api import consume from fedora_messaging.config import conf from build import ( get_docs_builder, - post_successful_build_comment, + announce_build_start, + announce_build_result, post_unsuccessful_build_comment ) -from sites import site_list +_log = logging.getLogger(__name__) +region = make_region().configure( + 'dogpile.cache.memory' + expiration_time = 3600, # Cache things for 1h +) + conf.setup_logging() -def print_message(message): +@region.cache_on_arguments() +def get_project_list(cnt=None): + """ Requests the main site.yaml used to build the documentation and + extract from it the list of proejcts that we need to trigger a build for """ - Print messages consumed by fedora messaging + url = "https://pagure.io/fedora-docs/docs-fp-o/raw/prod/f/site.yml" + cnt = cnt or 0 + if cnt >= 60: + raise Exception("Could not retrieve the site.yml from: %s" % url) - Args: a message object - """ - print(message.topic) - print(message.body['pullrequest']) + req = request.get(url) + if not req.ok: + time.sleep(1) + return get_project_list(cnt+=1) + + data = yaml.safe_load(req.text) + projects = [] + for source in data["content"]["sources"]: + src_url = source.get("url", "") + # For now we support only pagure.io, maybe we'll support github in + # the future, we'll see + if src_url.endswith(".git") and "pagure.io" in src_url: + projects.append(src_url) + + return projects -def build(message): +def consumer(message): """ - Build a pull request opened against fedora docs + This method is called for every message retrieved from the queue on the + rabbitmq server. Depending on the topic of the message (new PR, PR + rebased or PR updated), it will build the full documentation website + and comment on the PR whether it was successful at doing this or not. Args: a message object from Fedora Messages """ - if message.topic == "io.pagure.prod.pagure.pull-request.new": - pr_data = message.body['pullrequest'] - if pr_data['project']['full_url'] + '.git' in site_list: - try: - get_docs_builder() - post_successful_build_comment(pr_data) - except Exception as e: - print(f"Oops! {e} occured") - post_unsuccessful_build_comment(pr_data) - - if message.topic == "io.pagure.prod.pagure.pull-request.rebased" or \ - message.topic == 'io.pagure.prod.pagure.pull-request.updated': - pr_data = message.body['pullrequest'] - if pr_data['project']['full_url'] + '.git' in site_list: - get_docs_builder(pr_data) - - -if __name__ == "__main__": - conf.setup_logging() - consume(build) + _log.info("Processing message: %s - %s", message.id, message.topic) + + site_list = get_project_list() + + pr_data = message.body['pullrequest'] + project_to = pr_data['project']['full_url'] + '.git' + _log.info("Processing message about a PR at: %s", project_to) + if project_to in site_list: + announce_build_start(pr_data) + try: + target_dir_name = get_docs_builder(pr_data) + announce_build_result(pr_data, target_dir_name) + except Exception: + post_unsuccessful_build_comment(pr_data) + else: + _log.info("Ignoring %s, not in the projects of interest", project_to) diff --git a/build-scripts/sites.py b/build-scripts/sites.py deleted file mode 100644 index 0808368..0000000 --- a/build-scripts/sites.py +++ /dev/null @@ -1,42 +0,0 @@ -# List of sites used to build the fedora docs - -site_list = ( - "https://pagure.io/fedora-docs/release-docs-home.git", - "https://pagure.io/fedora-docs/install-guide.git", - "https://pagure.io/fedora-docs/system-administrators-guide.git", - "https://pagure.io/fedora-docs/quick-docs.git", - "https://pagure.io/fedora-docs/release-notes.git", - "https://pagure.io/mentored-projects.git", - "https://pagure.io/fedora-commops.git", - "https://pagure.io/Fedora-Council/council-docs.git", - "https://pagure.io/Fedora-Council/council-docs.git", - "https://pagure.io/Fedora-Council/status_reports.git", - "https://pagure.io/fedora-docs/modularity.git", - "https://github.com/fedora-silverblue/silverblue-docs.git", - "https://pagure.io/fedora-docs/documentation-contributors-guide.git", - "https://pagure.io/fedora-docs/flatpak.git", - "https://pagure.io/fedora-diversity.git", - "https://pagure.io/packaging-committee.git", - "https://pagure.io/mindshare.git", - "https://pagure.io/fesco/fesco-docs.git", - "https://pagure.io/fedora-iot/iot-docs.git", - "https://pagure.io/fedora-badges/docs.git", - "https://pagure.io/fedora-docs/remix-building.git", - "https://github.com/containers/docs.git", - "https://pagure.io/neuro-sig/documentation.git", - "https://pagure.io/sig-teleirc/infrastructure.git", - "https://pagure.io/fedora-ci/docs.git", - "https://pagure.io/java-packaging-howto.git", - "https://pagure.io/fedora-docs/taiga-docs.git", - "https://pagure.io/fedora-magazine.git", - "https://pagure.io/minimization.git", - "https://github.com/coreos/fedora-coreos-docs.git", - "https://pagure.io/cpe/rawhide-gating-docs.git", - "https://pagure.io/cpe/docs.git", - "https://pagure.io/fedora-docs/websites.git", - "https://pagure.io/fedora-pgm/pgm_docs.git", - "https://pagure.io/Ask-Fedora-SOP-docs.git", - "https://pagure.io/fedora-join/fedora-join-docs.git", - "https://pagure.io/fedora-docs/localization.git", - "https://pagure.io/fedora-l10n/docs.git" - ) diff --git a/requirements.txt b/requirements.txt index 13140be..1879e37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ GitPython PyYAML requests +dogpile.cache fedora-messaging python-dotenv python-crontab From f89a247853fe7f45d2339586ee062cd0fc80dde1 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:47:04 +0000 Subject: [PATCH 3/20] Drop a few dependencies from the dependency list as they are not used We only want in the dependency list dependencies that we use in the code. Signed-off-by: Pierre-Yves Chibon --- diff --git a/build-scripts/build.py b/build-scripts/build.py index 5b45b6b..63925c3 100644 --- a/build-scripts/build.py +++ b/build-scripts/build.py @@ -5,13 +5,10 @@ import tempfile import yaml import requests import shutil -from dotenv import load_dotenv import werkzeug.utils -load_dotenv() - DOCS_BUILDER_URL = "https://pagure.io/fedora-docs/docs-fp-o.git" DOCS_BUILDER_BRANCH = 'prod' PAGURE = 'https://pagure.io/' diff --git a/requirements.txt b/requirements.txt index 1879e37..93132d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,4 @@ PyYAML requests dogpile.cache fedora-messaging -python-dotenv -python-crontab -dnf -pycurl werkzeug From c647330c387b274c47a2ca3578fa32df7d0fcb49 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:47:05 +0000 Subject: [PATCH 4/20] Adjust the environment variable used to specify the API token for pagure In addition, we'll now let you know if you forgot to set one. Signed-off-by: Pierre-Yves Chibon --- diff --git a/build-scripts/build.py b/build-scripts/build.py index 63925c3..0820051 100644 --- a/build-scripts/build.py +++ b/build-scripts/build.py @@ -157,7 +157,13 @@ def comment_on_pagure(comment, url): for a pagure instance. Args: A dict object with information about a pull request """ - token = os.environ.get("api-key") + token = os.environ.get("PAGURE_API_KEY") + if not token: + raise Exception( + "The environment variable PAGURE_API_KEY is required, please " + "set it" + ) + data = { 'comment': comment } From ffd31ea82407f575ce9d5555619b8467e0bbd7a0 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:47:05 +0000 Subject: [PATCH 5/20] Move the example config.toml file outside of the sources The configuration file isn't part of the code, it's a configuration file so we're placing it one level above. Signed-off-by: Pierre-Yves Chibon --- diff --git a/build-scripts/config.toml b/build-scripts/config.toml deleted file mode 100644 index 4c1be7b..0000000 --- a/build-scripts/config.toml +++ /dev/null @@ -1,91 +0,0 @@ -# A basic configuration for Fedora's message broker, using the example callback -# which simply prints messages to standard output. -# -# This file is in the TOML format. -amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" -callback = "fedora_messaging.example:printer" - -[tls] -ca_cert = "/etc/fedora-messaging/cacert.pem" -keyfile = "/etc/fedora-messaging/fedora-key.pem" -certfile = "/etc/fedora-messaging/fedora-cert.pem" - -[client_properties] -app = "Fedora Docs CI" -# Some suggested extra fields: -# URL of the project that provides this consumer -app_url = "https://github.com/fedora-infra/fedora-messaging" -# Contact emails for the maintainer(s) of the consumer - in case the -# broker admin needs to contact them, for e.g. -app_contacts_email = ["jcline@fedoraproject.org"] - -[exchanges."amq.topic"] -type = "topic" -durable = true -auto_delete = false -arguments = {} - -# Queue names *must* be in the normal UUID format: run "uuidgen" and use the -# output as your queue name. If your queue is not exclusive, anyone can connect -# and consume from it, causing you to miss messages, so do not share your queue -# name. Any queues that are not auto-deleted on disconnect are garbage-collected -# after approximately one hour. -# -# If you require a stronger guarantee about delivery, please talk to Fedora's -# Infrastructure team. -[queues.77e3c5c9-51e4-4fba-9cee-bf41d5f1aaf6] -durable = false -auto_delete = true -exclusive = true -arguments = {} - -[[bindings]] -queue = "77e3c5c9-51e4-4fba-9cee-bf41d5f1aaf6" -exchange = "amq.topic" -routing_keys = ["io.pagure.prod.pagure.pull-request.new", "io.pagure.prod.pagure.pull-request.comment.added"] # Set this to the specific topics you are interested in. - -[consumer_config] -example_key = "for my consumer" - -[qos] -prefetch_size = 0 -prefetch_count = 25 - -[log_config] -version = 1 -disable_existing_loggers = true - -[log_config.formatters.simple] -format = "[%(levelname)s %(name)s] %(message)s" - -[log_config.handlers.console] -class = "logging.StreamHandler" -formatter = "simple" -stream = "ext://sys.stdout" - -[log_config.loggers.fedora_messaging] -level = "INFO" -propagate = false -handlers = ["console"] - -[log_config.loggers.twisted] -level = "INFO" -propagate = false -handlers = ["console"] - -[log_config.loggers.pika] -level = "WARNING" -propagate = false -handlers = ["console"] - -# If your consumer sets up a logger, you must add a configuration for it -# here in order for the messages to show up. e.g. if it set up a logger -# called 'example_printer', you could do: -#[log_config.loggers.example_printer] -#level = "INFO" -#propagate = false -#handlers = ["console"] - -[log_config.root] -level = "ERROR" -handlers = ["console"] \ No newline at end of file diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..4c1be7b --- /dev/null +++ b/config.toml @@ -0,0 +1,91 @@ +# A basic configuration for Fedora's message broker, using the example callback +# which simply prints messages to standard output. +# +# This file is in the TOML format. +amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" +callback = "fedora_messaging.example:printer" + +[tls] +ca_cert = "/etc/fedora-messaging/cacert.pem" +keyfile = "/etc/fedora-messaging/fedora-key.pem" +certfile = "/etc/fedora-messaging/fedora-cert.pem" + +[client_properties] +app = "Fedora Docs CI" +# Some suggested extra fields: +# URL of the project that provides this consumer +app_url = "https://github.com/fedora-infra/fedora-messaging" +# Contact emails for the maintainer(s) of the consumer - in case the +# broker admin needs to contact them, for e.g. +app_contacts_email = ["jcline@fedoraproject.org"] + +[exchanges."amq.topic"] +type = "topic" +durable = true +auto_delete = false +arguments = {} + +# Queue names *must* be in the normal UUID format: run "uuidgen" and use the +# output as your queue name. If your queue is not exclusive, anyone can connect +# and consume from it, causing you to miss messages, so do not share your queue +# name. Any queues that are not auto-deleted on disconnect are garbage-collected +# after approximately one hour. +# +# If you require a stronger guarantee about delivery, please talk to Fedora's +# Infrastructure team. +[queues.77e3c5c9-51e4-4fba-9cee-bf41d5f1aaf6] +durable = false +auto_delete = true +exclusive = true +arguments = {} + +[[bindings]] +queue = "77e3c5c9-51e4-4fba-9cee-bf41d5f1aaf6" +exchange = "amq.topic" +routing_keys = ["io.pagure.prod.pagure.pull-request.new", "io.pagure.prod.pagure.pull-request.comment.added"] # Set this to the specific topics you are interested in. + +[consumer_config] +example_key = "for my consumer" + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +# If your consumer sets up a logger, you must add a configuration for it +# here in order for the messages to show up. e.g. if it set up a logger +# called 'example_printer', you could do: +#[log_config.loggers.example_printer] +#level = "INFO" +#propagate = false +#handlers = ["console"] + +[log_config.root] +level = "ERROR" +handlers = ["console"] \ No newline at end of file From dfc249ea7fa7a2a554f9cc8dc398937001230b2a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:47:05 +0000 Subject: [PATCH 6/20] Move the cron job used to clean up the old builds of the doc This is an utility script that can (should) be ran as a cron job, it is not part of the sources themselves, so we're placing it one level higher in the project's structure. Signed-off-by: Pierre-Yves Chibon --- diff --git a/build-scripts/delete_builds.py b/build-scripts/delete_builds.py deleted file mode 100644 index 883ebf6..0000000 --- a/build-scripts/delete_builds.py +++ /dev/null @@ -1,55 +0,0 @@ -# importing the required modules -import os -import shutil -import time - -# main function -def main(): - - # specify the path - path = "/var/www/html" - - # specify the days - days = 15 - - # convert the current day to seconds - # time.time() returns current time since epoch (1970) in seconds - days_in_seconds = days * 24 * 60 * 60 - - # check whether the file is present in path or not - if os.path.exists(path): - # iterate over each list of folders in the path - for root_folder, folders, _ in os.walk(path): - for folder in folders: - # folder path - folder_path = os.path.join(root_folder, folder) - # comparing with the days - if time.time() - get_folder_age(folder_path) >= days_in_seconds: - # invoking the remove_folder function - remove_folder(folder_path) - else: - # file/folder is not found - print(f'"{path}" is not found') - - -def remove_folder(path): - # removing the folder - if not shutil.rmtree(path): - # success message - print(f"{path} is removed successfully") - else: - # failure message - print(f"Unable to delete the {path}") - - -def get_folder_age(path): - # getting ctime of the folder - # time will be in seconds - ctime = os.stat(path).st_ctime - - # returning the time - return ctime - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/delete_builds.py b/delete_builds.py new file mode 100644 index 0000000..883ebf6 --- /dev/null +++ b/delete_builds.py @@ -0,0 +1,55 @@ +# importing the required modules +import os +import shutil +import time + +# main function +def main(): + + # specify the path + path = "/var/www/html" + + # specify the days + days = 15 + + # convert the current day to seconds + # time.time() returns current time since epoch (1970) in seconds + days_in_seconds = days * 24 * 60 * 60 + + # check whether the file is present in path or not + if os.path.exists(path): + # iterate over each list of folders in the path + for root_folder, folders, _ in os.walk(path): + for folder in folders: + # folder path + folder_path = os.path.join(root_folder, folder) + # comparing with the days + if time.time() - get_folder_age(folder_path) >= days_in_seconds: + # invoking the remove_folder function + remove_folder(folder_path) + else: + # file/folder is not found + print(f'"{path}" is not found') + + +def remove_folder(path): + # removing the folder + if not shutil.rmtree(path): + # success message + print(f"{path} is removed successfully") + else: + # failure message + print(f"Unable to delete the {path}") + + +def get_folder_age(path): + # getting ctime of the folder + # time will be in seconds + ctime = os.stat(path).st_ctime + + # returning the time + return ctime + + +if __name__ == '__main__': + main() \ No newline at end of file From d5b4e6c6b61ce30ab80d240ed7a6c88a42bd5cea Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:47:05 +0000 Subject: [PATCH 7/20] Rename the folder from build-scripts to fedoradocs_ci This way we can turn it into a proper python module that can be installed and imported Signed-off-by: Pierre-Yves Chibon --- diff --git a/build-scripts/build.py b/build-scripts/build.py deleted file mode 100644 index 0820051..0000000 --- a/build-scripts/build.py +++ /dev/null @@ -1,171 +0,0 @@ -import os -import argparse -from git import Repo -import tempfile -import yaml -import requests -import shutil - -import werkzeug.utils - - -DOCS_BUILDER_URL = "https://pagure.io/fedora-docs/docs-fp-o.git" -DOCS_BUILDER_BRANCH = 'prod' -PAGURE = 'https://pagure.io/' - - -def get_data(): - """ - Use the argparse library to provide a user friendly cli. - Argsparse asks for a Pagure API containing info relating to a pull request - - Returns: - Data needed to build the docs site with with the new edit - """ - parser = argparse.ArgumentParser(description='Get the data for a particular for \ - a pull request related to a doc section.') - parser.add_argument("pull_request_api", help="Pagure API containing information \ - relating to a pull request") - - args = parser.parse_args() - - response = requests.get(f'{args.pull_request_api}') - pr_data = response.json() - return pr_data - - -def get_docs_builder(pr_data): - """ - Gets the fedora docs-fp-o used for building the entire docs site - - Args: A dict object with information about a pull request - """ - - target_dir_name = f"{0}-pr{pr_data['id']}".format( - werkzeug.utils.secure_filename(pr_data['project']['fullname']) - ) - - # Delete it early so people get a 404 while we're building it - if os.path.exists(f"/var/www/html/{target_dir_name}"): - shutil.rmtree(f"/var/www/html/{target_dir_name}") - - # Temporary directory to store the docs builder for preview in /tmp folder - with tempfile.TemporaryDirectory(prefix="docs-ci-%s-" % pr_data['id']) as temp_dir: - - # Use git library to clone docs-fp-o and branch into a temporary - Repo.clone_from(url=DOCS_BUILDER_URL, to_path=f'{temp_dir.name}', - branch=DOCS_BUILDER_BRANCH) - - # Change directory into the folder. - # Only way for the script to recognise site.yml - os.chdir(f'{temp_dir.name}') - - # Build the docs site with the data from the PR. Returns the playbook - # data from site.yml This is used when writing to Apache config - site_yml = build_docs(pr_data) - - shutil.move( - f"{temp_dir.name + site_yml['output']['dir'].strip('.)}", - f"/var/www/html/{target_dir_name}" - ) - - return target_dir_name - - -def build_docs(pr_data): - """ - Load the data from site.yml as a dict and append a fork of the - docs_repo to the list of sites to be built - """ - with open('site.yml') as f: - site_yml = yaml.load(f, Loader=yaml.SafeLoader) - - sources = site_yml['content']['sources'] - # Iterate through the sources and replace the docs upstream with the fork - for i in range(len(sources)): - if sources[i]['url'] == pr_data['project']['full_url']+'.git': - sources[i]['url'] = PAGURE + pr_data['repo_from']['fullname']+'.git' - # Antora builds 'master' branch by default, so if it's not master - # branch declare it in the playbook - if pr_data['branch_from'] != 'master': - if 'branches' in sources[i]: - # Sometimes it's a list of branches - if type(sources[i]['branches']) == list: - sources[i]['branches'].append(pr_data['branch_from']) - else: - sources[i]['branches'] = pr_data['branch_from'] - - with open('site.yml', 'w') as f: - yaml.dump(site_yml, f) - - os.system("./build.sh") - - return site_yml - - -def announce_build_start(pr_data): - """ - Posts a comment to the PR announcing that we've started building the - docs. - - Args: - A dict object with information about a pull request - Comment that should be posted - """ - url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" - - comment = f"Thank you for contributing to the documentation, we have " \ - "integrated, we will soon let you know where you can see it live!" - "started to build the full documentation website with these changes " \ - comment_on_pagure(comment, url) - - -def announce_build_result(pr_data, target_dir_name): - """ - Posts a comment under the PR with the link to the build - - Args: A dict object with information about a pull request - """ - url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" - base_url = "https://to-be-figured-out" - - comment = "Thank you for your contribution. Use the following link to see a" \ - f"preview of your contribution.\{base_url}/{target_dir_name}. "\ - "Do keep in mind that the build gets deleted if there is no update for more than a "\ - "period of 2 weeks." - - comment_on_pagure(comment, url) - - -def post_unsuccessful_build_comment(pr_data): - """ - Posts a comment under the PR when the build fails - - Args: A dict object with information about a pull request - """ - url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" - - comment = f"Thank you for your contribution. Unfortunately your PR did "\ - "not build for some reason. Does it build locally for you?" - - comment_on_pagure(pr_data, comment) - - -def comment_on_pagure(comment, url): - """ - Posts a specified comment to the specified url which is assumed to be - for a pagure instance. - Args: A dict object with information about a pull request - """ - token = os.environ.get("PAGURE_API_KEY") - if not token: - raise Exception( - "The environment variable PAGURE_API_KEY is required, please " - "set it" - ) - - data = { - 'comment': comment - } - headers = {'Authorization': f'token {token}'} - requests.post(url=url, data=data, headers=headers) diff --git a/build-scripts/consumer.py b/build-scripts/consumer.py deleted file mode 100644 index e6d240b..0000000 --- a/build-scripts/consumer.py +++ /dev/null @@ -1,87 +0,0 @@ -""" -This file contains a fedora-messaging consumers that is called by -fedora-messaging every time a message that matches the expected topic is -received from the message bus. - -It will then trigger the build of the documentation and let people know how -it did by commenting on the pull-request. -""" - -import logging -import requests -import time -import yaml - -from dogpile.cache import make_region - -from fedora_messaging.api import consume -from fedora_messaging.config import conf -from build import ( - get_docs_builder, - announce_build_start, - announce_build_result, - post_unsuccessful_build_comment -) - - -_log = logging.getLogger(__name__) -region = make_region().configure( - 'dogpile.cache.memory' - expiration_time = 3600, # Cache things for 1h -) - -conf.setup_logging() - - -@region.cache_on_arguments() -def get_project_list(cnt=None): - """ Requests the main site.yaml used to build the documentation and - extract from it the list of proejcts that we need to trigger a build for - """ - url = "https://pagure.io/fedora-docs/docs-fp-o/raw/prod/f/site.yml" - cnt = cnt or 0 - if cnt >= 60: - raise Exception("Could not retrieve the site.yml from: %s" % url) - - req = request.get(url) - if not req.ok: - time.sleep(1) - return get_project_list(cnt+=1) - - data = yaml.safe_load(req.text) - projects = [] - for source in data["content"]["sources"]: - src_url = source.get("url", "") - # For now we support only pagure.io, maybe we'll support github in - # the future, we'll see - if src_url.endswith(".git") and "pagure.io" in src_url: - projects.append(src_url) - - return projects - - -def consumer(message): - """ - This method is called for every message retrieved from the queue on the - rabbitmq server. Depending on the topic of the message (new PR, PR - rebased or PR updated), it will build the full documentation website - and comment on the PR whether it was successful at doing this or not. - - Args: a message object from Fedora Messages - """ - _log.info("Processing message: %s - %s", message.id, message.topic) - - site_list = get_project_list() - - pr_data = message.body['pullrequest'] - project_to = pr_data['project']['full_url'] + '.git' - _log.info("Processing message about a PR at: %s", project_to) - if project_to in site_list: - announce_build_start(pr_data) - try: - target_dir_name = get_docs_builder(pr_data) - announce_build_result(pr_data, target_dir_name) - except Exception: - post_unsuccessful_build_comment(pr_data) - else: - _log.info("Ignoring %s, not in the projects of interest", project_to) diff --git a/build-scripts/publisher.py b/build-scripts/publisher.py deleted file mode 100644 index e9b2974..0000000 --- a/build-scripts/publisher.py +++ /dev/null @@ -1,206 +0,0 @@ -from fedora_messaging import api, config -from build import post_comment - - -null = "null" -false = False -true = True - -# A sample PR message object for testing the build script. -topic = "io.pagure.prod.pagure.pull-request.new" -pr_data = { - "assignee": null, - "branch": "master", - "branch_from": "typo_fix", - "cached_merge_status": "unknown", - "closed_at": null, - "closed_by": null, - "comments": [], - "commit_start": "2be761e03c828fc2ab8709255d4f2946bb7e0e30", - "commit_stop": "2be761e03c828fc2ab8709255d4f2946bb7e0e30", - "date_created": "1614356479", - "full_url": "https://pagure.io/fedora-docs/quick-docs/pull-request/347", - "id": 347, - "initial_comment": "Ignore this PR. It's for testing purposes.", - "last_updated": "1614356479", - "project": { - "access_groups": { - "admin": [ - "fedora-docs" - ], - "collaborator": [], - "commit": [ - "quick-docs-committers" - ], - "ticket": [] - }, - "access_users": { - "admin": [ - "jflory7", - "mattdm" - ], - "collaborator": [], - "commit": [], - "owner": [ - "pbokoc" - ], - "ticket": [] - }, - "close_status": [ - "complete", - "duplicate", - "insufficient data", - "moved", - "not possible", - "out of scope", - "stale" - ], - "custom_keys": [], - "date_created": "1508967894", - "date_modified": "1590058045", - "description": "How-tos and other short-form documentation", - "full_url": "https://pagure.io/fedora-docs/quick-docs", - "fullname": "fedora-docs/quick-docs", - "id": 3273, - "milestones": {}, - "name": "quick-docs", - "namespace": "fedora-docs", - "parent": null, - "priorities": { - "": "", - "10": "needs review", - "20": "next meeting", - "30": "waiting on assignee", - "40": "waiting on external", - "50": "awaiting triage" - }, - "tags": [ - "docs", - "documentation" - ], - "url_path": "fedora-docs/quick-docs", - "user": { - "full_url": "https://pagure.io/user/pbokoc", - "fullname": "Petr Bokoc", - "name": "pbokoc", - "url_path": "user/pbokoc" - } - }, - "remote_git": null, - "repo_from": { - "access_groups": { - "admin": [], - "collaborator": [], - "commit": [], - "ticket": [] - }, - "access_users": { - "admin": [], - "collaborator": [], - "commit": [], - "owner": [ - "richardgreg" - ], - "ticket": [] - }, - "close_status": [], - "custom_keys": [], - "date_created": "1602135866", - "date_modified": "1602135866", - "description": "How-tos and other short-form documentation", - "full_url": "https://pagure.io/fork/richardgreg/fedora-docs/quick-docs", - "fullname": "forks/richardgreg/fedora-docs/quick-docs", - "id": 8768, - "milestones": {}, - "name": "quick-docs", - "namespace": "fedora-docs", - "parent": { - "access_groups": { - "admin": [ - "fedora-docs" - ], - "collaborator": [], - "commit": [ - "quick-docs-committers" - ], - "ticket": [] - }, - "access_users": { - "admin": [ - "jflory7", - "mattdm" - ], - "collaborator": [], - "commit": [], - "owner": [ - "pbokoc" - ], - "ticket": [] - }, - "close_status": [ - "complete", - "duplicate", - "insufficient data", - "moved", - "not possible", - "out of scope", - "stale" - ], - "custom_keys": [], - "date_created": "1508967894", - "date_modified": "1590058045", - "description": "How-tos and other short-form documentation", - "full_url": "https://pagure.io/fedora-docs/quick-docs", - "fullname": "fedora-docs/quick-docs", - "id": 3273, - "milestones": {}, - "name": "quick-docs", - "namespace": "fedora-docs", - "parent": null, - "priorities": { - "": "", - "10": "needs review", - "20": "next meeting", - "30": "waiting on assignee", - "40": "waiting on external", - "50": "awaiting triage" - }, - "tags": [ - "docs", - "documentation" - ], - "url_path": "fedora-docs/quick-docs", - "user": { - "full_url": "https://pagure.io/user/pbokoc", - "fullname": "Petr Bokoc", - "name": "pbokoc", - "url_path": "user/pbokoc" - } - }, - "priorities": {}, - "tags": [], - "url_path": "fork/richardgreg/fedora-docs/quick-docs", - "user": { - "full_url": "https://pagure.io/user/richardgreg", - "fullname": "Richard Gregory", - "name": "richardgreg", - "url_path": "user/richardgreg" - } - }, - "status": "Open", - "tags": [], - "threshold_reached": null, - "title": "Update index page.", - "uid": "defe5b8c5b9e44b398f84f7a2f4f77b4", - "updated_on": "1614356479", - "user": { - "full_url": "https://pagure.io/user/richardgreg", - "fullname": "Richard Gregory", - "name": "richardgreg", - "url_path": "user/richardgreg" - } - } - - -config.conf.setup_logging() -api.publish(api.Message(topic=topic, body={"pullrequest": pr_data})) diff --git a/fedoradocs_ci/build.py b/fedoradocs_ci/build.py new file mode 100644 index 0000000..0820051 --- /dev/null +++ b/fedoradocs_ci/build.py @@ -0,0 +1,171 @@ +import os +import argparse +from git import Repo +import tempfile +import yaml +import requests +import shutil + +import werkzeug.utils + + +DOCS_BUILDER_URL = "https://pagure.io/fedora-docs/docs-fp-o.git" +DOCS_BUILDER_BRANCH = 'prod' +PAGURE = 'https://pagure.io/' + + +def get_data(): + """ + Use the argparse library to provide a user friendly cli. + Argsparse asks for a Pagure API containing info relating to a pull request + + Returns: + Data needed to build the docs site with with the new edit + """ + parser = argparse.ArgumentParser(description='Get the data for a particular for \ + a pull request related to a doc section.') + parser.add_argument("pull_request_api", help="Pagure API containing information \ + relating to a pull request") + + args = parser.parse_args() + + response = requests.get(f'{args.pull_request_api}') + pr_data = response.json() + return pr_data + + +def get_docs_builder(pr_data): + """ + Gets the fedora docs-fp-o used for building the entire docs site + + Args: A dict object with information about a pull request + """ + + target_dir_name = f"{0}-pr{pr_data['id']}".format( + werkzeug.utils.secure_filename(pr_data['project']['fullname']) + ) + + # Delete it early so people get a 404 while we're building it + if os.path.exists(f"/var/www/html/{target_dir_name}"): + shutil.rmtree(f"/var/www/html/{target_dir_name}") + + # Temporary directory to store the docs builder for preview in /tmp folder + with tempfile.TemporaryDirectory(prefix="docs-ci-%s-" % pr_data['id']) as temp_dir: + + # Use git library to clone docs-fp-o and branch into a temporary + Repo.clone_from(url=DOCS_BUILDER_URL, to_path=f'{temp_dir.name}', + branch=DOCS_BUILDER_BRANCH) + + # Change directory into the folder. + # Only way for the script to recognise site.yml + os.chdir(f'{temp_dir.name}') + + # Build the docs site with the data from the PR. Returns the playbook + # data from site.yml This is used when writing to Apache config + site_yml = build_docs(pr_data) + + shutil.move( + f"{temp_dir.name + site_yml['output']['dir'].strip('.)}", + f"/var/www/html/{target_dir_name}" + ) + + return target_dir_name + + +def build_docs(pr_data): + """ + Load the data from site.yml as a dict and append a fork of the + docs_repo to the list of sites to be built + """ + with open('site.yml') as f: + site_yml = yaml.load(f, Loader=yaml.SafeLoader) + + sources = site_yml['content']['sources'] + # Iterate through the sources and replace the docs upstream with the fork + for i in range(len(sources)): + if sources[i]['url'] == pr_data['project']['full_url']+'.git': + sources[i]['url'] = PAGURE + pr_data['repo_from']['fullname']+'.git' + # Antora builds 'master' branch by default, so if it's not master + # branch declare it in the playbook + if pr_data['branch_from'] != 'master': + if 'branches' in sources[i]: + # Sometimes it's a list of branches + if type(sources[i]['branches']) == list: + sources[i]['branches'].append(pr_data['branch_from']) + else: + sources[i]['branches'] = pr_data['branch_from'] + + with open('site.yml', 'w') as f: + yaml.dump(site_yml, f) + + os.system("./build.sh") + + return site_yml + + +def announce_build_start(pr_data): + """ + Posts a comment to the PR announcing that we've started building the + docs. + + Args: + A dict object with information about a pull request + Comment that should be posted + """ + url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" + + comment = f"Thank you for contributing to the documentation, we have " \ + "integrated, we will soon let you know where you can see it live!" + "started to build the full documentation website with these changes " \ + comment_on_pagure(comment, url) + + +def announce_build_result(pr_data, target_dir_name): + """ + Posts a comment under the PR with the link to the build + + Args: A dict object with information about a pull request + """ + url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" + base_url = "https://to-be-figured-out" + + comment = "Thank you for your contribution. Use the following link to see a" \ + f"preview of your contribution.\{base_url}/{target_dir_name}. "\ + "Do keep in mind that the build gets deleted if there is no update for more than a "\ + "period of 2 weeks." + + comment_on_pagure(comment, url) + + +def post_unsuccessful_build_comment(pr_data): + """ + Posts a comment under the PR when the build fails + + Args: A dict object with information about a pull request + """ + url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" + + comment = f"Thank you for your contribution. Unfortunately your PR did "\ + "not build for some reason. Does it build locally for you?" + + comment_on_pagure(pr_data, comment) + + +def comment_on_pagure(comment, url): + """ + Posts a specified comment to the specified url which is assumed to be + for a pagure instance. + Args: A dict object with information about a pull request + """ + token = os.environ.get("PAGURE_API_KEY") + if not token: + raise Exception( + "The environment variable PAGURE_API_KEY is required, please " + "set it" + ) + + data = { + 'comment': comment + } + headers = {'Authorization': f'token {token}'} + requests.post(url=url, data=data, headers=headers) diff --git a/fedoradocs_ci/consumer.py b/fedoradocs_ci/consumer.py new file mode 100644 index 0000000..e6d240b --- /dev/null +++ b/fedoradocs_ci/consumer.py @@ -0,0 +1,87 @@ +""" +This file contains a fedora-messaging consumers that is called by +fedora-messaging every time a message that matches the expected topic is +received from the message bus. + +It will then trigger the build of the documentation and let people know how +it did by commenting on the pull-request. +""" + +import logging +import requests +import time +import yaml + +from dogpile.cache import make_region + +from fedora_messaging.api import consume +from fedora_messaging.config import conf +from build import ( + get_docs_builder, + announce_build_start, + announce_build_result, + post_unsuccessful_build_comment +) + + +_log = logging.getLogger(__name__) +region = make_region().configure( + 'dogpile.cache.memory' + expiration_time = 3600, # Cache things for 1h +) + +conf.setup_logging() + + +@region.cache_on_arguments() +def get_project_list(cnt=None): + """ Requests the main site.yaml used to build the documentation and + extract from it the list of proejcts that we need to trigger a build for + """ + url = "https://pagure.io/fedora-docs/docs-fp-o/raw/prod/f/site.yml" + cnt = cnt or 0 + if cnt >= 60: + raise Exception("Could not retrieve the site.yml from: %s" % url) + + req = request.get(url) + if not req.ok: + time.sleep(1) + return get_project_list(cnt+=1) + + data = yaml.safe_load(req.text) + projects = [] + for source in data["content"]["sources"]: + src_url = source.get("url", "") + # For now we support only pagure.io, maybe we'll support github in + # the future, we'll see + if src_url.endswith(".git") and "pagure.io" in src_url: + projects.append(src_url) + + return projects + + +def consumer(message): + """ + This method is called for every message retrieved from the queue on the + rabbitmq server. Depending on the topic of the message (new PR, PR + rebased or PR updated), it will build the full documentation website + and comment on the PR whether it was successful at doing this or not. + + Args: a message object from Fedora Messages + """ + _log.info("Processing message: %s - %s", message.id, message.topic) + + site_list = get_project_list() + + pr_data = message.body['pullrequest'] + project_to = pr_data['project']['full_url'] + '.git' + _log.info("Processing message about a PR at: %s", project_to) + if project_to in site_list: + announce_build_start(pr_data) + try: + target_dir_name = get_docs_builder(pr_data) + announce_build_result(pr_data, target_dir_name) + except Exception: + post_unsuccessful_build_comment(pr_data) + else: + _log.info("Ignoring %s, not in the projects of interest", project_to) diff --git a/fedoradocs_ci/publisher.py b/fedoradocs_ci/publisher.py new file mode 100644 index 0000000..e9b2974 --- /dev/null +++ b/fedoradocs_ci/publisher.py @@ -0,0 +1,206 @@ +from fedora_messaging import api, config +from build import post_comment + + +null = "null" +false = False +true = True + +# A sample PR message object for testing the build script. +topic = "io.pagure.prod.pagure.pull-request.new" +pr_data = { + "assignee": null, + "branch": "master", + "branch_from": "typo_fix", + "cached_merge_status": "unknown", + "closed_at": null, + "closed_by": null, + "comments": [], + "commit_start": "2be761e03c828fc2ab8709255d4f2946bb7e0e30", + "commit_stop": "2be761e03c828fc2ab8709255d4f2946bb7e0e30", + "date_created": "1614356479", + "full_url": "https://pagure.io/fedora-docs/quick-docs/pull-request/347", + "id": 347, + "initial_comment": "Ignore this PR. It's for testing purposes.", + "last_updated": "1614356479", + "project": { + "access_groups": { + "admin": [ + "fedora-docs" + ], + "collaborator": [], + "commit": [ + "quick-docs-committers" + ], + "ticket": [] + }, + "access_users": { + "admin": [ + "jflory7", + "mattdm" + ], + "collaborator": [], + "commit": [], + "owner": [ + "pbokoc" + ], + "ticket": [] + }, + "close_status": [ + "complete", + "duplicate", + "insufficient data", + "moved", + "not possible", + "out of scope", + "stale" + ], + "custom_keys": [], + "date_created": "1508967894", + "date_modified": "1590058045", + "description": "How-tos and other short-form documentation", + "full_url": "https://pagure.io/fedora-docs/quick-docs", + "fullname": "fedora-docs/quick-docs", + "id": 3273, + "milestones": {}, + "name": "quick-docs", + "namespace": "fedora-docs", + "parent": null, + "priorities": { + "": "", + "10": "needs review", + "20": "next meeting", + "30": "waiting on assignee", + "40": "waiting on external", + "50": "awaiting triage" + }, + "tags": [ + "docs", + "documentation" + ], + "url_path": "fedora-docs/quick-docs", + "user": { + "full_url": "https://pagure.io/user/pbokoc", + "fullname": "Petr Bokoc", + "name": "pbokoc", + "url_path": "user/pbokoc" + } + }, + "remote_git": null, + "repo_from": { + "access_groups": { + "admin": [], + "collaborator": [], + "commit": [], + "ticket": [] + }, + "access_users": { + "admin": [], + "collaborator": [], + "commit": [], + "owner": [ + "richardgreg" + ], + "ticket": [] + }, + "close_status": [], + "custom_keys": [], + "date_created": "1602135866", + "date_modified": "1602135866", + "description": "How-tos and other short-form documentation", + "full_url": "https://pagure.io/fork/richardgreg/fedora-docs/quick-docs", + "fullname": "forks/richardgreg/fedora-docs/quick-docs", + "id": 8768, + "milestones": {}, + "name": "quick-docs", + "namespace": "fedora-docs", + "parent": { + "access_groups": { + "admin": [ + "fedora-docs" + ], + "collaborator": [], + "commit": [ + "quick-docs-committers" + ], + "ticket": [] + }, + "access_users": { + "admin": [ + "jflory7", + "mattdm" + ], + "collaborator": [], + "commit": [], + "owner": [ + "pbokoc" + ], + "ticket": [] + }, + "close_status": [ + "complete", + "duplicate", + "insufficient data", + "moved", + "not possible", + "out of scope", + "stale" + ], + "custom_keys": [], + "date_created": "1508967894", + "date_modified": "1590058045", + "description": "How-tos and other short-form documentation", + "full_url": "https://pagure.io/fedora-docs/quick-docs", + "fullname": "fedora-docs/quick-docs", + "id": 3273, + "milestones": {}, + "name": "quick-docs", + "namespace": "fedora-docs", + "parent": null, + "priorities": { + "": "", + "10": "needs review", + "20": "next meeting", + "30": "waiting on assignee", + "40": "waiting on external", + "50": "awaiting triage" + }, + "tags": [ + "docs", + "documentation" + ], + "url_path": "fedora-docs/quick-docs", + "user": { + "full_url": "https://pagure.io/user/pbokoc", + "fullname": "Petr Bokoc", + "name": "pbokoc", + "url_path": "user/pbokoc" + } + }, + "priorities": {}, + "tags": [], + "url_path": "fork/richardgreg/fedora-docs/quick-docs", + "user": { + "full_url": "https://pagure.io/user/richardgreg", + "fullname": "Richard Gregory", + "name": "richardgreg", + "url_path": "user/richardgreg" + } + }, + "status": "Open", + "tags": [], + "threshold_reached": null, + "title": "Update index page.", + "uid": "defe5b8c5b9e44b398f84f7a2f4f77b4", + "updated_on": "1614356479", + "user": { + "full_url": "https://pagure.io/user/richardgreg", + "fullname": "Richard Gregory", + "name": "richardgreg", + "url_path": "user/richardgreg" + } + } + + +config.conf.setup_logging() +api.publish(api.Message(topic=topic, body={"pullrequest": pr_data})) From ab07d0a0c3c8425f6c99db44edaa1fa05ea55c1c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:47:05 +0000 Subject: [PATCH 8/20] Add an __init__.py file to fedoradocs_ci and include a version in it Signed-off-by: Pierre-Yves Chibon --- diff --git a/fedoradocs_ci/__init__.py b/fedoradocs_ci/__init__.py new file mode 100644 index 0000000..49a3f99 --- /dev/null +++ b/fedoradocs_ci/__init__.py @@ -0,0 +1,6 @@ +""" +This modules holds the code that is used for a simple CI service for the +fedora-docs project available at https://docs.fedoraproject.org. +""" + +__version__ = "0.0.1" From 86fc0269132a9ca7bcc2344cf79344426f9ead87 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:47:05 +0000 Subject: [PATCH 9/20] Small changes to the config.toml - Make the callback point to our consumer so we can actually consume messages - Adjust app_url so it points to our project - Reformat the routing_keys so they are easier to read - Add the topic for rebased pull-request to the list of topics of interest (ie: the routing_keys) Signed-off-by: Pierre-Yves Chibon --- diff --git a/config.toml b/config.toml index 4c1be7b..eedd44f 100644 --- a/config.toml +++ b/config.toml @@ -3,7 +3,7 @@ # # This file is in the TOML format. amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" -callback = "fedora_messaging.example:printer" +callback = "fedoradocs_ci.consumer:consume" [tls] ca_cert = "/etc/fedora-messaging/cacert.pem" @@ -14,7 +14,7 @@ certfile = "/etc/fedora-messaging/fedora-cert.pem" app = "Fedora Docs CI" # Some suggested extra fields: # URL of the project that provides this consumer -app_url = "https://github.com/fedora-infra/fedora-messaging" +app_url = "https://pagure.io/fedora-docs/fedora-docs-ci" # Contact emails for the maintainer(s) of the consumer - in case the # broker admin needs to contact them, for e.g. app_contacts_email = ["jcline@fedoraproject.org"] @@ -42,7 +42,11 @@ arguments = {} [[bindings]] queue = "77e3c5c9-51e4-4fba-9cee-bf41d5f1aaf6" exchange = "amq.topic" -routing_keys = ["io.pagure.prod.pagure.pull-request.new", "io.pagure.prod.pagure.pull-request.comment.added"] # Set this to the specific topics you are interested in. +routing_keys = [ + "io.pagure.prod.pagure.pull-request.new", + "io.pagure.prod.pagure.pull-request.comment.added", + "io.pagure.prod.pagure.pull-request.rebased," +] [consumer_config] example_key = "for my consumer" @@ -88,4 +92,4 @@ handlers = ["console"] [log_config.root] level = "ERROR" -handlers = ["console"] \ No newline at end of file +handlers = ["console"] From 3e56abba500d6ef6293dd4c1e9245a5dd2c25c1f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:47:05 +0000 Subject: [PATCH 10/20] Fix typos These are typos that we introduced while reformatting the code. I could amend the commits but this also shows that no-one makes no mistake and I'm keeping this here as a proof that errare humanum est. Signed-off-by: Pierre-Yves Chibon --- diff --git a/fedoradocs_ci/build.py b/fedoradocs_ci/build.py index 0820051..5fdc97c 100644 --- a/fedoradocs_ci/build.py +++ b/fedoradocs_ci/build.py @@ -65,7 +65,7 @@ def get_docs_builder(pr_data): site_yml = build_docs(pr_data) shutil.move( - f"{temp_dir.name + site_yml['output']['dir'].strip('.)}", + f"{temp_dir.name + site_yml['output']['dir'].strip('.')}", f"/var/www/html/{target_dir_name}" ) @@ -116,7 +116,7 @@ def announce_build_start(pr_data): comment = f"Thank you for contributing to the documentation, we have " \ "integrated, we will soon let you know where you can see it live!" - "started to build the full documentation website with these changes " \ + "started to build the full documentation website with these changes " comment_on_pagure(comment, url) diff --git a/fedoradocs_ci/consumer.py b/fedoradocs_ci/consumer.py index e6d240b..3e644ee 100644 --- a/fedoradocs_ci/consumer.py +++ b/fedoradocs_ci/consumer.py @@ -26,7 +26,7 @@ from build import ( _log = logging.getLogger(__name__) region = make_region().configure( - 'dogpile.cache.memory' + 'dogpile.cache.memory', expiration_time = 3600, # Cache things for 1h ) @@ -46,7 +46,7 @@ def get_project_list(cnt=None): req = request.get(url) if not req.ok: time.sleep(1) - return get_project_list(cnt+=1) + return get_project_list(cnt=cnt+1) data = yaml.safe_load(req.text) projects = [] From 9ae24231ab64c180235c0fd25ba8cc67879fc29a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:51:36 +0000 Subject: [PATCH 11/20] Rename a couple of method to reflect better what they actually do Signed-off-by: Pierre-Yves Chibon --- diff --git a/fedoradocs_ci/build.py b/fedoradocs_ci/build.py index 5fdc97c..067a79c 100644 --- a/fedoradocs_ci/build.py +++ b/fedoradocs_ci/build.py @@ -34,7 +34,7 @@ def get_data(): return pr_data -def get_docs_builder(pr_data): +def build_docs(pr_data): """ Gets the fedora docs-fp-o used for building the entire docs site @@ -62,7 +62,7 @@ def get_docs_builder(pr_data): # Build the docs site with the data from the PR. Returns the playbook # data from site.yml This is used when writing to Apache config - site_yml = build_docs(pr_data) + site_yml = configure_and_build_docs(pr_data) shutil.move( f"{temp_dir.name + site_yml['output']['dir'].strip('.')}", @@ -72,7 +72,7 @@ def get_docs_builder(pr_data): return target_dir_name -def build_docs(pr_data): +def configure_and_build_docs(pr_data): """ Load the data from site.yml as a dict and append a fork of the docs_repo to the list of sites to be built diff --git a/fedoradocs_ci/consumer.py b/fedoradocs_ci/consumer.py index 3e644ee..da9d385 100644 --- a/fedoradocs_ci/consumer.py +++ b/fedoradocs_ci/consumer.py @@ -16,10 +16,11 @@ from dogpile.cache import make_region from fedora_messaging.api import consume from fedora_messaging.config import conf -from build import ( - get_docs_builder, + +from fedoradocs_ci.build import ( announce_build_start, announce_build_result, + build_docs, post_unsuccessful_build_comment ) @@ -79,7 +80,7 @@ def consumer(message): if project_to in site_list: announce_build_start(pr_data) try: - target_dir_name = get_docs_builder(pr_data) + target_dir_name = build_docs(pr_data) announce_build_result(pr_data, target_dir_name) except Exception: post_unsuccessful_build_comment(pr_data) From 975837056e7f20e5d8a8d71dfd005aae5ba10b23 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:51:37 +0000 Subject: [PATCH 12/20] Adjust the name of the consumer function to what is in the config.toml and consumer:consume reads nicer than consumer:consumer so that's why we're fixing it here rather than in the config.toml Signed-off-by: Pierre-Yves Chibon --- diff --git a/fedoradocs_ci/consumer.py b/fedoradocs_ci/consumer.py index da9d385..9ba918f 100644 --- a/fedoradocs_ci/consumer.py +++ b/fedoradocs_ci/consumer.py @@ -61,7 +61,7 @@ def get_project_list(cnt=None): return projects -def consumer(message): +def consume(message): """ This method is called for every message retrieved from the queue on the rabbitmq server. Depending on the topic of the message (new PR, PR From d465e6592e23c04b64a47f2a42f1132ad0ebe777 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:51:37 +0000 Subject: [PATCH 13/20] Make the queue durable Even though the queue will not be really durable as we're using the public endpoint which get cleaned regularly, but at least it will leave messages in the queue while we restart we reboot/debug the program. Signed-off-by: Pierre-Yves Chibon --- diff --git a/config.toml b/config.toml index eedd44f..5736584 100644 --- a/config.toml +++ b/config.toml @@ -34,7 +34,7 @@ arguments = {} # If you require a stronger guarantee about delivery, please talk to Fedora's # Infrastructure team. [queues.77e3c5c9-51e4-4fba-9cee-bf41d5f1aaf6] -durable = false +durable = true auto_delete = true exclusive = true arguments = {} From 774c092ed3cf458914c38a3679b19c9262f3aad8 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:51:37 +0000 Subject: [PATCH 14/20] Fix typo when requesting the list of projects of interest Signed-off-by: Pierre-Yves Chibon --- diff --git a/fedoradocs_ci/consumer.py b/fedoradocs_ci/consumer.py index 9ba918f..ff51983 100644 --- a/fedoradocs_ci/consumer.py +++ b/fedoradocs_ci/consumer.py @@ -44,7 +44,7 @@ def get_project_list(cnt=None): if cnt >= 60: raise Exception("Could not retrieve the site.yml from: %s" % url) - req = request.get(url) + req = requests.get(url) if not req.ok: time.sleep(1) return get_project_list(cnt=cnt+1) From 8c68759bc810638e1a39f3036b2ebb7069285432 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:51:37 +0000 Subject: [PATCH 15/20] Add configuration for logging in fedoradocs_ci Signed-off-by: Pierre-Yves Chibon --- diff --git a/config.toml b/config.toml index 5736584..f42e159 100644 --- a/config.toml +++ b/config.toml @@ -90,6 +90,12 @@ handlers = ["console"] #propagate = false #handlers = ["console"] +[log_config.loggers.fedoradocs_ci] +level = "INFO" +propagate = false +handlers = ["console"] + + [log_config.root] level = "ERROR" handlers = ["console"] From 3df26c0cbaaa717f46225c029f8a17d07e9e2a3e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:54:22 +0000 Subject: [PATCH 16/20] Fix accessing the data in the JSON blob coming from the message on the bus Signed-off-by: Pierre-Yves Chibon --- diff --git a/fedoradocs_ci/build.py b/fedoradocs_ci/build.py index 067a79c..7f254be 100644 --- a/fedoradocs_ci/build.py +++ b/fedoradocs_ci/build.py @@ -112,7 +112,7 @@ def announce_build_start(pr_data): A dict object with information about a pull request Comment that should be posted """ - url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" + url = f"https://pagure.io/api/0/{pr_data['project']['fullname']}/comment" comment = f"Thank you for contributing to the documentation, we have " \ "integrated, we will soon let you know where you can see it live!" @@ -126,7 +126,7 @@ def announce_build_result(pr_data, target_dir_name): Args: A dict object with information about a pull request """ - url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" + url = f"https://pagure.io/api/0/{pr_data['project']['fullname']}/comment" base_url = "https://to-be-figured-out" comment = "Thank you for your contribution. Use the following link to see a" \ @@ -143,7 +143,7 @@ def post_unsuccessful_build_comment(pr_data): Args: A dict object with information about a pull request """ - url = f"https://pagure.io/api/0/{pr_data['fullname']}/comment" + url = f"https://pagure.io/api/0/{pr_data['project']['fullname']}/comment" comment = f"Thank you for your contribution. Unfortunately your PR did "\ "not build for some reason. Does it build locally for you?" From cde87dd8a55a43225b664adc89c9b2dfd4570519 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:54:51 +0000 Subject: [PATCH 17/20] Add time when logging to the console so we have an idea of speed Signed-off-by: Pierre-Yves Chibon --- diff --git a/config.toml b/config.toml index f42e159..1c0a0f2 100644 --- a/config.toml +++ b/config.toml @@ -60,7 +60,7 @@ version = 1 disable_existing_loggers = true [log_config.formatters.simple] -format = "[%(levelname)s %(name)s] %(message)s" +format = "[%(asctime)s - %(levelname)s %(name)s] %(message)s" [log_config.handlers.console] class = "logging.StreamHandler" From 6dc8252d12e03117fda88363abc39698aaa3628d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 14:57:28 +0000 Subject: [PATCH 18/20] Rename post_unsuccessful_build_comment to announce_unsuccessful_build Renaming this function makes it consistent with the other methods. Also log to the terminal when we fail to build the docs. Signed-off-by: Pierre-Yves Chibon --- diff --git a/fedoradocs_ci/build.py b/fedoradocs_ci/build.py index 7f254be..34c54cb 100644 --- a/fedoradocs_ci/build.py +++ b/fedoradocs_ci/build.py @@ -137,7 +137,7 @@ def announce_build_result(pr_data, target_dir_name): comment_on_pagure(comment, url) -def post_unsuccessful_build_comment(pr_data): +def announce_unsuccessful_build(pr_data): """ Posts a comment under the PR when the build fails @@ -148,7 +148,7 @@ def post_unsuccessful_build_comment(pr_data): comment = f"Thank you for your contribution. Unfortunately your PR did "\ "not build for some reason. Does it build locally for you?" - comment_on_pagure(pr_data, comment) + comment_on_pagure(comment, url) def comment_on_pagure(comment, url): diff --git a/fedoradocs_ci/consumer.py b/fedoradocs_ci/consumer.py index ff51983..643af4e 100644 --- a/fedoradocs_ci/consumer.py +++ b/fedoradocs_ci/consumer.py @@ -20,8 +20,8 @@ from fedora_messaging.config import conf from fedoradocs_ci.build import ( announce_build_start, announce_build_result, + announce_unsuccessful_build, build_docs, - post_unsuccessful_build_comment ) @@ -83,6 +83,7 @@ def consume(message): target_dir_name = build_docs(pr_data) announce_build_result(pr_data, target_dir_name) except Exception: - post_unsuccessful_build_comment(pr_data) + _log.exception("Failed to build the docs") + announce_unsuccessful_build(pr_data) else: _log.info("Ignoring %s, not in the projects of interest", project_to) From 9f54d456e1ec85848265727b781c472ca00c2a42 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 15:12:41 +0000 Subject: [PATCH 19/20] TemporaryDirectory returns a string, no need to call .name on them Also, log where we're working, that makes it easier to debug things while developing. Signed-off-by: Pierre-Yves Chibon --- diff --git a/fedoradocs_ci/build.py b/fedoradocs_ci/build.py index 34c54cb..8df444d 100644 --- a/fedoradocs_ci/build.py +++ b/fedoradocs_ci/build.py @@ -1,13 +1,15 @@ import os import argparse from git import Repo -import tempfile -import yaml -import requests +import logging import shutil +import tempfile +import requests import werkzeug.utils +import yaml +_log = logging.getLogger(__name__) DOCS_BUILDER_URL = "https://pagure.io/fedora-docs/docs-fp-o.git" DOCS_BUILDER_BRANCH = 'prod' @@ -51,21 +53,22 @@ def build_docs(pr_data): # Temporary directory to store the docs builder for preview in /tmp folder with tempfile.TemporaryDirectory(prefix="docs-ci-%s-" % pr_data['id']) as temp_dir: + _log.info("Working in %s", temp_dir) # Use git library to clone docs-fp-o and branch into a temporary - Repo.clone_from(url=DOCS_BUILDER_URL, to_path=f'{temp_dir.name}', + Repo.clone_from(url=DOCS_BUILDER_URL, to_path=f'{temp_dir}', branch=DOCS_BUILDER_BRANCH) # Change directory into the folder. # Only way for the script to recognise site.yml - os.chdir(f'{temp_dir.name}') + os.chdir(f'{temp_dir}') # Build the docs site with the data from the PR. Returns the playbook # data from site.yml This is used when writing to Apache config site_yml = configure_and_build_docs(pr_data) shutil.move( - f"{temp_dir.name + site_yml['output']['dir'].strip('.')}", + f"{temp_dir + site_yml['output']['dir'].strip('.')}", f"/var/www/html/{target_dir_name}" ) From 394a55da28a7811f42819c397324ee08309372d1 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 08 2021 15:16:34 +0000 Subject: [PATCH 20/20] Replace tabs with spaces in the delete_builds script Signed-off-by: Pierre-Yves Chibon --- diff --git a/delete_builds.py b/delete_builds.py index 883ebf6..2c11a3a 100644 --- a/delete_builds.py +++ b/delete_builds.py @@ -6,50 +6,50 @@ import time # main function def main(): - # specify the path - path = "/var/www/html" - - # specify the days - days = 15 - - # convert the current day to seconds - # time.time() returns current time since epoch (1970) in seconds - days_in_seconds = days * 24 * 60 * 60 - - # check whether the file is present in path or not - if os.path.exists(path): - # iterate over each list of folders in the path - for root_folder, folders, _ in os.walk(path): - for folder in folders: - # folder path - folder_path = os.path.join(root_folder, folder) + # specify the path + path = "/var/www/html" + + # specify the days + days = 15 + + # convert the current day to seconds + # time.time() returns current time since epoch (1970) in seconds + days_in_seconds = days * 24 * 60 * 60 + + # check whether the file is present in path or not + if os.path.exists(path): + # iterate over each list of folders in the path + for root_folder, folders, _ in os.walk(path): + for folder in folders: + # folder path + folder_path = os.path.join(root_folder, folder) # comparing with the days - if time.time() - get_folder_age(folder_path) >= days_in_seconds: + if time.time() - get_folder_age(folder_path) >= days_in_seconds: # invoking the remove_folder function - remove_folder(folder_path) - else: - # file/folder is not found - print(f'"{path}" is not found') + remove_folder(folder_path) + else: + # file/folder is not found + print(f'"{path}" is not found') def remove_folder(path): - # removing the folder - if not shutil.rmtree(path): - # success message - print(f"{path} is removed successfully") - else: - # failure message - print(f"Unable to delete the {path}") + # removing the folder + if not shutil.rmtree(path): + # success message + print(f"{path} is removed successfully") + else: + # failure message + print(f"Unable to delete the {path}") def get_folder_age(path): - # getting ctime of the folder - # time will be in seconds - ctime = os.stat(path).st_ctime + # getting ctime of the folder + # time will be in seconds + ctime = os.stat(path).st_ctime - # returning the time - return ctime + # returning the time + return ctime if __name__ == '__main__': - main() \ No newline at end of file + main()