PR#63: OpenShift Preparations for Packager Dashboard - fedora-qa/oraculum

fedora-qa / oraculum

#63 OpenShift Preparations for Packager Dashboard

Merged 3 years ago by frantisekz. Opened 3 years ago by frantisekz.

openshift_prep into master

OpenShift deployment preparations

František Zatloukal • 3 years ago

bdbcea1

Dockerfile

file modified

+13 -14

		`@@ -19,7 +19,7 @@`

		`USER root`

		`- RUN dnf -y install findutils python3-pip python3-setuptools python3-psycopg2 python3-mod_wsgi`
		`+ RUN dnf -y install findutils python3-pip python3-setuptools python3-psycopg2 python3-ipython`

		`# install dependencies extracted from setup.py and specfile`
		`COPY --from=builder /opt/app-root/src/oraculum/build/Dockerbuild/install_requires.list /opt/app-root/src/oraculum/`
		`@@ -31,31 +31,30 @@`
		`# install using --no-deps option to ensure nothing comes from PyPi`
		`RUN pip3 install --no-deps /opt/app-root/src/oraculum/`

		`- # fix apache config for container use`
		`- RUN sed -i 's#^WSGISocketPrefix .*#WSGISocketPrefix /var/run/wsgi#' /opt/app-root/src/oraculum/conf/oraculum.conf`
		`-`
		`# install launch script`
		`RUN install -p -m 0755 /opt/app-root/src/oraculum/container_start.sh /usr/bin/container_start`

		`# config files`
		`RUN install -d /usr/share/oraculum/conf \`
		`- && install -p -m 0644 /opt/app-root/src/oraculum/conf/oraculum.conf /usr/share/oraculum/conf/ \`
		`- && install -p -m 0644 /opt/app-root/src/oraculum/conf/oraculum.wsgi /usr/share/oraculum/ \`
		`- && install -d -m 0777 /etc/oraculum \`
		`- && install -p -m 0644 /opt/app-root/src/oraculum/conf/oraculum.conf /etc/httpd/conf.d/`
		`+ && install -p -m 0644 /opt/app-root/src/oraculum/conf/oraculum.conf.nginx /etc/nginx/conf.d/oraculum.conf \`
		`+ && install -d -m 0777 /etc/oraculum/`
		`+`
		`+ # write permissions for nginx conf`
		`+ RUN chmod -R a+w /etc/nginx/conf.d`

		`# alembic`
		`RUN install -p -m 0644 /opt/app-root/src/oraculum/alembic.ini /usr/share/oraculum/alembic.ini`
		`RUN cp -a /opt/app-root/src/oraculum/alembic /usr/share/oraculum/alembic`
		`RUN chmod -R 0755 /usr/share/oraculum/alembic`

		`- # EXPOSE 5005/tcp`
		`- EXPOSE 5005`
		`+ # EXPOSE 5000/tcp`
		`+ EXPOSE 5000`
		`+`
		`+ # We need to copy over nginx base conf without listen on :80`
		`+ RUN cp /opt/app-root/src/oraculum/conf/nginx_base.conf /etc/nginx/nginx.conf`

		- #RUN echo "SECRET_KEY = '`cat /dev/urandom \| tr -dc 'a-zA-Z0-9' \| fold -w 64 \| head -n 1`'" >> /etc/oraculum/settings.py
		`- #RUN echo "SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/oraculum.sqlite'" >> /etc/oraculum/settings.py`
		`- #RUN install -p -m 0644 /opt/app-root/src/oraculum/conf/client_secrets.json.example /etc/oraculum/client_secrets.json`
		`- #RUN echo "OIDC_CLIENT_SECRETS = '/etc/oraculum/client_secrets.json'" >> /etc/oraculum/settings.py`
		`+ RUN touch /etc/oraculum/settings.py`
		+ RUN echo "SECRET_KEY = '`cat /dev/urandom \| tr -dc 'a-zA-Z0-9' \| fold -w 64 \| head -n 1`'" >> /etc/oraculum/settings.py

		`CMD [ "runserver" ]`
		`ENTRYPOINT [ "/usr/bin/container_start" ]`

conf/nginx_base.conf

file added

+37

		`@@ -0,0 +1,37 @@`
		`+ # For more information on configuration, see:`
		`+ # * Official English Documentation: http://nginx.org/en/docs/`
		`+ # * Official Russian Documentation: http://nginx.org/ru/docs/`
		`+`
		`+ # user directive doesn't make sense in openshift/container deployment`
		`+ worker_processes auto;`
		`+ error_log /var/log/nginx/error.log;`
		`+ pid /tmp/nginx.pid;`
		`+ daemon off;`
		`+`
		`+ # Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.`
		`+ include /usr/share/nginx/modules/*.conf;`
		`+`
		`+ events {`
		`+ worker_connections 1024;`
		`+ }`
		`+`
		`+ http {`
		`+ log_format main '$remote_addr - $remote_user [$time_local] "$request" '`
		`+ '$status $body_bytes_sent "$http_referer" '`
		`+ '"$http_user_agent" "$http_x_forwarded_for"';`
		`+`
		`+ access_log /var/log/nginx/access.log main;`
		`+`
		`+ sendfile on;`
		`+ tcp_nopush on;`
		`+ tcp_nodelay on;`
		`+ keepalive_timeout 65;`
		`+ types_hash_max_size 4096;`
		`+`
		`+ include /etc/nginx/mime.types;`
		`+ default_type application/octet-stream;`
		`+`
		`+ include /etc/nginx/conf.d/*.conf;`
		`+`
		`+ }`
		`+`

conf/oraculum.conf.apache ~~conf/oraculum.conf~~

file renamed

file was moved with no change to the file

conf/oraculum.conf.nginx

file added

+12

		`@@ -0,0 +1,12 @@`
		`+ server {`
		`+ listen 5000;`
		`+ server_name _;`
		`+`
		`+ location / {`
		`+ proxy_set_header Host $http_host;`
		`+ proxy_set_header X-Real-IP $remote_addr;`
		`+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;`
		`+ proxy_set_header X-Forwarded-Proto $scheme;`
		`+ proxy_pass http://0.0.0.0:5001;`
		`+ }`
		`+ }`
		`\ No newline at end of file`

conf/oraculum.service

file added

		`@@ -0,0 +1,6 @@`
		`+ [Unit]`
		`+ Description=Gunicorn instance to serve oraculum`
		`+ After=network.target`
		`+`
		`+ [Service]`
		`+ ExecStart=gunicorn --workers 2 --daemon --bind 0.0.0.0:5001 oraculum`
		`\ No newline at end of file`

conf/oraculum.wsgi.apache ~~conf/oraculum.wsgi~~

file renamed

file was moved with no change to the file

container_start.sh

file modified

+41 -3

		`@@ -1,10 +1,48 @@`
		`#!/usr/bin/bash`
		`if [[ $1 == runserver ]]; then`
		`+ # Prepare database, this does nothing to already existing and latest db versions,`
		`+ # 11/10 sysadmins would recommend doing this every time`
		`+ oraculum init_db`
		`+ oraculum upgrade_db`
		`+`
		`+ gunicorn --workers 2 --bind 0.0.0.0:5001 oraculum`
		`+ fi`
		`+`
		`+ if [[ $1 == runproxy ]]; then`
		`+ # Set the url to the gunicorn worker`
		`+ sed -i "s,proxy_pass http://0.0.0.0:5001;,proxy_pass http://${ORACULUM_SERVICE_HOST}:5001;,g" /etc/nginx/conf.d/oraculum.conf`
		`+ nginx`
		`+ fi`
		`+`
		`+ if [[ $1 == runworker ]]; then`
		`+ celery -A oraculum.celery_app worker --concurrency=8`
		`+ fi`
		`+`
		`+ if [[ $1 == runbeat ]]; then`
		`+ celery -A oraculum.celery_app worker -B -s /tmp/celerybeat-schedule`
		`+ fi`
		`+`
		`+ if [[ $1 == debug ]]; then`
		`+ # Prepare development settings`
		`+ echo "SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/oraculum.sqlite'" >> /etc/oraculum/settings.py`
		`+ install -p -m 0644 /opt/app-root/src/oraculum/conf/client_secrets.json.example /etc/oraculum/client_secrets.json`
		`+ echo "OIDC_CLIENT_SECRETS = '/etc/oraculum/client_secrets.json'" >> /etc/oraculum/settings.py`
		`+`
		`# Prepare database`
		`oraculum init_db`
		`oraculum upgrade_db`

		`- mod_wsgi-express-3 start-server /usr/share/oraculum/oraculum.wsgi --user apache --group apache \`
		`- --port 5005 --threads 5 --include-file /etc/httpd/conf.d/oraculum.conf --log-level info \`
		`- --log-to-terminal --access-log --startup-log`
		`+ # Start redis and gunicorn`
		`+ redis-server --daemonize yes`
		`+ gunicorn --workers 2 --daemon --bind 0.0.0.0:5001 oraculum`
		`+`
		`+ # We want nginx to run in the background for debug run`
		`+ sed -i '/daemon/d' /etc/nginx/nginx.conf`
		`+`
		`+ # We want debug level logging`
		`+ echo "DEBUG = True" >> /etc/oraculum/settings.py`
		`+`
		`+ # And finally fire up nginx and beat`
		`+ nginx`
		`+ celery -A oraculum.celery_app worker -B`
		`fi`

oraculum.spec

file modified

+14 -9

		`@@ -1,8 +1,6 @@`
		`- %{?python_enable_dependency_generator}`
		`- %{!?_httpd_modconfdir: %{expand: %%global _httpd_modconfdir %%{_sysconfdir}/httpd/conf.d}}`
		`Name: oraculum`
		`# Don't forget to also change oraculum/__init__.py`
		`- Version: 0.0.3`
		`+ Version: 0.1.0`
		`Release: 1%{?dist}`
		`Summary: Backend and API for Fedora QA Dashboard`

		`@@ -15,6 +13,10 @@`
		`BuildRequires: python3-devel`
		`BuildRequires: python3-setuptools`

		`+ Requires: python3-gunicorn`
		`+ Requires: nginx`
		`+ Requires: redis`
		`+`
		`%description`
		`Backend and API for Fedora QA Dashboard`

		`@@ -27,15 +29,16 @@`
		`%install`
		`%py3_install`

		`- # apache and wsgi settings`
		`+ # service file and nginx settings`
		`mkdir -p %{buildroot}%{_datadir}/oraculum`
		`- cp -p conf/oraculum.wsgi %{buildroot}%{_datadir}/oraculum/.`

		`mkdir -p %{buildroot}%{_sysconfdir}/oraculum`
		`install conf/settings.py.example %{buildroot}%{_sysconfdir}/oraculum/settings.py`

		`- install -d -m 755 %{buildroot}%{_httpd_modconfdir}`
		`- install -p -m 644 conf/oraculum.conf %{buildroot}%{_httpd_modconfdir}/oraculum.conf`
		`+ install -d -m 755 %{buildroot}/etc/nginx/conf.d/`
		`+ install -p -m 644 conf/oraculum.conf.nginx %{buildroot}/etc/nginx/conf.d/oraculum.conf`
		`+`
		`+ install -Dpm 644 conf/oraculum.service %{buildroot}%{_unitdir}/oraculum.service`

		`%files`
		`%doc README.md conf/*`
		`@@ -45,12 +48,14 @@`
		`%{_bindir}/oraculum`
		`%dir %{_sysconfdir}/oraculum`
		`%dir %{_datadir}/oraculum`
		`- %{_datadir}/oraculum/*`

		`%config(noreplace) %{_sysconfdir}/oraculum/settings.py`
		`- %config(noreplace) %{_httpd_modconfdir}/oraculum.conf`
		`+ %config(noreplace) %{_sysconfdir}/nginx/conf.d/oraculum.conf`

		`%changelog`
		`+ * Mon Jul 27 2020 Frantisek Zatloukal <fzatlouk@redhat.com> - 0.1.0-1`
		`+ - Release 0.1.0`
		`+`
		`* Mon Feb 10 2020 Frantisek Zatloukal <fzatlouk@redhat.com> - 0.0.3-1`
		`- Release 0.0.3`

oraculum/__init__.py

file modified

+5 -3

		`@@ -33,7 +33,7 @@`
		`from . import config`

		`# the version as used in setup.py`
		`- __version__ = "0.0.3"`
		`+ __version__ = "0.1.0"`


		`def make_celery(app):`
		`@@ -162,8 +162,8 @@`
		`from .utils import cache_utils # NOQA: E402`
		`CACHE = cache_utils.Cached(max_cache_age=app.config['MAX_DB_AGE'])`
		`BUGZILLA = bugzilla.Bugzilla("bugzilla.redhat.com",`
		`- tokenfile=os.path.expanduser("~/.cache/python-bugzilla/token-%d" % os.getpid()),`
		`- cookiefile=os.path.expanduser("~/.cache/python-bugzilla/cookie-%d" % os.getpid()),`
		`+ tokenfile="/tmp/python-bugzilla-token-%d" % os.getpid(),`
		`+ cookiefile="/tmp/python-bugzilla-cookie-%d" % os.getpid(),`
		`force_rest=True)`

		`CORS(app, supports_credentials=True)`
		`@@ -180,3 +180,5 @@`
		`from oraculum.controllers import main # NOQA: E402`
		`from oraculum.controllers import oidc_login # NOQA: E402`
		`from oraculum.controllers import packager_dashboard # NOQA: E402`
		`+`
		`+ application = app`

oraculum/config.py

file modified

+30 -2

		`@@ -170,15 +170,43 @@`
		`temp.write(os.environ["CLIENT_SECRETS"])`
		`temp.close()`
		`config_object["OIDC_CLIENT_SECRETS"] = temp.name`
		`- except(KeyError):`
		`+ except KeyError:`
		`print("OpenShift mode enabled but required values couldn't be fetched. "`
		`"Check, if you have these variables defined in you env: "`
		`- "(POSTGRESQL_[USER, PASSWORD, DATABASE, SERVICE_HOST, SERVICE_PORT], SECRET_KEY, CLIENT_SECRETS)", file=sys.stderr)`
		`+ "(POSTGRESQL_[USER, PASSWORD, DATABASE, SERVICE_HOST, SERVICE_PORT], SECRET_KEY, CLIENT_SECRETS)",`
		`+ file=sys.stderr)`
		`+ sys.exit(1)`
		`+`
		`+ # Let's try redis`
		`+ try:`
		`+ config_object["CELERY_BROKER_URL"] = "redis://:%s@%s:%s" % (`
		`+ os.environ["REDIS_PASSWORD"],`
		`+ os.environ["REDIS_SERVICE_HOST"],`
		`+ os.environ["REDIS_SERVICE_PORT_REDIS"]`
		`+ )`
		`+ config_object["CELERY_RESULT_BACKEND"] = config_object["CELERY_BROKER_URL"]`
		`+ except KeyError:`
		`+ print("OpenShift mode enabled but Redis instance couldn't be reached."`
		`+ "Please, make sure REDIS_[PASSWORD, SERVICE_HOST, SERVICE_PORT_REDIS] are available in env.",`
		`+ file=sys.stderr)`
		`sys.exit(1)`

		`# Get some more data from OpenShift, if set`
		`if os.getenv("KANBAN_CONFIG"):`
		`config_object["KANBAN_CONFIG"] = os.getenv("KANBAN_CONFIG")`
		`+ if os.getenv("ALLOWED_PACKAGER_GROUPS"):`
		`+ config_object["ALLOWED_PACKAGER_GROUPS"] = os.getenv("ALLOWED_PACKAGER_GROUPS")`
		`+ if os.getenv("EPEL_RELEASES"):`
		`+ config_object["EPEL_RELEASES"] = os.getenv("EPEL_RELEASES")`
		`+ if os.getenv("SMTP_CONFIG"):`
		`+ smtp_config = os.getenv("SMTP_CONFIG").split(";")`
		`+ config_object["SEND_ERROR_EMAILS"] = True`
		`+ config_object["SMTP_SERVER"] = smtp_config[0]`
		`+ config_object["SMTP_PORT"] = smtp_config[1]`
		`+ config_object["SMTP_LOGIN"] = smtp_config[2]`
		`+ config_object["SMTP_SENDER"] = smtp_config[3]`
		`+ config_object["SMTP_PASSWORD"] = smtp_config[4]`
		`+ config_object["ADMIN_EMAILS"] = os.getenv("ADMIN_EMAILS")`

		`# Some final touches for oidc`
		`if os.getenv("OVERWRITE_REDIRECT_URI"):`

requirements.txt

file modified

+1 -1

		`@@ -16,7 +16,7 @@`
		`pygments`
		`python-bugzilla >= 2.4.0`
		`python-dateutil == 2.8.1`
		`- python-igraph >= 2.8.0`
		`+ python-igraph >= 0.8.0`
		`pytz`
		`requests`
		`celery`

setup.py

file modified

+1 -1

		`@@ -27,7 +27,7 @@`
		`author='Josef Skladanka',`
		`author_email='jskladan@redhat.com',`
		`license='GPLv2+',`
		`- packages=['oraculum', 'oraculum.controllers', 'oraculum.models', 'oraculum.utils', 'oraculum.data_providers'],`
		`+ packages=['oraculum', 'oraculum.action_providers', 'oraculum.controllers', 'oraculum.models', 'oraculum.utils'],`
		`package_dir={'oraculum': 'oraculum'},`
		`entry_points=dict(console_scripts=['oraculum=oraculum.cli:main']),`
		`include_package_data=True,`