PR#11: [UGLY] Deps from setup.py - fedora-qa/oraculum

fedora-qa / oraculum

#11 [UGLY] Deps from setup.py

Closed 4 years ago by jskladan. Opened 4 years ago by frantisekz.

deps into master

Multistage Dockerfile, random fixex

Josef Skladanka • 4 years ago

e150496

Extract dependencies from setup.py and specfile in Dockerfile

František Zatloukal • 4 years ago

d0f5326

.dockerignore

file added

+20

		`@@ -0,0 +1,20 @@`
		`+ venv/`
		`+ .venv/`
		`+`
		`+ # Python`
		`+ __pycache__/`
		`+ *.py[cod]`
		`+ *.egg-info`
		`+`
		`+ # Misc`
		`+ .vscode/`
		`+ *.swp`
		`+`
		`+ # rpm build`
		`+ build/`
		`+ dist/`
		`+ /*.egg-info`
		`+ /*.tar.gz`
		`+ /*.rpm`
		`+`
		`+`

.gitignore

file modified

		`@@ -3,6 +3,7 @@`
		`*.py[cod]`
		`*.egg-info`
		`venv`
		`+ .venv`

		`# Misc`
		`.vscode/`

Dockerfile

file modified

+21 -27

		`@@ -1,4 +1,14 @@`
		`- FROM fedora:32`
		`+ FROM fedora:32 AS builder`
		`+ USER root`
		`+`
		`+ RUN dnf -y install rpm-build pyp2rpm make git redhat-lsb-core`
		`+`
		`+ COPY . /opt/app-root/src/oraculum/`
		`+`
		`+ RUN cd /opt/app-root/src/oraculum/ && make clean && make dockerdeps`
		`+`
		`+`
		`+ FROM fedora:32 AS deployment`
		`LABEL \`
		`name="Oraculum" \`
		`vendor="Fedora QE" \`
		`@@ -9,27 +19,15 @@`

		`USER root`

		`- RUN dnf -y install findutils rpm-build python3-pip python3-mod_wsgi python3-pycurl \`
		`- && dnf -y install python3-alembic \`
		`- python3-flask \`
		`- python3-flask-caching \`
		`- python3-flask-cors \`
		`- python3-flask-login \`
		`- python3-flask-oidc \`
		`- python3-flask-sqlalchemy \`
		`- python3-icalendar \`
		`- python3-lxml \`
		`- python3-mod_wsgi \`
		`- python3-pygments \`
		`- python3-bugzilla \`
		`- python3-dateutil \`
		`- python3-bodhi-client \`
		`- python3-pytz \`
		`- python3-psycopg2 \`
		`- python3-requests \`
		`- python3-setuptools`
		`+ RUN dnf -y install findutils python3-pip python3-setuptools python3-psycopg2 python3-mod_wsgi`
		`+`
		`+ # install dependencies extracted from setup.py and specfile`
		`+ COPY --from=builder /opt/app-root/src/oraculum/build/Dockerbuild/install_requires.list /opt/app-root/src/oraculum/`
		`+ RUN cat /opt/app-root/src/oraculum/install_requires.list \| xargs -d '\n' dnf -y install && dnf clean all`
		`+`
		`+ # copy sources to the container`
		`+ COPY --from=builder /opt/app-root/src/oraculum/build/Dockerbuild/src/ /opt/app-root/src/oraculum/`

		`- COPY . /opt/app-root/src/oraculum/`
		`# install using --no-deps option to ensure nothing comes from PyPi`
		`RUN pip3 install --no-deps /opt/app-root/src/oraculum/`

		`@@ -51,17 +49,13 @@`
		`RUN cp -a /opt/app-root/src/oraculum/alembic /usr/share/oraculum/alembic`
		`RUN chmod -R 0755 /usr/share/oraculum/alembic`

		`- # clean up`
		`- RUN rm -rf /opt/app-root/src/oraculum \`
		`- && dnf -y autoremove findutils rpm-build \`
		`- && dnf clean all`
		`-`
		`# EXPOSE 5005/tcp`
		`EXPOSE 5005`

		#RUN echo "SECRET_KEY = '`cat /dev/urandom \| tr -dc 'a-zA-Z0-9' \| fold -w 64 \| head -n 1`'" >> /etc/oraculum/settings.py
		`#RUN echo "SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/oraculum.sqlite'" >> /etc/oraculum/settings.py`
		`- RUN echo "OIDC_CLIENT_SECRETS = '/etc/oraculum/client_secrets.json'" >> /etc/oraculum/settings.py`
		`+ #RUN install -p -m 0644 /opt/app-root/src/oraculum/conf/client_secrets.json.example /etc/oraculum/client_secrets.json`
		`+ #RUN echo "OIDC_CLIENT_SECRETS = '/etc/oraculum/client_secrets.json'" >> /etc/oraculum/settings.py`

		`CMD [ "runserver" ]`
		`ENTRYPOINT [ "/usr/bin/container_start" ]`

Makefile

file modified

+14

		`@@ -50,6 +50,19 @@`
		`curl --fail https://pagure.io/fedora-qa/qa-make/raw/master/f/Makefile -o Makefile.new`
		`if ! cmp Makefile Makefile.new ; then mv Makefile.new Makefile ; fi`

		`+ # Parses out dependencies from setup.py and specfile into build/Dockerbuild/install_requires.lis`
		`+ # Moves the product of make archive to a known location for the use within Dockerfile`
		`+ .PHONY: dockerdeps`
		`+ dockerdeps:`
		`+ @mkdir -p build/Dockerbuild`
		`+ @make archive`
		`+ @pyp2rpm -d build/$(VERSION)-$(RELEASE) build/$(VERSION)-$(RELEASE)/$(SRC)-$(VERSION).tar.gz > parsed.spec`
		`+ @rpm --query --requires --specfile parsed.spec > build/Dockerbuild/install_requires.list`
		`+ @rpm --query --requires --specfile $(SPECFILE) >> build/Dockerbuild/install_requires.list`
		`+ @tar xzf build/$(VERSION)-$(RELEASE)/$(SRC)-$(VERSION).tar.gz --directory=build/Dockerbuild/`
		`+ @mv build/Dockerbuild/$(SRC)-$(VERSION) build/Dockerbuild/src`
		`+ @rm parsed.spec build/$(VERSION)-$(RELEASE)/$(SRC)-$(VERSION).tar.gz`
		`+`
		`.PHONY: test`
		`.ONESHELL: test`
		`test: $(VENV)`
		`@@ -80,6 +93,7 @@`
		`.PHONY: docs`
		`docs:`
		`sphinx-build -b html -d docs/_build/doctrees docs/source docs/_build/html`
		`+ find docs/source/ -name '*.md' -exec cp {} docs/_build/html/ \;`

		`.PHONY: clean`
		`clean:`

container_start.sh

file modified

-2

		`@@ -1,7 +1,5 @@`
		`#!/usr/bin/bash`
		`if [[ $1 == runserver ]]; then`
		`- echo $CLIENT_SECRETS > /etc/oraculum/client_secrets.json`
		`-`
		`# Prepare database`
		`oraculum init_db`
		`oraculum upgrade_db`

oraculum/config.py

file modified

+9 -1

		`@@ -19,6 +19,7 @@`

		`import os`
		`import sys`
		`+ import tempfile`

		`class Config(object):`
		`DEBUG = True`
		`@@ -50,6 +51,7 @@`
		`PRODUCTION = True`
		`DEBUG = False`
		`FORCE_CACHED_DATA = True`
		`+ OIDC_CLIENT_SECRETS = '/etc/oraculum/client_secrets.json'`


		`class DevelopmentConfig(Config):`
		`@@ -75,7 +77,13 @@`
		`os.environ["POSTGRESQL_DATABASE"]`
		`)`
		`config_object["SECRET_KEY"] = os.environ["SECRET_KEY"]`
		`- config_object["CLIENT_SECRETS"] = os.environ["CLIENT_SECRETS"]`
		`+`
		`+ # Creates a temporary file containing the CLIENT_SECRETS data`
		`+ # and sets OIDC_CLIENT_SECRETS to the files path`
		`+ temp = tempfile.NamedTemporaryFile(mode="w+", delete=False)`
		`+ temp.write(os.environ["CLIENT_SECRETS"])`
		`+ temp.close()`
		`+ config_object["OIDC_CLIENT_SECRETS"] = temp.name`
		`except(KeyError):`
		`print("OpenShift mode enabled but required values couldn't be fetched. "`
		`"Check, if you have these variables defined in you env: "`

setup.py

file modified

+1 -2

		`@@ -22,7 +22,7 @@`

		`setup(name='oraculum',`
		`version=find_version('oraculum', '__init__.py'),`
		`- description='',`
		`+ description='Salty salmon...',`
		`author='Josef Skladanka',`
		`author_email='jskladan@redhat.com',`
		`license='GPLv2+',`
		`@@ -41,7 +41,6 @@`
		`'Flask-Sqlalchemy',`
		`'icalendar',`
		`'lxml',`
		`- 'mod_wsgi',`
		`'pygments',`
		`'python-bugzilla',`
		`'python-dateutil',`

frantisekz commented 4 years ago

.. commit works outside the docker env
... commit contains changes to make it work in docker build... god knows why is this needed

This is terrible, it should be burned and I wish not to see it ever again!

jskladan commented on line 29 of Dockerfile 4 years ago

As we discussed online - would be nicer to have the dockerdeps step dependent on the archive step in the Makefile.

jskladan commented on line 11 of Makefile 4 years ago

how about rpm --query --requires --specfile $(SPECFILE) instead?

jskladan commented on line 7 of Makefile 4 years ago

Hmm, I understand this is PoC, but how about no? :)

Based on the source code of pyp2rpm, using %{_topdir} instead of /root/rpmbuild should be OK, and will work even without root access ;) or you should be able to do this: @pyp2rpm -d build/$(VERSION)-$(RELEASE) build/$(VERSION)-$(RELEASE)/$(SRC)-$(VERSION).tar.gz and get rid of the mkdir/mv boilerplate. Not tested though, just went deep into the pyp2rpm code.

1 new commit added

There, fixed it for you :)

4 years ago

rebased onto d0f5326

4 years ago

1 new commit added

Multistage Dockerfile, random fixex

4 years ago

jskladan commented 4 years ago

I've fixed the code for you (not saying the Makefile/Dockerfile can't be better), and made the Dockerfile be a multistage build. tldr - we won't have unnecessary random packages from the RUN dnf -y install rpm-build pyp2rpm make git redhat-lsb-core in the final image.

I also changed the way CLIENT_SECRETS is handled. The way you have cleverly hidden it in the container_start.sh was... not ideal :)

@lbrabec please have a look
@frantisekz please test if it even works in openshift like this

THX!

lbrabec commented on line 11 of Dockerfile 4 years ago

just a nitpick, cat this be oneliner? RUN cd /opt/app-root/src/oraculum/ && make clean && make dockerdeps

lbrabec commented on line 6 of Makefile 4 years ago

Could you please add short comment here what's going on?

1 new commit added

Tweaks

4 years ago

frantisekz commented 4 years ago

just a nitpick, cat this be oneliner? RUN cd /opt/app-root/src/oraculum/ && make clean && make dockerdeps

Done

Could you please add short comment here what's going on?

Added simple one line comment. I think it's enough.... taking into account how are other parts of qa-make commented.

frantisekz commented 4 years ago

@frantisekz please test if it even works in openshift like this

Aaand it does not. I'll try to fix that :)

[Fri Apr 03 06:21:03.598751 2020] [wsgi:error] [pid 13:tid 140172142100800] Traceback (most recent call last):
[Fri Apr 03 06:21:03.598800 2020] [wsgi:error] [pid 13:tid 140172142100800]   File "/tmp/mod_wsgi-localhost:5005:1000760000/handler.wsgi", line 89, in <module>
[Fri Apr 03 06:21:03.598804 2020] [wsgi:error] [pid 13:tid 140172142100800]     handler = mod_wsgi.server.ApplicationHandler(entry_point,
[Fri Apr 03 06:21:03.598810 2020] [wsgi:error] [pid 13:tid 140172142100800]   File "/usr/lib64/python3.8/site-packages/mod_wsgi/server/__init__.py", line 1424, in __init__
[Fri Apr 03 06:21:03.598812 2020] [wsgi:error] [pid 13:tid 140172142100800]     exec(code, self.module.__dict__)
[Fri Apr 03 06:21:03.598818 2020] [wsgi:error] [pid 13:tid 140172142100800]   File "/usr/share/oraculum/oraculum.wsgi", line 15, in <module>
[Fri Apr 03 06:21:03.598820 2020] [wsgi:error] [pid 13:tid 140172142100800]     from oraculum import app as application
[Fri Apr 03 06:21:03.598825 2020] [wsgi:error] [pid 13:tid 140172142100800]   File "/usr/local/lib/python3.8/site-packages/oraculum/__init__.py", line 60, in <module>
[Fri Apr 03 06:21:03.598828 2020] [wsgi:error] [pid 13:tid 140172142100800]     config.openshift_config(app.config, openshift_production)
[Fri Apr 03 06:21:03.598833 2020] [wsgi:error] [pid 13:tid 140172142100800]   File "/usr/local/lib/python3.8/site-packages/oraculum/config.py", line 84, in openshift_config
[Fri Apr 03 06:21:03.598836 2020] [wsgi:error] [pid 13:tid 140172142100800]     temp.write(os.environ["CLIENT_SECRETS"])
[Fri Apr 03 06:21:03.598840 2020] [wsgi:error] [pid 13:tid 140172142100800]   File "/usr/lib64/python3.8/tempfile.py", line 474, in func_wrapper
[Fri Apr 03 06:21:03.598843 2020] [wsgi:error] [pid 13:tid 140172142100800]     return func(*args, **kwargs)
[Fri Apr 03 06:21:03.598854 2020] [wsgi:error] [pid 13:tid 140172142100800] TypeError: a bytes-like object is required, not 'str'

1 new commit added