From 3952bf6ff7d8b80742b1ff8ce0217f9db853c97c Mon Sep 17 00:00:00 2001 From: Ozan Unsal Date: Oct 06 2021 07:29:57 +0000 Subject: Create docker-compose file in order to run the ODCS locally Merges: https://pagure.io/odcs/pull-request/517 Jira: RHELCMP-6808 Signed-off-by: Ozan Unsal --- diff --git a/Dockerfile b/Dockerfile index bf8ebca..c4d2ef4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:31 +FROM fedora:34 LABEL \ name="ODCS application" \ vendor="ODCS developers" \ @@ -59,5 +59,3 @@ RUN pip3 install . --no-deps WORKDIR /tmp USER 1001 EXPOSE 8080 - -ENTRYPOINT celery-3 -A odcs.server.celery_tasks worker --loglevel=info -Q pungi_composes,pulp_composes diff --git a/README.md b/README.md index 76dd3ed..45d7cc6 100644 --- a/README.md +++ b/README.md @@ -295,3 +295,103 @@ $ ./submit_test_compose repo `pwd`/server/tests/repo ed ``` You should then see the backend process generating the compose and once it's done, the resulting compose in `./test_composes/latest-Unknown-1/compose/Temporary` directory. + +### Using docker-compose for creating a local setup + +You can create test setup for ODCS with the docker-compose file. This yaml file creates docker container for the backend and frontend setup of ODCS and run multiple services together. These services are; + +* **rabbitmq** (Handles the communication between backend and frontend) +* **postgres** (Creates the database where the localy generated composes are stored) +* **backend** (Backend service of ODCS) +* **frontend** (Frontend service of ODCS that handles the REST API) +* **static** (Apache service for making storage available) +* **beat** (Cronjob for backend to check the service status) + +In addition to these, there are also three docker volumes (`odcs_odcs-composes`, `odcs_odcs-postgres` and `odcs_rabbitmq`) are created. These are providing persistent storage for the services. + +This yaml file requires also an **.env** file that specified some enviroment variables for the configuration of frontend and backend. Such an **.env** file should be in the same path with the **docker-compose.yml** file and here are the necessary variables that need to be specified in this file; +``` +# Pyhton path +PYTHONPATH=/src/common:/src/server:/src/client + +# POSTGRES +POSTGRES_USER=odcs +POSTGRES_PASSWORD=password + +# PULP CONFIGURATION +PULP_SERVER_URL= +PULP_USERNAME= +PULP_PASSWORD= +RAW_CONFIG_URLS= + +# ODCS CONFIGURATION +ODCS_CONFIG_SECTION=DevConfiguration +ODCS_CONFIG_FILE=/src/server/conf/config.py +ODCS_CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672/ +ODCS_DB_URL=postgresql+psycopg2://odcs:password@postgres/odcs +# Directory where the generated composes are stored. This hast to match +# location where odcs-composes volume is mounted in frontend and backend +# containers. +ODCS_TARGET_DIR=/mnt/odcs +TARGET_DIR_URL=http://localhost:8080 + +# FLASK SETTINGS +# Force flask to reload application on change of source files. +FLASK_ENV=development +``` + +The services can be start with `sudo docker-compose up` command. If you have face an error or something that does not work correctly please use the following steps; + +1. Check whether there is already created docker volume. + + + $ sudo docker volume ls + Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. + DRIVER VOLUME NAME + + + If the output is like above, it means there isn't any volume yet and it is sufficient to run `sudo docker-compose up` and then `sudo docker-compose down` command. This creates the volumes and if you run the above command again the output becomes as follows; + + + $ sudo docker volume ls + Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. + DRIVER VOLUME NAME + local odcs_odcs-composes + local odcs_odcs-postgres + local odcs_odcs-rabbitmq + + +2. After this point we need to set the correct permission to `ocds_odcs-composes` volume. In order to find where is the actual location of the volume type the following + + + $ sudo docker volume inspect odcs_odcs-composes + Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. + [ + { + "Name": "odcs_odcs-composes", + "Driver": "local", + "Mountpoint": "/var/lib/containers/storage/volumes/odcs_odcs-composes/_data", + "CreatedAt": "2021-10-04T13:19:36.478636851+02:00", + "Labels": { + "io.podman.compose.project": "odcs" + }, + "Scope": "local", + "Options": {} + } + ] + + `"Mountpoint": "/var/lib/containers/storage/volumes/odcs_odcs-composes/_data"` shows the exact location of the corresponding volume. + + Add group write permission to the Mountpoint by + + + $ sudo chmod 775 /var/lib/containers/storage/volumes/odcs_odcs-composes/_data + + +3. In this step it is sufficient to run `sudo docker-compose up` command to start the services properly. + +Here are some REST calls for checking the ODCS. + +* `$ curl -s http://localhost:5000/api/1/composes/ | jq .` This call shows all the composes in db. +* `$ odcs --server http://localhost:5000/ create-pulp ` This call starts a pulp compose that match the given pulp content set +* `$ odcs --server http://localhost:5000/ create-raw-config --compose-type test my_raw_config master` This call starts a compose with the configuration defined as my_raw_config diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7414d99 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,90 @@ +version: '3' +services: + rabbitmq: + image: rabbitmq:3.9-management + environment: + RABBITMQ_NODENAME: rabbitmq@localhost + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + ports: + - 5672:5672 + - 15672:15672 + - 15692:15692 + volumes: + - odcs-rabbitmq:/var/lib/rabbitmq/:z + + postgres: + image: postgres:9 + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} + volumes: + - odcs-postgres:/var/lib/postgresql/data + + backend: + build: + context: . + dockerfile: ./Dockerfile + args: + cacert_url: https://password.corp.redhat.com/RH-IT-Root-CA.crt + env_file: + - .env + entrypoint: ["/bin/sh", "-c"] + command: + - | + celery-3 -b rabbitmq -A odcs.server.celery_tasks worker --loglevel=info -Q pungi_composes,pulp_composes,cleanup + volumes: + - ".:/src:z" + - odcs-composes:/mnt/odcs:z + depends_on: + - rabbitmq + - postgres + + frontend: + build: + context: . + dockerfile: ./Dockerfile + env_file: + - .env + entrypoint: ["/bin/sh","-c"] + command: + - | + sleep 10 + python3 /src/server/odcs/server/manage.py db upgrade + python3 /src/server/odcs/server/manage.py run --host 0.0.0.0 + depends_on: + - rabbitmq + - postgres + ports: + - 5000:5000 + volumes: + - ".:/src:z" + - odcs-composes:/mnt/odcs:z + + static: + image: httpd:2.4 + ports: + - 8080:80 + volumes: + - odcs-composes:/usr/local/apache2/htdocs/:z + + beat: + build: + context: . + dockerfile: ./Dockerfile + env_file: + - .env + entrypoint: ["/bin/sh", "-c"] + command: + - | + celery-3 -b rabbitmq -A odcs.server.celery_tasks beat --loglevel=debug + depends_on: + - rabbitmq + volumes: + - ".:/src:z" + - odcs-composes:/mnt/odcs:z + +volumes: + odcs-composes: + odcs-rabbitmq: + odcs-postgres: diff --git a/server/conf/config.py b/server/conf/config.py index 392ac94..b3ce57c 100644 --- a/server/conf/config.py +++ b/server/conf/config.py @@ -1,5 +1,6 @@ import errno -from os import path, makedirs +import json +from os import path, makedirs, environ # FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are @@ -14,7 +15,9 @@ dbdir = ( class BaseConfiguration(object): # Make this random (used to generate session keys) SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0" - SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "odcs.db")) + SQLALCHEMY_DATABASE_URI = environ.get( + "ODCS_DB_URL", "sqlite:///{0}".format(path.join(dbdir, "odcs.db")) + ) SQLALCHEMY_TRACK_MODIFICATIONS = False HOST = "127.0.0.1" @@ -79,7 +82,7 @@ class BaseConfiguration(object): # noauth: no authentication is enabled. Useful for development particularly. # kerberos: Kerberos authentication is enabled. # openidc: OpenIDC authentication is enabled. - AUTH_BACKEND = "" + AUTH_BACKEND = "noauth" # Used for Kerberos authentication and to query user's groups. # Format: ldap://hostname[:port] @@ -169,6 +172,17 @@ class BaseConfiguration(object): }, } + if "PULP_SERVER_URL" in environ: + PULP_SERVER_URL = environ.get("PULP_SERVER_URL") + if "PULP_USERNAME" in environ: + PULP_USERNAME = environ.get("PULP_USERNAME") + if "PULP_PASSWORD" in environ: + PULP_PASSWORD = environ.get("PULP_PASSWORD") + if "TARGET_DIR_URL" in environ: + TARGET_DIR_URL = environ.get("TARGET_DIR_URL") + if "RAW_CONFIG_URLS" in environ: + RAW_CONFIG_URLS = json.loads(environ.get("RAW_CONFIG_URLS")) + class DevConfiguration(BaseConfiguration): DEBUG = True @@ -178,7 +192,7 @@ class DevConfiguration(BaseConfiguration): # Global network-related values, in seconds NET_TIMEOUT = 5 NET_RETRY_INTERVAL = 1 - TARGET_DIR = path.join(dbdir, "test_composes") + TARGET_DIR = environ.get("ODCS_TARGET_DIR", path.join(dbdir, "test_composes")) try: makedirs(TARGET_DIR, mode=0o775) except OSError as ex: @@ -191,7 +205,7 @@ class DevConfiguration(BaseConfiguration): AUTH_BACKEND = "noauth" AUTH_OPENIDC_USERINFO_URI = "https://iddev.fedorainfracloud.org/openidc/UserInfo" - KOJI_PROFILE = "stg" + KOJI_PROFILE = "koji" RAW_CONFIG_WRAPPER_CONF_PATH = path.join(confdir, "raw_config_wrapper.conf") diff --git a/server/odcs/server/celery_tasks.py b/server/odcs/server/celery_tasks.py index 97acaa5..fe904ce 100644 --- a/server/odcs/server/celery_tasks.py +++ b/server/odcs/server/celery_tasks.py @@ -48,6 +48,7 @@ remove_expired_compose_thread = RemoveExpiredComposesThread() # Create the Celery app. if os.environ.get("ODCS_CELERY_BROKER_URL"): broker_url = os.environ["ODCS_CELERY_BROKER_URL"] + conf.celery_broker_url = broker_url elif conf.celery_broker_url: broker_url = conf.celery_broker_url else: