greenwave

Clone

#512 Use podman-compose by default if available
Merged 4 years ago by lholecek. Opened 4 years ago by lholecek.
lholecek/greenwave podman-compose  into  master

file modified
+1 -1 
@@ -34,5 +34,5 @@ 
 

  RUN rm -rf ./fedmsg.d
 

  USER 1001
 

  EXPOSE 8080
 

- ENTRYPOINT ["docker/docker-entrypoint.sh"]
 

+ ENTRYPOINT ["/src/docker/docker-entrypoint.sh"]
 

  CMD ["/usr/bin/gunicorn-3", "--workers", "8", "--bind", "0.0.0.0:8080", "--access-logfile", "-", "--enable-stdio-inheritance", "greenwave.wsgi:app"]

file modified
+55 -13 
@@ -1,5 +1,21 @@ 
 

- COMPOSE := docker-compose
 

+ # Use podman-compose by default if available.
 

+ ifeq (, $(shell which podman-compose))
 

+     COMPOSE := docker-compose
 

+     PODMAN := docker
 

+ else
 

+     COMPOSE := podman-compose
 

+     PODMAN := podman
 

+ endif
 

+ 

  BROWSER := xdg-open
 

+ SERVICE := dev
 

+ TEST_REQUIREMENTS := dev-requirements.txt
 

+ 

+ PYTHON := python3
 

+ PIP := $(PYTHON) -m pip
 

+ PYTEST := $(PYTHON) -m pytest --color=yes
 

+ FLAKE8 := $(PYTHON) -m flake8
 

+ PYLINT := $(PYTHON) -m pylint
 

  

  all: help
 

  
@@ -27,6 +43,22 @@ 
 

  	@echo '  make test - alias for "make pytest flake8 pylint"'
 

  	@echo
 

  	@echo '  make coverage [ARGS=".."] - generates and shows test code coverage'
 

+ 	@echo
 

+ 	@echo '  make test-waiverdb - run tests for WaiverDB'
 

+ 	@echo
 

+ 	@echo 'Variables:'
 

+ 	@echo
 

+ 	@echo '  COMPOSE=docker-compose|podman-compose'
 

+ 	@echo '    - docker-compose or podman-compose command'
 

+ 	@echo '      (default is "podman-compose" if available)'
 

+ 	@echo
 

+ 	@echo '  PODMAN=docker|podman'
 

+ 	@echo '    - docker or podman command'
 

+ 	@echo '      (default is "podman" if "podman-compose" is available)'
 

+ 	@echo
 

+ 	@echo '  SERVICE={dev|waiverdb|resultsdb|waiverdb-db|resultsdb-db|memcached}'
 

+ 	@echo '    - service for which to run `make exec` and similar (default is "dev")'
 

+ 	@echo '      Example: make exec SERVICE=waiverdb CMD=flake8'
 

  

  up:
 

  	$(COMPOSE) up -d
 
@@ -40,26 +72,36 @@ 
 

  recreate:
 

  	$(COMPOSE) up -d --force-recreate
 

  

- # Executes CMD in dev container.
 

- # Usage: make exec CMD="python3 -m pytest -x"
 

- exec: up
 

- 	$(COMPOSE) exec dev bash -c '$(CMD)'
 

+ exec:
 

+ 	$(PODMAN) exec greenwave_$(SERVICE)_1 bash -c '$(CMD)'
 

+ 

+ sudo:
 

+ 	$(PODMAN) exec -u root greenwave_$(SERVICE)_1 bash -c '$(CMD)'
 

  

- sudo: up
 

- 	$(COMPOSE) exec -u root dev bash -c '$(CMD)'
 

+ test: test_requirements pytest flake8 pylint
 

  

- test: pytest flake8 pylint
 

+ test_requirements:
 

+ 	$(MAKE) exec CMD="$(PIP) install --user -r $(TEST_REQUIREMENTS)"
 

  

  pytest:
 

  	$(MAKE) exec \
 

- 	    CMD="pip3 install --user -r dev-requirements.txt && COVERAGE_FILE=/home/dev/.coverage python3 -m pytest $(ARGS)"
 

+ 	    CMD="COVERAGE_FILE=/home/dev/.coverage-$(SERVICE) $(PYTEST) $(ARGS)"
 

  

  flake8:
 

- 	python -m flake8
 

+ 	$(FLAKE8)
 

  

  pylint:
 

- 	python -m pylint greenwave/
 

+ 	$(PYLINT) greenwave/
 

  

  coverage:
 

- 	$(MAKE) pytest ARGS="--cov-config .coveragerc --cov=greenwave --cov-report html:/home/dev/htmlcov $(ARGS)"
 

- 	$(BROWSER) docker/home/htmlcov/index.html
 

+ 	$(MAKE) pytest ARGS="--cov-config .coveragerc --cov=greenwave --cov-report html:/home/dev/htmlcov-$(SERVICE) $(ARGS)"
 

+ 	$(BROWSER) docker/home/htmlcov-$(SERVICE)/index.html
 

+ 

+ test-waiverdb:
 

+ 	$(MAKE) SERVICE=waiverdb TEST_REQUIREMENTS=test-requirements.txt test_requirements
 

+ 	$(MAKE) SERVICE=waiverdb pytest
 

+ 	$(MAKE) exec CMD="$(PIP) install --user flake8 && $(FLAKE8) waiverdb"
 

+ 

+ test-resultsdb:
 

+ 	$(MAKE) SERVICE=resultsdb TEST_REQUIREMENTS=requirements.txt test_requirements
 

+ 	$(MAKE) SERVICE=resultsdb pytest

file modified
+8 -4 
@@ -4,7 +4,7 @@ 
 

      build: .
 

      user: ${DEV_USER_ID:-1000}
 

      working_dir: /code
 

-     env_file: docker/dev.env
 

+     env_file: ["docker/dev.env"]
 

      command: ["bash", "-c", "./docker/dev-command.sh"]
 

      volumes:
 

        - ./:/code:ro,z
 
@@ -22,7 +22,7 @@ 
 

    resultsdb-db:
 

      image: postgres:9.5.2
 

      restart: always
 

-     env_file: docker/resultsdb-db.env
 

+     env_file: ["docker/resultsdb-db.env"]
 

      healthcheck:
 

        test: ["CMD-SHELL", "pg_isready -U postgres"]
 

        interval: 5s
 
@@ -33,8 +33,11 @@ 
 

      build:
 

        context: ../resultsdb
 

        dockerfile: openshift/Dockerfile
 

+     working_dir: /code
 

      command: ["bash", "-c", "/start.sh"]
 

      volumes:
 

+       - ../resultsdb:/code:ro,Z
 

+       - ./docker/home:/home/dev:Z
 

        - ./docker/resultsdb-command.sh:/start.sh:ro,z
 

        - ./docker/resultsdb-settings.py:/etc/resultsdb/settings.py:ro,z
 

        - ./docker/resultsdb.conf:/etc/httpd/conf.d/resultsdb.conf:ro,z
 
@@ -47,17 +50,18 @@ 
 

    waiverdb-db:
 

      image: postgres:9.5.2
 

      restart: always
 

-     env_file: docker/waiverdb-db.env
 

+     env_file: ["docker/waiverdb-db.env"]
 

  

    waiverdb:
 

      build:
 

        context: ../waiverdb
 

        dockerfile: openshift/containers/waiverdb/Dockerfile
 

      working_dir: /code
 

-     env_file: docker/waiverdb.env
 

+     env_file: ["docker/waiverdb.env"]
 

      command: ["bash", "-c", "/start.sh"]
 

      volumes:
 

        - ../waiverdb:/code:ro,Z
 

+       - ./docker/home:/home/dev:Z
 

        - ./docker/waiverdb-command.sh:/start.sh:ro,z
 

        - ./docker/waiverdb-settings.py:/etc/waiverdb/settings.py:ro,z
 

        - ./docker/client_secrets.json:/etc/secret/client_secrets.json:ro,z

file modified
+2 -1 
@@ -4,7 +4,8 @@ 
 

  # Check if UID is same as owner of the mounted home directory
 

  # (avoids creating files with wrong owner).
 

  real_dev_id="$(stat --format %u ~)"
 

- if [[ $real_dev_id != "$UID" ]]; then
 

+ # The owner of mounted directories is root (uid=0) when using podman.
 

+ if [[ ($real_dev_id == 0 && $UID == 0) || ($real_dev_id != 0 && $real_dev_id != "$UID") ]]; then
 

      echo "Set correct DEV_USER_ID in .env file (should be same as owner of docker/home)."
 

      exit 1
 

  fi

file modified
+1 
@@ -1,4 +1,5 @@ 
 

  POSTGRES_USER=waiverdb
 

  POSTGRES_PASSWORD=waiverdb
 

  POSTGRES_DB=waiverdb
 

+ PGPORT=5433
 

  POSTGRES_INITDB_ARGS=--auth=ident --auth=trust

file modified
+7 -1 
@@ -1,4 +1,10 @@ 
 

- DATABASE_URI = 'postgresql+psycopg2://waiverdb:waiverdb@waiverdb-db/waiverdb'
 

+ import os
 

+ 

+ DATABASE_URI = 'postgresql+psycopg2://waiverdb:waiverdb@waiverdb-db:5433/waiverdb'
 

+ 

+ if os.getenv('TEST') == 'true':
 

+     DATABASE_URI += '_test'
 

+ 

  HOST = '0.0.0.0'
 

  PORT = 5004
 

  #AUTH_METHOD = 'OIDC'

file modified
+1 
@@ -1,4 +1,5 @@ 
 

  DATABASE_PASSWORD=waiverdb
 

  SECRET_KEY=waiverdb
 

  WAIVERDB_CONFIG=/etc/waiverdb/settings.py
 

+ WAIVERDB_TEST_URL=http://waiverdb:5004/
 

  PYTEST_ADDOPTS=-o cache_dir=/tmp/.pytest_cache

Signed-off-by: Lukas Holecek hluk@email.cz

Not sure if this is a good idea since podman-compose is still in early phase of development and it's not that well tested and less user-friendly than docker-compose.

@gnaponie, @vmaljulin Can you review and check that it works for you? E.g. make up && make test

@lholecek, any chance we can make the use of podman-compose vs docker-compose configurable?

@lholecek, any chance we can make the use of podman-compose vs docker-compose configurable?

It is configurable:

make COMPOSE=docker-compose PODMAN=docker ...

rebased onto ba5b84041209246e6fb77cd638ae5a855caf7041
4 years ago

rebased onto 7c81be0932715f65b9d2a08c835637f971f6b35a
4 years ago

rebased onto 19ad60c11f7af63c2f91fb5852d7b8104471ec57
4 years ago

I think we can "switch" to podman. But maybe for a while we might keep the option of docker-compose too, and explain in the doc on how to "switch" back to docker-compose if the user wants to.
Can we add this?

There are still some stability issues with podman-compose.

One notable issue is that it's not possible to publish ports - it gives me following error:

ERRO[0000] error starting some container dependencies   
ERRO[0000] "error from slirp4netns while setting up port redirection: map[desc:bad request: add_hostfwd: slirp_add_hostfwd failed]"

I assume you need to be still root for this.

Workaround (depending on what you need to do) is to access API in the container:

# list policies
make exec CMD="curl localhost:8080/api/v1.0/policies"
# waiverdb version
make exec CMD="curl localhost:5004/api/v1.0/about/"

Maybe can we document this? We don't want people to waste time solving something you already figured out.

rebased onto 6eb38456c30ebcc2c1038e7a0e1b352beb36fa2b
4 years ago

rebased onto e3dbba7ef0587ae459d9d9c0496e54fdee26a649
4 years ago

rebased onto 95fa993d008523e34585a97dbd4540befce54859
4 years ago

After today's system update, publishing ports seems to work (no root necessary). :ok_hand:

rebased onto 6e437907f9a3f03788c4e888fe852013b5662293
4 years ago

rebased onto c2b76b977c60d2b21522b948461be6c7ab209b58
4 years ago

rebased onto b765ede13155cdcf2b6ecdcb4578a0d14c14753a
4 years ago

@gnaponie @lucarval Can you review again? I've also added another commit to enable testing waiverdb and resultsdb.

Maybe we should add to the installation guide in the documentation that the user should install podman, because only docker-compose is listed there.

I've pulled your code and tried.
I execute "make up -d"
and it tells me that docker isn't running. Isn't it supposed to start with podman? Yes, I have podman installed.

Maybe we should add to the installation guide in the documentation that the user should install podman, because only docker-compose is listed there.

Well, podman is not required, I would rather avoid adding more documentation if not strictly needed.

BTW, podman-compose is not yet in fedora package repos but there was a request to get it there and it might be already in rawhide. You can install it using pip3 install podman-compose but I don't think it's worth mentioning in developer docs.

I've pulled your code and tried.
I execute "make up -d"
and it tells me that docker isn't running. Isn't it supposed to start with podman? Yes, I have podman installed.

Mentioned in the docs:

Quick Tip: Use make to run frequent commands mentioned below. It uses podman-compose (if installed) instead of docker-compose.

2 new commits added

  • Allow running tests for resultsdb and waiverdb with docker-compose
  • Use podman-compose by default if available
4 years ago

I tried "make build" and I get this error...
any idea?

Error: error building at STEP "COPY . .": error copying "/home/gnaponie/proj/greenwave" to "/home/gnaponie/.local/share/containers/storage/overlay/1dfbb086f1589f509c9cc5ffc8d99c6d6b3ba9d4e3f3237d1b6af324145a93ea/merged/src": Error processing tar file(exit status 1): unexpected EOF
125
Edited 4 years ago by gnaponie

Not sure what exactly is the problem, but we should update Dockerfile for greenwave so it doesn't copy everything to the container (same thing can happen with waiverdb and resultsdb). It's not ideal, but we can use .dockerignore.

❯ podman run -it --rm localhost/greenwave_dev ls -la
total 444    
drwxr-xr-x.  2 root root  4096 Nov 14 11:17 .
drwxr-xr-x.  2 root root  4096 Nov 14 11:22 ..
-rw-rw-r--.  1 root root 31408 Nov 14 10:17 .coverage
-rw-rw-r--.  1 root root   215 Oct 15 05:30 .coveragerc
-rw-rw-r--.  1 root root    76 Mar 18  2019 .dockerignore
drwxrwxr-x. 32 root root  4096 Mar 15  2019 .eggs
-rw-r--r--.  1 root root    17 Apr 10  2019 .env
drwxrwxr-x.  9 root root  4096 Nov 14 10:19 .git
-rw-rw-r--.  1 root root   368 Nov  7 07:37 .gitignore    
drwxrwxr-x.  3 root root  4096 Jul 11 16:35 .pytest_cache
drwxrwxr-x.  8 root root  4096 Nov 14 10:17 .tox
drwxrwxr-x.  4 root root  4096 Aug 30 12:51 .vagrant
drwxrwxr-x.  5 root root  4096 Apr  9  2019 .venv
drwxrwxr-x.  2 root root  4096 Sep  4  2018 .vscode
...
drwxrwxr-x.  2 root root  4096 Nov 14 09:55 __pycache__
...
drwxrwxr-x.  5 root root  4096 Nov 14 10:19 docker
...

Oh .dockerignore is already there. Looks like podman doesn't support it.

Hmm, it doesn't work for me anymore -- some system update might have broke this. It comes down to this error:

❯ podman pod create --name=greenwave --share net -p 5004:5004 -p 5001:5001 -p 8080:8080             
Error: unable to create pod: unable to create pod cgroup for pod 90208560d9caf9408aa8bcdb6a4d8345549710ef7112eeb224f5437976df2c73: error creating cgroup user.slice/user-libpod_pod_90208560d9caf9408aa8bcdb6a4d8345549710ef7112eeb224f5
437976df2c73.slice: The name org.freedesktop.systemd1 was not provided by any .service files

Can anyone review? I have currently some issues with podman - I'll try to find out what went wrong later (I've recently uninstalled gdm and it might have removed/disabled some other important part).

Sorry Lukas, but this doesn't work for me... I tried "make up" and that's the end of the output:
http://pastebin.test.redhat.com/815050

Sorry Lukas, but this doesn't work for me... I tried "make up" and that's the end of the output:
http://pastebin.test.redhat.com/815050

Looks like it wants you to run make build explicitly.

Hey Lukas, I've tried again and it doesn't work.
I tried with make build and also make recreate. Using make down first.
But I get a lot of errors. Let me know if you need more details.

Were you able to run it correctly?

podman-compose down probably only gives you error messages because the pod and containers were not running. The exit code should be still 0 though.

@lholecek and I checked it together. It doesn't work for me, it might be because I still run on fedora 30 and I get some OCI runtime error.
We decided we're going to merge this PR, besides the documentation. So that people that want to use podman-compose, if it works for them, they can use it, otherwise docker-compose also works fine.

rebased onto f09637daef17287b3147bfe4c3c4dcd5501e6241
4 years ago

I've removed the podman-compose mention from the docs.

rebased onto f5e2530
4 years ago

As discussed, feel free to merge it if it works for you, as long as it doesn't break docker-compose.

Pull-Request has been merged by lholecek
4 years ago
Metadata
None
No Tags
Changes Summary 7
+1 -1
file changed
Dockerfile
+55 -13
file changed
Makefile
+8 -4
file changed
docker-compose.yml
+2 -1
file changed
docker/dev-command.sh
+1 -0
file changed
docker/waiverdb-db.env
+7 -1
file changed
docker/waiverdb-settings.py
+1 -0
file changed
docker/waiverdb.env
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.