From f3ba806a48fc245e9f3b51c64d3eb0c017e3d9f4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 29 2020 14:30:38 +0000 Subject: Rework the vagrant box to be closer to a production environment Signed-off-by: Pierre-Yves Chibon --- diff --git a/dev/Vagrantfile.example b/dev/Vagrantfile.example index f132f75..383f980 100644 --- a/dev/Vagrantfile.example +++ b/dev/Vagrantfile.example @@ -26,7 +26,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # If you would prefer to use NFS to share the directory uncomment this and configure NFS # config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_version: 4, nfs_udp: false config.vm.synced_folder ".", "/vagrant", disabled: true - config.vm.synced_folder ".", "/home/vagrant/devel", + config.vm.synced_folder ".", "/srv/pagure", type: "sshfs" # To cache update packages (which is helpful if frequently doing `vagrant destroy && vagrant up`) @@ -38,7 +38,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # sshfs_opts_append: "-o nonempty" # Comment this line if you would like to disable the automatic update during provisioning - config.vm.provision "shell", inline: "sudo dnf upgrade -y" + config.vm.provision "shell", inline: "sudo dnf upgrade -y || true" # bootstrap and run with ansible config.vm.provision "ansible" do |ansible| diff --git a/dev/ansible/roles/pagure-dev/files/bashrc b/dev/ansible/roles/pagure-dev/files/bashrc index 8e987c0..8eede33 100644 --- a/dev/ansible/roles/pagure-dev/files/bashrc +++ b/dev/ansible/roles/pagure-dev/files/bashrc @@ -11,9 +11,6 @@ echo "Reminder of the commands: * prestart: to restart all the services * plog: to access the logs of the 'pagure' service (the web server)" -# To avoid confusion, start in the `devel` directory -cd devel - # Uncomment the following line if you don't like systemctl's auto-paging feature: # export SYSTEMD_PAGER= @@ -24,25 +21,25 @@ cd devel export PAGURE_CONFIG=~/pagure.cfg pstart (){ - systemctl --user start pagure.service pagure-docs.service pagure_ci.service\ + sudo systemctl start pagure.service pagure-docs.service pagure_ci.service\ pagure_ev.service pagure_webhook.service pagure_worker.service echo 'The application is running on http://localhost:5000/' } pstop (){ - systemctl --user stop pagure.service pagure-docs.service pagure_ci.service\ + sudo systemctl stop pagure.service pagure-docs.service pagure_ci.service\ pagure_ev.service pagure_webhook.service pagure_worker.service } prestart (){ - systemctl --user restart pagure.service pagure-docs.service pagure_ci.service\ + sudo systemctl restart pagure.service pagure-docs.service pagure_ci.service\ pagure_ev.service pagure_webhook.service pagure_worker.service echo 'The application is running on http://localhost:5000/' } pstatus (){ - systemctl --user status pagure.service pagure-docs.service pagure_ci.service\ + sudo systemctl status pagure.service pagure-docs.service pagure_ci.service\ pagure_ev.service pagure_webhook.service pagure_worker.service } -alias plog="journalctl --user -lu pagure.service" +alias plog="sudo journalctl -lu pagure.service" diff --git a/dev/ansible/roles/pagure-dev/files/pagure-docs.service b/dev/ansible/roles/pagure-dev/files/pagure-docs.service index 76b9754..20d4004 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure-docs.service +++ b/dev/ansible/roles/pagure-dev/files/pagure-docs.service @@ -3,10 +3,11 @@ Description=Runs the Pagure documentation server After=network.target [Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=/home/vagrant/.virtualenvs/python3-pagure/bin/python \ - /home/vagrant/devel/rundocserver.py --host 0.0.0.0 +Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg" +ExecStart=python3 /srv/pagure/rundocserver.py --host 0.0.0.0 Type=simple +User=git +Group=git [Install] -WantedBy=default.target +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/files/pagure.cfg b/dev/ansible/roles/pagure-dev/files/pagure.cfg index b80af6f..00cca58 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure.cfg +++ b/dev/ansible/roles/pagure-dev/files/pagure.cfg @@ -21,7 +21,7 @@ SECRET_KEY='' ### url to the database server: #DB_URL=mysql://user:pass@host/db_name #DB_URL=postgres://user:pass@host/db_name -DB_URL = 'sqlite:////home/vagrant/pagure_data/pagure_dev.sqlite' +DB_URL = 'sqlite:////srv/git/pagure_dev.sqlite' ### The FAS group in which the admin of pagure are ADMIN_GROUP = ['sysadmin-main'] @@ -30,7 +30,7 @@ ADMIN_GROUP = ['sysadmin-main'] PAGURE_ADMIN_USERS = [] ### The URL at which the project is available. -APP_URL = 'http://localhost.localdomain/' +APP_URL = 'http://127.0.0.1:5000/' ### The URL at which the documentation of projects will be available ## This should be in a different domain to avoid XSS issues since we want ## to allow raw html to be displayed (different domain, ie not a sub-domain). @@ -41,13 +41,13 @@ EMAIL_SEND = False EMAIL_ERROR = 'vagrant@localhost' ### The URL to use to clone git repositories. -GIT_URL_SSH = 'ssh://vagrant@pagure-dev.example.com/' +GIT_URL_SSH = 'ssh://git@pagure-dev.example.com/' GIT_URL_GIT = 'http://pagure-dev.example.com:5000/' ### Folder containing to the git repos -STORAGE_ROOT = '/home/vagrant/pagure_data/' +STORAGE_ROOT = '/srv/git/' -GIT_FOLDER = os.path.join(STORAGE_ROOT, 'repos') +GIT_FOLDER = os.path.join(STORAGE_ROOT, 'repositories') ### Folder containing the clones for the remote pull-requests REMOTE_GIT_FOLDER = os.path.join(STORAGE_ROOT, 'remotes') @@ -55,25 +55,6 @@ REMOTE_GIT_FOLDER = os.path.join(STORAGE_ROOT, 'remotes') ### Whether to enable scanning for viruses in attachments VIRUS_SCAN_ATTACHMENTS = False -### Home folder of the gitolite user -### Folder where to run gl-compile-conf from -GITOLITE_HOME = '/home/vagrant/' - -### Configuration file for gitolite -GITOLITE_CONFIG = os.path.join(GITOLITE_HOME, '.gitolite/conf/gitolite.conf') - -### Version of gitolite used: 2 or 3? -GITOLITE_VERSION = 3 - -### Folder containing all the public ssh keys for gitolite -GITOLITE_KEYDIR = os.path.join(GITOLITE_HOME, '.gitolite/keydir/') - -### Path to the gitolite.rc file -GL_RC = '/home/vagrant/.gitolite.rc' - -### Path to the /bin directory where the gitolite tools can be found -GL_BINDIR = '/usr/bin/' - # SSH Information @@ -85,9 +66,6 @@ GL_BINDIR = '/usr/bin/' # Optional configuration -### Number of items displayed per page -# Used when listing items -ITEM_PER_PAGE = 50 ### Maximum size of the uploaded content # Used to limit the size of file attached to a ticket for example @@ -96,12 +74,6 @@ MAX_CONTENT_LENGTH = 4 * 1024 * 1024 # 4 megabytes ### Lenght for short commits ids or file hex SHORT_LENGTH = 6 -### List of blacklisted project names that can conflicts for pagure's URLs -### or other -BLACKLISTED_PROJECTS = [ - 'static', 'pv', 'releases', 'new', 'api', 'settings', - 'logout', 'login', 'users', 'groups', 'projects', 'about'] - ### IP addresses allowed to access the internal endpoints ### These endpoints are used by the milter and are security sensitive, thus ### the IP filter diff --git a/dev/ansible/roles/pagure-dev/files/pagure.service b/dev/ansible/roles/pagure-dev/files/pagure.service index aedda80..a46d99c 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure.service +++ b/dev/ansible/roles/pagure-dev/files/pagure.service @@ -3,9 +3,11 @@ Description=The Pagure web service After=network.target [Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=python3 %h/devel/runserver.py --host 0.0.0.0 --debug +Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg" +ExecStart=python3 /srv/pagure/runserver.py --host 0.0.0.0 --debug Type=simple +User=git +Group=git [Install] -WantedBy=default.target +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/files/pagure_ci.service b/dev/ansible/roles/pagure-dev/files/pagure_ci.service index ad82794..24112c5 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure_ci.service +++ b/dev/ansible/roles/pagure-dev/files/pagure_ci.service @@ -4,9 +4,11 @@ After=redis.target Documentation=https://pagure.io/pagure [Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=/home/vagrant/.virtualenvs/python3-pagure/bin/celery worker -A pagure.lib.tasks --loglevel=info -Q pagure_ci +Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg" +ExecStart=celery worker -A pagure.lib.tasks --loglevel=info -Q pagure_ci Type=simple +User=git +Group=git [Install] -WantedBy=default.target +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/files/pagure_ev.service b/dev/ansible/roles/pagure-dev/files/pagure_ev.service index 914c4d3..fa0dda6 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure_ev.service +++ b/dev/ansible/roles/pagure-dev/files/pagure_ev.service @@ -4,10 +4,11 @@ After=redis.target Documentation=https://pagure.io/pagure [Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=/home/vagrant/.virtualenvs/python3-pagure/bin/python \ - /home/vagrant/devel/pagure-ev/pagure_stream_server.py +Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg" +ExecStart=python3 /srv/pagure/pagure-ev/pagure_stream_server.py Type=simple +User=git +Group=git [Install] -WantedBy=default.target +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/files/pagure_webhook.service b/dev/ansible/roles/pagure-dev/files/pagure_webhook.service index f9a7c1c..997e515 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure_webhook.service +++ b/dev/ansible/roles/pagure-dev/files/pagure_webhook.service @@ -4,9 +4,11 @@ After=redis.target Documentation=https://pagure.io/pagure [Service] -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" -ExecStart=/home/vagrant/.virtualenvs/python3-pagure/bin/celery worker -A pagure.lib.tasks --loglevel=info -Q pagure_webhook +Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg" +ExecStart=celery worker -A pagure.lib.tasks --loglevel=info -Q pagure_webhook Type=simple +User=git +Group=git [Install] -WantedBy=default.target +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/files/pagure_worker.service b/dev/ansible/roles/pagure-dev/files/pagure_worker.service index ae1bdb8..d0bed44 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure_worker.service +++ b/dev/ansible/roles/pagure-dev/files/pagure_worker.service @@ -4,10 +4,12 @@ After=redis.target Documentation=https://pagure.io/pagure [Service] -ExecStart=/home/vagrant/.virtualenvs/python3-pagure/bin/celery worker -A pagure.lib.tasks --loglevel=info -Environment="PAGURE_CONFIG=/home/vagrant/pagure.cfg" +Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg" +ExecStart=celery worker -A pagure.lib.tasks --loglevel=info Type=simple +User=git +Group=git Restart=on-failure [Install] -WantedBy=default.target +WantedBy=multi-user.target diff --git a/dev/ansible/roles/pagure-dev/tasks/main.yml b/dev/ansible/roles/pagure-dev/tasks/main.yml index 7198069..91bbf2c 100644 --- a/dev/ansible/roles/pagure-dev/tasks/main.yml +++ b/dev/ansible/roles/pagure-dev/tasks/main.yml @@ -2,7 +2,7 @@ - include: clamav.yml - include: eventsource.yml -- include: gitolite.yml +#- include: gitolite.yml - include: milter.yml - include: postgres.yml @@ -16,6 +16,8 @@ - tmux - tree - vim-enhanced + - python3-pip + - httpd state: present - name: Install Pagure development packages @@ -30,6 +32,7 @@ - python3-chardet - python3-cryptography - python3-docutils + - python3-email-validator - python3-eventlet - python3-fedora-flask - python3-flask @@ -55,6 +58,11 @@ - redhat-rpm-config state: latest +- name: Install pagure for dev + command: pip3 install -e . + args: + chdir: "/srv/pagure/" + # Add various helpful configuration files - name: Install a custom bashrc become_user: "{{ ansible_env.SUDO_USER }}" @@ -75,66 +83,107 @@ become_user: "{{ ansible_env.SUDO_USER }}" copy: src=bash_history dest=/home/{{ ansible_env.SUDO_USER }}/.bash_history -- name: Install the pagure configuration - become_user: "{{ ansible_env.SUDO_USER }}" - copy: src=pagure.cfg dest=/home/{{ ansible_env.SUDO_USER }}/pagure.cfg -- name: Creates pagure data directories - become_user: "{{ ansible_env.SUDO_USER }}" - file: path=/home/{{ ansible_env.SUDO_USER }}/pagure_data/{{ item }} state=directory +# Configure pagure + +- name: Create the git user and group + command: useradd --create-home --home-dir=/srv/git/ git + creates=/srv/git/ + +- name: create the /attachments folder + file: state=directory + path=/srv/attachments + owner=git group=git mode=0775 + +- name: Adjust owner of /srv/git + file: name=/srv/git state=directory recurse=yes owner=git group=git + +- name: create all the directories used by pagure + file: state=directory + path={{ item }} + owner=git group=git mode=0775 with_items: - - forks - - docs - - tickets - - requests - - remotes - - repos + - /srv/git/repositories/ + - /srv/git/repositories/forks + - /srv/git/repositories/docs + - /srv/git/repositories/tickets + - /srv/git/repositories/requests + - /srv/git/remotes + - /var/www/releases + - /var/www/archives + - /srv/tmp + +- name: create the /etc/pagure folder for the config + file: state=directory + path=/etc/pagure + owner=git group=git mode=0775 + + +# Set things up for the mirroring feature + +- name: create the `paguremirroring` group + group: + name: paguremirroring + state: present + +- name: create the `paguremirroring` user + user: + name: paguremirroring + group: paguremirroring + groups: paguremirroring,git + shell: /bin/nologin + home: /srv/mirror + + +# Configure the web app + +- name: Install the pagure configuration + copy: + src: pagure.cfg + dest: /etc/pagure/pagure.cfg + owner: git + group: git + mode: 0644 - name: Add a working copy of alembic.ini - become_user: "{{ ansible_env.SUDO_USER }}" copy: - src: /home/{{ ansible_env.SUDO_USER }}/devel/files/alembic.ini - dest: /home/{{ ansible_env.SUDO_USER }}/alembic.ini + src: /srv/pagure/files/alembic.ini + dest: /etc/pagure/alembic.ini + owner: git + group: git + mode: 0644 remote_src: True - name: Configure alembic to use our development database - become_user: "{{ ansible_env.SUDO_USER }}" replace: - dest: /home/{{ ansible_env.SUDO_USER }}/alembic.ini + dest: /etc/pagure/alembic.ini regexp: "sqlalchemy.url = sqlite:////var/tmp/pagure_dev.sqlite" - replace: "sqlalchemy.url = sqlite:////home/{{ ansible_env.SUDO_USER }}/pagure_data/pagure_dev.sqlite" + replace: "sqlalchemy.url = sqlite:////srv/git/pagure_dev.sqlite" - name: Configure alembic to point to the pagure migration folder - become_user: "{{ ansible_env.SUDO_USER }}" replace: - dest: /home/{{ ansible_env.SUDO_USER }}/alembic.ini + dest: /etc/pagure/alembic.ini regexp: "script_location = /usr/share/pagure/alembic" - replace: "script_location = /home/vagrant/devel/alembic/" + replace: "script_location = /srv/pagure/alembic/" - name: Create the Pagure database - become_user: "{{ ansible_env.SUDO_USER }}" - command: python3 devel/createdb.py + become_user: git + command: python3 /srv/pagure/createdb.py + environment: + PAGURE_CONFIG: /etc/pagure/pagure.cfg args: - creates: /home/{{ ansible_env.SUDO_USER }}/pagure_data/pagure_dev.sqlite - chdir: "/home/{{ ansible_env.SUDO_USER }}/" + creates: /srv/git/pagure_dev.sqlite - name: Stamp the database with its current migration - become_user: "{{ ansible_env.SUDO_USER }}" - shell: PYTHONPATH=/home/{{ ansible_env.SUDO_USER }}/devel/ alembic-3 stamp $(PYTHONPATH=/home/{{ ansible_env.SUDO_USER }}/devel/ alembic-3 heads | awk '{ print $1 }') + become_user: git + shell: alembic-3 stamp $(alembic-3 heads | awk '{ print $1 }') args: - chdir: "/home/{{ ansible_env.SUDO_USER }}/" - -- name: Create systemd user unit directory - become_user: "{{ ansible_env.SUDO_USER }}" - file: - path: /home/{{ ansible_env.SUDO_USER }}/.config/systemd/user/ - state: directory + chdir: "/etc/pagure" - name: Install the Pagure service files for systemd - become_user: "{{ ansible_env.SUDO_USER }}" copy: src: "{{ item }}" - dest: /home/{{ ansible_env.SUDO_USER }}/.config/systemd/user/{{ item }} + dest: /etc/systemd/system/{{ item }} with_items: - pagure.service - pagure-docs.service @@ -143,15 +192,39 @@ - pagure_webhook.service - pagure_worker.service -- name: Enable and start the Pagure systemd services - become: False +- name: let paguremirroring read the pagure config + command: /usr/bin/setfacl -m user:paguremirroring:rx /etc/pagure/pagure.cfg + +- name: Add default facl so apache can read git repos + acl: default=yes etype=user entity=apache permissions="rx" name=/srv/git state=present + register: acl_updates + +- name: Manually fix current default ACLs since Ansible doesnt know recursive acls + when: acl_updates.changed + command: /usr/bin/setfacl -Rdm user:apache:rx /srv/git + +- name: Manually fix current ACLs since Ansible doesnt know recursive acls + when: acl_updates.changed + command: /usr/bin/setfacl -Rm user:apache:rx /srv/git + +- name: Turn off SELinux, this dev box is doing too many unwdily things + command: setenforce 0 + +- name: Turn off SELinux accross reboot + replace: + dest: /etc/selinux/config + regexp: "SELINUX=enforcing" + replace: "SELINUX=permissive" + +- name: Enable and start the all services needed systemd: daemon_reload: yes name: "{{ item }}" enabled: True state: started - scope: user with_items: + - httpd + - redis - pagure - pagure-docs - pagure_ci