From bbbd54cc8aa2f565ba63f6528ca7a0a7e1b11328 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Apr 09 2019 22:29:59 +0000 Subject: Add koschei --- diff --git a/host_vars/koschei.kos.kjnet.xyz b/host_vars/koschei.kos.kjnet.xyz new file mode 100644 index 0000000..f6443fe --- /dev/null +++ b/host_vars/koschei.kos.kjnet.xyz @@ -0,0 +1,15 @@ +# -*- yaml -*- + +kvm_host: dl4.kos.kjnet.xyz +volgroup: /dev/sysvol +lvm_size: 20000 +mem_size: 4096 +num_cpus: 4 + +koschei_pgsql_hostname: 172.22.13.2 + +koschei_backend_services: + - koschei-polling + - koschei-build-resolver + - koschei-repo-resolver + - koschei-scheduler diff --git a/inventory b/inventory index 2700824..bd47a09 100644 --- a/inventory +++ b/inventory @@ -1,5 +1,6 @@ [kos] koji.kos.kjnet.xyz +koschei.kos.kjnet.xyz [virthost] #dl1.kos.kjnet.xyz diff --git a/koschei.yml b/koschei.yml new file mode 100644 index 0000000..e569998 --- /dev/null +++ b/koschei.yml @@ -0,0 +1,21 @@ +- name: Spin up instance on AWS + hosts: koschei.kos.kjnet.xyz + gather_facts: false + roles: + - kvm + +- name: Minimal instance configuration required for Ansible to work + hosts: koschei.kos.kjnet.xyz + gather_facts: false + become: true + roles: + - ansible + +- name: Configure instance + hosts: koschei.kos.kjnet.xyz + gather_facts: true + vars_files: + - private/vars.yml + roles: + - koschei/frontend + - koschei/backend diff --git a/koschei/backend/files/systemd-memory-limit.conf b/koschei/backend/files/systemd-memory-limit.conf new file mode 100644 index 0000000..7c1c5fb --- /dev/null +++ b/koschei/backend/files/systemd-memory-limit.conf @@ -0,0 +1,2 @@ +[Service] +MemoryLimit=2G diff --git a/koschei/backend/handlers/main.yml b/koschei/backend/handlers/main.yml new file mode 100644 index 0000000..86c038a --- /dev/null +++ b/koschei/backend/handlers/main.yml @@ -0,0 +1,21 @@ +- name: reload systemd + command: systemctl daemon-reload + +- name: restart koschei-polling + action: service name=koschei-polling state=restarted + +- name: restart koschei-build-resolver + action: service name=koschei-build-resolver state=restarted + +- name: restart koschei-repo-resolver + action: service name=koschei-repo-resolver state=restarted + +- name: restart koschei-scheduler + action: service name=koschei-scheduler state=restarted + +- name: restart koschei-watcher + action: service name=koschei-watcher state=restarted + +- name: restart koschei backend services + service: name={{ item }} state=restarted + with_items: "{{ koschei_backend_services }}" diff --git a/koschei/backend/tasks/main.yml b/koschei/backend/tasks/main.yml new file mode 100644 index 0000000..2d9e748 --- /dev/null +++ b/koschei/backend/tasks/main.yml @@ -0,0 +1,72 @@ +- name: install packages + package: name={{ item }} state=present + with_items: + - koschei-admin + - koschei-backend + - postgresql + tags: + - koschei + - packages + +- name: create systemd config directories + file: > + path="/etc/systemd/system/{{ item }}.service.d" + state=directory + with_items: "{{ koschei_backend_services }}" + notify: + - reload systemd + tags: + - koschei + - config + - service + +- name: install systemd config files (memory) + copy: > + src=systemd-memory-limit.conf + dest=/etc/systemd/system/{{ item }}.service.d/memory-limit.conf + with_items: + - koschei-build-resolver + - koschei-repo-resolver + notify: + - reload systemd + - restart koschei-build-resolver + - restart koschei-repo-resolver + tags: + - koschei + - config + - service + +- name: install systemd config files (restart) + template: > + src=systemd-restart-limit.conf.j2 + dest=/etc/systemd/system/{{ item }}.service.d/restart.conf + with_items: "{{ koschei_backend_services }}" + notify: + - reload systemd + tags: + - koschei + - config + - service + +- name: install config files + template: > + src="{{ item }}.j2" + dest="/etc/koschei/{{ item }}" + owner=koschei + group=koschei + mode=0400 + with_items: + - config-admin.cfg + - config-backend.cfg + notify: + - restart koschei backend services + tags: + - koschei + - config + +- name: enable services to start on boot + service: name={{ item }} state=started enabled=true + with_items: "{{ koschei_backend_services }}" + tags: + - koschei + - service diff --git a/koschei/backend/templates/config-admin.cfg.j2 b/koschei/backend/templates/config-admin.cfg.j2 new file mode 100644 index 0000000..f864b08 --- /dev/null +++ b/koschei/backend/templates/config-admin.cfg.j2 @@ -0,0 +1,15 @@ +# Config for operations that require koschei credentials. +config = { + "database_config": { + "host": "{{ koschei_pgsql_hostname }}", + "database": "koschei", + "username": "koschei", + "password": "{{ koschei_db_password }}", + }, + "unpriv_db_username": "koschei", +} + +# Local Variables: +# mode: Python +# End: +# vi: ft=python diff --git a/koschei/backend/templates/config-backend.cfg.j2 b/koschei/backend/templates/config-backend.cfg.j2 new file mode 100644 index 0000000..ad8dd85 --- /dev/null +++ b/koschei/backend/templates/config-backend.cfg.j2 @@ -0,0 +1,71 @@ +# This is a config file for Koschei that can override values in default +# configuration in /usr/share/koschei/config.cfg. It is a python file expecting +# assignment to config dictionary which will be recursively merged with the +# default one. + +config = { + "database_config": { + "host": "{{ koschei_pgsql_hostname }}", + "username": "koschei", + "password": "{{ koschei_db_password }}", + "database": "koschei" + }, + "koji_config": { + "server": "http://172.22.13.40/kojihub", + "weburl": "http://172.22.13.40/koji", + "topurl": "http://172.22.13.40/kojifiles", + "login_method": "login", + "session_opts": { + "user": "koschei", + "password": "{{ koschei_koji_password }}", + }, + "max_builds": 16, + "build_arches": ['x86_64'], + "load_threshold": 1, + "task_priority": 30, + }, + "dependency": { + "build_group": "build", + "for_arch": "x86_64", + "repo_cache_items": 10, + "arches": ['x86_64', 'i386'], + "persist_chunk_size": 300, + "resolver_queue_size": 100, + "dependency_fetch_chunk_size": 150, + "dependency_cache_capacity": 50000, + "cache_l2_capacity": 100, + "ignore_weak_deps": True, + }, + "services": { + "polling": { + "interval": 1, # in seconds + }, + "build_resolver": { + "memory_limit": 1024**2, # kilobytes + }, + "repo_resolver": { + "memory_limit": 1024**2, # kilobytes + }, + "watcher": { + "memory_limit": 256*1024, # kilobytes + }, + }, + "priorities": { + "calculation_interval": 3600, # seconds + }, + "logging": { + "loggers": { + "": { + "handlers": ["stderr"], + }, + "fedmsg": { + "level": "ERROR", + }, + }, + }, +} + +# Local Variables: +# mode: Python +# End: +# vi: ft=python diff --git a/koschei/backend/templates/systemd-restart-limit.conf.j2 b/koschei/backend/templates/systemd-restart-limit.conf.j2 new file mode 100644 index 0000000..8611c98 --- /dev/null +++ b/koschei/backend/templates/systemd-restart-limit.conf.j2 @@ -0,0 +1,2 @@ +[Service] +RestartSec=10 diff --git a/koschei/frontend/handlers/main.yml b/koschei/frontend/handlers/main.yml new file mode 100644 index 0000000..a226fc8 --- /dev/null +++ b/koschei/frontend/handlers/main.yml @@ -0,0 +1,2 @@ +- name: reload httpd + action: service name=httpd state=reloaded diff --git a/koschei/frontend/tasks/main.yml b/koschei/frontend/tasks/main.yml new file mode 100644 index 0000000..5c7338e --- /dev/null +++ b/koschei/frontend/tasks/main.yml @@ -0,0 +1,50 @@ +- name: install packages + package: name={{ item }} state=present + with_items: + - koschei-frontend + tags: + - koschei + - packages + +- name: install config files + template: > + src="{{ item }}.j2" + dest="/etc/koschei/{{ item }}" + owner=koschei + group=koschei + mode=0400 + with_items: + - config-frontend.cfg + notify: + - reload httpd + tags: + - koschei + - config + +- name: install httpd config file + template: > + src="httpd.conf.j2" + dest="/etc/httpd/conf.d/koschei.conf" + notify: + - reload httpd + tags: + - koschei + - config + +- name: set httpd selinux booleans + seboolean: name={{ item }} state=yes persistent=yes + with_items: + - httpd_can_network_connect + - httpd_can_network_connect_db + - httpd_execmem + notify: + - reload httpd + tags: + - koschei + - selinux + +- name: enable httpd to start on boot + service: name=httpd state=started enabled=true + tags: + - koschei + - service diff --git a/koschei/frontend/templates/config-frontend.cfg.j2 b/koschei/frontend/templates/config-frontend.cfg.j2 new file mode 100644 index 0000000..2b9648a --- /dev/null +++ b/koschei/frontend/templates/config-frontend.cfg.j2 @@ -0,0 +1,53 @@ +# This is a config file for Koschei that can override values in default +# configuration in /usr/share/koschei/config.cfg. It is a python file expecting +# assignment to config dictionary which will be recursively merged with the +# default one. +config = { + "database_config": { + "host": "{{ koschei_pgsql_hostname }}", + "username": "koschei", + "password": "{{ koschei_db_password }}", + "database": "koschei" + }, + "koji_config": { + "topurl": "http://koji.kos.kjnet.xyz/kojifiles", + "weburl": "http://koji.kos.kjnet.xyz/koji", + }, + "flask": { + "SECRET_KEY": "{{ koschei_flask_secret_key }}", + }, + "logging": { + "loggers": { + "": { + "level": "INFO", + "handlers": ["stderr"], + }, + "requests": { + "level": "WARN", + }, + }, + }, + "frontend": { + "builds_per_page": 8, + }, + "links": [ + {"name": "Packages", + "url": "https://apps.fedoraproject.org/packages/{package.name}"}, + {"name": "Bodhi", + "url": "https://bodhi.fedoraproject.org/updates?packages={package.name}"}, + {"name": "Dist-git", + "url": "https://src.fedoraproject.org/rpms/{package.name}"}, + {"name": "Bugzilla", + "url": "https://bugzilla.redhat.com/buglist.cgi?product={package.collection.bugzilla_product}&component={package.name}"}, + {"name": "Koji", + "url": "https://koji.fedoraproject.org/koji/packageinfo?packageID={package.name}"}, + ], + "bugreport": { + "url": "https://bugzilla.redhat.com/enter_bug.cgi?{query}", + }, +} + +# Local Variables: +# mode: Python +# End: +# vi: ft=python diff --git a/koschei/frontend/templates/httpd.conf.j2 b/koschei/frontend/templates/httpd.conf.j2 new file mode 100644 index 0000000..1647097 --- /dev/null +++ b/koschei/frontend/templates/httpd.conf.j2 @@ -0,0 +1,20 @@ + + ServerName {{ inventory_hostname }} + + WSGIDaemonProcess koschei user=koschei group=koschei threads=5 home=/usr/share/koschei + + WSGIScriptAlias /koschei /usr/share/koschei/koschei.wsgi + Alias /koschei/static /usr/share/koschei/static/ + + + WSGIProcessGroup koschei + WSGIApplicationGroup %{GLOBAL} + Options All + AllowOverride All + Require all granted + + + + Require valid-user + + diff --git a/private-example/vars.yml b/private-example/vars.yml index e35145e..f963c95 100644 --- a/private-example/vars.yml +++ b/private-example/vars.yml @@ -2,3 +2,7 @@ db_password: xxx builder_password: xxx admin_password: xxx mizdebsk_password: xxx + +koschei_koji_password: xxx +koschei_db_password: xxx +koschei_flask_secret_key: xxx