#409 [stg] languages: initial deployment
Merged 3 years ago by smooge. Opened 3 years ago by darknao.
fedora-infra/ darknao/ansible raifort  into  main

@@ -782,3 +782,13 @@ 

      keephost: true

      tags: packager-dashboard

  

+   - role: httpd/reverseproxy

+     website: "languages{{ env_suffix }}.fedoraproject.org"

+     destname: os

+     balancer_name: os

+     targettype: openshift

+     balancer_members: "{{ openshift_masters }}"

+     keephost: true

+     when: env == "staging"

+     tags:

+     - languages

@@ -926,6 +926,14 @@ 

      when: env == "production"

  

    - role: httpd/website

+     site_name: languages.fedoraproject.org

+     server_aliases: [languages.stg.fedoraproject.org]

+     sslonly: true

+     cert_name: "{{wildcard_cert_name}}"

+     when: env == "staging"

+     tags: languages

+ 

+   - role: httpd/website

      site_name: fpdc.fedoraproject.org

      sslonly: true

      server_aliases: [fpdc.stg.fedoraproject.org]

@@ -0,0 +1,73 @@ 

+ - name: make the app be real

+   hosts: os_masters_stg[0]

+   user: root

+   gather_facts: False

+   

+   vars_files:

+     - /srv/web/infra/ansible/vars/global.yml

+     - "/srv/private/ansible/vars.yml"

+     - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml

+ 

+   roles:

+   - role: openshift/project

+     app: languages

+     description: Localization measurement and tooling

+     appowners:

+     - darknao

+     - jibecfed

+ 

+   - role: openshift/imagestream

+     app: languages

+     imagename: builder

+ 

+   - role: openshift/object

+     app: languages

+     objectname: buildconfig.yml

+     template: buildconfig.yml

+ 

+   - role: openshift/start-build

+     app: languages

+     buildname: build-latest

+ 

+   - role: openshift/start-build

+     app: languages

+     buildname: build-f33

+ 

+   - role: openshift/object

+     app: languages

+     template: service.yml

+     objectname: service.yml

+ 

+   - role: openshift/route

+     app: languages

+     routename: languages

+     host: "languages{{ env_suffix }}.fedoraproject.org"

+     serviceport: web

+     servicename: web

+ 

+   - role: openshift/object

+     app: languages

+     template: pvc.yml

+     objectname: pvc.yml

+ 

+   - role: openshift/object

+     app: languages

+     template: deploymentconfig.yml

+     objectname: deploymentconfig.yml

+ 

+   - role: openshift/object

+     app: languages

+     template: cron.yml

+     objectname: cron.yml

+ 

+ ###############################################

+ # actions to delete the project from OpenShift

+ ###############################################

+ # to run: sudo rbac-playbook -l os_masters_stg[0] -t delete openshift-apps/languages.yml

+   - role: openshift/object-delete

+     app: languages

+     objecttype: project

+     objectname: languages

+     tags:

+     - never

+     - delete

@@ -0,0 +1,1 @@ 

+ registry: "docker-registry.default.svc:5000"

@@ -0,0 +1,45 @@ 

+ apiVersion: v1

+ kind: BuildConfig

+ metadata:

+   labels:

+     build: build-f33

+   name: build-f33

+ spec:

+   successfulBuildsHistoryLimit: 2

+   failedBuildsHistoryLimit: 1

+   runPolicy: Serial

+   source:

+     git:

+       uri: "https://pagure.io/fedora-l10n/fedora-localization-statistics.git"

+       ref: main

+   strategy:

+     type: Docker

+     dockerStrategy:

+       dockerfilePath: docker/Dockerfile.33

+   output:

+     to:

+       kind: ImageStreamTag

+       name: builder:f33

+ ---

+ apiVersion: v1

+ kind: BuildConfig

+ metadata:

+   labels:

+     build: build-latest

+   name: build-latest

+ spec:

+   successfulBuildsHistoryLimit: 2

+   failedBuildsHistoryLimit: 1

+   runPolicy: Serial

+   source:

+     git:

+       uri: "https://pagure.io/fedora-l10n/fedora-localization-statistics.git"

+       ref: main

+   strategy:

+     type: Docker

+     dockerStrategy:

+       dockerfilePath: docker/Dockerfile.34

+   output:

+     to:

+       kind: ImageStreamTag

+       name: builder:latest

@@ -0,0 +1,229 @@ 

+ {% if env == "production" %}

+ apiVersion: batch/v1beta1

+ kind: CronJob

+ metadata:

+   name: extract-33

+ spec:

+   successfulJobsHistoryLimit: 3

+   failedJobsHistoryLimit: 2

+   concurrencyPolicy: Forbid

+   schedule: "0 4 * * *"

+   startingDeadlineSeconds: 500

+   jobTemplate:

+     spec:

+       backoffLimit: 0

+       template:

+         metadata:

+           labels:

+             parent: "cron_extract"

+         spec:

+           containers:

+           - name: extract

+             image: "{{ registry }}/{{ app }}/builder:f33"

+             imagePullPolicy: Always

+             command:

+             - ./build.py

+             - "--results"

+             - f33

+             volumeMounts:

+             - name: srpm-data

+               readOnly: false

+               mountPath: /src/results

+           volumes:

+           - name: srpm-data

+             persistentVolumeClaim:

+               claimName: "{{ app }}-storage"

+           restartPolicy: Never

+ ---

+ {% endif %}

+ apiVersion: batch/v1beta1

+ kind: CronJob

+ metadata:

+   name: extract-34

+ spec:

+   successfulJobsHistoryLimit: 3

+   failedJobsHistoryLimit: 2

+   concurrencyPolicy: Forbid

+   schedule: "0 4 * * *"

+   startingDeadlineSeconds: 500

+   jobTemplate:

+     spec:

+       backoffLimit: 0

+       template:

+         metadata:

+           labels:

+             parent: "cron_extract"

+         spec:

+           containers:

+           - name: extract

+             image: "{{ registry }}/{{ app }}/builder:latest"

+             imagePullPolicy: Always

+             command:

+             - ./build.py

+             - "--results"

+             - f34

+             volumeMounts:

+             - name: srpm-data

+               readOnly: false

+               mountPath: /src/results

+           volumes:

+           - name: srpm-data

+             persistentVolumeClaim:

+               claimName: "{{ app }}-storage"

+           restartPolicy: Never

+ ---

+ {% if env == "production" %}

+ apiVersion: batch/v1beta1

+ kind: CronJob

+ metadata:

+   name: stats-33

+ spec:

+   successfulJobsHistoryLimit: 3

+   failedJobsHistoryLimit: 2

+   concurrencyPolicy: Forbid

+   schedule: "0 20 * * *"

+   startingDeadlineSeconds: 500

+   jobTemplate:

+     spec:

+       backoffLimit: 0

+       template:

+         metadata:

+           labels:

+             parent: "cron_stats"

+         spec:

+           volumes:

+           - name: srpm-data

+             persistentVolumeClaim:

+               claimName: "{{ app }}-storage"

+           restartPolicy: Never

+           containers:

+           - name: stats

+             image: "{{ registry }}/{{ app }}/builder:latest"

+             imagePullPolicy: Always

+             command: ["/bin/sh", "-c"]

+             args:

+             - >

+               ./build_language_list.py --results f33 --refresh &&

+               ./build_language_list.py --results f33 --analyzealllang &&

+               ./build_tm.py --results f33 --compress &&

+               ./build_stats.py --results f33

+             volumeMounts:

+             - name: srpm-data

+               readOnly: false

+               mountPath: /src/results

+ ---

+ {% endif %}

+ apiVersion: batch/v1beta1

+ kind: CronJob

+ metadata:

+   name: stats-34

+ spec:

+   successfulJobsHistoryLimit: 3

+   failedJobsHistoryLimit: 2

+   concurrencyPolicy: Forbid

+   schedule: "0 10 * * *"

+   startingDeadlineSeconds: 500

+   jobTemplate:

+     spec:

+       backoffLimit: 0

+       template:

+         metadata:

+           labels:

+             parent: "cron_stats"

+         spec:

+           volumes:

+           - name: srpm-data

+             persistentVolumeClaim:

+               claimName: "{{ app }}-storage"

+           restartPolicy: Never

+           containers:

+           - name: stats

+             image: "{{ registry }}/{{ app }}/builder:latest"

+             imagePullPolicy: Always

+             command: ["/bin/sh", "-c"]

+             args:

+             - >

+               ./build_language_list.py --results f34 --refresh &&

+               ./build_language_list.py --results f34 --analyzealllang &&

+               ./build_tm.py --results f34 --compress &&

+               ./build_stats.py --results f34

+             volumeMounts:

+             - name: srpm-data

+               readOnly: false

+               mountPath: /src/results

+ ---

+ apiVersion: batch/v1beta1

+ kind: CronJob

+ metadata:

+   name: build-website

+ spec:

+   successfulJobsHistoryLimit: 3

+   failedJobsHistoryLimit: 2

+   concurrencyPolicy: Forbid

+   schedule: "0 */2 * * *"

+   startingDeadlineSeconds: 500

+   jobTemplate:

+     spec:

+       backoffLimit: 0

+       template:

+         metadata:

+           labels:

+             parent: "cron_stats"

+         spec:

+           volumes:

+           - name: srpm-data

+             persistentVolumeClaim:

+               claimName: "{{ app }}-storage"

+           restartPolicy: Never

+           containers:

+           - name: stats

+             image: "{{ registry }}/{{ app }}/builder:latest"

+             imagePullPolicy: Always

+             command: ["/bin/sh", "-c"]

+             args:

+             - >

+               ./build_website.py -v --results f33 ;

+               ./build_website.py -v --results f34 ;

+               cd website && hugo -D --baseURL https://languages{{ env_suffix }}.fedoraproject.org &&

+               cp -R public /src/results/

+             volumeMounts:

+             - name: srpm-data

+               readOnly: false

+               mountPath: /src/results

+ ---

+ apiVersion: batch/v1beta1

+ kind: CronJob

+ metadata:

+   name: stats-website

+ spec:

+   successfulJobsHistoryLimit: 3

+   failedJobsHistoryLimit: 2

+   concurrencyPolicy: Forbid

+   schedule: "0 * * * *"

+   startingDeadlineSeconds: 500

+   jobTemplate:

+     spec:

+       backoffLimit: 0

+       template:

+         metadata:

+           labels:

+             parent: "cron_stats"

+         spec:

+           restartPolicy: Never

+           containers:

+           - image: allinurl/goaccess

+             name: goaccess

+             imagePullPolicy: IfNotPresent

+             args:

+             - "-a"

+             - "--log-format=COMBINED"

+             - "--log-file=/html/logs/access.log"

+             - "--output=/html/public/stats.html"

+             ports:

+             volumeMounts:

+             - name: nginx-data

+               mountPath: /html

+           volumes:

+           - name: nginx-data

+             persistentVolumeClaim:

+               claimName: "{{ app }}-storage"

@@ -0,0 +1,39 @@ 

+ apiVersion: v1

+ kind: DeploymentConfig

+ metadata:

+   name: "web"

+   labels:

+     app: "web"

+ spec:

+   replicas: 1

+   selector:

+     app: "web"

+   template:

+     metadata:

+       labels:

+         app: "web"

+     spec:

+       containers:

+       - image: registry.redhat.io/rhel8/nginx-116

+         name: nginx

+         imagePullPolicy: IfNotPresent

+         args:

+         - nginx

+         - "-g"

+         - "daemon off;"

+         ports:

+         - containerPort: 8080

+           protocol: TCP

+         volumeMounts:

+         - name: nginx-data

+           mountPath: /opt/app-root/src

+           subPath: public

+         - name: nginx-data

+           subPath: logs

+           mountPath: /var/log/nginx

+       volumes:

+       - name: nginx-data

+         persistentVolumeClaim:

+           claimName: "{{ app }}-storage"

+       restartPolicy: Always

+ 

@@ -0,0 +1,15 @@ 

+ apiVersion: v1

+ kind: PersistentVolumeClaim

+ metadata:

+   name: "{{ app }}-storage"

+ spec:

+   accessModes:

+     - ReadWriteMany

+   resources:

+     requests:

+ {%    if env == 'staging' %}

+       storage: 40Gi

+ {%    else %}

+       storage: 70Gi

+ {%    endif %}

+   storageClassName: ""

@@ -0,0 +1,13 @@ 

+ apiVersion: v1

+ kind: Service

+ metadata:

+   name: web

+   labels:

+     app: web

+ spec:

+   selector:

+     app: web

+   ports:

+   - name: web

+     port: 8080

+     targetPort: 8080

related: fedora-infrastructure#9609

This should be deployed on staging only for now, and would be nice if we can get the rights to run the playbook ourself is that's possible.
It require a PV of 30G on staging, and a dns alias languages.stg.fedoraproject.org.

edit: tagging this as stg, since the playbook is now targeting staging only.

rebased onto 59ec051bf91e6cc27e613d3a6f5680d2add2d337

3 years ago

If it's meant to only go to stg for the moment, just adjust:hosts: os_masters[0]:os_masters_stg[0] to include only stg :)

not sure COPY will works here as it won't be present on the filesystem building the container

You're install two of the dependencies here from the latest commit of their default branch. Wouldn't specifying a tag or a commit be safer?

It's in the git repo, which is pulled just before the building process

/tmp is rw only by root on fedora image and I need it for some temporary files.
1777 doesn't seems weird for a /tmp to me, but I can use 1775 instead if needed.

@darknao : we can have a prod branch and a staging branch, but handling versions looks too early in the maturity of this software

would be nice if we can get the rights to run the playbook ourself is that's possible.

This is doable but permissions are added for groups rather than individuals. Are you already in a sysadmin-* group? If so which?

@jibecfed actually, this is more about the 2 dependencies we fetch from github :
https://github.com/WeblateOrg/language-data/archive/master.zip
https://github.com/WeblateOrg/translation-finder.git

@pingou i'm only in fi-apprentice, and I guess that's not enough

oh, i prefer not to use version for these, the owner of these package is re=
sponsive and does good work (it is the weblate dev)=2E No need to fear majo=
r regressions=2E
--
Jean-Baptiste

rebased onto 845febb4faa8ba0dccd65ffd56d3e91bd32a5d2f

3 years ago

rebased onto 524ea6691623e6a67327ace096dca2ddb5e73ed5

3 years ago

rebased onto 3b3c0c88928afe091808396deb37ab906782b55c

3 years ago

rebased onto 3b3c0c88928afe091808396deb37ab906782b55c

3 years ago

This looks good to me.

@pingou would you mind giving a +1 if you are happy your concerns have been addressed?

I can run this for you if you like. As pingou mentioned there would need to be either a new sysadmin-* group created for this or it would need to be adopted into another one which you could become a member of before we can give you permission to run it.

I'm curious, why using a loop here?

Should we use the same structure as the other playbook? It calls the different role in the roles section rather than as tasks, which allows dropping the include_role

I'm curious, why using a loop here?

For future uses. We'll add additional build, one for each fedora release (last 3 or 4 release, not determined yet). But yes, right now, it's not really useful. I can remove that.

Should we use the same structure as the other playbook? It calls the different role in the roles section rather than as tasks, which allows dropping the include_role

You're right. I'll make the changes.

For future uses. We'll add additional build, one for each fedora release (last 3 or 4 release, not determined yet). But yes, right now, it's not really useful. I can remove that.

If it'll be use for the future that's fine with me, I was just being curious if that was needed or was going to be needed

rebased onto ca41323156a1fb2178ed90457d8d5133b9d479f1

3 years ago

I've removed the loops (I end up using only two builds after all), and moved all include_role in roles section.

looking at existing playbook, I think we can still simplify, the vars can be dropped and I think the app variable needs to be pass to each role

looking at existing playbook, I think we can still simplify, the vars can be dropped and I think the app variable needs to be pass to each role

I'm not sure which vars you're talking about. The use of vars in role statements seems to be the preferred way on current ansible version.
The app var is defined globally to avoid redefining it on each role call.

I'm not sure which vars you're talking about. The use of vars in role statements seems to be the preferred way on current ansible version.
The app var is defined globally to avoid redefining it on each role call.

I was just comparing the structure used here to say: https://pagure.io/fedora-infra/ansible/blob/main/f/playbooks/openshift-apps/discourse2fedmsg.yml

why does it talks about docstranslation?

this can't be added here, the config.toml will change quite a lot over time
if you added it here because of the URL, you can use the --baseURL documented in https://gohugo.io/commands/hugo/#options

rebased onto 435f6e970ed0bcf6645556153f6533176af478b6

3 years ago

@jibecfed good catch, comment has been fixed and config file removed from configmap.
@pingou I've reverted back to the previous playbook structure for consistency. Let me know if that's ok for you.

rebased onto 00810945bd976972a0a52ec35feabffd5f164a29

3 years ago

rebased onto 8ca1dc79f9101e6c38d2aeb437ffa6fa1c339396

3 years ago

rebased onto f6edb981bf1212c1e4f5766aecce14422f44fcac

3 years ago

rebased onto 5b8316b

3 years ago

rebased onto 5b8316b

3 years ago

Pull-Request has been merged by smooge

3 years ago