#416 Add dev_scripts/find_images_to_rebuild.py and docs.
Merged 4 years ago by jkaluza. Opened 4 years ago by jkaluza.
jkaluza/freshmaker docs-update2  into  master

file modified
+2 -5
@@ -6,8 +6,5 @@ 

  on lot of other services. To be able to test some parts locally,

  we use few scripts stored in this directory.

  

- The scripts are just templates creating instances of particular Freshmaker

- classes so they can later be used for testing.

- 

- The intended use-case is that when developing some feature, you can test

why did you remove this part?

I originally stored only templates in this directory, but it now contains also ready-to-use scripts, so these can be committed.

- the feature using these scripts. You should *not* commit the changed script.

+ The scripts with "template_" prefix are just templates creating instances

+ of particular Freshmaker classes so they can later be used for testing.

@@ -0,0 +1,265 @@ 

+ # -*- coding: utf-8 -*-

+ 

+ import os

+ import tempfile

+ 

+ from freshmaker.config import all_, any_  # noqa

+ 

+ dbdir = os.path.abspath(os.path.dirname(__file__))

+ 

+ class BaseConfiguration(object):

+ 

+     # EDIT FOLLOWING OPTIONS BASED ON YOUR ENVIRONMENT:

+     # =================================================

+     #

+     # Base URL of git repository with source artifacts.

+     GIT_BASE_URL = "git://pkgs.fedoraproject.org"

+ 

+     # Read Koji configuration from profile instead of reading them from

+     # configuration file directly.

+     KOJI_PROFILE = "brew"

+ 

+     ERRATA_TOOL_SERVER_URL = "https://errata.localhost.tld"

+     PULP_SERVER_URL = "https://pulp.localhost.tld"

+     PULP_USERNAME = "username"

+     PULP_PASSWORD = "password"

+     LIGHTBLUE_SERVER_URL = "https://lightblue.localhost.tld"

+     LIGHTBLUE_CERTIFICATE = "./lb-freshmaker-cert.pem"

+     LIGHTBLUE_PRIVATE_KEY = "./lb-freshmaker-key.pem"

+     LIGHTBLUE_RELEASED_DEPENDENCIES_ONLY = True

+ 

+     HANDLER_BUILD_WHITELIST = {

+         'global': {

+             'image': any_(

+                 # In DRY_RUN mode, allow rebuilding any shipped-live advisory.

+                 # That's useful for testing when submitting rebuilds manually.

+                 {

+                     'advisory_state': 'SHIPPED_LIVE',

+                     'dry_run': True,

+                     'published': True,

+                 }

+             )

+         },

+     }

+ 

+ 

+     # DEFAULT VALUES:

+     # ===============

+     # Make this random (used to generate session keys)

+     SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0'

+     SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(os.path.join(

+         dbdir, 'freshmaker.db'))

+     SQLALCHEMY_TRACK_MODIFICATIONS = False

+ 

+     HOST = '0.0.0.0'

+     PORT = 5001

+ 

+     SERVER_NAME = 'localhost:5001'

+ 

+     DEBUG = True

+     # Global network-related values, in seconds

+     NET_TIMEOUT = 120

+     NET_RETRY_INTERVAL = 30

+ 

+     SYSTEM = 'koji'

+ 

+     # Available backends are: console, file, journal.

+     LOG_BACKEND = 'console'

+ 

+     # Available log levels are: debug, info, warn, error.

+     LOG_LEVEL = 'debug'

+ 

+     # Read Koji configuration from profile instead of reading them from

+     # configuration file directly.

+     KOJI_PROFILE = "brew"

+ 

+     # Settings for docker image rebuild handler

+     KOJI_CONTAINER_SCRATCH_BUILD = False

+ 

+     SSL_ENABLED = False

+ 

+     # whitelist for handlers to decide whether an artifact

+     # can be built.

+     #

+     # In format of:

+     #

+     # { <handler_name> :

+     #     { <artifact_type>: <rule(s)> }

+     # }

+     #

+     # The `handler_name` is usually set to "global" to affect all

+     # the handlers.

+     #

+     # The `rule(s)` part of a whitelist are dictionaries with key named as

+     # some artifact attribute. The value can be str, bool or list of strings.

+     # If it is list of strings, the rule matches if any string from the list

+     # matches the artifact attribute.

+     #

+     # The rule(s) can be also grouped using the any_() or all_() functions:

+     #

+     #     - The any_(rule_1, rule_2, ...) matches when any of the rules

+     #       matches.

+     #     - The all_(rule_1, rule_2, ...) matches when all the rules matches.

+     #

+     # For more information see <http://pagure.io/freshmaker>.

+     #

+     # Here is an example of allowing container images to be build as soon as

+     # an RHSA advisory with critical/important severity or with hightouch bug

+     # moves to SHIPPED_LIVE:

+     #

+     # HANDLER_BUILD_WHITELIST = {

+     #     "global": {

+     #         "image": all_(

+     #             {'advisory_name': 'RHSA-.*'

+     #              'advisory_state: 'SHIPPED_LIVE'},

+     #             any_(

+     #                 {'has_hightouch_bugs': True},

+     #                 {'severity': ['critical', 'important']}

+     #             )

+     #         )

+     #     },

+     # }

+ 

+     # whitelist for handlers to decide whether an artifact

+     # allowed to be built by whitelist should be build.

+     #

+     # The syntax is the same as for HANDLER_BUILD_WHITELIST, but any matched

+     # artifact will *not* be rebuild.

+     #

+     # HANDLER_BUILD_BLACKLIST = {

+     #     "global": {

+     #         "image": all_(

+     #             {'advisory_name': 'RHSA-.*'

+     #              'advisory_state: 'SHIPPED_LIVE'},

+     #             any_(

+     #                 {'has_hightouch_bugs': True},

+     #                 {'severity': ['critical', 'important']}

+     #             )

+     #         )

+     #     },

+     # }

+ 

+     # ODCS configs

+     # URL to ODCS to call APIs

+     ODCS_SERVER_URL = 'https://odcs.localhost/'

+     ODCS_VERIFY_SSL = True

+     # Valid authentication method would be kerberos or openidc

+     ODCS_AUTH_MECH = 'kerberos'

+     # When use openidc authentcation, set the openidc token for accessing ODCS

+     ODCS_OPENIDC_TOKEN = ''

+ 

+     # Kerberos authentication Settings used to authenticated freshmaker itself

+     # by other services

+ 

+     # Whether to use keytab to acquire credential cache. keytab should be used

+     # in a non-devel environment.

+     KRB_AUTH_USE_KEYTAB = True

+     # Principal used to acquire credential cache. When using a client keytab,

+     # this value must be present in that keytab file. Otherwise, principal must

+     # match the one in specified ccache file.

+     KRB_AUTH_PRINCIPAL = ''

+     # Path to freshmaker's client keytab file.

+     KRB_AUTH_CLIENT_KEYTAB = ''

+     # Path to credential cache file. This optional could be None when not using

+     # a client keytab to acquire credential.

+     KRB_AUTH_CCACHE_FILE = tempfile.mkstemp(

+         suffix=str(os.getpid()), prefix="freshmaker_cc_")

+ 

+     # Users are required to be in allowed_clients to generate composes,

+     # you can add group names or usernames (it can be normal user or host

+     # principal) into ALLOWED_CLIENTS. The group names are from ldap for

+     # kerberos users or FAS for openidc users.

+     ALLOWED_CLIENTS = {

+         'groups': [],

+         'users': [],

+     }

+ 

+     # Users in ADMINS are granted with admin permission.

+     ADMINS = {

+         'groups': [],

+         'users': [],

+     }

+ 

+     # Select which authentication backend to work with. There are 3 choices

+     # noauth: no authentication is enabled. Useful for development particularly.

+     # kerberos: Kerberos authentication is enabled.

+     # openidc: OpenIDC authentication is enabled.

+     AUTH_BACKEND = ''

+ 

+     # Used for Kerberos authentication and to query user's groups.

+     # Format: ldap://hostname[:port]

+     # For example: ldap://ldap.example.com/

+     AUTH_LDAP_SERVER = ''

+ 

+     # Group base to query groups from LDAP server.

+     # Generally, it would be, for example, ou=groups,dc=example,dc=com

+     AUTH_LDAP_GROUP_BASE = ''

+ 

+     AUTH_OPENIDC_USERINFO_URI = 'https://id.fedoraproject.org/openidc/UserInfo'

+ 

+     # OIDC base namespace

+     # See also section pagure.io/odcs in

+     # https://fedoraproject.org/wiki/Infrastructure/Authentication

+     OIDC_BASE_NAMESPACE = 'https://pagure.io/freshmaker/'

+ 

+     # Scope requested from Fedora Infra for permission of submitting request to

+     # run a new compose.

+     # See also: https://fedoraproject.org/wiki/Infrastructure/Authentication

+     # Add additional required scope in following list

+     AUTH_OPENIDC_REQUIRED_SCOPES = [

+         'openid',

+         'https://id.fedoraproject.org/scope/groups',

+     ]

+ 

+     # Select which messaging backend will be used, that could be fedmsg, amq,

+     # in_memory or rhmsg.

+     MESSAGING = 'fedmsg'

+     MESSAGING_BACKENDS = {

+         'fedmsg': {

+             'SERVICE': 'freshmaker',

+         },

+         'rhmsg': {

+             # Brokers to connect, e.g.

+             # ['amqps://host:5671', 'amqps://anotherhost:5671']

+             'BROKER_URLS': [],

+             # Path to certificate file used to authenticate freshmaker

+             'CERT_FILE': '',

+             # Path to private key file used to authenticate freshmaker

+             'KEY_FILE': '',

+             # Path to trusted CA certificate bundle.

+             'CA_CERT': '',

+             'TOPIC_PREFIX': 'VirtualTopic.eng.freshmaker',

+         },

+         'in_memory': {

+             'SERVICE': 'freshmaker',

+         }

+     }

+ 

+ 

+ class DevConfiguration(BaseConfiguration):

+     DEBUG = True

+     LOG_BACKEND = 'console'

+     LOG_LEVEL = 'debug'

+ 

+     MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.dev', 'org.fedoraproject.stg']

+ 

+     # Global network-related values, in seconds

+     NET_TIMEOUT = 5

+     NET_RETRY_INTERVAL = 1

+ 

+     KOJI_CONTAINER_SCRATCH_BUILD = True

+ 

+     LIGHTBLUE_VERIFY_SSL = False

+ 

+     # During development, we usually don't need a client keytab to acquire

+     # credential. Instead, kinit in default ccache with personal principal

+     # often.

+     KRB_AUTH_USE_KEYTAB = False

+     KRB_AUTH_PRINCIPAL = ''  # Should be in form name@REAL

+     # Use the default ccache

+     KRB_AUTH_CCACHE_FILE = None

+ 

+     AUTH_BACKEND = 'noauth'

+     AUTH_OPENIDC_USERINFO_URI = 'https://iddev.fedorainfracloud.org/openidc/UserInfo'

+ 

+ 

dev_scripts/template_errata.py dev_scripts/errata.py
file renamed
file was moved with no change to the file
dev_scripts/template_pulp.py dev_scripts/pulp.py
file renamed
file was moved with no change to the file
dev_scripts/templates_lightblue.py dev_scripts/lightblue.py
file renamed
file was moved with no change to the file
file added
+25
@@ -0,0 +1,25 @@ 

+ ===================

+ Development scripts

+ ===================

+ 

+ The dev_scripts_ directory contains few useful scripts which can be used to debug Freshmaker locally.

+ 

+ .. _dev_scripts: https://pagure.io/freshmaker/blob/master/f/dev_scripts

+ 

+ 

+ General configuration

+ =====================

+ 

+ The scripts are configured using the ``config.py`` configuration file. Before you execute any of the development scripts for the first time, you need to create your own ``config.py`` by renaming the ``config.py.template`` and editting the ``EDIT FOLLOWING OPTIONS BASED ON YOUR ENVIRONMENT`` section.

optional: could you put these on multiple lines so it is easier here to read it?

+ 

+ 

+ ``find_images_to_rebuild.py``

+ =============================

+ 

+ This script executes the ``RebuildImagesOnRPMAdvisoryChange`` handler which is responsible for rebuilding container images as a result of RPM RHSA release. This is the main work done by Freshmaker running in the infrastructure.

+ 

+ Usage: ``./find_images_to_rebuild.py <errata_advisory_id>``

+ 

+ The script queries Errata Tool, Pulp and Lightblue database to find out the list of container images which would be rebuilt by Freshmaker as a result of this RPM RHSA. It prints the list of container images and also initial set of image builds as submitted to Koji.

+ 

+ It also creates ``freshmaker.db`` which contains the Event and Artifact builds as they would be created in real infrastructure.

file modified
+1
@@ -18,6 +18,7 @@ 

     api_v1

     api_v2

     messaging_api

+    dev_scripts

  

  Indices and tables

  ==================

no initial comment

why did you remove this part?

optional: could you put these on multiple lines so it is easier here to read it?

Looks awesome!! Thank you

rebased onto 269c938

4 years ago

I originally stored only templates in this directory, but it now contains also ready-to-use scripts, so these can be committed.

Commit 218f4e7 fixes this pull-request

Pull-Request has been merged by jkaluza

4 years ago

Pull-Request has been merged by jkaluza

4 years ago