From b789d2d9b346b7e41f259f07d2afc31d04b6d21d Mon Sep 17 00:00:00 2001
From: Chenxiong Qi <cqi@redhat.com>
Date: Jun 28 2018 14:07:06 +0000
Subject: Build on multiple targets from stream branch


Signed-off-by: Chenxiong Qi <cqi@redhat.com>

---

diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py
index 4e1c9c7..26b5b10 100644
--- a/pyrpkg/__init__.py
+++ b/pyrpkg/__init__.py
@@ -21,6 +21,7 @@ import koji
 import logging
 import os
 import posixpath
+import random
 import re
 import rpm
 import shutil
@@ -397,6 +398,10 @@ class Commands(object):
             self.load_branch_merge()
         return(self._branch_merge)
 
+    @branch_merge.setter
+    def branch_merge(self, value):
+        self._branch_merge = value
+
     def load_branch_merge(self):
         """Find the remote tracking branch from the branch we're on.
 
@@ -824,7 +829,11 @@ class Commands(object):
 
         # If a site has a different naming scheme, this would be where
         # a site would override
-        self._target = '%s-candidate' % self.branch_merge
+        self._target = self.build_target(self.branch_merge)
+
+    def build_target(self, release):
+        """Map release to build target"""
+        return '%s-candidate' % release
 
     @property
     def container_build_target(self):
@@ -1950,8 +1959,9 @@ class Commands(object):
             self._get_namespace_anongiturl(repo_name or self.ns_repo_name),
             commit_hash or self.commithash)
 
-    def build(self, skip_tag=False, scratch=False, background=False,
-              url=None, chain=None, arches=None, sets=False, nvr_check=True):
+    def build(self, target=None, skip_tag=False, scratch=False,
+              background=False, url=None, chain=None, arches=None, sets=False,
+              nvr_check=True):
         """Initiate a build.  Available options are:
 
         skip_tag: Skip the tag action after the build
@@ -1992,10 +2002,14 @@ class Commands(object):
                     'Try option --srpm to make scratch build from local changes.')
                 raise rpkgError(msg)
             url = self.construct_build_url()
+
         # Check to see if the target is valid
-        build_target = self.kojisession.getBuildTarget(self.target)
+        # Allow choosing which build target to build against. If no one is
+        # specified, get the target from current release branch.
+        target = target or self.target
+        build_target = self.kojisession.getBuildTarget(target)
         if not build_target:
-            raise rpkgError('Unknown build target: %s' % self.target)
+            raise rpkgError('Unknown build target: %s' % target)
         # see if the dest tag is locked
         dest_tag = self.kojisession.getTag(build_target['dest_tag_name'])
         if not dest_tag:
@@ -2003,12 +2017,14 @@ class Commands(object):
                             % build_target['dest_tag_name'])
         if dest_tag['locked'] and not scratch:
             raise rpkgError('Destination tag %s is locked' % dest_tag['name'])
+
         if chain:
             cmd.append('chain-build')
             # We're chain building, make sure inheritance works
             self.check_inheritance(build_target, dest_tag)
         else:
             cmd.append('build')
+
         # define our dictionary for options
         opts = {}
         # Set a placeholder for the build priority
@@ -2032,7 +2048,7 @@ class Commands(object):
             cmd.append('--arch-override=%s' % ','.join(arches))
             opts['arch_override'] = ' '.join(arches)
 
-        cmd.append(self.target)
+        cmd.append(target)
 
         if url.endswith('.src.rpm'):
             srpm = os.path.basename(url)
@@ -2064,6 +2080,7 @@ class Commands(object):
                                     'Note: You can skip this check with'
                                     ' --skip-nvr-check. See help for more'
                                     ' info.' % self.nvr)
+
         # Now submit the task and get the task_id to return
         # Handle the chain build version
         if chain:
@@ -2079,19 +2096,26 @@ class Commands(object):
             # This next list comp is ugly, but it's how we properly get a :
             # put in between each build set
             cmd.extend(' : '.join([' '.join(build_sets) for build_sets in chain]).split())
-            self.log.info('Chain building %s + %s for %s', build_reference, chain[:-1], self.target)
+            self.log.info('Chain building %s + %s for %s', build_reference, chain[:-1], target)
             self.log.debug('Building chain %s for %s with options %s and a priority of %s',
-                           chain, self.target, opts, priority)
+                           chain, target, opts, priority)
             self.log.debug(' '.join(cmd))
-            task_id = self.kojisession.chainBuild(chain, self.target, opts, priority=priority)
+            task_id = self.kojisession.chainBuild(chain, target, opts, priority=priority)
         # Now handle the normal build
         else:
             cmd.append(url)
-            self.log.info('Building %s for %s', build_reference, self.target)
+            self.log.info('Building %s for %s', build_reference, target)
             self.log.debug('Building %s for %s with options %s and a priority of %s',
-                           url, self.target, opts, priority)
+                           url, target, opts, priority)
             self.log.debug(' '.join(cmd))
-            task_id = self.kojisession.build(url, self.target, opts, priority=priority)
+            if self.dry_run:
+                self.log.info(
+                    'DRY-RUN: self.kojisession.build("%s", "%s", %s, priority=%s)',
+                    url, target, opts, priority)
+                task_id = random.randint(12000, 12100)
+            else:
+                task_id = self.kojisession.build(url, target, opts, priority=priority)
+
         self.log.info('Created task: %s', task_id)
         self.log.info('Task info: %s/taskinfo?taskID=%s', self.kojiweburl, task_id)
         return task_id
diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py
index 35cfba3..c4020a4 100644
--- a/pyrpkg/cli.py
+++ b/pyrpkg/cli.py
@@ -14,18 +14,21 @@
 
 from __future__ import print_function
 import argparse
+import errno
 import getpass
+import itertools
 import logging
 import os
 import random
+import re
 import requests
-from requests.auth import HTTPBasicAuth
 import string
 import sys
 import time
-import re
+
 # For `_ArgumentParser'
 from gettext import gettext as _
+from requests.auth import HTTPBasicAuth
 
 import koji_cli.lib
 import pyrpkg.utils as utils
@@ -293,6 +296,7 @@ class cliClient(object):
             else:
                 self._cmd.ns = 'rpms'
 
+        self._cmd.dry_run = self.args.dry_run
         self._cmd.password = self.args.password
         self._cmd.runas = self.args.runas
         self._cmd.debug = self.args.debug
@@ -346,6 +350,10 @@ class cliClient(object):
         self.parser.add_argument('--config', '-C',
                                  default=None,
                                  help='Specify a config file to use')
+        self.parser.add_argument('--dry-run',
+                                 action='store_true',
+                                 default=False,
+                                 help='Dry run mode.')
         group = self.parser.add_mutually_exclusive_group()
         group.add_argument('--release',
                            dest='release',
@@ -509,10 +517,36 @@ class cliClient(object):
 
         build_parser = self.subparsers.add_parser(
             'build', help='Request build', parents=[self.build_parser_common],
-            description='This command requests a build of the package in the '
-                        'build system. By default it discovers the target '
-                        'to build for based on branch data, and uses the '
-                        'latest commit as the build source.')
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            description='''
+This command requests a build of the package in the build system. By default it
+discovers the target to build for based on branch data, and uses the latest
+commit as the build source.
+
+When build from a stream branch, {0} is able to submit builds according to
+configured releases in local package config file. If there is no such a file,
+global option --release would be required to help disovering the targetself.
+
+The local package config file is a hidden file with name ".package". You can
+create it for each stream branch with content:
+
+    [koji]
+    targets = specific-release|fedora|epel
+
+where specific release is one of master and active Fedora releases and EPEL,
+fedora is a shortcut for active Fedora releases, epel is another shortcut for
+EPEL. You can specify multiples in a single line. Here are some examples,
+each of which is valid in the config.
+
+    targets = master f28 el6
+    targets = fedora
+    targets = epel
+    targets = master f28 epel7
+    targets = master fedora epel
+
+Please note that, if you would like to build for rawhide, master should be
+specified explicitly.
+''')
         build_parser.add_argument(
             '--skip-nvr-check', action='store_false', default=True,
             dest='nvr_check',
@@ -1416,7 +1450,172 @@ see API KEY section of copr-cli(1) man page.
     def usage(self):
         self.parser.print_help()
 
-    def build(self, sets=None):
+    @staticmethod
+    def expand_release(rel, active_releases):
+        if rel == 'master':
+            return ['master']
+        elif rel == 'fedora':
+            return active_releases['fedora']
+        elif rel == 'epel':
+            return active_releases['epel']
+        elif rel in active_releases['fedora'] or rel in active_releases['epel']:
+            return [rel]
+        else:
+            return None
+
+    def read_releases_from_local_config(self, active_releases):
+        """Read configured releases from build config from repo"""
+        config_file = os.path.join(self.cmd.path, '.package')
+        if not os.path.exists(config_file):
+            self.log.warning('No local config file exists.')
+            self.log.warning('Create .package to specify build targets to '
+                             'build.')
+            return None
+        config = configparser.ConfigParser()
+        if not config.read([config_file]):
+            raise rpkgError('Package config .package is not accessible.')
+        if config.has_option('koji', 'targets'):
+            target_releases = config.get('koji', 'targets', raw=True)
+            expanded_releases = []
+            for rel in re.split(r'[ \t]+', target_releases.strip()):
+                expanded = self.expand_release(rel, active_releases)
+                if expanded:
+                    expanded_releases += expanded
+                else:
+                    self.log.error('Target %s is unknonw.', rel)
+            return sorted(set(expanded_releases))
+        else:
+            return None
+
+    def get_stream_branches(self, server_url):
+        query_args = {
+            'global_component': self.cmd.repo_name,
+            'fields': ['name', 'active'],
+            # XXX: use this type to query?
+            'type': 'module',
+        }
+        branches = self.query_pdc(server_url,
+                                  'component-branches',
+                                  params=query_args)
+        return [
+            item for item in branches
+            if not re.match(r'^(f|el|epel)\d+$', item['name']) and
+            item['name'] != 'master'
+        ]
+
+    @staticmethod
+    def is_stream_branch(stream_branches, name):
+        for branch_info in stream_branches:
+            if branch_info['name'] != name:
+                continue
+            if branch_info['active']:
+                return True
+            else:
+                raise rpkgError('Cannot build from stream branch {0} as it is '
+                                'inactive.'.format(name))
+        return False
+
+    @staticmethod
+    def query_pdc(server_url, endpoint, params, timeout=60):
+        api_url = '{0}/rest_api/v1/{1}/'.format(
+            server_url.rstrip('/'), endpoint.strip('/'))
+        query_args = params
+        while True:
+            try:
+                rv = requests.get(api_url, params=query_args, timeout=60)
+            except ConnectionError as error:
+                error_msg = ('The connection to PDC failed while trying to get '
+                            'the active release branches. The error was: {0}'
+                            .format(str(error)))
+                raise rpkgError(error_msg)
+
+            if not rv.ok:
+                base_error_msg = ('The following error occurred while trying to '
+                                  'get the active release branches in PDC: {0}')
+                raise rpkgError(base_error_msg.format(rv.text))
+
+            rv_json = rv.json()
+            for item in rv_json['results']:
+                yield item
+
+            if rv_json['next']:
+                # Clear the query_args because they are baked into the "next" URL
+                query_args = {}
+                api_url = rv_json['next']
+            else:
+                # We've gone through every page, so we can return the found
+                # branches
+                break
+
+    def get_active_releases(self, server_url):
+        """
+        Get the active Fedora release branches from PDC
+
+        :param url: a string of the URL to PDC
+        :return: a set containing the active Fedora release branches
+        """
+        query_args = {
+            'fields': ['short', 'version'],
+            'active': True
+        }
+        releases = {}
+
+        for product_version in self.query_pdc(server_url, 'product-versions',
+                                              params=query_args):
+            short_name = product_version['short']
+            version = product_version['version']
+
+            # If the version is not a digit we can ignore it (e.g. rawhide)
+            if not version.isdigit():
+                continue
+
+            if short_name == 'epel':
+                prefix = 'el' if version == '6' else 'epel'
+            elif short_name == 'fedora':
+                prefix = 'f'
+
+            release = '{0}{1}'.format(prefix, version)
+            releases.setdefault(short_name, []).append(release)
+
+        return releases
+
+    def build(self):
+        server_url = self.config.get('{0}.pdc'.format(self.name), 'url')
+        cur_branch = self.cmd.branch_merge
+        stream_branches = self.get_stream_branches(server_url)
+        self.log.debug('Got stream branches: %r',
+                       [item['name'] for item in stream_branches])
+        if self.is_stream_branch(stream_branches, cur_branch):
+            self.log.debug('Current branch is a stream branch.')
+            releases = self.read_releases_from_local_config(
+                self.get_active_releases(server_url))
+            if releases:
+                self.log.debug('Build on release targets: %r', releases)
+                tasks_ids = []
+                for release in releases:
+                    target = self.cmd.build_target(release)
+                    self.cmd.branch_merge = release
+                    task_id = self._build(target=target)
+                    tasks_ids.append(task_id)
+            else:
+                # If local config file is not created yet, or no build targets
+                # are not configured, let's build as normal.
+                tasks_ids = [self._build()]
+        else:
+            tasks_ids = [self._build()]
+
+        if self.args.nowait:
+            return
+
+        # Pass info off to our koji task watcher
+        if self.args.dry_run:
+            self.log.info(
+                'DRY-RUN: watch tasks: %s',
+                ', '.join((str(item) for item in tasks_ids)))
+        else:
+            return koji_cli.lib.watch_tasks(self.cmd.kojisession, tasks_ids)
+
+    def _build(self, sets=None, target=None):
         # We may have gotten arches by way of scratch build, so handle them
         arches = None
         if hasattr(self.args, 'arches'):
@@ -1427,9 +1626,6 @@ see API KEY section of copr-cli(1) man page.
         chain = None
         if hasattr(self.args, 'chain'):
             chain = self.args.chain
-        # Need to do something with BUILD_FLAGS or KOJI_FLAGS here for compat
-        if self.args.target:
-            self.cmd._target = self.args.target
         # handle uploading the srpm if we got one
         if hasattr(self.args, 'srpm') and self.args.srpm:
             # See if we need to generate the srpm first
@@ -1457,18 +1653,21 @@ see API KEY section of copr-cli(1) man page.
         nvr_check = True
         if hasattr(self.args, 'nvr_check'):
             nvr_check = self.args.nvr_check
-        task_id = self.cmd.build(self.args.skip_tag, self.args.scratch,
-                                 self.args.background, url, chain, arches,
-                                 sets, nvr_check)
-
-        # Log out of the koji session
-        self.cmd.kojisession.logout()
 
-        if self.args.nowait:
-            return
-
-        # Pass info off to our koji task watcher
-        return koji_cli.lib.watch_tasks(self.cmd.kojisession, [task_id])
+        try:
+            return self.cmd.build(
+                target=target,
+                skip_tag=self.args.skip_tag,
+                scratch=self.args.scratch,
+                background=self.args.background,
+                url=url,
+                chain=chain,
+                arches=arches,
+                sets=sets,
+                nvr_check=nvr_check)
+        finally:
+            # Log out of the koji session
+            self.cmd.kojisession.logout()
 
     def chainbuild(self):
         if self.cmd.module_name in self.args.package:
@@ -1517,7 +1716,14 @@ see API KEY section of copr-cli(1) man page.
         self.args.chain = urls
         self.args.skip_tag = False
         self.args.scratch = False
-        return self.build(sets=sets)
+
+        task_id = self._build(sets=sets)
+
+        if self.args.nowait:
+            return
+
+        # Pass info off to our koji task watcher
+        return koji_cli.lib.watch_tasks(self.cmd.kojisession, [task_id])
 
     def clean(self):
         dry = False
@@ -2020,7 +2226,14 @@ see API KEY section of copr-cli(1) man page.
         # A scratch build is just a build with --scratch
         self.args.scratch = True
         self.args.skip_tag = False
-        return self.build()
+
+        task_id = self._build()
+
+        if self.args.nowait:
+            return
+
+        # Pass info off to our koji task watcher
+        return koji_cli.lib.watch_tasks(self.cmd.kojisession, [task_id])
 
     def sources(self):
         """Download files listed in sources