+ #!/usr/bin/env python

+ 

+ import sys

+ import os

+ import time

+ 

+ import six

+ 

+ import koji

+ import koji.tasks

+ from koji_cli.lib import OptionParser, _, activate_session

+ 

+ 

+ def handle_batch_clone_tag():

+     """[admin] Clone tag with a batch size"""

+     usage = _("usage: %prog batch-clone-tag [options] <src-tag> <dst-tag>")

+     usage += _(

+         "\n(Specify the --help global option for a list of other help options)")

+     parser = OptionParser(usage=usage)

+     parser.add_option('-p', '--profile', default='koji',

+                       help='pick a profile')

+     parser.add_option('--config', action='store_true',

+                       help=_("Copy config from the source to the dest tag"))

+     parser.add_option('--groups', action='store_true',

+                       help=_("Copy group information"))

+     parser.add_option('--pkgs', action='store_true',

+                       help=_(

+                           "Copy package list from the source to the dest tag"))

+     parser.add_option('--builds', action='store_true',

+                       help=_("Tag builds into the dest tag"))

+     parser.add_option('--all', action='store_true',

+                       help=_("The same as --config --groups --pkgs --builds"))

+     parser.add_option('--latest-only', action='store_true',

+                       help=_("Tag only the latest build of each package"))

+     parser.add_option('--inherit-builds', action='store_true',

+                       help=_(

+                           "Include all builds inherited into the source tag into "

+                           "the dest tag"))

+     parser.add_option('--ts', type='int',

+                       help=_('Clone tag at a specific timestamp'))

+     parser.add_option('--event', type='int',

+                       help=_('Clone tag at a specific event'))

+     parser.add_option('--repo', type='int',

+                       help=_('Clone tag at a specific repo event'))

+     parser.add_option("-v", "--verbose", action="store_true",

+                       help=_("show changes"))

+     parser.add_option("-f", "--force", action="store_true",

+                       help=_("override tag locks if necessary"))

+     parser.add_option("-n", "--test", action="store_true", help=_("test mode"))

+     parser.add_option("--batch", type='int', default=1000,

+                       help=_("muiltCall batch size(Default: 1000)"))

+     (options, args) = parser.parse_args()

+ 

+     _koji = koji.get_profile_module(options.profile)

+     dir_opts = ('topdir', 'cert', 'serverca')

+     for name in dir_opts:

+         # expand paths here, so we don't have to worry about it later

+         value = os.path.expanduser(getattr(_koji.config, name))

+         setattr(_koji.config, name, value)

+     session_opts = koji.grab_session_options(_koji.config)

+     session = koji.ClientSession(_koji.config.server, session_opts)

+ 

+     if len(args) != 2:

+         parser.error(

+             _("This command takes two arguments: <src-tag> <dst-tag>"))

+         assert False  # pragma: no cover

+     activate_session(session, _koji.config)

+ 

+     if not session.hasPerm('admin') and not options.test:

+         print(_("This action requires admin privileges"))

+         return

+ 

+     if args[0] == args[1]:

+         sys.stdout.write('Source and destination tags must be different.\n')

+         return

+ 

+     if options.all:

+         options.config = options.groups = options.pkgs = options.builds = True

+ 

+     if options.batch <= 0:

+         parser.error(_("batch size must be bigger than zero"))

+ 

+     event = koji.util.eventFromOpts(session, options) or {}

+     if event:

+         event['timestr'] = time.asctime(time.localtime(event['ts']))

+         print(_("Cloning at event %(id)i (%(timestr)s)") % event)

+ 

+     # store tags.

+     srctag = session.getTag(args[0])

+     dsttag = session.getTag(args[1])

+     if not srctag:

+         sys.stdout.write("Unknown src-tag: %s\n" % args[0])

+         return

+     if (srctag['locked'] and not options.force) or (

+                     dsttag and dsttag['locked'] and not options.force):

+         print(_(

+             "Error: You are attempting to clone from or to a tag which is locked."))

+         print(_("Please use --force if this is what you really want to do."))

+         return

+ 

+     # init debug lists.

+     chgpkglist = []

+     chgbldlist = []

+     chggrplist = []

+     # case of brand new dst-tag.

+     if not dsttag:

+         if not options.config:

+             print(_('Cannot create tag without specifying --config'))

+             return

+         # create a new tag, copy srctag header.

+         if not options.test:

+             session.createTag(args[1], parent=None, arches=srctag['arches'],

+                               perm=srctag['perm_id'],

+                               locked=srctag['locked'],

+                               maven_support=srctag['maven_support'],

+                               maven_include_all=srctag['maven_include_all'])

+             newtag = session.getTag(args[1],

+                                     strict=True)  # store the new tag, need its asigned id.

+         # get pkglist of src-tag, including inherited packages.

+         if options.pkgs:

+             srcpkgs = session.listPackages(tagID=srctag['id'], inherited=True,

+                                            event=event.get('id'))

+             srcpkgs.sort(key=lambda x: x['package_name'])

+             if not options.test:

+                 session.multicall = True

+             for pkgs in srcpkgs:

+                 # for each package add one entry in the new tag.

+                 chgpkglist.append(('[new]', pkgs['package_name'],

+                                    pkgs['blocked'], pkgs['owner_name'],

+                                    pkgs['tag_name']))

+                 if not options.test:

+                     # add packages.

+                     session.packageListAdd(newtag['name'],

+                                            pkgs['package_name'],

+                                            owner=pkgs['owner_name'],

+                                            block=pkgs['blocked'],

+                                            extra_arches=pkgs['extra_arches'])

+             if not options.test:

+                 _batch_multicall(session, options.batch)

+         if options.builds:

+             # get --all latest builds from src tag

+             builds = session.listTagged(srctag['id'], event=event.get('id'),

+                                         inherit=options.inherit_builds,

+                                         latest=options.latest_only)

+             if not options.test:

+                 session.multicall = True

+             for build in builds:

+                 build['name'] = build[

+                     'package_name']  # add missing 'name' field.

+                 chgbldlist.append(('[new]', build['package_name'],

+                                    build['nvr'],

+                                    koji.BUILD_STATES[build['state']],

+                                    build['owner_name'], build['tag_name']))

+                 # copy latest builds into new tag

+                 if not options.test:

+                     session.tagBuildBypass(newtag['name'], build,

+                                            force=options.force)

+             if not options.test:

+                 _batch_multicall(session, options.batch)

+         if options.groups:

+             # Copy the group data

+             srcgroups = session.getTagGroups(srctag['name'],

+                                              event=event.get('id'))

+             if not options.test:

+                 session.multicall = True

+             for group in srcgroups:

+                 if not options.test:

+                     session.groupListAdd(newtag['name'], group['name'])

+                 for pkg in group['packagelist']:

+                     if not options.test:

+                         session.groupPackageListAdd(newtag['name'],

+                                                     group['name'],

+                                                     pkg['package'],

+                                                     block=pkg['blocked'])

+                     chggrplist.append(('[new]', pkg['package'], group['name']))

+             if not options.test:

+                 _batch_multicall(session, options.batch)

+     # case of existing dst-tag.

+     if dsttag:

+         # get fresh list of packages & builds into maps.

+         srcpkgs = {}

+         dstpkgs = {}

+         srclblds = {}

+         dstlblds = {}

+         srcgroups = {}

+         dstgroups = {}

+         if options.pkgs:

+             for pkg in session.listPackages(tagID=srctag['id'], inherited=True,

+                                             event=event.get('id')):

+                 srcpkgs[pkg['package_name']] = pkg

+             for pkg in session.listPackages(tagID=dsttag['id'],

+                                             inherited=True):

+                 dstpkgs[pkg['package_name']] = pkg

+         if options.builds:

+             src_builds = session.listTagged(srctag['id'],

+                                             event=event.get('id'),

+                                             inherit=options.inherit_builds,

+                                             latest=options.latest_only)

+             for build in src_builds:

+                 srclblds[build['nvr']] = build

+             for build in session.getLatestBuilds(dsttag['name']):

+                 dstlblds[build['nvr']] = build

+         if options.groups:

+             for group in session.getTagGroups(srctag['name'],

+                                               event=event.get('id')):

+                 srcgroups[group['name']] = group

+             for group in session.getTagGroups(dsttag['name']):

+                 dstgroups[group['name']] = group

+         # construct to-do lists.

+         paddlist = []  # list containing new packages to be added from src tag

+         for (package_name, pkg) in six.iteritems(srcpkgs):

+             if package_name not in dstpkgs:

+                 paddlist.append(pkg)

+         paddlist.sort(key=lambda x: x['package_name'])

+         pdellist = []  # list containing packages no more present in dst tag

+         for (package_name, pkg) in six.iteritems(dstpkgs):

+             if package_name not in srcpkgs:

+                 pdellist.append(pkg)

+         pdellist.sort(key=lambda x: x['package_name'])

+         baddlist = []  # list containing new builds to be added from src tag

+         for (nvr, lbld) in six.iteritems(srclblds):

+             if nvr not in dstlblds:

+                 baddlist.append(lbld)

+         baddlist.sort(key=lambda x: x['package_name'])

+         bdellist = []  # list containing new builds to be removed from src tag

+         for (nvr, lbld) in six.iteritems(dstlblds):

+             if nvr not in srclblds:

+                 bdellist.append(lbld)

+         bdellist.sort(key=lambda x: x['package_name'])

+         gaddlist = []  # list containing new groups to be added from src tag

+         for (grpname, group) in six.iteritems(srcgroups):

+             if grpname not in dstgroups:

+                 gaddlist.append(group)

+         gdellist = []  # list containing groups to be removed from src tag

+         for (grpname, group) in six.iteritems(dstgroups):

+             if grpname not in srcgroups:

+                 gdellist.append(group)

+         grpchanges = {}  # dict of changes to make in shared groups

+         for (grpname, group) in six.iteritems(srcgroups):

+             if grpname in dstgroups:

+                 grpchanges[grpname] = {'adds': [], 'dels': []}

+                 # Store whether group is inherited or not

+                 grpchanges[grpname]['inherited'] = False

+                 if group['tag_id'] != dsttag['id']:

+                     grpchanges[grpname]['inherited'] = True

+                 srcgrppkglist = []

+                 dstgrppkglist = []

+                 for pkg in group['packagelist']:

+                     srcgrppkglist.append(pkg['package'])

+                 for pkg in dstgroups[grpname]['packagelist']:

+                     dstgrppkglist.append(pkg['package'])

+                 for pkg in srcgrppkglist:

+                     if not pkg in dstgrppkglist:

+                         grpchanges[grpname]['adds'].append(pkg)

+                 for pkg in dstgrppkglist:

+                     if not pkg in srcgrppkglist:

+                         grpchanges[grpname]['dels'].append(pkg)

+         # ADD new packages.

+         if not options.test:

+             session.multicall = True

+         for pkg in paddlist:

+             chgpkglist.append(('[add]', pkg['package_name'],

+                                pkg['blocked'], pkg['owner_name'],

+                                pkg['tag_name']))

+             if not options.test:

+                 session.packageListAdd(dsttag['name'], pkg['package_name'],

+                                        owner=pkg['owner_name'],

+                                        block=pkg['blocked'],

+                                        extra_arches=pkg['extra_arches'])

+         if not options.test:

+             _batch_multicall(session, options.batch)

+         # ADD builds.

+         if not options.test:

+             session.multicall = True

+         for build in baddlist:

+             build['name'] = build['package_name']  # add missing 'name' field.

+             chgbldlist.append(('[add]', build['package_name'], build['nvr'],

+                                koji.BUILD_STATES[build['state']],

+                                build['owner_name'], build['tag_name']))

+             # copy latest builds into new tag.

+             if not options.test:

+                 session.tagBuildBypass(dsttag['name'], build,

+                                        force=options.force)

+         if not options.test:

+             _batch_multicall(session, options.batch)

+         # ADD groups.

+         if not options.test:

+             session.multicall = True

+         for group in gaddlist:

+             if not options.test:

+                 session.groupListAdd(dsttag['name'], group['name'],

+                                      force=options.force)

+             for pkg in group['packagelist']:

+                 if not options.test:

+                     session.groupPackageListAdd(dsttag['name'], group['name'],

+                                                 pkg['package'],

+                                                 force=options.force)

+                 chggrplist.append(('[new]', pkg['package'], group['name']))

+         if not options.test:

+             _batch_multicall(session, options.batch)

+         # ADD group pkgs.

+         if not options.test:

+             session.multicall = True

+         for group in grpchanges:

+             for pkg in grpchanges[group]['adds']:

+                 chggrplist.append(('[new]', pkg, group))

+                 if not options.test:

+                     session.groupPackageListAdd(dsttag['name'], group, pkg,

+                                                 force=options.force)

+         if not options.test:

+             _batch_multicall(session, options.batch)

+         # DEL builds.

+         if not options.test:

+             session.multicall = True

+         for build in bdellist:

+             # dont delete an inherited build.

+             if build['tag_name'] == dsttag['name']:

+                 build['name'] = build[

+                     'package_name']  # add missing 'name' field.

+                 chgbldlist.append(

+                     ('[del]', build['package_name'], build['nvr'],

+                      koji.BUILD_STATES[build['state']],

+                      build['owner_name'], build['tag_name']))

+                 # go on del builds from new tag.

+                 if not options.test:

+                     session.untagBuildBypass(dsttag['name'], build,

+                                              force=options.force)

+         if not options.test:

+             _batch_multicall(session, options.batch)

+         # DEL packages.

+         ninhrtpdellist = []

+         inhrtpdellist = []

+         for pkg in pdellist:

+             if pkg['tag_name'] == dsttag['name']:

+                 ninhrtpdellist.append(pkg)

+             else:

+                 inhrtpdellist.append(pkg)

+         session.multicall = True

+         # delete only non-inherited packages.

+         for pkg in ninhrtpdellist:

+             # check if package have owned builds inside.

+             session.listTagged(dsttag['name'], package=pkg['package_name'],

+                                inherit=False)

+         bump_builds = _batch_multicall(session, options.batch)

+         if not options.test:

+             session.multicall = True

+         for pkg, [builds] in zip(ninhrtpdellist, bump_builds):

+             # remove all its builds first if there are any.

+             for build in builds:

+                 build['name'] = build[

+                     'package_name']  # add missing 'name' field.

+                 chgbldlist.append(

+                     ('[del]', build['package_name'], build['nvr'],

+                      koji.BUILD_STATES[build['state']],

+                      build['owner_name'], build['tag_name']))

+                 # so delete latest build(s) from new tag.

+                 if not options.test:

+                     session.untagBuildBypass(dsttag['name'], build,

+                                              force=options.force)

+             # now safe to remove package itselfm since we resolved its builds.

+             chgpkglist.append(('[del]', pkg['package_name'], pkg['blocked'],

+                                pkg['owner_name'], pkg['tag_name']))

+             if not options.test:

+                 session.packageListRemove(dsttag['name'], pkg['package_name'],

+                                           force=False)

+         # mark as blocked inherited packages.

+         for pkg in inhrtpdellist:

+             chgpkglist.append(('[blk]', pkg['package_name'], pkg['blocked'],

+                                pkg['owner_name'], pkg['tag_name']))

+             if not options.test:

+                 session.packageListBlock(dsttag['name'], pkg['package_name'])

+         if not options.test:

+             _batch_multicall(session, options.batch)

+         # DEL groups.

+         if not options.test:

+             session.multicall = True

+         for group in gdellist:

+             # Only delete a group that isn't inherited

+             if group['tag_id'] == dsttag['id']:

+                 if not options.test:

+                     session.groupListRemove(dsttag['name'], group['name'],

+                                             force=options.force)

+                 for pkg in group['packagelist']:

+                     chggrplist.append(('[del]', pkg['package'], group['name']))

+             # mark as blocked inherited groups.

+             else:

+                 if not options.test:

+                     session.groupListBlock(dsttag['name'], group['name'])

+                 for pkg in group['packagelist']:

+                     chggrplist.append(('[blk]', pkg['package'], group['name']))

+         if not options.test:

+             _batch_multicall(session, options.batch)

+         # DEL group pkgs.

+         if not options.test:

+             session.multicall = True

+         for group in grpchanges:

+             for pkg in grpchanges[group]['dels']:

+                 # Only delete a group that isn't inherited

+                 if not grpchanges[group]['inherited']:

+                     chggrplist.append(('[del]', pkg, group))

+                     if not options.test:

+                         session.groupPackageListRemove(dsttag['name'], group,

+                                                        pkg,

+                                                        force=options.force)

+                 else:

+                     chggrplist.append(('[blk]', pkg, group))

+                     if not options.test:

+                         session.groupPackageListBlock(dsttag['name'], group,

+                                                       pkg)

+         if not options.test:

+             _batch_multicall(session, options.batch)

+     # print final list of actions.

+     if options.verbose:

+         pfmt = '    %-7s %-28s %-10s %-10s %-10s\n'

+         bfmt = '    %-7s %-28s %-40s %-10s %-10s %-10s\n'

+         gfmt = '    %-7s %-28s %-28s\n'

+         sys.stdout.write('\nList of changes:\n\n')

+         sys.stdout.write(

+             pfmt % ('Action', 'Package', 'Blocked', 'Owner', 'From Tag'))

+         sys.stdout.write(

+             pfmt % ('-' * 7, '-' * 28, '-' * 10, '-' * 10, '-' * 10))

+         for changes in chgpkglist:

+             sys.stdout.write(pfmt % changes)

+         sys.stdout.write('\n')

+         sys.stdout.write(bfmt % (

+             'Action', 'From/To Package', 'Latest Build(s)', 'State', 'Owner',

+             'From Tag'))

+         sys.stdout.write(

+             bfmt % ('-' * 7, '-' * 28, '-' * 40, '-' * 10, '-' * 10, '-' * 10))

+         for changes in chgbldlist:

+             sys.stdout.write(bfmt % changes)

+         sys.stdout.write('\n')

+         sys.stdout.write(gfmt % ('Action', 'Package', 'Group'))

+         sys.stdout.write(gfmt % ('-' * 7, '-' * 28, '-' * 28))

+         for changes in chggrplist:

+             sys.stdout.write(gfmt % changes)

+ 

+ 

+ def _batch_multicall(session, size):

+     if not session.multicall:

+         raise koji.GenericError(

+             'ClientSession.multicall must be set to True before calling multiCall()')

+     session.multicall = False

+     if len(session._calls) == 0:

+         return []

+     callgrp = [session._calls[i:i + size] for i in

+                xrange(0, len(session._calls), size)]

+     session._calls = []

+     ret = []

+     for c in callgrp:

+         ret.extend(session._callMethod('multiCall', (c,), {}))

+     return ret

+ 

+ 

+ if __name__ == '__main__':

+     handle_batch_clone_tag()