| |
@@ -0,0 +1,457 @@
|
| |
+ #!/usr/bin/env python
|
| |
+
|
| |
+ """
|
| |
+
|
| |
+ Utility script used to load the database based on the data present on disk
|
| |
+ for the pagure instance over Fedora's git repositories.
|
| |
+
|
| |
+ """
|
| |
+
|
| |
+
|
| |
+ import argparse
|
| |
+ import collections
|
| |
+ import os
|
| |
+
|
| |
+ import requests
|
| |
+ import pygit2
|
| |
+ from sqlalchemy.exc import SQLAlchemyError
|
| |
+
|
| |
+ if 'PAGURE_CONFIG' not in os.environ \
|
| |
+ and os.path.exists('/etc/pagure/pagure.cfg'):
|
| |
+ print('Using configuration file `/etc/pagure/pagure.cfg`')
|
| |
+ os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'
|
| |
+
|
| |
+ import pagure
|
| |
+ import pagure.lib
|
| |
+ import pagure.lib.model
|
| |
+
|
| |
+
|
| |
+ PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb/api/'
|
| |
+ FOLDER = '/srv/git/repositories/'
|
| |
+ MAPPING = {
|
| |
+ 'Fedora': 'rpms',
|
| |
+ 'Fedora EPEL': 'rpms',
|
| |
+ 'Fedora Container': 'container',
|
| |
+ 'Fedora EPEL Container': 'container',
|
| |
+ 'Fedora Modules': 'modules',
|
| |
+ }
|
| |
+ BLACKLIST = ['@provenpackager']
|
| |
+
|
| |
+
|
| |
+ def get_poc_of_pkgs(debug=False):
|
| |
+ """ Retrieve a dictionary giving the point of contact of each package
|
| |
+ in pkgdb.
|
| |
+ """
|
| |
+ if debug:
|
| |
+ print('Querying pkgdb')
|
| |
+ req = requests.get(PKGDB_URL + 'bugzilla').text
|
| |
+ if debug:
|
| |
+ print('Pkgdb data retrieved, getting POC')
|
| |
+ pkgs = {}
|
| |
+ for line in req.split('\n'):
|
| |
+ line = line.strip()
|
| |
+ if not line or line.startswith('#'):
|
| |
+ continue
|
| |
+ line = line.split('|')
|
| |
+ if len(line) < 4:
|
| |
+ continue
|
| |
+
|
| |
+ namespace = MAPPING[line[0].strip()]
|
| |
+ if namespace not in pkgs:
|
| |
+ pkgs[namespace] = {}
|
| |
+
|
| |
+ if line[1] not in pkgs[namespace]:
|
| |
+ pkgs[namespace][line[1]] = line[3].replace('group::', '@')
|
| |
+
|
| |
+ return pkgs
|
| |
+
|
| |
+
|
| |
+ def get_watch_tickets_of_pkgs(debug=False):
|
| |
+ """ Retrieve a dictionary giving the watch tickets for each package
|
| |
+ in pkgdb.
|
| |
+ """
|
| |
+ if debug:
|
| |
+ print('Querying pkgdb')
|
| |
+ req = requests.get(PKGDB_URL + 'bugzilla').text
|
| |
+ if debug:
|
| |
+ print('Pkgdb data retrieved, getting CC')
|
| |
+ pkgs = {}
|
| |
+ for line in req.split('\n'):
|
| |
+ line = line.strip()
|
| |
+ if not line or line.startswith('#'):
|
| |
+ continue
|
| |
+ line = line.split('|')
|
| |
+ if len(line) < 4:
|
| |
+ continue
|
| |
+
|
| |
+ namespace = MAPPING[line[0].strip()]
|
| |
+ if namespace not in pkgs:
|
| |
+ pkgs[namespace] = {}
|
| |
+
|
| |
+ if line[1] not in pkgs[namespace]:
|
| |
+ if line[-1]:
|
| |
+ users = line[-1].replace('group::', '@').split(',')
|
| |
+ users = [u.strip() for u in users if u.strip()]
|
| |
+ pkgs[namespace][line[1]] = users
|
| |
+
|
| |
+ return pkgs
|
| |
+
|
| |
+
|
| |
+ def get_watch_commits_of_pkgs(debug=False):
|
| |
+ """ Retrieve a dictionary giving the watch commit for each package
|
| |
+ in pkgdb.
|
| |
+ """
|
| |
+ if debug:
|
| |
+ print('Querying pkgdb')
|
| |
+ req = requests.get(PKGDB_URL + 'notify?namespace=1').text
|
| |
+ if debug:
|
| |
+ print('Pkgdb data retrieved, getting Watch commit')
|
| |
+ pkgs = {}
|
| |
+ for line in req.split('\n'):
|
| |
+ line = line.strip()
|
| |
+ if not line or line.startswith('#'):
|
| |
+ continue
|
| |
+ line = line.split('|')
|
| |
+ if len(line) < 3:
|
| |
+ continue
|
| |
+
|
| |
+ namespace = line[0].strip()
|
| |
+ if namespace not in pkgs:
|
| |
+ pkgs[namespace] = {}
|
| |
+
|
| |
+ if line[1] not in pkgs[namespace]:
|
| |
+ if line[-1]:
|
| |
+ users = line[-1].replace('group::', '@').split(',')
|
| |
+ users = [u.strip() for u in users if u.strip()]
|
| |
+ pkgs[namespace][line[1]] = users
|
| |
+
|
| |
+ return pkgs
|
| |
+
|
| |
+
|
| |
+ def get_maintainers(namespace, debug=False):
|
| |
+ """ Retrieve a dictionary giving the maintainers of each package
|
| |
+ in pkgdb.
|
| |
+ """
|
| |
+ if debug:
|
| |
+ print('Querying pkgdb')
|
| |
+ req = requests.get(PKGDB_URL + 'vcs?format=json&namespace=%s' % namespace)
|
| |
+ if debug:
|
| |
+ print('Pkgdb data retrieved, getting maintainers')
|
| |
+ pkgs = collections.defaultdict(set)
|
| |
+ data = req.json()[namespace]
|
| |
+ for pkg in data:
|
| |
+ pkgers = []
|
| |
+ grps = []
|
| |
+ for br in data[pkg]:
|
| |
+ pkgers += [
|
| |
+ p
|
| |
+ for p in data[pkg][br]['commit']['people']]
|
| |
+ grps += [
|
| |
+ '@%s' % g
|
| |
+ for g in data[pkg][br]['commit']['groups']
|
| |
+ if g != 'provenpackager'
|
| |
+ ]
|
| |
+ pkgs[pkg].update(pkgers)
|
| |
+ pkgs[pkg].update(grps)
|
| |
+
|
| |
+ return pkgs
|
| |
+
|
| |
+
|
| |
+ def set_up_users(pocs, maintainers, debug=False):
|
| |
+ """ Add the users and groups in the DB.
|
| |
+ """
|
| |
+ if debug:
|
| |
+ print('Adding the user to the DB')
|
| |
+
|
| |
+ all_users = set()
|
| |
+
|
| |
+ for ns in maintainers:
|
| |
+ for user_set in maintainers[ns].values():
|
| |
+ all_users.update(user_set)
|
| |
+
|
| |
+ for ns in pocs:
|
| |
+ all_users.update(set(pocs[ns].values()))
|
| |
+
|
| |
+ if 'releng' not in all_users:
|
| |
+ pagure.lib.set_up_user(
|
| |
+ session=pagure.SESSION,
|
| |
+ username='releng',
|
| |
+ fullname='Release Engineering',
|
| |
+ default_email='releng@fedoraproject.org',
|
| |
+ )
|
| |
+ pagure.SESSION.commit()
|
| |
+
|
| |
+ for user in sorted(all_users):
|
| |
+ if debug:
|
| |
+ print(user)
|
| |
+ try:
|
| |
+ if user.startswith('@') or user.startswith('group::'):
|
| |
+ if user.startswith('@'):
|
| |
+ name = user[1:]
|
| |
+ else:
|
| |
+ name = user[7:]
|
| |
+ pagure.lib.add_group(
|
| |
+ session=pagure.SESSION,
|
| |
+ group_name=name,
|
| |
+ display_name=name,
|
| |
+ description='Group %s' % name,
|
| |
+ group_type='user',
|
| |
+ user='releng',
|
| |
+ is_admin=False,
|
| |
+ blacklist=pagure.APP.config['BLACKLISTED_GROUPS'],
|
| |
+ )
|
| |
+ pagure.SESSION.commit()
|
| |
+ else:
|
| |
+ pagure.lib.set_up_user(
|
| |
+ session=pagure.SESSION,
|
| |
+ username=user,
|
| |
+ fullname=user,
|
| |
+ default_email='%s@fedoraproject.org' % user,
|
| |
+ )
|
| |
+ pagure.SESSION.commit()
|
| |
+ except SQLAlchemyError as err:
|
| |
+ pagure.SESSION.rollback()
|
| |
+ print('ERROR with user %s' % user)
|
| |
+ print(err)
|
| |
+
|
| |
+
|
| |
+ def main(pocs, maintainers, ccs, watch_commits, folder, namespace, debug=False):
|
| |
+ """
|
| |
+ Logic:
|
| |
+ - Query the list of maintainer/PoC from pkgdb
|
| |
+ - Set the users in the database based on the PoC retrieved
|
| |
+ - Browse the directory
|
| |
+ - For each git in the directory, create the project with the correct POC
|
| |
+
|
| |
+ """
|
| |
+
|
| |
+ for project in sorted(os.listdir(folder)):
|
| |
+ if not project.endswith('.git'):
|
| |
+ if debug:
|
| |
+ print(' -skip: not a git repository')
|
| |
+ continue
|
| |
+
|
| |
+ name = project.rsplit('.git', 1)[0]
|
| |
+ fullname = '%s/%s' % (namespace, name)
|
| |
+
|
| |
+ if name not in pocs:
|
| |
+ if debug:
|
| |
+ print(' -no pocs: setting poc to `orphan`')
|
| |
+ pocs[name] = 'orphan'
|
| |
+
|
| |
+ if debug:
|
| |
+ print(project, pocs[name])
|
| |
+
|
| |
+ try:
|
| |
+ if fullname in pagure.APP.config['BLACKLISTED_PROJECTS']:
|
| |
+ raise pagure.exceptions.RepoExistsException(
|
| |
+ 'No project "%s" are allowed to be created due to '
|
| |
+ 'potential conflicts in URLs with pagure itself' % name
|
| |
+ )
|
| |
+
|
| |
+ pkg_poc = pocs[name]
|
| |
+ if pkg_poc.startswith('@'):
|
| |
+ for user in maintainers[name]:
|
| |
+ user = user.strip()
|
| |
+ if not user:
|
| |
+ continue
|
| |
+ if not user.startswith('@'):
|
| |
+ if debug:
|
| |
+ print(' %s name had for POC: %s, switching to %s' % (
|
| |
+ name, pkg_poc, user))
|
| |
+ pkg_poc = user
|
| |
+ break
|
| |
+
|
| |
+ user_obj = pagure.lib.get_user(pagure.SESSION, pkg_poc)
|
| |
+ allowed_prefix = pagure.APP.config[
|
| |
+ 'ALLOWED_PREFIX'] + [grp for grp in user_obj.groups]
|
| |
+
|
| |
+ first_part, _, second_part = name.partition('/')
|
| |
+ if second_part and first_part not in allowed_prefix:
|
| |
+ raise pagure.exceptions.PagureException(
|
| |
+ 'The prefix of your project must be in the list of '
|
| |
+ 'allowed prefixes set by the admins of this pagure '
|
| |
+ 'instance, or the name of a group of which you are a '
|
| |
+ 'member.'
|
| |
+ )
|
| |
+
|
| |
+ gitfolder = pagure.APP.config['GIT_FOLDER']
|
| |
+ requestfolder = pagure.APP.config['REQUESTS_FOLDER']
|
| |
+ requestfolder = pagure.APP.config['REQUESTS_FOLDER']
|
| |
+
|
| |
+ project = pagure.lib.model.Project(
|
| |
+ name=name,
|
| |
+ namespace=namespace,
|
| |
+ description='The %s %s' % (name, namespace),
|
| |
+ url=None,
|
| |
+ avatar_email=None,
|
| |
+ user_id=user_obj.id,
|
| |
+ parent_id=None,
|
| |
+ hook_token=pagure.lib.login.id_generator(40)
|
| |
+ )
|
| |
+ pagure.SESSION.add(project)
|
| |
+ # Make sure we won't have SQLAlchemy error before we create
|
| |
+ # the repo
|
| |
+ pagure.SESSION.flush()
|
| |
+
|
| |
+ if name in maintainers:
|
| |
+ for maintainer in maintainers[name]:
|
| |
+ maintainer = maintainer.strip()
|
| |
+ if not maintainer:
|
| |
+ continue
|
| |
+ if debug:
|
| |
+ print(' - %s' % maintainer)
|
| |
+ if maintainer in project.contributors['commit']:
|
| |
+ if debug:
|
| |
+ print(' - %s has already commit' % maintainer)
|
| |
+ continue
|
| |
+ try:
|
| |
+ if maintainer.startswith('@') \
|
| |
+ or maintainer.startswith('group::'):
|
| |
+ if maintainer.startswith('@'):
|
| |
+ name = maintainer[1:]
|
| |
+ else:
|
| |
+ name = maintainer[7:]
|
| |
+ out = pagure.lib.add_group_to_project(
|
| |
+ session=pagure.SESSION,
|
| |
+ project=project,
|
| |
+ new_group=name,
|
| |
+ user='releng',
|
| |
+ access='commit',
|
| |
+ is_admin=True)
|
| |
+ if debug:
|
| |
+ print(' -', out)
|
| |
+ else:
|
| |
+ out = pagure.lib.add_user_to_project(
|
| |
+ session=pagure.SESSION,
|
| |
+ project=project,
|
| |
+ new_user=maintainer,
|
| |
+ user='releng',
|
| |
+ access='commit')
|
| |
+ if debug:
|
| |
+ print(' -', out)
|
| |
+ pagure.SESSION.commit()
|
| |
+ except pagure.exceptions.PagureException as err:
|
| |
+ print('ERROR with project %s' % fullname)
|
| |
+ print(err)
|
| |
+
|
| |
+ # Set-up the watch-ticket flags
|
| |
+ if name in ccs:
|
| |
+ for maintainer in ccs[name]:
|
| |
+ maintainer = maintainer.strip()
|
| |
+ if not maintainer:
|
| |
+ continue
|
| |
+ if debug:
|
| |
+ print(' - %s' % maintainer)
|
| |
+
|
| |
+ if maintainer.startswith('@'):
|
| |
+ if debug:
|
| |
+ print(' Skipping group')
|
| |
+ continue
|
| |
+
|
| |
+ watch = '1'
|
| |
+ if maintainer in maintainers[name]:
|
| |
+ watch = '3'
|
| |
+
|
| |
+ try:
|
| |
+ out = pagure.lib.update_watch_status(
|
| |
+ session=pagure.SESSION,
|
| |
+ project=project,
|
| |
+ user=maintainer,
|
| |
+ watch=watch
|
| |
+ )
|
| |
+ if debug:
|
| |
+ print(' -', out)
|
| |
+ except pagure.exceptions.PagureException as err:
|
| |
+ print('ERROR with project %s' % fullname)
|
| |
+ print(err)
|
| |
+
|
| |
+ # Set-up the watch-commits flags
|
| |
+ if name in watch_commits:
|
| |
+ for maintainer in watch_commits[name]:
|
| |
+ maintainer = maintainer.strip()
|
| |
+ if not maintainer:
|
| |
+ continue
|
| |
+ if debug:
|
| |
+ print(' - %s' % maintainer)
|
| |
+
|
| |
+ if maintainer.startswith('@'):
|
| |
+ if debug:
|
| |
+ print(' Skipping group')
|
| |
+ continue
|
| |
+
|
| |
+ watch = '2'
|
| |
+ if maintainer in maintainers.get(name, []) \
|
| |
+ or maintainer in ccs.get(name, []):
|
| |
+ watch = '3'
|
| |
+
|
| |
+ try:
|
| |
+ out = pagure.lib.update_watch_status(
|
| |
+ session=pagure.SESSION,
|
| |
+ project=project,
|
| |
+ user=maintainer,
|
| |
+ watch=watch
|
| |
+ )
|
| |
+ if debug:
|
| |
+ print(' -', out)
|
| |
+ except pagure.exceptions.PagureException as err:
|
| |
+ print('ERROR with project %s' % fullname)
|
| |
+ print(err)
|
| |
+
|
| |
+ gitrepo = os.path.join(gitfolder, '%s.git' % fullname)
|
| |
+ http_clone_file = os.path.join(gitrepo, 'git-daemon-export-ok')
|
| |
+ if not os.path.exists(http_clone_file):
|
| |
+ with open(http_clone_file, 'w') as stream:
|
| |
+ pass
|
| |
+
|
| |
+ requestrepo = os.path.join(requestfolder, project.path)
|
| |
+ if not os.path.exists(requestrepo):
|
| |
+ pygit2.init_repository(
|
| |
+ requestrepo, bare=True,
|
| |
+ mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP)
|
| |
+ elif debug:
|
| |
+ print(' - The requests repo "%s" already exists' % (
|
| |
+ project.path))
|
| |
+ pass
|
| |
+
|
| |
+ pagure.SESSION.commit()
|
| |
+ except pagure.exceptions.PagureException as err:
|
| |
+ print('ERROR with project %s' % fullname)
|
| |
+ print(err)
|
| |
+ except SQLAlchemyError as err: # pragma: no cover
|
| |
+ pagure.SESSION.rollback()
|
| |
+ print('ERROR (DB) with project %s' % project)
|
| |
+ print(err)
|
| |
+
|
| |
+
|
| |
+ if __name__ == '__main__':
|
| |
+ parser = argparse.ArgumentParser(
|
| |
+ description='Script creating projects on pagure based on the git '
|
| |
+ 'repos present in the specified folder and the pkgdb information.'
|
| |
+ )
|
| |
+ parser.add_argument(
|
| |
+ '--debug', dest='debug', action='store_true', default=False,
|
| |
+ help='Print the debugging output')
|
| |
+
|
| |
+ args = parser.parse_args()
|
| |
+
|
| |
+ pocs = get_poc_of_pkgs(debug=args.debug)
|
| |
+ ccs = get_watch_tickets_of_pkgs(debug=args.debug)
|
| |
+ watch_commit = get_watch_commits_of_pkgs(debug=args.debug)
|
| |
+ maintainers = {}
|
| |
+ for namespace in ['rpms', 'modules', 'container']:
|
| |
+ maintainers[namespace] = get_maintainers(namespace, debug=args.debug)
|
| |
+ set_up_users(pocs, maintainers, debug=args.debug)
|
| |
+
|
| |
+ for namespace in ['rpms', 'modules', 'container']:
|
| |
+
|
| |
+ folder = os.path.join(FOLDER, namespace)
|
| |
+ main(
|
| |
+ pocs=pocs[namespace],
|
| |
+ maintainers=maintainers[namespace],
|
| |
+ ccs=ccs[namespace],
|
| |
+ watch_commits=watch_commit[namespace],
|
| |
+ folder=folder,
|
| |
+ namespace=namespace,
|
| |
+ debug=args.debug
|
| |
+ )
|
| |
asaleh
commented 4 years ago
This is the script used to populate pagure's database based on what is
on disk and in pkgdb.
It is very Fedora-specific and thus belongs more here than in pagure
itself.
The PR is a rebased version of https://pagure.io/pagure-dist-git/pull-request/7 with python3 updates.