#1097 gather.py: use createrepo_c for creating repodata instead of obsolete createrepo python library
Merged 5 years ago by lsedlar. Opened 5 years ago by fepitre.
fepitre/pungi fix-createrepo  into  master

file modified
+20 -27
@@ -22,7 +22,6 @@ 

  import sys

  from fnmatch import fnmatch

  

- import createrepo

  import lockfile

  import urlgrabber.progress

  import yum
@@ -31,7 +30,7 @@ 

  import arch as arch_module

  import multilib_yum as multilib

  import pungi.util

- 

+ from pungi.wrappers.createrepo import CreaterepoWrapper

  

  class ReentrantYumLock(object):

      """ A lock that can be acquired multiple times by the same process. """
@@ -1405,39 +1404,33 @@ 

          basedir = os.path.join(self.destdir, self.config.get('pungi', 'version'))

          if subfile.startswith(basedir):

              return subfile.replace(basedir + os.path.sep, '')

-         

+ 

      def _makeMetadata(self, path, cachedir, comps=False, repoview=False, repoviewtitle=False,

                        baseurl=False, output=False, basedir=False, update=True,

                        compress_type=None):

          """Create repodata and repoview."""

-         

-         conf = createrepo.MetaDataConfig()

-         conf.cachedir = os.path.join(cachedir, 'createrepocache')

-         conf.update = update

-         conf.unique_md_filenames = True

+ 

+         # Define outputdir

          if output:

-             conf.outputdir = output

+             outputdir = output

          else:

-             conf.outputdir = path

-         conf.directory = path

-         conf.database = True

-         if comps:

-             conf.groupfile = comps

-         if basedir:

-             conf.basedir = basedir

-         if baseurl:

-             conf.baseurl = baseurl

-         if compress_type:

-             conf.compress_type = compress_type

+             outputdir = path

+ 

+         # Define revision if SOURCE_DATE_EPOCH exists in env

          if 'SOURCE_DATE_EPOCH' in os.environ:

-             conf.revision = os.environ['SOURCE_DATE_EPOCH']

-             conf.clamp_mtime_to = int(os.environ['SOURCE_DATE_EPOCH'])

-         repomatic = createrepo.MetaDataGenerator(conf)

+             revision = os.environ['SOURCE_DATE_EPOCH']

+         else:

+             revision = None

+ 

+         createrepo_wrapper = CreaterepoWrapper(createrepo_c=True)

+         createrepo = createrepo_wrapper.get_createrepo_cmd(directory=path, update=update, outputdir=outputdir,

+                                                            unique_md_filenames=True, database=True, groupfile=comps,

+                                                            basedir=basedir, baseurl=baseurl, revision=revision,

+                                                            compress_type=compress_type)

+ 

          self.logger.info('Making repodata')

-         repomatic.doPkgMetadata()

-         repomatic.doRepoMetadata()

-         repomatic.doFinalMove()

-         

+         pungi.util._doRunCommand(createrepo, self.logger)

+ 

          if repoview:

              # setup the repoview call

              repoview = ['/usr/bin/repoview']

file modified
+8 -2
@@ -28,13 +28,13 @@ 

              self.mergerepo = "mergerepo"

              self.modifyrepo = "modifyrepo"

  

-     def get_createrepo_cmd(self, directory, baseurl=None, outputdir=None, excludes=None,

+     def get_createrepo_cmd(self, directory, baseurl=None, outputdir=None, basedir=None, excludes=None,

                             pkglist=None, groupfile=None, cachedir=None, update=True,

                             update_md_path=None, skip_stat=False, checkts=False, split=False,

                             pretty=True, database=True, checksum=None, unique_md_filenames=True,

                             distro=None, content=None, repo=None, revision=None, deltas=False,

                             oldpackagedirs=None, num_deltas=None, workers=None, use_xz=False,

-                            extra_args=None):

+                            compress_type=None, extra_args=None):

          # groupfile = /path/to/comps.xml

  

          cmd = [self.createrepo, directory]
@@ -45,6 +45,9 @@ 

          if outputdir:

              cmd.append("--outputdir=%s" % outputdir)

  

+         if basedir:

+             cmd.append("--basedir=%s" % basedir)

+ 

          for i in force_list(excludes or []):

              cmd.append("--excludes=%s" % i)

  
@@ -118,6 +121,9 @@ 

          if use_xz:

              cmd.append("--xz")

  

+         if compress_type:

+             cmd.append("--compress-type=%s" % compress_type)

+ 

          if extra_args:

              cmd.extend(force_list(extra_args))

  

Currently --set-timestamp-to-revision is not available. It will be soon with this PR for createrepo_c: https://github.com/rpm-software-management/createrepo_c/pull/105.

We can wait for this feature or add it later.

Thanks, Frédéric, for dealing with it.
Removing createrepo should also be updated in the documentation. For example here: https://docs.pagure.org/pungi/contributing.html, where createrepo is noted.
Just question/suggestion, could some functionality from current CreaterepoWrapper be used/modified in this situation?

I will check also the documentation and remove any createrepo reference. If I understand you question, you mean to have a possibility to add extra options to createrepo_c when calling pungi?

Oh, I was not aware of CreaterepoWrapper which should be used here instead. I think I understand your point now. Is it what you meant?

I will check also the documentation and remove any createrepo reference. If I understand you question, you mean to have a possibility to add extra options to createrepo_c when calling pungi?

I have just meant to remove createrepo occurrences.

Oh, I was not aware of CreaterepoWrapper which should be used here instead. I think I understand your point now. Is it what you meant?

Yes. But I am not sure, whether it is or it is not a useful approach. Just look into CreaterepoWrapper and find if it is reasonable to reuse some functionality there.

Using CreaterepoWrapper is fine just tested it right now. I had to add 'basedir' and 'compress-type' option. I will adjust this PR soon. For createrepo occurence, maybe it worths to open a specific PR for removing this deprecated package as there a little work of refactor to do. What do you think?

rebased onto 0e399e6

5 years ago

2 new commits added

  • CreaterepoWrapper: add 'basedir' and 'compress-type' args for createrepo_c
  • gather.py: use createrepo_c for creating repodata instead of obsolete createrepo python library
5 years ago

I like the patch.

I would be okay with removing the references to old createrepo from documentation in another patch (it's really only two places in contributing guide, the configuration part refers to pungi-koji and that can optionally use old createrepo but only as an excutable).

Once --set-timestamp-from-revision is available, it might be good to enable it unconditionally. Again that should be a separate PR.

I agree with you. If you want, I can handle these two separate patches.

Commit f41c32e fixes this pull-request

Pull-Request has been merged by lsedlar

5 years ago

I'll handle the documentation, but I would appreciate if you could track the patch for timestamps. Thank you very much!