#27 change 'filterlist' to 'imagelist', use productmd formats
Merged 7 years ago by tibbs. Opened 7 years ago by adamwill.
adamwill/quick-fedora-mirror imagefilter  into  master

file modified
+16 -7
@@ -14,6 +14,12 @@ 

  import sys

  from scandir import scandir

  

+ # productmd is optional, needed only for the imagelist feature

+ try:

+     from productmd.images import SUPPORTED_IMAGE_FORMATS

+ except ImportError:

+     SUPPORTED_IMAGE_FORMATS = []

+ 

  

  def get_ftype(entry):

      """Return a simple indicator of the file type."""
@@ -58,8 +64,8 @@ 

      null = open(os.devnull, 'w')

      p = argparse.ArgumentParser(

          description='Generate a list of files and times, suitable for consumption by quick-fedora-mirror, '

-                     'and a much smaller list with packages, Device Tree boot files, HTML files and '

-                     'directories filtered out, for consumption by fedfind.')

+                     'and (optionally) a much smaller list of only files that match one of the productmd '

+                     ' supported image types, for use by fedfind.')

      p.add_argument('-c', '--checksum', action='store_true',

                     help='Include checksums of all repomd.xml files in the file list.')

      p.add_argument('-C', '--checksum-file', action='append', dest='checksum_files',
@@ -75,8 +81,9 @@ 

                     help='Filename of the file list with times (default: stdout).')

      p.add_argument('-f', '--filelist', type=argparse.FileType('w'), default=null,

                     help='Filename of the file list without times (default: no plain file list is generated).')

-     p.add_argument('-F', '--filterlist', type=argparse.FileType('w'), default=null,

-                    help='Filename of the filtered file list for fedfind (default: not generated).')

+     p.add_argument('-i', '--imagelist', type=argparse.FileType('w'), default=null,

+                    help='Filename of the image file list for fedfind (default: not generated). Requires '

+                    'the productmd library.')

  

      opts = p.parse_args()

  
@@ -99,6 +106,8 @@ 

  

  def main():

      opts = parseopts()

+     if opts.imagelist.name != os.devnull and not SUPPORTED_IMAGE_FORMATS:

+         sys.exit("--imagelist requires the productmd library!")

      checksums = {}

  

      os.chdir(opts.dir)
@@ -112,9 +121,9 @@ 

          # opts.filelist.write(entry.path + '\n')

          print(entry.path, file=opts.filelist)

          # write to filtered list if appropriate

-         skips = ('.rpm', '.drpm', '.dtb', '.html')

-         if not any(entry.path.endswith(skip) for skip in skips) and not (entry.is_dir()):

-             print(entry.path, file=opts.filterlist)

+         imgs = ['.{0}'.format(form) for form in SUPPORTED_IMAGE_FORMATS]

+         if any(entry.path.endswith(img) for img in imgs):

+             print(entry.path, file=opts.imagelist)

          if entry.name in opts.checksum_files:

              checksums[entry.path[2:]] = True

          info = entry.stat(follow_symlinks=False)

As an alternative to filtering out known not-image file types,
just import the list of known image file types from productmd
and filter out anything not in that when creating the list for
fedfind. Now call it 'imagelist' rather than 'filterlist'.

The advantage of this is it gives us the smallest possible list
and we don't have to worry about the filterlist file sizes
suddenly growing because a large number of some new type of
file starts showing up in the compose process (this would make
fedfind runs suddenly use a lot more bandwidth, but otherwise
wouldn't be terribly visible, which is kinda bad).

The disadvantage is it introduces a dependency on productmd
which we'd have to ensure we always account for on the systems
where create-filelist is run, and keep up to date so we don't
miss new image types.

This is an alternative to https://pagure.io/quick-fedora-mirror/pull-request/26 ; I'm giving us choices here. Please pick which approach you prefer and merge it.

@kevin said he'd prefer this version. We've checked that the productmd constant is the same in 1.0 (which is in EPEL 7) and 1.2 (which is in Fedora).

rebased

7 years ago

Revised to still work (but with --imagelist disabled) if productmd is unavailable.

Pull-Request has been merged by tibbs

7 years ago
Metadata