#1684 Add a script to check the dist tarball, and use it to fix the dist tarball
Merged a year ago by breilly. Opened 3 years ago by otaylor.
otaylor/fm-orchestrator tarball-dist  into  master

file modified
+8 -4
@@ -1,12 +1,16 @@ 

- global-exclude *.pyc *.pyo

  include LICENSE

  include README.rst

  include requirements.txt

+ include run-unittests.sh

  include test-requirements.txt

- recursive-include conf config.py cacert.pem *.conf *.cfg *.service

+ include tox.ini

+ recursive-include conf config.py cacert.pem *.conf *.cfg *.json *.service

  recursive-include docs *.txt *.rst *.md

- recursive-include module_build_service *

+ include module_build_service/migrations/README

+ include module_build_service/migrations/alembic.ini

+ include module_build_service/migrations/script.py.mako

  recursive-include fedmsg.d *

- recursive-include tests *.yaml *.json

+ recursive-include tests *.yaml *.json *.rst

  recursive-include tests/scm_data *

  recursive-include client *

+ global-exclude *.pyc *.pyo *.db

file added
+104
@@ -0,0 +1,104 @@ 

+ #!/usr/bin/python

+ 

+ from pathlib import Path

+ import re

+ import subprocess

+ import sys

+ 

+ 

+ if sys.stderr.isatty:

+     def info(*args):

+         print("\033[1m", file=sys.stderr, end="")

+         print(*args, file=sys.stderr, end="")

+         print("\033[0m", file=sys.stderr)

+ 

+ else:

+     def info(*args):

+         print(*args, file=sys.stderr)

+ 

+ 

+ def ignore_re(patterns):

+     return re.compile('^('

+                       + '|'.join(patterns.strip().split())

+                       + ')$')

+ 

+ 

+ # These are files that are generated as part of the distribution process

+ DIST_IGNORE_RE = ignore_re(r"""

+ module_build_service.egg-info/.*

+ setup.cfg

+ PKG-INFO

+ """)

+ 

+ 

+ # These files are in git, but we don't actually want to distribute them

+ GIT_IGNORE_RE = ignore_re(r"""

+ \.cico-pr.pipeline

+ \.dockerignore

+ \.gitignore

+ docker/.*

+ openshift/.*

+ make-dist-tarball.py

+ Vagrantfile

+ """)

+ 

+ 

+ # Find the version in setup.py, this determines the generated tarball name

+ 

+ version = None

+ with open("setup.py") as f:

+     for line in f:

+         m = re.match(r'\s*version\s*=\s*"([^"]+)', line)

+         if m:

+             version = m.group(1)

+ 

+ assert version, "Version not found in setup.py"

+ 

+ # Make the tarball

+ 

+ subprocess.check_call(['python3', 'setup.py', 'sdist'])

+ 

+ info("Checking distributed files")

+ 

+ # Read what files were included in the generated tarball, ignoring files

+ # based on DIST_IGNORE_RE

+ 

+ tarball = Path('dist') / f"module-build-service-{version}.tar.gz"

+ disted_files = set()

+ with subprocess.Popen(["tar", "tfz", tarball],

+                       stdout=subprocess.PIPE, encoding="UTF-8") as p:

+     for line in p.stdout:

+         filename = re.sub(r'^module-build-service-[\d.]+/', '', line.strip())

+         if filename and not filename.endswith("/") and not DIST_IGNORE_RE.match(filename):

+             disted_files.add(filename)

+ 

+ # And what files are checked into git, ignoring files

+ # based on GIT_IGNORE_RE

+ 

+ git_files = set()

+ with subprocess.Popen(["git", "ls-tree", "-r", "--name-only", "HEAD"],

+                       stdout=subprocess.PIPE, encoding="UTF-8") as p:

+     for line in p.stdout:

+         filename = line.strip()

+         if not GIT_IGNORE_RE.match(filename):

+             git_files.add(filename)

+ 

+ # Compare and tell the user about differences

+ 

+ disted_extra = disted_files - git_files

+ git_extra = git_files - disted_files

+ 

+ if not disted_extra and not git_extra:

+     info(tarball, "- OK")

+ else:

+     if disted_extra:

+         info("Extra distributed files")

+         for f in sorted(disted_extra):

+             print('  ', f, file=sys.stderr)

+ 

+     if git_extra:

+         info("Extra files in git, not distributed")

+         for f in sorted(git_extra):

+             print('  ', f, file=sys.stderr)

+ 

+     sys.exit(1)

empty or binary file added
empty or binary file added

This deals with the problem discussed with @breilly with extra files being distributed. I needed a fix here to make a snapshot tarball ... Help me, I can't stop filing MBS PR's... :-)


  • Add a script make-dist-tarball.py that runs 'setup.py sdist' and compares
    what gets distributed with git and with a set of patterns of extra files
    added during distribution, and of git files that we don't want to
    distribute.

  • Extend MANIFEST.in or add init.py files to distribute:

    • run-unittests.sh
    • tox.in
    • conf/client_secrets.json
    • tests/test_builder
    • tests/integration
  • Don't include all of module_build_service - this is too prone to backup
    files and development trash files; instead add just the files we need
    that aren't *.py

  • Move global-excludes to the end - they modify the existing list of files;
    and add *.db - for some reason, .mbs_local_build.db files were
    getting disted.

Build f09eb6c386584f1a0ddf055707818b81696d1ea5 FAILED!
Rebase or make new commits to rebuild.

rebased onto bfc9548

2 years ago

rebased onto c1a28de

a year ago

rebased onto 3e1fe9e

a year ago

rebased onto ad4927f

a year ago

Commit e437ddf fixes this pull-request

Pull-Request has been merged by breilly

a year ago

Pull-Request has been merged by breilly

a year ago