#438 Ignore files in a cloned repository
Merged 3 years ago by onosek. Opened 3 years ago by onosek.
onosek/rpkg git_excludes  into  master

file modified
+20 -1
@@ -96,7 +96,8 @@ 


                   koji_config_type='config', user=None,

                   dist=None, target=None, quiet=False,

-                  distgit_namespaced=False, realms=None, lookaside_namespaced=False):

+                  distgit_namespaced=False, realms=None, lookaside_namespaced=False,

+                  git_excludes=None):

          """Init the object and some configuration details."""


          # Path to operate on, most often pwd
@@ -215,6 +216,8 @@ 

          self.module_api_url = None

          # Namespaces for which retirement is blocked by default.

          self.block_retire_ns = ['rpms']

+         # Git excludes patterns

+         self.git_excludes = git_excludes or []


      # Define properties here

      # Properties allow us to "lazy load" various attributes, which also means
@@ -1543,6 +1546,8 @@ 

          conf_git = git.Git(os.path.join(path, git_dir))

          self._clone_config(conf_git, repo)


+         self._add_git_excludes(os.path.join(path, git_dir))




      def get_base_repo(self, repo):
@@ -1652,6 +1657,20 @@ 

                  if confline:



+     def _add_git_excludes(self, conf_dir):

+         """

+         Add a list of patterns from config into the config file in a git

+         repository. This list excludes some files or dirs to be tracked by

+         git. This list usually includes files that are automatically generated.

+         These changes are valid just for local git repository.

+         """

+         git_excludes_path = os.path.join(conf_dir, '.git/info/exclude')

+         git_excludes = GitIgnore(git_excludes_path)

+         for item in self.git_excludes:

+             git_excludes.add(item)

+         git_excludes.write()

+         self.log.debug('Git-excludes patterns were added into %s' % git_excludes_path)


      def commit(self, message=None, file=None, files=[], signoff=False):

          """Commit changes to a repository (optionally found at path)


file modified
+7 -1
@@ -266,6 +266,11 @@ 

                  'Format argument module is deprecated in anongiturl. '

                  'Please use "repo" instead.')


+         # Read line separated list of git excludes patterns

+         git_excludes = [excl

+                         for excl in items.get("git_excludes", '').split('\n')

+                         if excl]


          # Create the cmd object

          self._cmd = self.site.Commands(self.args.path,

@@ -283,7 +288,8 @@ 




-                                        lookaside_namespaced=la_namespaced

+                                        lookaside_namespaced=la_namespaced,

+                                        git_excludes=git_excludes



          if self.args.module_name:

@@ -33,6 +33,11 @@ 

              bz.default-component %(module)s

              sendemail.to %(module)s-owner@fedoraproject.org


+         self.git_excludes = [

+             'i386/',

+             'noarch/',

+             'build*.log',

+         ]

          self.user = 'TODO'

          self.dist = 'TODO'

          self.target = 'TODO'

@@ -72,6 +72,29 @@ 




+     def test_clone_anonymous_git_excludes(self):

+         self.make_new_git(self.module)


+         altpath = tempfile.mkdtemp(prefix='rpkg-tests.')


+         cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash,

+                               self.lookaside_cgi, self.gitbaseurl,

+                               self.anongiturl, self.branchre, self.kojiprofile,

+                               self.build_client, self.user, self.dist,

+                               self.target, self.quiet,

+                               git_excludes=self.git_excludes)

+         cmd.clone(self.module, anon=True)


+         moduledir = os.path.join(self.path, self.module)

+         self.assertTrue(os.path.isfile(os.path.join(moduledir, '.git/info/exclude')))


+         with open(os.path.join(moduledir, '.git/info/exclude')) as git_exclude_file:

+             all_excludes = git_exclude_file.read()

+             for pattern in self.git_excludes:

+                 self.assertIn(pattern, all_excludes)


+         shutil.rmtree(altpath)


      def test_clone_anonymous_with_branch(self):


                            branches=['rpkg-tests-1', 'rpkg-tests-2'])

Git will ignore automatically generated files. Ignored patterns can be
specified in rhpkg/fedpkg config. Patterns are applied in
'.git/info/exclude' file only when repository is cloned. And changes are
valid only for local repository.

Fixes: #355

Signed-off-by: Ondrej Nosek onosek@redhat.com

When is this file created?

Other files to consider: .build-*.log and results_*, which are created by mockbuild.
Also clog would be nice.

When is this file created?
just for example. No functional reason.

rebased onto bb1a5318ac53f5dc78fbd681a04eced8cc004d92

3 years ago

Other files to consider: .build-.log and results_, which are created by mockbuild.
Also clog would be nice.


Suggest: more short version: git_excludes or []

I'm thinking if rpkg.conf is a good place to read these patterns from the point of view of extensible purpose for downstream package tools like fedpkg. Let's use fedpkg as an example in the discussion below.

When running fedpkg, rpkg code reads fedpkg.conf instead of rpkg.conf, so git_excludes has to be duplicated in fedpkg.conf. On the other hand, fedpkg also has some other patterns that could be ignored, for example bodhi.template. To extend the list, fedpkg has to duplicate the list and then add new ones. Once there is any new pattern to be ignored, the duplicaiton must happen again. This may not be a good thing.

The patterns listed in git_excludes are generated or required by the code logic in whatever rpkg or fedpkg, they are not configurable actually. I think defining and extending the pattern list through the pyrpkg.Commands class inheritance would be a better way, e.g.

class Command(object):
    git_exclude_patterns = ['i386/', 'noarch/', 'result_*', ...]

# in fedpkg
class Commands(pyrpkg.Commands):
    pyrpkg.Commands.git_exclude_patterns += ['bodhi.template']

@cqi, maybe I do not understand the whole picture. Do we have a hierarchy/duplicate problem? rpkg configuration here is a fake and serves just for example and reminder (I can delete if needed). Real patterns should be defined directly in rhpkg and fedpkg configuration files. So another PR have to be opened for this in fedpkg just to modify configuration file.

Real patterns should be defined directly in rhpkg and fedpkg configuration files. So another PR have to be opened for this in fedpkg just to modify configuration file.

Will both rhpkg and fedpkg have same git_excludes list as the one added to rpkg.conf? And what about rpkg-tool rfpkg, centpkg, rpkg-client or other package tools built on top of rpkg?

If you intend to allow downstream package tool developers or user him/herself to select which directories and files generated by rpkg should be ignore, not all by default, it makes sense to read the list from config file.

rebased onto 230a5e273ac1b2fe61d075dcadeef226ac24097b

3 years ago

I think we are on the same thought. rpkg processes patterns from rhpkg/fedpkg (or other) config.
Config in this PR serves as an example. I removed its changes not to confuse anybody.
In parallel, I will open PR for fedpkg to present this.

rebased onto 0de15d2e55d9fa0196a2b0e6b02558df6cd49711

3 years ago

This return seems not needed.

One minor nitpick aside, this looks good to me.

rebased onto da320b4

3 years ago

Pull-Request has been merged by onosek

3 years ago