rpkg

Clone

#24 If no args specified for new-sources or upload cmds, upload all binary sources found in project folder
Opened 8 years ago by dsilakov. Modified 7 years ago
dsilakov/rpkg master  into  master

Merge
file modified
+37 -1 
@@ -69,7 +69,8 @@ 
 

      # This shouldn't change... often
 

      UPLOADEXTS = ['tar', 'gz', 'bz2', 'lzma', 'xz', 'Z', 'zip', 'tff',
 

                    'bin', 'tbz', 'tbz2', 'tgz', 'tlz', 'txz', 'pdf', 'rpm',
 

-                   'jar', 'war', 'db', 'cpio', 'jisp', 'egg', 'gem', 'spkg']
 

+                   'jar', 'war', 'db', 'cpio', 'jisp', 'egg', 'gem', 'spkg',
 

+                   'oxt', 'xpi']
 

  

      def __init__(self, path, lookaside, lookasidehash, lookaside_cgi,
 

                   gitbaseurl, anongiturl, branchre, kojiconfig,
 
@@ -2194,6 +2195,40 @@ 
 

                             config_dir)
 

              self._cleanup_tmp_dir(config_dir)
 

  

+     def choose_uploads(self):
 

+         """
 

+         Choose files from the project folder that should be uploaded
 

+         to lookaside cache - parse Source fields from the spec file
 

+         and choose existing binary files from that set
 

+         """
 

+         ts = rpm.TransactionSet()
 

+         try:
 

+             sources_all = ts.parseSpec(os.path.join(self.path, self.spec)).sources
 

+         except:
 

+             raise rpkgError("Error parsing spec file %s" % self.spec)
 

+ 

+         upload_files = []
 

+ 

+         # A flag distinguishing "Source" from "Patch"
 

+         src_flag = 1
 

+ 

+         for src in sources_all:
 

+             name = os.path.basename(src[0])
 

+             # Process only source file that exist in the project folder
 

+             if not os.path.isfile(os.path.join(self.path, name)):
 

+                 continue
 

+             if not src[2] & src_flag:
 

+                 continue
 

+             # Upload file if its extension is in UPLOADEXTS
 

+             if not name.rsplit('.')[-1] in self.UPLOADEXTS:
 

+                 continue
 

+             upload_files.append(name)
 

+ 

+         if not upload_files:
 

+             raise rpkgError("Didn't find any files to be uploaded!")
 

+ 

+         return upload_files
 

+ 

      def upload(self, files, replace=False):
 

          """Upload source file(s) in the lookaside cache
 

  
@@ -2466,3 +2501,4 @@ 
 

                            ' %s file', self.osbs_config_filename)
 

          else:
 

              self.log.info('Nothing to be done')
 

+

file modified
+17 -8 
@@ -613,8 +613,10 @@ 
 

              description='This will upload new source files to the lookaside '
 

                          'cache and remove any existing ones. The "sources" '
 

                          'and .gitignore files will be updated with the new '
 

-                         'uploaded file(s).')
 

-         self.new_sources_parser.add_argument('files', nargs='+')
 

+                         'uploaded file(s). If no arguments are provided, all '
 

+                         'binary files from the project folder that are '
 

+                         'mentioned in the spec file will be uploaded.')
 

+         self.new_sources_parser.add_argument('files', nargs='*')
 

          self.new_sources_parser.set_defaults(
 

              command=self.new_sources, replace=True)
 

  
@@ -780,7 +782,10 @@ 
 

              conflict_handler='resolve', help='Upload source files',
 

              description='This command will add a new source archive to the '
 

                          'lookaside cache. The sources and .gitignore file '
 

-                         'will be updated with the new file(s).')
 

+                         'will be updated with the new file(s). If no '
 

+                         'arguments are provided, all binary files from the '
 

+                         'project folder that are mentioned in the spec file '
 

+                         'will be uploaded.')
 

          upload_parser.set_defaults(command=self.new_sources, replace=False)
 

  

      def register_verify_files(self):
 
@@ -1156,11 +1161,15 @@ 
 

          print(self.cmd.new())
 

  

      def new_sources(self):
 

-         # Check to see if the files passed exist
 

-         for file in self.args.files:
 

-             if not os.path.isfile(file):
 

-                 raise Exception('Path does not exist or is '
 

-                                 'not a file: %s' % file)
 

+         # If no files were passed, upload all binaries mentioned in spec
 

+         if not self.args.files:
 

+             self.args.files = self.cmd.choose_uploads()
 

+         else:
 

+             # Check to see if the files passed exist
 

+             for file in self.args.files:
 

+                 if not os.path.isfile(file):
 

+                     raise Exception('Path does not exist or is '
 

+                                     'not a file: %s' % file)
 

          self.cmd.upload(self.args.files, replace=self.args.replace)
 

          self.log.info("Source upload succeeded. Don't forget to commit the "
 

                        "sources file")

file added
+53 
@@ -0,0 +1,53 @@ 
 

+ import os
 

+ import shutil
 

+ import tempfile
 

+ 

+ from . import CommandTestCase
 

+ 

+ 

+ class CommandUploadTestCase(CommandTestCase):
 

+     def test_choose_upload(self):
 

+         self.make_new_git(self.module)
 

+ 

+         import pyrpkg
 

+         cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash,
 

+                               self.lookaside_cgi, self.gitbaseurl,
 

+                               self.anongiturl, self.branchre, self.kojiconfig,
 

+                               self.build_client, self.user, self.dist,
 

+                               self.target, self.quiet)
 

+         cmd.clone(self.module, anon=True)
 

+ 

+         moduledir = os.path.join(self.path, self.module)
 

+         self.assertTrue(os.path.isdir(os.path.join(moduledir, '.git')))
 

+         cmd.path = moduledir
 

+ 

+         # Create a dummy (but valid) spec file which has two sources:
 

+         # * test-1.0.tar.gz (in the form of '%{name}-%{version}')
 

+         # * valid_source.tar.gz
 

+         test_spec = os.path.join(moduledir, 'test.spec')
 

+         f = open(test_spec, "w")
 

+         f.write("Name: test\nVersion: 1.0\nRelease: 1\nSummary: test\n")
 

+         f.write("License: test\nGroup: test\nURL: test\n")
 

+         f.write("Source0: %{name}-%{version}.tar.gz\n")
 

+         f.write("Source1: valid_source.tar.gz\n")
 

+         f.write("%description\ntest\n%prep\n%setup -q\n%build\n\n%files\n")
 

+         f.close()
 

+         try:
 

+             owd = os.getcwd()
 

+         except:
 

+             owd = "."
 

+         os.chdir(moduledir)
 

+         os.system("git add test.spec")
 

+         # Create 3 tar balls: test-1.0, valid_source and wrong_file
 

+         os.system("tar cf test-1.0.tar.gz test.spec")
 

+         os.system("tar cf valid_source.tar.gz test.spec")
 

+         os.system("tar cf wrong_file.tar.gz test.spec")
 

+         cmd.commit(message="Test_commit")
 

+         # We expect that rpkg will pick up only test-1.0.tar.gz and
 

+         # valid_source.tar.gz for upload
 

+         files = cmd.choose_uploads()
 

+         self.assertTrue("test-1.0.tar.gz" in files)
 

+         self.assertTrue("valid_source.tar.gz" in files)
 

+         self.assertFalse("wrong_file.tar.gz" in files)
 

+         os.chdir(owd)
 

+

no initial comment

What problems does this PR aim to fix? Please add some tests.

1 new commit added

  • Added a test to check new choose_upload function
7 years ago

Added a test case that demonstrates expected behavior.

This improvement is targeted for lazy people like me who don't want to manually specify files for upload. I just want the tool to decide which files from the working folder should be uploaded. If file extension is among self.UPLOADEXTS and this file is mentioned in spec file as one of the sources, then let's upload it to the lookaside cache. And ignore files that are not mentioned in spec - it is very unlikely that they are required for the build (this sometimes happens that I have garbage files in the folder - e.g., tarballs of different package versions I was experimenting with).

1 new commit added

  • Catch exceptions in getcwd during test_upload check
7 years ago

rebased
7 years ago

One concern is, "binary sources" is too general, and it might be possible to upload unecessary binary files in some cases because we cannot assume what concrete files (not file type) would be generated in package repository. It should be much safer by specifying files to upload explicitly I think.

I agree that "binary sources" is a very general term. However, the patch assumes that "binaries" are files with extensions from UPLOADEXTS array - this approach is already used in rpkg when uploading srpms, see _srpmdetails() function. So with this patch, "upload" and "new_sources" commands get behavior similar to the one already used for importing srpms.

Finally, the old behavior remains in place - one still can explicitly specify files to upload.

rebased onto c97709f
5 years ago

rebased onto 29796aa
5 years ago
Changes Summary 3
+37 -1
file changed
src/pyrpkg/__init__.py
+17 -8
file changed
src/pyrpkg/cli.py
+53
file added
test/commands/test_upload.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.