fm-orchestrator

Clone
Source Code
GIT

#1081 Import original source modulemd file as stored in dist-git to CG build.
Merged 5 years ago by jkaluza. Opened 5 years ago by jkaluza.
jkaluza/fm-orchestrator cg-src-modulemd  into  master

file modified
+64 -10 
@@ -42,6 +42,7 @@ 
 

  import pungi.arch
 

  

  from module_build_service import conf, log, build_logs, Modulemd, glib
 

+ from module_build_service.scm import SCM
 

  

  logging.basicConfig(level=logging.DEBUG)
 

  
@@ -366,16 +367,27 @@ 
 

          # Read the modulemd file to get the filesize/checksum and also
 

          # parse it to get the Modulemd instance.
 

          mmd_path = os.path.join(output_path, mmd_filename)
 

-         with open(mmd_path) as mmd_f:
 

-             data = mmd_f.read()
 

-             mmd = Modulemd.Module().new_from_string(data)
 

-             ret['filename'] = mmd_filename
 

-             ret['filesize'] = len(data)
 

-             ret['checksum'] = hashlib.md5(data.encode('utf-8')).hexdigest()
 

+         try:
 

+             with open(mmd_path) as mmd_f:
 

+                 data = mmd_f.read()
 

+                 mmd = Modulemd.Module().new_from_string(data)
 

+                 ret['filename'] = mmd_filename
 

+                 ret['filesize'] = len(data)
 

+                 ret['checksum'] = hashlib.md5(data.encode('utf-8')).hexdigest()
 

+         except IOError:
 

+             if arch == "src":
 

+                 # This might happen in case the Module is submitted directly
 

+                 # using the yaml without SCM URL. This should never happen
 

+                 # when building production-ready modules using Koji, but in
 

+                 # theory it is possible.
 

+                 log.warn("No modulemd.src.txt found.")
 

+                 return
 

+             else:
 

+                 raise
 

  

          components = []
 

-         if arch == "noarch":
 

-             # For generic noarch modulemd, include all the RPMs.
 

+         if arch in ["noarch", "src"]:
 

+             # For generic noarch/src modulemd, include all the RPMs.
 

              for rpm in self.rpms:
 

                  components.append(
 

                      self._koji_rpm_to_component_record(rpm))
 
@@ -397,8 +409,10 @@ 
 

  

      def _get_output(self, output_path):
 

          ret = []
 

-         for arch in self.arches + ["noarch"]:
 

-             ret.append(self._get_arch_mmd_output(output_path, arch))
 

+         for arch in self.arches + ["noarch", "src"]:
 

+             mmd_dict = self._get_arch_mmd_output(output_path, arch)
 

+             if mmd_dict:
 

+                 ret.append(mmd_dict)
 

  

          try:
 

              log_path = os.path.join(output_path, "build.log")
 
@@ -613,6 +627,43 @@ 
 

  

          return unicode(mmd.dumps())
 

  

+     def _download_source_modulemd(self, mmd, output_path):
 

+         """
 

+         Fetches the original source modulemd file from SCM URL stored in the
 

+         XMD section of `mmd` and stores it to filename referenced by `output_path`.
 

+ 

+         This method does nothing if SCM URL is not set in the `mmd`.
 

+ 

+         :param Modulemd mmd: Modulemd instance.
 

+         :param str output_path: Full path to file into which the original modulemd
 

+             file will be stored.
 

+         """
 

+         xmd = glib.from_variant_dict(mmd.get_xmd())
 

+         commit = xmd.get("mbs", {}).get("commit")
 

+         scmurl = xmd.get("mbs", {}).get("scmurl")
 

+         if not commit or not scmurl:
 

+             log.warn("%r: xmd['mbs'] does not contain 'commit' or 'scmurl'.", self.module)
 

+             return
 

+ 

+         td = None
 

+         try:
 

+             log.info("Fetching %s (%s) to get the source modulemd.yaml", scmurl, commit)
 

+             td = tempfile.mkdtemp()
 

+             scm = SCM(scmurl)
 

+             scm.commit = commit
 

+             scm.checkout(td)
 

+             fn = scm.get_module_yaml()
 

+             log.info("Writing source modulemd.yaml to %r" % output_path)
 

+             shutil.copy(fn, output_path)
 

+         finally:
 

+             try:
 

+                 if td is not None:
 

+                     shutil.rmtree(td)
 

+             except Exception as e:
 

+                 log.warn(
 

+                     "Failed to remove temporary directory {!r}: {}".format(
 

+                         td, str(e)))
 

+ 

      def _prepare_file_directory(self):
 

          """ Creates a temporary directory that will contain all the files
 

          mentioned in the outputs section
 
@@ -625,6 +676,9 @@ 
 

          with open(mmd_path, "w") as mmd_f:
 

              mmd_f.write(self.mmd)
 

  

+         mmd_path = os.path.join(prepdir, "modulemd.src.txt")
 

+         self._download_source_modulemd(self.module.mmd(), mmd_path)
 

+ 

          for arch in self.arches:
 

              mmd_path = os.path.join(prepdir, "modulemd.%s.txt" % arch)
 

              log.info("Writing %s modulemd.yaml to %r" % (arch, mmd_path))

file modified
+14 
@@ -35,6 +35,7 @@ 
 

  import kobo.rpmlib
 

  

  from tests import init_data
 

+ from tests.test_views.test_views import FakeSCM
 

  

  from module_build_service.builder.KojiContentGenerator import KojiContentGenerator
 

  
@@ -630,3 +631,16 @@ 
 

              assert pkg.get_cache() is None
 

  

          assert "mbs" not in mmd.get_xmd().keys()
 

+ 

+     @patch('module_build_service.builder.KojiContentGenerator.SCM')
 

+     def test_prepare_file_directory_modulemd_src(self, mocked_scm):
 

+         FakeSCM(mocked_scm, 'testmodule', 'testmodule_init.yaml',
 

+                 '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
 

+         mmd = self.cg.module.mmd()
 

+         mmd.set_xmd(glib.dict_values({"mbs": {
 

+             "commit": "foo",
 

+             "scmurl": "git://localhost/modules/foo.git#master"}}))
 

+         self.cg.module.modulemd = mmd.dumps()
 

+         file_dir = self.cg._prepare_file_directory()
 

+         with open(path.join(file_dir, "modulemd.src.txt")) as mmd:
 

+             assert len(mmd.read()) == 1337

There is a need to ship the unchanged source file which was used to build
a module build from legal reasons. The KojiContentGenerator is extended
in this commit to fetch it from SCM URL using the scm module and later
attach it as modulemd.src.txt to Koji CG build.

Remove this line?

Can this method be renamed to _fetch_and_store_source_modulemd? Because it actually does two things, fetch and store the source modulemd. Or, move the store action out of this method?

rebased onto e8832cb3d4aa764a99cfd5ee47a28d6b50f79ded
5 years ago

rebased onto 8feea0a6f16bc9eff5e719afe3c21ffe47c3b836
5 years ago

@cqi: I've renamed that method to _download_source_modulemd and renamed mmd_path arg to output_path. I think it makes it clearer that the file is really stored locally.

Moving storing to different method would make things more complex, because we would need to read the downloaded file (it is stored on filesystem by the SCM class) and then write it elsewhere again. Using shutil.copy in the current method is simpler.

@jkaluza this new name is much better :thumbsup:

rebased onto f5e3c81
5 years ago

Rebased, going to merge.

Pull-Request has been merged by jkaluza
5 years ago
Changes Summary 2
+64 -10
file changed
module_build_service/builder/KojiContentGenerator.py
+14 -0
file changed
tests/test_content_generator.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.