#1081 Import original source modulemd file as stored in dist-git to CG build.
Merged 8 months ago by jkaluza. Opened 8 months ago by jkaluza.
jkaluza/fm-orchestrator cg-src-modulemd  into  master

@@ -42,6 +42,7 @@ 

  import pungi.arch

  

  from module_build_service import conf, log, build_logs, Modulemd, glib

+ from module_build_service.scm import SCM

  

  logging.basicConfig(level=logging.DEBUG)

  

@@ -366,16 +367,27 @@ 

          # Read the modulemd file to get the filesize/checksum and also

          # parse it to get the Modulemd instance.

          mmd_path = os.path.join(output_path, mmd_filename)

-         with open(mmd_path) as mmd_f:

-             data = mmd_f.read()

-             mmd = Modulemd.Module().new_from_string(data)

-             ret['filename'] = mmd_filename

-             ret['filesize'] = len(data)

-             ret['checksum'] = hashlib.md5(data.encode('utf-8')).hexdigest()

+         try:

+             with open(mmd_path) as mmd_f:

+                 data = mmd_f.read()

+                 mmd = Modulemd.Module().new_from_string(data)

+                 ret['filename'] = mmd_filename

+                 ret['filesize'] = len(data)

+                 ret['checksum'] = hashlib.md5(data.encode('utf-8')).hexdigest()

+         except IOError:

+             if arch == "src":

+                 # This might happen in case the Module is submitted directly

+                 # using the yaml without SCM URL. This should never happen

+                 # when building production-ready modules using Koji, but in

+                 # theory it is possible.

+                 log.warn("No modulemd.src.txt found.")

+                 return

+             else:

+                 raise

  

          components = []

-         if arch == "noarch":

-             # For generic noarch modulemd, include all the RPMs.

+         if arch in ["noarch", "src"]:

+             # For generic noarch/src modulemd, include all the RPMs.

              for rpm in self.rpms:

                  components.append(

                      self._koji_rpm_to_component_record(rpm))

@@ -397,8 +409,10 @@ 

  

      def _get_output(self, output_path):

          ret = []

-         for arch in self.arches + ["noarch"]:

-             ret.append(self._get_arch_mmd_output(output_path, arch))

+         for arch in self.arches + ["noarch", "src"]:

+             mmd_dict = self._get_arch_mmd_output(output_path, arch)

+             if mmd_dict:

+                 ret.append(mmd_dict)

  

          try:

              log_path = os.path.join(output_path, "build.log")

@@ -613,6 +627,43 @@ 

  

          return unicode(mmd.dumps())

  

+     def _download_source_modulemd(self, mmd, output_path):

+         """

+         Fetches the original source modulemd file from SCM URL stored in the

+         XMD section of `mmd` and stores it to filename referenced by `output_path`.

+ 

+         This method does nothing if SCM URL is not set in the `mmd`.

+ 

+         :param Modulemd mmd: Modulemd instance.

+         :param str output_path: Full path to file into which the original modulemd

+             file will be stored.

+         """

+         xmd = glib.from_variant_dict(mmd.get_xmd())

+         commit = xmd.get("mbs", {}).get("commit")

+         scmurl = xmd.get("mbs", {}).get("scmurl")

+         if not commit or not scmurl:

+             log.warn("%r: xmd['mbs'] does not contain 'commit' or 'scmurl'.", self.module)

+             return

+ 

+         td = None

+         try:

+             log.info("Fetching %s (%s) to get the source modulemd.yaml", scmurl, commit)

+             td = tempfile.mkdtemp()

+             scm = SCM(scmurl)

+             scm.commit = commit

+             scm.checkout(td)

+             fn = scm.get_module_yaml()

+             log.info("Writing source modulemd.yaml to %r" % output_path)

+             shutil.copy(fn, output_path)

+         finally:

+             try:

+                 if td is not None:

+                     shutil.rmtree(td)

+             except Exception as e:

+                 log.warn(

+                     "Failed to remove temporary directory {!r}: {}".format(

+                         td, str(e)))

+ 

      def _prepare_file_directory(self):

          """ Creates a temporary directory that will contain all the files

          mentioned in the outputs section

@@ -625,6 +676,9 @@ 

          with open(mmd_path, "w") as mmd_f:

              mmd_f.write(self.mmd)

  

+         mmd_path = os.path.join(prepdir, "modulemd.src.txt")

+         self._download_source_modulemd(self.module.mmd(), mmd_path)

+ 

          for arch in self.arches:

              mmd_path = os.path.join(prepdir, "modulemd.%s.txt" % arch)

              log.info("Writing %s modulemd.yaml to %r" % (arch, mmd_path))

@@ -35,6 +35,7 @@ 

  import kobo.rpmlib

  

  from tests import init_data

+ from tests.test_views.test_views import FakeSCM

  

  from module_build_service.builder.KojiContentGenerator import KojiContentGenerator

  

@@ -630,3 +631,16 @@ 

              assert pkg.get_cache() is None

  

          assert "mbs" not in mmd.get_xmd().keys()

+ 

+     @patch('module_build_service.builder.KojiContentGenerator.SCM')

+     def test_prepare_file_directory_modulemd_src(self, mocked_scm):

+         FakeSCM(mocked_scm, 'testmodule', 'testmodule_init.yaml',

+                 '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

+         mmd = self.cg.module.mmd()

+         mmd.set_xmd(glib.dict_values({"mbs": {

+             "commit": "foo",

+             "scmurl": "git://localhost/modules/foo.git#master"}}))

+         self.cg.module.modulemd = mmd.dumps()

+         file_dir = self.cg._prepare_file_directory()

+         with open(path.join(file_dir, "modulemd.src.txt")) as mmd:

+             assert len(mmd.read()) == 1337

There is a need to ship the unchanged source file which was used to build
a module build from legal reasons. The KojiContentGenerator is extended
in this commit to fetch it from SCM URL using the scm module and later
attach it as modulemd.src.txt to Koji CG build.

Can this method be renamed to _fetch_and_store_source_modulemd? Because it actually does two things, fetch and store the source modulemd. Or, move the store action out of this method?

rebased onto e8832cb3d4aa764a99cfd5ee47a28d6b50f79ded

8 months ago

rebased onto 8feea0a6f16bc9eff5e719afe3c21ffe47c3b836

8 months ago

@cqi: I've renamed that method to _download_source_modulemd and renamed mmd_path arg to output_path. I think it makes it clearer that the file is really stored locally.

Moving storing to different method would make things more complex, because we would need to read the downloaded file (it is stored on filesystem by the SCM class) and then write it elsewhere again. Using shutil.copy in the current method is simpler.

@jkaluza this new name is much better :thumbsup:

rebased onto f5e3c81

8 months ago

Rebased, going to merge.

Pull-Request has been merged by jkaluza

8 months ago