From f47a938a0a23e198320f14d1665d7f530f25df55 Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Nov 28 2017 18:52:58 +0000 Subject: [PATCH 1/3] Use yaml.safe_dump_all in dumps_all() To match the usage of yaml.safe_dump in dumps() --- diff --git a/modulemd/__init__.py b/modulemd/__init__.py index 6c92b7e..86d43f1 100644 --- a/modulemd/__init__.py +++ b/modulemd/__init__.py @@ -98,7 +98,7 @@ def dumps_all(l): :param list l: List of ModuleMetadata instances """ - return yaml.dump_all([x.dumpd() for x in l], explicit_start=True) + return yaml.safe_dump_all([x.dumpd() for x in l], explicit_start=True) class ModuleMetadata(object): """Class representing the whole module.""" From 1a7b656e87b168dda74ceecc88ef81ca8ecb55f7 Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Nov 28 2017 18:57:37 +0000 Subject: [PATCH 2/3] Add internal method to dump into an OrderedDict ...rather than a plain unordered dict(). This is a no-op for now but will be used in upcoming changes. --- diff --git a/modulemd/__init__.py b/modulemd/__init__.py index 86d43f1..2620a46 100644 --- a/modulemd/__init__.py +++ b/modulemd/__init__.py @@ -41,6 +41,7 @@ Example usage: """ import sys +from collections import OrderedDict import datetime import dateutil.parser import yaml @@ -341,17 +342,17 @@ class ModuleMetadata(object): with open(f, "w") as outfile: outfile.write(data) - def dumpd(self): - """Dumps the metadata into a dictionary. + def _dumpd_ordered(self): + """Dumps the metadata into a OrderedDict. - :rtype: dict + :rtype: collections.OrderedDict """ - doc = dict() + doc = OrderedDict() # header doc["document"] = "modulemd" doc["version"] = self.mdversion # data - d = dict() + d = OrderedDict() if self.name: d["name"] = self.name if self.stream: @@ -366,18 +367,18 @@ class ModuleMetadata(object): d["description"] = self.description if self.eol: d["eol"] = str(self.eol) - d["license"] = dict() + d["license"] = OrderedDict() d["license"]["module"] = sorted(list(self.module_licenses)) if self.content_licenses: d["license"]["content"] = sorted(list(self.content_licenses)) if self.buildrequires or self.requires: - d["dependencies"] = dict() + d["dependencies"] = OrderedDict() if self.buildrequires: d["dependencies"]["buildrequires"] = self.buildrequires if self.requires: d["dependencies"]["requires"] = self.requires if self.community or self.documentation or self.tracker: - d["references"] = dict() + d["references"] = OrderedDict() if self.community: d["references"]["community"] = self.community if self.documentation: @@ -387,39 +388,39 @@ class ModuleMetadata(object): if self.xmd: d["xmd"] = self.xmd if self.profiles: - d["profiles"] = dict() + d["profiles"] = OrderedDict() for profile in self.profiles.keys(): if self.profiles[profile].description: if profile not in d["profiles"]: - d["profiles"][profile] = dict() + d["profiles"][profile] = OrderedDict() d["profiles"][profile]["description"] = \ str(self.profiles[profile].description) if self.profiles[profile].rpms: if profile not in d["profiles"]: - d["profiles"][profile] = dict() + d["profiles"][profile] = OrderedDict() d["profiles"][profile]["rpms"] = \ sorted(list(self.profiles[profile].rpms)) if self.api: - d["api"] = dict() + d["api"] = OrderedDict() if self.api.rpms: d["api"]["rpms"] = sorted(list(self.api.rpms)) if self.filter: - d["filter"] = dict() + d["filter"] = OrderedDict() if self.filter.rpms: d["filter"]["rpms"] = sorted(list(self.filter.rpms)) if self.buildopts: - d["buildopts"] = dict() + d["buildopts"] = OrderedDict() if self.buildopts.rpms: - d["buildopts"]["rpms"] = dict() + d["buildopts"]["rpms"] = OrderedDict() if self.buildopts.rpms.macros: d["buildopts"]["rpms"]["macros"] = \ self.buildopts.rpms.macros if self.components: - d["components"] = dict() + d["components"] = OrderedDict() if self.components.rpms: - d["components"]["rpms"] = dict() + d["components"]["rpms"] = OrderedDict() for p in self.components.rpms.values(): - extra = dict() + extra = OrderedDict() extra["rationale"] = p.rationale if p.buildorder: extra["buildorder"] = p.buildorder @@ -435,9 +436,9 @@ class ModuleMetadata(object): extra["multilib"] = sorted(list(p.multilib)) d["components"]["rpms"][p.name] = extra if self.components.modules: - d["components"]["modules"] = dict() + d["components"]["modules"] = OrderedDict() for p in self.components.modules.values(): - extra = dict() + extra = OrderedDict() extra["rationale"] = p.rationale if p.buildorder: extra["buildorder"] = p.buildorder @@ -447,12 +448,34 @@ class ModuleMetadata(object): extra["ref"] = p.ref d["components"]["modules"][p.name] = extra if self.artifacts: - d["artifacts"] = dict() + d["artifacts"] = OrderedDict() if self.artifacts.rpms: d["artifacts"]["rpms"] = sorted(list(self.artifacts.rpms)) doc["data"] = d return doc + def dumpd(self): + """Dumps the metadata into a dictionary. + + :rtype: dict + """ + def _convert_ordered(orig, new): + """Recurse over a nested OrderedDict, converting each to + a dict() + """ + for key, val in orig.items(): + if not isinstance(val, OrderedDict): + new[key] = val + continue + + new[key] = dict() + _convert_ordered(val, new[key]) + + ordered = self._dumpd_ordered() + converted = dict() + _convert_ordered(ordered, converted) + return converted + def dumps(self): """Dumps the metadata into a string. From 362ffeb159728d72000f9eb2b29a9c6844097374 Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Nov 28 2017 19:01:01 +0000 Subject: [PATCH 3/3] Preserve modulemd field ordering in dumps/dumps_all Right now, serializing a modulemd object to string with dumps() produces alphabetic ordering in the yaml document, which doesn't match expected modulemd conventions. For example, fedmod rpm2module will put document: and version: tags at the bottom of the output Internally modulemd already populates the dumped dict() in the appropriate order, but dict() doesn't preserve ordering, and yaml sorts the fields anyways. Instead, pass an OrderedDict to yaml, and add a custom representer to teach yaml how to handle it. --- diff --git a/modulemd/__init__.py b/modulemd/__init__.py index 2620a46..1d2c31e 100644 --- a/modulemd/__init__.py +++ b/modulemd/__init__.py @@ -40,8 +40,8 @@ Example usage: mmd.dump("out.yaml") """ -import sys from collections import OrderedDict +import sys import datetime import dateutil.parser import yaml @@ -61,6 +61,22 @@ from modulemd.profile import ModuleProfile supported_mdversions = ( 1, ) +# From https://stackoverflow.com/a/16782282 +# Enable yaml handling of OrderedDict, rather than serialize +# dict values alphabetically +def _represent_ordereddict(dumper, data): + value = [] + + for item_key, item_value in data.items(): + node_key = dumper.represent_data(item_key) + node_value = dumper.represent_data(item_value) + + value.append((node_key, node_value)) + + return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) +yaml.representer.SafeRepresenter.add_representer(OrderedDict, + _represent_ordereddict) + def load_all(f): """Loads a metadata file containing multiple modulemd documents into a list of ModuleMetadata instances. @@ -99,7 +115,8 @@ def dumps_all(l): :param list l: List of ModuleMetadata instances """ - return yaml.safe_dump_all([x.dumpd() for x in l], explicit_start=True) + return yaml.safe_dump_all([x._dumpd_ordered() for x in l], + explicit_start=True) class ModuleMetadata(object): """Class representing the whole module.""" @@ -481,7 +498,7 @@ class ModuleMetadata(object): :rtype: str """ - return yaml.safe_dump(self.dumpd(), default_flow_style=False) + return yaml.safe_dump(self._dumpd_ordered(), default_flow_style=False) @property def mdversion(self):