From 8a735301114012783cdc86f6aaa1e19eed13e4c9 Mon Sep 17 00:00:00 2001
From: iamcourtney <cpacheco@redhat.com>
Date: Sep 18 2016 23:59:12 +0000
Subject: [PATCH 1/4] Updating modulemd filtering code and updating test scripts


---

diff --git a/modulemd/__init__.py b/modulemd/__init__.py
index fed4706..6909154 100644
--- a/modulemd/__init__.py
+++ b/modulemd/__init__.py
@@ -41,6 +41,7 @@ Example usage:
 """
 
 import yaml
+import ast
 
 from modulemd.components import ModuleComponents
 from modulemd.content import ModuleContent
@@ -170,9 +171,30 @@ class ModuleMetadata(object):
                         if "multilib" in e:
                             extras["multilib"] = e["multilib"]
                         self.components.rpms.add_package(p, **extras)
+
                 if "filter" in yml["data"]["components"]["rpms"]:
-                    self.components.rpms.filter = \
-                        set(yml["data"]["components"]["rpms"]["filter"])
+                    filtered_packages = dict()
+                    for filt in yml["data"]["components"]["rpms"]["filter"]:
+                        for p, e in yml["data"]["components"]["rpms"]["packages"].items():
+                            if (p == filt) or (filt in filtered_packages): #exclude
+                                continue
+                            if "archs" in yml["data"]["components"]["rpms"]["filter"][filt]:
+                                archs = yml["data"]["components"]["rpms"]["filter"][filt]["archs"]
+                                if archs[0] == ".+":
+                                    archs = ["*"]
+                                    filt = filt + ".+"
+                            else:
+                                archs = ["*"]
+                            filtered_packages[filt] = archs
+                    #Combine like-architectures by switching dict keys with dict values
+                    filtered = dict()
+                    for p in filtered_packages:
+                        for a in filtered_packages[p]:
+                            if a not in filtered:
+                                filtered[a] = [p]
+                            else:
+                                filtered[a].append(p)
+                    self.components.rpms.filter = set([str(filtered).replace("'",'"')])
 
     def dump(self, f):
         """Dumps the metadata into the supplied file.
@@ -261,8 +283,30 @@ class ModuleMetadata(object):
                         data["data"]["components"]["rpms"]["packages"][p] = \
                             extra
                 if self.components.rpms.filter:
+                    filter_list = list(self.components.rpms.filter)[0]
+                    filter_str = ast.literal_eval(filter_list)
+                    if isinstance(filter_str, str):
+                        filter_dict = ast.literal_eval(filter_str[0])
+                    else:
+                        filter_dict = filter_str
+                    filter_dict_reversed = dict()
+                    for a in filter_dict:
+                        for p in filter_dict[a]:
+                            arches = dict()
+                            if p not in filter_dict_reversed:
+                                if ".+" in p:
+                                    (name, empty) = p.split(".+")
+                                    arches["archs"] = [".+"]
+                                    filter_dict_reversed[name] = arches
+                                else:
+                                    arches["archs"] = [a]
+                                    filter_dict_reversed[p] = arches
+                            else:
+                                filter_dict_reversed[p]["archs"].append(a)
                     data["data"]["components"]["rpms"]["filter"] = \
-                        list(self.components.rpms.filter)
+                        filter_dict_reversed
+                    
+
         return yaml.safe_dump(data)
 
     def validate(self):
diff --git a/tests/test_basic.py b/tests/test_basic.py
index e9ad61c..fea7f3f 100755
--- a/tests/test_basic.py
+++ b/tests/test_basic.py
@@ -59,7 +59,7 @@ class TestBasic(unittest.TestCase):
         cls.mmd.components.rpms.dependencies = True
         cls.mmd.components.rpms.api = set([ "api" ])
         cls.mmd.components.rpms.packages = { "rpm" : { "rationale" : "" } }
-        cls.mmd.components.rpms.filter = set([ "filter_1", "filter_2" ])
+        cls.mmd.components.rpms.filter = set(['{"x86_64": ["bar"], "*": ["bar-doc2"], "ppc64": ["bar-doc", "foo-doc"], "ppc64le": ["foo-doc"]}'])
 
     def test_mdversion(self):
         self.assertIn(self.mmd.mdversion, modulemd.supported_mdversions)
@@ -126,7 +126,7 @@ class TestBasic(unittest.TestCase):
         self.assertEqual(self.mmd.components.rpms.packages, { "rpm" : { "rationale" : "" } })
 
     def test_rpm_filter(self):
-        self.assertEqual(self.mmd.components.rpms.filter, set([ "filter_1", "filter_2" ]))
+        self.assertEqual(self.mmd.components.rpms.filter, set(['{"x86_64": ["bar"], "*": ["bar-doc2"], "ppc64": ["bar-doc", "foo-doc"], "ppc64le": ["foo-doc"]}']))
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_io.py b/tests/test_io.py
index 887d0b8..d82deeb 100755
--- a/tests/test_io.py
+++ b/tests/test_io.py
@@ -68,7 +68,7 @@ class TestIO(unittest.TestCase):
                           "repository" : "golf",
                           "cache" : "hotel" } } )
         self.assertEqual(mmd.components.rpms.filter,
-            set([ "filter_1", "filter_2" ]))
+            set(['{"x86_64": ["beta", "alfa"], "*": ["epsilon.+", "delta"], "ppc64": ["alfa"]}']))
 
     def test_loads(self, yaml=None):
         mmd = modulemd.ModuleMetadata()
@@ -120,8 +120,9 @@ class TestIO(unittest.TestCase):
                                 repository: golf
                                 cache: hotel
                         filter:
-                               - filter_1
-                               - filter_2
+                            delta:
+                                 archs:
+                                      - x86_64
         """
         if not yaml:
             yaml = document
@@ -157,7 +158,7 @@ class TestIO(unittest.TestCase):
                           "repository" : "golf",
                           "cache" : "hotel" } } )
         self.assertEqual(mmd.components.rpms.filter,
-            set([ "filter_1", "filter_2" ]))
+            set(['{"x86_64": ["delta"]}']))
 
     def test_dump(self):
         mmd = modulemd.ModuleMetadata()
@@ -189,8 +190,7 @@ class TestIO(unittest.TestCase):
         mmd.components.rpms.add_package("bravo", rationale="bravo rationale",
             arches=["charlie", "delta"], multilib=["echo"],
             commit="foxtrot", repository="golf", cache="hotel")
-        mmd.components.rpms.add_filter("filter_1")
-        mmd.components.rpms.add_filter("filter_2")
+        mmd.components.rpms.add_filter('{"x86_64": ["beta", "alfa"], "*": ["epsilon.+", "delta"], "ppc64": ["alfa"]}')
         mmd.dump("tests/dump.yaml")
         self.test_load(filename="tests/dump.yaml")
 
@@ -224,8 +224,7 @@ class TestIO(unittest.TestCase):
         mmd.components.rpms.add_package("bravo", rationale="bravo rationale",
             arches=["charlie", "delta"], multilib=["echo"],
             commit="foxtrot", repository="golf", cache="hotel")
-        mmd.components.rpms.add_filter("filter_1")
-        mmd.components.rpms.add_filter("filter_2")
+        mmd.components.rpms.add_filter('{"x86_64": ["delta"]}')
         self.test_loads(yaml=mmd.dumps())
 
 if __name__ == "__main__":
diff --git a/tests/test_rpms.py b/tests/test_rpms.py
index 6801194..3788e21 100755
--- a/tests/test_rpms.py
+++ b/tests/test_rpms.py
@@ -116,17 +116,17 @@ class TestRPMs(unittest.TestCase):
         self.assertEqual(self.mr.api, set([]))
 
     def test_add_filter(self):
-        self.assertNotIn("AddRPMAPI", self.mr.filter)
-        self.mr.add_filter("AddRPMAPI")
-        self.assertIn("AddRPMAPI", self.mr.filter)
+        self.assertNotIn('{ "*": [foo] }', self.mr.filter)
+        self.mr.add_filter('{ "*": [foo] }')
+        self.assertIn('{ "*": [foo] }', self.mr.filter)
 
     def test_del_filter(self):
-        self.mr.filter = set(["DelRPMAPI"])
-        self.mr.del_filter("DelRPMAPI")
-        self.assertNotIn("DelRPMAPI", self.mr.filter)
+        self.mr.filter = set('{ "*": [bar] }')
+        self.mr.del_filter('{ "*": [bar] }')
+        self.assertNotIn('{ "*": [bar] }', self.mr.filter)
 
     def test_clear_filter(self):
-        self.mr.filter = set(["ClearRPMAPI"])
+        self.mr.filter = set(['{ "*": [foo] }', '{ "*": [bar] }'])
         self.mr.clear_filter()
         self.assertEqual(self.mr.filter, set([]))
 

From 88dbecf56b02c703fa22674c0712ff471f2406b3 Mon Sep 17 00:00:00 2001
From: iamcourtney <cpacheco@redhat.com>
Date: Sep 19 2016 07:42:34 +0000
Subject: [PATCH 2/4] Fixing typos: archs --> arches


---

diff --git a/modulemd/__init__.py b/modulemd/__init__.py
index 6909154..b207820 100644
--- a/modulemd/__init__.py
+++ b/modulemd/__init__.py
@@ -178,14 +178,14 @@ class ModuleMetadata(object):
                         for p, e in yml["data"]["components"]["rpms"]["packages"].items():
                             if (p == filt) or (filt in filtered_packages): #exclude
                                 continue
-                            if "archs" in yml["data"]["components"]["rpms"]["filter"][filt]:
-                                archs = yml["data"]["components"]["rpms"]["filter"][filt]["archs"]
-                                if archs[0] == ".+":
-                                    archs = ["*"]
+                            if "arches" in yml["data"]["components"]["rpms"]["filter"][filt]:
+                                arches = yml["data"]["components"]["rpms"]["filter"][filt]["arches"]
+                                if arches[0] == ".+":
+                                    arches = ["*"]
                                     filt = filt + ".+"
                             else:
-                                archs = ["*"]
-                            filtered_packages[filt] = archs
+                                arches = ["*"]
+                            filtered_packages[filt] = arches
                     #Combine like-architectures by switching dict keys with dict values
                     filtered = dict()
                     for p in filtered_packages:
@@ -296,13 +296,13 @@ class ModuleMetadata(object):
                             if p not in filter_dict_reversed:
                                 if ".+" in p:
                                     (name, empty) = p.split(".+")
-                                    arches["archs"] = [".+"]
+                                    arches["arches"] = [".+"]
                                     filter_dict_reversed[name] = arches
                                 else:
-                                    arches["archs"] = [a]
+                                    arches["arches"] = [a]
                                     filter_dict_reversed[p] = arches
                             else:
-                                filter_dict_reversed[p]["archs"].append(a)
+                                filter_dict_reversed[p]["arches"].append(a)
                     data["data"]["components"]["rpms"]["filter"] = \
                         filter_dict_reversed
                     
diff --git a/tests/test.yaml b/tests/test.yaml
index eaf4176..e42056f 100644
--- a/tests/test.yaml
+++ b/tests/test.yaml
@@ -40,8 +40,19 @@ data:
                                 - alfa
                                 - alfa-extras
                         filter:
-                                - filter_1
-                                - filter_2
+                                alfa:
+                                        arches:
+                                                - ppc64
+                                                - x86_64
+                                beta:
+                                        arches:
+                                                - x86_64
+                                delta:
+                                        arches:
+                                                - "*"
+                                epsilon:
+                                        arches:
+                                                - ".+"
                         packages:
                                 alfa:
                                         rationale: alfa rationale
diff --git a/tests/test_io.py b/tests/test_io.py
index d82deeb..474d356 100755
--- a/tests/test_io.py
+++ b/tests/test_io.py
@@ -121,7 +121,7 @@ class TestIO(unittest.TestCase):
                                 cache: hotel
                         filter:
                             delta:
-                                 archs:
+                                 arches:
                                       - x86_64
         """
         if not yaml:

From aef85a5d2dcf2edb950ecc3b02f5298908d03cd2 Mon Sep 17 00:00:00 2001
From: iamcourtney <cpacheco@redhat.com>
Date: Sep 19 2016 10:04:57 +0000
Subject: [PATCH 3/4] Fixing bug with filtering


---

diff --git a/modulemd/__init__.py b/modulemd/__init__.py
index b207820..18d9102 100644
--- a/modulemd/__init__.py
+++ b/modulemd/__init__.py
@@ -176,7 +176,9 @@ class ModuleMetadata(object):
                     filtered_packages = dict()
                     for filt in yml["data"]["components"]["rpms"]["filter"]:
                         for p, e in yml["data"]["components"]["rpms"]["packages"].items():
-                            if (p == filt) or (filt in filtered_packages): #exclude
+                            if ((p == filt) or 
+                                (filt in filtered_packages) or 
+                                (filt in yml["data"]["components"]["rpms"]["packages"])): #exclude
                                 continue
                             if "arches" in yml["data"]["components"]["rpms"]["filter"][filt]:
                                 arches = yml["data"]["components"]["rpms"]["filter"][filt]["arches"]
diff --git a/tests/test_io.py b/tests/test_io.py
index 474d356..612e4a0 100755
--- a/tests/test_io.py
+++ b/tests/test_io.py
@@ -68,7 +68,7 @@ class TestIO(unittest.TestCase):
                           "repository" : "golf",
                           "cache" : "hotel" } } )
         self.assertEqual(mmd.components.rpms.filter,
-            set(['{"x86_64": ["beta", "alfa"], "*": ["epsilon.+", "delta"], "ppc64": ["alfa"]}']))
+            set(['{"x86_64": ["beta"], "*": ["epsilon.+", "delta"]}']))
 
     def test_loads(self, yaml=None):
         mmd = modulemd.ModuleMetadata()
@@ -190,7 +190,7 @@ class TestIO(unittest.TestCase):
         mmd.components.rpms.add_package("bravo", rationale="bravo rationale",
             arches=["charlie", "delta"], multilib=["echo"],
             commit="foxtrot", repository="golf", cache="hotel")
-        mmd.components.rpms.add_filter('{"x86_64": ["beta", "alfa"], "*": ["epsilon.+", "delta"], "ppc64": ["alfa"]}')
+        mmd.components.rpms.add_filter('{"x86_64": ["beta"], "*": ["epsilon.+", "delta"]}')
         mmd.dump("tests/dump.yaml")
         self.test_load(filename="tests/dump.yaml")
 

From c9cdc3edbc916b08081dba1c0d23fa370505e23f Mon Sep 17 00:00:00 2001
From: Courtney Pacheco <cpacheco@redhat.com>
Date: Oct 10 2016 13:54:07 +0000
Subject: [PATCH 4/4] Updating code to follow new coding guidelines


---

diff --git a/modulemd/__init__.py b/modulemd/__init__.py
index 18d9102..6ed88b1 100644
--- a/modulemd/__init__.py
+++ b/modulemd/__init__.py
@@ -172,30 +172,47 @@ class ModuleMetadata(object):
                             extras["multilib"] = e["multilib"]
                         self.components.rpms.add_package(p, **extras)
 
+                #Iterate through the filtered packages (if they exist)
                 if "filter" in yml["data"]["components"]["rpms"]:
-                    filtered_packages = dict()
-                    for filt in yml["data"]["components"]["rpms"]["filter"]:
-                        for p, e in yml["data"]["components"]["rpms"]["packages"].items():
+
+                    #Grab list of RPMs, filters, and packages, and create a dict
+                    #to store all the filtered packages' arches
+                    filtered_packages = {}
+                    rpms_yaml = yml["data"]["components"]["rpms"]
+                    filter_yaml = rpms_yaml["filter"]
+                    packages_yaml = rpms_yaml["packages"]
+
+                    #Iterate through all the filters
+                    for filtkey, filt in  filter_yaml:
+                        for p in packages_yaml:
+
+                            #Exclude this package
                             if ((p == filt) or 
-                                (filt in filtered_packages) or 
-                                (filt in yml["data"]["components"]["rpms"]["packages"])): #exclude
+                                (filtkey in filtered_packages) or 
+                                (filt in packages_yaml)):
                                 continue
-                            if "arches" in yml["data"]["components"]["rpms"]["filter"][filt]:
-                                arches = yml["data"]["components"]["rpms"]["filter"][filt]["arches"]
+
+                            arches = filt.get("arches")
+                            
+                            #This case is for excluding secondary architectures
+                            if arches:
                                 if arches[0] == ".+":
                                     arches = ["*"]
                                     filt = filt + ".+"
-                            else:
+                            else: #Otherwise, exclude all package architectures
                                 arches = ["*"]
                             filtered_packages[filt] = arches
+
                     #Combine like-architectures by switching dict keys with dict values
-                    filtered = dict()
-                    for p in filtered_packages:
-                        for a in filtered_packages[p]:
+                    filtered = {}
+                    for pkey, pvalue in filtered_packages:
+                        for a in pvalue:
                             if a not in filtered:
-                                filtered[a] = [p]
+                                filtered[a] = [pkey]
                             else:
-                                filtered[a].append(p)
+                                filtered[a].append(pkey)
+                    
+                    #This is for json formatting. (Python is picky about ' vs ")
                     self.components.rpms.filter = set([str(filtered).replace("'",'"')])
 
     def dump(self, f):

