From 8a735301114012783cdc86f6aaa1e19eed13e4c9 Mon Sep 17 00:00:00 2001 From: iamcourtney Date: Sep 18 2016 23:59:12 +0000 Subject: [PATCH 1/4] Updating modulemd filtering code and updating test scripts --- diff --git a/modulemd/__init__.py b/modulemd/__init__.py index fed4706..6909154 100644 --- a/modulemd/__init__.py +++ b/modulemd/__init__.py @@ -41,6 +41,7 @@ Example usage: """ import yaml +import ast from modulemd.components import ModuleComponents from modulemd.content import ModuleContent @@ -170,9 +171,30 @@ class ModuleMetadata(object): if "multilib" in e: extras["multilib"] = e["multilib"] self.components.rpms.add_package(p, **extras) + if "filter" in yml["data"]["components"]["rpms"]: - self.components.rpms.filter = \ - set(yml["data"]["components"]["rpms"]["filter"]) + filtered_packages = dict() + for filt in yml["data"]["components"]["rpms"]["filter"]: + for p, e in yml["data"]["components"]["rpms"]["packages"].items(): + if (p == filt) or (filt in filtered_packages): #exclude + continue + if "archs" in yml["data"]["components"]["rpms"]["filter"][filt]: + archs = yml["data"]["components"]["rpms"]["filter"][filt]["archs"] + if archs[0] == ".+": + archs = ["*"] + filt = filt + ".+" + else: + archs = ["*"] + filtered_packages[filt] = archs + #Combine like-architectures by switching dict keys with dict values + filtered = dict() + for p in filtered_packages: + for a in filtered_packages[p]: + if a not in filtered: + filtered[a] = [p] + else: + filtered[a].append(p) + self.components.rpms.filter = set([str(filtered).replace("'",'"')]) def dump(self, f): """Dumps the metadata into the supplied file. @@ -261,8 +283,30 @@ class ModuleMetadata(object): data["data"]["components"]["rpms"]["packages"][p] = \ extra if self.components.rpms.filter: + filter_list = list(self.components.rpms.filter)[0] + filter_str = ast.literal_eval(filter_list) + if isinstance(filter_str, str): + filter_dict = ast.literal_eval(filter_str[0]) + else: + filter_dict = filter_str + filter_dict_reversed = dict() + for a in filter_dict: + for p in filter_dict[a]: + arches = dict() + if p not in filter_dict_reversed: + if ".+" in p: + (name, empty) = p.split(".+") + arches["archs"] = [".+"] + filter_dict_reversed[name] = arches + else: + arches["archs"] = [a] + filter_dict_reversed[p] = arches + else: + filter_dict_reversed[p]["archs"].append(a) data["data"]["components"]["rpms"]["filter"] = \ - list(self.components.rpms.filter) + filter_dict_reversed + + return yaml.safe_dump(data) def validate(self): diff --git a/tests/test_basic.py b/tests/test_basic.py index e9ad61c..fea7f3f 100755 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -59,7 +59,7 @@ class TestBasic(unittest.TestCase): cls.mmd.components.rpms.dependencies = True cls.mmd.components.rpms.api = set([ "api" ]) cls.mmd.components.rpms.packages = { "rpm" : { "rationale" : "" } } - cls.mmd.components.rpms.filter = set([ "filter_1", "filter_2" ]) + cls.mmd.components.rpms.filter = set(['{"x86_64": ["bar"], "*": ["bar-doc2"], "ppc64": ["bar-doc", "foo-doc"], "ppc64le": ["foo-doc"]}']) def test_mdversion(self): self.assertIn(self.mmd.mdversion, modulemd.supported_mdversions) @@ -126,7 +126,7 @@ class TestBasic(unittest.TestCase): self.assertEqual(self.mmd.components.rpms.packages, { "rpm" : { "rationale" : "" } }) def test_rpm_filter(self): - self.assertEqual(self.mmd.components.rpms.filter, set([ "filter_1", "filter_2" ])) + self.assertEqual(self.mmd.components.rpms.filter, set(['{"x86_64": ["bar"], "*": ["bar-doc2"], "ppc64": ["bar-doc", "foo-doc"], "ppc64le": ["foo-doc"]}'])) if __name__ == "__main__": unittest.main() diff --git a/tests/test_io.py b/tests/test_io.py index 887d0b8..d82deeb 100755 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -68,7 +68,7 @@ class TestIO(unittest.TestCase): "repository" : "golf", "cache" : "hotel" } } ) self.assertEqual(mmd.components.rpms.filter, - set([ "filter_1", "filter_2" ])) + set(['{"x86_64": ["beta", "alfa"], "*": ["epsilon.+", "delta"], "ppc64": ["alfa"]}'])) def test_loads(self, yaml=None): mmd = modulemd.ModuleMetadata() @@ -120,8 +120,9 @@ class TestIO(unittest.TestCase): repository: golf cache: hotel filter: - - filter_1 - - filter_2 + delta: + archs: + - x86_64 """ if not yaml: yaml = document @@ -157,7 +158,7 @@ class TestIO(unittest.TestCase): "repository" : "golf", "cache" : "hotel" } } ) self.assertEqual(mmd.components.rpms.filter, - set([ "filter_1", "filter_2" ])) + set(['{"x86_64": ["delta"]}'])) def test_dump(self): mmd = modulemd.ModuleMetadata() @@ -189,8 +190,7 @@ class TestIO(unittest.TestCase): mmd.components.rpms.add_package("bravo", rationale="bravo rationale", arches=["charlie", "delta"], multilib=["echo"], commit="foxtrot", repository="golf", cache="hotel") - mmd.components.rpms.add_filter("filter_1") - mmd.components.rpms.add_filter("filter_2") + mmd.components.rpms.add_filter('{"x86_64": ["beta", "alfa"], "*": ["epsilon.+", "delta"], "ppc64": ["alfa"]}') mmd.dump("tests/dump.yaml") self.test_load(filename="tests/dump.yaml") @@ -224,8 +224,7 @@ class TestIO(unittest.TestCase): mmd.components.rpms.add_package("bravo", rationale="bravo rationale", arches=["charlie", "delta"], multilib=["echo"], commit="foxtrot", repository="golf", cache="hotel") - mmd.components.rpms.add_filter("filter_1") - mmd.components.rpms.add_filter("filter_2") + mmd.components.rpms.add_filter('{"x86_64": ["delta"]}') self.test_loads(yaml=mmd.dumps()) if __name__ == "__main__": diff --git a/tests/test_rpms.py b/tests/test_rpms.py index 6801194..3788e21 100755 --- a/tests/test_rpms.py +++ b/tests/test_rpms.py @@ -116,17 +116,17 @@ class TestRPMs(unittest.TestCase): self.assertEqual(self.mr.api, set([])) def test_add_filter(self): - self.assertNotIn("AddRPMAPI", self.mr.filter) - self.mr.add_filter("AddRPMAPI") - self.assertIn("AddRPMAPI", self.mr.filter) + self.assertNotIn('{ "*": [foo] }', self.mr.filter) + self.mr.add_filter('{ "*": [foo] }') + self.assertIn('{ "*": [foo] }', self.mr.filter) def test_del_filter(self): - self.mr.filter = set(["DelRPMAPI"]) - self.mr.del_filter("DelRPMAPI") - self.assertNotIn("DelRPMAPI", self.mr.filter) + self.mr.filter = set('{ "*": [bar] }') + self.mr.del_filter('{ "*": [bar] }') + self.assertNotIn('{ "*": [bar] }', self.mr.filter) def test_clear_filter(self): - self.mr.filter = set(["ClearRPMAPI"]) + self.mr.filter = set(['{ "*": [foo] }', '{ "*": [bar] }']) self.mr.clear_filter() self.assertEqual(self.mr.filter, set([])) From 88dbecf56b02c703fa22674c0712ff471f2406b3 Mon Sep 17 00:00:00 2001 From: iamcourtney Date: Sep 19 2016 07:42:34 +0000 Subject: [PATCH 2/4] Fixing typos: archs --> arches --- diff --git a/modulemd/__init__.py b/modulemd/__init__.py index 6909154..b207820 100644 --- a/modulemd/__init__.py +++ b/modulemd/__init__.py @@ -178,14 +178,14 @@ class ModuleMetadata(object): for p, e in yml["data"]["components"]["rpms"]["packages"].items(): if (p == filt) or (filt in filtered_packages): #exclude continue - if "archs" in yml["data"]["components"]["rpms"]["filter"][filt]: - archs = yml["data"]["components"]["rpms"]["filter"][filt]["archs"] - if archs[0] == ".+": - archs = ["*"] + if "arches" in yml["data"]["components"]["rpms"]["filter"][filt]: + arches = yml["data"]["components"]["rpms"]["filter"][filt]["arches"] + if arches[0] == ".+": + arches = ["*"] filt = filt + ".+" else: - archs = ["*"] - filtered_packages[filt] = archs + arches = ["*"] + filtered_packages[filt] = arches #Combine like-architectures by switching dict keys with dict values filtered = dict() for p in filtered_packages: @@ -296,13 +296,13 @@ class ModuleMetadata(object): if p not in filter_dict_reversed: if ".+" in p: (name, empty) = p.split(".+") - arches["archs"] = [".+"] + arches["arches"] = [".+"] filter_dict_reversed[name] = arches else: - arches["archs"] = [a] + arches["arches"] = [a] filter_dict_reversed[p] = arches else: - filter_dict_reversed[p]["archs"].append(a) + filter_dict_reversed[p]["arches"].append(a) data["data"]["components"]["rpms"]["filter"] = \ filter_dict_reversed diff --git a/tests/test.yaml b/tests/test.yaml index eaf4176..e42056f 100644 --- a/tests/test.yaml +++ b/tests/test.yaml @@ -40,8 +40,19 @@ data: - alfa - alfa-extras filter: - - filter_1 - - filter_2 + alfa: + arches: + - ppc64 + - x86_64 + beta: + arches: + - x86_64 + delta: + arches: + - "*" + epsilon: + arches: + - ".+" packages: alfa: rationale: alfa rationale diff --git a/tests/test_io.py b/tests/test_io.py index d82deeb..474d356 100755 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -121,7 +121,7 @@ class TestIO(unittest.TestCase): cache: hotel filter: delta: - archs: + arches: - x86_64 """ if not yaml: From aef85a5d2dcf2edb950ecc3b02f5298908d03cd2 Mon Sep 17 00:00:00 2001 From: iamcourtney Date: Sep 19 2016 10:04:57 +0000 Subject: [PATCH 3/4] Fixing bug with filtering --- diff --git a/modulemd/__init__.py b/modulemd/__init__.py index b207820..18d9102 100644 --- a/modulemd/__init__.py +++ b/modulemd/__init__.py @@ -176,7 +176,9 @@ class ModuleMetadata(object): filtered_packages = dict() for filt in yml["data"]["components"]["rpms"]["filter"]: for p, e in yml["data"]["components"]["rpms"]["packages"].items(): - if (p == filt) or (filt in filtered_packages): #exclude + if ((p == filt) or + (filt in filtered_packages) or + (filt in yml["data"]["components"]["rpms"]["packages"])): #exclude continue if "arches" in yml["data"]["components"]["rpms"]["filter"][filt]: arches = yml["data"]["components"]["rpms"]["filter"][filt]["arches"] diff --git a/tests/test_io.py b/tests/test_io.py index 474d356..612e4a0 100755 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -68,7 +68,7 @@ class TestIO(unittest.TestCase): "repository" : "golf", "cache" : "hotel" } } ) self.assertEqual(mmd.components.rpms.filter, - set(['{"x86_64": ["beta", "alfa"], "*": ["epsilon.+", "delta"], "ppc64": ["alfa"]}'])) + set(['{"x86_64": ["beta"], "*": ["epsilon.+", "delta"]}'])) def test_loads(self, yaml=None): mmd = modulemd.ModuleMetadata() @@ -190,7 +190,7 @@ class TestIO(unittest.TestCase): mmd.components.rpms.add_package("bravo", rationale="bravo rationale", arches=["charlie", "delta"], multilib=["echo"], commit="foxtrot", repository="golf", cache="hotel") - mmd.components.rpms.add_filter('{"x86_64": ["beta", "alfa"], "*": ["epsilon.+", "delta"], "ppc64": ["alfa"]}') + mmd.components.rpms.add_filter('{"x86_64": ["beta"], "*": ["epsilon.+", "delta"]}') mmd.dump("tests/dump.yaml") self.test_load(filename="tests/dump.yaml") From c9cdc3edbc916b08081dba1c0d23fa370505e23f Mon Sep 17 00:00:00 2001 From: Courtney Pacheco Date: Oct 10 2016 13:54:07 +0000 Subject: [PATCH 4/4] Updating code to follow new coding guidelines --- diff --git a/modulemd/__init__.py b/modulemd/__init__.py index 18d9102..6ed88b1 100644 --- a/modulemd/__init__.py +++ b/modulemd/__init__.py @@ -172,30 +172,47 @@ class ModuleMetadata(object): extras["multilib"] = e["multilib"] self.components.rpms.add_package(p, **extras) + #Iterate through the filtered packages (if they exist) if "filter" in yml["data"]["components"]["rpms"]: - filtered_packages = dict() - for filt in yml["data"]["components"]["rpms"]["filter"]: - for p, e in yml["data"]["components"]["rpms"]["packages"].items(): + + #Grab list of RPMs, filters, and packages, and create a dict + #to store all the filtered packages' arches + filtered_packages = {} + rpms_yaml = yml["data"]["components"]["rpms"] + filter_yaml = rpms_yaml["filter"] + packages_yaml = rpms_yaml["packages"] + + #Iterate through all the filters + for filtkey, filt in filter_yaml: + for p in packages_yaml: + + #Exclude this package if ((p == filt) or - (filt in filtered_packages) or - (filt in yml["data"]["components"]["rpms"]["packages"])): #exclude + (filtkey in filtered_packages) or + (filt in packages_yaml)): continue - if "arches" in yml["data"]["components"]["rpms"]["filter"][filt]: - arches = yml["data"]["components"]["rpms"]["filter"][filt]["arches"] + + arches = filt.get("arches") + + #This case is for excluding secondary architectures + if arches: if arches[0] == ".+": arches = ["*"] filt = filt + ".+" - else: + else: #Otherwise, exclude all package architectures arches = ["*"] filtered_packages[filt] = arches + #Combine like-architectures by switching dict keys with dict values - filtered = dict() - for p in filtered_packages: - for a in filtered_packages[p]: + filtered = {} + for pkey, pvalue in filtered_packages: + for a in pvalue: if a not in filtered: - filtered[a] = [p] + filtered[a] = [pkey] else: - filtered[a].append(p) + filtered[a].append(pkey) + + #This is for json formatting. (Python is picky about ' vs ") self.components.rpms.filter = set([str(filtered).replace("'",'"')]) def dump(self, f):