From 9afb8e6801499e91ee65889a63ae37e9238e0891 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Oct 17 2018 08:14:04 +0000 Subject: fus: Write solvables to file Instead of having one giant command line, write the long list into a file and pass that to the solver. The items on the input list are sorted for easy processing. JIRA: COMPOSE-3012 Signed-off-by: Lubomír Sedlář --- diff --git a/pungi/paths.py b/pungi/paths.py index 47b7129..3d92159 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -127,6 +127,17 @@ class WorkPaths(object): path = os.path.join(path, file_name) return path + def fus_conf(self, arch, variant, iteration, create_dir=True): + """ + Examples: + work/x86_64/fus/Server-solvables.x86_64.conf + """ + file_name = "%s-solvables-%d.%s.conf" % (variant.uid, iteration, arch) + path = os.path.join(self.topdir(arch, create_dir=create_dir), "fus") + if create_dir: + makedirs(path) + return os.path.join(path, file_name) + def pungi_log(self, arch=None, variant=None, create_dir=True, source_name=None): """ Examples: diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index 2bab297..faec83e 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -206,12 +206,13 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): while True: step += 1 + conf_file = self.compose.paths.work.fus_conf(arch, variant, step) + fus.write_config(conf_file, sorted(modules), sorted(input_packages)) cmd = fus.get_cmd( + conf_file, tree_arch_to_yum_arch(arch), repos, pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant), - input_packages, - modules, platform=platform, filter_packages=filter_packages, ) diff --git a/pungi/wrappers/fus.py b/pungi/wrappers/fus.py index 755c944..0c8991f 100644 --- a/pungi/wrappers/fus.py +++ b/pungi/wrappers/fus.py @@ -26,11 +26,10 @@ Pungi). def get_cmd( + conf_file, arch, repos, lookasides, - packages, - modules, platform=None, filter_packages=None, ): @@ -49,14 +48,19 @@ def get_cmd( for pkg in sorted(filter_packages or []): cmd.append("--exclude=%s" % pkg) - for module in modules: - cmd.append("module(%s)" % module) - - cmd.extend(packages) + cmd.append("@%s" % conf_file) return cmd +def write_config(conf_file, modules, packages): + with open(conf_file, "w") as f: + for module in modules: + f.write("module(%s)\n" % module) + for pkg in packages: + f.write("%s\n" % pkg) + + def _prep_path(path): """Strip file:// from the path if present or raise exception for other protocols. diff --git a/tests/helpers.py b/tests/helpers.py index be4a6bd..5537ce9 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -32,6 +32,14 @@ class BaseTestCase(unittest.TestCase): fromfile='EXPECTED', tofile='ACTUAL')) self.assertEqual(diff, '', 'Files differ:\n' + diff) + def assertFileContent(self, fn, expected): + with open(fn, 'rb') as f: + lines = f.read().decode('utf-8').splitlines() + diff = '\n'.join(difflib.unified_diff( + lines, expected.splitlines(), fromfile='EXPECTED', tofile='ACTUAL') + ) + self.assertEqual(diff, '', 'Files differ:\n' + diff) + class PungiTestCase(BaseTestCase): def setUp(self): diff --git a/tests/test_fus_wrapper.py b/tests/test_fus_wrapper.py index 81a754a..3555a4e 100644 --- a/tests/test_fus_wrapper.py +++ b/tests/test_fus_wrapper.py @@ -5,6 +5,7 @@ try: except ImportError: import unittest import tempfile +from textwrap import dedent import os import sys @@ -13,21 +14,20 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi.wrappers import fus -from .helpers import touch +from .helpers import touch, PungiTestCase class TestGetCmd(unittest.TestCase): def test_minimum_command(self): - cmd = fus.get_cmd("x86_64", [], [], [], [], []) - self.assertEqual(cmd, ["fus", "--verbose", "--arch", "x86_64"]) + cmd = fus.get_cmd("conf", "x86_64", repos=[], lookasides=[]) + self.assertEqual(cmd, ["fus", "--verbose", "--arch", "x86_64", "@conf"]) def test_full_command(self): cmd = fus.get_cmd( + "conf", "x86_64", ["/tmp/first", "/tmp/second"], ["/tmp/fst", "/tmp/snd"], - ["pkg"], - ["mod:1.0"], platform="f29", filter_packages=["foo", "bar"], ) @@ -45,31 +45,62 @@ class TestGetCmd(unittest.TestCase): "--platform=f29", "--exclude=bar", "--exclude=foo", - "module(mod:1.0)", - "pkg", + "@conf", ], ) def test_strip_file_protocol(self): - cmd = fus.get_cmd("x86_64", ["file:///tmp"], [], [], [], []) + cmd = fus.get_cmd("conf", "x86_64", ["file:///tmp"], []) self.assertEqual( - cmd, ["fus", "--verbose", "--arch", "x86_64", "--repo=repo-0,repo,/tmp"] + cmd, + [ + "fus", + "--verbose", + "--arch", + "x86_64", + "--repo=repo-0,repo,/tmp", + "@conf", + ], ) def test_fail_on_http_repo(self): with self.assertRaises(ValueError): - fus.get_cmd("x86_64", ["http:///tmp"], [], [], [], []) + fus.get_cmd("conf", "x86_64", ["http:///tmp"], []) def test_strip_file_protocol_lookaside(self): - cmd = fus.get_cmd("x86_64", [], ["file:///r"], [], [], []) + cmd = fus.get_cmd("conf", "x86_64", [], ["file:///r"]) self.assertEqual( cmd, - ["fus", "--verbose", "--arch", "x86_64", "--repo=lookaside-0,lookaside,/r"] + [ + "fus", + "--verbose", + "--arch", + "x86_64", + "--repo=lookaside-0,lookaside,/r", + "@conf", + ], ) def test_fail_on_http_repo_lookaside(self): with self.assertRaises(ValueError): - fus.get_cmd("x86_64", [], ["http:///tmp"], [], [], []) + fus.get_cmd("conf", "x86_64", [], ["http:///tmp"]) + + +class TestWriteConfig(PungiTestCase): + + def test_write_sorted_mix(self): + f = os.path.join(self.topdir, "solvables") + fus.write_config(f, ["moda:master"], ["pkg", "foo"]) + self.assertFileContent( + f, + dedent( + """\ + module(moda:master) + pkg + foo + """ + ), + ) class TestParseOutput(unittest.TestCase): diff --git a/tests/test_gather_method_hybrid.py b/tests/test_gather_method_hybrid.py index 8ba39c9..354241f 100644 --- a/tests/test_gather_method_hybrid.py +++ b/tests/test_gather_method_hybrid.py @@ -292,6 +292,7 @@ class ModifiedMagicMock(mock.MagicMock): ) +@mock.patch("pungi.wrappers.fus.write_config") @mock.patch("pungi.wrappers.fus.parse_output") @mock.patch("pungi.wrappers.fus.get_cmd", new_callable=ModifiedMagicMock) @mock.patch("pungi.phases.gather.methods.method_hybrid.run") @@ -307,8 +308,14 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): self.logfile2 = os.path.join( self.compose.topdir, "logs/x86_64/hybrid-depsolver-Server-iter-2.x86_64.log" ) + self.config1 = os.path.join( + self.compose.topdir, "work/x86_64/fus/Server-solvables-1.x86_64.conf" + ) + self.config2 = os.path.join( + self.compose.topdir, "work/x86_64/fus/Server-solvables-2.x86_64.conf" + ) - def test_with_modules(self, run, gc, po): + def test_with_modules(self, run, gc, po, wc): self.compose.has_comps = False self.compose.variants["Server"].arch_mmds["x86_64"] = { "mod:master": mock.Mock( @@ -347,21 +354,23 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ], ) self.assertEqual( + wc.call_args_list, [mock.call(self.config1, ["mod:master"], [])], + ) + self.assertEqual( gc.call_args_list, [ mock.call( + self.config1, "x86_64", [self._repo("repo"), self._repo("module_repo_Server")], [], - [], - ["mod:master"], platform="pl", filter_packages=["foo"], ) ], ) - def test_with_modules_with_devel(self, run, gc, po): + def test_with_modules_with_devel(self, run, gc, po, wc): self.compose.has_comps = False self.compose.variants["Server"].arch_mmds["x86_64"] = { "mod:master": mock.Mock( @@ -398,21 +407,24 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ], ) self.assertEqual( + wc.call_args_list, + [mock.call(self.config1, ["mod-devel:master", "mod:master"], [])], + ) + self.assertEqual( gc.call_args_list, [ mock.call( + self.config1, "x86_64", [self._repo("repo"), self._repo("module_repo_Server")], [], - [], - ["mod:master", "mod-devel:master"], platform="pl", filter_packages=["foo"], ) ], ) - def test_with_comps(self, run, gc, po): + def test_with_comps(self, run, gc, po, wc): po.return_value = (mock.Mock(), mock.Mock()) res = self.phase.run_solver( self.compose.variants["Server"], @@ -433,21 +445,23 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ], ) self.assertEqual( + wc.call_args_list, [mock.call(self.config1, [], ["pkg"])], + ) + self.assertEqual( gc.call_args_list, [ mock.call( + self.config1, "x86_64", [self._repo("repo")], [], - ["pkg"], - [], platform=None, filter_packages=[], ) ], ) - def test_with_langpacks(self, run, gc, po): + def test_with_langpacks(self, run, gc, po, wc): self.phase.langpacks = {"pkg": set(["pkg-en"])} final = [("pkg-1.0-1", "x86_64", []), ("pkg-en-1.0-1", "noarch", [])] po.side_effect = [([("pkg-1.0-1", "x86_64", [])], set()), (final, [])] @@ -476,23 +490,28 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ], ) self.assertEqual( + wc.call_args_list, + [ + mock.call(self.config1, [], ["pkg"]), + mock.call(self.config2, [], ["pkg", "pkg-en"]), + ], + ) + self.assertEqual( gc.call_args_list, [ mock.call( + self.config1, "x86_64", [self._repo("repo")], [], - ["pkg"], - [], platform=None, filter_packages=["foo"], ), mock.call( + self.config2, "x86_64", [self._repo("repo")], [], - ["pkg", "pkg-en"], - [], platform=None, filter_packages=["foo"], ), @@ -500,7 +519,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ) @mock.patch("pungi.phases.gather.methods.method_hybrid.cr") - def test_multilib_devel(self, cr, run, gc, po): + def test_multilib_devel(self, cr, run, gc, po, wc): self.phase.arch = "x86_64" self.phase.multilib_methods = ["devel"] self.phase.multilib = mock.Mock() @@ -551,23 +570,28 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ], ) self.assertEqual( + wc.call_args_list, + [ + mock.call(self.config1, [], ["foo", "pkg-devel"]), + mock.call(self.config2, [], ["foo", "pkg-devel", "pkg-devel.i686"]), + ], + ) + self.assertEqual( gc.call_args_list, [ mock.call( + self.config1, "x86_64", [self._repo("repo")], [], - ["pkg-devel", "foo"], - [], platform=None, filter_packages=[], ), mock.call( + self.config2, "x86_64", [self._repo("repo")], [], - ["pkg-devel", "foo", "pkg-devel.i686"], - [], platform=None, filter_packages=[], ), @@ -575,7 +599,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ) @mock.patch("pungi.phases.gather.methods.method_hybrid.cr") - def test_multilib_runtime(self, cr, run, gc, po): + def test_multilib_runtime(self, cr, run, gc, po, wc): packages = { "abc": NamedMock( name="foo", @@ -652,23 +676,28 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ], ) self.assertEqual( + wc.call_args_list, + [ + mock.call(self.config1, [], ["foo", "pkg-devel"]), + mock.call(self.config2, [], ["foo", "foo.i686", "pkg-devel"]), + ], + ) + self.assertEqual( gc.call_args_list, [ mock.call( + self.config1, "x86_64", [self._repo("repo")], [], - ["pkg-devel", "foo"], - [], platform=None, filter_packages=[], ), mock.call( + self.config2, "x86_64", [self._repo("repo")], [], - ["pkg-devel", "foo", "foo.i686"], - [], platform=None, filter_packages=[], ),