From a446593698e75143ea077abf921c6a0ae8c34e8a Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Sep 03 2019 07:31:18 +0000 Subject: pkgset: Update Koji source to create multiple pkgsets With this patch, there should be a separate package set for each tag that is consumed. Generally each module will create a separate package set, with the exception of -devel modules that will be in the same set as their non-devel version. Variants no longer need to keep their own package set objects. Instead they now include a set of package set names that should be used for the variant. This can replace the whitelist mechanism of deps gather method. JIRA: COMPOSE-3620 Signed-off-by: Lubomír Sedlář --- diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index e647fb5..7cfb010 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -96,7 +96,8 @@ def write_pungi_config(compose, arch, variant, packages, groups, filter_packages repos = {} for i, pkgset in enumerate(package_sets or []): - repos["pungi-repo-%d" % i] = pkgset.paths[arch] + if not variant.pkgsets or pkgset.name in variant.pkgsets: + repos["pungi-repo-%d" % i] = pkgset.paths[arch] if compose.has_comps: repos["comps-repo"] = compose.paths.work.comps_repo(arch=arch, variant=variant) if variant.type == "optional": diff --git a/pungi/phases/pkgset/__init__.py b/pungi/phases/pkgset/__init__.py index 46974fb..cf01ff9 100644 --- a/pungi/phases/pkgset/__init__.py +++ b/pungi/phases/pkgset/__init__.py @@ -28,5 +28,4 @@ class PkgsetPhase(PhaseBase): PkgsetSourceContainer.register_module(sources) container = PkgsetSourceContainer() SourceClass = container[pkgset_source] - package_sets, self.path_prefix = SourceClass(self.compose)() - self.package_sets = [package_sets] + self.package_sets, self.path_prefix = SourceClass(self.compose)() diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index 6332863..02defd4 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -111,17 +111,27 @@ def run_create_global_repo(compose, cmd, logfile): compose.log_info("[DONE ] %s", msg) -def create_arch_repos(compose, path_prefix, paths, pkgset): +def create_arch_repos(compose, path_prefix, paths, pkgset, mmds): run_in_threads( _create_arch_repo, - [(compose, arch, path_prefix, paths, pkgset) for arch in compose.get_arches()], + [ + ( + compose, + arch, + path_prefix, + paths, + pkgset, + mmds.get(arch) if mmds else None, + ) + for arch in compose.get_arches() + ], threads=compose.conf["createrepo_num_threads"], ) def _create_arch_repo(worker_thread, args, task_num): """Create a single pkgset repo for given arch.""" - compose, arch, path_prefix, paths, pkgset = args + compose, arch, path_prefix, paths, pkgset, mmd = args createrepo_c = compose.conf["createrepo_c"] createrepo_checksum = compose.conf["createrepo_checksum"] repo = CreaterepoWrapper(createrepo_c=createrepo_c) @@ -199,7 +210,7 @@ class MaterializedPackageSet(object): yield self.package_sets[arch][file_path] @classmethod - def create(klass, compose, pkgset_global, path_prefix): + def create(klass, compose, pkgset_global, path_prefix, mmd=None): """Create per-arch pkgsets and create repodata for each arch.""" repo_dir_global = compose.paths.work.pkgset_repo( pkgset_global.name, arch="global" @@ -221,7 +232,7 @@ class MaterializedPackageSet(object): t.join() - create_arch_repos(compose, path_prefix, paths, pkgset_global) + create_arch_repos(compose, path_prefix, paths, pkgset_global, mmd) return klass(package_sets, paths) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 4c41a7a..237b604 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -188,9 +187,9 @@ class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase): def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): event_info = get_koji_event_info(compose, koji_wrapper) - pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) + pkgsets = populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) - return MaterializedPackageSet.create(compose, pkgset_global, path_prefix) + return pkgsets def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=False): @@ -276,6 +277,10 @@ def _get_modules_from_koji(compose, koji_wrapper, event, variant, variant_tags): # This is needed in createrepo phase where metadata is exposed by producmd variant.module_uid_to_koji_tag[nsvc] = tag + tag_to_mmd.setdefault(tag, {}) + for arch in variant.arch_mmds: + tag_to_mmd[tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc]) + module_msg = ( "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' " "(as a result of querying module '{module}')" @@ -452,6 +459,10 @@ def _get_modules_from_koji_tags(compose, koji_wrapper, event_id, variant, varian # This is needed in createrepo phase where metadata is exposed by producmd variant.module_uid_to_koji_tag[nsvc] = module_tag + tag_to_mmd.setdefault(module_tag, {}) + for arch in variant.arch_mmds: + tag_to_mmd[module_tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc]) + module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( variant=variant, tag=module_tag, module=build["nvr"]) compose.log_info("%s" % module_msg) @@ -524,19 +535,12 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): # there are some packages with invalid sigkeys, it raises an exception. allow_invalid_sigkeys = compose.conf["gather_method"] == "deps" + tag_to_mmd = {} + for variant in compose.all_variants.values(): - # pkgset storing the packages belonging to this particular variant. - variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( - "TODO", - koji_wrapper, - compose.conf["sigkeys"], - logger=compose._logger, - arches=all_arches, - ) variant_tags[variant] = [] - # Get the modules from Koji tag or from PDC, depending on - # configuration. + # Get the modules from Koji tag modular_koji_tags = variant.get_modular_koji_tags() if (variant.modules or modular_koji_tags) and not Modulemd: raise ValueError( diff --git a/pungi/phases/pkgset/sources/source_repos.py b/pungi/phases/pkgset/sources/source_repos.py index 1fab1ca..f00f79d 100644 --- a/pungi/phases/pkgset/sources/source_repos.py +++ b/pungi/phases/pkgset/sources/source_repos.py @@ -112,9 +112,9 @@ def get_pkgset_from_repos(compose): flist = sorted(set(flist)) pkgset_global = populate_global_pkgset(compose, flist, path_prefix) - package_sets = MaterializedPackageSet.create(compose, pkgset_global, path_prefix) + package_set = MaterializedPackageSet.create(compose, pkgset_global, path_prefix) - return package_sets, path_prefix + return [package_set], path_prefix def populate_global_pkgset(compose, file_list, path_prefix):