From eed2aa2753a9a9d5a3031eef57757b7615f9f0f5 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Sep 03 2019 07:31:18 +0000 Subject: pkgset: Add object representing a package set on disk Once a package set repo is written to disk, let's use this object that connects the repository path with the mapping of packages. This change also makes it explicit where the dependency on package set repos are. In the original codebase, any part of code could generate a path to the repo, even if that repo has not yet been written. Signed-off-by: Lubomír Sedlář --- diff --git a/pungi/paths.py b/pungi/paths.py index cdf312e..67c24d1 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -218,7 +218,7 @@ class WorkPaths(object): makedirs(path) return path - def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True): + def package_list(self, arch=None, variant=None, pkgset=None, pkg_type=None, create_dir=True): """ Examples: work/x86_64/package_list/x86_64.conf @@ -230,6 +230,8 @@ class WorkPaths(object): file_name = "%s.%s" % (variant, arch) else: file_name = "%s" % arch + if pkgset: + file_name += "." + pkgset.name if pkg_type is not None: file_name += ".%s" % pkg_type file_name += ".conf" diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 4462191..a3fad6a 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -656,8 +656,7 @@ def get_system_release_packages(compose, arch, variant, package_sets): system_release_packages = set() for pkgset in package_sets or []: - for i in pkgset.get(arch, []): - pkg = pkgset[arch][i] + for pkg in pkgset.iter_packages(arch): if pkg.is_system_release: system_release_packages.add(pkg) diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index 9312803..e647fb5 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -94,9 +94,9 @@ def write_pungi_config(compose, arch, variant, packages, groups, filter_packages "Writing pungi config (arch: %s, variant: %s): %s", arch, variant, pungi_cfg ) - repos = { - "pungi-repo": compose.paths.work.arch_repo(arch=arch), - } + repos = {} + for i, pkgset in enumerate(package_sets or []): + repos["pungi-repo-%d" % i] = pkgset.paths[arch] if compose.has_comps: repos["comps-repo"] = compose.paths.work.comps_repo(arch=arch, variant=variant) if variant.type == "optional": diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index 8fd9696..6332863 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -30,86 +30,124 @@ from pungi.phases.createrepo import add_modular_metadata def populate_arch_pkgsets(compose, path_prefix, global_pkgset): result = {} - exclusive_noarch = compose.conf['pkgset_exclusive_arch_considers_noarch'] + exclusive_noarch = compose.conf["pkgset_exclusive_arch_considers_noarch"] for arch in compose.get_arches(): - compose.log_info("Populating package set for arch: %s" % arch) + compose.log_info("Populating package set for arch: %s", arch) is_multilib = is_arch_multilib(compose.conf, arch) arches = get_valid_arches(arch, is_multilib, add_src=True) - pkgset = pungi.phases.pkgset.pkgsets.PackageSetBase(compose.conf["sigkeys"], logger=compose._logger, arches=arches) + pkgset = pungi.phases.pkgset.pkgsets.PackageSetBase( + global_pkgset.name, + compose.conf["sigkeys"], + logger=compose._logger, + arches=arches, + ) pkgset.merge(global_pkgset, arch, arches, exclusive_noarch=exclusive_noarch) - pkgset.save_file_list(compose.paths.work.package_list(arch=arch), remove_path_prefix=path_prefix) + pkgset.save_file_list( + compose.paths.work.package_list(arch=arch, pkgset=global_pkgset), + remove_path_prefix=path_prefix, + ) result[arch] = pkgset return result -def get_create_global_repo_cmd(compose, path_prefix): +def get_create_global_repo_cmd(compose, path_prefix, repo_dir_global, pkgset): createrepo_c = compose.conf["createrepo_c"] createrepo_checksum = compose.conf["createrepo_checksum"] repo = CreaterepoWrapper(createrepo_c=createrepo_c) - repo_dir_global = compose.paths.work.arch_repo(arch="global") + + pkgset.save_file_list( + compose.paths.work.package_list(arch="global", pkgset=pkgset), + remove_path_prefix=path_prefix, + ) # find an old compose suitable for repodata reuse old_compose_path = None update_md_path = None if compose.old_composes: + is_layered = compose.ci_base.release.is_layered old_compose_path = find_old_compose( compose.old_composes, compose.ci_base.release.short, compose.ci_base.release.version, compose.ci_base.release.type_suffix, - compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None, - compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None, + compose.ci_base.base_product.short if is_layered else None, + compose.ci_base.base_product.version if is_layered else None, ) if old_compose_path is None: - compose.log_info("No suitable old compose found in: %s" % compose.old_composes) + compose.log_info( + "No suitable old compose found in: %s", compose.old_composes + ) else: - repo_dir = compose.paths.work.arch_repo(arch="global") - rel_path = relative_path(repo_dir, os.path.abspath(compose.topdir).rstrip("/") + "/") + repo_dir = compose.paths.work.pkgset_repo(pkgset.name, arch="global") + rel_path = relative_path( + repo_dir, os.path.abspath(compose.topdir).rstrip("/") + "/" + ) old_repo_dir = os.path.join(old_compose_path, rel_path) if os.path.isdir(old_repo_dir): - compose.log_info("Using old repodata from: %s" % old_repo_dir) + compose.log_info("Using old repodata from: %s", old_repo_dir) update_md_path = old_repo_dir - # IMPORTANT: must not use --skip-stat here -- to make sure that correctly signed files are pulled in - cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=False, skip_stat=False, - pkglist=compose.paths.work.package_list(arch="global"), outputdir=repo_dir_global, - baseurl="file://%s" % path_prefix, workers=compose.conf["createrepo_num_workers"], - update_md_path=update_md_path, checksum=createrepo_checksum) + # IMPORTANT: must not use --skip-stat here -- to make sure that correctly + # signed files are pulled in + cmd = repo.get_createrepo_cmd( + path_prefix, + update=True, + database=False, + skip_stat=False, + pkglist=compose.paths.work.package_list(arch="global", pkgset=pkgset), + outputdir=repo_dir_global, + baseurl="file://%s" % path_prefix, + workers=compose.conf["createrepo_num_workers"], + update_md_path=update_md_path, + checksum=createrepo_checksum, + ) return cmd -def run_create_global_repo(compose, cmd): +def run_create_global_repo(compose, cmd, logfile): msg = "Running createrepo for the global package set" - compose.log_info("[BEGIN] %s" % msg) + compose.log_info("[BEGIN] %s", msg) + run(cmd, logfile=logfile, show_cmd=True) + compose.log_info("[DONE ] %s", msg) - run(cmd, logfile=compose.paths.log.log_file("global", "arch_repo"), show_cmd=True) - compose.log_info("[DONE ] %s" % msg) - -def create_arch_repos(compose, path_prefix): +def create_arch_repos(compose, path_prefix, paths, pkgset): run_in_threads( _create_arch_repo, - [(compose, arch, path_prefix) for arch in compose.get_arches()], - threads=compose.conf['createrepo_num_threads'], + [(compose, arch, path_prefix, paths, pkgset) for arch in compose.get_arches()], + threads=compose.conf["createrepo_num_threads"], ) def _create_arch_repo(worker_thread, args, task_num): """Create a single pkgset repo for given arch.""" - compose, arch, path_prefix = args + compose, arch, path_prefix, paths, pkgset = args createrepo_c = compose.conf["createrepo_c"] createrepo_checksum = compose.conf["createrepo_checksum"] repo = CreaterepoWrapper(createrepo_c=createrepo_c) - repo_dir_global = compose.paths.work.arch_repo(arch="global") - repo_dir = compose.paths.work.arch_repo(arch=arch) + repo_dir_global = compose.paths.work.pkgset_repo(pkgset.name, arch="global") + repo_dir = compose.paths.work.pkgset_repo(pkgset.name, arch=arch) + paths[arch] = repo_dir msg = "Running createrepo for arch '%s'" % arch - compose.log_info("[BEGIN] %s" % msg) - cmd = repo.get_createrepo_cmd(path_prefix, update=True, database=False, skip_stat=True, - pkglist=compose.paths.work.package_list(arch=arch), outputdir=repo_dir, - baseurl="file://%s" % path_prefix, workers=compose.conf["createrepo_num_workers"], - update_md_path=repo_dir_global, checksum=createrepo_checksum) - run(cmd, logfile=compose.paths.log.log_file(arch, "arch_repo"), show_cmd=True) + compose.log_info("[BEGIN] %s", msg) + cmd = repo.get_createrepo_cmd( + path_prefix, + update=True, + database=False, + skip_stat=True, + pkglist=compose.paths.work.package_list(arch=arch, pkgset=pkgset), + outputdir=repo_dir, + baseurl="file://%s" % path_prefix, + workers=compose.conf["createrepo_num_workers"], + update_md_path=repo_dir_global, + checksum=createrepo_checksum, + ) + run( + cmd, + logfile=compose.paths.log.log_file(arch, "arch_repo_%s" % pkgset.name), + show_cmd=True, + ) # Add modulemd to the repo for all modules in all variants on this architecture. if Modulemd: mod_index = collect_module_defaults(compose.paths.work.module_defaults_dir()) @@ -118,26 +156,74 @@ def _create_arch_repo(worker_thread, args, task_num): for module_stream in variant.arch_mmds.get(arch, {}).values(): mod_index.add_module_stream(module_stream) add_modular_metadata( - repo, repo_dir, mod_index, compose.paths.log.log_file(arch, "arch_repo_modulemd") + repo, + repo_dir, + mod_index, + compose.paths.log.log_file(arch, "arch_repo_modulemd"), ) - compose.log_info("[DONE ] %s" % msg) + compose.log_info("[DONE ] %s", msg) + +class MaterializedPackageSet(object): + """A wrapper for PkgsetBase object that represents the package set created + as repos on the filesystem. + """ -def materialize_pkgset(compose, pkgset_global, path_prefix): - """Create per-arch pkgsets and create repodata for each arch.""" - cmd = get_create_global_repo_cmd(compose, path_prefix) - t = threading.Thread(target=run_create_global_repo, args=(compose, cmd)) - t.start() + def __init__(self, package_sets, paths): + self.package_sets = package_sets + self.paths = paths + + @property + def name(self): + return self.package_sets["global"].name + + def __getitem__(self, key): + """Direct access to actual package set for particular arch.""" + return self.package_sets[key] + + def get(self, arch, default=None): + """Get package set for particular arch.""" + return self.package_sets.get(arch, default or []) + + def iter_packages(self, arch=None): + """Yield all packages in the set, optionally filtering for some arch + only. + """ + if not arch: + for arch in self.package_sets: + for file_path in self.get(arch): + yield self.package_sets[arch][file_path] + else: + for file_path in self.get(arch): + yield self.package_sets[arch][file_path] + + @classmethod + def create(klass, compose, pkgset_global, path_prefix): + """Create per-arch pkgsets and create repodata for each arch.""" + repo_dir_global = compose.paths.work.pkgset_repo( + pkgset_global.name, arch="global" + ) + paths = {"global": repo_dir_global} + cmd = get_create_global_repo_cmd( + compose, path_prefix, repo_dir_global, pkgset_global + ) + logfile = compose.paths.log.log_file( + "global", "arch_repo.%s" % pkgset_global.name + ) + t = threading.Thread( + target=run_create_global_repo, args=(compose, cmd, logfile) + ) + t.start() - package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global) - package_sets["global"] = pkgset_global + package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global) + package_sets["global"] = pkgset_global - t.join() + t.join() - create_arch_repos(compose, path_prefix) + create_arch_repos(compose, path_prefix, paths, pkgset_global) - return package_sets + return klass(package_sets, paths) def get_all_arches(compose): diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 7a79daf..4c41a7a 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -30,7 +30,7 @@ from pungi.arch import getBaseArch from pungi.util import retry, find_old_compose from pungi import Modulemd -from pungi.phases.pkgset.common import materialize_pkgset, get_all_arches +from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches from pungi.phases.gather import get_packages_to_gather import pungi.phases.pkgset.source @@ -190,9 +190,7 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): event_info = get_koji_event_info(compose, koji_wrapper) pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) - package_sets = materialize_pkgset(compose, pkgset_global, path_prefix) - - return package_sets + return MaterializedPackageSet.create(compose, pkgset_global, path_prefix) def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=False): diff --git a/pungi/phases/pkgset/sources/source_repos.py b/pungi/phases/pkgset/sources/source_repos.py index 0e0ef5c..1fab1ca 100644 --- a/pungi/phases/pkgset/sources/source_repos.py +++ b/pungi/phases/pkgset/sources/source_repos.py @@ -23,7 +23,7 @@ import pungi.phases.pkgset.pkgsets from pungi.util import makedirs from pungi.wrappers.pungi import PungiWrapper -from pungi.phases.pkgset.common import materialize_pkgset, get_all_arches +from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches from pungi.phases.gather import get_prepopulate_packages, get_packages_to_gather from pungi.linker import LinkerPool @@ -112,7 +112,7 @@ def get_pkgset_from_repos(compose): flist = sorted(set(flist)) pkgset_global = populate_global_pkgset(compose, flist, path_prefix) - package_sets = materialize_pkgset(compose, pkgset_global, path_prefix) + package_sets = MaterializedPackageSet.create(compose, pkgset_global, path_prefix) return package_sets, path_prefix