diff --git a/tests/test_pulp.py b/tests/test_pulp.py index 7f7862f..8522a5a 100644 --- a/tests/test_pulp.py +++ b/tests/test_pulp.py @@ -63,17 +63,18 @@ def fixture_mock_repo(): ("dist_id_2", "dist_type_id_2"), ], ubi_population=None, + population_sources=None ) @pytest.fixture(name='mock_package') def fixture_mock_package(): - yield Package("foo-pkg", "foo-pkg.rpm") + yield Package("foo-pkg", "foo-pkg.rpm", "src_repo_id") @pytest.fixture(name='mock_mdd') def fixture_mock_mdd(): - yield ModuleDefaults("virt", "rhel", {"2.6": ["common"]}) + yield ModuleDefaults("virt", "rhel", {"2.6": ["common"]}, "src_repo_id") @pytest.fixture(name='mock_response_for_async_req') diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index 346fd94..0ea6e23 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -27,9 +27,9 @@ @pytest.fixture(name='ubi_repo_set') def fixture_ubi_repo_set(): - yield UbiRepoSet(RepoSet(get_test_repo(repo_id="foo-rpms"), - get_test_repo(repo_id="foo-source"), - get_test_repo(repo_id="foo-debug")), + yield UbiRepoSet(RepoSet([get_test_repo(repo_id="foo-rpms")], + [get_test_repo(repo_id="foo-source")], + [get_test_repo(repo_id="foo-debug")]), RepoSet(get_test_repo(repo_id="ubi-foo-rpms"), get_test_repo(repo_id="ubi-foo-source"), get_test_repo(repo_id="ubi-foo-debug"))) @@ -37,8 +37,8 @@ def fixture_ubi_repo_set(): @pytest.fixture(name='ubi_repo_set_no_debug') def fixture_ubi_repo_set_no_debug(): - yield UbiRepoSet(RepoSet(get_test_repo(repo_id="foo-rpms"), - get_test_repo(repo_id="foo-source"), + yield UbiRepoSet(RepoSet([get_test_repo(repo_id="foo-rpms")], + [get_test_repo(repo_id="foo-source")], None), RepoSet(get_test_repo(repo_id="ubi-foo-rpms"), get_test_repo(repo_id="ubi-foo-source"), @@ -68,6 +68,7 @@ def get_test_repo(**kwargs): kwargs.get('platform_full_version'), kwargs.get('distributors_ids_type_ids'), kwargs.get('ubi_population'), + kwargs.get('population_sources'), ) @@ -75,6 +76,7 @@ def get_test_pkg(**kwargs): return Package( kwargs.get('name'), kwargs.get('filename'), + kwargs.get('src_repo_id'), sourcerpm_filename=kwargs.get('sourcerpm_filename'), is_modular=kwargs.get('is_modular', False), ) @@ -89,11 +91,16 @@ def get_test_mod(**kwargs): kwargs.get('arch', ''), kwargs.get('packages', ''), kwargs.get('profiles', ''), + kwargs.get('src_repo_id'), ) def get_test_mod_defaults(**kwargs): - return ModuleDefaults(kwargs['name'], kwargs['stream'], kwargs['profiles']) + return ModuleDefaults(kwargs['name'], + kwargs['stream'], + kwargs['profiles'], + kwargs.get('src_repo_id'), + ) def test_get_output_repo_ids(ubi_repo_set): @@ -111,20 +118,6 @@ def test_get_output_repo_ids_no_debug(ubi_repo_set_no_debug): def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner, caplog): # Don't actually query Pulp for repos mocked_search_repo_by_cs.side_effect = [ - # Input repos - rhel-8-for-x86_64-appstream - [get_test_repo( - repo_id="rhel-8-for-x86_64-appstream-rpms", - content_set="rhel-8-for-x86_64-appstream-rpms", - ), ], - [get_test_repo( - repo_id="rhel-8-for-x86_64-appstream-source-rpms", - content_set="rhel-8-for-x86_64-appstream-source-rpms", - ), ], - [get_test_repo( - repo_id="rhel-8-for-x86_64-appstream-debug-rpms", - content_set="rhel-8-for-x86_64-appstream-debug-rpms", - ), ], - # Output repos - rhel-8-for-x86_64-appstream [get_test_repo( repo_id="ubi-8-for-x86_64-appstream-rpms", @@ -142,36 +135,52 @@ def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner, ubi_population=True ), ], - # Input repos - rhel-7-server + # Input repos - rhel-8-for-x86_64-appstream [get_test_repo( - repo_id="rhel-7-server-rpms__7_DOT_2__x86_64", - content_set="rhel-7-server-rpms", + repo_id="rhel-8-for-x86_64-appstream-rpms", + content_set="rhel-8-for-x86_64-appstream-rpms", ), ], [get_test_repo( - repo_id="rhel-7-server-source-rpms__7_DOT_2__x86_64", - content_set="rhel-7-server-source-rpms", + repo_id="rhel-8-for-x86_64-appstream-source-rpms", + content_set="rhel-8-for-x86_64-appstream-source-rpms", ), ], [get_test_repo( - repo_id="rhel-7-server-debuginfo-rpms__7_DOT_2__x86_64", - content_set="rhel-7-server-debuginfo-rpms", + repo_id="rhel-8-for-x86_64-appstream-debug-rpms", + content_set="rhel-8-for-x86_64-appstream-debug-rpms", ), ], # Output repos - rhel-7-server [get_test_repo( repo_id="ubi-7-server-rpms__7_DOT_2__x86_64", content_set="ubi-7-server-rpms", - ubi_population=True + ubi_population=False ), ], [get_test_repo( repo_id="ubi-7-server-source-rpms__7_DOT_2__x86_64", content_set="ubi-7-server-source-rpms", - ubi_population=False + ubi_population=True + # doesn't matter here, it sufficient to have ubi_population==False at rpm binary repo + # for skipping whole repo triplet ), ], [get_test_repo( repo_id="ubi-7-server-debuginfo-rpms__7_DOT_2__x86_64", content_set="ubi-7-server-debuginfo-rpms", ubi_population=False ), ], + + # Input repos - rhel-7-server + [get_test_repo( + repo_id="rhel-7-server-rpms__7_DOT_2__x86_64", + content_set="rhel-7-server-rpms", + ), ], + [get_test_repo( + repo_id="rhel-7-server-source-rpms__7_DOT_2__x86_64", + content_set="rhel-7-server-source-rpms", + ), ], + [get_test_repo( + repo_id="rhel-7-server-debuginfo-rpms__7_DOT_2__x86_64", + content_set="rhel-7-server-debuginfo-rpms", + ), ], ] # Attempt to populate both invalid and valid repo sets @@ -180,10 +189,47 @@ def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner, # Should've only run once assert mocked_ubipop_runner.call_count == 1 - # For rhel-8-for-x86_64-appstream - assert "Skipping rhel-8-for-x86_64-appstream" not in caplog.text - # Not for rhel-7-server - assert "Skipping rhel-7-server-rpms" in caplog.text + # For ubi-8 + assert "ubi-8-for-x86_64-appstream-rpms" not in caplog.text + # Not for ubi-7-server + assert "ubi-7-server-rpms__7_DOT_2__x86_64" in caplog.text + + +@patch("ubipop._pulp_client.Pulp.search_repo_by_id") +def test_get_population_sources_repo_note(mocked_search_repo_by_id): + repo = get_test_repo( + repo_id="rhel-8-for-x86_64-appstream-rpms", + content_set="rhel-8-for-x86_64-appstream-rpms", + population_sources=['src_1', 'src_2'] + ) + ubipop = UbiPopulate("foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=TEST_DATA_DIR) + mocked_search_repo_by_id.side_effect = [[get_test_repo(repo_id="src_1", + content_set="src_1_cs", + ) + ], + [get_test_repo(repo_id="src_2", + content_set="src_2_cs", + ) + ], + ] + repos = ubipop._get_population_sources(repo, None) # pylint: disable=protected-access + assert len(repos) == 2 + + +@patch("ubipop._pulp_client.Pulp.search_repo_by_cs") +def test_get_population_sources_by_search(search_repo_by_cs): + repo = get_test_repo( + repo_id="rhel-8-for-x86_64-appstream-rpms", + content_set="rhel-8-for-x86_64-appstream-rpms", + ) + ubipop = UbiPopulate("foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=TEST_DATA_DIR) + search_repo_by_cs.side_effect = [[get_test_repo(repo_id="src_1", + content_set="src_1_cs", + ) + ], + ] + repos = ubipop._get_population_sources(repo, None) # pylint: disable=protected-access + assert len(repos) == 1 def test_get_packages_from_module_by_name(mock_ubipop_runner): @@ -776,12 +822,14 @@ def test_create_srpms_output_set(mock_ubipop_runner): name="tomcatjss", filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", sourcerpm_filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.src.rpm", + src_repo_id="foo-rpms", ), # blacklisted get_test_pkg( name="kernel", filename="kernel-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", sourcerpm_filename="kernel.src.rpm", + src_repo_id="foo-rpms", ), # blacklisted but referenced in some module get_test_pkg( @@ -789,6 +837,7 @@ def test_create_srpms_output_set(mock_ubipop_runner): filename="foo-pkg-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", sourcerpm_filename="foo-pkg-7.3.6-1.el8+1944+b6c8e16f.src.rpm", is_modular=True, + src_repo_id="foo-rpms", ), ] @@ -822,9 +871,9 @@ def test_create_srpms_output_set_missings_srpm_reference(capsys, set_logging, mo @pytest.fixture(name='mock_get_repo_pairs') def fixture_mock_get_repo_pairs(ubi_repo_set): - with patch('ubipop.UbiPopulate._get_input_and_output_repo_pairs') as get_repo_pairs: - get_repo_pairs.return_value = [ubi_repo_set] - yield get_repo_pairs + with patch('ubipop.UbiPopulate._get_ubi_repo_sets') as get_ubi_repo_sets: + get_ubi_repo_sets.return_value = [ubi_repo_set] + yield get_ubi_repo_sets @pytest.fixture(name='mock_run_ubi_population') @@ -914,22 +963,26 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content_ft): assert len(modules.units) == 1 assert modules.units[0].name == "test_md" assert modules.dst_repo.repo_id == "ubi-foo-rpms" - assert modules.src_repo.repo_id == "foo-rpms" + assert len(modules.src_repos) == 1 + assert modules.src_repos[0].repo_id == "foo-rpms" assert len(rpms.units) == 1 assert rpms.units[0].name == "test_rpm" assert rpms.dst_repo.repo_id == "ubi-foo-rpms" - assert rpms.src_repo.repo_id == "foo-rpms" + assert len(rpms.src_repos) == 1 + assert rpms.src_repos[0].repo_id == "foo-rpms" assert len(srpms.units) == 1 assert srpms.units[0].name == "test_srpm" assert srpms.dst_repo.repo_id == "ubi-foo-source" - assert srpms.src_repo.repo_id == "foo-source" + assert len(srpms.src_repos) == 1 + assert srpms.src_repos[0].repo_id == "foo-source" assert len(debug_rpms.units) == 1 assert debug_rpms.units[0].name == "test_debug_pkg" assert debug_rpms.dst_repo.repo_id == "ubi-foo-debug" - assert debug_rpms.src_repo.repo_id == "foo-debug" + assert len(debug_rpms.src_repos) == 1 + assert debug_rpms.src_repos[0].repo_id == "foo-debug" # secondly, check correct unassociations, there should 1 unit of each type unassociated modules, rpms, srpms, debug_rpms = unassociations @@ -951,14 +1004,14 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content_ft): assert len(mdd_association.units) == 1 assert mdd_association.dst_repo.repo_id == 'ubi-foo-rpms' - assert mdd_association.src_repo.repo_id == 'foo-rpms' + assert len(mdd_association.src_repos) == 1 + assert mdd_association.src_repos[0].repo_id == 'foo-rpms' assert len(mdd_unassociation.units) == 1 assert mdd_unassociation.units[0].name == 'mdd_current' assert mdd_unassociation.dst_repo.repo_id == 'ubi-foo-rpms' - def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content_ft): mock_ubipop_runner.repos.modules = { "test": [ @@ -1018,7 +1071,9 @@ def test_log_pulp_action(capsys, set_logging, mock_ubipop_runner): set_logging.addHandler(logging.StreamHandler(sys.stdout)) src_repo = get_test_repo(repo_id='test_src') dst_repo = get_test_repo(repo_id='test_dst') - associations = [AssociateActionModules([get_test_mod(name="test_assoc")], dst_repo, src_repo)] + associations = [AssociateActionModules([get_test_mod(name="test_assoc", + src_repo_id=src_repo.repo_id)], + dst_repo, [src_repo])] unassociations = [UnassociateActionModules([get_test_mod(name="test_unassoc")], dst_repo)] mock_ubipop_runner.log_pulp_actions(associations, unassociations) @@ -1034,7 +1089,7 @@ def test_log_pulp_action_no_actions(capsys, set_logging, mock_ubipop_runner): set_logging.addHandler(logging.StreamHandler(sys.stdout)) src_repo = get_test_repo(repo_id='test_src') dst_repo = get_test_repo(repo_id='test_dst') - associations = [AssociateActionModules([], dst_repo, src_repo)] + associations = [AssociateActionModules([], dst_repo, [src_repo])] unassociations = [UnassociateActionModules([], dst_repo)] mock_ubipop_runner.log_pulp_actions(associations, unassociations) @@ -1042,7 +1097,7 @@ def test_log_pulp_action_no_actions(capsys, set_logging, mock_ubipop_runner): assoc_line, unassoc_line = out.split('\n', 1) assert err == "" - assert assoc_line.strip() == "No association expected for modules from test_src to test_dst" + assert assoc_line.strip() == "No association expected for modules from ['test_src'] to test_dst" assert unassoc_line.strip() == "No unassociation expected for modules from test_dst" @@ -1088,7 +1143,7 @@ def test_associate_units(mock_ubipop_runner): dst_repo = get_test_repo(repo_id='test_dst') associations = [ - AssociateActionModules([get_test_mod(name="test_assoc")], dst_repo, src_repo), + AssociateActionModules([get_test_mod(name="test_assoc")], dst_repo, [src_repo]), ] mock_ubipop_runner.pulp.associate_modules.return_value = ["task_id"] @@ -1108,7 +1163,7 @@ def test_associate_unassociate_md_defaults(mock_ubipop_runner): stream='rhel', profiles={'2.5': ["common"]}, ), - ], dst_repo, src_repo) + ], dst_repo, [src_repo]) unassociations = UnassociateActionModuleDefaults([ get_test_mod_defaults( diff --git a/tests/test_utils.py b/tests/test_utils.py index ee39e1b..ad271be 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,17 +16,17 @@ def test_raise_not_implemented_pulp_action(): units = ["unit1", "unit2"] - repo = Repo("test", "1", "test-rpms", "2", None, None) + repo = Repo("test", "1", "test-rpms", "2", None, None, None) action = PulpAction(units, repo) - pytest.raises(NotImplementedError, action.get_action, None) + pytest.raises(NotImplementedError, action.get_actions, None) def test_raise_not_implemented_associate_action(): units = ["unit1", "unit2"] - repo = Repo("test", "1", "test-rpms", "2", None, None) - src_repo = Repo("test", "1", "test-rpms", "2", None, None) + repo = Repo("test", "1", "test-rpms", "2", None, None, None) + src_repo = Repo("test", "1", "test-rpms", "2", None, None, None) action = AssociateAction(units, repo, src_repo) - pytest.raises(NotImplementedError, action.get_action, None) + pytest.raises(NotImplementedError, action.get_actions, None) @pytest.mark.parametrize("klass, method", [ @@ -35,17 +35,25 @@ def test_raise_not_implemented_associate_action(): (AssociateActionRpms, "associate_packages"), ]) def test_get_action_associate(klass, method): - units = ["unit1", "unit2"] - dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None) - src_repo = Repo("test_src", "1", "test_src-rpms", "2", None, None) - action = klass(units, dst_repo, src_repo) - associate_action, src_repo_current, dst_repo_current, current_units = \ - action.get_action(MagicMock()) + mocked_unit_1 = MagicMock() + mocked_unit_1.associate_source_repo_id = "test_src_1" + mocked_unit_2 = MagicMock() + mocked_unit_2.associate_source_repo_id = "test_src_2" + units = [mocked_unit_1, mocked_unit_2] + dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None, None) + src_repos = [Repo("test_src_1", "1", "test_src-rpms", "2", None, None, None), + Repo("test_src_2", "1", "test_src-rpms", "2", None, None, None)] + action = klass(units, dst_repo, src_repos) + actions = action.get_actions(MagicMock()) - assert "mock." + method in str(associate_action) - assert current_units == units - assert dst_repo_current.repo_id == dst_repo.repo_id - assert src_repo_current.repo_id == src_repo.repo_id + for action in actions: + associate_action, src_repo_current, dst_repo_current, current_units = action + assert "mock." + method in str(associate_action) + assert len(current_units) == 1 + assert current_units == [u for u in units + if u.associate_source_repo_id == src_repo_current.repo_id] + assert dst_repo_current.repo_id == dst_repo.repo_id + assert src_repo_current.repo_id == current_units[0].associate_source_repo_id @pytest.mark.parametrize("klass, method", [ @@ -55,9 +63,9 @@ def test_get_action_associate(klass, method): ]) def test_get_action_unassociate(klass, method): units = ["unit1", "unit2"] - dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None) + dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None, None) action = klass(units, dst_repo) - associate_action, dst_repo_current, current_units = action.get_action(MagicMock()) + associate_action, dst_repo_current, current_units = action.get_actions(MagicMock())[0] assert "mock." + method in str(associate_action) assert current_units == units diff --git a/ubipop/__init__.py b/ubipop/__init__.py index 4d3582e..d9f5f1f 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -137,7 +137,8 @@ def populate_ubi_repos(self): for config in self.ubiconfig_list: try: - repo_pairs = self._get_input_and_output_repo_pairs(config) + repo_pairs = self._get_ubi_repo_sets(config) + except RepoMissing: _LOG.warning("Skipping current content triplet, some repos are missing") continue @@ -153,58 +154,72 @@ def populate_ubi_repos(self): for repo in out_repos: f.write(repo.strip() + '\n') - def _get_input_and_output_repo_pairs(self, ubiconfig_item): + def _get_ubi_repo_sets(self, ubi_config_item): """ - Determines pairs of input and output repos and also find correct source and debuginfo - counterpart of repos. + Searches for ubi repository triplet (binary rpm, srpm, debug) for + one ubi config item and tries to determine their population sources + (input repositories). Returns list UbiRepoSet objects that provides + input and output repositories that are used for population process. """ + rpm_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, + ubi_config_item.content_sets.rpm.output) + source_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, + ubi_config_item.content_sets.srpm.output) + debug_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, + ubi_config_item.content_sets.debuginfo.output) - rpms_cs = ubiconfig_item.content_sets.rpm - source_cs = ubiconfig_item.content_sets.srpm - debug_cs = ubiconfig_item.content_sets.debuginfo + ubi_repo_sets = [] - _LOG.info( - "Getting input repos for input content sets:\n\t%s\n\t%s\n\t%s", - rpms_cs.input, source_cs.input, debug_cs.input) + for out_rpm_repo in rpm_repos_ft.result(): - in_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, rpms_cs.input) - in_source_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, source_cs.input) - in_debug_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, debug_cs.input) + if not out_rpm_repo.ubi_population: + # it is sufficient to check only binary from repo triplet for disabling population + _LOG.debug( + "Skipping population for output binary repo and " + "related source and debug repos:\n\t%s", + out_rpm_repo.repo_id) + continue - _LOG.info( - "Getting output repos for output content sets:\n\t%s\n\t%s\n\t%s", - rpms_cs.output, source_cs.output, debug_cs.output) + out_source_repo = self.get_repo_counterpart(out_rpm_repo, source_repos_ft.result()) + out_debug_repo = self.get_repo_counterpart(out_rpm_repo, debug_repos_ft.result()) - out_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, rpms_cs.output) - out_source_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, source_cs.output) - out_debug_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, debug_cs.output) + in_rpm_repos = self._get_population_sources(out_rpm_repo, + ubi_config_item.content_sets.rpm.input) + in_source_repos = self._get_population_sources(out_source_repo, + ubi_config_item.content_sets.srpm.input) + in_debug_repos = self._get_population_sources(out_debug_repo, + ubi_config_item.content_sets.debuginfo + .input) - repo_pairs = [] - for input_repo in in_repos_ft.result(): - in_rpm = input_repo - in_source = self._get_repo_counterpart(input_repo, in_source_repos_ft.result()) - in_debug_info = self._get_repo_counterpart(input_repo, in_debug_repos_ft.result()) + out_repos = (out_rpm_repo, out_source_repo, out_debug_repo) + in_repos = (in_rpm_repos, in_source_repos, in_debug_repos) - out_rpm = self._get_repo_counterpart(input_repo, out_repos_ft.result()) - out_source = self._get_repo_counterpart(input_repo, out_source_repos_ft.result()) - out_debug_info = self._get_repo_counterpart(input_repo, out_debug_repos_ft.result()) + ubi_repo_sets.append(UbiRepoSet(RepoSet(*in_repos), RepoSet(*out_repos))) - in_repos = (in_rpm, in_source, in_debug_info) - out_repos = (out_rpm, out_source, out_debug_info) + return ubi_repo_sets - # Skip repos sets containing output repos which should not be populated - if not all([r.ubi_population is True for r in out_repos]): - _LOG.debug( - "Skipping %s repo set, population disabled for output repos(s):\n\t%s", - in_rpm.content_set, - "\n\t".join(r.repo_id for r in out_repos if r.ubi_population is False)) - continue + def _get_population_sources(self, repo, input_cs): + src_repos = [] + if repo.population_sources: + fts = [self._executor.submit(self.pulp.search_repo_by_id, r) + for r in repo.population_sources] - repo_pairs.append(UbiRepoSet(RepoSet(*in_repos), RepoSet(*out_repos))) + for ft in as_completed(fts): + repo = ft.result() + if repo: + src_repos.append(repo[0]) + else: + in_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, input_cs) + repo = self.get_repo_counterpart(repo, in_repos_ft.result()) + src_repos.append(repo) - return repo_pairs + return src_repos - def _get_repo_counterpart(self, input_repo, repos_to_match): + @staticmethod + def get_repo_counterpart(input_repo, repos_to_match): + """ + Finds counterpart of input_repo in repos_to_match list by arch and platform_full_version. + """ for repo in repos_to_match: if input_repo.arch == repo.arch and \ input_repo.platform_full_version == repo.platform_full_version: @@ -221,13 +236,13 @@ def __init__(self, pulp, output_repo_set, ubiconfig_item, dry_run, executor): def _match_modules(self): # Add matching modules - fts = {} for module in self.ubiconfig.modules: - fts[self._executor.submit(self.pulp.search_modules, - self.repos.in_repos.rpm, module.name, - str(module.stream))] = \ - (module.name + str(module.stream), module.profiles) + for in_repo_rpm in self.repos.in_repos.rpm: + fts[self._executor.submit(self.pulp.search_modules, + in_repo_rpm, module.name, + str(module.stream))] = \ + (module.name + str(module.stream), module.profiles) for ft in as_completed(fts): input_modules = ft.result() @@ -266,16 +281,19 @@ def _match_module_defaults(self): fts = {} for _, modules in self.repos.modules.items(): for md in modules: - fts[self._executor.submit(self.pulp.search_module_defaults, - self.repos.in_repos.rpm, md.name, - str(md.stream))] = md.name + str(md.stream) + for in_repo_rpm in self.repos.in_repos.rpm: + fts[self._executor.submit(self.pulp.search_module_defaults, + in_repo_rpm, md.name, + str(md.stream))] = md.name + str(md.stream) for ft in as_completed(fts): module_defaults = ft.result() if module_defaults: self.repos.module_defaults[fts[ft]].extend(module_defaults) def _get_pkgs_from_all_modules(self): - modules = self.pulp.search_modules(self.repos.in_repos.rpm) + modules = [] + for in_repo_rpm in self.repos.in_repos.rpm: + modules.extend(self.pulp.search_modules(in_repo_rpm)) pkgs = set() regex = r'\d+:' reg = re.compile(regex) @@ -287,7 +305,7 @@ def _get_pkgs_from_all_modules(self): return pkgs - def _match_packages(self, repo, packages_dict): + def _match_packages(self, input_repos, packages_dict): """ Add matching packages from whitelist Globbing package name is not supported @@ -297,8 +315,9 @@ def _match_packages(self, repo, packages_dict): for package_pattern in self.ubiconfig.packages.whitelist: name = package_pattern.name arch = None if package_pattern.arch in ('*', None) else package_pattern.arch - fts[(self._executor.submit(self.pulp.search_rpms, - repo, name, arch))] = name + for repo in input_repos: + fts[(self._executor.submit(self.pulp.search_rpms, + repo, name, arch))] = name for ft in as_completed(fts): packages = ft.result() @@ -381,12 +400,18 @@ def _create_srpms_output_set(self): if package.sourcerpm_filename is None: _LOG.warning("Package %s doesn't reference its source rpm", package.name) continue + in_repo = [r for r in self.repos.in_repos.rpm + if r.repo_id == package.associate_source_repo_id][0] + + associate_src_repo = UbiPopulate.get_repo_counterpart(in_repo, + self.repos.in_repos.source) self.repos.source_rpms[package.name].append( Package( package.name, package.sourcerpm_filename, - is_modular=package.is_modular) + is_modular=package.is_modular, + src_repo_id=associate_src_repo.repo_id) ) blacklisted_srpms = self.get_blacklisted_packages( @@ -554,7 +579,7 @@ def _associate_unassociate_units(self, action_list): fts = [] for action in action_list: if action.units: - fts.append(self._executor.submit(*action.get_action(self.pulp))) + fts.extend([self._executor.submit(*a) for a in action.get_actions(self.pulp)]) return fts @@ -603,12 +628,13 @@ def log_pulp_actions(self, associations, unassociations): for item in associations: if item.units: for unit in item.units: - _LOG.info("Would associate %s from %s to %s", unit, item.src_repo.repo_id, + _LOG.info("Would associate %s from %s to %s", unit, + unit.associate_source_repo_id, item.dst_repo.repo_id) else: _LOG.info("No association expected for %s from %s to %s", item.TYPE, - item.src_repo.repo_id, + [r.repo_id for r in item.src_repos], item.dst_repo.repo_id) for item in unassociations: @@ -748,26 +774,32 @@ def get_packages_from_module(self, input_modules, package_name=None): continue rpm_filename = rpm_without_epoch + '.rpm' - # Check existence of rpm in binary rpm repo - rpms = self.pulp.search_rpms(self.repos.in_repos.rpm, filename=rpm_filename) + # Check existence of rpm in binary rpm repos + rpms = [] + for in_repo_rpm in self.repos.in_repos.rpm: + res = self.pulp.search_rpms(in_repo_rpm, filename=rpm_filename) + if res: + rpms.extend(res) if rpms: rpms[0].is_modular = True ret_rpms.append(rpms[0]) else: - # Check existence of rpm in debug repo - debug_rpms = self.pulp.search_rpms( - self.repos.in_repos.debug, - filename=rpm_filename, - ) + # Check existence of rpm in debug repos + debug_rpms = [] + for in_repo_debug in self.repos.in_repos.debug: + res = self.pulp.search_rpms(in_repo_debug, filename=rpm_filename) + if res: + debug_rpms.extend(res) if debug_rpms: debug_rpms[0].is_modular = True ret_debug_rpms.append(debug_rpms[0]) else: _LOG.warning("RPM %s is unavailable in input repos %s %s, skipping", - rpm_filename, self.repos.in_repos.rpm.repo_id, - self.repos.in_repos.debug.repo_id + rpm_filename, + [r.repo_id for r in self.repos.in_repos.rpm.repo_id], + [r.repo_id for r in self.repos.in_repos.debug] ) return ret_rpms, ret_debug_rpms diff --git a/ubipop/_pulp_client.py b/ubipop/_pulp_client.py index 98e6958..6c1e6e6 100644 --- a/ubipop/_pulp_client.py +++ b/ubipop/_pulp_client.py @@ -97,6 +97,7 @@ def _search_repo(self, criteria): platform_full_version=notes['platform_full_version'], dist_ids_type_ids=dist_info, ubi_population=ubi_population, + population_sources=notes.get('population_sources') )) return repos @@ -141,7 +142,7 @@ def search_rpms(self, repo, name=None, arch=None, name_globbing=False, filename= ret.raise_for_status() for item in ret.json(): metadata = item['metadata'] - rpms.append(Package(metadata['name'], metadata['filename'], + rpms.append(Package(metadata['name'], metadata['filename'], repo.repo_id, sourcerpm_filename=metadata.get('sourcerpm'))) return rpms @@ -160,7 +161,8 @@ def search_modules(self, repo, name=None, stream=None): metadata = item['metadata'] modules.append(Module(metadata['name'], metadata['stream'], metadata['version'], metadata['context'], - metadata['arch'], metadata['artifacts'], metadata['profiles'])) + metadata['arch'], metadata['artifacts'], metadata['profiles'], + repo.repo_id)) return modules def search_module_defaults(self, repo, name=None, stream=None): @@ -176,7 +178,7 @@ def search_module_defaults(self, repo, name=None, stream=None): for item in ret.json(): metadata = item['metadata'] module_defaults.append(ModuleDefaults(metadata['name'], metadata['stream'], - metadata['profiles'])) + metadata['profiles'], repo.repo_id)) return module_defaults def wait_for_tasks(self, task_id_list, delay=5.0): @@ -311,17 +313,18 @@ def publish_repo(self, repo): class Repo(object): def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids, - ubi_population): + ubi_population, population_sources): self.repo_id = repo_id self.arch = arch self.content_set = content_set self.platform_full_version = platform_full_version self.distributors_ids_type_ids_tuples = dist_ids_type_ids self.ubi_population = ubi_population + self.population_sources = population_sources class Package(object): - def __init__(self, name, filename, sourcerpm_filename=None, is_modular=False): + def __init__(self, name, filename, src_repo_id, sourcerpm_filename=None, is_modular=False): self.name = name self.filename = filename self.sourcerpm_filename = sourcerpm_filename @@ -329,6 +332,7 @@ def __init__(self, name, filename, sourcerpm_filename=None, is_modular=False): # return name, ver, rel, epoch, arch _, self.version, self.release, self.epoch, _ = split_filename(self.filename) self.evr_tuple = (self.epoch, self.version, self.release) + self.associate_source_repo_id = src_repo_id def __lt__(self, other): return label_compare(self.evr_tuple, other.evr_tuple) < 0 @@ -353,7 +357,7 @@ def __str__(self): class Module(object): - def __init__(self, name, stream, version, context, arch, packages, profiles): + def __init__(self, name, stream, version, context, arch, packages, profiles, src_repo_id): self.name = name self.stream = stream self.version = version @@ -361,6 +365,7 @@ def __init__(self, name, stream, version, context, arch, packages, profiles): self.arch = arch self.packages = packages self.profiles = profiles + self.associate_source_repo_id = src_repo_id @property def nsvca(self): @@ -385,10 +390,11 @@ class ModuleDefaults(object): if someone asks to enable 'ruby:2.5' for some repo without specifing profiles, will get 'common' profile by defualt """ - def __init__(self, name, stream, profiles): + def __init__(self, name, stream, profiles, src_repo_id): self.name = name self.stream = stream self.profiles = profiles # a dict such as {'4.046':['common']} + self.associate_source_repo_id = src_repo_id def __str__(self): return self.name diff --git a/ubipop/_utils.py b/ubipop/_utils.py index 62387fa..adbe294 100644 --- a/ubipop/_utils.py +++ b/ubipop/_utils.py @@ -37,56 +37,85 @@ def __init__(self, units, dst_repo): self.units = units self.dst_repo = dst_repo - def get_action(self, pulp_client_inst): + def get_actions(self, pulp_client_inst): raise NotImplementedError class AssociateAction(PulpAction): - def __init__(self, units, dst_repo, src_repo): + def __init__(self, units, dst_repo, src_repos): super(AssociateAction, self).__init__(units, dst_repo) - self.src_repo = src_repo + self.src_repos = src_repos - def get_action(self, pulp_client_inst): + def _map_src_repo_to_unit(self): + src_repo_unit_map = {} + for unit in self.units: + src_repo_unit_map.setdefault(unit.associate_source_repo_id, []).append(unit) + + return src_repo_unit_map + + def _get_repo_obj(self, repo_id): + for repo in self.src_repos: + if repo_id == repo.repo_id: + return repo + + def get_actions(self, pulp_client_inst): raise NotImplementedError class AssociateActionModules(AssociateAction): TYPE = "modules" - def get_action(self, pulp_client_inst): - return pulp_client_inst.associate_modules, self.src_repo, self.dst_repo, self.units + def get_actions(self, pulp_client_inst): + actions = [] + for src_repo_id, units in self._map_src_repo_to_unit().items(): + actions.append((pulp_client_inst.associate_modules, self._get_repo_obj(src_repo_id), + self.dst_repo, units)) + + return actions class UnassociateActionModules(PulpAction): TYPE = "modules" - def get_action(self, pulp_client_inst): - return pulp_client_inst.unassociate_modules, self.dst_repo, self.units + def get_actions(self, pulp_client_inst): + return [(pulp_client_inst.unassociate_modules, self.dst_repo, self.units)] class AssociateActionModuleDefaults(AssociateAction): TYPE = "module_defaults" - def get_action(self, pulp_client_inst): - return pulp_client_inst.associate_module_defaults, self.src_repo, self.dst_repo, self.units + def get_actions(self, pulp_client_inst): + actions = [] + for src_repo_id, units in self._map_src_repo_to_unit().items(): + actions.append((pulp_client_inst.associate_module_defaults, + self._get_repo_obj(src_repo_id), + self.dst_repo, units)) + + return actions class UnassociateActionModuleDefaults(PulpAction): TYPE = "module_defaults" - def get_action(self, pulp_client_inst): - return pulp_client_inst.unassociate_module_defaults, self.dst_repo, self.units + def get_actions(self, pulp_client_inst): + return [(pulp_client_inst.unassociate_module_defaults, self.dst_repo, self.units)] class AssociateActionRpms(AssociateAction): TYPE = "packages" - def get_action(self, pulp_client_inst): - return pulp_client_inst.associate_packages, self.src_repo, self.dst_repo, self.units + def get_actions(self, pulp_client_inst): + actions = [] + for src_repo_id, units in self._map_src_repo_to_unit().items(): + actions.append( + (pulp_client_inst.associate_packages, self._get_repo_obj(src_repo_id), + self.dst_repo, units)) + + return actions class UnassociateActionRpms(PulpAction): TYPE = "packages" - def get_action(self, pulp_client_inst): - return pulp_client_inst.unassociate_packages, self.dst_repo, self.units + def get_actions(self, pulp_client_inst): + return [(pulp_client_inst.unassociate_packages, self.dst_repo, self.units)]