diff --git a/changelog/66559.changed.md b/changelog/66559.changed.md new file mode 100644 index 00000000000..4772189f18c --- /dev/null +++ b/changelog/66559.changed.md @@ -0,0 +1 @@ +Included Salt extensions in Salt-SSH thin archive diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 7c8ad3e8a07..ef1d0e7a93d 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -1667,6 +1667,8 @@ Pass a list of importable Python modules that are typically located in the `site-packages` Python directory so they will be also always included into the Salt Thin, once generated. +.. conf_master:: min_extra_mods + ``min_extra_mods`` ------------------ @@ -1674,6 +1676,47 @@ Default: None Identical as `thin_extra_mods`, only applied to the Salt Minimal. +.. conf_master:: thin_exclude_saltexts + +``thin_exclude_saltexts`` +------------------------- + +Default: False + +By default, Salt-SSH autodiscovers Salt extensions in the current Python environment +and adds them to the Salt Thin. This disables that behavior. + +.. note:: + + When the list of modules/extensions to include in the Salt Thin changes + for any reason (e.g. Saltext was added/removed, :conf_master:`thin_exclude_saltexts`, + :conf_master:`thin_saltext_allowlist` or :conf_master:`thin_saltext_blocklist` + was changed), you typically need to regenerate the Salt Thin by passing + ``--regen-thin`` to the next Salt-SSH invocation. + +.. conf_master:: thin_saltext_allowlist + +``thin_saltext_allowlist`` +-------------------------- + +Default: None + +A list of Salt extension **distribution** names which are allowed to be +included in the Salt Thin (when :conf_master:`thin_exclude_saltexts` +is inactive) and they are discovered. Any extension not in this list +will be excluded. If unset, all discovered extensions are added, +unless present in :conf_master:`thin_saltext_blocklist`. + +.. conf_master:: thin_saltext_blocklist + +``thin_saltext_blocklist`` +-------------------------- + +Default: None + +A list of Salt extension **distribution** names which should never be +included in the Salt Thin (when :conf_master:`thin_exclude_saltexts` +is inactive). .. _master-security-settings: diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index c20bbd88719..6064b7b0064 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -315,6 +315,9 @@ def __init__(self, opts): extra_mods=self.opts.get("thin_extra_mods"), overwrite=self.opts["regen_thin"], extended_cfg=self.opts.get("ssh_ext_alternatives"), + exclude_saltexts=self.opts.get("thin_exclude_saltexts", False), + saltext_allowlist=self.opts.get("thin_saltext_allowlist"), + saltext_blocklist=self.opts.get("thin_saltext_blocklist"), ) self.mods = mod_data(self.fsclient) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 72f5c44f2b7..27a3d3d8000 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -907,6 +907,9 @@ def _gather_buffer_space(): # Thin and minimal Salt extra modules "thin_extra_mods": str, "min_extra_mods": str, + "thin_exclude_saltexts": bool, + "thin_saltext_allowlist": (type(None), list), + "thin_saltext_blocklist": list, # Default returners minion should use. List or comma-delimited string "return": (str, list), # TLS/SSL connection options. This could be set to a dictionary containing arguments @@ -1630,6 +1633,9 @@ def _gather_buffer_space(): "memcache_debug": False, "thin_extra_mods": "", "min_extra_mods": "", + "thin_exclude_saltexts": False, + "thin_saltext_allowlist": None, + "thin_saltext_blocklist": [], "ssl": None, "extmod_whitelist": {}, "extmod_blacklist": {}, diff --git a/salt/utils/hashutils.py b/salt/utils/hashutils.py index 4969465acbe..5364c883db7 100644 --- a/salt/utils/hashutils.py +++ b/salt/utils/hashutils.py @@ -196,6 +196,19 @@ def add(self, path): for chunk in iter(lambda: ifile.read(self.__buff), b""): self.__digest.update(chunk) + def add_data(self, data): + """ + Update digest with the file content directly. + + :param data: + :return: + """ + try: + data = data.encode("utf8") + except AttributeError: + pass + self.__digest.update(data) + def digest(self): """ Get digest. diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index fc2eabc9a24..b269745641e 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -3176,6 +3176,13 @@ def _mixin_setup(self): "to be included into Thin Salt." ), ) + self.add_option( + "--thin-exclude-saltexts", + default=False, + action="store_true", + dest="thin_exclude_saltexts", + help="Exclude Salt extension modules from generated Thin Salt.", + ) self.add_option( "-v", "--verbose", diff --git a/salt/utils/thin.py b/salt/utils/thin.py index a760fcc02f5..8596253749e 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -2,9 +2,12 @@ Generate the salt thin tarball from the installed python files """ +import contextlib import contextvars as py_contextvars import copy import importlib.util +import inspect +import io import logging import os import shutil @@ -13,6 +16,7 @@ import sys import tarfile import tempfile +import types import zipfile import distro @@ -25,6 +29,7 @@ import salt import salt.exceptions +import salt.utils.entrypoints import salt.utils.files import salt.utils.hashutils import salt.utils.json @@ -224,18 +229,19 @@ def _is_shareable(mod): return os.path.basename(mod) in shareable -def _add_dependency(container, obj): +def _add_dependency(container, obj, namespace=None): """ Add a dependency to the top list. :param obj: :param is_file: + :param namespace: Optional tuple of parent namespaces for namespace packages :return: """ if os.path.basename(obj.__file__).split(".")[0] == "__init__": - container.append(os.path.dirname(obj.__file__)) + container.append((os.path.dirname(obj.__file__), namespace)) else: - container.append(obj.__file__.replace(".pyc", ".py")) + container.append((obj.__file__.replace(".pyc", ".py"), None)) def gte(): @@ -454,9 +460,9 @@ def get_tops(extra_mods="", so_mods=""): moddir, modname = os.path.split(locals()[mod].__file__) base, _ = os.path.splitext(modname) if base == "__init__": - tops.append(moddir) + tops.append((moddir, None)) else: - tops.append(os.path.join(moddir, base + ".py")) + tops.append((os.path.join(moddir, base + ".py"), None)) except ImportError as err: log.error( 'Unable to import extra-module "%s": %s', mod, err, exc_info=True @@ -465,8 +471,8 @@ def get_tops(extra_mods="", so_mods=""): for mod in [m for m in so_mods.split(",") if m]: try: locals()[mod] = __import__(mod) - tops.append(locals()[mod].__file__) - except ImportError as err: + tops.append((locals()[mod].__file__, None)) + except ImportError: log.error('Unable to import so-module "%s"', mod, exc_info=True) return tops @@ -583,6 +589,138 @@ def _pack_alternative(extended_cfg, digest_collector, tfp): tfp.add(os.path.join(root, name), arcname=arcname) +@contextlib.contextmanager +def _catch_entry_points_exception(entry_point): + context = types.SimpleNamespace(exception_caught=False) + try: + yield context + except Exception as exc: # pylint: disable=broad-except + context.exception_caught = True + entry_point_details = salt.utils.entrypoints.name_and_version_from_entry_point( + entry_point + ) + log.error( + "Error processing Salt Extension %s(version: %s): %s", + entry_point_details.name, + entry_point_details.version, + exc, + exc_info_on_loglevel=logging.DEBUG, + ) + + +def _get_package_root_mod(mod): + """ + Given an imported module, find the topmost module + that is not a namespace package. + Returns a tuple of (root_mod, tuple), where the + second value is a tuple of parent namespaces. + Needed for saltext discovery if the entrypoint is not + part of the root module. + """ + parts = mod.__name__.split(".") + level = 0 + while level < len(parts): + root_mod_name = ".".join(parts[: level + 1]) + root_mod = sys.modules[root_mod_name] + # importlib.machinery.NamespaceLoader requires Python 3.11+ + if type(root_mod.__path__) is list: + return root_mod, tuple(parts[:level]) + level += 1 + raise RuntimeError(f"Unable to determine package root mod for {mod}") + + +def _discover_saltexts(allowlist=None, blocklist=None): + mods = [] + loaded_saltexts = {} + blocklist = blocklist or [] + + for entry_point in salt.utils.entrypoints.iter_entry_points("salt.loader"): + if allowlist is not None and entry_point.dist.name not in allowlist: + log.debug( + "Skipping entry point '%s' of '%s': not in allowlist", + entry_point.name, + entry_point.dist.name, + ) + continue + if entry_point.dist.name in blocklist: + log.debug( + "Skipping entry point '%s' of '%s': in blocklist", + entry_point.name, + entry_point.dist.name, + ) + continue + with _catch_entry_points_exception(entry_point) as ctx: + loaded_entry_point = entry_point.load() + if ctx.exception_caught: + continue + if not isinstance(loaded_entry_point, (types.FunctionType, types.ModuleType)): + log.debug( + "Skipping entry point '%s' of '%s': Not a function/module", + entry_point.name, + entry_point.dist.name, + ) + continue + if entry_point.dist.name not in loaded_saltexts: + try: + # We could get this via entry_point.dist._path.name, but that is hacky + dist_name = next( + iter( + file.parent.name + for file in entry_point.dist.files + if file.parent.suffix == ".dist-info" + ) + ) + except StopIteration: + # This should never happen since we have the data to arrive here + log.debug( + "Skipping entry point '%s' of '%s': Failed discovering dist-info", + entry_point.name, + entry_point.dist.name, + ) + continue + loaded_saltexts[entry_point.dist.name] = { + "name": dist_name, + "entrypoints": {}, + } + + mod = inspect.getmodule(loaded_entry_point) + with _catch_entry_points_exception(entry_point) as ctx: + root_mod, namespace = _get_package_root_mod(mod) + if ctx.exception_caught: + continue + + loaded_saltexts[entry_point.dist.name]["entrypoints"][ + entry_point.name + ] = entry_point.value + _add_dependency(mods, root_mod, namespace=namespace) + + # We need the mods to be in a deterministic order for the hash digest later + return list(sorted(set(mods))), loaded_saltexts + + +def _pack_saltext_dists(saltext_dists, digest_collector, tfp): + """ + Take the output of discover_saltexts and add appropriate entry point definitions + for the loader to be able to discover the extensions. + """ + # Again, we need this to execute in a deterministic order for the hash digest + for dist in sorted(saltext_dists): + data = saltext_dists[dist] + if not data["entrypoints"]: + log.debug("No entrypoints for distribution '%s'", dist) + continue + log.debug("Packing entrypoints for distribution '%s'", dist) + defs = ( + "[salt.loader]\n" + + "\n".join(f"{name} = {val}" for name, val in data["entrypoints"].items()) + + "\n" + ).encode("utf-8") + info = tarfile.TarInfo(name="py3/" + data["name"] + "/entry_points.txt") + info.size = len(defs) + tfp.addfile(tarinfo=info, fileobj=io.BytesIO(defs)) + digest_collector.add_data(defs) + + def gen_thin( cachedir, extra_mods="", @@ -591,6 +729,9 @@ def gen_thin( absonly=True, compress="gzip", extended_cfg=None, + exclude_saltexts=False, + saltext_allowlist=None, + saltext_blocklist=None, ): """ Generate the salt-thin tarball and print the location of the tarball @@ -657,9 +798,24 @@ def gen_thin( else: return thintar - tops_failure_msg = "Failed %s tops for Python binary %s." tops_py_version_mapping = {} tops = get_tops(extra_mods=extra_mods, so_mods=so_mods) + if not exclude_saltexts: + if compress != "gzip": + # The reason being that we're generating the filtered entrypoints + # and adding them from memory - if this is deemed as unnecessary, + # we would need the absolute path to the entry_points.txt file for + # the distribution, which is only available as a protected attribute. + # Salt-SSH never overrides `compress` from gzip though. + log.warning("Cannot include saltexts in thin when compression is not gzip") + exclude_saltexts = True + else: + mods, saltext_dists = _discover_saltexts( + allowlist=saltext_allowlist, blocklist=saltext_blocklist + ) + # Deduplicate in case some saltexts were passed in thin_extra_modules + tops.extend(mod for mod in mods if mod not in tops) + tops_py_version_mapping[sys.version_info.major] = tops with salt.utils.files.fopen(pymap_cfg, "wb") as fp_: @@ -688,7 +844,7 @@ def gen_thin( # Pack default data log.debug("Packing default libraries based on current Salt version") for py_ver, tops in tops_py_version_mapping.items(): - for top in tops: + for top, namespace in tops: if absonly and not os.path.isabs(top): continue base = os.path.basename(top) @@ -715,7 +871,9 @@ def gen_thin( for name in files: if not name.endswith((".pyc", ".pyo")): digest_collector.add(os.path.join(root, name)) - arcname = os.path.join(site_pkg_dir, root, name) + arcname = os.path.join( + site_pkg_dir, *(namespace or ()), root, name + ) if hasattr(tfp, "getinfo"): try: # This is a little slow but there's no clear way to detect duplicates @@ -732,6 +890,9 @@ def gen_thin( shutil.rmtree(tempdir) tempdir = None + if not exclude_saltexts: + log.debug("Packing saltext distribution entrypoints") + _pack_saltext_dists(saltext_dists, digest_collector, tfp) if extended_cfg: log.debug("Packing libraries based on alternative Salt versions") _pack_alternative(extended_cfg, digest_collector, tfp) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index d1cf35d9d76..cf299415c7a 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -99,6 +99,12 @@ discovery: false #enable_ssh_minions: True #ignore_host_keys: True +# Ensure pytest-salt-factories is not included +# in the thin tar during integration tests +# (it defines a saltext, which are autodiscovered by default) +thin_saltext_blocklist: + - pytest-salt-factories + sdbetcd: driver: etcd etcd.host: 127.0.0.1 diff --git a/tests/pytests/integration/ssh/test_saltext.py b/tests/pytests/integration/ssh/test_saltext.py new file mode 100644 index 00000000000..cbb0ff5ede8 --- /dev/null +++ b/tests/pytests/integration/ssh/test_saltext.py @@ -0,0 +1,120 @@ +import json +import shutil +from pathlib import Path + +import pytest + +from tests.support.helpers import SaltVirtualEnv +from tests.support.pytest.helpers import FakeSaltExtension + + +@pytest.fixture(scope="module") +def salt_extension(tmp_path_factory): + with FakeSaltExtension( + tmp_path_factory=tmp_path_factory, name="salt-ext-ssh-test" + ) as extension: + yield extension + + +@pytest.fixture(scope="module") +def namespaced_salt_extension(tmp_path_factory): + with FakeSaltExtension( + tmp_path_factory=tmp_path_factory, + name="saltext.ssh-test2", + virtualname="barbaz", + ) as extension: + yield extension + + +@pytest.fixture(scope="module") +def namespaced_salt_extension_2(tmp_path_factory): + with FakeSaltExtension( + tmp_path_factory=tmp_path_factory, + name="saltext.ssh-test3", + virtualname="wut", + ) as extension: + yield extension + + +@pytest.fixture(scope="module") +def venv( + tmp_path_factory, + salt_extension, + namespaced_salt_extension, + namespaced_salt_extension_2, +): + venv_dir = tmp_path_factory.mktemp("saltext-ssh-test-venv") + saltexts = (salt_extension, namespaced_salt_extension, namespaced_salt_extension_2) + try: + with SaltVirtualEnv(venv_dir=venv_dir) as _venv: + for saltext in saltexts: + _venv.install(str(saltext.srcdir)) + installed_packages = _venv.get_installed_packages() + for saltext in saltexts: + assert saltext.name in installed_packages + yield _venv + finally: + shutil.rmtree(venv_dir, ignore_errors=True) + + +@pytest.fixture(params=({},)) +def saltext_conf(request, salt_master): + with pytest.helpers.temp_file( + "saltext_ssh.conf", + json.dumps(request.param), + Path(salt_master.config_dir) / "master.d", + ): + yield request.param + + +@pytest.fixture +def args(venv, salt_master, salt_ssh_roster_file, sshd_config_dir): + return [ + venv.venv_bin_dir / "salt-ssh", + f"--config-dir={salt_master.config_dir}", + f"--roster-file={salt_ssh_roster_file}", + f"--priv={sshd_config_dir / 'client_key'}", + "--regen-thin", + "localhost", + ] + + +@pytest.mark.parametrize( + "saltext_conf", + ( + {}, + {"thin_saltext_allowlist": ["salt-ext-ssh-test", "saltext.ssh-test3"]}, + {"thin_saltext_blocklist": ["saltext.ssh-test2"]}, + ), + indirect=True, +) +def test_saltexts_are_available_on_target(venv, args, saltext_conf): + ext1_args = args + ["foobar.echo1", "foo"] + res = venv.run(*ext1_args, check=True) + assert res.stdout == "localhost:\n foo\n" + ext2_args = args + ["barbaz.echo1", "bar"] + res = venv.run(*ext2_args, check=False) + if ( + "thin_saltext_allowlist" not in saltext_conf + and "thin_saltext_blocklist" not in saltext_conf + ): + assert res.returncode == 0 + assert res.stdout == "localhost:\n bar\n" + else: + assert res.returncode > 0 + assert "'barbaz.echo1' is not available" in res.stdout + ext3_args = args + ["wut.echo1", "wat"] + res = venv.run(*ext3_args, check=True) + assert res.stdout == "localhost:\n wat\n" + + +@pytest.mark.usefixtures("saltext_conf") +@pytest.mark.parametrize( + "saltext_conf", ({"thin_exclude_saltexts": True},), indirect=True +) +def test_saltexts_can_be_excluded(venv, args): + for ext in ("foobar", "barbaz", "wut"): + ext_args = args + [f"{ext}.echo1", "foo"] + res = venv.run(*ext_args, check=False) + assert res.returncode > 0 + assert f"'{ext}.echo1' is not available" in res.stdout diff --git a/tests/pytests/unit/utils/test_thin.py b/tests/pytests/unit/utils/test_thin.py index adab8ecf68d..8398dfea02d 100644 --- a/tests/pytests/unit/utils/test_thin.py +++ b/tests/pytests/unit/utils/test_thin.py @@ -1,9 +1,15 @@ +import importlib +import os +import sys + import pytest +import saltfactories.utils.saltext import salt.exceptions import salt.utils.stringutils import salt.utils.thin from tests.support.mock import MagicMock, patch +from tests.support.pytest.helpers import FakeSaltExtension def _mock_popen(return_value=None, side_effect=None, returncode=0): @@ -77,3 +83,45 @@ def test_get_ext_tops(version): else: assert not [x for x in ret["namespace"]["dependencies"] if "distro" in x] assert [x for x in ret["namespace"]["dependencies"] if "msgpack" in x] + + +def test_get_package_root_mod(): + res = salt.utils.thin._get_package_root_mod(saltfactories.utils.saltext) + assert res[0] is saltfactories + assert res[1] == () + + +@pytest.fixture +def namespaced_saltext(tmp_path_factory): + with FakeSaltExtension( + tmp_path_factory=tmp_path_factory, + name="saltext.wut", + ) as extension: + try: + sys.path.insert(0, str(extension.srcdir / "src")) + yield extension + finally: + sys.path.pop(0) + + +def test_get_namespaced_package_root_mod(namespaced_saltext): + saltext = importlib.import_module(namespaced_saltext.name) + res = salt.utils.thin._get_package_root_mod(saltext) + assert res[0].__name__ == namespaced_saltext.name + assert res[1] == ("saltext",) + + +def test_discover_saltexts(): + """ + pytest-salt-factories provides a saltext, which can be discovered here. + """ + mods, dists = salt.utils.thin._discover_saltexts() + assert mods + assert any(mod.endswith(f"{os.sep}saltfactories") and not ns for mod, ns in mods) + assert dists + dist = "pytest-salt-factories" + assert dist in dists + assert "entrypoints" in dists[dist] + assert "name" in dists[dist] + assert dists[dist]["name"].startswith("pytest_salt_factories") + assert dists[dist]["name"].endswith(".dist-info") diff --git a/tests/support/pytest/helpers.py b/tests/support/pytest/helpers.py index d544837e73b..d959cb9fdce 100644 --- a/tests/support/pytest/helpers.py +++ b/tests/support/pytest/helpers.py @@ -445,6 +445,7 @@ class FakeSaltExtension: name = attr.ib() pkgname = attr.ib(init=False) srcdir = attr.ib(init=False) + virtualname = attr.ib(default="foobar") @srcdir.default def _srcdir(self): @@ -480,9 +481,9 @@ def _laydown_files(self): if not setup_cfg.exists(): setup_cfg.write_text( textwrap.dedent( - """\ + f"""\ [metadata] - name = {0} + name = {self.name} version = 1.0 description = Salt Extension Test author = Pedro @@ -503,27 +504,30 @@ def _laydown_files(self): [options] zip_safe = False include_package_data = True - packages = find: + package_dir = + =src + packages = find{'_namespace' if '.' in self.pkgname else ''}: python_requires = >= 3.5 setup_requires = wheel setuptools>=50.3.2 + [options.packages.find] + where = src + [options.entry_points] salt.loader= - module_dirs = {1} - runner_dirs = {1}.loader:get_runner_dirs - states_dirs = {1}.loader:get_state_dirs - wheel_dirs = {1}.loader:get_new_style_entry_points - """.format( - self.name, self.pkgname - ) + module_dirs = {self.pkgname} + runner_dirs = {self.pkgname}.loader:get_runner_dirs + states_dirs = {self.pkgname}.loader:get_state_dirs + wheel_dirs = {self.pkgname}.loader:get_new_style_entry_points + """ ) ) - extension_package_dir = self.srcdir / self.pkgname + extension_package_dir = self.srcdir.joinpath("src", *self.pkgname.split(".")) if not extension_package_dir.exists(): - extension_package_dir.mkdir() + extension_package_dir.mkdir(parents=True) extension_package_dir.joinpath("__init__.py").write_text("") extension_package_dir.joinpath("loader.py").write_text( textwrap.dedent( @@ -550,10 +554,10 @@ def get_new_style_entry_points(): runners1_dir = extension_package_dir / "runners1" runners1_dir.mkdir() runners1_dir.joinpath("__init__.py").write_text("") - runners1_dir.joinpath("foobar1.py").write_text( + runners1_dir.joinpath(f"{self.virtualname}1.py").write_text( textwrap.dedent( - """\ - __virtualname__ = "foobar" + f"""\ + __virtualname__ = "{self.virtualname}" def __virtual__(): return True @@ -567,10 +571,10 @@ def echo1(string): runners2_dir = extension_package_dir / "runners2" runners2_dir.mkdir() runners2_dir.joinpath("__init__.py").write_text("") - runners2_dir.joinpath("foobar2.py").write_text( + runners2_dir.joinpath(f"{self.virtualname}2.py").write_text( textwrap.dedent( - """\ - __virtualname__ = "foobar" + f"""\ + __virtualname__ = "{self.virtualname}" def __virtual__(): return True @@ -584,10 +588,10 @@ def echo2(string): modules_dir = extension_package_dir / "modules" modules_dir.mkdir() modules_dir.joinpath("__init__.py").write_text("") - modules_dir.joinpath("foobar1.py").write_text( + modules_dir.joinpath(f"{self.virtualname}1.py").write_text( textwrap.dedent( - """\ - __virtualname__ = "foobar" + f"""\ + __virtualname__ = "{self.virtualname}" def __virtual__(): return True @@ -597,10 +601,10 @@ def echo1(string): """ ) ) - modules_dir.joinpath("foobar2.py").write_text( + modules_dir.joinpath(f"{self.virtualname}2.py").write_text( textwrap.dedent( - """\ - __virtualname__ = "foobar" + f"""\ + __virtualname__ = "{self.virtualname}" def __virtual__(): return True @@ -614,10 +618,10 @@ def echo2(string): wheel_dir = extension_package_dir / "the_wheel_modules" wheel_dir.mkdir() wheel_dir.joinpath("__init__.py").write_text("") - wheel_dir.joinpath("foobar1.py").write_text( + wheel_dir.joinpath(f"{self.virtualname}1.py").write_text( textwrap.dedent( - """\ - __virtualname__ = "foobar" + f"""\ + __virtualname__ = "{self.virtualname}" def __virtual__(): return True @@ -627,10 +631,10 @@ def echo1(string): """ ) ) - wheel_dir.joinpath("foobar2.py").write_text( + wheel_dir.joinpath(f"{self.virtualname}2.py").write_text( textwrap.dedent( - """\ - __virtualname__ = "foobar" + f"""\ + __virtualname__ = "{self.virtualname}" def __virtual__(): return True @@ -644,16 +648,16 @@ def echo2(string): states_dir = extension_package_dir / "states1" states_dir.mkdir() states_dir.joinpath("__init__.py").write_text("") - states_dir.joinpath("foobar1.py").write_text( + states_dir.joinpath(f"{self.virtualname}1.py").write_text( textwrap.dedent( - """\ - __virtualname__ = "foobar" + f"""\ + __virtualname__ = "{self.virtualname}" def __virtual__(): return True def echoed(string): - ret = {"name": name, "changes": {}, "result": True, "comment": string} + ret = {{"name": name, "changes": {{}}, "result": True, "comment": string}} return ret """ ) @@ -662,10 +666,10 @@ def echoed(string): utils_dir = extension_package_dir / "utils" utils_dir.mkdir() utils_dir.joinpath("__init__.py").write_text("") - utils_dir.joinpath("foobar1.py").write_text( + utils_dir.joinpath(f"{self.virtualname}1.py").write_text( textwrap.dedent( - """\ - __virtualname__ = "foobar" + f"""\ + __virtualname__ = "{self.virtualname}" def __virtual__(): return True diff --git a/tests/unit/utils/test_thin.py b/tests/unit/utils/test_thin.py index b9c7781af6d..23e5939c117 100644 --- a/tests/unit/utils/test_thin.py +++ b/tests/unit/utils/test_thin.py @@ -325,7 +325,7 @@ def test_add_dep_path(self): for pth in ["/foo/bar.py", "/something/else/__init__.py"]: thin._add_dependency(container, type("obj", (), {"__file__": pth})()) assert "__init__" not in container[1] - assert container == ["/foo/bar.py", "/something/else"] + assert container == [("/foo/bar.py", None), ("/something/else", None)] def test_thin_path(self): """ @@ -479,7 +479,7 @@ def test_get_tops(self): if salt.utils.thin.has_immutables: base_tops.extend(["immutables"]) tops = [] - for top in thin.get_tops(extra_mods="foo,bar"): + for top, _ in thin.get_tops(extra_mods="foo,bar"): if top.find("/") != -1: spl = "/" else: @@ -594,7 +594,7 @@ def test_get_tops_extra_mods(self): MagicMock(side_effect=[type("foo", (), foo), type("bar", (), bar)]), ): tops = [] - for top in thin.get_tops(extra_mods="foo,bar"): + for top, _ in thin.get_tops(extra_mods="foo,bar"): if top.find("/") != -1: spl = "/" else: @@ -712,7 +712,7 @@ def test_get_tops_so_mods(self): ), ): tops = [] - for top in thin.get_tops(so_mods="foo,bar"): + for top, _ in thin.get_tops(so_mods="foo,bar"): if top.find("/") != -1: spl = "/" else: @@ -776,7 +776,12 @@ def test_gen_thin_fails_ancient_python_version(self): @patch("salt.utils.files.fopen", MagicMock()) @patch("salt.utils.thin._get_salt_call", MagicMock()) @patch("salt.utils.thin._get_ext_namespaces", MagicMock()) - @patch("salt.utils.thin.get_tops", MagicMock(return_value=["/foo3", "/bar3"])) + @patch( + "salt.utils.thin.get_tops", + MagicMock( + return_value=[("/foo3", None), ("/bar3", None), ("/ns/package", ("ns",))] + ), + ) @patch("salt.utils.thin.get_ext_tops", MagicMock(return_value={})) @patch("salt.utils.thin.os.path.isfile", MagicMock()) @patch("salt.utils.thin.os.path.isdir", MagicMock(return_value=True)) @@ -825,9 +830,13 @@ def test_gen_thin_compression_fallback_py3(self): @patch("salt.utils.thin.log", MagicMock()) @patch("salt.utils.thin.os.makedirs", MagicMock()) @patch("salt.utils.files.fopen", MagicMock()) + @patch("salt.utils.thin._discover_saltexts", MagicMock(return_value=([], {}))) @patch("salt.utils.thin._get_salt_call", MagicMock()) @patch("salt.utils.thin._get_ext_namespaces", MagicMock()) - @patch("salt.utils.thin.get_tops", MagicMock(return_value=["/foo3", "/bar3"])) + @patch( + "salt.utils.thin.get_tops", + MagicMock(return_value=[("/foo3", None), ("/bar3", None)]), + ) @patch("salt.utils.thin.get_ext_tops", MagicMock(return_value={})) @patch("salt.utils.thin.os.path.isfile", MagicMock()) @patch("salt.utils.thin.os.path.isdir", MagicMock(return_value=False)) @@ -875,9 +884,15 @@ def test_gen_thin_control_files_written_py3(self): @patch("salt.utils.thin.log", MagicMock()) @patch("salt.utils.thin.os.makedirs", MagicMock()) @patch("salt.utils.files.fopen", MagicMock()) + @patch("salt.utils.thin._discover_saltexts", MagicMock(return_value=([], {}))) @patch("salt.utils.thin._get_salt_call", MagicMock()) @patch("salt.utils.thin._get_ext_namespaces", MagicMock()) - @patch("salt.utils.thin.get_tops", MagicMock(return_value=["/salt", "/bar3"])) + @patch( + "salt.utils.thin.get_tops", + MagicMock( + return_value=[("/salt", None), ("/bar3", None), ("/ns/package", ("ns",))] + ), + ) @patch("salt.utils.thin.get_ext_tops", MagicMock(return_value={})) @patch("salt.utils.thin.os.path.isfile", MagicMock()) @patch("salt.utils.thin.os.path.isdir", MagicMock(return_value=True)) @@ -921,8 +936,12 @@ def test_gen_thin_main_content_files_written_py3(self): for py in ("py3", "pyall"): for i in range(1, 4): files.append(os.path.join(py, "root", f"r{i}")) + if py == "py3": + files.append(os.path.join(py, "ns", "root", f"r{i}")) for i in range(4, 7): files.append(os.path.join(py, "root2", f"r{i}")) + if py == "py3": + files.append(os.path.join(py, "ns", "root2", f"r{i}")) for cl in thin.tarfile.open().method_calls[:-6]: arcname = cl[2].get("arcname") self.assertIn(arcname, files) @@ -933,6 +952,7 @@ def test_gen_thin_main_content_files_written_py3(self): @patch("salt.utils.thin.log", MagicMock()) @patch("salt.utils.thin.os.makedirs", MagicMock()) @patch("salt.utils.files.fopen", MagicMock()) + @patch("salt.utils.thin._discover_saltexts", MagicMock(return_value=([], {}))) @patch("salt.utils.thin._get_salt_call", MagicMock()) @patch("salt.utils.thin._get_ext_namespaces", MagicMock()) @patch("salt.utils.thin.get_tops", MagicMock(return_value=[])) @@ -1060,9 +1080,13 @@ def test_get_supported_py_config_ext_tops(self): @patch("salt.utils.thin.log", MagicMock()) @patch("salt.utils.thin.os.makedirs", MagicMock()) @patch("salt.utils.files.fopen", MagicMock()) + @patch("salt.utils.thin._discover_saltexts", MagicMock(return_value=([], {}))) @patch("salt.utils.thin._get_salt_call", MagicMock()) @patch("salt.utils.thin._get_ext_namespaces", MagicMock()) - @patch("salt.utils.thin.get_tops", MagicMock(return_value=["/foo3", "/bar3"])) + @patch( + "salt.utils.thin.get_tops", + MagicMock(return_value=[("/foo3", None), ("/bar3", None)]), + ) @patch("salt.utils.thin.get_ext_tops", MagicMock(return_value={})) @patch("salt.utils.thin.os.path.isfile", MagicMock()) @patch("salt.utils.thin.os.path.isdir", MagicMock(return_value=False))