diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 2c072df51..87bbaf979 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -729,7 +729,10 @@ def run_prepare_scripts(context: Context, build: bool) -> None: with ( mount_build_overlay(context) if build else contextlib.nullcontext(), - finalize_source_mounts(context.config, ephemeral=context.config.build_sources_ephemeral) as sources, + finalize_source_mounts( + context.config, + ephemeral=bool(context.config.build_sources_ephemeral), + ) as sources, finalize_config_json(context.config) as json, ): if build: @@ -871,7 +874,10 @@ def run_postinst_scripts(context: Context) -> None: env |= context.config.environment with ( - finalize_source_mounts(context.config, ephemeral=context.config.build_sources_ephemeral) as sources, + finalize_source_mounts( + context.config, + ephemeral=bool(context.config.build_sources_ephemeral), + ) as sources, finalize_config_json(context.config) as json, ): for script in context.config.postinst_scripts: @@ -937,7 +943,10 @@ def run_finalize_scripts(context: Context) -> None: env |= context.config.environment with ( - finalize_source_mounts(context.config, ephemeral=context.config.build_sources_ephemeral) as sources, + finalize_source_mounts( + context.config, + ephemeral=bool(context.config.build_sources_ephemeral), + ) as sources, finalize_config_json(context.config) as json, ): for script in context.config.finalize_scripts: @@ -989,7 +998,10 @@ def run_postoutput_scripts(context: Context) -> None: env["PROFILES"] = " ".join(context.config.profiles) with ( - finalize_source_mounts(context.config, ephemeral=context.config.build_sources_ephemeral) as sources, + finalize_source_mounts( + context.config, + ephemeral=bool(context.config.build_sources_ephemeral), + ) as sources, finalize_config_json(context.config) as json, ): for script in context.config.postoutput_scripts: diff --git a/mkosi/config.py b/mkosi/config.py index a1a24dca9..ef3606d37 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -354,6 +354,15 @@ def __bool__(self) -> bool: return self != Incremental.no +class BuildSourcesEphemeral(StrEnum): + yes = enum.auto() + no = enum.auto() + buildcache = enum.auto() + + def __bool__(self) -> bool: + return self != BuildSourcesEphemeral.no + + class Architecture(StrEnum): alpha = enum.auto() arc = enum.auto() @@ -1897,7 +1906,7 @@ class Config: repart_offline: bool history: bool build_sources: list[ConfigTree] - build_sources_ephemeral: bool + build_sources_ephemeral: BuildSourcesEphemeral environment: dict[str, str] environment_files: list[Path] with_tests: bool @@ -3360,11 +3369,15 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple ), ConfigSetting( dest="build_sources_ephemeral", - metavar="BOOL", + nargs="?", section="Build", - parse=config_parse_boolean, + parse=config_make_enum_parser_with_boolean( + BuildSourcesEphemeral, yes=BuildSourcesEphemeral.yes, no=BuildSourcesEphemeral.no + ), + default=BuildSourcesEphemeral.no, help="Make build sources ephemeral when running scripts", scope=SettingScope.universal, + choices=BuildSourcesEphemeral.values(), ), ConfigSetting( dest="environment", @@ -4943,7 +4956,7 @@ def summary(config: Config) -> str: Repart Offline: {yes_no(config.repart_offline)} Save History: {yes_no(config.history)} Build Sources: {line_join_list(config.build_sources)} - Build Sources Ephemeral: {yes_no(config.build_sources_ephemeral)} + Build Sources Ephemeral: {config.build_sources_ephemeral} Script Environment: {line_join_list(env)} Environment Files: {line_join_list(config.environment_files)} Run Tests in Build Scripts: {yes_no(config.with_tests)} @@ -5134,6 +5147,7 @@ def uki_profile_transformer( Firmware: enum_transformer, SecureBootSignTool: enum_transformer, Incremental: enum_transformer, + BuildSourcesEphemeral: enum_transformer, Optional[Distribution]: optional_enum_transformer, list[ManifestFormat]: enum_list_transformer, Verb: enum_transformer, diff --git a/mkosi/mounts.py b/mkosi/mounts.py index 3b83d6e92..edca3c3e0 100644 --- a/mkosi/mounts.py +++ b/mkosi/mounts.py @@ -6,9 +6,10 @@ import tempfile from collections.abc import Iterator, Sequence from pathlib import Path -from typing import Optional +from typing import Optional, Union -from mkosi.config import Config +from mkosi.config import BuildSourcesEphemeral, Config +from mkosi.log import die from mkosi.sandbox import OverlayOperation from mkosi.util import PathString, flatten @@ -56,7 +57,11 @@ def mount_overlay( @contextlib.contextmanager -def finalize_source_mounts(config: Config, *, ephemeral: bool) -> Iterator[list[PathString]]: +def finalize_source_mounts( + config: Config, + *, + ephemeral: Union[BuildSourcesEphemeral, bool], +) -> Iterator[list[PathString]]: with contextlib.ExitStack() as stack: options: list[PathString] = [] @@ -64,12 +69,24 @@ def finalize_source_mounts(config: Config, *, ephemeral: bool) -> Iterator[list[ src, dst = t.with_prefix("/work/src") if ephemeral: - upperdir = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="volatile-overlay"))) - os.chmod(upperdir, src.stat().st_mode) + if ephemeral == BuildSourcesEphemeral.buildcache: + if config.build_dir is None: + die( + "BuildSourcesEphemeral=buildcache was configured, but no build directory exists.", # noqa: E501 + hint="Configure BuildDirectory= or create mkosi.builddir.", + ) + assert config.build_dir + upperdir = config.build_dir / f"mkosi.buildovl.{src.name}" + upperdir.mkdir(mode=src.stat().st_mode, exist_ok=True) + else: + upperdir = Path( + stack.enter_context(tempfile.TemporaryDirectory(prefix="volatile-overlay.")) + ) + os.chmod(upperdir, src.stat().st_mode) workdir = Path( stack.enter_context( - tempfile.TemporaryDirectory(dir=upperdir.parent, prefix=f"{upperdir.name}-workdir") + tempfile.TemporaryDirectory(dir=upperdir.parent, prefix=f"{upperdir.name}-workdir.") ) ) diff --git a/mkosi/resources/man/mkosi.1.md b/mkosi/resources/man/mkosi.1.md index e255c884c..880374cb9 100644 --- a/mkosi/resources/man/mkosi.1.md +++ b/mkosi/resources/man/mkosi.1.md @@ -1476,12 +1476,17 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, working directory is mounted to `/work/src`. `BuildSourcesEphemeral=`, `--build-sources-ephemeral=` -: Takes a boolean. Disabled by default. Configures whether changes to - source directories (the working directory and configured using - `BuildSources=`) are persisted. If enabled, all source directories - will be reset to their original state every time after running all +: Takes a boolean or the special value `buildcache`. Disabled by default. Configures whether changes to + source directories, the working directory and configured using `BuildSources=`, are persisted. If + enabled, all source directories will be reset to their original state every time after running all scripts of a specific type (except sync scripts). + 💥💣💥 If set to `buildcache` the overlay is not discarded when running build scripts, but saved to the + build directory, configured via `BuildDirectory=`, and will be reused on subsequent runs. The overlay is + still discarded for all other scripts. This option can be used to implement more advanced caching of + builds, but can lead to unexpected states of the source directory. When using this option, a build + directory must be configured. 💥💣💥 + `Environment=`, `--environment=` : Adds variables to the environment that package managers and the prepare/build/postinstall/finalize scripts are executed with. Takes diff --git a/tests/test_json.py b/tests/test_json.py index 3d164f53a..00a138a4c 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -14,6 +14,7 @@ ArtifactOutput, BiosBootloader, Bootloader, + BuildSourcesEphemeral, Cacheonly, CertificateSource, CertificateSourceType, @@ -117,7 +118,7 @@ def test_config() -> None: "Target": "/frob" } ], - "BuildSourcesEphemeral": true, + "BuildSourcesEphemeral": "yes", "CDROM": false, "CPUs": 2, "CacheDirectory": "/is/this/the/cachedir", @@ -424,7 +425,7 @@ def test_config() -> None: build_dir=None, build_packages=["pkg1", "pkg2"], build_scripts=[Path("/path/to/buildscript")], - build_sources_ephemeral=True, + build_sources_ephemeral=BuildSourcesEphemeral.yes, build_sources=[ConfigTree(Path("/qux"), Path("/frob"))], cache_dir=Path("/is/this/the/cachedir"), cacheonly=Cacheonly.always,