Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add plugin hook to nfpm backend so plugins can inject nfpm package field values #21822

Merged
merged 7 commits into from
Jan 16, 2025
28 changes: 28 additions & 0 deletions docs/notes/2.25.x.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,34 @@ python_requirement(name="black", resolve="your-resolve-name", requirements=["bla

The previously deprecated `[shell-setup].tailor` option has now been removed. See [`[shell-setup].tailor_sources`](https://www.pantsbuild.org/2.25/reference/subsystems/shell-setup#tailor_sources) and [`[shell-setup].tailor_shunit2_tests`](https://www.pantsbuild.org/2.25/reference/subsystems/shell#tailor_shunit2_tests) to update.

#### nFPM

The nFPM backend has a new plugin hook that allows plugins to inject field values that are used to generate nfpm config. To use this, a plugin needs to implement `InjectNfpmPackageFieldsRequest`:

```python
from pants.backend.nfpm.fields.version import NfpmVersionField, NfpmVersionReleaseField
from pants.backend.nfpm.util_rules.inject_config import InjectedNfpmPackageFields, InjectNfpmPackageFieldsRequest
from pants.engine.internals.native_engine import Address, Field
from pants.engine.rules import rule

class MyCustomInjectFieldsRequest(InjectNfpmPackageFieldsRequest):
@classmethod
def is_applicable(cls, target) -> bool:
# this could check the target's address, packager, package_name field, etc.
return True

@rule
def inject_my_custom_fields(request: MyCustomInjectFieldsRequest) -> InjectedNfpmPackageFields:
# this could get the version from a file
version = "9.8.7-dev+git"
release = 6
fields: list[Field] = [
NfpmVersionField(version, request.target.address),
NfpmVersionReleaseField(release, request.target.address),
]
return InjectedNfpmPackageFields(fields, address=request.target.address)
```

### Plugin API changes

The version of Python used by Pants itself is now [3.11](https://docs.python.org/3/whatsnew/3.11.html) (up from 3.9).
Expand Down
21 changes: 17 additions & 4 deletions src/python/pants/backend/nfpm/field_sets.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from pants.backend.nfpm.target_types import APK_FIELDS, ARCHLINUX_FIELDS, DEB_FIELDS, RPM_FIELDS
from pants.core.goals.package import PackageFieldSet
from pants.engine.fs import FileEntry
from pants.engine.internals.native_engine import Field
from pants.engine.rules import collect_rules
from pants.engine.target import DescriptionField, FieldSet, Target
from pants.engine.unions import UnionRule, union
Expand All @@ -48,7 +49,13 @@ class NfpmPackageFieldSet(PackageFieldSet, metaclass=ABCMeta):
description: DescriptionField
scripts: NfpmPackageScriptsField

def nfpm_config(self, tgt: Target, *, default_mtime: str | None) -> dict[str, Any]:
def nfpm_config(
self,
tgt: Target,
injected_fields: FrozenDict[type[Field], Field],
*,
default_mtime: str | None,
) -> dict[str, Any]:
config: dict[str, Any] = {
# pants handles any globbing before passing contents to nFPM.
"disable_globbing": True,
Expand Down Expand Up @@ -84,7 +91,7 @@ def fill_nested(_nfpm_alias: str, value: Any) -> None:
# field opted out of being included in this config (like dependencies)
continue

field_value = tgt[field].value
field_value = injected_fields.get(field, tgt[field]).value
# NB: This assumes that nfpm fields have 'none_is_valid_value=False'.
if not field.required and field_value is None:
# Omit any undefined optional values unless default applied.
Expand Down Expand Up @@ -137,8 +144,14 @@ class NfpmRpmPackageFieldSet(NfpmPackageFieldSet):
required_fields = RPM_FIELDS
ghost_contents: NfpmRpmGhostContents

def nfpm_config(self, tgt: Target, *, default_mtime: str | None) -> dict[str, Any]:
config = super().nfpm_config(tgt, default_mtime=default_mtime)
def nfpm_config(
self,
tgt: Target,
injected_fields: FrozenDict[type[Field], Field],
*,
default_mtime: str | None,
) -> dict[str, Any]:
config = super().nfpm_config(tgt, injected_fields, default_mtime=default_mtime)
config["contents"].extend(self.ghost_contents.nfpm_contents)
return config

Expand Down
36 changes: 28 additions & 8 deletions src/python/pants/backend/nfpm/field_sets_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,13 @@
)
from pants.engine.addresses import Address
from pants.engine.target import DescriptionField
from pants.util.frozendict import FrozenDict

MTIME = NfpmPackageMtimeField.default

ADDRESS = Address("", target_name="t")
INJECTED = FrozenDict({NfpmVersionField: NfpmVersionField("9.8.7", ADDRESS)})


def test_generate_nfpm_config_for_apk():
depends = [
Expand All @@ -67,7 +71,7 @@ def test_generate_nfpm_config_for_apk():
NfpmLicenseField.alias: "MIT",
NfpmApkDependsField.alias: depends,
},
Address("", target_name="t"),
ADDRESS,
)
expected_nfpm_config = {
"disable_globbing": True,
Expand All @@ -90,9 +94,13 @@ def test_generate_nfpm_config_for_apk():
}

field_set = NfpmApkPackageFieldSet.create(tgt)
nfpm_config = field_set.nfpm_config(tgt, default_mtime=MTIME)
nfpm_config = field_set.nfpm_config(tgt, FrozenDict({}), default_mtime=MTIME)
assert nfpm_config == expected_nfpm_config

nfpm_config_2 = field_set.nfpm_config(tgt, INJECTED, default_mtime=MTIME)
assert nfpm_config["version"] != nfpm_config_2["version"]
assert nfpm_config_2["version"] == "9.8.7"


def test_generate_nfpm_config_for_archlinux():
depends = [
Expand All @@ -114,7 +122,7 @@ def test_generate_nfpm_config_for_archlinux():
NfpmLicenseField.alias: "MIT",
NfpmArchlinuxDependsField.alias: depends,
},
Address("", target_name="t"),
ADDRESS,
)
expected_nfpm_config = {
"disable_globbing": True,
Expand All @@ -137,9 +145,13 @@ def test_generate_nfpm_config_for_archlinux():
}

field_set = NfpmArchlinuxPackageFieldSet.create(tgt)
nfpm_config = field_set.nfpm_config(tgt, default_mtime=MTIME)
nfpm_config = field_set.nfpm_config(tgt, FrozenDict({}), default_mtime=MTIME)
assert nfpm_config == expected_nfpm_config

nfpm_config_2 = field_set.nfpm_config(tgt, INJECTED, default_mtime=MTIME)
assert nfpm_config["version"] != nfpm_config_2["version"]
assert nfpm_config_2["version"] == "9.8.7"


def test_generate_nfpm_config_for_deb():
depends = [
Expand All @@ -164,7 +176,7 @@ def test_generate_nfpm_config_for_deb():
NfpmDebFieldsField.alias: {"Urgency": "high (critical for landlubbers)"},
NfpmDebTriggersField.alias: {"interest_noawait": ["some-trigger", "other-trigger"]},
},
Address("", target_name="t"),
ADDRESS,
)
expected_nfpm_config = {
"disable_globbing": True,
Expand Down Expand Up @@ -193,9 +205,13 @@ def test_generate_nfpm_config_for_deb():
}

field_set = NfpmDebPackageFieldSet.create(tgt)
nfpm_config = field_set.nfpm_config(tgt, default_mtime=MTIME)
nfpm_config = field_set.nfpm_config(tgt, FrozenDict({}), default_mtime=MTIME)
assert nfpm_config == expected_nfpm_config

nfpm_config_2 = field_set.nfpm_config(tgt, INJECTED, default_mtime=MTIME)
assert nfpm_config["version"] != nfpm_config_2["version"]
assert nfpm_config_2["version"] == "9.8.7"


def test_generate_nfpm_config_for_rpm():
depends = [
Expand All @@ -219,7 +235,7 @@ def test_generate_nfpm_config_for_rpm():
NfpmRpmPrefixesField.alias: ["/", "/usr", "/opt/treasure"],
NfpmRpmGhostContents.alias: ["/var/log/captains.log"],
},
Address("", target_name="t"),
ADDRESS,
)
expected_nfpm_config = {
"disable_globbing": True,
Expand Down Expand Up @@ -247,5 +263,9 @@ def test_generate_nfpm_config_for_rpm():
}

field_set = NfpmRpmPackageFieldSet.create(tgt)
nfpm_config = field_set.nfpm_config(tgt, default_mtime=MTIME)
nfpm_config = field_set.nfpm_config(tgt, FrozenDict({}), default_mtime=MTIME)
assert nfpm_config == expected_nfpm_config

nfpm_config_2 = field_set.nfpm_config(tgt, INJECTED, default_mtime=MTIME)
assert nfpm_config["version"] != nfpm_config_2["version"]
assert nfpm_config_2["version"] == "9.8.7"
2 changes: 1 addition & 1 deletion src/python/pants/backend/nfpm/fields/contents.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ def compute_value(
if dst_dupes:
raise InvalidFieldException(
help_text(
lambda: f"""
f"""
'{cls._dst_alias}' must be unique in '{cls.alias}', but
found duplicate entries for: {repr(dst_dupes)}
"""
Expand Down
2 changes: 2 additions & 0 deletions src/python/pants/backend/nfpm/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
generate_nfpm_yaml,
)
from pants.backend.nfpm.util_rules.generate_config import rules as generate_config_rules
from pants.backend.nfpm.util_rules.inject_config import rules as inject_config_rules
from pants.backend.nfpm.util_rules.sandbox import (
NfpmContentSandboxRequest,
populate_nfpm_content_sandbox,
Expand Down Expand Up @@ -160,6 +161,7 @@ def rules():
return [
*package.rules(),
*field_sets_rules(),
*inject_config_rules(),
*generate_config_rules(),
*sandbox_rules(),
*collect_rules(),
Expand Down
11 changes: 10 additions & 1 deletion src/python/pants/backend/nfpm/util_rules/generate_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
from pants.backend.nfpm.fields.contents import NfpmContentFileSourceField, NfpmContentSrcField
from pants.backend.nfpm.subsystem import NfpmSubsystem
from pants.backend.nfpm.target_types import NfpmContentFile
from pants.backend.nfpm.util_rules.inject_config import (
NfpmPackageTargetWrapper,
determine_injected_nfpm_package_fields,
)
from pants.core.goals.package import TraverseIfNotPackageTarget
from pants.engine.fs import CreateDigest, FileContent, FileEntry
from pants.engine.internals.graph import find_valid_field_sets
Expand Down Expand Up @@ -74,7 +78,12 @@ async def generate_nfpm_yaml(

# Fist get the config that can be constructed from the target itself.
nfpm_package_target = transitive_targets.roots[0]
config = request.field_set.nfpm_config(nfpm_package_target, default_mtime=default_mtime)
injected_fields = await determine_injected_nfpm_package_fields(
NfpmPackageTargetWrapper(nfpm_package_target), union_membership
)
config = request.field_set.nfpm_config(
nfpm_package_target, injected_fields.field_values, default_mtime=default_mtime
)

# Second, gather package contents from hydrated deps.
contents: list[NfpmContent] = config["contents"]
Expand Down
147 changes: 147 additions & 0 deletions src/python/pants/backend/nfpm/util_rules/inject_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
# Copyright 2025 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Iterable

from pants.backend.nfpm.fields.scripts import NfpmPackageScriptsField
from pants.engine.environment import EnvironmentName
from pants.engine.internals.native_engine import Address, Field
from pants.engine.internals.selectors import Get
from pants.engine.rules import collect_rules, rule
from pants.engine.target import Target
from pants.engine.unions import UnionMembership, union
from pants.util.frozendict import FrozenDict
from pants.util.strutil import softwrap


@dataclass(frozen=True)
class InjectedNfpmPackageFields:
"""The injected fields that should be used instead of the target's fields.

Though any field can technically be provided (except "scripts" which is banned),
only nfpm package metadata fields will have an impact. Passing other fields are
silently ignored. For example, "dependencies", and "output_path" are not used
when generating nfpm config, so they will be ignored; "sources" is not a valid
field for nfpm package targets, so it will also be ignored.

The "scripts" field is special in that it has dependency inference tied to it.
If you write your own dependency inference rule (possibly based on a custom
field you've added to the nfpm package target), then you can pass
_allow_banned_fields=True to allow injection of the "scripts" field.
"""

field_values: FrozenDict[type[Field], Field]

def __init__(
self,
fields: Iterable[Field],
*,
address: Address,
_allow_banned_fields: bool = False,
) -> None:
super().__init__()
if not _allow_banned_fields:
aliases = [field.alias for field in fields]
for alias in {
NfpmPackageScriptsField.alias, # if _allow_banned_fields, the plugin author must handle scripts deps.
}:
if alias in aliases:
raise ValueError(
softwrap(
f"""
{alias} cannot be an injected nfpm package field for {address} to avoid
breaking dependency inference.
"""
)
)
# Ignore any fields that do not have a value (assuming nfpm fields have 'none_is_valid_value=False').
field_values = {type(field): field for field in fields if field.value is not None}
object.__setattr__(
self,
"field_values",
FrozenDict(
sorted(
field_values.items(),
key=lambda field_type_to_val_pair: field_type_to_val_pair[0].alias,
)
),
)


# Note: This only exists as a hook for additional logic for nFPM config generation, e.g. for plugin
# authors. To resolve `InjectedNfpmPackageFields`, call `determine_injected_nfpm_package_fields`,
# which handles running any custom implementations vs. using the default implementation.
@union(in_scope_types=[EnvironmentName])
@dataclass(frozen=True)
class InjectNfpmPackageFieldsRequest(ABC):
"""A request to inject nFPM config for nfpm_package_* targets.

By default, Pants will use the nfpm_package_* fields in the BUILD file unchanged to generate the
nfpm.yaml config file for nFPM. To customize this, subclass `InjectNfpmPackageFieldsRequest`,
register `UnionRule(InjectNfpmPackageFieldsRequest, MyCustomInjectNfpmPackageFieldsRequest)`,
and add a rule that takes your subclass as a parameter and returns `InjectedNfpmPackageFields`.
"""

target: Target

@classmethod
@abstractmethod
def is_applicable(cls, target: Target) -> bool:
"""Whether to use this InjectNfpmPackageFieldsRequest implementation for this target."""


@dataclass(frozen=True)
class NfpmPackageTargetWrapper:
"""Nfpm Package target Wrapper.

This is not meant to be used by plugin authors.
"""

target: Target


@rule
async def determine_injected_nfpm_package_fields(
wrapper: NfpmPackageTargetWrapper, union_membership: UnionMembership
) -> InjectedNfpmPackageFields:
target = wrapper.target
inject_nfpm_config_requests = union_membership.get(InjectNfpmPackageFieldsRequest)
applicable_inject_nfpm_config_requests = tuple(
request for request in inject_nfpm_config_requests if request.is_applicable(target)
)

# If no provided implementations, fall back to our default implementation that simply returns
# what the user explicitly specified in the BUILD file.
if not applicable_inject_nfpm_config_requests:
return InjectedNfpmPackageFields((), address=target.address)

if len(applicable_inject_nfpm_config_requests) > 1:
possible_requests = sorted(
plugin.__name__ for plugin in applicable_inject_nfpm_config_requests
)
raise ValueError(
softwrap(
f"""
Multiple registered `InjectNfpmPackageFieldsRequest`s can work on the target
{target.address}, and it's ambiguous which to use: {possible_requests}

Please activate fewer implementations, or make the classmethod `is_applicable()`
more precise so that only one implementation is applicable for this target.
"""
)
)
inject_nfpm_config_request_type = applicable_inject_nfpm_config_requests[0]
inject_nfpm_config_request: InjectNfpmPackageFieldsRequest = inject_nfpm_config_request_type(target) # type: ignore[abstract]
return await Get(
InjectedNfpmPackageFields, InjectNfpmPackageFieldsRequest, inject_nfpm_config_request
)


def rules():
return [
*collect_rules(),
]
Loading
Loading