diff --git a/rpm.mk b/rpm.mk index 95485e6335..bc08fe3d87 100644 --- a/rpm.mk +++ b/rpm.mk @@ -12,6 +12,8 @@ JEMALLOC_URL ?= $(shell rpmspec -P $(RPMBUILD)/SPECS/389-ds-base.spec | awk '/^S JEMALLOC_TARBALL ?= $(shell basename "$(JEMALLOC_URL)") BUNDLE_JEMALLOC = 1 NODE_MODULES_TEST = src/cockpit/389-console/package-lock.json +NODE_MODULES_PATH = src/cockpit/389-console/ +CARGO_PATH = src/ GIT_TAG = ${TAG} # Some sanitizers are supported only by clang @@ -42,8 +44,8 @@ download-cargo-dependencies: cargo fetch --manifest-path=./src/Cargo.toml tar -czf vendor.tar.gz vendor -bundle-rust: - python3 rpm/bundle-rust-downstream.py ./src/Cargo.lock $(DS_SPECFILE) ./vendor +bundle-rust-npm: + python3 rpm/bundle-rust-npm.py $(CARGO_PATH) $(NODE_MODULES_PATH) $(DS_SPECFILE) --backup-specfile install-node-modules: ifeq ($(COCKPIT_ON), 1) diff --git a/rpm/bundle-rust-downstream.py b/rpm/bundle-rust-downstream.py deleted file mode 100644 index ba0a58a154..0000000000 --- a/rpm/bundle-rust-downstream.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/python3 - -# --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2021 Red Hat, Inc. -# All rights reserved. -# -# License: GPL (version 3 or any later version). -# See LICENSE for details. -# --- END COPYRIGHT BLOCK --- -# -# PYTHON_ARGCOMPLETE_OK - -import os -import sys -import time -import signal -import argparse -import argcomplete -import shutil -import toml -from lib389.cli_base import setup_script_logger -from rust2rpm import licensing - -SPECFILE_COMMENT_LINE = 'Bundled cargo crates list' -START_LINE = f"##### {SPECFILE_COMMENT_LINE} - START #####\n" -END_LINE = f"##### {SPECFILE_COMMENT_LINE} - END #####\n" - - -parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description="""Add 'Provides: bundled(crate(foo)) = version' to a Fedora based specfile. -Additionally, add a helper comment with a comulated License metainfo which is based on Cargo.lock file content.""") - -parser.add_argument('-v', '--verbose', - help="Display verbose operation tracing during command execution", - action='store_true', default=False) - -parser.add_argument('cargo_lock_file', - help="The path to Cargo.lock file.") -parser.add_argument('spec_file', - help="The path to spec file that will be modified.") -parser.add_argument('vendor_dir', - help="The path to the vendor directory file that will be modified.") -parser.add_argument('--backup-specfile', - help="Make a backup of the downstream specfile.", - action='store_true', default=False) - - -# handle a control-c gracefully -def signal_handler(signal, frame): - print('\n\nExiting...') - sys.exit(0) - - -def get_license_list(vendor_dir): - license_list = list() - for root, _, files in os.walk(vendor_dir): - for file in files: - name = os.path.join(root, file) - if os.path.isfile(name) and "Cargo.toml" in name: - with open(name, "r") as file: - contents = file.read() - data = toml.loads(contents) - license, warning = licensing.translate_license_fedora(data["package"]["license"]) - - # Normalise - license = license.replace("/", " or ").replace(" / ", " or ") - license = license.replace("Apache-2.0", "ASL 2.0") - license = license.replace("WITH LLVM-exception", "with exceptions") - if "or" in license or "and" in license: - license = f"({license})" - if license == "(MIT or ASL 2.0)": - license = "(ASL 2.0 or MIT)" - - if license not in license_list: - if warning is not None: - # Ignore known warnings - if not warning.endswith("LLVM-exception!") and \ - not warning.endswith("MIT/Apache-2.0!"): - print(f"{license}: {warning}") - license_list.append(license) - return " and ".join(license_list) - - -def backup_specfile(spec_file): - time_now = time.strftime("%Y%m%d_%H%M%S") - log.info(f"Backing up file {spec_file} to {spec_file}.{time_now}") - shutil.copy2(spec_file, f"{spec_file}.{time_now}") - - -def replace_license(spec_file, license_string): - result = [] - with open(spec_file, "r") as file: - contents = file.readlines() - for line in contents: - if line.startswith("License: "): - result.append("# IMPORTANT - Check if it looks right. Additionally, " - "compare with the original line. Then, remove this comment and # FIX ME - part.\n") - result.append(f"# FIX ME - License: GPLv3+ and {license_string}\n") - else: - result.append(line) - with open(spec_file, "w") as file: - file.writelines(result) - log.info(f"Licenses are successfully updated - {spec_file}") - - -def clean_specfile(spec_file): - result = [] - remove_lines = False - cleaned = False - with open(spec_file, "r") as file: - contents = file.readlines() - - log.info(f"Remove '{SPECFILE_COMMENT_LINE}' content from {spec_file}") - for line in contents: - if line == START_LINE: - remove_lines = True - log.debug(f"Remove '{START_LINE}' from {spec_file}") - elif line == END_LINE: - remove_lines = False - cleaned = True - log.debug(f"Remove '{END_LINE}' from {spec_file}") - elif not remove_lines: - result.append(line) - else: - log.debug(f"Remove '{line}' from {spec_file}") - - with open(spec_file, "w") as file: - file.writelines(result) - return cleaned - - -def write_provides_bundled_crate(cargo_lock_file, spec_file, cleaned): - # Generate 'Provides' out of cargo_lock_file - with open(cargo_lock_file, "r") as file: - contents = file.read() - data = toml.loads(contents) - provides_lines = [] - for package in data["package"]: - provides_lines.append(f"Provides: bundled(crate({package['name']})) = {package['version'].replace('-', '_')}\n") - - # Find a line index where 'Provides' ends - with open(spec_file, "r") as file: - spec_file_lines = file.readlines() - last_provides = -1 - for i in range(0, len(spec_file_lines)): - if spec_file_lines[i].startswith("%description"): - break - if spec_file_lines[i].startswith("Provides:"): - last_provides = i - - # Insert the generated 'Provides' to the specfile - log.info(f"Add the fresh '{SPECFILE_COMMENT_LINE}' content to {spec_file}") - i = last_provides + 2 - spec_file_lines.insert(i, START_LINE) - for line in sorted(provides_lines): - i = i + 1 - log.debug(f"Adding '{line[:-1]}' as a line {i} to buffer") - spec_file_lines.insert(i, line) - i = i + 1 - spec_file_lines.insert(i, END_LINE) - - # Insert an empty line if we haven't cleaned the old content - # (as the old content already has an extra empty line that wasn't removed) - if not cleaned: - i = i + 1 - spec_file_lines.insert(i, "\n") - - log.debug(f"Commit the buffer to {spec_file}") - with open(spec_file, "w") as file: - file.writelines(spec_file_lines) - - -if __name__ == '__main__': - args = parser.parse_args() - log = setup_script_logger('bundle-rust-downstream', args.verbose) - - log.debug("389-ds-base Rust Crates to Bundled Downstream Specfile tool") - log.debug(f"Called with: {args}") - - if not os.path.exists(args.spec_file): - log.info(f"File doesn't exists: {args.spec_file}") - sys.exit(1) - if not os.path.exists(args.cargo_lock_file): - log.info(f"File doesn't exists: {args.cargo_lock_file}") - sys.exit(1) - - if args.backup_specfile: - backup_specfile(args.spec_file) - - cleaned = clean_specfile(args.spec_file) - write_provides_bundled_crate(args.cargo_lock_file, args.spec_file, cleaned) - license_string = get_license_list(args.vendor_dir) - replace_license(args.spec_file, license_string) - log.info(f"Specfile {args.spec_file} is successfully modified! Please:\n" - "1. Open the specfile with your editor of choice\n" - "2. Make sure that Provides with bundled crates are correct\n" - "3. Follow the instructions for 'License:' field and remove the helper comments") - diff --git a/rpm/bundle-rust-npm.py b/rpm/bundle-rust-npm.py new file mode 100644 index 0000000000..37d1806596 --- /dev/null +++ b/rpm/bundle-rust-npm.py @@ -0,0 +1,270 @@ +#!/usr/bin/python3 +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2024 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- +# +# PYTHON_ARGCOMPLETE_OK + +import os +import sys +import subprocess +import time +import signal +import argparse +import argcomplete +import shutil +import json +import re +from typing import List, Dict, Tuple, Set +from lib389.cli_base import setup_script_logger +from lib389.utils import ensure_list_str, ensure_str + +SPECFILE_COMMENT_LINE = 'Bundled cargo crates list' +START_LINE = f"##### {SPECFILE_COMMENT_LINE} - START #####\n" +END_LINE = f"##### {SPECFILE_COMMENT_LINE} - END #####\n" + +IGNORED_RUST_PACKAGES: Set[str] = {"librslapd", "librnsslapd", "slapd", "slapi_r_plugin", "entryuuid", "entryuuid_syntax", "pwdchan"} +IGNORED_NPM_PACKAGES: Set[str] = {"389-console"} +PACKAGE_REGEX = re.compile(r"(.*)@(.*)") + +parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""Add 'Provides: bundled(crate(foo)) = version' 'Provides: bundled(npm(bar)) = version' to a Fedora based specfile. +Additionally, add a helper comment with a comulated License metainfo which is based on Cargo.lock and Package-lock.json files content. +You need to have 'cargo install cargo-license' and 'dnf install npm' to be able to run this script.""") + +parser.add_argument('-v', '--verbose', + help="Display verbose operation tracing during command execution", + action='store_true', default=False) + +parser.add_argument('cargo_path', + help="The path to the directory with Cargo.lock file.") +parser.add_argument('npm_path', + help="The path to the directory with Package-lock.json file.") +parser.add_argument('spec_file', + help="The path to spec file that will be modified.") +parser.add_argument('--backup-specfile', + help="Make a backup of the downstream specfile.", + action='store_true', default=False) + + +# handle a control-c gracefully +def signal_handler(signal, frame): + """Exits the script gracefully on SIGINT.""" + print('\n\nExiting...') + sys.exit(0) + + +def backup_specfile(spec_file: str): + """Creates a backup of the specfile with a timestamp.""" + try: + time_now = time.strftime("%Y%m%d_%H%M%S") + log.info(f"Backing up file {spec_file} to {spec_file}.{time_now}") + shutil.copy2(spec_file, f"{spec_file}.{time_now}") + except IOError as e: + log.error(f"Failed to backup specfile: {e}") + sys.exit(1) + + +def run_cmd(cmd): + """Executes a command and returns its output.""" + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + args = ' '.join(ensure_list_str(result.args)) + stdout = ensure_str(result.stdout) + stderr = ensure_str(result.stderr) + log.debug(f"CMD: {args} returned {result.returncode} STDOUT: {stdout} STDERR: {stderr}") + return stdout + + +def process_rust_crates(output: str) -> Dict[str, Tuple[str, str]]: + """Processes the output from cargo-license to extract crate information.""" + crates = json.loads(output) + return {crate['name']: (enclose_if_contains_or(crate['license']), crate['version']) + for crate in crates if crate['name'] not in IGNORED_RUST_PACKAGES} + + +def process_npm_packages(output: str) -> Dict[str, Tuple[str, str]]: + """Processes the output from license-checker to extract npm package information.""" + packages = json.loads(output) + processed_packages = {} + for package, data in packages.items(): + package_name, package_version = PACKAGE_REGEX.match(package).groups() + if package_name not in IGNORED_NPM_PACKAGES: + npm_license = process_npm_license(data['licenses']) + # Check if the package is 'pause-stream' and if the license is 'Apache2' + # If so, replace it with 'Apache-2.0' to match the license in Upstream + # It is a workaround till the pause-stream's fix is released + if package_name == "pause-stream" and "Apache2" in npm_license: + npm_license = npm_license.replace("Apache2", "Apache-2.0") + # Check if the package is 'argparse' and if the license is 'Python-2.0' + # If so, replace it with 'PSF-2.0' as sugested here: + # https://gitlab.com/fedora/legal/fedora-license-data/-/issues/470 + # It is a workaround till the issue resolved + if package_name == "argparse" and "Python-2.0" in npm_license: + npm_license = npm_license.replace("Python-2.0", "PSF-2.0") + processed_packages[package_name] = (npm_license, package_version) + + return processed_packages + + +def process_npm_license(license_data) -> str: + """Formats the license data for npm packages.""" + npm_license = license_data if isinstance(license_data, str) else ' OR '.join(license_data) + return enclose_if_contains_or(npm_license) + + +def enclose_if_contains_or(license_str: str) -> str: + """Enclose the license string in parentheses if it contains 'OR'.""" + return f"({license_str})" if 'OR' in license_str and not license_str.startswith('(') else license_str + + +def build_provides_lines(rust_crates: Dict[str, Tuple[str, str]], npm_packages: Dict[str, Tuple[str, str]]) -> list[str]: + """Builds lines to be added to the spec file for provided packages.""" + provides_lines = [f"Provides: bundled(crate({crate})) = {version.replace('-', '_')}\n" + for crate, (_, version) in rust_crates.items()] + provides_lines += [f"Provides: bundled(npm({package})) = {version.replace('-', '_')}\n" + for package, (_, version) in npm_packages.items()] + return provides_lines + + +def create_license_line(rust_crates: Dict[str, Tuple[str, str]], npm_packages: Dict[str, Tuple[str, str]]) -> str: + """Creates a line for the spec file with combined license information.""" + licenses = {license for _, (license, _) in {**rust_crates, **npm_packages}.items() if license} + return " AND ".join(sorted(licenses)) + + +def replace_license(spec_file: str, license_string: str): + """Replaces the license section in the spec file with a new license string and + adds a comment for manual review and adjustment. + """ + result = [] + with open(spec_file, "r") as file: + contents = file.readlines() + for line in contents: + if line.startswith("License: "): + result.append("# IMPORTANT - Check if it looks right. Additionally, " + "compare with the original line. Then, remove this comment and # FIX ME - part.\n") + result.append(f"# FIX ME - License: GPL-3.0-or-later AND {license_string}\n") + else: + result.append(line) + with open(spec_file, "w") as file: + file.writelines(result) + log.info(f"Licenses are successfully updated - {spec_file}") + + +def clean_specfile(spec_file: str) -> bool: + """Cleans up the spec file by removing the previous bundled package information. + Returns a boolean indicating if the clean-up was successful. + """ + result = [] + remove_lines = False + cleaned = False + with open(spec_file, "r") as file: + contents = file.readlines() + + log.info(f"Remove '{SPECFILE_COMMENT_LINE}' content from {spec_file}") + for line in contents: + if line == START_LINE: + remove_lines = True + log.debug(f"Remove '{START_LINE}' from {spec_file}") + elif line == END_LINE: + remove_lines = False + cleaned = True + log.debug(f"Remove '{END_LINE}' from {spec_file}") + elif not remove_lines: + result.append(line) + else: + log.debug(f"Remove '{line}' from {spec_file}") + + with open(spec_file, "w") as file: + file.writelines(result) + return cleaned + + +def write_provides_bundled(provides_lines: List[str], spec_file: str, cleaned: bool): + """Writes bundled package information to the spec file. + Includes generated 'Provides' lines and marks the section for easy future modification. + """ + # Find a line index where 'Provides' ends + with open(spec_file, "r") as file: + spec_file_lines = file.readlines() + last_provides = -1 + for i in range(0, len(spec_file_lines)): + if spec_file_lines[i].startswith("%description"): + break + if spec_file_lines[i].startswith("Provides:"): + last_provides = i + + # Insert the generated 'Provides' to the specfile + log.info(f"Add the fresh '{SPECFILE_COMMENT_LINE}' content to {spec_file}") + i = last_provides + 2 + spec_file_lines.insert(i, START_LINE) + for line in sorted(provides_lines): + i = i + 1 + log.debug(f"Adding '{line[:-1]}' as a line {i} to buffer") + spec_file_lines.insert(i, line) + i = i + 1 + spec_file_lines.insert(i, END_LINE) + + # Insert an empty line if we haven't cleaned the old content + # (as the old content already has an extra empty line that wasn't removed) + if not cleaned: + i = i + 1 + spec_file_lines.insert(i, "\n") + + log.debug(f"Commit the buffer to {spec_file}") + with open(spec_file, "w") as file: + file.writelines(spec_file_lines) + + +if __name__ == '__main__': + args = parser.parse_args() + log = setup_script_logger('bundle-rust-npm', args.verbose) + + log.debug("389-ds-base Rust Crates and Node Modules to Bundled Downstream Specfile tool") + log.debug(f"Called with: {args}") + + if args.backup_specfile: + backup_specfile(args.spec_file) + + if not os.path.isdir(args.cargo_path): + log.error(f"Path {args.cargo_path} does not exist or is not a directory") + sys.exit(1) + if not os.path.isdir(args.npm_path): + log.error(f"Path {args.npm_path} does not exist or is not a directory") + sys.exit(1) + + if shutil.which("cargo-license") is None: + log.error("cargo-license is not installed. Please install it with 'cargo install cargo-license' and try again.") + sys.exit(1) + if shutil.which("npm") is None: + log.error("npm is not installed. Please install it with 'dnf install npm' and try again.") + sys.exit(1) + + rust_output = run_cmd(["cargo", "license", "--json", "--current-dir", args.cargo_path]) + npm_output = run_cmd(["npx", "license-checker", "--production", "--json", "--start", args.npm_path]) + + if rust_output is None or npm_output is None: + log.error("Failed to process dependencies. Ensure cargo-license and license-checker are installed and accessible. " + "Also, ensure that Cargo.lock and Package-lock.json files exist in the respective directories.") + sys.exit(1) + + cleaned = clean_specfile(args.spec_file) + + rust_crates = process_rust_crates(rust_output) + npm_packages = process_npm_packages(npm_output) + provides_lines = build_provides_lines(rust_crates, npm_packages) + + write_provides_bundled(provides_lines, args.spec_file, cleaned) + + license_string = create_license_line(rust_crates, npm_packages) + replace_license(args.spec_file, license_string) + log.info(f"Specfile {args.spec_file} is successfully modified! Please:\n" + "1. Open the specfile with your editor of choice\n" + "2. Make sure that Provides with bundled crates are correct\n" + "3. Follow the instructions for 'License:' field and remove the helper comments") +