From be69ac8459b2bad28685d08dad9488c0dfdcec4b Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Wed, 1 Jul 2026 14:08:33 +0200 Subject: [PATCH 01/13] elbepack: add VM-free alternative to elbe initvm submit Add a new top-level "elbe build" command as alternative to "elbe initvm submit". It drives the project manager directly in-process, without any daemon or SOAP communication. It is meant to be used inside a container that already provides the required isolation. Signed-off-by: Florian Kauer --- debian/python3-elbe-common.install | 1 + debian/python3-elbe-control.install | 1 + docs/elbe-build.rst | 121 +++++++++++++ docs/index.rst | 1 + elbepack/buildsubmitaction.py | 10 +- elbepack/commands/build.py | 47 ++++++ elbepack/initvmaction.py | 4 +- elbepack/localbuildaction.py | 159 ++++++++++++++++++ newsfragments/+elbe-build-command.feature.rst | 4 + 9 files changed, 343 insertions(+), 5 deletions(-) create mode 100644 docs/elbe-build.rst create mode 100644 elbepack/commands/build.py create mode 100644 elbepack/localbuildaction.py create mode 100644 newsfragments/+elbe-build-command.feature.rst diff --git a/debian/python3-elbe-common.install b/debian/python3-elbe-common.install index 13dbe9b86..190fa90c2 100644 --- a/debian/python3-elbe-common.install +++ b/debian/python3-elbe-common.install @@ -30,6 +30,7 @@ usr/lib/python3.*/*-packages/elbepack/initvm.py usr/lib/python3.*/*-packages/elbepack/initvmaction.py usr/lib/python3.*/*-packages/elbepack/isooptions.py usr/lib/python3.*/*-packages/elbepack/licencexml.py +usr/lib/python3.*/*-packages/elbepack/localbuildaction.py usr/lib/python3.*/*-packages/elbepack/log.py usr/lib/python3.*/*-packages/elbepack/packers.py usr/lib/python3.*/*-packages/elbepack/paths.py diff --git a/debian/python3-elbe-control.install b/debian/python3-elbe-control.install index 397cdccab..6c858c1ef 100644 --- a/debian/python3-elbe-control.install +++ b/debian/python3-elbe-control.install @@ -1,3 +1,4 @@ +usr/lib/python3.*/*-packages/elbepack/commands/build.py usr/lib/python3.*/*-packages/elbepack/commands/control.py usr/lib/python3.*/*-packages/elbepack/commands/initvm.py usr/lib/python3.*/*-packages/elbepack/commands/pbuilder.py diff --git a/docs/elbe-build.rst b/docs/elbe-build.rst new file mode 100644 index 000000000..e7ecf55f1 --- /dev/null +++ b/docs/elbe-build.rst @@ -0,0 +1,121 @@ +************************ +elbe-build +************************ + +NAME +==== + +elbe-build - Build a root filesystem from an ELBE XML file, without +requiring an initvm. + +SYNOPSIS +======== + + :: + + elbe build [options] | + +DESCRIPTION +=========== + +This command builds an ELBE project directly, without encapsulating the +build into an initvm and without any daemon or SOAP communication. It +runs the whole build in-process, and is meant to be used inside a +container (or any other environment) that already provides the +isolation an initvm would otherwise provide. It is therefore the +VM-free alternative to *elbe initvm submit*. + +Since it is meant for container builds where there is no initvm, +packages needed only for the initvm are always excluded from the +generated CDROMs. + +OPTIONS +======= + +--skip-download + After the build has finished, the generated files are normally + copied out of the project directory to *--build-dir*. This step is + skipped, when this option is specified. + +--build-dir + Directory name where the generated and downloaded files should be + saved and where the internal build cache is kept. The default is to + generate a directory with a timestamp in the current working directory. + +--skip-build-bin + Skip building binary repository CDROM, for exact reproduction. + +--skip-build-sources + Skip building source CDROM. + +--keep-files + Don’t delete elbe project files after a build. The project directory + is printed during the build. + +--writeproject + Write project name to . + +--build-sdk + Also build an SDK. + +--base-image + Use a base image instead of debootstrap as the starting point for a rootfilesystem (experimental). + +XML OPTIONS +=========== + +These options are passed through to an implicit invocation of +*elbe preprocess*, which is run on the given xmlfile before the build. + +-v , --variants + comma separated list of variants; enable only tags with empty or + given variant. + +-p , --proxy + add proxy to mirrors + +Examples +======== + +*elbe build* is meant to be run inside a container. The example +container definition in *contrib/containerfile* provides a +ready-to-use build environment for this. + +- Build the container image, installing elbe from the published elbe + archive: + + :: + + $ cd contrib/containerfile + $ make build + +- Alternatively, build the container image with ELBE packages + built from the current checkout, instead of the published ones: + + :: + + $ cd contrib/containerfile + $ make build-local + +- Run the build in a container, with the current directory + mounted as */build*. The container is removed again once the build + finishes: + + :: + + $ podman run --rm \ + -v $(pwd):/work:Z \ + elbe-buildenv-image \ + elbe build --skip-build-bin --skip-build-sources \ + /work/tests/base-extended/simple-validation/image-base-trixie.xml \ + --build-dir /work/build + +SEE ALSO +======== + +``elbe-initvm(1)``, ``elbe-preprocess(1)`` + +ELBE +==== + +Part of the ``elbe(1)`` suite diff --git a/docs/index.rst b/docs/index.rst index 9d552da95..4e1e59294 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ If you are new to ELBE, we recommend starting with the elbe elbe-add + elbe-build elbe-check_updates elbe-cyclonedx-sbom elbe-initvm diff --git a/elbepack/buildsubmitaction.py b/elbepack/buildsubmitaction.py index dab925cd0..bb627c0d1 100644 --- a/elbepack/buildsubmitaction.py +++ b/elbepack/buildsubmitaction.py @@ -50,15 +50,17 @@ def extract_cdrom(cdrom): return tmp +def add_output_argument(f): + return add_argument('--output', dest='outdir', + type=os.path.abspath, + help='directory where to save downloaded Files')(f) + + def add_submit_arguments(f): f = add_argument('--skip-download', action='store_true', dest='skip_download', default=False, help='Skip downloading generated Files')(f) - f = add_argument('--output', dest='outdir', - type=os.path.abspath, - help='directory where to save downloaded Files')(f) - f = add_argument('--skip-build-bin', dest='build_bin', action='store_false', default=True, help='Skip building Binary Repository CDROM, for exact Reproduction')(f) diff --git a/elbepack/commands/build.py b/elbepack/commands/build.py new file mode 100644 index 000000000..e40c0f183 --- /dev/null +++ b/elbepack/commands/build.py @@ -0,0 +1,47 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import argparse +import datetime +import os + +from elbepack.buildsubmitaction import add_submit_arguments, extract_cdrom +from elbepack.cli import add_argument, add_arguments_from_decorated_function +from elbepack.commands.preprocess import add_xmlpreprocess_passthrough_arguments +from elbepack.localbuildaction import local_build_with_repodir_and_dl_result + + +@add_submit_arguments +@add_argument( + '--build-dir', dest='build_dir', type=os.path.abspath, + help='directory where to save output files and the internal build cache ' + '(default is a timestamped directory in the current working directory)') +@add_argument('input', metavar=' | ') +def _build(args): + if args.build_dir is None: + args.build_dir = os.path.abspath( + 'elbe-build-' + datetime.datetime.now().strftime('%Y%m%d-%H%M%S')) + + cdrom = None + xmlfile = args.input + if xmlfile.endswith('.iso'): + tmp = extract_cdrom(xmlfile) + cdrom = xmlfile + xmlfile = tmp.fname('source.xml') + elif not xmlfile.endswith('.xml'): + args.parser.error('Unknown file ending (use either xml or iso)') + + local_build_with_repodir_and_dl_result(xmlfile, cdrom, args.base_image, args) + + +def run_command(argv): + aparser = argparse.ArgumentParser(prog='elbe build') + + add_xmlpreprocess_passthrough_arguments(aparser) + add_arguments_from_decorated_function(aparser, _build) + + args = aparser.parse_args(argv) + args.parser = aparser + + _build(args) diff --git a/elbepack/initvmaction.py b/elbepack/initvmaction.py index d15d16114..b54026d0f 100644 --- a/elbepack/initvmaction.py +++ b/elbepack/initvmaction.py @@ -12,7 +12,7 @@ import elbepack import elbepack.initvm -from elbepack.buildsubmitaction import add_submit_arguments, extract_cdrom +from elbepack.buildsubmitaction import add_output_argument, add_submit_arguments, extract_cdrom from elbepack.cli import CliError, add_argument, with_cli_details from elbepack.config import add_argument_sshport, add_arguments_soapclient from elbepack.elbexml import ValidationError @@ -228,6 +228,7 @@ def _submit_and_dl_result(control, xmlfile, cdrom, base_image, args, xmlfile_bas help=argparse.SUPPRESS) @add_submit_arguments @add_argument('--size', help='Disk size', type=size_to_int) +@add_output_argument @add_argument('input', nargs='?', metavar=' | ') def _create(args): # Upgrade from older versions which used tmux @@ -316,6 +317,7 @@ def _create(args): @_add_initvm_from_args_arguments @add_submit_arguments +@add_output_argument @add_argument('input', metavar=' | ') def _submit(args): initvm = _initvm_from_args(args) diff --git a/elbepack/localbuildaction.py b/elbepack/localbuildaction.py new file mode 100644 index 000000000..287aad789 --- /dev/null +++ b/elbepack/localbuildaction.py @@ -0,0 +1,159 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import os +import shutil +import sys +import textwrap +import time + +from elbepack.cli import CliError, with_cli_details +from elbepack.loopcheck import check_loop_mount_requirements +from elbepack.projectmanager import ProjectManager +from elbepack.repodir import Repodir, RepodirError +from elbepack.xmlpreprocess import preprocess_file + +prog = os.path.basename(sys.argv[0]) + + +def local_build_with_repodir_and_dl_result(xmlfile, cdrom, base_image, args): + os.makedirs(args.build_dir, exist_ok=True) + fname = f'elbe-repodir-{time.time_ns()}.xml' + preprocess_xmlfile = os.path.join(args.build_dir, fname) + try: + with Repodir(xmlfile, preprocess_xmlfile): + _local_build_and_dl_result(preprocess_xmlfile, cdrom, base_image, args) + except RepodirError as err: + raise with_cli_details(err, 127, 'elbe repodir failed') + + +def _wait_busy(pm, prjdir): + while True: + is_busy, msg = pm.project_is_busy(prjdir) + + if msg: + print(msg) + continue + + if not is_busy: + break + + time.sleep(0.1) + + # exited the loop -> the project is not busy anymore, + # check, whether everything is ok. + prj = pm.db.get_project_data(prjdir) + if prj.status != 'build_done': + raise CliError(191, f'Project build was not successful, current status: {prj.status}') + + +def _local_build_and_dl_result(xmlfile, cdrom, base_image, args): + cache_dir = os.path.join(args.build_dir, 'cache') + pm = ProjectManager(cache_dir) + try: + with preprocess_file(xmlfile, variants=args.variants, sshport=args.sshport, + soapport=args.soapport) as xmlfile: + prjdir = pm.create_project(xmlfile) + + if args.writeproject: + with open(args.writeproject, 'w') as wpf: + wpf.write(prjdir) + + if cdrom is not None: + print('Copying CDROM into project. This might take a while') + shutil.copy(cdrom, os.path.join(prjdir, 'uploaded_cdrom.iso')) + pm.set_upload_cdrom(prjdir) + print('Copy finished') + + uploaded_base_image_path = None + if base_image is not None: + print('Copying base image into project. This might take a while') + uploaded_base_image_path = os.path.join(prjdir, 'uploaded_base_image.img') + shutil.copy(base_image, uploaded_base_image_path) + print('Copy finished') + + pm.build_project(prjdir, args.build_bin, args.build_sources, bool(cdrom), + uploaded_base_image_path) + + print('Build started, waiting till it finishes') + + try: + _wait_busy(pm, prjdir) + except Exception as e: + raise with_cli_details(e, 133, textwrap.dedent(f""" + Build Failed + + The project will not be deleted. + Its files are available at: + {prjdir} """)) + + print('') + print('Build finished !') + print('') + + if args.build_sdk: + pm.build_sdk(prjdir) + + print('SDK Build started, waiting till it finishes') + + try: + _wait_busy(pm, prjdir) + except Exception: + print('Waiting for the SDK build Failed', file=sys.stderr) + print('', file=sys.stderr) + print('The project will not be deleted.', file=sys.stderr) + print('Its files are available at:', file=sys.stderr) + print(prjdir, file=sys.stderr) + print('', file=sys.stderr) + sys.exit(135) + + print('') + print('SDK Build finished !') + print('') + + try: + with open(os.path.join(prjdir, 'validation.txt'), 'rb') as f: + shutil.copyfileobj(f, sys.stdout.buffer) + sys.stdout.buffer.flush() + except Exception: + print( + 'Project failed to generate validation.txt', + file=sys.stderr) + print('Getting log.txt', file=sys.stderr) + try: + with open(os.path.join(prjdir, 'log.txt'), 'rb') as f: + shutil.copyfileobj(f, sys.stdout.buffer) + sys.stdout.buffer.flush() + except Exception as e: + raise with_cli_details(e, 137, textwrap.dedent('Failed to dump log.txt')) + sys.exit(136) + + files = pm.db.get_project_files(prjdir) + + if args.skip_download: + print('') + print('Listing available files:') + print('') + for file in files: + print(f'{file.name}\t{file.description}') + + print('') + print(f'Files are available at: {prjdir}') + else: + print('') + print('Getting generated Files') + print('') + + print(f'Saving generated Files to {args.build_dir}') + + os.makedirs(args.build_dir, exist_ok=True) + for file in files: + shutil.copy(os.path.join(prjdir, file.name), + os.path.join(args.build_dir, os.path.basename(file.name))) + print(f'{file.name}\t{file.description}') + + if not args.keep_files: + pm.del_project(prjdir) + finally: + pm.stop() diff --git a/newsfragments/+elbe-build-command.feature.rst b/newsfragments/+elbe-build-command.feature.rst new file mode 100644 index 000000000..3b1c1e5c6 --- /dev/null +++ b/newsfragments/+elbe-build-command.feature.rst @@ -0,0 +1,4 @@ +Add a new ``elbe build`` command as a VM-free alternative to ``elbe initvm submit``. +It drives the project manager directly in-process, without any daemon or SOAP +communication, and is meant to be used inside a container (or any other environment) +that already provides the isolation an initvm would otherwise provide. From 76f1917dc2b7df0313cc7f64c1dbc045d1c61c00 Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Mon, 20 Jul 2026 14:46:30 +0200 Subject: [PATCH 02/13] elbepack: exclude initvm packages for container builds Adding initvm packages to the CDROMs is not reasonable when there is no initvm. Therefore, exclude them. Also, provide this option as CLI parameter (default off) for initvm builds for consistency. Signed-off-by: Florian Kauer --- elbepack/asyncworker.py | 12 ++++++-- elbepack/cdroms.py | 55 ++++++++++++++++++++-------------- elbepack/commands/build.py | 2 ++ elbepack/daemons/soap/esoap.py | 9 ++++-- elbepack/elbeproject.py | 42 ++++++++++++++++---------- elbepack/initvmaction.py | 6 +++- elbepack/localbuildaction.py | 4 +-- elbepack/projectmanager.py | 9 +++--- elbepack/repomanager.py | 25 +++++++++++----- 9 files changed, 105 insertions(+), 59 deletions(-) diff --git a/elbepack/asyncworker.py b/elbepack/asyncworker.py index a2b3daff7..7f62ecd86 100644 --- a/elbepack/asyncworker.py +++ b/elbepack/asyncworker.py @@ -58,6 +58,9 @@ def execute(self, db): class BuildSDKJob(AsyncWorkerJob): + def __init__(self, project, exclude_initvm_pkgs=False): + super().__init__(project) + self.exclude_initvm_pkgs = exclude_initvm_pkgs def enqueue(self, queue, db): db.set_busy(self.project.builddir, @@ -70,7 +73,7 @@ def execute(self, db): success = self.build_failed try: logging.info('Build SDK started') - self.project.build_sdk() + self.project.build_sdk(exclude_initvm_pkgs=self.exclude_initvm_pkgs) except Exception: logging.exception('Build SDK Failed') else: @@ -143,12 +146,14 @@ def execute(self, db): class BuildJob(AsyncWorkerJob): - def __init__(self, project, build_bin, build_src, skip_pbuilder, base_image_path): + def __init__(self, project, build_bin, build_src, skip_pbuilder, + base_image_path, exclude_initvm_pkgs=False): super().__init__(project) self.build_bin = build_bin self.build_src = build_src self.skip_pbuilder = skip_pbuilder self.base_image_path = base_image_path + self.exclude_initvm_pkgs = exclude_initvm_pkgs def enqueue(self, queue, db): db.set_busy(self.project.builddir, @@ -166,7 +171,8 @@ def execute(self, db): build_bin=self.build_bin, build_sources=self.build_src, skip_pbuild=self.skip_pbuilder, - base_image_path=self.base_image_path) + base_image_path=self.base_image_path, + exclude_initvm_pkgs=self.exclude_initvm_pkgs) except (DebootstrapException, AptCacheCommitError, AptCacheUpdateError) as e: if isinstance(e, DebootstrapException): err = 'Debootstrap failed to install the base rootfilesystem.' diff --git a/elbepack/cdroms.py b/elbepack/cdroms.py index 4544fc9ca..dd7d4be7f 100644 --- a/elbepack/cdroms.py +++ b/elbepack/cdroms.py @@ -42,7 +42,8 @@ def add_source_pkg(repo, component, cache, pkg, version, forbid): def mk_source_cdrom(components, codename, init_codename, target, cdrom_size=CDROM_SIZE, xml=None, - mirror='http://deb.debian.org/debian'): + mirror='http://deb.debian.org/debian', + exclude_initvm_pkgs=False): os.makedirs(SOURCES_DIR, exist_ok=True) make_writable_by_apt(SOURCES_DIR) @@ -94,12 +95,15 @@ def mk_source_cdrom(components, codename, # with the bin repo, because the src cdrom can be split # into multiple cdroms - for dirpath, _, filenames in os.walk(SOURCES_DIR): - for filename in filenames: - if not filename.endswith('.dsc'): - continue + if not exclude_initvm_pkgs: + for dirpath, _, filenames in os.walk(SOURCES_DIR): + for filename in filenames: + if not filename.endswith('.dsc'): + continue - repos['main'].include_init_dsc(os.path.join(dirpath, filename), 'initvm') + repos['main'].include_init_dsc(os.path.join(dirpath, filename), 'initvm') + else: + logging.info('Skipping initvm source packages as requested by --exclude-initvm-pkgs') for repo in repos.values(): repo.finalize() @@ -132,7 +136,7 @@ def mk_source_cdrom(components, codename, options=options)) for component, repo in repos.items()] -def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target): +def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target, exclude_initvm_pkgs=False): rfs.mkdir_p(BINARIES_ADDED_DIR) make_writable_by_apt(rfs.fname(BINARIES_ADDED_DIR), passwd_root=rfs) @@ -150,19 +154,23 @@ def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target): # initvm repo has been built upon initvm creation # just copy it. the repo __init__() afterwards will # not touch the repo config, nor generate a new key. - try: - do(f'cp -av {INITVM_BIN_REPO_DIR} "{repo_path}"') - except subprocess.CalledProcessError: - # When INITVM_BIN_REPO_DIR has not been created - # (because the initvm install was an old version or somthing, - # log an error, and continue with an empty directory. - logging.exception('%s does not exist\n' - 'The generated CDROM will not contain initvm pkgs\n' - 'This happened because the initvm was probably\n' - 'generated with --skip-build-bin', - INITVM_BIN_REPO_DIR) - + if exclude_initvm_pkgs: + logging.info('Skipping initvm packages as requested by --exclude-initvm-pkgs') do(f'mkdir -p "{repo_path}"') + else: + try: + do(f'cp -av {INITVM_BIN_REPO_DIR} "{repo_path}"') + except subprocess.CalledProcessError: + # When INITVM_BIN_REPO_DIR has not been created + # (because the initvm install was an old version or somthing, + # log an error, and continue with an empty directory. + logging.exception('%s does not exist\n' + 'The generated CDROM will not contain initvm pkgs\n' + 'This happened because the initvm was probably\n' + 'generated with --skip-build-bin', + INITVM_BIN_REPO_DIR) + + do(f'mkdir -p "{repo_path}"') repo = CdromInitRepo(init_codename, repo_path, mirror) @@ -222,10 +230,11 @@ def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target): xml.xml.write(repo_path / 'source.xml') # copy initvm-cdrom.gz and vmlinuz - copyfile(os.path.join(INSTALLER_DIR, 'initrd-cdrom.gz'), - repo_path / 'initrd-cdrom.gz') - copyfile(os.path.join(INSTALLER_DIR, 'vmlinuz'), - repo_path / 'vmlinuz') + if not exclude_initvm_pkgs: + copyfile(os.path.join(INSTALLER_DIR, 'initrd-cdrom.gz'), + repo_path / 'initrd-cdrom.gz') + copyfile(os.path.join(INSTALLER_DIR, 'vmlinuz'), + repo_path / 'vmlinuz') target_repo_path.joinpath('.aptignr').touch() diff --git a/elbepack/commands/build.py b/elbepack/commands/build.py index e40c0f183..b72ca4d4c 100644 --- a/elbepack/commands/build.py +++ b/elbepack/commands/build.py @@ -19,6 +19,8 @@ '(default is a timestamped directory in the current working directory)') @add_argument('input', metavar=' | ') def _build(args): + args.exclude_initvm_pkgs = True + if args.build_dir is None: args.build_dir = os.path.abspath( 'elbe-build-' + datetime.datetime.now().strftime('%Y%m%d-%H%M%S')) diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py index 165dea8f5..a1108f23e 100644 --- a/elbepack/daemons/soap/esoap.py +++ b/elbepack/daemons/soap/esoap.py @@ -162,10 +162,13 @@ def build_sdk(self, builddir): def build_cdroms(self, builddir, build_bin, build_src): self.app.pm.build_cdroms(builddir, build_bin, build_src) - @rpc(String, Boolean, Boolean, Boolean, String) - def build(self, builddir, build_bin, build_src, skip_pbuilder, base_image_path): + @rpc(String, Boolean, Boolean, Boolean, String, Boolean) + def build(self, builddir, build_bin, build_src, skip_pbuilder, + base_image_path, exclude_initvm_pkgs): - self.app.pm.build_project(builddir, build_bin, build_src, skip_pbuilder, base_image_path) + self.app.pm.build_project(builddir, build_bin, build_src, + skip_pbuilder, base_image_path, + exclude_initvm_pkgs) @rpc(String, Boolean, Boolean, String) def build_pbuilder(self, builddir, cross, noccache, ccachesize): diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py index 798d787d4..b4986a17c 100644 --- a/elbepack/elbeproject.py +++ b/elbepack/elbeproject.py @@ -249,7 +249,7 @@ def _dpkg_query(*args): yield file - def build_sysroot(self): + def build_sysroot(self, exclude_initvm_pkgs=False): do(['rm', '-rf', self.sysrootpath]) do(['mkdir', self.sysrootpath]) @@ -263,7 +263,8 @@ def build_sysroot(self): self.xml.add_target_package('libc6-dbg') self.xml.add_target_package('gdbserver') - self.install_packages(sysrootenv, buildenv=False) + self.install_packages(sysrootenv, buildenv=False, + exclude_initvm_pkgs=exclude_initvm_pkgs) # ignore packages from debootstrap tpkgs = self.xml.get_target_packages() @@ -388,7 +389,7 @@ def build_host_sysroot(self, pkgs, hostsysrootpath): host_sysrootenv.rfs.rmtree('/tmp') host_sysrootenv.rfs.rmtree('/var') - def build_sdk(self): + def build_sdk(self, exclude_initvm_pkgs=False): triplet = self.xml.defs['triplet'] elfcode = self.xml.defs['elfcode'] @@ -407,7 +408,7 @@ def build_sdk(self): host_pkglist.append('gdb-multiarch') # build target sysroot including libs and headers for the target - self.build_sysroot() + self.build_sysroot(exclude_initvm_pkgs=exclude_initvm_pkgs) sdktargetpath = os.path.join(self.sdkpath, 'sysroots', 'target') do(['mkdir', '-p', sdktargetpath]) do(['tar', 'xJf', os.path.join(self.builddir, 'sysroot.tar.xz'), '-C', sdktargetpath], @@ -469,7 +470,7 @@ def pbuild(self, p): def build_cdroms(self, build_bin=True, build_sources=False, cdrom_size=None, - tgt_pkg_lst=None): + tgt_pkg_lst=None, exclude_initvm_pkgs=False): self.repo_images = [] @@ -502,7 +503,8 @@ def build_cdroms(self, build_bin=True, self.codename, init_codename, self.xml, - self.builddir) + self.builddir, + exclude_initvm_pkgs=exclude_initvm_pkgs) if build_sources: if not cdrom_size and self.xml.has('src-cdrom/size'): cdrom_size = size_to_int(self.xml.text('src-cdrom/size')) @@ -560,6 +562,7 @@ def build_cdroms(self, build_bin=True, self.codename, init_codename, self.builddir, + exclude_initvm_pkgs=exclude_initvm_pkgs, **kwargs): self.repo_images += iso except SystemError as e: @@ -567,7 +570,8 @@ def build_cdroms(self, build_bin=True, validation.error(str(e)) def build(self, build_bin=False, build_sources=False, cdrom_size=None, - skip_pkglist=False, skip_pbuild=False, base_image_path=None): + skip_pkglist=False, skip_pbuild=False, base_image_path=None, + exclude_initvm_pkgs=False): # Write the log header self.write_log_header() @@ -605,7 +609,7 @@ def build(self, build_bin=False, build_sources=False, cdrom_size=None, # Install packages if not skip_pkglist: - self.install_packages(self.buildenv) + self.install_packages(self.buildenv, exclude_initvm_pkgs=exclude_initvm_pkgs) try: self.buildenv.rfs.dump_elbeversion(self.xml) @@ -642,7 +646,8 @@ def build(self, build_bin=False, build_sources=False, cdrom_size=None, # install packages for buildenv if not skip_pkglist: - self.install_packages(self.buildenv, buildenv=True) + self.install_packages(self.buildenv, buildenv=True, + exclude_initvm_pkgs=exclude_initvm_pkgs) # Write source.xml try: @@ -689,7 +694,9 @@ def build(self, build_bin=False, build_sources=False, cdrom_size=None, self.targetfs.part_target(self.builddir, grub_version, grub_fw_type) - self.build_cdroms(build_bin, build_sources, cdrom_size, tgt_pkg_lst=tgt_pkgs) + self.build_cdroms(build_bin, build_sources, cdrom_size, + tgt_pkg_lst=tgt_pkgs, + exclude_initvm_pkgs=exclude_initvm_pkgs) if self.postbuild_file: logging.info('Postbuild script') @@ -987,7 +994,7 @@ def copy_initvmnode(self): logging.exception('%s is available. But it does not ' 'contain an initvm node', SOURCE_XML) - def install_packages(self, target, buildenv=False): + def install_packages(self, target, buildenv=False, exclude_initvm_pkgs=False): # to workaround debian bug no. 872543 if self.xml.prj.has('noauth'): @@ -1012,10 +1019,12 @@ def install_packages(self, target, buildenv=False): if target.need_dumpdebootstrap: dump_debootstrappkgs(self.xml, self.get_rpcaptcache(env=target)) - dump_initvmpkgs(self.xml) + if not exclude_initvm_pkgs: + dump_initvmpkgs(self.xml) target.need_dumpdebootstrap = False - self.copy_initvmnode() + if not exclude_initvm_pkgs: + self.copy_initvmnode() else: sourcepath = os.path.join(self.builddir, 'source.xml') source = ElbeXML(sourcepath, @@ -1026,9 +1035,10 @@ def install_packages(self, target, buildenv=False): try: self.xml.get_initvmnode_from(source) except NoInitvmNode: - logging.warning('source.xml is available. ' - 'But it does not contain an initvm node') - self.copy_initvmnode() + if not exclude_initvm_pkgs: + logging.warning('source.xml is available. ' + 'But it does not contain an initvm node') + self.copy_initvmnode() # Seed /etc, we need /etc/hosts for hostname -f to work correctly if not buildenv: diff --git a/elbepack/initvmaction.py b/elbepack/initvmaction.py index b54026d0f..befef9ea5 100644 --- a/elbepack/initvmaction.py +++ b/elbepack/initvmaction.py @@ -127,7 +127,7 @@ def _submit_and_dl_result(control, xmlfile, cdrom, base_image, args, xmlfile_bas print('Upload finished') control.service.build(prjdir, args.build_bin, args.build_sources, bool(cdrom), - uploaded_base_image_path) + uploaded_base_image_path, args.exclude_initvm_pkgs) print('Build started, waiting till it finishes') @@ -318,6 +318,10 @@ def _create(args): @_add_initvm_from_args_arguments @add_submit_arguments @add_output_argument +@add_argument( + '--exclude-initvm-pkgs', action='store_true', dest='exclude_initvm_pkgs', + default=False, + help='Exclude initvm packages from CDROM generation') @add_argument('input', metavar=' | ') def _submit(args): initvm = _initvm_from_args(args) diff --git a/elbepack/localbuildaction.py b/elbepack/localbuildaction.py index 287aad789..8dca8bcb1 100644 --- a/elbepack/localbuildaction.py +++ b/elbepack/localbuildaction.py @@ -74,7 +74,7 @@ def _local_build_and_dl_result(xmlfile, cdrom, base_image, args): print('Copy finished') pm.build_project(prjdir, args.build_bin, args.build_sources, bool(cdrom), - uploaded_base_image_path) + uploaded_base_image_path, args.exclude_initvm_pkgs) print('Build started, waiting till it finishes') @@ -93,7 +93,7 @@ def _local_build_and_dl_result(xmlfile, cdrom, base_image, args): print('') if args.build_sdk: - pm.build_sdk(prjdir) + pm.build_sdk(prjdir, args.exclude_initvm_pkgs) print('SDK Build started, waiting till it finishes') diff --git a/elbepack/projectmanager.py b/elbepack/projectmanager.py index 9815c1399..7be84dbd5 100644 --- a/elbepack/projectmanager.py +++ b/elbepack/projectmanager.py @@ -108,10 +108,11 @@ def build_project( build_bin, build_src, skip_pbuilder, - base_image_path): + base_image_path, + exclude_initvm_pkgs=False): ep = self.open_project(builddir, allow_busy=False) self.worker.enqueue(BuildJob(ep, build_bin, build_src, - skip_pbuilder, base_image_path)) + skip_pbuilder, base_image_path, exclude_initvm_pkgs)) def update_pbuilder(self, builddir): ep = self.open_project(builddir, allow_busy=False) @@ -153,9 +154,9 @@ def build_sysroot(self, builddir): ep = self.open_project(builddir, allow_busy=False) self.worker.enqueue(BuildSysrootJob(ep)) - def build_sdk(self, builddir): + def build_sdk(self, builddir, exclude_initvm_pkgs=False): ep = self.open_project(builddir, allow_busy=False) - self.worker.enqueue(BuildSDKJob(ep)) + self.worker.enqueue(BuildSDKJob(ep, exclude_initvm_pkgs)) def build_cdroms(self, builddir, build_bin, build_src): ep = self.open_project(builddir, allow_busy=False) diff --git a/elbepack/repomanager.py b/elbepack/repomanager.py index a490ef018..c9430ed7b 100644 --- a/elbepack/repomanager.py +++ b/elbepack/repomanager.py @@ -77,6 +77,8 @@ def __init__( self.attrs = [repo_attr] elif init_attr is not None: self.attrs = [init_attr] + else: + self.attrs = [] self.origin = origin self.description = description @@ -86,10 +88,16 @@ def __init__( # if repo exists retrive the keyid otherwise # generate a new key and generate repository config if self.volume.is_dir(): - repo_conf = self.volume.joinpath('conf', 'distributions').read_text() - for lic in repo_conf.splitlines(): - if lic.startswith('SignWith'): - self.keyid = lic.split()[1] + conf_dist = self.volume.joinpath('conf', 'distributions') + if conf_dist.is_file(): + repo_conf = conf_dist.read_text() + for lic in repo_conf.splitlines(): + if lic.startswith('SignWith'): + self.keyid = lic.split()[1] + else: + # Directory exists but no repo config, so treat as new repository + self.keyid = generate_elbe_internal_key() + self.gen_repo_conf() else: self.keyid = generate_elbe_internal_key() self.gen_repo_conf() @@ -360,9 +368,12 @@ class CdromInitRepo(RepoBase): def __init__(self, init_codename, path, mirror='http://deb.debian.org/debian'): - init_attrs = RepoAttributes( - init_codename, 'amd64', [ - 'main', 'main/debian-installer'], mirror) + if init_codename is not None: + init_attrs = RepoAttributes( + init_codename, 'amd64', [ + 'main', 'main/debian-installer'], mirror) + else: + init_attrs = None super().__init__(path, None, init_attrs, 'Elbe', 'Elbe InitVM Cdrom Repo') From a81cea78f09d9bb937fb7ff604bd109048529c84 Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Tue, 28 Jul 2026 08:31:40 +0200 Subject: [PATCH 03/13] elbepack: thread gnupg_home through repo and signing helpers The gnupg home directory used for signing and verifying repositories was hardcoded to /var/cache/elbe/gnupg throughout the signing helpers and repository classes. Pass it explicitly instead, so callers can point them at a project-specific keyring. Signed-off-by: Florian Kauer --- elbepack/cdroms.py | 13 ++++-- elbepack/commands/fetch_initvm_pkgs.py | 6 ++- elbepack/commands/remove_sign.py | 4 +- elbepack/commands/sign.py | 4 +- elbepack/dump.py | 4 +- elbepack/egpg.py | 27 ++++++------ elbepack/elbeproject.py | 5 ++- elbepack/finetuning.py | 16 ++++--- elbepack/paths.py | 5 +++ elbepack/repomanager.py | 58 ++++++++++++++------------ elbepack/updated.py | 4 +- elbepack/updatepkg.py | 2 +- 12 files changed, 85 insertions(+), 63 deletions(-) diff --git a/elbepack/cdroms.py b/elbepack/cdroms.py index dd7d4be7f..1776fac62 100644 --- a/elbepack/cdroms.py +++ b/elbepack/cdroms.py @@ -14,7 +14,8 @@ from elbepack.archivedir import archive_tmpfile from elbepack.isooptions import get_iso_options from elbepack.paths import ( - BINARIES_ADDED_DIR, BINARIES_MAIN_DIR, INITVM_BIN_REPO_DIR, INSTALLER_DIR, SOURCES_DIR, + BINARIES_ADDED_DIR, BINARIES_MAIN_DIR, INITVM_BIN_REPO_DIR, INITVM_GNUPG_HOME, + INSTALLER_DIR, SOURCES_DIR, ) from elbepack.repomanager import CdromBinRepo, CdromInitRepo, CdromSrcRepo from elbepack.rpcaptcache import get_rpcaptcache @@ -57,6 +58,8 @@ def mk_source_cdrom(components, codename, except KeyError: pass + gnupg_home = os.path.join(target, 'gnupg') + repos = {} for component in components.keys(): @@ -72,7 +75,7 @@ def mk_source_cdrom(components, codename, make_writable_by_apt(rfs.fname(SOURCES_DIR), passwd_root=rfs) repo = CdromSrcRepo(codename, init_codename, os.path.join(target, f'srcrepo-{component}'), - cdrom_size, mirror) + cdrom_size, gnupg_home, mirror) repos[component] = repo for pkg, version in pkg_lst: add_source_pkg(repo, component, @@ -148,6 +151,8 @@ def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target, exclude_ini else: mirror = 'http://deb.debian.org/debian' + gnupg_home = os.path.join(target, 'gnupg') + repo_path = pathlib.Path(target, 'binrepo') target_repo_path = repo_path / 'targetrepo' @@ -172,10 +177,10 @@ def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target, exclude_ini do(f'mkdir -p "{repo_path}"') - repo = CdromInitRepo(init_codename, repo_path, mirror) + repo = CdromInitRepo(init_codename, repo_path, INITVM_GNUPG_HOME, mirror) target_repo = CdromBinRepo(arch, codename, None, - target_repo_path, mirror) + target_repo_path, gnupg_home, mirror) if xml is not None: cache = get_rpcaptcache(rfs, arch) diff --git a/elbepack/commands/fetch_initvm_pkgs.py b/elbepack/commands/fetch_initvm_pkgs.py index 0e7a08daa..f6d9a926b 100644 --- a/elbepack/commands/fetch_initvm_pkgs.py +++ b/elbepack/commands/fetch_initvm_pkgs.py @@ -15,6 +15,7 @@ from elbepack.aptpkgutils import fetch_source, get_corresponding_source_packages from elbepack.aptprogress import ElbeAcquireProgress from elbepack.dump import get_initvm_pkglist +from elbepack.egpg import INITVM_GNUPG_HOME from elbepack.elbexml import ElbeXML, ValidationError from elbepack.imgutils import mount from elbepack.log import elbe_logging @@ -90,7 +91,7 @@ def run_command(argv): # Binary Repo # - repo = CdromInitRepo(init_codename, args.binrepo, mirror) + repo = CdromInitRepo(init_codename, args.binrepo, INITVM_GNUPG_HOME, mirror) os.makedirs(args.archive, exist_ok=True) @@ -131,7 +132,8 @@ def run_command(argv): # Source Repo # - repo = CdromSrcRepo(init_codename, init_codename, args.srcrepo, 0, mirror) + repo = CdromSrcRepo(init_codename, init_codename, args.srcrepo, 0, + INITVM_GNUPG_HOME, mirror) os.makedirs(args.srcarchive, exist_ok=True) # a cdrom build does not have sources diff --git a/elbepack/commands/remove_sign.py b/elbepack/commands/remove_sign.py index c582d351e..b634c1830 100644 --- a/elbepack/commands/remove_sign.py +++ b/elbepack/commands/remove_sign.py @@ -4,7 +4,7 @@ import argparse -from elbepack.egpg import unsign_file +from elbepack.egpg import INITVM_GNUPG_HOME, unsign_file def run_command(argv): @@ -13,7 +13,7 @@ def run_command(argv): args = parser.parse_args(argv) - fname = unsign_file(args.file) + fname = unsign_file(args.file, INITVM_GNUPG_HOME) if fname: print(f'unsigned file: {fname}') else: diff --git a/elbepack/commands/sign.py b/elbepack/commands/sign.py index 47e89e560..68bad6638 100644 --- a/elbepack/commands/sign.py +++ b/elbepack/commands/sign.py @@ -4,7 +4,7 @@ import argparse -from elbepack.egpg import sign_file +from elbepack.egpg import INITVM_GNUPG_HOME, sign_file def run_command(argv): @@ -14,4 +14,4 @@ def run_command(argv): args = parser.parse_args(argv) - sign_file(args.file, args.fingerprint) + sign_file(args.file, args.fingerprint, INITVM_GNUPG_HOME) diff --git a/elbepack/dump.py b/elbepack/dump.py index 2143ec2f9..71cef272c 100644 --- a/elbepack/dump.py +++ b/elbepack/dump.py @@ -183,7 +183,7 @@ def check_full_pkgs(pkgs, fullpkgs, cache): validation.info('No Errors found') -def elbe_report(xml, buildenv, cache, targetfs): +def elbe_report(xml, buildenv, cache, targetfs, builddir): rfs = buildenv.rfs @@ -244,7 +244,7 @@ def elbe_report(xml, buildenv, cache, targetfs): mt_index_postarch = mt_index if xml.has('target/finetuning'): - do_finetuning(xml, buildenv, targetfs) + do_finetuning(xml, buildenv, targetfs, builddir) mt_index_post_fine = targetfs.mtime_snap() else: mt_index_post_fine = mt_index_postarch diff --git a/elbepack/egpg.py b/elbepack/egpg.py index 9dd39fed8..a955aab62 100644 --- a/elbepack/egpg.py +++ b/elbepack/egpg.py @@ -17,6 +17,7 @@ from gpg.constants import PROTOCOL_OpenPGP, sig, sigsum from gpg.errors import GPGMEError, InvalidSigners, KeyNotFound +from elbepack.paths import INITVM_GNUPG_HOME, TARGET_GNUPG_HOME # noqa: F401 from elbepack.shellhelper import env_add @@ -148,7 +149,7 @@ def check_signature(ctx, signature): return status -def unsign_file(fname): +def unsign_file(fname, gnupg_home): # check for .gpg extension and create an output filename without it if len(fname) <= 4 or fname[len(fname) - 4:] != '.gpg': print('The input file needs a .gpg extension') @@ -159,7 +160,7 @@ def unsign_file(fname): ctx = core.Context() ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + gnupg_home) ctx.set_armor(False) overall_status = OverallStatus() @@ -190,14 +191,14 @@ def unsign_file(fname): return None -def sign(infile, outfile, fingerprint): +def sign(infile, outfile, fingerprint, gnupg_home): ctx = core.Context() try: ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + gnupg_home) except GPGMEError as E: print("Error: Can't set engine info - %s", E) return @@ -232,16 +233,16 @@ def sign(infile, outfile, fingerprint): fd.write(signature) -def sign_file(fname, fingerprint): +def sign_file(fname, fingerprint, gnupg_home): outfilename = fname + '.gpg' - sign(fname, outfilename, fingerprint) + sign(fname, outfilename, fingerprint, gnupg_home) -def get_fingerprints(): +def get_fingerprints(gnupg_home): ctx = core.Context() ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + gnupg_home) keys = ctx.op_keylist_all(None, False) fingerprints = [] for k in keys: @@ -259,8 +260,8 @@ def get_fingerprints(): EOT = 4294967295 -def generate_elbe_internal_key(): - gpg_agent_conf = pathlib.Path('/var/cache/elbe/gnupg/gpg-agent.conf') +def generate_elbe_internal_key(gnupg_home): + gpg_agent_conf = pathlib.Path(gnupg_home, 'gpg-agent.conf') gpg_agent_conf.parent.mkdir(mode=0o700, parents=True, exist_ok=True) gpg_agent_conf.write_text('allow-preset-passphrase\n' f'default-cache-ttl {EOT}\n' @@ -269,18 +270,18 @@ def generate_elbe_internal_key(): ctx = core.Context() ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + gnupg_home) ctx.op_genkey(elbe_internal_key_param, None, None) key = ctx.op_genkey_result() return key.fpr -def export_key(fingerprint, outfile): +def export_key(fingerprint, outfile, gnupg_home): subprocess.run([ '/usr/bin/gpg', '-a', '-o', outfile, '--export', fingerprint, - ], check=True, env=env_add({'GNUPGHOME': '/var/cache/elbe/gnupg'})) + ], check=True, env=env_add({'GNUPGHOME': gnupg_home})) def unarmor_openpgp_keyring(armored): diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py index b4986a17c..fc8eeaaee 100644 --- a/elbepack/elbeproject.py +++ b/elbepack/elbeproject.py @@ -165,7 +165,8 @@ def __init__( self.name = self.xml.text('project/name') self.repo = ProjectRepo(self.arch, self.codename, - os.path.join(self.builddir, 'repo')) + os.path.join(self.builddir, 'repo'), + os.path.join(self.builddir, 'gnupg')) # Create BuildEnv instance, if the chroot directory exists and # has an etc/elbe_version @@ -658,7 +659,7 @@ def build(self, build_bin=False, build_sources=False, cdrom_size=None, # Elbe report cache = self.get_rpcaptcache() - tgt_pkgs = elbe_report(self.xml, self.buildenv, cache, self.targetfs) + tgt_pkgs = elbe_report(self.xml, self.buildenv, cache, self.targetfs, self.builddir) # chroot' licenses self.gen_licenses('chroot', self.buildenv, diff --git a/elbepack/finetuning.py b/elbepack/finetuning.py index d780d4ad5..e19339dbd 100644 --- a/elbepack/finetuning.py +++ b/elbepack/finetuning.py @@ -20,6 +20,7 @@ from gpg import core from gpg.constants import PROTOCOL_OpenPGP +from elbepack.egpg import TARGET_GNUPG_HOME from elbepack.filesystem import Filesystem from elbepack.imgutils import losetup, mount from elbepack.packers import default_packer, packers @@ -42,7 +43,8 @@ def __init__(self, node): def execute(self, _buildenv, _target): raise NotImplementedError('execute() not implemented') - def execute_prj(self, buildenv, target, _builddir): + def execute_prj(self, buildenv, target, builddir): + self.builddir = builddir self.execute(buildenv, target) @@ -368,7 +370,7 @@ def execute(self, buildenv, target): ctx = core.Context() ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + os.path.join(self.builddir, 'gnupg')) ctx.set_armor(True) ctx.op_export(fp, 0, gpgdata) gpgdata.seek(0, os.SEEK_SET) @@ -378,10 +380,10 @@ def execute(self, buildenv, target): with open((target.path + '/pub.key'), 'wb') as tkey: tkey.write(key) - target.mkdir_p('/var/cache/elbe/gnupg', mode=0o700) + target.mkdir_p(TARGET_GNUPG_HOME, mode=0o700) with target: do(['gpg', '--import', target.path + '/pub.key'], - env_add={'GNUPGHOME': f'{target.path}/var/cache/elbe/gnupg'}) + env_add={'GNUPGHOME': target.path + TARGET_GNUPG_HOME}) logging.info('generate base repo') @@ -406,7 +408,8 @@ def execute(self, buildenv, target): logging.exception('Package %s-%s missing name or version', pkg.name, pkg.installed_version) r = UpdateRepo(target.xml, - target.path + REPOS_BASE_DIR) + target.path + REPOS_BASE_DIR, + os.path.join(self.builddir, 'gnupg')) for d in buildenv.rfs.glob('tmp/pkgs/*.deb'): r.includedeb(d, 'main') @@ -697,7 +700,7 @@ def execute(self, buildenv, _target): f.write('\n'.join(src_lst)) -def do_finetuning(xml, buildenv, target): +def do_finetuning(xml, buildenv, target, builddir): if not xml.has('target/finetuning'): return @@ -705,6 +708,7 @@ def do_finetuning(xml, buildenv, target): for i in xml.node('target/finetuning'): try: action = _action_for_node(i) + action.builddir = builddir action.execute(buildenv, target) except KeyError: logging.exception("Unimplemented finetuning action '%s'", diff --git a/elbepack/paths.py b/elbepack/paths.py index 8944e1c8e..03a174934 100644 --- a/elbepack/paths.py +++ b/elbepack/paths.py @@ -8,6 +8,7 @@ BINARIES_MAIN_DIR = f'{ELBE_CACHE_DIR}/binaries/main' BINARIES_ADDED_DIR = f'{ELBE_CACHE_DIR}/binaries/added' INITVM_BIN_REPO_DIR = f'{ELBE_CACHE_DIR}/initvm-bin-repo' +INITVM_GNUPG_HOME = f'{ELBE_CACHE_DIR}/gnupg' INITVM_SRC_REPO_DIR = f'{ELBE_CACHE_DIR}/initvm-src-repo' INSTALLER_DIR = f'{ELBE_CACHE_DIR}/installer' INSTALLER_VMLINUZ = f'{INSTALLER_DIR}/vmlinuz' @@ -23,3 +24,7 @@ DB_PATH = ELBE_CACHE_DIR PRE_SCRIPT = f'{ELBE_CACHE_DIR}/pre.sh' POST_SCRIPT = f'{ELBE_CACHE_DIR}/post.sh' + +# Path on deployed target system. Just happens to be the same +# in the build system, but keep separate because there is no relation. +TARGET_GNUPG_HOME = '/var/cache/elbe/gnupg' diff --git a/elbepack/repomanager.py b/elbepack/repomanager.py index c9430ed7b..03a38791f 100644 --- a/elbepack/repomanager.py +++ b/elbepack/repomanager.py @@ -63,9 +63,11 @@ def __init__( repo_attr, origin, description, + gnupg_home, maxsize=None): self.vol_path = path + self.gnupg_home = gnupg_home self.volume_count = 0 self.init_attr = init_attr @@ -96,12 +98,15 @@ def __init__( self.keyid = lic.split()[1] else: # Directory exists but no repo config, so treat as new repository - self.keyid = generate_elbe_internal_key() + self.keyid = generate_elbe_internal_key(self.gnupg_home) self.gen_repo_conf() else: - self.keyid = generate_elbe_internal_key() + self.keyid = generate_elbe_internal_key(self.gnupg_home) self.gen_repo_conf() + def _reprepro(self, args): + do(['reprepro', *args], env_add={'GNUPGHOME': self.gnupg_home}) + def get_volume_path(self, volume): if self.maxsize: if volume >= 0: @@ -168,20 +173,17 @@ def gen_repo_conf(self): fp.write('\n') - export_key(self.keyid, self.volume / 'repo.pub') + export_key(self.keyid, self.volume / 'repo.pub', self.gnupg_home) if need_update: - do(['reprepro', '--export=force', '--basedir', self.volume, 'update'], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro(['--export=force', '--basedir', self.volume, 'update']) else: for att in self.attrs: - do(['reprepro', '--basedir', self.volume, 'export', att.codename], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro(['--basedir', self.volume, 'export', att.codename]) def finalize(self): for att in self.attrs: - do(['reprepro', '--basedir', self.volume, 'export', att.codename], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro(['--basedir', self.volume, 'export', att.codename]) def _includedeb(self, path, codename, components=None, prio=None): if self.maxsize: @@ -202,7 +204,7 @@ def _includedeb(self, path, codename, components=None, prio=None): components = [components] global_opt.extend(['--component', '|'.join(components)]) - do(['reprepro', *global_opt, 'includedeb', codename, path]) + self._reprepro([*global_opt, 'includedeb', codename, path]) def includedeb(self, path, components=None, pkgname=None, force=False, prio=None): # pkgname needs only to be specified if force is enabled @@ -240,7 +242,7 @@ def _include(self, path, codename, components=None): components = [components] global_opt.extend(['--component', '|'.join(components)]) - do(['reprepro', *global_opt, 'include', codename, path]) + self._reprepro([*global_opt, 'include', codename, path]) def _removedeb(self, pkgname, codename, components=None): @@ -252,8 +254,7 @@ def _removedeb(self, pkgname, codename, components=None): components = [components] global_opt.extend(['--component', '|'.join(components)]) - do(['reprepro', *global_opt, 'remove', codename, pkgname], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro([*global_opt, 'remove', codename, pkgname]) def removedeb(self, pkgname, components=None): self._removedeb(pkgname, self.repo_attr.codename, components) @@ -262,8 +263,7 @@ def _removesrc(self, srcname, codename): global_opt = ['--basedir', self.volume] - do(['reprepro', *global_opt, 'removesrc', codename, srcname], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro([*global_opt, 'removesrc', codename, srcname]) def removesrc(self, path): with open(path) as fp: @@ -305,7 +305,7 @@ def _includedsc(self, path, codename, components=None): components = [components] global_opt.extend(['--component', '|'.join(components)]) - do(['reprepro', *global_opt, 'includedsc', codename, path]) + self._reprepro([*global_opt, 'includedsc', codename, path]) def includedsc(self, path, components=None, force=False): try: @@ -353,7 +353,7 @@ def volume_indexes(self): class UpdateRepo(RepoBase): - def __init__(self, xml, path): + def __init__(self, xml, path, gnupg_home): self.xml = xml arch = xml.text('project/arch', key='arch') @@ -361,11 +361,11 @@ def __init__(self, xml, path): repo_attrs = RepoAttributes(codename, arch, 'main') - super().__init__(path, None, repo_attrs, 'Update', 'Update') + super().__init__(path, None, repo_attrs, 'Update', 'Update', gnupg_home) class CdromInitRepo(RepoBase): - def __init__(self, init_codename, path, + def __init__(self, init_codename, path, gnupg_home, mirror='http://deb.debian.org/debian'): if init_codename is not None: @@ -375,7 +375,7 @@ def __init__(self, init_codename, path, else: init_attrs = None - super().__init__(path, None, init_attrs, 'Elbe', 'Elbe InitVM Cdrom Repo') + super().__init__(path, None, init_attrs, 'Elbe', 'Elbe InitVM Cdrom Repo', gnupg_home) class CdromBinRepo(RepoBase): @@ -385,6 +385,7 @@ def __init__( codename, init_codename, path, + gnupg_home, mirror='http://deb.debian.org/debian'): repo_attrs = RepoAttributes(codename, arch, ['main', 'added'], mirror) @@ -395,11 +396,12 @@ def __init__( else: init_attrs = None - super().__init__(path, init_attrs, repo_attrs, 'Elbe', 'Elbe Binary Cdrom Repo') + super().__init__(path, init_attrs, repo_attrs, 'Elbe', 'Elbe Binary Cdrom Repo', + gnupg_home) class CdromSrcRepo(RepoBase): - def __init__(self, codename, init_codename, path, maxsize, + def __init__(self, codename, init_codename, path, maxsize, gnupg_home, mirror='http://deb.debian.org/debian'): repo_attrs = RepoAttributes(codename, @@ -419,16 +421,18 @@ def __init__(self, codename, init_codename, path, maxsize, else: init_attrs = None - super().__init__(path, init_attrs, repo_attrs, 'Elbe', 'Elbe Source Cdrom Repo', maxsize) + super().__init__(path, init_attrs, repo_attrs, 'Elbe', 'Elbe Source Cdrom Repo', + gnupg_home, maxsize) class ToolchainRepo(RepoBase): - def __init__(self, arch, codename, path): + def __init__(self, arch, codename, path, gnupg_home): repo_attrs = RepoAttributes(codename, arch, 'main') - super().__init__(path, None, repo_attrs, 'toolchain', 'Toolchain binary packages Repo') + super().__init__(path, None, repo_attrs, 'toolchain', 'Toolchain binary packages Repo', + gnupg_home) class ProjectRepo(RepoBase): - def __init__(self, arch, codename, path): + def __init__(self, arch, codename, path, gnupg_home): repo_attrs = RepoAttributes(codename, [arch, 'amd64', 'source'], 'main') - super().__init__(path, None, repo_attrs, 'Local', 'Self build packages Repo') + super().__init__(path, None, repo_attrs, 'Local', 'Self build packages Repo', gnupg_home) diff --git a/elbepack/updated.py b/elbepack/updated.py index 0d9fa2ad2..1a28865f4 100644 --- a/elbepack/updated.py +++ b/elbepack/updated.py @@ -31,7 +31,7 @@ ElbeInstallProgress, ElbeOpProgress, ) -from elbepack.egpg import unsign_file +from elbepack.egpg import TARGET_GNUPG_HOME, unsign_file from elbepack.paths import ( DOWNGRADE_ALLOWED_FILE, ELBE_CACHE_DIR, POST_SCRIPT, PRE_SCRIPT, UPDATE_STATE_FILE, @@ -571,7 +571,7 @@ def handle_update_file(upd_file, status, remove=False): _, extension = os.path.splitext(upd_file) if extension == '.gpg': - fname = unsign_file(upd_file) + fname = unsign_file(upd_file, TARGET_GNUPG_HOME) if remove: os.remove(upd_file) if fname: diff --git a/elbepack/updatepkg.py b/elbepack/updatepkg.py index 6ecdad4ec..3cd24c8d9 100644 --- a/elbepack/updatepkg.py +++ b/elbepack/updatepkg.py @@ -105,7 +105,7 @@ def gen_update_pkg(project, xml_filename, upd_filename, if xml_filename: repodir = os.path.join(update, 'repo') - repo = UpdateRepo(xml, repodir) + repo = UpdateRepo(xml, repodir, os.path.join(project.builddir, 'gnupg')) for fname in fnamelist: path = os.path.join( From 54c6114622120dead787c45dd96533a4c97b51e3 Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Wed, 1 Jul 2026 14:43:14 +0200 Subject: [PATCH 04/13] add minimal VM-free ELBE build container With the recent additions, it is no longer necessary to spawn an initvm to build ELBE images. Therefore, provide a container that enables this use case and replace the existing legacy containerfile with it. Signed-off-by: Florian Kauer --- .dockerignore | 6 ++ contrib/containerfile/Containerfile | 31 +++++++++ contrib/containerfile/Containerfile.in | 67 ------------------- contrib/containerfile/Containerfile.local | 52 ++++++++++++++ contrib/containerfile/Makefile | 65 ++++-------------- contrib/containerfile/README.md | 48 ------------- .../+vmless-build-container.feature.rst | 2 + 7 files changed, 104 insertions(+), 167 deletions(-) create mode 100644 .dockerignore create mode 100644 contrib/containerfile/Containerfile delete mode 100644 contrib/containerfile/Containerfile.in create mode 100644 contrib/containerfile/Containerfile.local delete mode 100644 contrib/containerfile/README.md create mode 100644 newsfragments/+vmless-build-container.feature.rst diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..69e73aa36 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.mypy_cache +.pytest_cache +*.deb +*.buildinfo +*.changes diff --git a/contrib/containerfile/Containerfile b/contrib/containerfile/Containerfile new file mode 100644 index 000000000..210fc1cc9 --- /dev/null +++ b/contrib/containerfile/Containerfile @@ -0,0 +1,31 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +FROM debian:trixie-slim + +USER root +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + elbe-archive-keyring + +RUN echo 'deb [signed-by=/usr/share/keyrings/elbe-archive-keyring.gpg] http://debian.linutronix.de/elbe bullseye main' \ + > /etc/apt/sources.list.d/elbe.list && \ + apt-get update -y && \ + apt-get install -y --no-install-recommends \ + python3-elbe-daemon \ + python3-elbe-soap \ + python3-elbe-control \ + locales && \ + apt-get clean -y + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen + +VOLUME [ "/build" ] + +USER root +WORKDIR /build diff --git a/contrib/containerfile/Containerfile.in b/contrib/containerfile/Containerfile.in deleted file mode 100644 index 9aa586b24..000000000 --- a/contrib/containerfile/Containerfile.in +++ /dev/null @@ -1,67 +0,0 @@ -# ELBE - Debian Based Embedded Rootfilesystem Builder -# SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2014-2015 Silvio Fricke -# SPDX-FileCopyrightText: 2018 Linutronix GmbH - -# This Containerfile generates an image for the elbe buildsystem -FROM debian:trixie-slim - -USER root -ENV DEBIAN_FRONTEND noninteractive - -# update, upgrade and install elbe runtime-dependencies -RUN apt-get update -y -RUN apt-get install -y --no-install-recommends \ - -o Dpkg::Options::="--force-confnew" \ - ca-certificates \ - sudo \ - vim-nox \ - elbe-archive-keyring \ - gnupg \ - python3-setuptools \ - python3-yaml \ - python3-jsonschema \ - locales \ - gcc \ - g++ \ - diffstat \ - texinfo \ - gawk \ - chrpath \ - python3-mako \ - fuseiso9660 \ - aptly \ - debian-archive-keyring \ - qemu-system-x86 - -RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen - -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -# install current elbe -RUN echo 'deb [signed-by=/usr/share/keyrings/elbe-archive-keyring.gpg] http://debian.linutronix.de/elbe bullseye main' >> /etc/apt/sources.list -RUN apt-get update -y -RUN apt-get install -y --no-install-recommends \ - elbe \ - elbe-doc -RUN apt-get clean -y -RUN rm -rf /var/lib/apt/lists/* - -# create elbe user -RUN groupadd -g @KVMGID@ -o -r kvm-elbe -RUN useradd -d /home/elbe -l -U -G kvm-elbe,libvirt -m -s /bin/bash -o -u @USERID@ elbe -RUN echo "root:elbe" | chpasswd -RUN echo "elbe:elbe" | chpasswd - -VOLUME [ "/sys/fs/cgroup" ] -VOLUME [ "/elbe" ] -VOLUME [ "/var/cache/elbe" ] - -# sudo for elbe -RUN echo "%elbe ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/elbegrp -RUN chmod 0440 /etc/sudoers.d/elbegrp - - -CMD [ "/bin/bash" ] diff --git a/contrib/containerfile/Containerfile.local b/contrib/containerfile/Containerfile.local new file mode 100644 index 000000000..c263df77d --- /dev/null +++ b/contrib/containerfile/Containerfile.local @@ -0,0 +1,52 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +FROM debian:trixie-slim AS builder + +USER root +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates + +WORKDIR /usr/src/elbe +COPY . . + +RUN apt-get update -y && \ + apt-get build-dep -y . && \ + DEB_BUILD_OPTIONS=nocheck sphinxflags= dpkg-buildpackage -us -uc -b && \ + mkdir -p /out && mv ../*.deb /out/ + +FROM debian:trixie-slim + +USER root +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates + +COPY --from=builder /out/*.deb /tmp/local-debs/ +RUN apt-get update -y && apt-get upgrade -y + +RUN apt-get install -y --no-install-recommends \ + locales \ + /tmp/local-debs/elbe-schema_*.deb \ + /tmp/local-debs/python3-elbe-common_*.deb \ + /tmp/local-debs/python3-elbe-bin_*.deb \ + /tmp/local-debs/python3-elbe-buildenv_*.deb \ + /tmp/local-debs/python3-elbe-daemon_*.deb \ + /tmp/local-debs/python3-elbe-soap_*.deb \ + /tmp/local-debs/python3-elbe-control_*.deb && \ + rm -rf /tmp/local-debs && \ + apt-get clean -y + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen + +VOLUME [ "/build" ] + +USER root +WORKDIR /build diff --git a/contrib/containerfile/Makefile b/contrib/containerfile/Makefile index 3a9f6a934..735d006ba 100644 --- a/contrib/containerfile/Makefile +++ b/contrib/containerfile/Makefile @@ -1,12 +1,9 @@ # ELBE - Debian Based Embedded Rootfilesystem Builder # SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2015 Silvio Fricke -# SPDX-FileCopyrightText: 2018 Linutronix GmbH +# SPDX-FileCopyrightText: 2026 Linutronix GmbH -IMAGENAME ?= elbe-devel-image -CONTAINERNAME ?= elbe-devel -KVMGID ?= $(shell ls -n /dev/kvm | awk '{ print $$4 }') -UID ?= $(shell id -u) +IMAGENAME ?= elbe-buildenv-image +CONTAINERNAME ?= elbe-buildenv # Container engine to use. Defaults to podman if installed, else docker. ENGINE ?= $(shell command -v podman >/dev/null 2>&1 && echo podman || echo docker) @@ -14,11 +11,6 @@ ENGINE ?= $(shell command -v podman >/dev/null 2>&1 && echo podman || echo docke # Common container run options RUN_OPTS = \ --cap-add SYS_ADMIN \ - --security-opt seccomp=unconfined \ - --security-opt apparmor=unconfined \ - --group-add kvm \ - --device /dev/kvm \ - --device /dev/fuse \ --rm \ --interactive \ --tty @@ -32,51 +24,20 @@ ifeq ($(ENGINE),podman) RUN_OPTS += --userns=keep-id endif -# container commands build: - test -c /dev/kvm || ( echo "/dev/kvm not found" && false ) - test -c /dev/fuse || ( echo "/dev/fuse not found" && false ) - test -n "$(KVMGID)" || ( echo "detecting groupid of /dev/kvm failed" && false ) - sed -e "s#@KVMGID@#$(KVMGID)#g" \ - -e "s#@USERID@#$(UID)#g" \ - Containerfile.in > Containerfile $(ENGINE) build --build-arg http_proxy=$(http_proxy) \ --build-arg https_proxy=$(https_proxy) \ --build-arg no_proxy=$(no_proxy) \ - --no-cache \ - -f Containerfile \ + --network=slirp4netns \ -t $(IMAGENAME) . - rm Containerfile -start-devel: - $(ENGINE) ps | grep $(CONTAINERNAME)$$ || \ - $(ENGINE) run --name $(CONTAINERNAME) \ - -e http_proxy=$(http_proxy) \ - -e https_proxy=$(https_proxy) \ - -e no_proxy=$(no_proxy) \ - -v $(realpath ../../.):/var/cache/elbe/devel -w /var/cache/elbe/devel \ - $(RUN_OPTS) \ - $(IMAGENAME) - -start: - $(ENGINE) ps | grep $(CONTAINERNAME)$$ || \ - $(ENGINE) run --name $(CONTAINERNAME) \ - -e http_proxy=$(http_proxy) \ - -e https_proxy=$(https_proxy) \ - -e no_proxy=$(no_proxy) \ - $(RUN_OPTS) \ - $(IMAGENAME) - -stop: - -$(ENGINE) stop $(CONTAINERNAME) - -stoprm: stop - -$(ENGINE) rm $(CONTAINERNAME) - -clean: stoprm - -$(ENGINE) rmi $(IMAGENAME) - -connect: start - $(ENGINE) exec -tiu $(UID) $(CONTAINERNAME) /bin/bash +# Builds the elbe .deb packages from this checkout first +build-local: + $(ENGINE) build --build-arg http_proxy=$(http_proxy) \ + --build-arg https_proxy=$(https_proxy) \ + --build-arg no_proxy=$(no_proxy) \ + --network=slirp4netns \ + -f Containerfile.local \ + -t $(IMAGENAME) ../.. -.PHONY: build start stop stoprm clean connect +.PHONY: build build-local diff --git a/contrib/containerfile/README.md b/contrib/containerfile/README.md deleted file mode 100644 index 0a002edb7..000000000 --- a/contrib/containerfile/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Dockerfile for elbe - - -# SPDX-FileCopyrightText: 2018 Linutronix GmbH -?> - -[elbe][elb] is a debian based system to generate root-filesystems for embedded -devices. - -[docker][doc] and [podman][pod] are open-source projects to easily create -lightweight, portable, self-sufficient containers from any application. - -This is a Containerfile to generate an ELBE development and runtime environment for -systems that are not Debian based or where you prefer a clean separation. - -[doc]: https://www.docker.io "Docker Homepage" -[pod]: https://podman.io "Podman Homepage" -[elb]: http://elbe-rfs.org "ELBE Homepage" - -## Dependencies - -You need `podman` or `docker` installed, plus `make`. `podman` is used by -default if it is found on the `PATH`; set `ENGINE=docker` to force Docker -instead (e.g. `make ENGINE=docker build`). - -## usage - -A `Makefile` with some handy targets are provided. Per default the image name -is `elbe-devel-image` and a started container name is `elbe-devel`. This names are -changeable via `IMAGENAME` and `CONTAINERNAME` environment variables. - -* `build`: build the image -* `start` start a container, and use packaged elbe -* `start-devel` start a container, mounts the elbe git-archive to `/var/cache/elbe` -* `stop`: stop a running container -* `stoprm`: stop and remove the container -* `connect`: attach a new terminal to a running container - -After `connect` you can find the elbe git repository under `/elbe`. - -## passwords - - root: elbe - elbe: elbe - diff --git a/newsfragments/+vmless-build-container.feature.rst b/newsfragments/+vmless-build-container.feature.rst new file mode 100644 index 000000000..5bd0ca980 --- /dev/null +++ b/newsfragments/+vmless-build-container.feature.rst @@ -0,0 +1,2 @@ +Provide a container (``contrib/containerfile``) that builds ELBE images with the +new ``elbe build`` command, without spawning an initvm. From 4a317605ce33365097642287e838f0d20dc74930 Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Mon, 27 Jul 2026 09:44:59 +0200 Subject: [PATCH 05/13] efilesystem: shellhelper: move _Mount from imgutils.py to shellhelper.py The rbind-based pseudo filesystem mounting for chroot() needs the _Mount helper, but imgutils.py cannot be imported from shellhelper.py without creating a cyclic dependency. Move _Mount there first. Signed-off-by: Florian Kauer --- elbepack/commands/fetch_initvm_pkgs.py | 2 +- elbepack/efilesystem.py | 3 +- elbepack/finetuning.py | 4 +-- elbepack/hdimg.py | 4 +-- elbepack/imgutils.py | 45 -------------------------- elbepack/shellhelper.py | 45 ++++++++++++++++++++++++++ 6 files changed, 51 insertions(+), 52 deletions(-) diff --git a/elbepack/commands/fetch_initvm_pkgs.py b/elbepack/commands/fetch_initvm_pkgs.py index f6d9a926b..17a367f59 100644 --- a/elbepack/commands/fetch_initvm_pkgs.py +++ b/elbepack/commands/fetch_initvm_pkgs.py @@ -17,12 +17,12 @@ from elbepack.dump import get_initvm_pkglist from elbepack.egpg import INITVM_GNUPG_HOME from elbepack.elbexml import ElbeXML, ValidationError -from elbepack.imgutils import mount from elbepack.log import elbe_logging from elbepack.paths import ( BINARIES_MAIN_DIR, INITVM_BIN_REPO_DIR, INITVM_SRC_REPO_DIR, SOURCES_DIR, ) from elbepack.repomanager import CdromInitRepo, CdromSrcRepo +from elbepack.shellhelper import mount def run_command(argv): diff --git a/elbepack/efilesystem.py b/elbepack/efilesystem.py index 653a2c2d4..0dded9b8f 100644 --- a/elbepack/efilesystem.py +++ b/elbepack/efilesystem.py @@ -15,10 +15,9 @@ from elbepack.filesystem import Filesystem from elbepack.fstab import fstabentry -from elbepack.imgutils import mount from elbepack.licencexml import copyright_xml from elbepack.packers import default_packer -from elbepack.shellhelper import chroot, do +from elbepack.shellhelper import chroot, do, mount from elbepack.version import elbe_version diff --git a/elbepack/finetuning.py b/elbepack/finetuning.py index e19339dbd..08fac9c29 100644 --- a/elbepack/finetuning.py +++ b/elbepack/finetuning.py @@ -22,10 +22,10 @@ from elbepack.egpg import TARGET_GNUPG_HOME from elbepack.filesystem import Filesystem -from elbepack.imgutils import losetup, mount +from elbepack.imgutils import losetup from elbepack.packers import default_packer, packers from elbepack.paths import DOWNGRADE_ALLOWED_FILE, REPOS_BASE_DIR -from elbepack.shellhelper import ELBE_LOGGING, chroot, do, env_add, get_env_with_sbin, run +from elbepack.shellhelper import ELBE_LOGGING, chroot, do, env_add, get_env_with_sbin, mount, run from elbepack.treeutils import strip_leading_whitespace_from_lines diff --git a/elbepack/hdimg.py b/elbepack/hdimg.py index f190a6007..46ac9bed4 100644 --- a/elbepack/hdimg.py +++ b/elbepack/hdimg.py @@ -16,8 +16,8 @@ from elbepack.filesystem import Filesystem, size_to_int from elbepack.fstab import fstabentry, hdpart, mountpoint_dict -from elbepack.imgutils import dd, losetup, mount -from elbepack.shellhelper import chroot, do +from elbepack.imgutils import dd, losetup +from elbepack.shellhelper import chroot, do, mount def mkfs_mtd(mtd, fslabel, target): diff --git a/elbepack/imgutils.py b/elbepack/imgutils.py index 5c8051d4c..0b6c8fe1a 100644 --- a/elbepack/imgutils.py +++ b/elbepack/imgutils.py @@ -46,50 +46,5 @@ def losetup(dev, extra_args=[]): do(['losetup', '--detach', loopdev], check=False) -class _Mount: - # This is not using contextlib.contextmanager as it will be pass to our - # RPCAPTCache which uses the pickle serialization. - # The generator by contextlib.contextmanager is not compatible with pickle. - def __init__(self, device, target, *, bind=False, type=None, options=None, log_output=True, - force_writable=False): - self.log_output = log_output - self.target = target - - cmd = ['mount'] - if bind: - cmd.append('--bind') - - if options is not None: - cmd.extend(['-o', options]) - - if force_writable: - cmd.append('--rw') - - if type is not None: - cmd.extend(['-t', type]) - - if device is None: - device = 'none' - - cmd.extend([device, target]) - - self.cmd = cmd - - def _run_cmd(self, cmd, *args, **kwargs): - if self.log_output: - do(cmd, *args, **kwargs) - else: - subprocess.run(cmd, *args, **kwargs) - - def __enter__(self): - self._run_cmd(self.cmd) - - def __exit__(self, exc_type, exc_value, traceback): - self._run_cmd(['umount', self.target], check=False) - - -mount = _Mount - - def dd(args, /, **kwargs): do(['dd', *[f'{k}={v}' for k, v in args.items()]], **kwargs) diff --git a/elbepack/shellhelper.py b/elbepack/shellhelper.py index 722a1a908..0b29ef585 100644 --- a/elbepack/shellhelper.py +++ b/elbepack/shellhelper.py @@ -152,6 +152,51 @@ def _target_path(directory): return FALLBACK_PATH +class _Mount: + # This is not using contextlib.contextmanager as it will be pass to our + # RPCAPTCache which uses the pickle serialization. + # The generator by contextlib.contextmanager is not compatible with pickle. + def __init__(self, device, target, *, bind=False, type=None, options=None, log_output=True, + force_writable=False): + self.log_output = log_output + self.target = target + + cmd = ['mount'] + if bind: + cmd.append('--bind') + + if options is not None: + cmd.extend(['-o', options]) + + if force_writable: + cmd.append('--rw') + + if type is not None: + cmd.extend(['-t', type]) + + if device is None: + device = 'none' + + cmd.extend([device, target]) + + self.cmd = cmd + + def _run_cmd(self, cmd, *args, **kwargs): + if self.log_output: + do(cmd, *args, **kwargs) + else: + subprocess.run(cmd, *args, **kwargs) + + def __enter__(self): + self._run_cmd(self.cmd) + + def __exit__(self, exc_type, exc_value, traceback): + self._run_cmd(['umount', self.target], check=False) + + +mount = _Mount + + def chroot(directory, cmd, /, *, env_add=None, **kwargs): """chroot() - Wrapper around do(). From e39876148704d615877d7cbc53a3bfd26286ee89 Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Mon, 27 Jul 2026 09:44:59 +0200 Subject: [PATCH 06/13] efilesystem: rbind pseudo filesystems instead of creating them from scratch Several operations require a chroot with the pseudo filesystems (/proc, /sys and /dev) in it. Inside a container, the container runtime already protects parts of it (like /proc/kcore) by overlaying it with a null bind mount. Doing a fresh mount from scratch (e.g. mount -t proc None foo) inside a completely new mount and user namespace, would expose the content again, so the kernel has explicit procection against it (-> mount_too_revealing). But we do not need fresh pseudo filesystems. We can just rebind what we already have. The only issue with that is that we need to scope it more tightly around the operations that actually require this setup with chroot and pseudo filesystems, otherwise we would fill up the newly created filesystem with stuff from the pseudo filesystems we do not need. Signed-off-by: Florian Kauer --- elbepack/efilesystem.py | 49 ++++++++++++++++++++++++----------------- elbepack/rpcaptcache.py | 14 ++++++++++++ elbepack/shellhelper.py | 40 +++++++++++++++++++++++++++------ 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/elbepack/efilesystem.py b/elbepack/efilesystem.py index 0dded9b8f..d4456e415 100644 --- a/elbepack/efilesystem.py +++ b/elbepack/efilesystem.py @@ -17,7 +17,7 @@ from elbepack.fstab import fstabentry from elbepack.licencexml import copyright_xml from elbepack.packers import default_packer -from elbepack.shellhelper import chroot, do, mount +from elbepack.shellhelper import bind_mount_pseudo_filesystems, chroot, do from elbepack.version import elbe_version @@ -368,16 +368,6 @@ def __enter__(self): for excursion in excursions ] - if self.path != '/': - self._exitstack.enter_context( - mount(None, self.fname('/proc'), type='proc', log_output=False)) - self._exitstack.enter_context( - mount(None, self.fname('/sys'), type='sysfs', log_output=False)) - self._exitstack.enter_context( - mount('/dev', self.fname('/dev'), bind=True, log_output=False)) - self._exitstack.enter_context( - mount('/dev/pts', self.fname('/dev/pts'), bind=True, log_output=False)) - return self def __exit__(self, typ, value, traceback): @@ -392,6 +382,16 @@ def end_excursion(self, origin): excursion_context.end() return + def _switch_to_chroot(self): + os.chdir(self.path) + if self.path != '/': + os.chroot(self) + + def _switch_to_real_root(self): + os.fchdir(self.cwd) + if self.path != '/': + os.chroot('.') + def enter_chroot(self): assert not self.inchroot @@ -399,24 +399,33 @@ def enter_chroot(self): os.environ['LANGUAGE'] = 'C' os.environ['LC_ALL'] = 'C' - os.chdir(self.path) + self._switch_to_chroot() self.inchroot = True - if self.path == '/': - return - - os.chroot(self) - def leave_chroot(self): assert self.inchroot - os.fchdir(self.cwd) - + self._switch_to_real_root() self.inchroot = False + + @contextlib.contextmanager + def mount_pseudo_filesystems(self): + assert self.inchroot + if self.path == '/': + yield return - os.chroot('.') + self._switch_to_real_root() + try: + with bind_mount_pseudo_filesystems(self.path): + self._switch_to_chroot() + try: + yield + finally: + self._switch_to_real_root() + finally: + self._switch_to_chroot() class TargetFs(ChRootFilesystem): diff --git a/elbepack/rpcaptcache.py b/elbepack/rpcaptcache.py index bc6623636..c00f59a9f 100644 --- a/elbepack/rpcaptcache.py +++ b/elbepack/rpcaptcache.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # SPDX-FileCopyrightText: 2014-2018 Linutronix GmbH +import functools import os import sys import time @@ -62,6 +63,14 @@ def __init__(self, rfs): self.finalizer = Finalize(self, self.rfs.leave_chroot, exitpriority=10) +def _with_pseudo_filesystems(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + with self.rfs.mount_pseudo_filesystems(): + return func(self, *args, **kwargs) + return wrapper + + @MyMan.register('RPCAPTCache') class RPCAPTCache(InChRootObject): @@ -208,14 +217,17 @@ def mark_delete(self, pkgname): p = self.cache[pkgname] p.mark_delete(purge=True) + @_with_pseudo_filesystems def update(self): self.cache.update(fetch_progress=ElbeAcquireProgress()) self.cache.open(progress=ElbeOpProgress()) + @_with_pseudo_filesystems def fetch_archives(self): print('Fetching packages...') self.cache.fetch_archives(ElbeAcquireProgress()) + @_with_pseudo_filesystems def commit(self): os.environ['DEBIAN_FRONTEND'] = 'noninteractive' os.environ['DEBONF_NONINTERACTIVE_SEEN'] = 'true' @@ -268,6 +280,7 @@ def get_pkg(self, pkgname): def get_corresponding_source_packages(self, pkg_lst=None, *, include_built_using=True): return get_corresponding_source_packages(self.cache, pkg_lst, include_built_using) + @_with_pseudo_filesystems def download_binary(self, pkgname, path, version=None): p = self.cache[pkgname] if version is None: @@ -277,6 +290,7 @@ def download_binary(self, pkgname, path, version=None): rel_filename = pkgver.fetch_binary(path, ElbeAcquireProgress()) return self.rfs.fname(rel_filename) + @_with_pseudo_filesystems def download_source(self, src_name, src_version, dest_dir): return self.rfs.fname(fetch_source(src_name, src_version, dest_dir, ElbeAcquireProgress())) diff --git a/elbepack/shellhelper.py b/elbepack/shellhelper.py index 0b29ef585..e2fbfdb13 100644 --- a/elbepack/shellhelper.py +++ b/elbepack/shellhelper.py @@ -156,14 +156,18 @@ class _Mount: # This is not using contextlib.contextmanager as it will be pass to our # RPCAPTCache which uses the pickle serialization. # The generator by contextlib.contextmanager is not compatible with pickle. - def __init__(self, device, target, *, bind=False, type=None, options=None, log_output=True, - force_writable=False): + def __init__(self, device, target, *, bind=False, rbind=False, type=None, options=None, + log_output=True, force_writable=False): self.log_output = log_output self.target = target + self.rbind = rbind + cmd = ['mount'] if bind: cmd.append('--bind') + elif rbind: + cmd.append('--rbind') if options is not None: cmd.extend(['-o', options]) @@ -189,14 +193,35 @@ def _run_cmd(self, cmd, *args, **kwargs): def __enter__(self): self._run_cmd(self.cmd) + if self.rbind: + # Detach the bind-mounted subtree from the shared + # propagation group. Without this, unmounting it in + # __exit__ can propagate back and unmount the + # corresponding mounts at the source. + self._run_cmd(['mount', '--make-rprivate', self.target]) def __exit__(self, exc_type, exc_value, traceback): - self._run_cmd(['umount', self.target], check=False) + cmd = ['umount', '--lazy', self.target] if self.rbind else ['umount', self.target] + self._run_cmd(cmd, check=False) mount = _Mount +@contextlib.contextmanager +def bind_mount_pseudo_filesystems(directory): + if directory == '/': + yield + return + + with contextlib.ExitStack() as stack: + for src in ['/proc', '/sys', '/dev']: + target = os.path.join(directory, src.lstrip('/')) + os.makedirs(target, exist_ok=True) + stack.enter_context(mount(src, target, rbind=True, log_output=False)) + yield + + def chroot(directory, cmd, /, *, env_add=None, **kwargs): """chroot() - Wrapper around do(). @@ -221,10 +246,11 @@ def chroot(directory, cmd, /, *, env_add=None, **kwargs): if env_add: new_env.update(env_add) - if _is_shell_cmd(cmd): - do(['/usr/sbin/chroot', directory, '/bin/sh', '-c', cmd], env_add=new_env, **kwargs) - else: - do(['/usr/sbin/chroot', directory] + cmd, env_add=new_env, **kwargs) + with bind_mount_pseudo_filesystems(directory): + if _is_shell_cmd(cmd): + do(['/usr/sbin/chroot', directory, '/bin/sh', '-c', cmd], env_add=new_env, **kwargs) + else: + do(['/usr/sbin/chroot', directory] + cmd, env_add=new_env, **kwargs) def env_add(d): From 1edb61ce9ce8f96e9b5de9b1f56d21f8bfbb8243 Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Thu, 30 Jul 2026 10:02:43 +0200 Subject: [PATCH 07/13] elbepack: imgutils: do not depend on udev if it does not exist Inside a container, we do not have udev running. Therefore, there is no need to wait for potential interference with udev. Instead, create the device nodes directly from the information in sysfs. Signed-off-by: Florian Kauer --- elbepack/imgutils.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/elbepack/imgutils.py b/elbepack/imgutils.py index 0b6c8fe1a..e45ea6218 100644 --- a/elbepack/imgutils.py +++ b/elbepack/imgutils.py @@ -4,12 +4,30 @@ import contextlib import fcntl +import os import pathlib +import stat import subprocess from elbepack.shellhelper import ELBE_LOGGING, do, run +def _udev_available(): + return pathlib.Path('/run/udev/control').is_socket() + + +def _mknod_from_sysfs(device_name): + devpath = f'/dev/{device_name}' + if os.path.exists(devpath): + return + + dev_attr = pathlib.Path('/sys/class/block', device_name, 'dev').read_text().strip() + major, minor = (int(x) for x in dev_attr.split(':')) + with contextlib.suppress(FileExistsError): + os.mknod(devpath, mode=0o660 | stat.S_IFBLK, device=os.makedev(major, minor)) + os.chmod(devpath, 0o660) + + def _wait_on_udev_for_device_and_partitions(device): # The callers expect the udev symlinks of the loop device and its # partitions to be present. @@ -21,13 +39,19 @@ def _wait_on_udev_for_device_and_partitions(device): # However udev processing triggers a rescan of the partitions, removing # the entries for a short time. Prevent udev from doing so while we iterate. fcntl.flock(f, fcntl.LOCK_EX) - partitions = [ - '/dev/' + entry.name + partition_names = [ + entry.name for entry in pathlib.Path('/sys/class/block', device_name).iterdir() if entry.name.startswith(device_name) ] + if not _udev_available(): + for name in (device_name, *partition_names): + _mknod_from_sysfs(name) + return + # All partitions need to be mentioned explicitly. + partitions = ['/dev/' + name for name in partition_names] subprocess.run(['udevadm', 'wait', device, *partitions], check=True, timeout=30) From c3359a1f21d2dbebc60c6162c552708603a4efc1 Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Thu, 30 Jul 2026 10:02:43 +0200 Subject: [PATCH 08/13] elbepack: imgutils: create by-uuid symlinks for device nodes created without udev Setup UUID as needed e.g. for grub, because that is usually also done by udev. So without udev, we need a replacement for that functionality. Signed-off-by: Florian Kauer --- elbepack/imgutils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/elbepack/imgutils.py b/elbepack/imgutils.py index e45ea6218..d81c92d40 100644 --- a/elbepack/imgutils.py +++ b/elbepack/imgutils.py @@ -28,6 +28,22 @@ def _mknod_from_sysfs(device_name): os.chmod(devpath, 0o660) +def _symlink_by_uuid_from_blkid(device_name): + devpath = f'/dev/{device_name}' + blkid = subprocess.run( + ['blkid', '-s', 'UUID', '-o', 'value', devpath], + stdout=subprocess.PIPE, check=False, + ) + uuid = blkid.stdout.decode('ascii').strip() + if blkid.returncode != 0 or not uuid: + return + + by_uuid_dir = pathlib.Path('/dev/disk/by-uuid') + by_uuid_dir.mkdir(parents=True, exist_ok=True) + with contextlib.suppress(FileExistsError): + (by_uuid_dir / uuid).symlink_to(devpath) + + def _wait_on_udev_for_device_and_partitions(device): # The callers expect the udev symlinks of the loop device and its # partitions to be present. @@ -48,6 +64,7 @@ def _wait_on_udev_for_device_and_partitions(device): if not _udev_available(): for name in (device_name, *partition_names): _mknod_from_sysfs(name) + _symlink_by_uuid_from_blkid(name) return # All partitions need to be mentioned explicitly. From 3793bb69719c29907454021b2bb6b1950afcae69 Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Sat, 25 Jul 2026 09:16:27 +0200 Subject: [PATCH 09/13] container: avoid the need for explicitly adding CAP_SYS_ADMIN By unsharing into a new user and mount namespace, we get sufficient permissions to perform (bind) mounts needed for the build process. With this, we can avoid the need for an explicit --cap-add CAP_SYS_ADMIN Signed-off-by: Florian Kauer --- contrib/containerfile/Containerfile | 9 ++++++++- contrib/containerfile/Containerfile.local | 9 ++++++++- contrib/containerfile/entrypoint.sh | 20 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100755 contrib/containerfile/entrypoint.sh diff --git a/contrib/containerfile/Containerfile b/contrib/containerfile/Containerfile index 210fc1cc9..26543da71 100644 --- a/contrib/containerfile/Containerfile +++ b/contrib/containerfile/Containerfile @@ -11,7 +11,8 @@ RUN apt-get update -y && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ ca-certificates \ - elbe-archive-keyring + elbe-archive-keyring \ + tini RUN echo 'deb [signed-by=/usr/share/keyrings/elbe-archive-keyring.gpg] http://debian.linutronix.de/elbe bullseye main' \ > /etc/apt/sources.list.d/elbe.list && \ @@ -29,3 +30,9 @@ VOLUME [ "/build" ] USER root WORKDIR /build + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Use tini for reaping child processes. +ENTRYPOINT ["tini", "--", "/entrypoint.sh"] diff --git a/contrib/containerfile/Containerfile.local b/contrib/containerfile/Containerfile.local index c263df77d..245aba386 100644 --- a/contrib/containerfile/Containerfile.local +++ b/contrib/containerfile/Containerfile.local @@ -27,7 +27,8 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ - ca-certificates + ca-certificates \ + tini COPY --from=builder /out/*.deb /tmp/local-debs/ RUN apt-get update -y && apt-get upgrade -y @@ -50,3 +51,9 @@ VOLUME [ "/build" ] USER root WORKDIR /build + +COPY contrib/containerfile/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Use tini for reaping child processes. +ENTRYPOINT ["tini", "--", "/entrypoint.sh"] diff --git a/contrib/containerfile/entrypoint.sh b/contrib/containerfile/entrypoint.sh new file mode 100755 index 000000000..b46e23787 --- /dev/null +++ b/contrib/containerfile/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH +set -e + +# Check if we already have CAP_SYS_ADMIN +# In that case we assume we are already running rootful, +# so no need for unshare, but this mode also enables usage +# of loop devices, so perform mknod to enable its usage. +capeff=$(grep '^CapEff:' /proc/self/status | cut -f2) +if [ $(( 0x$capeff & 0x200000 )) -ne 0 ]; then + i=0 + while [ "$i" -lt 64 ]; do + [ -e "/dev/loop$i" ] || mknod -m 660 "/dev/loop$i" b 7 "$i" 2>/dev/null || true + i=$((i + 1)) + done + exec "$@" +fi + +exec unshare --user --map-root-user --map-users=all --map-groups=all --mount -- "$@" From 5b96212ada250a6bfd70278eb37c91546b925e07 Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Fri, 24 Jul 2026 23:17:53 +0200 Subject: [PATCH 10/13] elbepack: localbuildaction: check the need for rootful container Several actions still require losetup or other actions that require a rootful container (such as mknod). Instead of starting the build and then later failing with an error message, check beforehand for critical elements in the XML and print a respective error message. With proper setup and high privileges it is actually possible to use loop devices or mknod in containers. Provide the respective documentation how. Signed-off-by: Florian Kauer --- debian/python3-elbe-common.install | 1 + docs/elbe-build.rst | 56 ++++++++ elbepack/localbuildaction.py | 10 +- elbepack/rootcheck.py | 98 ++++++++++++++ elbepack/tests/test_rootcheck.py | 203 +++++++++++++++++++++++++++++ 5 files changed, 364 insertions(+), 4 deletions(-) create mode 100644 elbepack/rootcheck.py create mode 100644 elbepack/tests/test_rootcheck.py diff --git a/debian/python3-elbe-common.install b/debian/python3-elbe-common.install index 190fa90c2..c1ac2dd76 100644 --- a/debian/python3-elbe-common.install +++ b/debian/python3-elbe-common.install @@ -32,6 +32,7 @@ usr/lib/python3.*/*-packages/elbepack/isooptions.py usr/lib/python3.*/*-packages/elbepack/licencexml.py usr/lib/python3.*/*-packages/elbepack/localbuildaction.py usr/lib/python3.*/*-packages/elbepack/log.py +usr/lib/python3.*/*-packages/elbepack/rootcheck.py usr/lib/python3.*/*-packages/elbepack/packers.py usr/lib/python3.*/*-packages/elbepack/paths.py usr/lib/python3.*/*-packages/elbepack/pkgutils.py diff --git a/docs/elbe-build.rst b/docs/elbe-build.rst index e7ecf55f1..22a554aad 100644 --- a/docs/elbe-build.rst +++ b/docs/elbe-build.rst @@ -110,6 +110,62 @@ ready-to-use build environment for this. /work/tests/base-extended/simple-validation/image-base-trixie.xml \ --build-dir /work/build + +Rootful Containers +================== + +Some project XML features make *elbe build* create a Linux loop device +to loop-mount a disk or partition image: + +- ```` inside a ````/```` target image +- ```` (or the ````) inside a + partition's ```` +- a ```` project-finetuning action containing + ``copy_from_partition``, ``copy_to_partition``, or ``command`` + +Creating a loop device requires access to */dev/loop-control*, which +is a host-kernel-wide privilege. Additionally, ``CAP_SYS_ADMIN`` must be +enabled in the **initial user namespace** (not in a namespace created +via ``unshare``), which means **rootless containers cannot create loop +devices**. + +Also, some XMLs might require mknod which is also not possible +in rootless containers. + +If the project XML uses one of the features above, *elbe build* +checks the requirements upfront and aborts immediately. + +If your XML needs these features, either: + +- build via *elbe initvm submit* instead, which runs inside a full + virtual machine with real root privileges, or + +- run the container with elevated privileges (i.e. no rootless container). + Add the following to the *podman run*/*docker run* invocation and run with + elevated privileges (e.g. via ``pkexec`` (PolicyKit)). + + :: + + --cap-add SYS_ADMIN --cap-add MKNOD --device-cgroup-rule='b *:* rmw' \ + --security-opt apparmor=unconfined + + Note that since this is now a rootful container and it there are much more + options for security vulnerabilities to manifest. Also, the networking might + be differently set up, so you might need to add ``--network slirp4netns`` + (or ``--network host`` if ``slirp4netns`` is not installed). + + All together, the command would look like this: + + :: + + pkexec podman run --rm \ + -v $(pwd):/work:Z \ + --cap-add SYS_ADMIN --cap-add MKNOD --device-cgroup-rule='b *:* rmw' \ + --security-opt apparmor=unconfined \ + --network slirp4netns \ + elbe-buildenv-image \ + elbe build /work/myimage.xml --build-dir /work/build + SEE ALSO ======== diff --git a/elbepack/localbuildaction.py b/elbepack/localbuildaction.py index 8dca8bcb1..8fb66e099 100644 --- a/elbepack/localbuildaction.py +++ b/elbepack/localbuildaction.py @@ -9,9 +9,9 @@ import time from elbepack.cli import CliError, with_cli_details -from elbepack.loopcheck import check_loop_mount_requirements from elbepack.projectmanager import ProjectManager from elbepack.repodir import Repodir, RepodirError +from elbepack.rootcheck import check_rootful_requirements from elbepack.xmlpreprocess import preprocess_file prog = os.path.basename(sys.argv[0]) @@ -23,7 +23,8 @@ def local_build_with_repodir_and_dl_result(xmlfile, cdrom, base_image, args): preprocess_xmlfile = os.path.join(args.build_dir, fname) try: with Repodir(xmlfile, preprocess_xmlfile): - _local_build_and_dl_result(preprocess_xmlfile, cdrom, base_image, args) + _local_build_and_dl_result(preprocess_xmlfile, cdrom, base_image, args, + xmlfile_base=xmlfile) except RepodirError as err: raise with_cli_details(err, 127, 'elbe repodir failed') @@ -48,12 +49,13 @@ def _wait_busy(pm, prjdir): raise CliError(191, f'Project build was not successful, current status: {prj.status}') -def _local_build_and_dl_result(xmlfile, cdrom, base_image, args): +def _local_build_and_dl_result(xmlfile, cdrom, base_image, args, xmlfile_base=None): cache_dir = os.path.join(args.build_dir, 'cache') pm = ProjectManager(cache_dir) try: with preprocess_file(xmlfile, variants=args.variants, sshport=args.sshport, - soapport=args.soapport) as xmlfile: + soapport=args.soapport, xmlfile_base=xmlfile_base) as xmlfile: + check_rootful_requirements(xmlfile) prjdir = pm.create_project(xmlfile) if args.writeproject: diff --git a/elbepack/rootcheck.py b/elbepack/rootcheck.py new file mode 100644 index 000000000..6ddcb96aa --- /dev/null +++ b/elbepack/rootcheck.py @@ -0,0 +1,98 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import logging +import os +import subprocess +import tempfile +import textwrap + +from elbepack.cli import CliError +from elbepack.finetuning import LosetupAction +from elbepack.imgutils import losetup +from elbepack.treeutils import etree + + +def _loop_mount_reasons(xml): + reasons = [] + + for grub in xml.all('target/images/*/grub-install'): + hd = grub.get_parent() + reasons.append( + f"<{hd.tag}> image '{hd.text('name')}' uses , which " + 'loop-mounts the disk image to run grub-install in a chroot') + + for bylabel in xml.all('target/fstab/bylabel'): + if (bylabel.has('fs/fs-finetuning/device-command') + or bylabel.has('fs/fs-finetuning/path-command')): + reasons.append( + f"filesystem '{bylabel.text('label')}' uses " + '(or the deprecated ) with a device-command/' + 'path-command, which loop-mounts that partition image') + + for ls in xml.all('target/project-finetuning/losetup'): + if any(child.tag in LosetupAction.needs_loop_device for child in ls): + reasons.append( + f" contains an action " + 'that needs a loop device') + + return reasons + + +def _mknod_reasons(xml): + reasons = [] + + for node in xml.all('target/finetuning/mknod'): + reasons.append( + f"{node.et.text} " + 'creates a device node, which needs to run rootful (real root / ' + 'CAP_MKNOD, and in a container without a remapped user namespace)') + + return reasons + + +def xml_needs_rootful(xml): + return _loop_mount_reasons(xml) + _mknod_reasons(xml) + + +def loop_mount_available(): + with tempfile.NamedTemporaryFile(prefix='elbe-loopcheck-') as f: + f.truncate(1024 * 1024) + try: + with losetup(f.name): + pass + except subprocess.CalledProcessError as e: + logging.debug('loop-mount preflight probe failed: %s', e) + return False + return True + + +def check_rootful_requirements(xmlfile): + xml = etree(xmlfile) + loop_reasons = _loop_mount_reasons(xml) + mknod_reasons = _mknod_reasons(xml) + + problems = [] + if loop_reasons and not loop_mount_available(): + problems.append(( + 'Linux loop devices, but this process is not able to create ' + 'them (e.g. no access to /dev/loop-control)', loop_reasons)) + if mknod_reasons and os.geteuid() != 0: + problems.append(( + 'to create device nodes (mknod), but this process is not ' + 'running as root', mknod_reasons)) + + if not problems: + return + + body = '\n\n'.join( + f'This build needs {desc}:\n' + '\n'.join(f' - {r}' for r in reasons) + for desc, reasons in problems) + + raise CliError(message=textwrap.dedent(f""" + {body} + + See the documentation for the elbe build command how to + run a container with the necessary privileges. + """)) diff --git a/elbepack/tests/test_rootcheck.py b/elbepack/tests/test_rootcheck.py new file mode 100644 index 000000000..6908567f2 --- /dev/null +++ b/elbepack/tests/test_rootcheck.py @@ -0,0 +1,203 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import contextlib +import subprocess + +import pytest + +from elbepack.cli import CliError +from elbepack.rootcheck import ( + check_rootful_requirements, + loop_mount_available, + xml_needs_rootful, +) +from elbepack.treeutils import etree + + +def _xml(s): + return etree(None, string=s) + + +def test_needs_no_rootful_for_plain_xml(): + assert xml_needs_rootful(_xml('')) == [] + + +@pytest.mark.parametrize('hd_tag', ['msdoshd', 'gpthd']) +def test_needs_rootful_for_grub_install(hd_tag): + xml = _xml(f""" + <{hd_tag}> + disk.img + + + """) + reasons = xml_needs_rootful(xml) + assert len(reasons) == 1 + assert 'grub-install' in reasons[0] + + +@pytest.mark.parametrize('command_tag', ['device-command', 'path-command']) +def test_needs_rootful_for_fs_finetuning(command_tag): + xml = _xml(f""" + + + + ext4 + <{command_tag}>echo hi + + + """) + reasons = xml_needs_rootful(xml) + assert len(reasons) == 1 + assert 'rootfs' in reasons[0] + + +def test_needs_no_rootful_for_fs_finetuning_file_command(): + xml = _xml(""" + + + + ext4 + echo hi + + + """) + assert xml_needs_rootful(xml) == [] + + +@pytest.mark.parametrize('child_xml', [ + 'out.bin', + 'in.bin', + 'true', +]) +def test_needs_rootful_for_specific_losetups(child_xml): + xml = _xml(f""" + + {child_xml} + + """) + assert len(xml_needs_rootful(xml)) == 1 + + +def test_needs_no_rootful_for_some_losetups(): + xml = _xml(""" + + + out.img + 83 + in.img + + + """) + assert xml_needs_rootful(xml) == [] + + +def test_loop_mount_available_true(monkeypatch): + @contextlib.contextmanager + def fake_losetup(dev, extra_args=[]): + yield '/dev/loop0' + + monkeypatch.setattr('elbepack.rootcheck.losetup', fake_losetup) + assert loop_mount_available() is True + + +def test_loop_mount_available_false_on_called_process_error(monkeypatch): + @contextlib.contextmanager + def fake_losetup(dev, extra_args=[]): + raise subprocess.CalledProcessError(1, ['losetup']) + yield # pragma: no cover + + monkeypatch.setattr('elbepack.rootcheck.losetup', fake_losetup) + assert loop_mount_available() is False + + +def test_check_rootful_requirements_skips_probe_when_nothing_needed(tmp_path, monkeypatch): + def fail(*a, **k): + raise AssertionError('loop_mount_available should not be called') + + monkeypatch.setattr('elbepack.rootcheck.loop_mount_available', fail) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text('') + check_rootful_requirements(str(xmlfile)) + + +def test_check_rootful_requirements_passes_when_available(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.loop_mount_available', lambda: True) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + disk.img + + """) + check_rootful_requirements(str(xmlfile)) + + +def test_check_rootful_requirements_raises_when_loop_unavailable(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.loop_mount_available', lambda: False) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + disk.img + + """) + with pytest.raises(CliError) as exc_info: + check_rootful_requirements(str(xmlfile)) + assert 'grub-install' in str(exc_info.value) + assert 'elbe build command' in str(exc_info.value) + + +def test_needs_rootful_detects_mknod(): + xml = _xml(""" + + /dev/tty + + """) + reasons = xml_needs_rootful(xml) + assert len(reasons) == 1 + assert 'mknod' in reasons[0] + assert '/dev/tty' in reasons[0] + + +def test_check_rootful_requirements_raises_when_mknod_and_not_root(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.os.geteuid', lambda: 1000) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + /dev/tty + + """) + with pytest.raises(CliError) as exc_info: + check_rootful_requirements(str(xmlfile)) + assert 'mknod' in str(exc_info.value) + + +def test_check_rootful_requirements_passes_when_mknod_and_root(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.os.geteuid', lambda: 0) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + /dev/tty + + """) + check_rootful_requirements(str(xmlfile)) + + +def test_check_rootful_requirements_combines_loop_and_mknod_problems(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.loop_mount_available', lambda: False) + monkeypatch.setattr('elbepack.rootcheck.os.geteuid', lambda: 1000) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + + disk.img + + + /dev/tty + + + """) + with pytest.raises(CliError) as exc_info: + check_rootful_requirements(str(xmlfile)) + assert 'grub-install' in str(exc_info.value) + assert 'mknod' in str(exc_info.value) From d05b2034ca0fb1af0b277d93607b29194b3f2aad Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Tue, 28 Jul 2026 05:42:04 +0200 Subject: [PATCH 11/13] test: use /var/tmp instead of /tmp for pytest /tmp is usually RAM-backed, so when putting build artifacts there, it will fill up very quickly and exhaust the RAM. Therefore, use /var/tmp per default. On Debian systems /var/tmp will still follow a retention policy, so any potential leftovers will be removed after 30 days (in the default configuration). Signed-off-by: Florian Kauer --- conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conftest.py b/conftest.py index a1f8204ff..9182d85c0 100644 --- a/conftest.py +++ b/conftest.py @@ -35,6 +35,9 @@ def pytest_configure(config): if warnings: os.environ.setdefault('PYTHONWARNINGS', ' '.join(warnings)) + # use /var/tmp to avoid filling up the RAM with large build artifacts + os.environ.setdefault('TMPDIR', '/var/tmp') + def pytest_collection_modifyitems(config, items): if config.getoption('--runslow'): From 511b2ae38f3d804c07441204a1b273fe0598242a Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Mon, 20 Jul 2026 10:42:28 +0200 Subject: [PATCH 12/13] elbepack: tests: introduce build_driver abstraction for xml tests Split the test_xml to only contain code that is not directly related to the initvm. This will enable us in the next step to add further tests that will use the container-based approach. Signed-off-by: Florian Kauer --- elbepack/tests/test_xml.py | 160 +++++------------------------ elbepack/tests/test_xml_initvm.py | 161 ++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 137 deletions(-) create mode 100644 elbepack/tests/test_xml_initvm.py diff --git a/elbepack/tests/test_xml.py b/elbepack/tests/test_xml.py index c0818e8eb..2d47ea0ce 100644 --- a/elbepack/tests/test_xml.py +++ b/elbepack/tests/test_xml.py @@ -2,107 +2,41 @@ # SPDX-License-Identifier: GPL-3.0-or-later # SPDX-FileCopyrightText: 2020 Linutronix GmbH -import contextlib -import functools -import io + +# Only contains templates used by other test files +__test__ = False + import pathlib -import subprocess import pytest from elbepack.main import run_elbe_subcommand -from elbepack.tests import parametrize_xml_test_files, xml_test_files - - -here = pathlib.Path(__file__).parent - - -@pytest.fixture(scope='module') -def initvm(tmp_path_factory, request): - initvm_dir = tmp_path_factory.mktemp('initvm-') / 'initvm' - use_initvm = request.config.getoption('--elbe-use-initvm') - - if use_initvm in {'libvirt', 'existing'}: - qemu_arg = [] - elif use_initvm == 'qemu': - qemu_arg = ['--qemu'] - else: - raise ValueError(use_initvm) - - def initvm_func(subcmd, *args): - run_elbe_subcommand(['initvm', subcmd, '--directory', initvm_dir, *qemu_arg, *args]) - - def destroy_initvm(): - with contextlib.suppress(Exception): - initvm_func('stop') - with contextlib.suppress(Exception): - initvm_func('destroy') +from elbepack.tests import xml_test_files - if use_initvm == 'existing': - yield initvm_func - return - try: - initvm_func('create', '--fail-on-warning') - except Exception as e: - # If the fixture setup fails, pytest will try to create the fixture for - # each test. This is very slow and unlikely to work, so remember the failure. - def error_func(*args, _initvm_exception, **kwargs): - raise RuntimeError('initvm setup failed') from _initvm_exception - destroy_initvm() - yield functools.partial(error_func, _initvm_exception=e) - else: - try: - yield initvm_func - finally: - destroy_initvm() +CHECK_BUILD_VARIANTS = ('schema', 'cdrom', 'img', 'sdk') - -def _delete_project(uuid): - with contextlib.suppress(Exception): - run_elbe_subcommand(['control', 'del_project', uuid]) +_EXTENDED_XML = ( + pathlib.Path('tests') / 'base-extended' / 'simple-validation' / 'image-extended.xml' +) @pytest.fixture(scope='module', params=xml_test_files('simple'), ids=lambda f: f.name) -def simple_build(request, initvm, tmp_path_factory): - build_dir = tmp_path_factory.mktemp('build_dir') - prj = build_dir / 'uuid.prj' - - initvm( - 'submit', request.param, - '--output', build_dir, - '--keep-files', '--build-sdk', - '--writeproject', prj, - ) - - uuid = prj.read_text() - - with contextlib.redirect_stdout(io.StringIO()) as stdout: - run_elbe_subcommand(['control', 'list_projects']) - - if uuid not in stdout.getvalue(): - raise RuntimeError('Project was not created') - - yield build_dir - - _delete_project(uuid) +def simple_build(request, tmp_path_factory, build_driver): + workdir = tmp_path_factory.mktemp('build_dir') + return build_driver.submit(request, request.param, workdir, build_sdk=True) @pytest.mark.slow -@pytest.mark.parametrize('check_build', ('schema', 'cdrom', 'img', 'sdk')) +@pytest.mark.parametrize('check_build', CHECK_BUILD_VARIANTS) def test_simple_build(simple_build, check_build): run_elbe_subcommand(['check-build', check_build, simple_build]) @pytest.mark.slow -def test_rebuild(initvm, simple_build, tmp_path_factory): +def test_rebuild(build_driver, simple_build, tmp_path_factory): build_dir = tmp_path_factory.mktemp('build_dir') - - initvm( - 'submit', '--skip-build-source', - '--output', build_dir, - simple_build / 'bin-cdrom.iso', - ) + build_driver.rebuild(simple_build / 'bin-cdrom.iso', build_dir) @pytest.mark.slow @@ -110,65 +44,17 @@ def test_check_updates(simple_build): run_elbe_subcommand(['check_updates', simple_build / 'source.xml']) -def _prjrepo_list_packages(uuid): - with contextlib.redirect_stdout(io.StringIO()) as stdout: - run_elbe_subcommand(['prjrepo', 'list_packages', uuid]) - - return stdout.getvalue() - - -@pytest.mark.slow -@parametrize_xml_test_files('xml', 'pbuilder') -def test_pbuilder_build(initvm, xml, tmp_path, request): - build_dir = tmp_path - prj = build_dir / 'uuid.prj' - - run_elbe_subcommand(['pbuilder', 'create', '--xmlfile', xml, '--writeproject', prj]) - - uuid = prj.read_text() - request.addfinalizer(lambda: _delete_project(uuid)) - - # Not necessary, to test the command. - run_elbe_subcommand(['pbuilder', 'update', '--project', uuid]) - run_elbe_subcommand(['control', 'wait_busy', uuid]) - - assert _prjrepo_list_packages(uuid) == '' - - for package in ['libgpio', 'gpiotest']: - subprocess.run(['git', 'clone', f'https://github.com/Linutronix/{package}.git'], - check=True, cwd=build_dir) - run_elbe_subcommand(['pbuilder', 'build', '--project', uuid, - '--source', build_dir.joinpath(package), - '--output', build_dir.joinpath('out')]) - - assert _prjrepo_list_packages(uuid) == ( - 'gpiotest_1.0_amd64.deb\n' - 'libgpio-dev_3.0.1_amd64.deb\n' - 'libgpio3-dbgsym_3.0.1_amd64.deb\n' - 'libgpio3_3.0.1_amd64.deb\n' - ) - - run_elbe_subcommand(['prjrepo', 'upload_pkg', uuid, here / 'equivs-dummy_1.0_all.deb']) - - assert _prjrepo_list_packages(uuid) == ( - 'equivs-dummy_1.0_all.deb\n' - 'gpiotest_1.0_amd64.deb\n' - 'libgpio-dev_3.0.1_amd64.deb\n' - 'libgpio3-dbgsym_3.0.1_amd64.deb\n' - 'libgpio3_3.0.1_amd64.deb\n' - ) - - @pytest.mark.slow -def test_base_extended_build(simple_build, initvm, tmp_path): - tests_dir = pathlib.Path('tests') / 'base-extended' / 'simple-validation' - extended_xml_path = tests_dir / 'image-extended.xml' +def test_base_extended_build(request, build_driver, simple_build, tmp_path): base_build_image = simple_build / 'base-rootfs.tgz' - extended_build = tmp_path / 'extended-build' if not base_build_image.exists(): pytest.skip('No base image tarball was produced') - initvm('submit', '--output', extended_build, '--skip-build-bin', '--skip-build-sources', - '--base-image', base_build_image, extended_xml_path) - run_elbe_subcommand(['check-build', 'img', extended_build]) + extended_build = tmp_path / 'extended-build' + extended_build.mkdir() + build_dir = build_driver.submit( + request, _EXTENDED_XML, extended_build, + skip_cdrom=True, base_image=base_build_image, + ) + run_elbe_subcommand(['check-build', 'img', build_dir]) diff --git a/elbepack/tests/test_xml_initvm.py b/elbepack/tests/test_xml_initvm.py new file mode 100644 index 000000000..432d3d42a --- /dev/null +++ b/elbepack/tests/test_xml_initvm.py @@ -0,0 +1,161 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2020 Linutronix GmbH + +import contextlib +import functools +import io +import pathlib +import subprocess + +import pytest + +from elbepack.main import run_elbe_subcommand +from elbepack.tests import parametrize_xml_test_files +from elbepack.tests.test_xml import ( # noqa: F401 + simple_build, + test_base_extended_build, + test_check_updates, + test_rebuild, + test_simple_build, +) + + +here = pathlib.Path(__file__).parent + + +@pytest.fixture(scope='module') +def initvm(tmp_path_factory, request): + initvm_dir = tmp_path_factory.mktemp('initvm-') / 'initvm' + use_initvm = request.config.getoption('--elbe-use-initvm') + + if use_initvm in {'libvirt', 'existing'}: + qemu_arg = [] + elif use_initvm == 'qemu': + qemu_arg = ['--qemu'] + else: + raise ValueError(use_initvm) + + def initvm_func(subcmd, *args): + run_elbe_subcommand(['initvm', subcmd, '--directory', initvm_dir, *qemu_arg, *args]) + + def destroy_initvm(): + with contextlib.suppress(Exception): + initvm_func('stop') + with contextlib.suppress(Exception): + initvm_func('destroy') + + if use_initvm == 'existing': + yield initvm_func + return + + try: + initvm_func('create', '--fail-on-warning') + except Exception as e: + # If the fixture setup fails, pytest will try to create the fixture for + # each test. This is very slow and unlikely to work, so remember the failure. + def error_func(*args, _initvm_exception, **kwargs): + raise RuntimeError('initvm setup failed') from _initvm_exception + destroy_initvm() + yield functools.partial(error_func, _initvm_exception=e) + else: + try: + yield initvm_func + finally: + destroy_initvm() + + +def _delete_project(uuid): + with contextlib.suppress(Exception): + run_elbe_subcommand(['control', 'del_project', uuid]) + + +@pytest.fixture(scope='module') +def build_driver(initvm): + class _InitvmDriver: + def submit( + self, request, xml_file, build_dir, *, + build_sdk=False, skip_cdrom=False, base_image=None, + ): + prj = build_dir / 'uuid.prj' + + args = ['submit', xml_file, '--output', build_dir, + '--keep-files', '--writeproject', prj] + if build_sdk: + args.append('--build-sdk') + if skip_cdrom: + args += ['--skip-build-bin', '--skip-build-sources'] + if base_image: + args += ['--base-image', base_image] + + initvm(*args) + + uuid = prj.read_text() + + with contextlib.redirect_stdout(io.StringIO()) as stdout: + run_elbe_subcommand(['control', 'list_projects']) + + if uuid not in stdout.getvalue(): + raise RuntimeError('Project was not created') + + request.addfinalizer(lambda: _delete_project(uuid)) + + return build_dir + + def rebuild(self, iso_path, build_dir): + initvm( + 'submit', '--skip-build-source', + '--output', build_dir, + iso_path, + ) + + return _InitvmDriver() + + +def _prjrepo_list_packages(uuid): + with contextlib.redirect_stdout(io.StringIO()) as stdout: + run_elbe_subcommand(['prjrepo', 'list_packages', uuid]) + + return stdout.getvalue() + + +@pytest.mark.slow +@parametrize_xml_test_files('xml', 'pbuilder') +def test_pbuilder_build(initvm, xml, tmp_path, request): + build_dir = tmp_path + prj = build_dir / 'uuid.prj' + + run_elbe_subcommand(['pbuilder', 'create', '--xmlfile', xml, '--writeproject', prj]) + + uuid = prj.read_text() + request.addfinalizer(lambda: _delete_project(uuid)) + + # Not necessary, to test the command. + run_elbe_subcommand(['pbuilder', 'update', '--project', uuid]) + run_elbe_subcommand(['control', 'wait_busy', uuid]) + + assert _prjrepo_list_packages(uuid) == '' + + for package in ['libgpio', 'gpiotest']: + subprocess.run(['git', 'clone', f'https://github.com/Linutronix/{package}.git'], + check=True, cwd=build_dir) + run_elbe_subcommand(['pbuilder', 'build', '--project', uuid, + '--source', build_dir.joinpath(package), + '--output', build_dir.joinpath('out')]) + + assert _prjrepo_list_packages(uuid) == ( + 'gpiotest_1.0_amd64.deb\n' + 'libgpio-dev_3.0.1_amd64.deb\n' + 'libgpio3-dbgsym_3.0.1_amd64.deb\n' + 'libgpio3_3.0.1_amd64.deb\n' + ) + + run_elbe_subcommand(['prjrepo', 'upload_pkg', uuid, here / 'equivs-dummy_1.0_all.deb']) + + assert _prjrepo_list_packages(uuid) == ( + 'equivs-dummy_1.0_all.deb\n' + 'gpiotest_1.0_amd64.deb\n' + 'libgpio-dev_3.0.1_amd64.deb\n' + 'libgpio3-dbgsym_3.0.1_amd64.deb\n' + 'libgpio3_3.0.1_amd64.deb\n' + ) From 720036972ca1698d11842a5db845801b8d828ffd Mon Sep 17 00:00:00 2001 From: Florian Kauer Date: Mon, 20 Jul 2026 10:42:28 +0200 Subject: [PATCH 13/13] elbepack: tests: provide test for build without initvm Provide a test driver to run the same tests as for the initvm in test_xml.py for the container-based build as well. Signed-off-by: Florian Kauer --- elbepack/tests/test_xml_container.py | 150 +++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 elbepack/tests/test_xml_container.py diff --git a/elbepack/tests/test_xml_container.py b/elbepack/tests/test_xml_container.py new file mode 100644 index 000000000..22157fdf4 --- /dev/null +++ b/elbepack/tests/test_xml_container.py @@ -0,0 +1,150 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import os +import pathlib +import shutil +import subprocess + +import pytest + +from elbepack.buildsubmitaction import extract_cdrom +from elbepack.rootcheck import xml_needs_rootful +from elbepack.tests.test_xml import ( # noqa: F401 + simple_build, + test_base_extended_build, + test_check_updates, + test_rebuild, + test_simple_build, +) +from elbepack.treeutils import etree + +_IMAGE_NAME = 'elbe-buildenv-image' +_REPO_ROOT = pathlib.Path(__file__).parent.parent.parent +_CONTAINERFILE_DIR = _REPO_ROOT / 'contrib' / 'containerfile' +_TESTS_ROOT = _REPO_ROOT / 'tests' + + +@pytest.fixture(scope='module') +def elbe_buildenv_image(): + subprocess.run( + ['make', 'build-local', f'BUILD_DIR={_REPO_ROOT}'], + cwd=_CONTAINERFILE_DIR, check=True) + + return _IMAGE_NAME + + +def _get_build_container_opts(needs_rootful): + opts = [] + + if os.geteuid() == 0: + if needs_rootful: + opts.extend([ + '--cap-add', 'SYS_ADMIN', + '--cap-add', 'MKNOD', + '--device-cgroup-rule', 'b *:* rmw', + ]) + else: + opts.extend(['--userns', 'auto']) + + opts.extend(['--security-opt', 'apparmor=unconfined']) + + if shutil.which('slirp4netns'): + opts.extend(['--network', 'slirp4netns']) + else: + opts.extend(['--network', 'host']) + + return opts + + +def _run_build(elbe_buildenv_image, workdir, xml_name, build_args=(), base_image=None, + source_dir=None): + """Run ELBE build in container, skipping if loop devices are required but unavailable.""" + input_dir = workdir if source_dir is None else source_dir + xml_path = input_dir / xml_name + + if xml_name.endswith('.iso'): + # rebuild-from-iso: extract source.xml for the rootful pre-check, the same + # way elbe build itself does internally for iso rebuilds. + extracted = extract_cdrom(xml_path) + rootcheck_xml = extracted.fname('source.xml') + else: + rootcheck_xml = xml_path + + xml = etree(rootcheck_xml) + needs_rootful = bool(xml_needs_rootful(xml)) + + extra_opts = _get_build_container_opts(needs_rootful) + + if needs_rootful and os.geteuid() != 0: + pytest.skip( + 'XML file requires rootful container. Rerun as root (e.g. `sudo pytest ...`)' + ) + + mounts = [] + if source_dir is None: + xml_container_path = f'/work/{xml_name}' + else: + mounts += ['-v', f'{_TESTS_ROOT.resolve()}:/src:z,ro'] + unresolved_xml_path = xml_path.parent.resolve() / xml_path.name + xml_container_path = f'/src/{unresolved_xml_path.relative_to(_TESTS_ROOT.resolve())}' + + if xml_path.is_symlink(): + mounts += ['-v', f'{xml_path.resolve()}:{xml_container_path}:z,ro'] + + build_cmd = [ + 'podman', 'run', '--rm', + *mounts, + '-v', f'{workdir}:/work:Z,U', + *extra_opts, + elbe_buildenv_image, + 'elbe', 'build', xml_container_path, '--build-dir', '/work/build', + *build_args, + ] + + if base_image: + build_cmd.extend(['--base-image', f'/work/{base_image}']) + + result = subprocess.run(build_cmd, check=False) + + if result.returncode != 0: + pytest.fail(f'ELBE build failed for {xml_name}. See {workdir} for build state') + + build_dir = workdir / 'build' + assert (build_dir / 'source.xml').exists() + assert (build_dir / 'validation.txt').exists() + return build_dir + + +@pytest.fixture(scope='module') +def build_driver(elbe_buildenv_image): + class _ContainerDriver: + def submit( + self, request, xml_file, build_dir, *, + build_sdk=False, skip_cdrom=False, base_image=None, + ): + build_args = [] + if skip_cdrom: + build_args += ['--skip-build-bin', '--skip-build-sources'] + if build_sdk: + build_args.append('--build-sdk') + + base_image_name = None + if base_image: + build_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(base_image, build_dir) + base_image_name = base_image.name + + return _run_build( + elbe_buildenv_image, build_dir, xml_file.name, + build_args=build_args, base_image=base_image_name, + source_dir=xml_file.parent, + ) + + def rebuild(self, iso_path, build_dir): + shutil.copy(iso_path, build_dir) + _run_build(elbe_buildenv_image, build_dir, iso_path.name, + build_args=['--skip-build-sources']) + + return _ContainerDriver()