diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..69e73aa36 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.mypy_cache +.pytest_cache +*.deb +*.buildinfo +*.changes diff --git a/conftest.py b/conftest.py index a1f8204ff..9182d85c0 100644 --- a/conftest.py +++ b/conftest.py @@ -35,6 +35,9 @@ def pytest_configure(config): if warnings: os.environ.setdefault('PYTHONWARNINGS', ' '.join(warnings)) + # use /var/tmp to avoid filling up the RAM with large build artifacts + os.environ.setdefault('TMPDIR', '/var/tmp') + def pytest_collection_modifyitems(config, items): if config.getoption('--runslow'): diff --git a/contrib/containerfile/Containerfile b/contrib/containerfile/Containerfile new file mode 100644 index 000000000..26543da71 --- /dev/null +++ b/contrib/containerfile/Containerfile @@ -0,0 +1,38 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +FROM debian:trixie-slim + +USER root +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + elbe-archive-keyring \ + tini + +RUN echo 'deb [signed-by=/usr/share/keyrings/elbe-archive-keyring.gpg] http://debian.linutronix.de/elbe bullseye main' \ + > /etc/apt/sources.list.d/elbe.list && \ + apt-get update -y && \ + apt-get install -y --no-install-recommends \ + python3-elbe-daemon \ + python3-elbe-soap \ + python3-elbe-control \ + locales && \ + apt-get clean -y + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen + +VOLUME [ "/build" ] + +USER root +WORKDIR /build + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Use tini for reaping child processes. +ENTRYPOINT ["tini", "--", "/entrypoint.sh"] diff --git a/contrib/containerfile/Containerfile.in b/contrib/containerfile/Containerfile.in deleted file mode 100644 index 9aa586b24..000000000 --- a/contrib/containerfile/Containerfile.in +++ /dev/null @@ -1,67 +0,0 @@ -# ELBE - Debian Based Embedded Rootfilesystem Builder -# SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2014-2015 Silvio Fricke -# SPDX-FileCopyrightText: 2018 Linutronix GmbH - -# This Containerfile generates an image for the elbe buildsystem -FROM debian:trixie-slim - -USER root -ENV DEBIAN_FRONTEND noninteractive - -# update, upgrade and install elbe runtime-dependencies -RUN apt-get update -y -RUN apt-get install -y --no-install-recommends \ - -o Dpkg::Options::="--force-confnew" \ - ca-certificates \ - sudo \ - vim-nox \ - elbe-archive-keyring \ - gnupg \ - python3-setuptools \ - python3-yaml \ - python3-jsonschema \ - locales \ - gcc \ - g++ \ - diffstat \ - texinfo \ - gawk \ - chrpath \ - python3-mako \ - fuseiso9660 \ - aptly \ - debian-archive-keyring \ - qemu-system-x86 - -RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen - -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -# install current elbe -RUN echo 'deb [signed-by=/usr/share/keyrings/elbe-archive-keyring.gpg] http://debian.linutronix.de/elbe bullseye main' >> /etc/apt/sources.list -RUN apt-get update -y -RUN apt-get install -y --no-install-recommends \ - elbe \ - elbe-doc -RUN apt-get clean -y -RUN rm -rf /var/lib/apt/lists/* - -# create elbe user -RUN groupadd -g @KVMGID@ -o -r kvm-elbe -RUN useradd -d /home/elbe -l -U -G kvm-elbe,libvirt -m -s /bin/bash -o -u @USERID@ elbe -RUN echo "root:elbe" | chpasswd -RUN echo "elbe:elbe" | chpasswd - -VOLUME [ "/sys/fs/cgroup" ] -VOLUME [ "/elbe" ] -VOLUME [ "/var/cache/elbe" ] - -# sudo for elbe -RUN echo "%elbe ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/elbegrp -RUN chmod 0440 /etc/sudoers.d/elbegrp - - -CMD [ "/bin/bash" ] diff --git a/contrib/containerfile/Containerfile.local b/contrib/containerfile/Containerfile.local new file mode 100644 index 000000000..245aba386 --- /dev/null +++ b/contrib/containerfile/Containerfile.local @@ -0,0 +1,59 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +FROM debian:trixie-slim AS builder + +USER root +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates + +WORKDIR /usr/src/elbe +COPY . . + +RUN apt-get update -y && \ + apt-get build-dep -y . && \ + DEB_BUILD_OPTIONS=nocheck sphinxflags= dpkg-buildpackage -us -uc -b && \ + mkdir -p /out && mv ../*.deb /out/ + +FROM debian:trixie-slim + +USER root +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + tini + +COPY --from=builder /out/*.deb /tmp/local-debs/ +RUN apt-get update -y && apt-get upgrade -y + +RUN apt-get install -y --no-install-recommends \ + locales \ + /tmp/local-debs/elbe-schema_*.deb \ + /tmp/local-debs/python3-elbe-common_*.deb \ + /tmp/local-debs/python3-elbe-bin_*.deb \ + /tmp/local-debs/python3-elbe-buildenv_*.deb \ + /tmp/local-debs/python3-elbe-daemon_*.deb \ + /tmp/local-debs/python3-elbe-soap_*.deb \ + /tmp/local-debs/python3-elbe-control_*.deb && \ + rm -rf /tmp/local-debs && \ + apt-get clean -y + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen + +VOLUME [ "/build" ] + +USER root +WORKDIR /build + +COPY contrib/containerfile/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Use tini for reaping child processes. +ENTRYPOINT ["tini", "--", "/entrypoint.sh"] diff --git a/contrib/containerfile/Makefile b/contrib/containerfile/Makefile index 3a9f6a934..735d006ba 100644 --- a/contrib/containerfile/Makefile +++ b/contrib/containerfile/Makefile @@ -1,12 +1,9 @@ # ELBE - Debian Based Embedded Rootfilesystem Builder # SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2015 Silvio Fricke -# SPDX-FileCopyrightText: 2018 Linutronix GmbH +# SPDX-FileCopyrightText: 2026 Linutronix GmbH -IMAGENAME ?= elbe-devel-image -CONTAINERNAME ?= elbe-devel -KVMGID ?= $(shell ls -n /dev/kvm | awk '{ print $$4 }') -UID ?= $(shell id -u) +IMAGENAME ?= elbe-buildenv-image +CONTAINERNAME ?= elbe-buildenv # Container engine to use. Defaults to podman if installed, else docker. ENGINE ?= $(shell command -v podman >/dev/null 2>&1 && echo podman || echo docker) @@ -14,11 +11,6 @@ ENGINE ?= $(shell command -v podman >/dev/null 2>&1 && echo podman || echo docke # Common container run options RUN_OPTS = \ --cap-add SYS_ADMIN \ - --security-opt seccomp=unconfined \ - --security-opt apparmor=unconfined \ - --group-add kvm \ - --device /dev/kvm \ - --device /dev/fuse \ --rm \ --interactive \ --tty @@ -32,51 +24,20 @@ ifeq ($(ENGINE),podman) RUN_OPTS += --userns=keep-id endif -# container commands build: - test -c /dev/kvm || ( echo "/dev/kvm not found" && false ) - test -c /dev/fuse || ( echo "/dev/fuse not found" && false ) - test -n "$(KVMGID)" || ( echo "detecting groupid of /dev/kvm failed" && false ) - sed -e "s#@KVMGID@#$(KVMGID)#g" \ - -e "s#@USERID@#$(UID)#g" \ - Containerfile.in > Containerfile $(ENGINE) build --build-arg http_proxy=$(http_proxy) \ --build-arg https_proxy=$(https_proxy) \ --build-arg no_proxy=$(no_proxy) \ - --no-cache \ - -f Containerfile \ + --network=slirp4netns \ -t $(IMAGENAME) . - rm Containerfile -start-devel: - $(ENGINE) ps | grep $(CONTAINERNAME)$$ || \ - $(ENGINE) run --name $(CONTAINERNAME) \ - -e http_proxy=$(http_proxy) \ - -e https_proxy=$(https_proxy) \ - -e no_proxy=$(no_proxy) \ - -v $(realpath ../../.):/var/cache/elbe/devel -w /var/cache/elbe/devel \ - $(RUN_OPTS) \ - $(IMAGENAME) - -start: - $(ENGINE) ps | grep $(CONTAINERNAME)$$ || \ - $(ENGINE) run --name $(CONTAINERNAME) \ - -e http_proxy=$(http_proxy) \ - -e https_proxy=$(https_proxy) \ - -e no_proxy=$(no_proxy) \ - $(RUN_OPTS) \ - $(IMAGENAME) - -stop: - -$(ENGINE) stop $(CONTAINERNAME) - -stoprm: stop - -$(ENGINE) rm $(CONTAINERNAME) - -clean: stoprm - -$(ENGINE) rmi $(IMAGENAME) - -connect: start - $(ENGINE) exec -tiu $(UID) $(CONTAINERNAME) /bin/bash +# Builds the elbe .deb packages from this checkout first +build-local: + $(ENGINE) build --build-arg http_proxy=$(http_proxy) \ + --build-arg https_proxy=$(https_proxy) \ + --build-arg no_proxy=$(no_proxy) \ + --network=slirp4netns \ + -f Containerfile.local \ + -t $(IMAGENAME) ../.. -.PHONY: build start stop stoprm clean connect +.PHONY: build build-local diff --git a/contrib/containerfile/README.md b/contrib/containerfile/README.md deleted file mode 100644 index 0a002edb7..000000000 --- a/contrib/containerfile/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Dockerfile for elbe - - -# SPDX-FileCopyrightText: 2018 Linutronix GmbH -?> - -[elbe][elb] is a debian based system to generate root-filesystems for embedded -devices. - -[docker][doc] and [podman][pod] are open-source projects to easily create -lightweight, portable, self-sufficient containers from any application. - -This is a Containerfile to generate an ELBE development and runtime environment for -systems that are not Debian based or where you prefer a clean separation. - -[doc]: https://www.docker.io "Docker Homepage" -[pod]: https://podman.io "Podman Homepage" -[elb]: http://elbe-rfs.org "ELBE Homepage" - -## Dependencies - -You need `podman` or `docker` installed, plus `make`. `podman` is used by -default if it is found on the `PATH`; set `ENGINE=docker` to force Docker -instead (e.g. `make ENGINE=docker build`). - -## usage - -A `Makefile` with some handy targets are provided. Per default the image name -is `elbe-devel-image` and a started container name is `elbe-devel`. This names are -changeable via `IMAGENAME` and `CONTAINERNAME` environment variables. - -* `build`: build the image -* `start` start a container, and use packaged elbe -* `start-devel` start a container, mounts the elbe git-archive to `/var/cache/elbe` -* `stop`: stop a running container -* `stoprm`: stop and remove the container -* `connect`: attach a new terminal to a running container - -After `connect` you can find the elbe git repository under `/elbe`. - -## passwords - - root: elbe - elbe: elbe - diff --git a/contrib/containerfile/entrypoint.sh b/contrib/containerfile/entrypoint.sh new file mode 100755 index 000000000..b46e23787 --- /dev/null +++ b/contrib/containerfile/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH +set -e + +# Check if we already have CAP_SYS_ADMIN +# In that case we assume we are already running rootful, +# so no need for unshare, but this mode also enables usage +# of loop devices, so perform mknod to enable its usage. +capeff=$(grep '^CapEff:' /proc/self/status | cut -f2) +if [ $(( 0x$capeff & 0x200000 )) -ne 0 ]; then + i=0 + while [ "$i" -lt 64 ]; do + [ -e "/dev/loop$i" ] || mknod -m 660 "/dev/loop$i" b 7 "$i" 2>/dev/null || true + i=$((i + 1)) + done + exec "$@" +fi + +exec unshare --user --map-root-user --map-users=all --map-groups=all --mount -- "$@" diff --git a/debian/python3-elbe-common.install b/debian/python3-elbe-common.install index 13dbe9b86..c1ac2dd76 100644 --- a/debian/python3-elbe-common.install +++ b/debian/python3-elbe-common.install @@ -30,7 +30,9 @@ usr/lib/python3.*/*-packages/elbepack/initvm.py usr/lib/python3.*/*-packages/elbepack/initvmaction.py usr/lib/python3.*/*-packages/elbepack/isooptions.py usr/lib/python3.*/*-packages/elbepack/licencexml.py +usr/lib/python3.*/*-packages/elbepack/localbuildaction.py usr/lib/python3.*/*-packages/elbepack/log.py +usr/lib/python3.*/*-packages/elbepack/rootcheck.py usr/lib/python3.*/*-packages/elbepack/packers.py usr/lib/python3.*/*-packages/elbepack/paths.py usr/lib/python3.*/*-packages/elbepack/pkgutils.py diff --git a/debian/python3-elbe-control.install b/debian/python3-elbe-control.install index 397cdccab..6c858c1ef 100644 --- a/debian/python3-elbe-control.install +++ b/debian/python3-elbe-control.install @@ -1,3 +1,4 @@ +usr/lib/python3.*/*-packages/elbepack/commands/build.py usr/lib/python3.*/*-packages/elbepack/commands/control.py usr/lib/python3.*/*-packages/elbepack/commands/initvm.py usr/lib/python3.*/*-packages/elbepack/commands/pbuilder.py diff --git a/docs/elbe-build.rst b/docs/elbe-build.rst new file mode 100644 index 000000000..22a554aad --- /dev/null +++ b/docs/elbe-build.rst @@ -0,0 +1,177 @@ +************************ +elbe-build +************************ + +NAME +==== + +elbe-build - Build a root filesystem from an ELBE XML file, without +requiring an initvm. + +SYNOPSIS +======== + + :: + + elbe build [options] | + +DESCRIPTION +=========== + +This command builds an ELBE project directly, without encapsulating the +build into an initvm and without any daemon or SOAP communication. It +runs the whole build in-process, and is meant to be used inside a +container (or any other environment) that already provides the +isolation an initvm would otherwise provide. It is therefore the +VM-free alternative to *elbe initvm submit*. + +Since it is meant for container builds where there is no initvm, +packages needed only for the initvm are always excluded from the +generated CDROMs. + +OPTIONS +======= + +--skip-download + After the build has finished, the generated files are normally + copied out of the project directory to *--build-dir*. This step is + skipped, when this option is specified. + +--build-dir + Directory name where the generated and downloaded files should be + saved and where the internal build cache is kept. The default is to + generate a directory with a timestamp in the current working directory. + +--skip-build-bin + Skip building binary repository CDROM, for exact reproduction. + +--skip-build-sources + Skip building source CDROM. + +--keep-files + Don’t delete elbe project files after a build. The project directory + is printed during the build. + +--writeproject + Write project name to . + +--build-sdk + Also build an SDK. + +--base-image + Use a base image instead of debootstrap as the starting point for a rootfilesystem (experimental). + +XML OPTIONS +=========== + +These options are passed through to an implicit invocation of +*elbe preprocess*, which is run on the given xmlfile before the build. + +-v , --variants + comma separated list of variants; enable only tags with empty or + given variant. + +-p , --proxy + add proxy to mirrors + +Examples +======== + +*elbe build* is meant to be run inside a container. The example +container definition in *contrib/containerfile* provides a +ready-to-use build environment for this. + +- Build the container image, installing elbe from the published elbe + archive: + + :: + + $ cd contrib/containerfile + $ make build + +- Alternatively, build the container image with ELBE packages + built from the current checkout, instead of the published ones: + + :: + + $ cd contrib/containerfile + $ make build-local + +- Run the build in a container, with the current directory + mounted as */build*. The container is removed again once the build + finishes: + + :: + + $ podman run --rm \ + -v $(pwd):/work:Z \ + elbe-buildenv-image \ + elbe build --skip-build-bin --skip-build-sources \ + /work/tests/base-extended/simple-validation/image-base-trixie.xml \ + --build-dir /work/build + + +Rootful Containers +================== + +Some project XML features make *elbe build* create a Linux loop device +to loop-mount a disk or partition image: + +- ```` inside a ````/```` target image +- ```` (or the ````) inside a + partition's ```` +- a ```` project-finetuning action containing + ``copy_from_partition``, ``copy_to_partition``, or ``command`` + +Creating a loop device requires access to */dev/loop-control*, which +is a host-kernel-wide privilege. Additionally, ``CAP_SYS_ADMIN`` must be +enabled in the **initial user namespace** (not in a namespace created +via ``unshare``), which means **rootless containers cannot create loop +devices**. + +Also, some XMLs might require mknod which is also not possible +in rootless containers. + +If the project XML uses one of the features above, *elbe build* +checks the requirements upfront and aborts immediately. + +If your XML needs these features, either: + +- build via *elbe initvm submit* instead, which runs inside a full + virtual machine with real root privileges, or + +- run the container with elevated privileges (i.e. no rootless container). + Add the following to the *podman run*/*docker run* invocation and run with + elevated privileges (e.g. via ``pkexec`` (PolicyKit)). + + :: + + --cap-add SYS_ADMIN --cap-add MKNOD --device-cgroup-rule='b *:* rmw' \ + --security-opt apparmor=unconfined + + Note that since this is now a rootful container and it there are much more + options for security vulnerabilities to manifest. Also, the networking might + be differently set up, so you might need to add ``--network slirp4netns`` + (or ``--network host`` if ``slirp4netns`` is not installed). + + All together, the command would look like this: + + :: + + pkexec podman run --rm \ + -v $(pwd):/work:Z \ + --cap-add SYS_ADMIN --cap-add MKNOD --device-cgroup-rule='b *:* rmw' \ + --security-opt apparmor=unconfined \ + --network slirp4netns \ + elbe-buildenv-image \ + elbe build /work/myimage.xml --build-dir /work/build + +SEE ALSO +======== + +``elbe-initvm(1)``, ``elbe-preprocess(1)`` + +ELBE +==== + +Part of the ``elbe(1)`` suite diff --git a/docs/index.rst b/docs/index.rst index 9d552da95..4e1e59294 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ If you are new to ELBE, we recommend starting with the elbe elbe-add + elbe-build elbe-check_updates elbe-cyclonedx-sbom elbe-initvm diff --git a/elbepack/asyncworker.py b/elbepack/asyncworker.py index a2b3daff7..7f62ecd86 100644 --- a/elbepack/asyncworker.py +++ b/elbepack/asyncworker.py @@ -58,6 +58,9 @@ def execute(self, db): class BuildSDKJob(AsyncWorkerJob): + def __init__(self, project, exclude_initvm_pkgs=False): + super().__init__(project) + self.exclude_initvm_pkgs = exclude_initvm_pkgs def enqueue(self, queue, db): db.set_busy(self.project.builddir, @@ -70,7 +73,7 @@ def execute(self, db): success = self.build_failed try: logging.info('Build SDK started') - self.project.build_sdk() + self.project.build_sdk(exclude_initvm_pkgs=self.exclude_initvm_pkgs) except Exception: logging.exception('Build SDK Failed') else: @@ -143,12 +146,14 @@ def execute(self, db): class BuildJob(AsyncWorkerJob): - def __init__(self, project, build_bin, build_src, skip_pbuilder, base_image_path): + def __init__(self, project, build_bin, build_src, skip_pbuilder, + base_image_path, exclude_initvm_pkgs=False): super().__init__(project) self.build_bin = build_bin self.build_src = build_src self.skip_pbuilder = skip_pbuilder self.base_image_path = base_image_path + self.exclude_initvm_pkgs = exclude_initvm_pkgs def enqueue(self, queue, db): db.set_busy(self.project.builddir, @@ -166,7 +171,8 @@ def execute(self, db): build_bin=self.build_bin, build_sources=self.build_src, skip_pbuild=self.skip_pbuilder, - base_image_path=self.base_image_path) + base_image_path=self.base_image_path, + exclude_initvm_pkgs=self.exclude_initvm_pkgs) except (DebootstrapException, AptCacheCommitError, AptCacheUpdateError) as e: if isinstance(e, DebootstrapException): err = 'Debootstrap failed to install the base rootfilesystem.' diff --git a/elbepack/buildsubmitaction.py b/elbepack/buildsubmitaction.py index dab925cd0..bb627c0d1 100644 --- a/elbepack/buildsubmitaction.py +++ b/elbepack/buildsubmitaction.py @@ -50,15 +50,17 @@ def extract_cdrom(cdrom): return tmp +def add_output_argument(f): + return add_argument('--output', dest='outdir', + type=os.path.abspath, + help='directory where to save downloaded Files')(f) + + def add_submit_arguments(f): f = add_argument('--skip-download', action='store_true', dest='skip_download', default=False, help='Skip downloading generated Files')(f) - f = add_argument('--output', dest='outdir', - type=os.path.abspath, - help='directory where to save downloaded Files')(f) - f = add_argument('--skip-build-bin', dest='build_bin', action='store_false', default=True, help='Skip building Binary Repository CDROM, for exact Reproduction')(f) diff --git a/elbepack/cdroms.py b/elbepack/cdroms.py index 4544fc9ca..1776fac62 100644 --- a/elbepack/cdroms.py +++ b/elbepack/cdroms.py @@ -14,7 +14,8 @@ from elbepack.archivedir import archive_tmpfile from elbepack.isooptions import get_iso_options from elbepack.paths import ( - BINARIES_ADDED_DIR, BINARIES_MAIN_DIR, INITVM_BIN_REPO_DIR, INSTALLER_DIR, SOURCES_DIR, + BINARIES_ADDED_DIR, BINARIES_MAIN_DIR, INITVM_BIN_REPO_DIR, INITVM_GNUPG_HOME, + INSTALLER_DIR, SOURCES_DIR, ) from elbepack.repomanager import CdromBinRepo, CdromInitRepo, CdromSrcRepo from elbepack.rpcaptcache import get_rpcaptcache @@ -42,7 +43,8 @@ def add_source_pkg(repo, component, cache, pkg, version, forbid): def mk_source_cdrom(components, codename, init_codename, target, cdrom_size=CDROM_SIZE, xml=None, - mirror='http://deb.debian.org/debian'): + mirror='http://deb.debian.org/debian', + exclude_initvm_pkgs=False): os.makedirs(SOURCES_DIR, exist_ok=True) make_writable_by_apt(SOURCES_DIR) @@ -56,6 +58,8 @@ def mk_source_cdrom(components, codename, except KeyError: pass + gnupg_home = os.path.join(target, 'gnupg') + repos = {} for component in components.keys(): @@ -71,7 +75,7 @@ def mk_source_cdrom(components, codename, make_writable_by_apt(rfs.fname(SOURCES_DIR), passwd_root=rfs) repo = CdromSrcRepo(codename, init_codename, os.path.join(target, f'srcrepo-{component}'), - cdrom_size, mirror) + cdrom_size, gnupg_home, mirror) repos[component] = repo for pkg, version in pkg_lst: add_source_pkg(repo, component, @@ -94,12 +98,15 @@ def mk_source_cdrom(components, codename, # with the bin repo, because the src cdrom can be split # into multiple cdroms - for dirpath, _, filenames in os.walk(SOURCES_DIR): - for filename in filenames: - if not filename.endswith('.dsc'): - continue + if not exclude_initvm_pkgs: + for dirpath, _, filenames in os.walk(SOURCES_DIR): + for filename in filenames: + if not filename.endswith('.dsc'): + continue - repos['main'].include_init_dsc(os.path.join(dirpath, filename), 'initvm') + repos['main'].include_init_dsc(os.path.join(dirpath, filename), 'initvm') + else: + logging.info('Skipping initvm source packages as requested by --exclude-initvm-pkgs') for repo in repos.values(): repo.finalize() @@ -132,7 +139,7 @@ def mk_source_cdrom(components, codename, options=options)) for component, repo in repos.items()] -def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target): +def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target, exclude_initvm_pkgs=False): rfs.mkdir_p(BINARIES_ADDED_DIR) make_writable_by_apt(rfs.fname(BINARIES_ADDED_DIR), passwd_root=rfs) @@ -144,30 +151,36 @@ def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target): else: mirror = 'http://deb.debian.org/debian' + gnupg_home = os.path.join(target, 'gnupg') + repo_path = pathlib.Path(target, 'binrepo') target_repo_path = repo_path / 'targetrepo' # initvm repo has been built upon initvm creation # just copy it. the repo __init__() afterwards will # not touch the repo config, nor generate a new key. - try: - do(f'cp -av {INITVM_BIN_REPO_DIR} "{repo_path}"') - except subprocess.CalledProcessError: - # When INITVM_BIN_REPO_DIR has not been created - # (because the initvm install was an old version or somthing, - # log an error, and continue with an empty directory. - logging.exception('%s does not exist\n' - 'The generated CDROM will not contain initvm pkgs\n' - 'This happened because the initvm was probably\n' - 'generated with --skip-build-bin', - INITVM_BIN_REPO_DIR) - + if exclude_initvm_pkgs: + logging.info('Skipping initvm packages as requested by --exclude-initvm-pkgs') do(f'mkdir -p "{repo_path}"') + else: + try: + do(f'cp -av {INITVM_BIN_REPO_DIR} "{repo_path}"') + except subprocess.CalledProcessError: + # When INITVM_BIN_REPO_DIR has not been created + # (because the initvm install was an old version or somthing, + # log an error, and continue with an empty directory. + logging.exception('%s does not exist\n' + 'The generated CDROM will not contain initvm pkgs\n' + 'This happened because the initvm was probably\n' + 'generated with --skip-build-bin', + INITVM_BIN_REPO_DIR) + + do(f'mkdir -p "{repo_path}"') - repo = CdromInitRepo(init_codename, repo_path, mirror) + repo = CdromInitRepo(init_codename, repo_path, INITVM_GNUPG_HOME, mirror) target_repo = CdromBinRepo(arch, codename, None, - target_repo_path, mirror) + target_repo_path, gnupg_home, mirror) if xml is not None: cache = get_rpcaptcache(rfs, arch) @@ -222,10 +235,11 @@ def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target): xml.xml.write(repo_path / 'source.xml') # copy initvm-cdrom.gz and vmlinuz - copyfile(os.path.join(INSTALLER_DIR, 'initrd-cdrom.gz'), - repo_path / 'initrd-cdrom.gz') - copyfile(os.path.join(INSTALLER_DIR, 'vmlinuz'), - repo_path / 'vmlinuz') + if not exclude_initvm_pkgs: + copyfile(os.path.join(INSTALLER_DIR, 'initrd-cdrom.gz'), + repo_path / 'initrd-cdrom.gz') + copyfile(os.path.join(INSTALLER_DIR, 'vmlinuz'), + repo_path / 'vmlinuz') target_repo_path.joinpath('.aptignr').touch() diff --git a/elbepack/commands/build.py b/elbepack/commands/build.py new file mode 100644 index 000000000..b72ca4d4c --- /dev/null +++ b/elbepack/commands/build.py @@ -0,0 +1,49 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import argparse +import datetime +import os + +from elbepack.buildsubmitaction import add_submit_arguments, extract_cdrom +from elbepack.cli import add_argument, add_arguments_from_decorated_function +from elbepack.commands.preprocess import add_xmlpreprocess_passthrough_arguments +from elbepack.localbuildaction import local_build_with_repodir_and_dl_result + + +@add_submit_arguments +@add_argument( + '--build-dir', dest='build_dir', type=os.path.abspath, + help='directory where to save output files and the internal build cache ' + '(default is a timestamped directory in the current working directory)') +@add_argument('input', metavar=' | ') +def _build(args): + args.exclude_initvm_pkgs = True + + if args.build_dir is None: + args.build_dir = os.path.abspath( + 'elbe-build-' + datetime.datetime.now().strftime('%Y%m%d-%H%M%S')) + + cdrom = None + xmlfile = args.input + if xmlfile.endswith('.iso'): + tmp = extract_cdrom(xmlfile) + cdrom = xmlfile + xmlfile = tmp.fname('source.xml') + elif not xmlfile.endswith('.xml'): + args.parser.error('Unknown file ending (use either xml or iso)') + + local_build_with_repodir_and_dl_result(xmlfile, cdrom, args.base_image, args) + + +def run_command(argv): + aparser = argparse.ArgumentParser(prog='elbe build') + + add_xmlpreprocess_passthrough_arguments(aparser) + add_arguments_from_decorated_function(aparser, _build) + + args = aparser.parse_args(argv) + args.parser = aparser + + _build(args) diff --git a/elbepack/commands/fetch_initvm_pkgs.py b/elbepack/commands/fetch_initvm_pkgs.py index 0e7a08daa..17a367f59 100644 --- a/elbepack/commands/fetch_initvm_pkgs.py +++ b/elbepack/commands/fetch_initvm_pkgs.py @@ -15,13 +15,14 @@ from elbepack.aptpkgutils import fetch_source, get_corresponding_source_packages from elbepack.aptprogress import ElbeAcquireProgress from elbepack.dump import get_initvm_pkglist +from elbepack.egpg import INITVM_GNUPG_HOME from elbepack.elbexml import ElbeXML, ValidationError -from elbepack.imgutils import mount from elbepack.log import elbe_logging from elbepack.paths import ( BINARIES_MAIN_DIR, INITVM_BIN_REPO_DIR, INITVM_SRC_REPO_DIR, SOURCES_DIR, ) from elbepack.repomanager import CdromInitRepo, CdromSrcRepo +from elbepack.shellhelper import mount def run_command(argv): @@ -90,7 +91,7 @@ def run_command(argv): # Binary Repo # - repo = CdromInitRepo(init_codename, args.binrepo, mirror) + repo = CdromInitRepo(init_codename, args.binrepo, INITVM_GNUPG_HOME, mirror) os.makedirs(args.archive, exist_ok=True) @@ -131,7 +132,8 @@ def run_command(argv): # Source Repo # - repo = CdromSrcRepo(init_codename, init_codename, args.srcrepo, 0, mirror) + repo = CdromSrcRepo(init_codename, init_codename, args.srcrepo, 0, + INITVM_GNUPG_HOME, mirror) os.makedirs(args.srcarchive, exist_ok=True) # a cdrom build does not have sources diff --git a/elbepack/commands/remove_sign.py b/elbepack/commands/remove_sign.py index c582d351e..b634c1830 100644 --- a/elbepack/commands/remove_sign.py +++ b/elbepack/commands/remove_sign.py @@ -4,7 +4,7 @@ import argparse -from elbepack.egpg import unsign_file +from elbepack.egpg import INITVM_GNUPG_HOME, unsign_file def run_command(argv): @@ -13,7 +13,7 @@ def run_command(argv): args = parser.parse_args(argv) - fname = unsign_file(args.file) + fname = unsign_file(args.file, INITVM_GNUPG_HOME) if fname: print(f'unsigned file: {fname}') else: diff --git a/elbepack/commands/sign.py b/elbepack/commands/sign.py index 47e89e560..68bad6638 100644 --- a/elbepack/commands/sign.py +++ b/elbepack/commands/sign.py @@ -4,7 +4,7 @@ import argparse -from elbepack.egpg import sign_file +from elbepack.egpg import INITVM_GNUPG_HOME, sign_file def run_command(argv): @@ -14,4 +14,4 @@ def run_command(argv): args = parser.parse_args(argv) - sign_file(args.file, args.fingerprint) + sign_file(args.file, args.fingerprint, INITVM_GNUPG_HOME) diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py index 165dea8f5..a1108f23e 100644 --- a/elbepack/daemons/soap/esoap.py +++ b/elbepack/daemons/soap/esoap.py @@ -162,10 +162,13 @@ def build_sdk(self, builddir): def build_cdroms(self, builddir, build_bin, build_src): self.app.pm.build_cdroms(builddir, build_bin, build_src) - @rpc(String, Boolean, Boolean, Boolean, String) - def build(self, builddir, build_bin, build_src, skip_pbuilder, base_image_path): + @rpc(String, Boolean, Boolean, Boolean, String, Boolean) + def build(self, builddir, build_bin, build_src, skip_pbuilder, + base_image_path, exclude_initvm_pkgs): - self.app.pm.build_project(builddir, build_bin, build_src, skip_pbuilder, base_image_path) + self.app.pm.build_project(builddir, build_bin, build_src, + skip_pbuilder, base_image_path, + exclude_initvm_pkgs) @rpc(String, Boolean, Boolean, String) def build_pbuilder(self, builddir, cross, noccache, ccachesize): diff --git a/elbepack/dump.py b/elbepack/dump.py index 2143ec2f9..71cef272c 100644 --- a/elbepack/dump.py +++ b/elbepack/dump.py @@ -183,7 +183,7 @@ def check_full_pkgs(pkgs, fullpkgs, cache): validation.info('No Errors found') -def elbe_report(xml, buildenv, cache, targetfs): +def elbe_report(xml, buildenv, cache, targetfs, builddir): rfs = buildenv.rfs @@ -244,7 +244,7 @@ def elbe_report(xml, buildenv, cache, targetfs): mt_index_postarch = mt_index if xml.has('target/finetuning'): - do_finetuning(xml, buildenv, targetfs) + do_finetuning(xml, buildenv, targetfs, builddir) mt_index_post_fine = targetfs.mtime_snap() else: mt_index_post_fine = mt_index_postarch diff --git a/elbepack/efilesystem.py b/elbepack/efilesystem.py index 653a2c2d4..d4456e415 100644 --- a/elbepack/efilesystem.py +++ b/elbepack/efilesystem.py @@ -15,10 +15,9 @@ from elbepack.filesystem import Filesystem from elbepack.fstab import fstabentry -from elbepack.imgutils import mount from elbepack.licencexml import copyright_xml from elbepack.packers import default_packer -from elbepack.shellhelper import chroot, do +from elbepack.shellhelper import bind_mount_pseudo_filesystems, chroot, do from elbepack.version import elbe_version @@ -369,16 +368,6 @@ def __enter__(self): for excursion in excursions ] - if self.path != '/': - self._exitstack.enter_context( - mount(None, self.fname('/proc'), type='proc', log_output=False)) - self._exitstack.enter_context( - mount(None, self.fname('/sys'), type='sysfs', log_output=False)) - self._exitstack.enter_context( - mount('/dev', self.fname('/dev'), bind=True, log_output=False)) - self._exitstack.enter_context( - mount('/dev/pts', self.fname('/dev/pts'), bind=True, log_output=False)) - return self def __exit__(self, typ, value, traceback): @@ -393,6 +382,16 @@ def end_excursion(self, origin): excursion_context.end() return + def _switch_to_chroot(self): + os.chdir(self.path) + if self.path != '/': + os.chroot(self) + + def _switch_to_real_root(self): + os.fchdir(self.cwd) + if self.path != '/': + os.chroot('.') + def enter_chroot(self): assert not self.inchroot @@ -400,24 +399,33 @@ def enter_chroot(self): os.environ['LANGUAGE'] = 'C' os.environ['LC_ALL'] = 'C' - os.chdir(self.path) + self._switch_to_chroot() self.inchroot = True - if self.path == '/': - return - - os.chroot(self) - def leave_chroot(self): assert self.inchroot - os.fchdir(self.cwd) - + self._switch_to_real_root() self.inchroot = False + + @contextlib.contextmanager + def mount_pseudo_filesystems(self): + assert self.inchroot + if self.path == '/': + yield return - os.chroot('.') + self._switch_to_real_root() + try: + with bind_mount_pseudo_filesystems(self.path): + self._switch_to_chroot() + try: + yield + finally: + self._switch_to_real_root() + finally: + self._switch_to_chroot() class TargetFs(ChRootFilesystem): diff --git a/elbepack/egpg.py b/elbepack/egpg.py index 9dd39fed8..a955aab62 100644 --- a/elbepack/egpg.py +++ b/elbepack/egpg.py @@ -17,6 +17,7 @@ from gpg.constants import PROTOCOL_OpenPGP, sig, sigsum from gpg.errors import GPGMEError, InvalidSigners, KeyNotFound +from elbepack.paths import INITVM_GNUPG_HOME, TARGET_GNUPG_HOME # noqa: F401 from elbepack.shellhelper import env_add @@ -148,7 +149,7 @@ def check_signature(ctx, signature): return status -def unsign_file(fname): +def unsign_file(fname, gnupg_home): # check for .gpg extension and create an output filename without it if len(fname) <= 4 or fname[len(fname) - 4:] != '.gpg': print('The input file needs a .gpg extension') @@ -159,7 +160,7 @@ def unsign_file(fname): ctx = core.Context() ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + gnupg_home) ctx.set_armor(False) overall_status = OverallStatus() @@ -190,14 +191,14 @@ def unsign_file(fname): return None -def sign(infile, outfile, fingerprint): +def sign(infile, outfile, fingerprint, gnupg_home): ctx = core.Context() try: ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + gnupg_home) except GPGMEError as E: print("Error: Can't set engine info - %s", E) return @@ -232,16 +233,16 @@ def sign(infile, outfile, fingerprint): fd.write(signature) -def sign_file(fname, fingerprint): +def sign_file(fname, fingerprint, gnupg_home): outfilename = fname + '.gpg' - sign(fname, outfilename, fingerprint) + sign(fname, outfilename, fingerprint, gnupg_home) -def get_fingerprints(): +def get_fingerprints(gnupg_home): ctx = core.Context() ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + gnupg_home) keys = ctx.op_keylist_all(None, False) fingerprints = [] for k in keys: @@ -259,8 +260,8 @@ def get_fingerprints(): EOT = 4294967295 -def generate_elbe_internal_key(): - gpg_agent_conf = pathlib.Path('/var/cache/elbe/gnupg/gpg-agent.conf') +def generate_elbe_internal_key(gnupg_home): + gpg_agent_conf = pathlib.Path(gnupg_home, 'gpg-agent.conf') gpg_agent_conf.parent.mkdir(mode=0o700, parents=True, exist_ok=True) gpg_agent_conf.write_text('allow-preset-passphrase\n' f'default-cache-ttl {EOT}\n' @@ -269,18 +270,18 @@ def generate_elbe_internal_key(): ctx = core.Context() ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + gnupg_home) ctx.op_genkey(elbe_internal_key_param, None, None) key = ctx.op_genkey_result() return key.fpr -def export_key(fingerprint, outfile): +def export_key(fingerprint, outfile, gnupg_home): subprocess.run([ '/usr/bin/gpg', '-a', '-o', outfile, '--export', fingerprint, - ], check=True, env=env_add({'GNUPGHOME': '/var/cache/elbe/gnupg'})) + ], check=True, env=env_add({'GNUPGHOME': gnupg_home})) def unarmor_openpgp_keyring(armored): diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py index 798d787d4..fc8eeaaee 100644 --- a/elbepack/elbeproject.py +++ b/elbepack/elbeproject.py @@ -165,7 +165,8 @@ def __init__( self.name = self.xml.text('project/name') self.repo = ProjectRepo(self.arch, self.codename, - os.path.join(self.builddir, 'repo')) + os.path.join(self.builddir, 'repo'), + os.path.join(self.builddir, 'gnupg')) # Create BuildEnv instance, if the chroot directory exists and # has an etc/elbe_version @@ -249,7 +250,7 @@ def _dpkg_query(*args): yield file - def build_sysroot(self): + def build_sysroot(self, exclude_initvm_pkgs=False): do(['rm', '-rf', self.sysrootpath]) do(['mkdir', self.sysrootpath]) @@ -263,7 +264,8 @@ def build_sysroot(self): self.xml.add_target_package('libc6-dbg') self.xml.add_target_package('gdbserver') - self.install_packages(sysrootenv, buildenv=False) + self.install_packages(sysrootenv, buildenv=False, + exclude_initvm_pkgs=exclude_initvm_pkgs) # ignore packages from debootstrap tpkgs = self.xml.get_target_packages() @@ -388,7 +390,7 @@ def build_host_sysroot(self, pkgs, hostsysrootpath): host_sysrootenv.rfs.rmtree('/tmp') host_sysrootenv.rfs.rmtree('/var') - def build_sdk(self): + def build_sdk(self, exclude_initvm_pkgs=False): triplet = self.xml.defs['triplet'] elfcode = self.xml.defs['elfcode'] @@ -407,7 +409,7 @@ def build_sdk(self): host_pkglist.append('gdb-multiarch') # build target sysroot including libs and headers for the target - self.build_sysroot() + self.build_sysroot(exclude_initvm_pkgs=exclude_initvm_pkgs) sdktargetpath = os.path.join(self.sdkpath, 'sysroots', 'target') do(['mkdir', '-p', sdktargetpath]) do(['tar', 'xJf', os.path.join(self.builddir, 'sysroot.tar.xz'), '-C', sdktargetpath], @@ -469,7 +471,7 @@ def pbuild(self, p): def build_cdroms(self, build_bin=True, build_sources=False, cdrom_size=None, - tgt_pkg_lst=None): + tgt_pkg_lst=None, exclude_initvm_pkgs=False): self.repo_images = [] @@ -502,7 +504,8 @@ def build_cdroms(self, build_bin=True, self.codename, init_codename, self.xml, - self.builddir) + self.builddir, + exclude_initvm_pkgs=exclude_initvm_pkgs) if build_sources: if not cdrom_size and self.xml.has('src-cdrom/size'): cdrom_size = size_to_int(self.xml.text('src-cdrom/size')) @@ -560,6 +563,7 @@ def build_cdroms(self, build_bin=True, self.codename, init_codename, self.builddir, + exclude_initvm_pkgs=exclude_initvm_pkgs, **kwargs): self.repo_images += iso except SystemError as e: @@ -567,7 +571,8 @@ def build_cdroms(self, build_bin=True, validation.error(str(e)) def build(self, build_bin=False, build_sources=False, cdrom_size=None, - skip_pkglist=False, skip_pbuild=False, base_image_path=None): + skip_pkglist=False, skip_pbuild=False, base_image_path=None, + exclude_initvm_pkgs=False): # Write the log header self.write_log_header() @@ -605,7 +610,7 @@ def build(self, build_bin=False, build_sources=False, cdrom_size=None, # Install packages if not skip_pkglist: - self.install_packages(self.buildenv) + self.install_packages(self.buildenv, exclude_initvm_pkgs=exclude_initvm_pkgs) try: self.buildenv.rfs.dump_elbeversion(self.xml) @@ -642,7 +647,8 @@ def build(self, build_bin=False, build_sources=False, cdrom_size=None, # install packages for buildenv if not skip_pkglist: - self.install_packages(self.buildenv, buildenv=True) + self.install_packages(self.buildenv, buildenv=True, + exclude_initvm_pkgs=exclude_initvm_pkgs) # Write source.xml try: @@ -653,7 +659,7 @@ def build(self, build_bin=False, build_sources=False, cdrom_size=None, # Elbe report cache = self.get_rpcaptcache() - tgt_pkgs = elbe_report(self.xml, self.buildenv, cache, self.targetfs) + tgt_pkgs = elbe_report(self.xml, self.buildenv, cache, self.targetfs, self.builddir) # chroot' licenses self.gen_licenses('chroot', self.buildenv, @@ -689,7 +695,9 @@ def build(self, build_bin=False, build_sources=False, cdrom_size=None, self.targetfs.part_target(self.builddir, grub_version, grub_fw_type) - self.build_cdroms(build_bin, build_sources, cdrom_size, tgt_pkg_lst=tgt_pkgs) + self.build_cdroms(build_bin, build_sources, cdrom_size, + tgt_pkg_lst=tgt_pkgs, + exclude_initvm_pkgs=exclude_initvm_pkgs) if self.postbuild_file: logging.info('Postbuild script') @@ -987,7 +995,7 @@ def copy_initvmnode(self): logging.exception('%s is available. But it does not ' 'contain an initvm node', SOURCE_XML) - def install_packages(self, target, buildenv=False): + def install_packages(self, target, buildenv=False, exclude_initvm_pkgs=False): # to workaround debian bug no. 872543 if self.xml.prj.has('noauth'): @@ -1012,10 +1020,12 @@ def install_packages(self, target, buildenv=False): if target.need_dumpdebootstrap: dump_debootstrappkgs(self.xml, self.get_rpcaptcache(env=target)) - dump_initvmpkgs(self.xml) + if not exclude_initvm_pkgs: + dump_initvmpkgs(self.xml) target.need_dumpdebootstrap = False - self.copy_initvmnode() + if not exclude_initvm_pkgs: + self.copy_initvmnode() else: sourcepath = os.path.join(self.builddir, 'source.xml') source = ElbeXML(sourcepath, @@ -1026,9 +1036,10 @@ def install_packages(self, target, buildenv=False): try: self.xml.get_initvmnode_from(source) except NoInitvmNode: - logging.warning('source.xml is available. ' - 'But it does not contain an initvm node') - self.copy_initvmnode() + if not exclude_initvm_pkgs: + logging.warning('source.xml is available. ' + 'But it does not contain an initvm node') + self.copy_initvmnode() # Seed /etc, we need /etc/hosts for hostname -f to work correctly if not buildenv: diff --git a/elbepack/finetuning.py b/elbepack/finetuning.py index d780d4ad5..08fac9c29 100644 --- a/elbepack/finetuning.py +++ b/elbepack/finetuning.py @@ -20,11 +20,12 @@ from gpg import core from gpg.constants import PROTOCOL_OpenPGP +from elbepack.egpg import TARGET_GNUPG_HOME from elbepack.filesystem import Filesystem -from elbepack.imgutils import losetup, mount +from elbepack.imgutils import losetup from elbepack.packers import default_packer, packers from elbepack.paths import DOWNGRADE_ALLOWED_FILE, REPOS_BASE_DIR -from elbepack.shellhelper import ELBE_LOGGING, chroot, do, env_add, get_env_with_sbin, run +from elbepack.shellhelper import ELBE_LOGGING, chroot, do, env_add, get_env_with_sbin, mount, run from elbepack.treeutils import strip_leading_whitespace_from_lines @@ -42,7 +43,8 @@ def __init__(self, node): def execute(self, _buildenv, _target): raise NotImplementedError('execute() not implemented') - def execute_prj(self, buildenv, target, _builddir): + def execute_prj(self, buildenv, target, builddir): + self.builddir = builddir self.execute(buildenv, target) @@ -368,7 +370,7 @@ def execute(self, buildenv, target): ctx = core.Context() ctx.set_engine_info(PROTOCOL_OpenPGP, None, - '/var/cache/elbe/gnupg') + os.path.join(self.builddir, 'gnupg')) ctx.set_armor(True) ctx.op_export(fp, 0, gpgdata) gpgdata.seek(0, os.SEEK_SET) @@ -378,10 +380,10 @@ def execute(self, buildenv, target): with open((target.path + '/pub.key'), 'wb') as tkey: tkey.write(key) - target.mkdir_p('/var/cache/elbe/gnupg', mode=0o700) + target.mkdir_p(TARGET_GNUPG_HOME, mode=0o700) with target: do(['gpg', '--import', target.path + '/pub.key'], - env_add={'GNUPGHOME': f'{target.path}/var/cache/elbe/gnupg'}) + env_add={'GNUPGHOME': target.path + TARGET_GNUPG_HOME}) logging.info('generate base repo') @@ -406,7 +408,8 @@ def execute(self, buildenv, target): logging.exception('Package %s-%s missing name or version', pkg.name, pkg.installed_version) r = UpdateRepo(target.xml, - target.path + REPOS_BASE_DIR) + target.path + REPOS_BASE_DIR, + os.path.join(self.builddir, 'gnupg')) for d in buildenv.rfs.glob('tmp/pkgs/*.deb'): r.includedeb(d, 'main') @@ -697,7 +700,7 @@ def execute(self, buildenv, _target): f.write('\n'.join(src_lst)) -def do_finetuning(xml, buildenv, target): +def do_finetuning(xml, buildenv, target, builddir): if not xml.has('target/finetuning'): return @@ -705,6 +708,7 @@ def do_finetuning(xml, buildenv, target): for i in xml.node('target/finetuning'): try: action = _action_for_node(i) + action.builddir = builddir action.execute(buildenv, target) except KeyError: logging.exception("Unimplemented finetuning action '%s'", diff --git a/elbepack/hdimg.py b/elbepack/hdimg.py index f190a6007..46ac9bed4 100644 --- a/elbepack/hdimg.py +++ b/elbepack/hdimg.py @@ -16,8 +16,8 @@ from elbepack.filesystem import Filesystem, size_to_int from elbepack.fstab import fstabentry, hdpart, mountpoint_dict -from elbepack.imgutils import dd, losetup, mount -from elbepack.shellhelper import chroot, do +from elbepack.imgutils import dd, losetup +from elbepack.shellhelper import chroot, do, mount def mkfs_mtd(mtd, fslabel, target): diff --git a/elbepack/imgutils.py b/elbepack/imgutils.py index 5c8051d4c..d81c92d40 100644 --- a/elbepack/imgutils.py +++ b/elbepack/imgutils.py @@ -4,12 +4,46 @@ import contextlib import fcntl +import os import pathlib +import stat import subprocess from elbepack.shellhelper import ELBE_LOGGING, do, run +def _udev_available(): + return pathlib.Path('/run/udev/control').is_socket() + + +def _mknod_from_sysfs(device_name): + devpath = f'/dev/{device_name}' + if os.path.exists(devpath): + return + + dev_attr = pathlib.Path('/sys/class/block', device_name, 'dev').read_text().strip() + major, minor = (int(x) for x in dev_attr.split(':')) + with contextlib.suppress(FileExistsError): + os.mknod(devpath, mode=0o660 | stat.S_IFBLK, device=os.makedev(major, minor)) + os.chmod(devpath, 0o660) + + +def _symlink_by_uuid_from_blkid(device_name): + devpath = f'/dev/{device_name}' + blkid = subprocess.run( + ['blkid', '-s', 'UUID', '-o', 'value', devpath], + stdout=subprocess.PIPE, check=False, + ) + uuid = blkid.stdout.decode('ascii').strip() + if blkid.returncode != 0 or not uuid: + return + + by_uuid_dir = pathlib.Path('/dev/disk/by-uuid') + by_uuid_dir.mkdir(parents=True, exist_ok=True) + with contextlib.suppress(FileExistsError): + (by_uuid_dir / uuid).symlink_to(devpath) + + def _wait_on_udev_for_device_and_partitions(device): # The callers expect the udev symlinks of the loop device and its # partitions to be present. @@ -21,13 +55,20 @@ def _wait_on_udev_for_device_and_partitions(device): # However udev processing triggers a rescan of the partitions, removing # the entries for a short time. Prevent udev from doing so while we iterate. fcntl.flock(f, fcntl.LOCK_EX) - partitions = [ - '/dev/' + entry.name + partition_names = [ + entry.name for entry in pathlib.Path('/sys/class/block', device_name).iterdir() if entry.name.startswith(device_name) ] + if not _udev_available(): + for name in (device_name, *partition_names): + _mknod_from_sysfs(name) + _symlink_by_uuid_from_blkid(name) + return + # All partitions need to be mentioned explicitly. + partitions = ['/dev/' + name for name in partition_names] subprocess.run(['udevadm', 'wait', device, *partitions], check=True, timeout=30) @@ -46,50 +87,5 @@ def losetup(dev, extra_args=[]): do(['losetup', '--detach', loopdev], check=False) -class _Mount: - # This is not using contextlib.contextmanager as it will be pass to our - # RPCAPTCache which uses the pickle serialization. - # The generator by contextlib.contextmanager is not compatible with pickle. - def __init__(self, device, target, *, bind=False, type=None, options=None, log_output=True, - force_writable=False): - self.log_output = log_output - self.target = target - - cmd = ['mount'] - if bind: - cmd.append('--bind') - - if options is not None: - cmd.extend(['-o', options]) - - if force_writable: - cmd.append('--rw') - - if type is not None: - cmd.extend(['-t', type]) - - if device is None: - device = 'none' - - cmd.extend([device, target]) - - self.cmd = cmd - - def _run_cmd(self, cmd, *args, **kwargs): - if self.log_output: - do(cmd, *args, **kwargs) - else: - subprocess.run(cmd, *args, **kwargs) - - def __enter__(self): - self._run_cmd(self.cmd) - - def __exit__(self, exc_type, exc_value, traceback): - self._run_cmd(['umount', self.target], check=False) - - -mount = _Mount - - def dd(args, /, **kwargs): do(['dd', *[f'{k}={v}' for k, v in args.items()]], **kwargs) diff --git a/elbepack/initvmaction.py b/elbepack/initvmaction.py index d15d16114..befef9ea5 100644 --- a/elbepack/initvmaction.py +++ b/elbepack/initvmaction.py @@ -12,7 +12,7 @@ import elbepack import elbepack.initvm -from elbepack.buildsubmitaction import add_submit_arguments, extract_cdrom +from elbepack.buildsubmitaction import add_output_argument, add_submit_arguments, extract_cdrom from elbepack.cli import CliError, add_argument, with_cli_details from elbepack.config import add_argument_sshport, add_arguments_soapclient from elbepack.elbexml import ValidationError @@ -127,7 +127,7 @@ def _submit_and_dl_result(control, xmlfile, cdrom, base_image, args, xmlfile_bas print('Upload finished') control.service.build(prjdir, args.build_bin, args.build_sources, bool(cdrom), - uploaded_base_image_path) + uploaded_base_image_path, args.exclude_initvm_pkgs) print('Build started, waiting till it finishes') @@ -228,6 +228,7 @@ def _submit_and_dl_result(control, xmlfile, cdrom, base_image, args, xmlfile_bas help=argparse.SUPPRESS) @add_submit_arguments @add_argument('--size', help='Disk size', type=size_to_int) +@add_output_argument @add_argument('input', nargs='?', metavar=' | ') def _create(args): # Upgrade from older versions which used tmux @@ -316,6 +317,11 @@ def _create(args): @_add_initvm_from_args_arguments @add_submit_arguments +@add_output_argument +@add_argument( + '--exclude-initvm-pkgs', action='store_true', dest='exclude_initvm_pkgs', + default=False, + help='Exclude initvm packages from CDROM generation') @add_argument('input', metavar=' | ') def _submit(args): initvm = _initvm_from_args(args) diff --git a/elbepack/localbuildaction.py b/elbepack/localbuildaction.py new file mode 100644 index 000000000..8fb66e099 --- /dev/null +++ b/elbepack/localbuildaction.py @@ -0,0 +1,161 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import os +import shutil +import sys +import textwrap +import time + +from elbepack.cli import CliError, with_cli_details +from elbepack.projectmanager import ProjectManager +from elbepack.repodir import Repodir, RepodirError +from elbepack.rootcheck import check_rootful_requirements +from elbepack.xmlpreprocess import preprocess_file + +prog = os.path.basename(sys.argv[0]) + + +def local_build_with_repodir_and_dl_result(xmlfile, cdrom, base_image, args): + os.makedirs(args.build_dir, exist_ok=True) + fname = f'elbe-repodir-{time.time_ns()}.xml' + preprocess_xmlfile = os.path.join(args.build_dir, fname) + try: + with Repodir(xmlfile, preprocess_xmlfile): + _local_build_and_dl_result(preprocess_xmlfile, cdrom, base_image, args, + xmlfile_base=xmlfile) + except RepodirError as err: + raise with_cli_details(err, 127, 'elbe repodir failed') + + +def _wait_busy(pm, prjdir): + while True: + is_busy, msg = pm.project_is_busy(prjdir) + + if msg: + print(msg) + continue + + if not is_busy: + break + + time.sleep(0.1) + + # exited the loop -> the project is not busy anymore, + # check, whether everything is ok. + prj = pm.db.get_project_data(prjdir) + if prj.status != 'build_done': + raise CliError(191, f'Project build was not successful, current status: {prj.status}') + + +def _local_build_and_dl_result(xmlfile, cdrom, base_image, args, xmlfile_base=None): + cache_dir = os.path.join(args.build_dir, 'cache') + pm = ProjectManager(cache_dir) + try: + with preprocess_file(xmlfile, variants=args.variants, sshport=args.sshport, + soapport=args.soapport, xmlfile_base=xmlfile_base) as xmlfile: + check_rootful_requirements(xmlfile) + prjdir = pm.create_project(xmlfile) + + if args.writeproject: + with open(args.writeproject, 'w') as wpf: + wpf.write(prjdir) + + if cdrom is not None: + print('Copying CDROM into project. This might take a while') + shutil.copy(cdrom, os.path.join(prjdir, 'uploaded_cdrom.iso')) + pm.set_upload_cdrom(prjdir) + print('Copy finished') + + uploaded_base_image_path = None + if base_image is not None: + print('Copying base image into project. This might take a while') + uploaded_base_image_path = os.path.join(prjdir, 'uploaded_base_image.img') + shutil.copy(base_image, uploaded_base_image_path) + print('Copy finished') + + pm.build_project(prjdir, args.build_bin, args.build_sources, bool(cdrom), + uploaded_base_image_path, args.exclude_initvm_pkgs) + + print('Build started, waiting till it finishes') + + try: + _wait_busy(pm, prjdir) + except Exception as e: + raise with_cli_details(e, 133, textwrap.dedent(f""" + Build Failed + + The project will not be deleted. + Its files are available at: + {prjdir} """)) + + print('') + print('Build finished !') + print('') + + if args.build_sdk: + pm.build_sdk(prjdir, args.exclude_initvm_pkgs) + + print('SDK Build started, waiting till it finishes') + + try: + _wait_busy(pm, prjdir) + except Exception: + print('Waiting for the SDK build Failed', file=sys.stderr) + print('', file=sys.stderr) + print('The project will not be deleted.', file=sys.stderr) + print('Its files are available at:', file=sys.stderr) + print(prjdir, file=sys.stderr) + print('', file=sys.stderr) + sys.exit(135) + + print('') + print('SDK Build finished !') + print('') + + try: + with open(os.path.join(prjdir, 'validation.txt'), 'rb') as f: + shutil.copyfileobj(f, sys.stdout.buffer) + sys.stdout.buffer.flush() + except Exception: + print( + 'Project failed to generate validation.txt', + file=sys.stderr) + print('Getting log.txt', file=sys.stderr) + try: + with open(os.path.join(prjdir, 'log.txt'), 'rb') as f: + shutil.copyfileobj(f, sys.stdout.buffer) + sys.stdout.buffer.flush() + except Exception as e: + raise with_cli_details(e, 137, textwrap.dedent('Failed to dump log.txt')) + sys.exit(136) + + files = pm.db.get_project_files(prjdir) + + if args.skip_download: + print('') + print('Listing available files:') + print('') + for file in files: + print(f'{file.name}\t{file.description}') + + print('') + print(f'Files are available at: {prjdir}') + else: + print('') + print('Getting generated Files') + print('') + + print(f'Saving generated Files to {args.build_dir}') + + os.makedirs(args.build_dir, exist_ok=True) + for file in files: + shutil.copy(os.path.join(prjdir, file.name), + os.path.join(args.build_dir, os.path.basename(file.name))) + print(f'{file.name}\t{file.description}') + + if not args.keep_files: + pm.del_project(prjdir) + finally: + pm.stop() diff --git a/elbepack/paths.py b/elbepack/paths.py index 8944e1c8e..03a174934 100644 --- a/elbepack/paths.py +++ b/elbepack/paths.py @@ -8,6 +8,7 @@ BINARIES_MAIN_DIR = f'{ELBE_CACHE_DIR}/binaries/main' BINARIES_ADDED_DIR = f'{ELBE_CACHE_DIR}/binaries/added' INITVM_BIN_REPO_DIR = f'{ELBE_CACHE_DIR}/initvm-bin-repo' +INITVM_GNUPG_HOME = f'{ELBE_CACHE_DIR}/gnupg' INITVM_SRC_REPO_DIR = f'{ELBE_CACHE_DIR}/initvm-src-repo' INSTALLER_DIR = f'{ELBE_CACHE_DIR}/installer' INSTALLER_VMLINUZ = f'{INSTALLER_DIR}/vmlinuz' @@ -23,3 +24,7 @@ DB_PATH = ELBE_CACHE_DIR PRE_SCRIPT = f'{ELBE_CACHE_DIR}/pre.sh' POST_SCRIPT = f'{ELBE_CACHE_DIR}/post.sh' + +# Path on deployed target system. Just happens to be the same +# in the build system, but keep separate because there is no relation. +TARGET_GNUPG_HOME = '/var/cache/elbe/gnupg' diff --git a/elbepack/projectmanager.py b/elbepack/projectmanager.py index 9815c1399..7be84dbd5 100644 --- a/elbepack/projectmanager.py +++ b/elbepack/projectmanager.py @@ -108,10 +108,11 @@ def build_project( build_bin, build_src, skip_pbuilder, - base_image_path): + base_image_path, + exclude_initvm_pkgs=False): ep = self.open_project(builddir, allow_busy=False) self.worker.enqueue(BuildJob(ep, build_bin, build_src, - skip_pbuilder, base_image_path)) + skip_pbuilder, base_image_path, exclude_initvm_pkgs)) def update_pbuilder(self, builddir): ep = self.open_project(builddir, allow_busy=False) @@ -153,9 +154,9 @@ def build_sysroot(self, builddir): ep = self.open_project(builddir, allow_busy=False) self.worker.enqueue(BuildSysrootJob(ep)) - def build_sdk(self, builddir): + def build_sdk(self, builddir, exclude_initvm_pkgs=False): ep = self.open_project(builddir, allow_busy=False) - self.worker.enqueue(BuildSDKJob(ep)) + self.worker.enqueue(BuildSDKJob(ep, exclude_initvm_pkgs)) def build_cdroms(self, builddir, build_bin, build_src): ep = self.open_project(builddir, allow_busy=False) diff --git a/elbepack/repomanager.py b/elbepack/repomanager.py index a490ef018..03a38791f 100644 --- a/elbepack/repomanager.py +++ b/elbepack/repomanager.py @@ -63,9 +63,11 @@ def __init__( repo_attr, origin, description, + gnupg_home, maxsize=None): self.vol_path = path + self.gnupg_home = gnupg_home self.volume_count = 0 self.init_attr = init_attr @@ -77,6 +79,8 @@ def __init__( self.attrs = [repo_attr] elif init_attr is not None: self.attrs = [init_attr] + else: + self.attrs = [] self.origin = origin self.description = description @@ -86,14 +90,23 @@ def __init__( # if repo exists retrive the keyid otherwise # generate a new key and generate repository config if self.volume.is_dir(): - repo_conf = self.volume.joinpath('conf', 'distributions').read_text() - for lic in repo_conf.splitlines(): - if lic.startswith('SignWith'): - self.keyid = lic.split()[1] + conf_dist = self.volume.joinpath('conf', 'distributions') + if conf_dist.is_file(): + repo_conf = conf_dist.read_text() + for lic in repo_conf.splitlines(): + if lic.startswith('SignWith'): + self.keyid = lic.split()[1] + else: + # Directory exists but no repo config, so treat as new repository + self.keyid = generate_elbe_internal_key(self.gnupg_home) + self.gen_repo_conf() else: - self.keyid = generate_elbe_internal_key() + self.keyid = generate_elbe_internal_key(self.gnupg_home) self.gen_repo_conf() + def _reprepro(self, args): + do(['reprepro', *args], env_add={'GNUPGHOME': self.gnupg_home}) + def get_volume_path(self, volume): if self.maxsize: if volume >= 0: @@ -160,20 +173,17 @@ def gen_repo_conf(self): fp.write('\n') - export_key(self.keyid, self.volume / 'repo.pub') + export_key(self.keyid, self.volume / 'repo.pub', self.gnupg_home) if need_update: - do(['reprepro', '--export=force', '--basedir', self.volume, 'update'], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro(['--export=force', '--basedir', self.volume, 'update']) else: for att in self.attrs: - do(['reprepro', '--basedir', self.volume, 'export', att.codename], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro(['--basedir', self.volume, 'export', att.codename]) def finalize(self): for att in self.attrs: - do(['reprepro', '--basedir', self.volume, 'export', att.codename], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro(['--basedir', self.volume, 'export', att.codename]) def _includedeb(self, path, codename, components=None, prio=None): if self.maxsize: @@ -194,7 +204,7 @@ def _includedeb(self, path, codename, components=None, prio=None): components = [components] global_opt.extend(['--component', '|'.join(components)]) - do(['reprepro', *global_opt, 'includedeb', codename, path]) + self._reprepro([*global_opt, 'includedeb', codename, path]) def includedeb(self, path, components=None, pkgname=None, force=False, prio=None): # pkgname needs only to be specified if force is enabled @@ -232,7 +242,7 @@ def _include(self, path, codename, components=None): components = [components] global_opt.extend(['--component', '|'.join(components)]) - do(['reprepro', *global_opt, 'include', codename, path]) + self._reprepro([*global_opt, 'include', codename, path]) def _removedeb(self, pkgname, codename, components=None): @@ -244,8 +254,7 @@ def _removedeb(self, pkgname, codename, components=None): components = [components] global_opt.extend(['--component', '|'.join(components)]) - do(['reprepro', *global_opt, 'remove', codename, pkgname], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro([*global_opt, 'remove', codename, pkgname]) def removedeb(self, pkgname, components=None): self._removedeb(pkgname, self.repo_attr.codename, components) @@ -254,8 +263,7 @@ def _removesrc(self, srcname, codename): global_opt = ['--basedir', self.volume] - do(['reprepro', *global_opt, 'removesrc', codename, srcname], - env_add={'GNUPGHOME': '/var/cache/elbe/gnupg'}) + self._reprepro([*global_opt, 'removesrc', codename, srcname]) def removesrc(self, path): with open(path) as fp: @@ -297,7 +305,7 @@ def _includedsc(self, path, codename, components=None): components = [components] global_opt.extend(['--component', '|'.join(components)]) - do(['reprepro', *global_opt, 'includedsc', codename, path]) + self._reprepro([*global_opt, 'includedsc', codename, path]) def includedsc(self, path, components=None, force=False): try: @@ -345,7 +353,7 @@ def volume_indexes(self): class UpdateRepo(RepoBase): - def __init__(self, xml, path): + def __init__(self, xml, path, gnupg_home): self.xml = xml arch = xml.text('project/arch', key='arch') @@ -353,18 +361,21 @@ def __init__(self, xml, path): repo_attrs = RepoAttributes(codename, arch, 'main') - super().__init__(path, None, repo_attrs, 'Update', 'Update') + super().__init__(path, None, repo_attrs, 'Update', 'Update', gnupg_home) class CdromInitRepo(RepoBase): - def __init__(self, init_codename, path, + def __init__(self, init_codename, path, gnupg_home, mirror='http://deb.debian.org/debian'): - init_attrs = RepoAttributes( - init_codename, 'amd64', [ - 'main', 'main/debian-installer'], mirror) + if init_codename is not None: + init_attrs = RepoAttributes( + init_codename, 'amd64', [ + 'main', 'main/debian-installer'], mirror) + else: + init_attrs = None - super().__init__(path, None, init_attrs, 'Elbe', 'Elbe InitVM Cdrom Repo') + super().__init__(path, None, init_attrs, 'Elbe', 'Elbe InitVM Cdrom Repo', gnupg_home) class CdromBinRepo(RepoBase): @@ -374,6 +385,7 @@ def __init__( codename, init_codename, path, + gnupg_home, mirror='http://deb.debian.org/debian'): repo_attrs = RepoAttributes(codename, arch, ['main', 'added'], mirror) @@ -384,11 +396,12 @@ def __init__( else: init_attrs = None - super().__init__(path, init_attrs, repo_attrs, 'Elbe', 'Elbe Binary Cdrom Repo') + super().__init__(path, init_attrs, repo_attrs, 'Elbe', 'Elbe Binary Cdrom Repo', + gnupg_home) class CdromSrcRepo(RepoBase): - def __init__(self, codename, init_codename, path, maxsize, + def __init__(self, codename, init_codename, path, maxsize, gnupg_home, mirror='http://deb.debian.org/debian'): repo_attrs = RepoAttributes(codename, @@ -408,16 +421,18 @@ def __init__(self, codename, init_codename, path, maxsize, else: init_attrs = None - super().__init__(path, init_attrs, repo_attrs, 'Elbe', 'Elbe Source Cdrom Repo', maxsize) + super().__init__(path, init_attrs, repo_attrs, 'Elbe', 'Elbe Source Cdrom Repo', + gnupg_home, maxsize) class ToolchainRepo(RepoBase): - def __init__(self, arch, codename, path): + def __init__(self, arch, codename, path, gnupg_home): repo_attrs = RepoAttributes(codename, arch, 'main') - super().__init__(path, None, repo_attrs, 'toolchain', 'Toolchain binary packages Repo') + super().__init__(path, None, repo_attrs, 'toolchain', 'Toolchain binary packages Repo', + gnupg_home) class ProjectRepo(RepoBase): - def __init__(self, arch, codename, path): + def __init__(self, arch, codename, path, gnupg_home): repo_attrs = RepoAttributes(codename, [arch, 'amd64', 'source'], 'main') - super().__init__(path, None, repo_attrs, 'Local', 'Self build packages Repo') + super().__init__(path, None, repo_attrs, 'Local', 'Self build packages Repo', gnupg_home) diff --git a/elbepack/rootcheck.py b/elbepack/rootcheck.py new file mode 100644 index 000000000..6ddcb96aa --- /dev/null +++ b/elbepack/rootcheck.py @@ -0,0 +1,98 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import logging +import os +import subprocess +import tempfile +import textwrap + +from elbepack.cli import CliError +from elbepack.finetuning import LosetupAction +from elbepack.imgutils import losetup +from elbepack.treeutils import etree + + +def _loop_mount_reasons(xml): + reasons = [] + + for grub in xml.all('target/images/*/grub-install'): + hd = grub.get_parent() + reasons.append( + f"<{hd.tag}> image '{hd.text('name')}' uses , which " + 'loop-mounts the disk image to run grub-install in a chroot') + + for bylabel in xml.all('target/fstab/bylabel'): + if (bylabel.has('fs/fs-finetuning/device-command') + or bylabel.has('fs/fs-finetuning/path-command')): + reasons.append( + f"filesystem '{bylabel.text('label')}' uses " + '(or the deprecated ) with a device-command/' + 'path-command, which loop-mounts that partition image') + + for ls in xml.all('target/project-finetuning/losetup'): + if any(child.tag in LosetupAction.needs_loop_device for child in ls): + reasons.append( + f" contains an action " + 'that needs a loop device') + + return reasons + + +def _mknod_reasons(xml): + reasons = [] + + for node in xml.all('target/finetuning/mknod'): + reasons.append( + f"{node.et.text} " + 'creates a device node, which needs to run rootful (real root / ' + 'CAP_MKNOD, and in a container without a remapped user namespace)') + + return reasons + + +def xml_needs_rootful(xml): + return _loop_mount_reasons(xml) + _mknod_reasons(xml) + + +def loop_mount_available(): + with tempfile.NamedTemporaryFile(prefix='elbe-loopcheck-') as f: + f.truncate(1024 * 1024) + try: + with losetup(f.name): + pass + except subprocess.CalledProcessError as e: + logging.debug('loop-mount preflight probe failed: %s', e) + return False + return True + + +def check_rootful_requirements(xmlfile): + xml = etree(xmlfile) + loop_reasons = _loop_mount_reasons(xml) + mknod_reasons = _mknod_reasons(xml) + + problems = [] + if loop_reasons and not loop_mount_available(): + problems.append(( + 'Linux loop devices, but this process is not able to create ' + 'them (e.g. no access to /dev/loop-control)', loop_reasons)) + if mknod_reasons and os.geteuid() != 0: + problems.append(( + 'to create device nodes (mknod), but this process is not ' + 'running as root', mknod_reasons)) + + if not problems: + return + + body = '\n\n'.join( + f'This build needs {desc}:\n' + '\n'.join(f' - {r}' for r in reasons) + for desc, reasons in problems) + + raise CliError(message=textwrap.dedent(f""" + {body} + + See the documentation for the elbe build command how to + run a container with the necessary privileges. + """)) diff --git a/elbepack/rpcaptcache.py b/elbepack/rpcaptcache.py index bc6623636..c00f59a9f 100644 --- a/elbepack/rpcaptcache.py +++ b/elbepack/rpcaptcache.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # SPDX-FileCopyrightText: 2014-2018 Linutronix GmbH +import functools import os import sys import time @@ -62,6 +63,14 @@ def __init__(self, rfs): self.finalizer = Finalize(self, self.rfs.leave_chroot, exitpriority=10) +def _with_pseudo_filesystems(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + with self.rfs.mount_pseudo_filesystems(): + return func(self, *args, **kwargs) + return wrapper + + @MyMan.register('RPCAPTCache') class RPCAPTCache(InChRootObject): @@ -208,14 +217,17 @@ def mark_delete(self, pkgname): p = self.cache[pkgname] p.mark_delete(purge=True) + @_with_pseudo_filesystems def update(self): self.cache.update(fetch_progress=ElbeAcquireProgress()) self.cache.open(progress=ElbeOpProgress()) + @_with_pseudo_filesystems def fetch_archives(self): print('Fetching packages...') self.cache.fetch_archives(ElbeAcquireProgress()) + @_with_pseudo_filesystems def commit(self): os.environ['DEBIAN_FRONTEND'] = 'noninteractive' os.environ['DEBONF_NONINTERACTIVE_SEEN'] = 'true' @@ -268,6 +280,7 @@ def get_pkg(self, pkgname): def get_corresponding_source_packages(self, pkg_lst=None, *, include_built_using=True): return get_corresponding_source_packages(self.cache, pkg_lst, include_built_using) + @_with_pseudo_filesystems def download_binary(self, pkgname, path, version=None): p = self.cache[pkgname] if version is None: @@ -277,6 +290,7 @@ def download_binary(self, pkgname, path, version=None): rel_filename = pkgver.fetch_binary(path, ElbeAcquireProgress()) return self.rfs.fname(rel_filename) + @_with_pseudo_filesystems def download_source(self, src_name, src_version, dest_dir): return self.rfs.fname(fetch_source(src_name, src_version, dest_dir, ElbeAcquireProgress())) diff --git a/elbepack/shellhelper.py b/elbepack/shellhelper.py index 722a1a908..e2fbfdb13 100644 --- a/elbepack/shellhelper.py +++ b/elbepack/shellhelper.py @@ -152,6 +152,76 @@ def _target_path(directory): return FALLBACK_PATH +class _Mount: + # This is not using contextlib.contextmanager as it will be pass to our + # RPCAPTCache which uses the pickle serialization. + # The generator by contextlib.contextmanager is not compatible with pickle. + def __init__(self, device, target, *, bind=False, rbind=False, type=None, options=None, + log_output=True, force_writable=False): + self.log_output = log_output + self.target = target + + self.rbind = rbind + + cmd = ['mount'] + if bind: + cmd.append('--bind') + elif rbind: + cmd.append('--rbind') + + if options is not None: + cmd.extend(['-o', options]) + + if force_writable: + cmd.append('--rw') + + if type is not None: + cmd.extend(['-t', type]) + + if device is None: + device = 'none' + + cmd.extend([device, target]) + + self.cmd = cmd + + def _run_cmd(self, cmd, *args, **kwargs): + if self.log_output: + do(cmd, *args, **kwargs) + else: + subprocess.run(cmd, *args, **kwargs) + + def __enter__(self): + self._run_cmd(self.cmd) + if self.rbind: + # Detach the bind-mounted subtree from the shared + # propagation group. Without this, unmounting it in + # __exit__ can propagate back and unmount the + # corresponding mounts at the source. + self._run_cmd(['mount', '--make-rprivate', self.target]) + + def __exit__(self, exc_type, exc_value, traceback): + cmd = ['umount', '--lazy', self.target] if self.rbind else ['umount', self.target] + self._run_cmd(cmd, check=False) + + +mount = _Mount + + +@contextlib.contextmanager +def bind_mount_pseudo_filesystems(directory): + if directory == '/': + yield + return + + with contextlib.ExitStack() as stack: + for src in ['/proc', '/sys', '/dev']: + target = os.path.join(directory, src.lstrip('/')) + os.makedirs(target, exist_ok=True) + stack.enter_context(mount(src, target, rbind=True, log_output=False)) + yield + + def chroot(directory, cmd, /, *, env_add=None, **kwargs): """chroot() - Wrapper around do(). @@ -176,10 +246,11 @@ def chroot(directory, cmd, /, *, env_add=None, **kwargs): if env_add: new_env.update(env_add) - if _is_shell_cmd(cmd): - do(['/usr/sbin/chroot', directory, '/bin/sh', '-c', cmd], env_add=new_env, **kwargs) - else: - do(['/usr/sbin/chroot', directory] + cmd, env_add=new_env, **kwargs) + with bind_mount_pseudo_filesystems(directory): + if _is_shell_cmd(cmd): + do(['/usr/sbin/chroot', directory, '/bin/sh', '-c', cmd], env_add=new_env, **kwargs) + else: + do(['/usr/sbin/chroot', directory] + cmd, env_add=new_env, **kwargs) def env_add(d): diff --git a/elbepack/tests/test_rootcheck.py b/elbepack/tests/test_rootcheck.py new file mode 100644 index 000000000..6908567f2 --- /dev/null +++ b/elbepack/tests/test_rootcheck.py @@ -0,0 +1,203 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import contextlib +import subprocess + +import pytest + +from elbepack.cli import CliError +from elbepack.rootcheck import ( + check_rootful_requirements, + loop_mount_available, + xml_needs_rootful, +) +from elbepack.treeutils import etree + + +def _xml(s): + return etree(None, string=s) + + +def test_needs_no_rootful_for_plain_xml(): + assert xml_needs_rootful(_xml('')) == [] + + +@pytest.mark.parametrize('hd_tag', ['msdoshd', 'gpthd']) +def test_needs_rootful_for_grub_install(hd_tag): + xml = _xml(f""" + <{hd_tag}> + disk.img + + + """) + reasons = xml_needs_rootful(xml) + assert len(reasons) == 1 + assert 'grub-install' in reasons[0] + + +@pytest.mark.parametrize('command_tag', ['device-command', 'path-command']) +def test_needs_rootful_for_fs_finetuning(command_tag): + xml = _xml(f""" + + + + ext4 + <{command_tag}>echo hi + + + """) + reasons = xml_needs_rootful(xml) + assert len(reasons) == 1 + assert 'rootfs' in reasons[0] + + +def test_needs_no_rootful_for_fs_finetuning_file_command(): + xml = _xml(""" + + + + ext4 + echo hi + + + """) + assert xml_needs_rootful(xml) == [] + + +@pytest.mark.parametrize('child_xml', [ + 'out.bin', + 'in.bin', + 'true', +]) +def test_needs_rootful_for_specific_losetups(child_xml): + xml = _xml(f""" + + {child_xml} + + """) + assert len(xml_needs_rootful(xml)) == 1 + + +def test_needs_no_rootful_for_some_losetups(): + xml = _xml(""" + + + out.img + 83 + in.img + + + """) + assert xml_needs_rootful(xml) == [] + + +def test_loop_mount_available_true(monkeypatch): + @contextlib.contextmanager + def fake_losetup(dev, extra_args=[]): + yield '/dev/loop0' + + monkeypatch.setattr('elbepack.rootcheck.losetup', fake_losetup) + assert loop_mount_available() is True + + +def test_loop_mount_available_false_on_called_process_error(monkeypatch): + @contextlib.contextmanager + def fake_losetup(dev, extra_args=[]): + raise subprocess.CalledProcessError(1, ['losetup']) + yield # pragma: no cover + + monkeypatch.setattr('elbepack.rootcheck.losetup', fake_losetup) + assert loop_mount_available() is False + + +def test_check_rootful_requirements_skips_probe_when_nothing_needed(tmp_path, monkeypatch): + def fail(*a, **k): + raise AssertionError('loop_mount_available should not be called') + + monkeypatch.setattr('elbepack.rootcheck.loop_mount_available', fail) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text('') + check_rootful_requirements(str(xmlfile)) + + +def test_check_rootful_requirements_passes_when_available(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.loop_mount_available', lambda: True) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + disk.img + + """) + check_rootful_requirements(str(xmlfile)) + + +def test_check_rootful_requirements_raises_when_loop_unavailable(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.loop_mount_available', lambda: False) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + disk.img + + """) + with pytest.raises(CliError) as exc_info: + check_rootful_requirements(str(xmlfile)) + assert 'grub-install' in str(exc_info.value) + assert 'elbe build command' in str(exc_info.value) + + +def test_needs_rootful_detects_mknod(): + xml = _xml(""" + + /dev/tty + + """) + reasons = xml_needs_rootful(xml) + assert len(reasons) == 1 + assert 'mknod' in reasons[0] + assert '/dev/tty' in reasons[0] + + +def test_check_rootful_requirements_raises_when_mknod_and_not_root(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.os.geteuid', lambda: 1000) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + /dev/tty + + """) + with pytest.raises(CliError) as exc_info: + check_rootful_requirements(str(xmlfile)) + assert 'mknod' in str(exc_info.value) + + +def test_check_rootful_requirements_passes_when_mknod_and_root(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.os.geteuid', lambda: 0) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + /dev/tty + + """) + check_rootful_requirements(str(xmlfile)) + + +def test_check_rootful_requirements_combines_loop_and_mknod_problems(tmp_path, monkeypatch): + monkeypatch.setattr('elbepack.rootcheck.loop_mount_available', lambda: False) + monkeypatch.setattr('elbepack.rootcheck.os.geteuid', lambda: 1000) + xmlfile = tmp_path / 'x.xml' + xmlfile.write_text(""" + + + disk.img + + + /dev/tty + + + """) + with pytest.raises(CliError) as exc_info: + check_rootful_requirements(str(xmlfile)) + assert 'grub-install' in str(exc_info.value) + assert 'mknod' in str(exc_info.value) diff --git a/elbepack/tests/test_xml.py b/elbepack/tests/test_xml.py index c0818e8eb..2d47ea0ce 100644 --- a/elbepack/tests/test_xml.py +++ b/elbepack/tests/test_xml.py @@ -2,107 +2,41 @@ # SPDX-License-Identifier: GPL-3.0-or-later # SPDX-FileCopyrightText: 2020 Linutronix GmbH -import contextlib -import functools -import io + +# Only contains templates used by other test files +__test__ = False + import pathlib -import subprocess import pytest from elbepack.main import run_elbe_subcommand -from elbepack.tests import parametrize_xml_test_files, xml_test_files - - -here = pathlib.Path(__file__).parent - - -@pytest.fixture(scope='module') -def initvm(tmp_path_factory, request): - initvm_dir = tmp_path_factory.mktemp('initvm-') / 'initvm' - use_initvm = request.config.getoption('--elbe-use-initvm') - - if use_initvm in {'libvirt', 'existing'}: - qemu_arg = [] - elif use_initvm == 'qemu': - qemu_arg = ['--qemu'] - else: - raise ValueError(use_initvm) - - def initvm_func(subcmd, *args): - run_elbe_subcommand(['initvm', subcmd, '--directory', initvm_dir, *qemu_arg, *args]) - - def destroy_initvm(): - with contextlib.suppress(Exception): - initvm_func('stop') - with contextlib.suppress(Exception): - initvm_func('destroy') +from elbepack.tests import xml_test_files - if use_initvm == 'existing': - yield initvm_func - return - try: - initvm_func('create', '--fail-on-warning') - except Exception as e: - # If the fixture setup fails, pytest will try to create the fixture for - # each test. This is very slow and unlikely to work, so remember the failure. - def error_func(*args, _initvm_exception, **kwargs): - raise RuntimeError('initvm setup failed') from _initvm_exception - destroy_initvm() - yield functools.partial(error_func, _initvm_exception=e) - else: - try: - yield initvm_func - finally: - destroy_initvm() +CHECK_BUILD_VARIANTS = ('schema', 'cdrom', 'img', 'sdk') - -def _delete_project(uuid): - with contextlib.suppress(Exception): - run_elbe_subcommand(['control', 'del_project', uuid]) +_EXTENDED_XML = ( + pathlib.Path('tests') / 'base-extended' / 'simple-validation' / 'image-extended.xml' +) @pytest.fixture(scope='module', params=xml_test_files('simple'), ids=lambda f: f.name) -def simple_build(request, initvm, tmp_path_factory): - build_dir = tmp_path_factory.mktemp('build_dir') - prj = build_dir / 'uuid.prj' - - initvm( - 'submit', request.param, - '--output', build_dir, - '--keep-files', '--build-sdk', - '--writeproject', prj, - ) - - uuid = prj.read_text() - - with contextlib.redirect_stdout(io.StringIO()) as stdout: - run_elbe_subcommand(['control', 'list_projects']) - - if uuid not in stdout.getvalue(): - raise RuntimeError('Project was not created') - - yield build_dir - - _delete_project(uuid) +def simple_build(request, tmp_path_factory, build_driver): + workdir = tmp_path_factory.mktemp('build_dir') + return build_driver.submit(request, request.param, workdir, build_sdk=True) @pytest.mark.slow -@pytest.mark.parametrize('check_build', ('schema', 'cdrom', 'img', 'sdk')) +@pytest.mark.parametrize('check_build', CHECK_BUILD_VARIANTS) def test_simple_build(simple_build, check_build): run_elbe_subcommand(['check-build', check_build, simple_build]) @pytest.mark.slow -def test_rebuild(initvm, simple_build, tmp_path_factory): +def test_rebuild(build_driver, simple_build, tmp_path_factory): build_dir = tmp_path_factory.mktemp('build_dir') - - initvm( - 'submit', '--skip-build-source', - '--output', build_dir, - simple_build / 'bin-cdrom.iso', - ) + build_driver.rebuild(simple_build / 'bin-cdrom.iso', build_dir) @pytest.mark.slow @@ -110,65 +44,17 @@ def test_check_updates(simple_build): run_elbe_subcommand(['check_updates', simple_build / 'source.xml']) -def _prjrepo_list_packages(uuid): - with contextlib.redirect_stdout(io.StringIO()) as stdout: - run_elbe_subcommand(['prjrepo', 'list_packages', uuid]) - - return stdout.getvalue() - - -@pytest.mark.slow -@parametrize_xml_test_files('xml', 'pbuilder') -def test_pbuilder_build(initvm, xml, tmp_path, request): - build_dir = tmp_path - prj = build_dir / 'uuid.prj' - - run_elbe_subcommand(['pbuilder', 'create', '--xmlfile', xml, '--writeproject', prj]) - - uuid = prj.read_text() - request.addfinalizer(lambda: _delete_project(uuid)) - - # Not necessary, to test the command. - run_elbe_subcommand(['pbuilder', 'update', '--project', uuid]) - run_elbe_subcommand(['control', 'wait_busy', uuid]) - - assert _prjrepo_list_packages(uuid) == '' - - for package in ['libgpio', 'gpiotest']: - subprocess.run(['git', 'clone', f'https://github.com/Linutronix/{package}.git'], - check=True, cwd=build_dir) - run_elbe_subcommand(['pbuilder', 'build', '--project', uuid, - '--source', build_dir.joinpath(package), - '--output', build_dir.joinpath('out')]) - - assert _prjrepo_list_packages(uuid) == ( - 'gpiotest_1.0_amd64.deb\n' - 'libgpio-dev_3.0.1_amd64.deb\n' - 'libgpio3-dbgsym_3.0.1_amd64.deb\n' - 'libgpio3_3.0.1_amd64.deb\n' - ) - - run_elbe_subcommand(['prjrepo', 'upload_pkg', uuid, here / 'equivs-dummy_1.0_all.deb']) - - assert _prjrepo_list_packages(uuid) == ( - 'equivs-dummy_1.0_all.deb\n' - 'gpiotest_1.0_amd64.deb\n' - 'libgpio-dev_3.0.1_amd64.deb\n' - 'libgpio3-dbgsym_3.0.1_amd64.deb\n' - 'libgpio3_3.0.1_amd64.deb\n' - ) - - @pytest.mark.slow -def test_base_extended_build(simple_build, initvm, tmp_path): - tests_dir = pathlib.Path('tests') / 'base-extended' / 'simple-validation' - extended_xml_path = tests_dir / 'image-extended.xml' +def test_base_extended_build(request, build_driver, simple_build, tmp_path): base_build_image = simple_build / 'base-rootfs.tgz' - extended_build = tmp_path / 'extended-build' if not base_build_image.exists(): pytest.skip('No base image tarball was produced') - initvm('submit', '--output', extended_build, '--skip-build-bin', '--skip-build-sources', - '--base-image', base_build_image, extended_xml_path) - run_elbe_subcommand(['check-build', 'img', extended_build]) + extended_build = tmp_path / 'extended-build' + extended_build.mkdir() + build_dir = build_driver.submit( + request, _EXTENDED_XML, extended_build, + skip_cdrom=True, base_image=base_build_image, + ) + run_elbe_subcommand(['check-build', 'img', build_dir]) diff --git a/elbepack/tests/test_xml_container.py b/elbepack/tests/test_xml_container.py new file mode 100644 index 000000000..22157fdf4 --- /dev/null +++ b/elbepack/tests/test_xml_container.py @@ -0,0 +1,150 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2026 Linutronix GmbH + +import os +import pathlib +import shutil +import subprocess + +import pytest + +from elbepack.buildsubmitaction import extract_cdrom +from elbepack.rootcheck import xml_needs_rootful +from elbepack.tests.test_xml import ( # noqa: F401 + simple_build, + test_base_extended_build, + test_check_updates, + test_rebuild, + test_simple_build, +) +from elbepack.treeutils import etree + +_IMAGE_NAME = 'elbe-buildenv-image' +_REPO_ROOT = pathlib.Path(__file__).parent.parent.parent +_CONTAINERFILE_DIR = _REPO_ROOT / 'contrib' / 'containerfile' +_TESTS_ROOT = _REPO_ROOT / 'tests' + + +@pytest.fixture(scope='module') +def elbe_buildenv_image(): + subprocess.run( + ['make', 'build-local', f'BUILD_DIR={_REPO_ROOT}'], + cwd=_CONTAINERFILE_DIR, check=True) + + return _IMAGE_NAME + + +def _get_build_container_opts(needs_rootful): + opts = [] + + if os.geteuid() == 0: + if needs_rootful: + opts.extend([ + '--cap-add', 'SYS_ADMIN', + '--cap-add', 'MKNOD', + '--device-cgroup-rule', 'b *:* rmw', + ]) + else: + opts.extend(['--userns', 'auto']) + + opts.extend(['--security-opt', 'apparmor=unconfined']) + + if shutil.which('slirp4netns'): + opts.extend(['--network', 'slirp4netns']) + else: + opts.extend(['--network', 'host']) + + return opts + + +def _run_build(elbe_buildenv_image, workdir, xml_name, build_args=(), base_image=None, + source_dir=None): + """Run ELBE build in container, skipping if loop devices are required but unavailable.""" + input_dir = workdir if source_dir is None else source_dir + xml_path = input_dir / xml_name + + if xml_name.endswith('.iso'): + # rebuild-from-iso: extract source.xml for the rootful pre-check, the same + # way elbe build itself does internally for iso rebuilds. + extracted = extract_cdrom(xml_path) + rootcheck_xml = extracted.fname('source.xml') + else: + rootcheck_xml = xml_path + + xml = etree(rootcheck_xml) + needs_rootful = bool(xml_needs_rootful(xml)) + + extra_opts = _get_build_container_opts(needs_rootful) + + if needs_rootful and os.geteuid() != 0: + pytest.skip( + 'XML file requires rootful container. Rerun as root (e.g. `sudo pytest ...`)' + ) + + mounts = [] + if source_dir is None: + xml_container_path = f'/work/{xml_name}' + else: + mounts += ['-v', f'{_TESTS_ROOT.resolve()}:/src:z,ro'] + unresolved_xml_path = xml_path.parent.resolve() / xml_path.name + xml_container_path = f'/src/{unresolved_xml_path.relative_to(_TESTS_ROOT.resolve())}' + + if xml_path.is_symlink(): + mounts += ['-v', f'{xml_path.resolve()}:{xml_container_path}:z,ro'] + + build_cmd = [ + 'podman', 'run', '--rm', + *mounts, + '-v', f'{workdir}:/work:Z,U', + *extra_opts, + elbe_buildenv_image, + 'elbe', 'build', xml_container_path, '--build-dir', '/work/build', + *build_args, + ] + + if base_image: + build_cmd.extend(['--base-image', f'/work/{base_image}']) + + result = subprocess.run(build_cmd, check=False) + + if result.returncode != 0: + pytest.fail(f'ELBE build failed for {xml_name}. See {workdir} for build state') + + build_dir = workdir / 'build' + assert (build_dir / 'source.xml').exists() + assert (build_dir / 'validation.txt').exists() + return build_dir + + +@pytest.fixture(scope='module') +def build_driver(elbe_buildenv_image): + class _ContainerDriver: + def submit( + self, request, xml_file, build_dir, *, + build_sdk=False, skip_cdrom=False, base_image=None, + ): + build_args = [] + if skip_cdrom: + build_args += ['--skip-build-bin', '--skip-build-sources'] + if build_sdk: + build_args.append('--build-sdk') + + base_image_name = None + if base_image: + build_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(base_image, build_dir) + base_image_name = base_image.name + + return _run_build( + elbe_buildenv_image, build_dir, xml_file.name, + build_args=build_args, base_image=base_image_name, + source_dir=xml_file.parent, + ) + + def rebuild(self, iso_path, build_dir): + shutil.copy(iso_path, build_dir) + _run_build(elbe_buildenv_image, build_dir, iso_path.name, + build_args=['--skip-build-sources']) + + return _ContainerDriver() diff --git a/elbepack/tests/test_xml_initvm.py b/elbepack/tests/test_xml_initvm.py new file mode 100644 index 000000000..432d3d42a --- /dev/null +++ b/elbepack/tests/test_xml_initvm.py @@ -0,0 +1,161 @@ +# ELBE - Debian Based Embedded Rootfilesystem Builder +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2020 Linutronix GmbH + +import contextlib +import functools +import io +import pathlib +import subprocess + +import pytest + +from elbepack.main import run_elbe_subcommand +from elbepack.tests import parametrize_xml_test_files +from elbepack.tests.test_xml import ( # noqa: F401 + simple_build, + test_base_extended_build, + test_check_updates, + test_rebuild, + test_simple_build, +) + + +here = pathlib.Path(__file__).parent + + +@pytest.fixture(scope='module') +def initvm(tmp_path_factory, request): + initvm_dir = tmp_path_factory.mktemp('initvm-') / 'initvm' + use_initvm = request.config.getoption('--elbe-use-initvm') + + if use_initvm in {'libvirt', 'existing'}: + qemu_arg = [] + elif use_initvm == 'qemu': + qemu_arg = ['--qemu'] + else: + raise ValueError(use_initvm) + + def initvm_func(subcmd, *args): + run_elbe_subcommand(['initvm', subcmd, '--directory', initvm_dir, *qemu_arg, *args]) + + def destroy_initvm(): + with contextlib.suppress(Exception): + initvm_func('stop') + with contextlib.suppress(Exception): + initvm_func('destroy') + + if use_initvm == 'existing': + yield initvm_func + return + + try: + initvm_func('create', '--fail-on-warning') + except Exception as e: + # If the fixture setup fails, pytest will try to create the fixture for + # each test. This is very slow and unlikely to work, so remember the failure. + def error_func(*args, _initvm_exception, **kwargs): + raise RuntimeError('initvm setup failed') from _initvm_exception + destroy_initvm() + yield functools.partial(error_func, _initvm_exception=e) + else: + try: + yield initvm_func + finally: + destroy_initvm() + + +def _delete_project(uuid): + with contextlib.suppress(Exception): + run_elbe_subcommand(['control', 'del_project', uuid]) + + +@pytest.fixture(scope='module') +def build_driver(initvm): + class _InitvmDriver: + def submit( + self, request, xml_file, build_dir, *, + build_sdk=False, skip_cdrom=False, base_image=None, + ): + prj = build_dir / 'uuid.prj' + + args = ['submit', xml_file, '--output', build_dir, + '--keep-files', '--writeproject', prj] + if build_sdk: + args.append('--build-sdk') + if skip_cdrom: + args += ['--skip-build-bin', '--skip-build-sources'] + if base_image: + args += ['--base-image', base_image] + + initvm(*args) + + uuid = prj.read_text() + + with contextlib.redirect_stdout(io.StringIO()) as stdout: + run_elbe_subcommand(['control', 'list_projects']) + + if uuid not in stdout.getvalue(): + raise RuntimeError('Project was not created') + + request.addfinalizer(lambda: _delete_project(uuid)) + + return build_dir + + def rebuild(self, iso_path, build_dir): + initvm( + 'submit', '--skip-build-source', + '--output', build_dir, + iso_path, + ) + + return _InitvmDriver() + + +def _prjrepo_list_packages(uuid): + with contextlib.redirect_stdout(io.StringIO()) as stdout: + run_elbe_subcommand(['prjrepo', 'list_packages', uuid]) + + return stdout.getvalue() + + +@pytest.mark.slow +@parametrize_xml_test_files('xml', 'pbuilder') +def test_pbuilder_build(initvm, xml, tmp_path, request): + build_dir = tmp_path + prj = build_dir / 'uuid.prj' + + run_elbe_subcommand(['pbuilder', 'create', '--xmlfile', xml, '--writeproject', prj]) + + uuid = prj.read_text() + request.addfinalizer(lambda: _delete_project(uuid)) + + # Not necessary, to test the command. + run_elbe_subcommand(['pbuilder', 'update', '--project', uuid]) + run_elbe_subcommand(['control', 'wait_busy', uuid]) + + assert _prjrepo_list_packages(uuid) == '' + + for package in ['libgpio', 'gpiotest']: + subprocess.run(['git', 'clone', f'https://github.com/Linutronix/{package}.git'], + check=True, cwd=build_dir) + run_elbe_subcommand(['pbuilder', 'build', '--project', uuid, + '--source', build_dir.joinpath(package), + '--output', build_dir.joinpath('out')]) + + assert _prjrepo_list_packages(uuid) == ( + 'gpiotest_1.0_amd64.deb\n' + 'libgpio-dev_3.0.1_amd64.deb\n' + 'libgpio3-dbgsym_3.0.1_amd64.deb\n' + 'libgpio3_3.0.1_amd64.deb\n' + ) + + run_elbe_subcommand(['prjrepo', 'upload_pkg', uuid, here / 'equivs-dummy_1.0_all.deb']) + + assert _prjrepo_list_packages(uuid) == ( + 'equivs-dummy_1.0_all.deb\n' + 'gpiotest_1.0_amd64.deb\n' + 'libgpio-dev_3.0.1_amd64.deb\n' + 'libgpio3-dbgsym_3.0.1_amd64.deb\n' + 'libgpio3_3.0.1_amd64.deb\n' + ) diff --git a/elbepack/updated.py b/elbepack/updated.py index 0d9fa2ad2..1a28865f4 100644 --- a/elbepack/updated.py +++ b/elbepack/updated.py @@ -31,7 +31,7 @@ ElbeInstallProgress, ElbeOpProgress, ) -from elbepack.egpg import unsign_file +from elbepack.egpg import TARGET_GNUPG_HOME, unsign_file from elbepack.paths import ( DOWNGRADE_ALLOWED_FILE, ELBE_CACHE_DIR, POST_SCRIPT, PRE_SCRIPT, UPDATE_STATE_FILE, @@ -571,7 +571,7 @@ def handle_update_file(upd_file, status, remove=False): _, extension = os.path.splitext(upd_file) if extension == '.gpg': - fname = unsign_file(upd_file) + fname = unsign_file(upd_file, TARGET_GNUPG_HOME) if remove: os.remove(upd_file) if fname: diff --git a/elbepack/updatepkg.py b/elbepack/updatepkg.py index 6ecdad4ec..3cd24c8d9 100644 --- a/elbepack/updatepkg.py +++ b/elbepack/updatepkg.py @@ -105,7 +105,7 @@ def gen_update_pkg(project, xml_filename, upd_filename, if xml_filename: repodir = os.path.join(update, 'repo') - repo = UpdateRepo(xml, repodir) + repo = UpdateRepo(xml, repodir, os.path.join(project.builddir, 'gnupg')) for fname in fnamelist: path = os.path.join( diff --git a/newsfragments/+elbe-build-command.feature.rst b/newsfragments/+elbe-build-command.feature.rst new file mode 100644 index 000000000..3b1c1e5c6 --- /dev/null +++ b/newsfragments/+elbe-build-command.feature.rst @@ -0,0 +1,4 @@ +Add a new ``elbe build`` command as a VM-free alternative to ``elbe initvm submit``. +It drives the project manager directly in-process, without any daemon or SOAP +communication, and is meant to be used inside a container (or any other environment) +that already provides the isolation an initvm would otherwise provide. diff --git a/newsfragments/+vmless-build-container.feature.rst b/newsfragments/+vmless-build-container.feature.rst new file mode 100644 index 000000000..5bd0ca980 --- /dev/null +++ b/newsfragments/+vmless-build-container.feature.rst @@ -0,0 +1,2 @@ +Provide a container (``contrib/containerfile``) that builds ELBE images with the +new ``elbe build`` command, without spawning an initvm.