diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index 9130c973b..01f67afb8 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -62,9 +62,11 @@ cd "$BASEDIR" gzip -dc $SOURCE_TARBALL | tar -xf - ln -s cfengine-3* core -log_debug "UNPACKING MASTERFILES TARBALL AND SYMLINKING masterfiles/" +log_debug "UNPACKING MASTERFILES TARBALL: $MASTERFILES_TARBALL AND SYMLINKING masterfiles/" # shellcheck disable=SC2086 # > Double quote to prevent globbing and word splitting. # We want globbing here gzip -dc $MASTERFILES_TARBALL | tar -xf - -ln -s cfengine-masterfiles-* masterfiles +# the masterfiles tarball would match cfengine-masterfiles-* so find the unpacked dir specifically +_cfmpfdir=$(find -type d -name 'cfengine-masterfiles-*') +ln -s "$_cfmpfdir" masterfiles diff --git a/ci/cfengine-build-host-setup.cf b/ci/cfengine-build-host-setup.cf index 46e4b2890..a8205af36 100644 --- a/ci/cfengine-build-host-setup.cf +++ b/ci/cfengine-build-host-setup.cf @@ -378,6 +378,11 @@ bundle agent cfengine_build_host_setup comment => "note: centos-7 has installed instead of --installed argument, and that works on rhel-8 and rhel-9 so go with the sub-command instead of option"; commands: + sshd_hardened:: + 'kill -1 $(pgrep -f "sshd -D")' + handle => "sshd_restarted", + contain => in_shell, + comment => "Reload sshd config with SIGHUP(1) to apply hardened configuration"; have_tmp_mount:: "mount -o remount,size=5G /tmp" comment => "We could check if /tmp was size 5G but not worth the trouble since this remount call just sets the maximum size of the tmpfs in virtual memory.", @@ -651,12 +656,6 @@ jenkins ALL=NOPASSWD: /usr/bin/podman comment => "Verify KbdInteractiveAuthentication (OpenSSH 8.7+) or ChallengeResponseAuthentication (older) is disabled"; services: - sshd_hardened:: - "$(sshd_service_name)" - service_policy => "restart", - handle => "sshd_restarted", - comment => "Restart sshd to apply hardened configuration"; - any:: "fail2ban" service_policy => "start", diff --git a/ci/initialize-build-host.sh b/ci/initialize-build-host.sh index 90bbc9392..7080d2dd4 100644 --- a/ci/initialize-build-host.sh +++ b/ci/initialize-build-host.sh @@ -136,9 +136,10 @@ do sleep 10 done -echo '========================================= PRINTING CLOUD-INIT LOG ===================================================' -sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true -echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================' +# TODO, instead of printing this out ALWAYS, print it out only in case of errors ENT-14372 +#echo '========================================= PRINTING CLOUD-INIT LOG ===================================================' +#sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true +#echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================' if [ $attempts -le 0 ] then @@ -147,9 +148,10 @@ then exit 1 fi -echo '=========================================== CURRENT ENVIRONMENT =====================================================' -export -echo '========================================= CURRENT ENVIRONMENT END ===================================================' +# TODO only print current environment on errors, maybe save the environment NOW and then show a diff at ERROR +#echo '=========================================== CURRENT ENVIRONMENT =====================================================' +#export +#echo '========================================= CURRENT ENVIRONMENT END ===================================================' # Disable TTY requirement. This normally happens in initialize-user-data.sh, but # for hosts that do not support cloud user data, it may not have happened diff --git a/ci/setup-cfengine-build-host.sh b/ci/setup-cfengine-build-host.sh index cac131fde..0928b5695 100755 --- a/ci/setup-cfengine-build-host.sh +++ b/ci/setup-cfengine-build-host.sh @@ -26,6 +26,9 @@ rm -rf cfengine-masterfiles* function cleanup() { set -e + set -x + [ -f /var/log/messages ] && tail /var/log/messages + command -v journalctl >/dev/null && journalctl | grep -P '(error|fail)' if command -v apt >/dev/null 2>&1; then # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false # Replace systemctl with a no-op stub that always succeeds. We can't @@ -33,7 +36,7 @@ function cleanup() { # single multi-call uutils binary that dispatches on argv[0], so it # would fail with "coreutils: unknown program 'systemctl'". rm -f /bin/systemctl - printf '#!/bin/sh\nexit 0\n' > /bin/systemctl + printf '#!/bin/sh\nexit 0\n' >/bin/systemctl chmod +x /bin/systemctl apt remove -y cfengine-nova || true elif command -v yum >/dev/null 2>&1; then @@ -95,9 +98,12 @@ if [ -f /etc/os-release ]; then yum update --assumeyes alias software='yum install --assumeyes' elif grep -q debian /etc/os-release; then + DEBIAN_FRONTEND=noninteractive apt update + # sometimes the /boot partition is too small to handle kernel upgrade regenerations of initrd and related files on ubuntu, so allow failure first DEBIAN_FRONTEND=noninteractive apt upgrade --yes || true DEBIAN_FRONTEND=noninteractive apt autoremove --yes + # and now perform the upgrade a second time after hopefully autoremove cleans up /boot partition of kernel files that cause failure DEBIAN_FRONTEND=noninteractive apt upgrade --yes DEBIAN_FRONTEND=noninteractive apt autoremove --yes @@ -141,8 +147,13 @@ fi # Here we start replacing the use of CFEngine policy with scripts. See ENT-14330 if [ -f /etc/cfengine-bootstrap-pr-host.flag ]; then - "$thisdir"/setup-bootstrap-host.sh - exit + "$thisdir"/setup-bootstrap-host.sh + exit +fi + +if [ -f /etc/cfengine-containers-host.flag ]; then + "$thisdir"/setup-ci-host.sh + exit fi if grep -q ubuntu /etc/os-release; then @@ -217,6 +228,23 @@ if ! /var/cfengine/bin/cf-agent -V 2>/dev/null; then else _VERSION="" fi + + # ENT-14373, migrate any cf-remote cache/config files to avoid dirs_exist_ok problems on old pythons + if [ -d "$HOME"/.cfengine/cf-remote/json ]; then + mkdir -p "$HOME"/.cache/cfengine/cf-remote + mv "$HOME"/.cfengine/cf-remote/json "$HOME"/.cache/cfengine/cf-remote/ + fi + if [ -d "$HOME"/.cfengine/cf-remote/packages ]; then + mkdir -p "$HOME"/.cache/cfengine/cf-remote + mv "$HOME"/.cfengine/cf-remote/packages "$HOME"/.cache/cfengine/cf-remote/ + fi + if [ -d "$HOME"/.cfengine/cf-remote ]; then + mkdir -p "$HOME"/.config/cfengine/cf-remote + mv "$HOME"/.cfengine/cf-remote/* "$HOME"/.config/cfengine/cf-remote/ + fi + + # We are passing a two-token string and need it to stay two tokens for proper argument parsing in $_VERSION + # shellcheck disable=SC2086 cf-remote --log-level info $_VERSION install --clients localhost || true fi diff --git a/ci/setup-ci-host.sh b/ci/setup-ci-host.sh new file mode 100755 index 000000000..c88f5d7a3 --- /dev/null +++ b/ci/setup-ci-host.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +set -e +shopt -s expand_aliases +thisdir="$(dirname "$0")" + +packages="" # a space separated list of packages to install +function add-pkg() { + packages+=" $*" +} + +# we setup some vars for platform versions to make it easier to make choice later +# default version is 0 so that a check can be [ "$debian" -gt "12" ] and that will skip non-debians and such +redhat=0 +debian=0 +ubuntu=0 +suse=0 +# shellcheck disable=SC2034 +solaris=0 +# shellcheck disable=SC2034 +hpux=0 +# shellcheck disable=SC2034 +aix=0 + +if [ -f /etc/os-release ]; then + source /etc/os-release + if grep -q rhel /etc/os-release; then + yum update --assumeyes + alias packages='yum install --assumeyes' + redhat="$VERSION_ID" + elif grep -q debian /etc/os-release; then + alias packages='DEBIAN_FRONTEND=noninteractive apt install --yes' + debian="$VERSION_ID" + elif grep -q suse /etc/os-release; then + alias packages='zypper install -y' + # shellcheck disable=SC2034 + suse="$VERSION_ID" + else + echo "Unknown platform ID $ID. Need this information in order to update/upgrade distribution packages." + exit 1 + fi +elif [ -f /etc/redhat-release ]; then + alias packages='yum install --assumeyes' + # shellcheck disable=SC1091 + source /etc/redhat-release + redhat="$VERSION_ID" +else + echo "No /etc/os-release or /etc/redhat-release so cant determine platform." + exit 1 +fi + +if [ -f /etc/cfengine-containers-host.flag ]; then + if [ "$debian" -ge "12" ]; then + # in jenkins, CONTAINER labeled nodes are capable of running container builds like valgrind-check and static-check + add-pkg unzip # linux-install-groovy.sh needs unzip to unpack the groovy distribution archive + add-pkg buildah + add-pkg jq + add-pkg make + add-pkg parallel + add-pkg podman + if ! command -v groovy; then + bash "$thisdir"/linux-install-groovy.sh + fi + + # NOPASSWD is needed for various tools related to container jobs + rm -rf /etc/sudoers.d/999-local + cat >/etc/sudoers.d/999-local </dev/null; then + if [ ! -f /etc/cfengine-containers-host.flag ]; then + sysctl kernel.core_pattern='|/lib/systemd/systemd-coredump %p %u %g %s %t %e' + fi +fi + +"$thisdir"/linux-install-jdk.sh # the script should skip if sufficient java is already installed + +# leech2 build toolchain host +if [ "$ubuntu" -ge 20 ] || [ "$debian" -ge 12 ] || [ "$redhat" -ge 7 ]; then + "$thisdir"/linux-install-protobuf.sh + # TODO if mingw then pass along x86_64-pc-windows-gnu as an arg to install rust + "$thisdir"/linux-install-rust.sh +fi + +if [ "$redhat" -ge 7 ]; then + yum groups install -y 'Development Tools' +fi + +if [ "$redhat" = 8 ]; then + sudo rpm -iv https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-1.23-1.fc29.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-libs-1.23-1.fc29.x86_64.rpm +fi + +if [ "$redhat" -gt 7 ]; then + yum install --assumeyes https://dl.fedoraproject.org/pub/epel/epel-release-latest-"$redhat".noarch.rpm +fi