From e0785c1c3d6af51d7f502c2b02139dc4a2203320 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 11:45:03 -0400 Subject: [PATCH 01/38] Use matrix with parametrized ci.yaml for future os upgrades --- .github/workflows/ci.yml | 16 +++++++++------- script/cibuild-create-packages | 11 ++++++++--- script/cibuild-create-packages-focal | 28 ---------------------------- script/helpers/folding.sh | 6 +++--- 4 files changed, 20 insertions(+), 41 deletions(-) delete mode 100755 script/cibuild-create-packages-focal diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59845efb..3527e735 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,17 +7,19 @@ permissions: jobs: package-build: - runs-on: ubuntu-latest - + strategy: + fail-fast: false + matrix: + distro: [focal] steps: - uses: actions/checkout@v6 - - name: Run package build focal - run: script/cibuild-create-packages-focal + - name: Run package build ${{ matrix.distro }} + run: script/cibuild-create-packages ${{ matrix.distro }} - name: Tar files - run: tar -cvf glb-director.tar $GITHUB_WORKSPACE/tmp/build + run: tar -cvf glb-director-${{ matrix.distro }}.tar $GITHUB_WORKSPACE/tmp/build - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: glb-director - path: glb-director.tar \ No newline at end of file + name: glb-director-${{ matrix.distro }} + path: glb-director-${{ matrix.distro }}.tar diff --git a/script/cibuild-create-packages b/script/cibuild-create-packages index 8cf9fa34..e0f3dfd0 100755 --- a/script/cibuild-create-packages +++ b/script/cibuild-create-packages @@ -7,9 +7,14 @@ cd "$(dirname "$0")/.." . script/helpers/folding.sh +DISTRO="$1" +if [ -z "$DISTRO" ]; then + DISTRO="focal" +fi + begin_fold "Preparing Docker build environment" ( - docker build -t glb-director-build-stretch -f script/Dockerfile.stretch script + docker build -t glb-director-build-$DISTRO -f "script/Dockerfile.$DISTRO" script ) end_fold @@ -21,8 +26,8 @@ begin_fold "Building packages" docker run --rm \ --volume "$HOSTPATH":/glb-director \ - "glb-director-build-stretch" \ + "glb-director-build-$DISTRO" \ bash -c "cd /glb-director && make BUILDDIR=/glb-director/tmp/build clean mkdeb" ) -end_fold +end_fold \ No newline at end of file diff --git a/script/cibuild-create-packages-focal b/script/cibuild-create-packages-focal deleted file mode 100755 index adc26467..00000000 --- a/script/cibuild-create-packages-focal +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -e - -HOSTPATH=$(cd $(dirname "$0") && cd .. && pwd) -cd "$(dirname "$0")/.." - -. script/helpers/folding.sh - -begin_fold "Preparing Docker build environment" -( - docker build -t glb-director-build-focal -f script/Dockerfile.focal script -) -end_fold - -begin_fold "Building packages" -( - # prep - rm -rf tmp/build/ - mkdir -p tmp/build/ - - docker run --rm \ - --volume "$HOSTPATH":/glb-director \ - "glb-director-build-focal" \ - bash -c "cd /glb-director && - make BUILDDIR=/glb-director/tmp/build clean mkdeb" -) -end_fold diff --git a/script/helpers/folding.sh b/script/helpers/folding.sh index 0c387304..774bf222 100644 --- a/script/helpers/folding.sh +++ b/script/helpers/folding.sh @@ -1,9 +1,9 @@ #!/bin/bash begin_fold() { - echo "%%%FOLD {$*}%%%" + echo "::group::$*" } end_fold() { - echo "%%%END FOLD%%%" -} + echo "::endgroup::" +} \ No newline at end of file From 4a524ecef458b97ba48711932027e9e24c50dabf Mon Sep 17 00:00:00 2001 From: Mark Penny <163456289+mpenny-github@users.noreply.github.com> Date: Tue, 19 May 2026 12:02:31 -0400 Subject: [PATCH 02/38] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- script/cibuild-create-packages | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/script/cibuild-create-packages b/script/cibuild-create-packages index e0f3dfd0..eaae0bcd 100755 --- a/script/cibuild-create-packages +++ b/script/cibuild-create-packages @@ -12,9 +12,15 @@ if [ -z "$DISTRO" ]; then DISTRO="focal" fi +DOCKERFILE="script/Dockerfile.$DISTRO" +if [ ! -f "$DOCKERFILE" ]; then + echo "Error: unsupported distro '$DISTRO' or missing Dockerfile '$DOCKERFILE'." >&2 + exit 1 +fi + begin_fold "Preparing Docker build environment" ( - docker build -t glb-director-build-$DISTRO -f "script/Dockerfile.$DISTRO" script + docker build -t glb-director-build-$DISTRO -f "$DOCKERFILE" script ) end_fold From aaf989c6b9b099b761c2fd97cdcae355fe35504a Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 12:16:29 -0400 Subject: [PATCH 03/38] Enfore that the docker container should use amd64 version (even when run on arm64 hosts. Pin to a specific focal version to harden security --- script/Dockerfile.focal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/Dockerfile.focal b/script/Dockerfile.focal index 2208dbb5..a5287c0a 100644 --- a/script/Dockerfile.focal +++ b/script/Dockerfile.focal @@ -1,4 +1,4 @@ -FROM ubuntu:focal +FROM --platform=linux/amd64 ubuntu:focal@sha256:8feb4d8ca5354def3d8fce243717141ce31e2c428701f6682bd2fafe15388214 RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries From ab35fa7e54e847d859b1d968d586204a9af36b3d Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 12:24:32 -0400 Subject: [PATCH 04/38] Create containerized testing framework --- .github/workflows/test.yml | 72 ++++++++++++++++++++++++++++++++++++++ script/test | 18 ++++++++++ script/test-local | 43 +++++++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 script/test create mode 100755 script/test-local diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..60cfe3b2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,72 @@ +name: Tests + +on: + push: + branches: + - main + - master + pull_request: + branches: + - main + - master + +permissions: + contents: read + +jobs: + build-images: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + distro: [focal] + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build image + run: | + docker build --file script/Dockerfile.${{ matrix.distro }} --tag glb-director-build-${{ matrix.distro }}:latest . + docker save glb-director-build-${{ matrix.distro }}:latest --output glb-director-build-${{ matrix.distro }}.tar + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.distro }} + path: glb-director-build-${{ matrix.distro }}.tar + retention-days: 1 + + + + test: + needs: build-images + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-suite: [director, director-xdp, healthcheck, redirect] + distro: [focal] + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Download image artifact + uses: actions/download-artifact@v4 + with: + name: build-${{ matrix.distro }} + + - name: Load Docker image + run: | + docker load --input glb-director-build-${{ matrix.distro }}.tar + + - name: Run test suite in container + run: | + docker run --rm \ + --privileged \ + --volume $(pwd):/workspace \ + --workdir /workspace \ + glb-director-build-${{ matrix.distro }}:latest \ + bash -c "cd /workspace/src/glb-${{ matrix.test-suite }} && script/test" \ No newline at end of file diff --git a/script/test b/script/test new file mode 100644 index 00000000..020dcb99 --- /dev/null +++ b/script/test @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Runs a Clang static analysis build (scan-build) over the project to detect +# potential bugs at compile time. Automatically detects the available version +# of scan-build installed in the environment. +set -euo pipefail + +if command -v scan-build-10 >/dev/null 2>&1; then + SCAN_BUILD=scan-build-10 +elif command -v scan-build >/dev/null 2>&1; then + SCAN_BUILD=scan-build +elif command -v scan-build-14 >/dev/null 2>&1; then + SCAN_BUILD=scan-build-14 +else + echo "scan-build is not installed" >&2 + exit 1 +fi + +"$SCAN_BUILD" make diff --git a/script/test-local b/script/test-local new file mode 100755 index 00000000..76fb2ec3 --- /dev/null +++ b/script/test-local @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +DISTRO="${1}" + +if [[ -z "$DISTRO" ]]; then + echo "Usage: $0 " + echo " e.g. $0 focal" + echo " e.g. $0 noble" + exit 1 +fi + +HOSTPATH=$(cd "$(dirname "$0")/.." && pwd) +IMAGE="glb-director-build-${DISTRO}:latest" +DOCKERFILE="script/Dockerfile.${DISTRO}" + +if [[ ! -f "${HOSTPATH}/${DOCKERFILE}" ]]; then + echo "ERROR: Dockerfile not found: ${DOCKERFILE}" + exit 1 +fi + +cd "$HOSTPATH" + +echo "==> Building Docker image for ${DISTRO}..." +docker build --platform linux/amd64 --file "${DOCKERFILE}" --tag "${IMAGE}" . + +TEST_SUITES=(director director-xdp healthcheck redirect) + +for suite in "${TEST_SUITES[@]}"; do + echo "" + echo "==> Running test suite: glb-${suite} (${DISTRO})" + docker run --rm \ + --platform linux/amd64 \ + --privileged \ + --volume "$(pwd):/workspace" \ + --workdir /workspace \ + "${IMAGE}" \ + bash -c "cd /workspace/src/glb-${suite} && script/test" +done + +echo "" +echo "==> All test suites passed for ${DISTRO}." From 3bbf390ed0e01b08e619598e6eaf8e306800d291 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 13:32:23 -0400 Subject: [PATCH 05/38] Update the Dockerfile with the requirements for running in a container --- script/Dockerfile.focal | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/script/Dockerfile.focal b/script/Dockerfile.focal index a5287c0a..948b23ec 100644 --- a/script/Dockerfile.focal +++ b/script/Dockerfile.focal @@ -30,7 +30,7 @@ RUN gem install rake fpm # XDP # linux-libc-dev must be upgraded to get a bpf.h that matches what we use. the rest match what we do in Vagrant for testing. RUN apt-get update && apt install -y apt-transport-https curl software-properties-common -RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-10 +RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-10 clang-tools-10 # Hack because the kernel headers are not installed in the right place (linuxkit vs generic) RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname -r) @@ -38,3 +38,21 @@ RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname # Hack for C99 math RUN sed -i '1s/^/#define __USE_C99_MATH\n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h RUN sed -i '2s/^/#include \n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h + +# Python test dependencies (scapy/nose etc.) used by the test suites. +RUN apt-get update && apt-get install -y python3 python3-pip python3-dev +COPY requirements.txt /tmp/requirements.txt +RUN pip3 install --no-cache-dir -r /tmp/requirements.txt + +# valgrind is required by the glb-director test suite +RUN apt-get update && apt-get install -y valgrind + +# netcat and jq are required by the glb-healthcheck test suite +RUN apt-get update && apt-get install -y netcat jq + +# DPDK KNI kernel module. Needed at test runtime (`modprobe rte_kni`). +# Building the .ko requires the running kernel's headers and is therefore +# only loaded at container runtime by the test harness on hosts where the +# kernel supports it (e.g. real Linux CI runners). The dkms package is +# installed here so the source/module is available inside the container. +RUN apt-get update && apt-get install -y dpdk-rte-kni-dkms || true From e2fb5d3c8d9d1225f7fd2c0cccebc290bde49fd8 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 13:33:20 -0400 Subject: [PATCH 06/38] Update python tests with python3 syntax --- src/glb-director/tests/glb_test_utils.py | 27 ++++++++++--------- src/glb-director/tests/test_cli_tool.py | 18 ++++++------- .../tests/test_director_classify_v6.py | 2 +- .../tests/test_rendezvous_table.py | 12 ++++----- .../tests/glb_test_remote_snoop.py | 2 +- src/glb-redirect/tests/glb_test_utils.py | 4 +-- .../tests/test_glb_redirect_v4_on_v4.py | 2 +- .../tests/test_glb_redirect_v6_on_v4.py | 2 +- src/scapy-glb-gue/glb_scapy/__init__.py | 2 +- src/scapy-glb-gue/glb_scapy/glb_gue_scapy.py | 2 +- 10 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index e0bae561..ef3f525f 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -18,7 +18,8 @@ import logging logging.getLogger("scapy.runtime").setLevel(logging.ERROR) -from scapy.all import sniff, sendp, Ether, IP, IPv6, L2ListenSocket, MTU, Packet, UDP, TCP, bind_layers, ICMP, ICMPv6PacketTooBig +from scapy.all import sniff, sendp, Ether, IP, IPv6, MTU, Packet, UDP, TCP, bind_layers, ICMP, ICMPv6PacketTooBig, conf +from scapy.arch.linux import L2ListenSocket from pyroute2 import IPRoute, NetlinkError from nose.tools import assert_equals import subprocess, time @@ -79,7 +80,7 @@ def setup(self, iface): stderr=subprocess.STDOUT, ) - print('launched as pid', self.director.pid) + print(('launched as pid', self.director.pid)) ip = IPRoute() @@ -116,7 +117,7 @@ def reload(self): self.director.send_signal(signal.SIGUSR1) def kni(self): - return L2ListenSocket(iface=GLBDirectorTestBase.IFACE_NAME_KNI, promisc=True) + return L2Socket(iface=GLBDirectorTestBase.IFACE_NAME_KNI, promisc=True) class SystemdNotify(object): def __init__(self, unix_path): @@ -194,7 +195,7 @@ def launch_director(self): env=notify_director.updated_env(), ) - print('launched as pid', self.director.pid) + print(('launched as pid', self.director.pid)) notify_director.wait() @@ -299,7 +300,7 @@ def get_initial_director_config(cls): @classmethod def update_running_forwarding_tables(cls, config): - f = open('tests/test-tables.json', 'wb') + f = open('tests/test-tables.json', 'w') f.write(json.dumps(config, indent=4)) f.close() @@ -334,7 +335,7 @@ def setup_class(cls): GLBDirectorTestBase.py_side_mac = dict(ip.link('get', index=ip.link_lookup(ifname=cls.IFACE_NAME_PY))[0]['attrs'])['IFLA_ADDRESS'] - with open('tests/director-config.json', 'wb') as f: + with open('tests/director-config.json', 'w') as f: f.write(json.dumps(cls.get_initial_director_config(), indent=4)) # set up a statsd receiver @@ -353,7 +354,7 @@ def setup_class(cls): GLBDirectorTestBase.backend.setup_pyside(iface=cls.IFACE_NAME_PY) # prepare our listener for return traffic from director - GLBDirectorTestBase.eth_tx = L2ListenSocket(iface=cls.IFACE_NAME_PY, promisc=True) + GLBDirectorTestBase.eth_tx = L2Socket(iface=cls.IFACE_NAME_PY, promisc=True) GLBDirectorTestBase.kni_tx = GLBDirectorTestBase.backend.kni() @classmethod @@ -375,12 +376,12 @@ def sendp(self, *args, **kwargs): sendp(*args, **kwargs) def wait_for_packet(self, iface, condition, timeout_seconds=5): - print('Waiting for packets on', iface.iff, 'with timeout', timeout_seconds) + print(('Waiting for packets on', iface.iff, 'with timeout', timeout_seconds)) try: with timeout(timeout_seconds): while True: packet = iface.recv(MTU) - print(repr(packet)) + print((repr(packet))) if condition(packet): return packet except: @@ -414,7 +415,7 @@ def expect_metrics(self, spec): spec_matches = set() for metric_name, metric_value, metric_type, metric_tags in self.stream_statsd_metrics(timeout=1): metric_key = (metric_name, metric_tags) - print metric_key + print(metric_key) if metric_key in spec: assert spec[metric_key](metric_value), "Metric {} had unexpected value {}".format(metric_key, repr(metric_value)) spec_matches.add(metric_key) @@ -456,7 +457,7 @@ def pkt_sport(self, key, src_addr=None, dst_addr=None, src_port=None, dst_port=N def route_for_packet(self, test_packet, fields): field_data = self._fields_for_packet(test_packet) table = self._table_for_bind(field_data['dst_addr'], field_data['dst_port']) - rt = GLBRendezvousTable(table['seed'].decode('hex')) + rt = GLBRendezvousTable(bytes.fromhex(table['seed'])) hosts = self._hosts_for_table(table) hash_key_bytes = self._key_for_bind(field_data['dst_addr'], field_data['dst_port']) @@ -465,7 +466,7 @@ def route_for_packet(self, test_packet, fields): return rt.forwarding_table_entry(hash_row, hosts)[:2] def _hosts_for_table(self, table): - return map(lambda b: b['ip'], table['backends']) + return [b['ip'] for b in table['backends']] def _table_for_bind(self, dest_ip, dest_port): config = GLBDirectorTestBase.running_forwarding_config @@ -482,7 +483,7 @@ def _key_for_bind(self, dest_ip, dest_port): if table is None: return None else: - return table['hash_key'].decode('hex').rjust(16, '\x00') + return bytes.fromhex(table['hash_key']).rjust(16, b'\x00') def _fields_for_packet(self, packet): ether = packet diff --git a/src/glb-director/tests/test_cli_tool.py b/src/glb-director/tests/test_cli_tool.py index cb3b8abf..3a9f6af8 100644 --- a/src/glb-director/tests/test_cli_tool.py +++ b/src/glb-director/tests/test_cli_tool.py @@ -63,11 +63,11 @@ def write_example_config(self): def get_example_table_reference_implementation(self, table_index): table_config = self.get_example_config()['tables'][table_index] - return GLBRendezvousTable(table_config['seed'].decode('hex')) + return GLBRendezvousTable(bytes.fromhex(table_config['seed'])) def get_example_table_hosts(self, table_index): table_config = self.get_example_config()['tables'][table_index] - return map(lambda b: b['ip'], table_config['backends']) + return [b['ip'] for b in table_config['backends']] def test_generate_configs(self): self.write_example_config() @@ -75,7 +75,7 @@ def test_generate_configs(self): subprocess.check_call(['cli/glb-director-cli', 'build-config', 'tests/test-config.json', 'tests/test-config.bin']) f = open('tests/test-config.bin', 'rb') - assert_equals(f.read(4), 'GLBD') + assert_equals(f.read(4), b'GLBD') num_table_entries = 0x10000 max_num_backends = 0x100 @@ -109,7 +109,7 @@ def test_generate_configs(self): assert_equals(inet_addr, socket.inet_pton(socket.AF_INET6, backend['ip'])) else: assert_equals(inet_family, 1) - assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, backend['ip']).ljust(16, '\x00')) + assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, backend['ip']).ljust(16, b'\x00')) assert_equals(be_state, 1) assert_equals(be_health, 1) @@ -128,14 +128,14 @@ def test_generate_configs(self): assert_equals(ip_bits, 128) else: assert_equals(inet_family, 1) - assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, bind['ip']).ljust(16, '\x00')) + assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, bind['ip']).ljust(16, b'\x00')) assert_equals(ip_bits, 32) assert_equals(bind_port_start, bind['port']) assert_equals(bind_port_end, bind['port']) assert_equals(bind_proto, 6 if bind['proto'] == 'tcp' else 17) # validate hash key for source hashing - assert_equals(f.read(16), table['hash_key'].decode('hex').rjust(16, '\x00')) + assert_equals(f.read(16), bytes.fromhex(table['hash_key']).rjust(16, b'\x00')) # validate table entries for table_index in range(num_table_entries): @@ -149,10 +149,10 @@ def test_generate_configs(self): assert_equals(actual_first_ips, expected_first_ips[:2]) - # forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + # forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') # table = GLBRendezvousTable(forwarding_table_seed) - # assert_equals(table.calculate_forwarding_table_row_seed(0x0000).encode('hex'), '491c53a72df4c837') - # assert_equals(table.calculate_forwarding_table_row_seed(0xffff).encode('hex'), 'f223c0cc65161620') + # assert_equals(table.calculate_forwarding_table_row_seed(0x0000).hex(), '491c53a72df4c837') + # assert_equals(table.calculate_forwarding_table_row_seed(0xffff).hex(), 'f223c0cc65161620') def test_atomic_write_no_temp_file_remains(self): """Verify that no temporary file is left behind after a successful build.""" diff --git a/src/glb-director/tests/test_director_classify_v6.py b/src/glb-director/tests/test_director_classify_v6.py index f40c6208..2d82ad4e 100644 --- a/src/glb-director/tests/test_director_classify_v6.py +++ b/src/glb-director/tests/test_director_classify_v6.py @@ -45,7 +45,7 @@ def test_01_route_classified_v6(self): assert_equals(glb_gue.private_data[0].hops, ['6.7.8.9']) inner_ip = glb_gue.payload - print repr(inner_ip) + print(repr(inner_ip)) assert isinstance(inner_ip, IPv6) # Expecting the inner IPv6 packet assert_equals(inner_ip.src, 'fd91:79d3:d621::1234') assert_equals(inner_ip.dst, 'fdb4:98ce:52d4::42') diff --git a/src/glb-director/tests/test_rendezvous_table.py b/src/glb-director/tests/test_rendezvous_table.py index a891831f..42233331 100644 --- a/src/glb-director/tests/test_rendezvous_table.py +++ b/src/glb-director/tests/test_rendezvous_table.py @@ -22,10 +22,10 @@ class TestGLBRendezvousTable(): def test_row_seeds(self): """GLBRendezvousTable correctly calculates valid row seeds""" - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) - assert_equals(table.calculate_forwarding_table_row_seed(0x0000).encode('hex'), '491c53a72df4c837') - assert_equals(table.calculate_forwarding_table_row_seed(0xffff).encode('hex'), 'f223c0cc65161620') + assert_equals(table.calculate_forwarding_table_row_seed(0x0000).hex(), '491c53a72df4c837') + assert_equals(table.calculate_forwarding_table_row_seed(0xffff).hex(), 'f223c0cc65161620') def test_order_hosts_0000(self): """ @@ -37,7 +37,7 @@ def test_order_hosts_0000(self): 1.1.1.4 6f022ce1ea607e16 """ - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] @@ -54,7 +54,7 @@ def test_order_hosts_ffff(self): 1.1.1.4 a1f610df9fbb2025 """ - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] @@ -71,7 +71,7 @@ def test_order_hosts_bb44(self): 1.1.1.4 0676eaf9cb7d2f85 """ - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] diff --git a/src/glb-redirect/tests/glb_test_remote_snoop.py b/src/glb-redirect/tests/glb_test_remote_snoop.py index 1bc18a58..c010447d 100644 --- a/src/glb-redirect/tests/glb_test_remote_snoop.py +++ b/src/glb-redirect/tests/glb_test_remote_snoop.py @@ -49,7 +49,7 @@ def recv(self, recv_filter, timeout=10): pkt_ether = Ether(pkt_raw) pkt = pkt_ether.payload - if self.debug: print("got packet from {}: {}".format(self.remote_host, repr(pkt))) + if self.debug: print(("got packet from {}: {}".format(self.remote_host, repr(pkt)))) if recv_filter(pkt): if self.debug: print(" -> match!") return pkt diff --git a/src/glb-redirect/tests/glb_test_utils.py b/src/glb-redirect/tests/glb_test_utils.py index f1f303cd..6d4526f6 100644 --- a/src/glb-redirect/tests/glb_test_utils.py +++ b/src/glb-redirect/tests/glb_test_utils.py @@ -25,7 +25,7 @@ def _sendrecv6(self, pkt, **kwargs): s.close() if len(ret) == 0: assert False, "Expected to receive a response packet, but none received." - print "Received packet:", repr(ret[0]) + print("Received packet:", repr(ret[0])) return ret[0] def _sendrecvmany4(self, pkt, **kwargs): @@ -36,7 +36,7 @@ def _sendrecvmany4(self, pkt, **kwargs): if len(ret) == 0: assert False, "Expected to receive a response packet, but none received." for pkt in ret: - print "Received packet:", repr(pkt) + print("Received packet:", repr(pkt)) return ret def _sendrecv4(self, pkt, **kwargs): diff --git a/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py b/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py index e8616817..23b7682b 100644 --- a/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py +++ b/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py @@ -41,7 +41,7 @@ def test_00_icmp_accepted(self): # expect a ICMP echo response back from self.PROXY_HOST (decapsulated) resp_ip = self._sendrecv4(pkt, filter='host {} and icmp'.format(dst)) - print repr(resp_ip) + print(repr(resp_ip)) assert isinstance(resp_ip, IP) assert_equals(resp_ip.src, dst) assert_equals(resp_ip.dst, self.SELF_HOST) diff --git a/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py b/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py index 8a5970d6..84c78831 100644 --- a/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py +++ b/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py @@ -45,7 +45,7 @@ def test_00_icmp_accepted(self): GLBGUE(private_data=GLBGUEChainedRouting(hops=[self.ALT_HOST])) / \ IPv6(src=self.SELF_HOST_V6, dst=self.V4_TO_V6[dst]) / \ ICMPv6EchoRequest() - print repr(pkt) + print(repr(pkt)) # expect a ICMP echo response back from self.PROXY_HOST (decapsulated) resp_ip = self._sendrecv6(pkt, lfilter=lambda p: isinstance(p, IPv6) and isinstance(p.payload, ICMPv6EchoReply)) diff --git a/src/scapy-glb-gue/glb_scapy/__init__.py b/src/scapy-glb-gue/glb_scapy/__init__.py index 213434c5..f23dcde2 100644 --- a/src/scapy-glb-gue/glb_scapy/__init__.py +++ b/src/scapy-glb-gue/glb_scapy/__init__.py @@ -15,4 +15,4 @@ # You should have received a copy of the GNU General Public License # along with scapy-glb-gue. If not, see . -from glb_gue_scapy import GLBGUEChainedRouting, GLBGUE +from .glb_gue_scapy import GLBGUEChainedRouting, GLBGUE diff --git a/src/scapy-glb-gue/glb_scapy/glb_gue_scapy.py b/src/scapy-glb-gue/glb_scapy/glb_gue_scapy.py index beb31019..3c023057 100644 --- a/src/scapy-glb-gue/glb_scapy/glb_gue_scapy.py +++ b/src/scapy-glb-gue/glb_scapy/glb_gue_scapy.py @@ -36,7 +36,7 @@ class GLBGUE(Packet): name = "GLBGUE" fields_desc = [BitField("version", 0, 2), BitField("control_msg", 0, 1), - BitFieldLenField("hlen", None, 5, length_of='private_data', adjust=lambda pkt, x: (x / 4)), + BitFieldLenField("hlen", None, 5, length_of='private_data', adjust=lambda pkt, x: (x // 4)), BitField("protocol", 0, 8), BitField("flags", 0, 16), PacketListField("private_data", [], GLBGUEChainedRouting, length_from=lambda p:p.hlen * 4) From 0225418e74a702dd0b132ce16c7a777abfea371a Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 13:39:17 -0400 Subject: [PATCH 07/38] Add tcpdump to Dockerfile. Skip tests that can't be run locally --- script/Dockerfile.focal | 2 +- src/glb-director/tests/glb_test_utils.py | 14 ++- src/glb-director/tests/test-config.json | 104 +++++++++++------------ 3 files changed, 64 insertions(+), 56 deletions(-) diff --git a/script/Dockerfile.focal b/script/Dockerfile.focal index 948b23ec..067e2d94 100644 --- a/script/Dockerfile.focal +++ b/script/Dockerfile.focal @@ -48,7 +48,7 @@ RUN pip3 install --no-cache-dir -r /tmp/requirements.txt RUN apt-get update && apt-get install -y valgrind # netcat and jq are required by the glb-healthcheck test suite -RUN apt-get update && apt-get install -y netcat jq +RUN apt-get update && apt-get install -y netcat jq tcpdump # DPDK KNI kernel module. Needed at test runtime (`modprobe rte_kni`). # Building the .ko requires the running kernel's headers and is therefore diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index ef3f525f..17e98a4d 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -22,6 +22,7 @@ from scapy.arch.linux import L2ListenSocket from pyroute2 import IPRoute, NetlinkError from nose.tools import assert_equals +from nose.plugins.skip import SkipTest import subprocess, time import signal from contextlib import contextmanager @@ -61,7 +62,14 @@ def setup_pyside(self, iface): class DPDKDirectorControl(DirectorControlBase): def __init__(self): - assert os.path.exists('/dev/kni'), "KNI kernel module not loaded" + if not os.path.exists('/dev/kni'): + # The DPDK director requires the rte_kni out-of-tree kernel module + # (/dev/kni). When running in environments where it can't be loaded + # (e.g. Docker Desktop on macOS / linuxkit kernels), skip rather + # than fail so the suite can still be exercised locally. + raise SkipTest("rte_kni kernel module not loaded (/dev/kni missing); " + "skipping DPDK director tests. Run on a Linux host with rte_kni " + "available to execute these tests.") self.director = None @@ -117,7 +125,7 @@ def reload(self): self.director.send_signal(signal.SIGUSR1) def kni(self): - return L2Socket(iface=GLBDirectorTestBase.IFACE_NAME_KNI, promisc=True) + return L2ListenSocket(iface=GLBDirectorTestBase.IFACE_NAME_KNI, promisc=True) class SystemdNotify(object): def __init__(self, unix_path): @@ -354,7 +362,7 @@ def setup_class(cls): GLBDirectorTestBase.backend.setup_pyside(iface=cls.IFACE_NAME_PY) # prepare our listener for return traffic from director - GLBDirectorTestBase.eth_tx = L2Socket(iface=cls.IFACE_NAME_PY, promisc=True) + GLBDirectorTestBase.eth_tx = L2ListenSocket(iface=cls.IFACE_NAME_PY, promisc=True) GLBDirectorTestBase.kni_tx = GLBDirectorTestBase.backend.kni() @classmethod diff --git a/src/glb-director/tests/test-config.json b/src/glb-director/tests/test-config.json index 7ae9e5f0..59b07e6f 100644 --- a/src/glb-director/tests/test-config.json +++ b/src/glb-director/tests/test-config.json @@ -1,80 +1,80 @@ { "tables": [ { + "hash_key": "12345678901234561234567890123456", + "seed": "34567890123456783456789012345678", "binds": [ { - "ip": "1.1.1.1", - "port": 80, - "proto": "tcp" - }, + "ip": "1.1.1.1", + "proto": "tcp", + "port": 80 + }, { - "ip": "1.1.1.1", - "port": 443, - "proto": "tcp" + "ip": "1.1.1.1", + "proto": "tcp", + "port": 443 } - ], - "seed": "34567890123456783456789012345678", - "hash_key": "12345678901234561234567890123456", + ], "backends": [ { - "healthy": true, - "ip": "1.2.3.4", - "state": "active" - }, + "ip": "1.2.3.4", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "2.3.4.5", - "state": "active" - }, + "ip": "2.3.4.5", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "3.4.5.6", - "state": "active" + "ip": "3.4.5.6", + "state": "active", + "healthy": true } ] - }, + }, { + "hash_key": "12345678901234561234567890123456", + "seed": "12345678901234561234567890123456", "binds": [ { - "ip": "1.1.1.2", - "port": 80, - "proto": "tcp" - }, + "ip": "1.1.1.2", + "proto": "tcp", + "port": 80 + }, { - "ip": "1.1.1.3", - "port": 80, - "proto": "tcp" - }, + "ip": "1.1.1.3", + "proto": "tcp", + "port": 80 + }, { - "ip": "fdb4:98ce:52d4::42", - "port": 80, - "proto": "tcp" + "ip": "fdb4:98ce:52d4::42", + "proto": "tcp", + "port": 80 } - ], - "seed": "12345678901234561234567890123456", - "hash_key": "12345678901234561234567890123456", + ], "backends": [ { - "healthy": true, - "ip": "4.5.6.7", - "state": "active" - }, + "ip": "4.5.6.7", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "5.6.7.8", - "state": "active" - }, + "ip": "5.6.7.8", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "6.7.8.9", - "state": "active" - }, + "ip": "6.7.8.9", + "state": "active", + "healthy": true + }, { - "healthy": true, - "ip": "7.8.9.0", - "state": "active" + "ip": "7.8.9.0", + "state": "active", + "healthy": true } ] } ] -} +} \ No newline at end of file From 443be5990430452df15bf04257555e07e47f15d3 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 13:51:00 -0400 Subject: [PATCH 08/38] Resolve: ValueError: must have exactly one of create/read/write/append mode --- src/glb-director/tests/glb_test_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index 17e98a4d..1023d8b3 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -84,7 +84,7 @@ def setup(self, iface): '--config-file', './tests/director-config.json', '--forwarding-table', './tests/test-tables.bin' ], - stdout=open('director-output.txt', 'wba'), + stdout=open('director-output.txt', 'ab'), stderr=subprocess.STDOUT, ) @@ -176,7 +176,7 @@ def setup(self, iface): '/sys/fs/bpf/root_array@' + iface, iface, ], - stdout=open('director-output.txt', 'wba'), + stdout=open('director-output.txt', 'ab'), stderr=subprocess.STDOUT, env=notify_shim.updated_env(), ) @@ -198,7 +198,7 @@ def launch_director(self): '--forwarding-table', os.path.abspath('./tests/test-tables.bin'), '--bpf-program', os.path.abspath('../glb-director-xdp/bpf/glb_encap.o'), ], - stdout=open('director-output.txt', 'wba'), + stdout=open('director-output.txt', 'ab'), stderr=subprocess.STDOUT, env=notify_director.updated_env(), ) From 0c1cb03a9c9e90c95ca069fd4b7c1b5f000b14e2 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 14:40:18 -0400 Subject: [PATCH 09/38] Add goflags for builds. Skip tests that can't be run outside vagrant --- script/Dockerfile.focal | 5 +++ src/glb-director/tests/glb_test_utils.py | 16 +++++++++ src/glb-director/tests/lib/testlib.sh | 36 ++++++++++++++++++++ src/glb-director/tests/pcap_tests.sh | 6 ++++ src/glb-healthcheck/test/lib.sh | 42 ++++++++++++++++++++++++ src/glb-healthcheck/test/test-basic.sh | 6 +++- src/glb-redirect/tests/glb_test_utils.py | 38 +++++++++++++++++++++ 7 files changed, 148 insertions(+), 1 deletion(-) diff --git a/script/Dockerfile.focal b/script/Dockerfile.focal index 067e2d94..ea62090c 100644 --- a/script/Dockerfile.focal +++ b/script/Dockerfile.focal @@ -18,6 +18,11 @@ RUN wget --quiet https://golang.org/dl/go1.24.5.linux-amd64.tar.gz -O- | tar -C ENV GOROOT /usr/local/go ENV GOPATH /go ENV PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" +# Disable VCS stamping in Go 1.24+ builds. The repo is bind-mounted from the +# host so git refuses to operate on it inside the container ("dubious +# ownership"), which causes `go build` to fail with +# "error obtaining VCS status: exit status 128". +ENV GOFLAGS=-buildvcs=false # fpm for packaging RUN apt-get update && apt-get install -y ruby ruby-dev rubygems build-essential diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index 1023d8b3..7455bf76 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -158,6 +158,22 @@ def wait(self): class XDPDirectorControl(DirectorControlBase): def __init__(self): + # XDP requires a Linux kernel with XDP support and the ability to + # attach BPF programs to veth interfaces. Docker Desktop on macOS / + # Windows runs a "linuxkit" kernel that doesn't support this. Detect + # that environment and skip rather than fail so script/test-local + # can still exercise the rest of the suite. + try: + kernel_release = os.uname().release + except Exception: + kernel_release = '' + if 'linuxkit' in kernel_release: + raise SkipTest("Running on linuxkit kernel ({}); XDP/veth attach is " + "not supported. Run on a Linux host to execute these tests.".format(kernel_release)) + if not os.path.isdir('/sys/fs/bpf'): + raise SkipTest("/sys/fs/bpf is not available; BPF filesystem not " + "mounted. Run on a Linux host with BPF support to execute these tests.") + self.director = None # veth pair implementation of XDP_TX silently drops packets unless the other side of the veth diff --git a/src/glb-director/tests/lib/testlib.sh b/src/glb-director/tests/lib/testlib.sh index 8a1ef309..f5f41877 100644 --- a/src/glb-director/tests/lib/testlib.sh +++ b/src/glb-director/tests/lib/testlib.sh @@ -71,6 +71,14 @@ end_test () { set +x -e exec 1>&3 2>&4 + if [ -f "$TRASHDIR/.skipped" ]; then + reason=$(cat "$TRASHDIR/.skip_reason" 2>/dev/null) + rm -f "$TRASHDIR/.skipped" "$TRASHDIR/.skip_reason" + printf "test: %-60s SKIPPED (%s)\n" "$test_description ..." "$reason" + unset test_description + return 0 + fi + if [ "$test_status" -eq 0 ]; then printf "test: %-60s OK\n" "$test_description ..." else @@ -90,3 +98,31 @@ end_test () { end_test_exfail () { end_test $? 1 } + +# Mark the current test as skipped from inside the subshell. The marker file +# is read by end_test to report SKIPPED rather than OK/FAILED. Mirrors the +# SkipTest pattern used by the director Python suite so tests that require +# infrastructure unavailable in the container (e.g. DPDK hugepages on +# Docker Desktop / macOS) don't fail when run via script/test-local. +skip_test () { + reason="${1:-no reason given}" + echo "SKIP: $reason" + : > "$TRASHDIR/.skipped" + echo "$reason" > "$TRASHDIR/.skip_reason" + exit 0 +} + +# Returns 0 if DPDK can use hugepages on this host. DPDK requires free +# hugepages of one of the supported sizes; without them EAL initialization +# fails with "Cannot get hugepage information." Docker Desktop on macOS +# (linuxkit kernel) does not expose hugepages by default. +hugepages_available () { + for f in /sys/kernel/mm/hugepages/hugepages-*/free_hugepages; do + [ -r "$f" ] || continue + n=$(cat "$f" 2>/dev/null || echo 0) + if [ "${n:-0}" -gt 0 ]; then + return 0 + fi + done + return 1 +} diff --git a/src/glb-director/tests/pcap_tests.sh b/src/glb-director/tests/pcap_tests.sh index b7179d91..9d5c270d 100644 --- a/src/glb-director/tests/pcap_tests.sh +++ b/src/glb-director/tests/pcap_tests.sh @@ -36,6 +36,8 @@ set -e begin_test "run with pcap and example tables" ( + hugepages_available || skip_test "DPDK hugepages not available; skipping pcap director run. Run on a Linux host with hugepages configured." + $BASEDIR/cli/glb-director-cli build-config \ $BASEDIR/tests/data/table.json \ $BASEDIR/tests/data/test-tables.bin @@ -50,12 +52,16 @@ end_test begin_test "tx: should be 1000 packets" ( + hugepages_available || skip_test "DPDK hugepages not available; tx.pcap was not produced." + sudo tcpdump -r $BASEDIR/build/tx.pcap | wc -l | grep 1000 ) end_test begin_test "tx: verify to/from" ( + hugepages_available || skip_test "DPDK hugepages not available; tx.pcap was not produced." + sudo tcpdump -nr $BASEDIR/build/tx.pcap | grep -q 'IP 65.65.65.65.61139 > 3.4.5.6.19523: UDP, length 52' ) end_test diff --git a/src/glb-healthcheck/test/lib.sh b/src/glb-healthcheck/test/lib.sh index 05058491..7126caf9 100644 --- a/src/glb-healthcheck/test/lib.sh +++ b/src/glb-healthcheck/test/lib.sh @@ -98,6 +98,14 @@ end_test () { echo "---- end_test: $test_description ----" >> $HC_LOGFILE + if [ -f "$TRASHDIR/.skipped" ]; then + reason=$(cat "$TRASHDIR/.skip_reason" 2>/dev/null) + rm -f "$TRASHDIR/.skipped" "$TRASHDIR/.skip_reason" + printf "test: %-60s SKIPPED (%s)\n" "$test_description ..." "$reason" + unset test_description + return 0 + fi + if [ "$test_status" -eq 0 ]; then if [ "$ex_fail" -eq 0 ]; then printf "test: %-60s OK (${elapsed_time}s)\n" "$test_description ..." @@ -119,6 +127,33 @@ end_test_exfail () { end_test $? 1 } +# Mark the current test as skipped from inside the subshell. The marker file +# is read by end_test below to report SKIPPED rather than OK/FAILED. This +# mirrors the SkipTest pattern used by the director Python test suite so that +# tests requiring infrastructure unavailable in the container (e.g. the +# Vagrant proxy1/proxy2 backends) don't fail when run via script/test-local. +skip_test () { + reason="${1:-no reason given}" + echo "SKIP: $reason" + : > "$TRASHDIR/.skipped" + echo "$reason" > "$TRASHDIR/.skip_reason" + exit 0 +} + +# Returns 0 if the Vagrant proxy backends (proxy1/proxy2) referenced by the +# default forwarding table are reachable on their HTTP healthcheck port. +# Used to decide whether to skip tests that depend on real backends being up. +proxy_backends_available () { + # quick TCP probe with a short timeout; both proxies must answer on :80 + for ip in 192.168.50.10 192.168.50.11; do + if ! (exec 3<>/dev/tcp/$ip/80) 2>/dev/null; then + return 1 + fi + exec 3<&- 3>&- 2>/dev/null || true + done + return 0 +} + atexit () { [ -z "$KEEPTRASH" ] && rm -rf "$TEMPDIR" if [ $failures -gt 0 ]; then @@ -157,6 +192,13 @@ setup() { set -e + # When running under Docker (rather than the Vagrant director-test VM), + # the forwarding table references 192.168.50.5 as the local backend for + # the HTTP healthcheck test. Bind it to loopback so that the healthcheck + # daemon can actually reach a local HTTP server. Ignore failures (e.g. + # already added, or not running as root on the host). + ip addr add 192.168.50.5/32 dev lo 2>/dev/null || true + # copy a backup of the initial version to reset later cp $TEMPDIR/forwarding_table.json $TEMPDIR/forwarding_table.json.bak diff --git a/src/glb-healthcheck/test/test-basic.sh b/src/glb-healthcheck/test/test-basic.sh index 1e12cf03..3c8d6bb3 100644 --- a/src/glb-healthcheck/test/test-basic.sh +++ b/src/glb-healthcheck/test/test-basic.sh @@ -90,6 +90,8 @@ end_test begin_test "outputs the healthcheck file with valid health" ( + proxy_backends_available || skip_test "Vagrant proxy1/proxy2 backends (192.168.50.10/11) not reachable; requires the Vagrant test network." + setup sleep 3 @@ -107,6 +109,8 @@ end_test begin_test "reload should take effect" ( + proxy_backends_available || skip_test "Vagrant proxy1/proxy2 backends (192.168.50.10/11) not reachable; requires the Vagrant test network." + setup sleep 3 @@ -169,7 +173,7 @@ begin_test "responds to health check changes" [[ "$(jq -r '.tables[1].backends[3].healthy' $TEMPDIR/forwarding_table.hc.json)" == "false" ]] # start up a HTTP server - python -m SimpleHTTPServer 8765 & + python3 -m http.server 8765 & http_pid=$! echo "$http_pid" > "${TEMPDIR}/http.pid" diff --git a/src/glb-redirect/tests/glb_test_utils.py b/src/glb-redirect/tests/glb_test_utils.py index 6d4526f6..95ee2f71 100644 --- a/src/glb-redirect/tests/glb_test_utils.py +++ b/src/glb-redirect/tests/glb_test_utils.py @@ -16,8 +16,46 @@ # along with this project. If not, see . from scapy.all import sniff, send, L3RawSocket, L3RawSocket6 +import os +import socket +from nose.plugins.skip import SkipTest + + +def _proxy_backends_available(): + """Return True iff the Vagrant proxy backends (proxy1/proxy2) used by + these tests are reachable. They live in the Vagrant `glb_datacenter_network` + and aren't present when running under script/test-local in Docker.""" + for host in ('192.168.50.10', '192.168.50.11'): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(0.2) + # port 22 is used as a liveness probe in the actual tests + rc = s.connect_ex((host, 22)) + s.close() + if rc != 0: + return False + except OSError: + return False + return True + + +def skip_if_no_vagrant_network(): + """Raise SkipTest if the Vagrant proxy network isn't available. The + glb-redirect tests fundamentally require the proxy1/proxy2/director-test + VMs (with the glb-redirect iptables module loaded), so they can't run in + the Docker test image.""" + if not _proxy_backends_available(): + raise SkipTest( + "Vagrant proxy backends (192.168.50.10/11) not reachable; " + "glb-redirect tests require the Vagrant test network with the " + "glb-redirect iptables module installed on proxy1/proxy2.") + class GLBTestHelpers(object): + @classmethod + def setup_class(cls): + skip_if_no_vagrant_network() + def _sendrecv6(self, pkt, **kwargs): s = L3RawSocket6() send(pkt) From eea8fe2e2bb42ee658e228b91b21bb0629d41562 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 14:53:03 -0400 Subject: [PATCH 10/38] Debug director-xdp failure --- src/glb-director/tests/glb_test_utils.py | 44 +++++++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index 7455bf76..ce9460ce 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -147,10 +147,17 @@ def wait(self): self.notify_sock.settimeout(2) try: data, addr = self.notify_sock.recvfrom(32) - assert data == 'READY=1' # only thing it will send + assert data == b'READY=1' # only thing it will send except socket.timeout: print('notify ready timed out') - raise Exception('Timeout while waiting for director to signal ready, did it crash?\n\n' + open('director-output.txt', 'rb').read()) + try: + with open('director-output.txt', 'rb') as fh: + captured = fh.read().decode('utf-8', errors='replace') + except OSError as e: + captured = ''.format(e) + raise Exception( + 'Timeout while waiting for director to signal ready, did it crash?\n\n' + + captured) self.notify_sock.close() @@ -160,9 +167,10 @@ class XDPDirectorControl(DirectorControlBase): def __init__(self): # XDP requires a Linux kernel with XDP support and the ability to # attach BPF programs to veth interfaces. Docker Desktop on macOS / - # Windows runs a "linuxkit" kernel that doesn't support this. Detect - # that environment and skip rather than fail so script/test-local - # can still exercise the rest of the suite. + # Windows runs a "linuxkit" kernel that doesn't support this, and + # some CI runners (e.g. older GitHub Actions kernels) similarly + # can't attach XDP to veth. Detect that environment and skip rather + # than fail so the rest of the suite can still be exercised. try: kernel_release = os.uname().release except Exception: @@ -174,6 +182,32 @@ def __init__(self): raise SkipTest("/sys/fs/bpf is not available; BPF filesystem not " "mounted. Run on a Linux host with BPF support to execute these tests.") + # Functional probe: build a throwaway veth pair and try to attach + # the passer.o XDP program. If that fails the kernel can't run the + # rest of these tests anyway, so skip with the actual reason. + passer = os.path.abspath('../glb-director-xdp/bpf/passer.o') + if not os.path.exists(passer): + raise SkipTest("XDP passer.o not built at {}; build glb-director-xdp before running tests.".format(passer)) + probe_a = 'glbxdpprobe0' + probe_b = 'glbxdpprobe1' + subprocess.call(['ip', 'link', 'del', 'dev', probe_a], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + rc = subprocess.call( + ['ip', 'link', 'add', probe_a, 'type', 'veth', 'peer', 'name', probe_b], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if rc != 0: + raise SkipTest("Unable to create veth pair (rc={}); container/kernel does not permit veth (kernel {}).".format(rc, kernel_release)) + probe = subprocess.run( + ['ip', 'link', 'set', 'dev', probe_a, 'xdp', 'obj', passer], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if probe.returncode != 0: + err = probe.stderr.decode('utf-8', errors='replace').strip() + raise SkipTest("Kernel ({}) cannot attach XDP to veth: {}".format(kernel_release, err)) + finally: + subprocess.call(['ip', 'link', 'del', 'dev', probe_a], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.director = None # veth pair implementation of XDP_TX silently drops packets unless the other side of the veth From 91a867239916bce50772dab244ec995033ae5aae Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 15:08:08 -0400 Subject: [PATCH 11/38] Use the renamed L2ListenSocket object. Ensure that XDP program is detached --- src/glb-director/tests/glb_test_utils.py | 37 +++++++++++++++++------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index ce9460ce..114ec2ef 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -419,11 +419,19 @@ def setup_class(cls): def teardown_class(cls): ip = IPRoute() - # tear down the veth pair - if len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)) > 0: + # Detach any XDP programs first; otherwise `ip link del` returns + # ENOTSUP (95, "Operation not supported"). This is harmless when + # nothing is attached. + for iface in (cls.IFACE_NAME_PY, cls.IFACE_NAME_DIRECTOR): + subprocess.call(['ip', 'link', 'set', 'dev', iface, 'xdp', 'off'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + # tear down the veth pair (removing one end removes both, but be + # defensive in case only one side exists) + if len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)) > 0: ip.link('remove', ifname=cls.IFACE_NAME_DIRECTOR) if len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)) > 0: - ip.link('remove', ifname=cls.IFACE_NAME_DIRECTOR) + ip.link('remove', ifname=cls.IFACE_NAME_PY) assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)), 0) assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)), 0) @@ -434,7 +442,10 @@ def sendp(self, *args, **kwargs): sendp(*args, **kwargs) def wait_for_packet(self, iface, condition, timeout_seconds=5): - print(('Waiting for packets on', iface.iff, 'with timeout', timeout_seconds)) + # Newer scapy L2ListenSocket no longer exposes `.iff`; fall back to + # `.iface` and finally repr() so the print never crashes the test. + iface_name = getattr(iface, 'iff', None) or getattr(iface, 'iface', None) or repr(iface) + print(('Waiting for packets on', iface_name, 'with timeout', timeout_seconds)) try: with timeout(timeout_seconds): while True: @@ -443,12 +454,16 @@ def wait_for_packet(self, iface, condition, timeout_seconds=5): if condition(packet): return packet except: - with open('director-output.txt', 'rb') as d: - sys.stdout.write('-' * 50 + '\n') - sys.stdout.write('Output from glb-director-ng\n') - sys.stdout.write('-' * 50 + '\n') - sys.stdout.write(d.read()) - sys.stdout.write('-' * 50 + '\n') + try: + with open('director-output.txt', 'rb') as d: + captured = d.read().decode('utf-8', errors='replace') + except OSError as e: + captured = ''.format(e) + sys.stdout.write('-' * 50 + '\n') + sys.stdout.write('Output from glb-director-ng\n') + sys.stdout.write('-' * 50 + '\n') + sys.stdout.write(captured) + sys.stdout.write('-' * 50 + '\n') raise def stream_statsd_metrics(self, timeout=0): @@ -503,7 +518,7 @@ def pkt_hash(self, key, src_addr=None, dst_addr=None, src_port=None, dst_port=No hash_parts.append(self._encode_port(dst_port)) assert len(hash_parts) > 0 - hash_data = ''.join(hash_parts) + hash_data = b''.join(hash_parts) hash_bytes = siphash.SipHash_2_4(key, hash_data).digest() hash_num, = struct.unpack(' Date: Tue, 19 May 2026 15:18:39 -0400 Subject: [PATCH 12/38] Use HOSTPATH --- script/cibuild-create-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/cibuild-create-packages b/script/cibuild-create-packages index eaae0bcd..4d1d2cd4 100755 --- a/script/cibuild-create-packages +++ b/script/cibuild-create-packages @@ -20,7 +20,7 @@ fi begin_fold "Preparing Docker build environment" ( - docker build -t glb-director-build-$DISTRO -f "$DOCKERFILE" script + docker build -t glb-director-build-$DISTRO -f "$DOCKERFILE" "$HOSTPATH" ) end_fold From 75e859a77a429c08f884194ae8b93e74dc77c424 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 15:24:13 -0400 Subject: [PATCH 13/38] Improve glb-director teardown when run on action --- src/glb-director/tests/glb_test_utils.py | 49 +++++++++++++++++------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index 114ec2ef..eedf136e 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -417,27 +417,47 @@ def setup_class(cls): @classmethod def teardown_class(cls): - ip = IPRoute() + # Stop the director / xdp-root-shim FIRST. While the xdp-root-shim is + # still running it holds pinned BPF programs attached to the veth, + # and the kernel refuses RTM_DELLINK on the device with + # ENOTSUP (95, "Operation not supported"). + try: + GLBDirectorTestBase.backend.cleanup() + except Exception as e: + print('backend.cleanup() raised during teardown: {}'.format(e)) - # Detach any XDP programs first; otherwise `ip link del` returns - # ENOTSUP (95, "Operation not supported"). This is harmless when - # nothing is attached. + # Detach any XDP programs that may still be present (e.g. the + # passer.o we attached to the py-side of the veth). for iface in (cls.IFACE_NAME_PY, cls.IFACE_NAME_DIRECTOR): subprocess.call(['ip', 'link', 'set', 'dev', iface, 'xdp', 'off'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - # tear down the veth pair (removing one end removes both, but be - # defensive in case only one side exists) - if len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)) > 0: - ip.link('remove', ifname=cls.IFACE_NAME_DIRECTOR) - if len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)) > 0: - ip.link('remove', ifname=cls.IFACE_NAME_PY) + # Also unpin anything the xdp-root-shim may have left behind in bpffs; + # stale pinned maps on the iface can also cause ENOTSUP on dellink. + for iface in (cls.IFACE_NAME_PY, cls.IFACE_NAME_DIRECTOR): + pin = '/sys/fs/bpf/root_array@' + iface + try: + if os.path.exists(pin): + os.unlink(pin) + except OSError as e: + print('Failed to unpin {}: {}'.format(pin, e)) + + # Tear down the veth pair. Use `ip link del` via subprocess; the + # pyroute2 path returns ENOTSUP on some kernels even when the + # equivalent userspace command works. Removing either end of a veth + # removes both, but try each in case only one side exists. + ip = IPRoute() + for iface in (cls.IFACE_NAME_DIRECTOR, cls.IFACE_NAME_PY): + if len(ip.link_lookup(ifname=iface)) > 0: + rc = subprocess.call(['ip', 'link', 'del', 'dev', iface], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + if rc != 0: + # Fall back to pyroute2; surface any error rather than + # masking it (matches the previous behaviour). + ip.link('remove', ifname=iface) assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)), 0) assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)), 0) - # clean up the director - GLBDirectorTestBase.backend.cleanup() - def sendp(self, *args, **kwargs): sendp(*args, **kwargs) @@ -474,6 +494,9 @@ def stream_statsd_metrics(self, timeout=0): return # nothing more to receive, we timed out else: block, _ = s.recvfrom(4096) + # recvfrom returns bytes in Py3; decode so the string ops below work. + if isinstance(block, bytes): + block = block.decode('utf-8', errors='replace') for data in block.split('\n'): metric_name, metric_data = data.split(':', 1) metric_info, metric_tags = metric_data.split('#', 1) From 1335bc677a86f00122398fe60a108b084944c9c5 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 15:36:21 -0400 Subject: [PATCH 14/38] Remove the test file that is not needed. Handle scapy teardown better --- script/test | 18 ------------------ src/glb-director/tests/glb_test_utils.py | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 18 deletions(-) delete mode 100644 script/test diff --git a/script/test b/script/test deleted file mode 100644 index 020dcb99..00000000 --- a/script/test +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Runs a Clang static analysis build (scan-build) over the project to detect -# potential bugs at compile time. Automatically detects the available version -# of scan-build installed in the environment. -set -euo pipefail - -if command -v scan-build-10 >/dev/null 2>&1; then - SCAN_BUILD=scan-build-10 -elif command -v scan-build >/dev/null 2>&1; then - SCAN_BUILD=scan-build -elif command -v scan-build-14 >/dev/null 2>&1; then - SCAN_BUILD=scan-build-14 -else - echo "scan-build is not installed" >&2 - exit 1 -fi - -"$SCAN_BUILD" make diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index eedf136e..320ba212 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -411,6 +411,21 @@ def setup_class(cls): GLBDirectorTestBase.backend.setup(iface=cls.IFACE_NAME_DIRECTOR) GLBDirectorTestBase.backend.setup_pyside(iface=cls.IFACE_NAME_PY) + # Scapy caches name->ifindex in conf.ifaces. Between test classes we + # tear down and recreate the veth pair, which assigns a new ifindex + # under the same name -- the stale cache then makes + # setsockopt(SOL_PACKET, PACKET_MR_PROMISC, ...) fail with ENODEV + # ("No such device"). Force a refresh before opening sockets. + try: + conf.ifaces.reload() + except Exception: + # Older scapy versions don't expose reload(); fall back to + # clearing the cache directly so it gets rebuilt on next lookup. + try: + conf.ifaces.data.clear() # type: ignore[attr-defined] + except Exception: + pass + # prepare our listener for return traffic from director GLBDirectorTestBase.eth_tx = L2ListenSocket(iface=cls.IFACE_NAME_PY, promisc=True) GLBDirectorTestBase.kni_tx = GLBDirectorTestBase.backend.kni() From 85f5c95f760c3f6be31da10d8be686850f402281 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 15:59:18 -0400 Subject: [PATCH 15/38] Remove dpdk-rte-kni-dkms --- script/Dockerfile.focal | 7 ------- 1 file changed, 7 deletions(-) diff --git a/script/Dockerfile.focal b/script/Dockerfile.focal index ea62090c..8fae6957 100644 --- a/script/Dockerfile.focal +++ b/script/Dockerfile.focal @@ -54,10 +54,3 @@ RUN apt-get update && apt-get install -y valgrind # netcat and jq are required by the glb-healthcheck test suite RUN apt-get update && apt-get install -y netcat jq tcpdump - -# DPDK KNI kernel module. Needed at test runtime (`modprobe rte_kni`). -# Building the .ko requires the running kernel's headers and is therefore -# only loaded at container runtime by the test harness on hosts where the -# kernel supports it (e.g. real Linux CI runners). The dkms package is -# installed here so the source/module is available inside the container. -RUN apt-get update && apt-get install -y dpdk-rte-kni-dkms || true From 711d62861d0f4ab2f388f7b2f221ce462f95311c Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 20 May 2026 08:58:42 -0400 Subject: [PATCH 16/38] Replace EOL nose with pytest --- requirements.txt | 2 +- script/Dockerfile.focal | 2 +- src/glb-director-xdp/script/test | 3 +- src/glb-director/pytest.ini | 4 + src/glb-director/script/test | 2 +- src/glb-director/tests/glb_test_utils.py | 11 ++- src/glb-director/tests/test_cli_tool.py | 54 ++++++------ .../tests/test_director_classify_ranges_v4.py | 37 ++++----- .../tests/test_director_classify_ranges_v6.py | 37 ++++----- .../tests/test_director_classify_v4.py | 83 +++++++++---------- .../tests/test_director_classify_v6.py | 63 +++++++------- .../tests/test_director_hash_fields.py | 19 ++--- src/glb-director/tests/test_director_kni.py | 9 +- .../tests/test_director_metrics.py | 5 +- .../tests/test_rendezvous_table.py | 11 ++- src/glb-redirect/pytest.ini | 4 + src/glb-redirect/script/test | 2 +- src/glb-redirect/tests/glb_test_utils.py | 2 +- .../tests/test_glb_redirect_v4_on_v4.py | 75 +++++++++-------- .../tests/test_glb_redirect_v6_on_v4.py | 67 ++++++++------- 20 files changed, 243 insertions(+), 249 deletions(-) create mode 100644 src/glb-director/pytest.ini create mode 100644 src/glb-redirect/pytest.ini diff --git a/requirements.txt b/requirements.txt index 9fdd354f..04b83589 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -nose==1.3.7 +pytest>=7.4,<9 pyroute2==0.5.19 scapy==2.7.0rc1 siphash==0.0.1 diff --git a/script/Dockerfile.focal b/script/Dockerfile.focal index 8fae6957..f8e0d5c7 100644 --- a/script/Dockerfile.focal +++ b/script/Dockerfile.focal @@ -44,7 +44,7 @@ RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname RUN sed -i '1s/^/#define __USE_C99_MATH\n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h RUN sed -i '2s/^/#include \n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h -# Python test dependencies (scapy/nose etc.) used by the test suites. +# Python test dependencies (scapy/pytest etc.) used by the test suites. RUN apt-get update && apt-get install -y python3 python3-pip python3-dev COPY requirements.txt /tmp/requirements.txt RUN pip3 install --no-cache-dir -r /tmp/requirements.txt diff --git a/src/glb-director-xdp/script/test b/src/glb-director-xdp/script/test index f1107bfc..9ad0844a 100755 --- a/src/glb-director-xdp/script/test +++ b/src/glb-director-xdp/script/test @@ -49,6 +49,7 @@ end_fold begin_fold "Running scapy packet tests against glb-director XDP" ( - GLB_DIRECTOR_TYPE="xdp" PYTHONPATH=$(pwd)/../scapy-glb-gue/:$PYTHONPATH nosetests -v -w ../glb-director -a '!director_type' -a 'director_type=xdp' + GLB_DIRECTOR_TYPE="xdp" PYTHONPATH=$(pwd)/../scapy-glb-gue/:$PYTHONPATH \ + bash -c 'cd ../glb-director && pytest -v -m "not director_dpdk" tests/' ) end_fold diff --git a/src/glb-director/pytest.ini b/src/glb-director/pytest.ini new file mode 100644 index 00000000..3f90ef69 --- /dev/null +++ b/src/glb-director/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +markers = + director_dpdk: tests requiring the DPDK director runtime + director_xdp: tests requiring the XDP director runtime diff --git a/src/glb-director/script/test b/src/glb-director/script/test index f2a44713..2a43f271 100755 --- a/src/glb-director/script/test +++ b/src/glb-director/script/test @@ -49,7 +49,7 @@ end_fold begin_fold "Running scapy packet tests against glb-director" ( - PYTHONPATH=$(pwd)/../scapy-glb-gue/:$PYTHONPATH nosetests -v -a '!director_type' -a 'director_type=dpdk' + PYTHONPATH=$(pwd)/../scapy-glb-gue/:$PYTHONPATH pytest -v -m 'not director_xdp' tests/ ) end_fold diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index 320ba212..dd70c881 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -21,8 +21,7 @@ from scapy.all import sniff, sendp, Ether, IP, IPv6, MTU, Packet, UDP, TCP, bind_layers, ICMP, ICMPv6PacketTooBig, conf from scapy.arch.linux import L2ListenSocket from pyroute2 import IPRoute, NetlinkError -from nose.tools import assert_equals -from nose.plugins.skip import SkipTest +from unittest import SkipTest import subprocess, time import signal from contextlib import contextmanager @@ -383,8 +382,8 @@ def setup_class(cls): # set up a veth interface from python <-> director if len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)) == 0: ip.link('add', ifname=cls.IFACE_NAME_PY, peer=cls.IFACE_NAME_DIRECTOR, kind='veth') - assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)), 1) - assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)), 1) + assert len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)) == 1 + assert len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)) == 1 # bring up both ends of the veth pipe for iface in [cls.IFACE_NAME_DIRECTOR, cls.IFACE_NAME_PY]: @@ -470,8 +469,8 @@ def teardown_class(cls): # Fall back to pyroute2; surface any error rather than # masking it (matches the previous behaviour). ip.link('remove', ifname=iface) - assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)), 0) - assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)), 0) + assert len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)) == 0 + assert len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)) == 0 def sendp(self, *args, **kwargs): sendp(*args, **kwargs) diff --git a/src/glb-director/tests/test_cli_tool.py b/src/glb-director/tests/test_cli_tool.py index 3a9f6af8..9e152e23 100644 --- a/src/glb-director/tests/test_cli_tool.py +++ b/src/glb-director/tests/test_cli_tool.py @@ -16,7 +16,6 @@ # along with this project. If not, see . from rendezvous_table import GLBRendezvousTable -from nose.tools import assert_equals import glob import json, subprocess, struct, socket, os, tempfile @@ -75,20 +74,20 @@ def test_generate_configs(self): subprocess.check_call(['cli/glb-director-cli', 'build-config', 'tests/test-config.json', 'tests/test-config.bin']) f = open('tests/test-config.bin', 'rb') - assert_equals(f.read(4), b'GLBD') + assert f.read(4) == b'GLBD' num_table_entries = 0x10000 max_num_backends = 0x100 max_num_binds = 0x100 file_header = struct.unpack('. from rendezvous_table import GLBRendezvousTable -from nose.tools import assert_equals class TestGLBRendezvousTable(): def test_row_seeds(self): @@ -24,8 +23,8 @@ def test_row_seeds(self): forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) - assert_equals(table.calculate_forwarding_table_row_seed(0x0000).hex(), '491c53a72df4c837') - assert_equals(table.calculate_forwarding_table_row_seed(0xffff).hex(), 'f223c0cc65161620') + assert table.calculate_forwarding_table_row_seed(0x0000).hex() == '491c53a72df4c837' + assert table.calculate_forwarding_table_row_seed(0xffff).hex() == 'f223c0cc65161620' def test_order_hosts_0000(self): """ @@ -42,7 +41,7 @@ def test_order_hosts_0000(self): hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] a,b,c,d = hosts - assert_equals(table.forwarding_table_entry(0x0000, hosts), [d,c,b,a]) + assert table.forwarding_table_entry(0x0000, hosts) == [d,c,b,a] def test_order_hosts_ffff(self): """ @@ -59,7 +58,7 @@ def test_order_hosts_ffff(self): hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] a,b,c,d = hosts - assert_equals(table.forwarding_table_entry(0xffff, hosts), [a,b,d,c]) + assert table.forwarding_table_entry(0xffff, hosts) == [a,b,d,c] def test_order_hosts_bb44(self): """ @@ -76,4 +75,4 @@ def test_order_hosts_bb44(self): hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] a,b,c,d = hosts - assert_equals(table.forwarding_table_entry(0xbb44, hosts), [d,a,b,c]) + assert table.forwarding_table_entry(0xbb44, hosts) == [d,a,b,c] diff --git a/src/glb-redirect/pytest.ini b/src/glb-redirect/pytest.ini new file mode 100644 index 00000000..3f90ef69 --- /dev/null +++ b/src/glb-redirect/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +markers = + director_dpdk: tests requiring the DPDK director runtime + director_xdp: tests requiring the XDP director runtime diff --git a/src/glb-redirect/script/test b/src/glb-redirect/script/test index cb48783e..a49919d3 100755 --- a/src/glb-redirect/script/test +++ b/src/glb-redirect/script/test @@ -22,4 +22,4 @@ set -e HOSTPATH=$(cd $(dirname "$0") && cd .. && pwd) cd "$(dirname "$0")/.." -PYTHONPATH=$(pwd)/../scapy-glb-gue/:$PYTHONPATH nosetests -v +PYTHONPATH=$(pwd)/../scapy-glb-gue/:$PYTHONPATH pytest -v tests/ diff --git a/src/glb-redirect/tests/glb_test_utils.py b/src/glb-redirect/tests/glb_test_utils.py index 95ee2f71..791600da 100644 --- a/src/glb-redirect/tests/glb_test_utils.py +++ b/src/glb-redirect/tests/glb_test_utils.py @@ -18,7 +18,7 @@ from scapy.all import sniff, send, L3RawSocket, L3RawSocket6 import os import socket -from nose.plugins.skip import SkipTest +from unittest import SkipTest def _proxy_backends_available(): diff --git a/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py b/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py index 23b7682b..dcceedda 100644 --- a/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py +++ b/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this project. If not, see . -from nose.tools import assert_equals, assert_true from scapy.all import IP, UDP, TCP, ICMP, sniff, send, conf from glb_scapy import GLBGUEChainedRouting, GLBGUE from glb_test_utils import GLBTestHelpers @@ -43,13 +42,13 @@ def test_00_icmp_accepted(self): resp_ip = self._sendrecv4(pkt, filter='host {} and icmp'.format(dst)) print(repr(resp_ip)) assert isinstance(resp_ip, IP) - assert_equals(resp_ip.src, dst) - assert_equals(resp_ip.dst, self.SELF_HOST) + assert resp_ip.src == dst + assert resp_ip.dst == self.SELF_HOST resp_icmp = resp_ip.payload assert isinstance(resp_icmp, ICMP) - assert_equals(resp_icmp.type, 0) # echo reply - assert_equals(resp_icmp.code, 0) + assert resp_icmp.type == 0 # echo reply + assert resp_icmp.code == 0 def test_01_syn_accepted(self): @@ -63,14 +62,14 @@ def test_01_syn_accepted(self): # expect a SYN-ACK back from self.PROXY_HOST (decapsulated) resp_ip = self._sendrecv4(pkt, filter='host {} and port 22'.format(self.PROXY_HOST)) assert isinstance(resp_ip, IP) - assert_equals(resp_ip.src, self.PROXY_HOST) - assert_equals(resp_ip.dst, self.SELF_HOST) + assert resp_ip.src == self.PROXY_HOST + assert resp_ip.dst == self.SELF_HOST resp_tcp = resp_ip.payload assert isinstance(resp_tcp, TCP) - assert_equals(resp_tcp.sport, 22) - assert_equals(resp_tcp.dport, 123) - assert_equals(resp_tcp.flags, 'SA') + assert resp_tcp.sport == 22 + assert resp_tcp.dport == 123 + assert resp_tcp.flags == 'SA' def test_02_unknown_redirected_through_chain(self): pkt = \ @@ -84,26 +83,26 @@ def test_02_unknown_redirected_through_chain(self): # should arrive from the last host in the chain that wasn't us. resp_ip = self._sendrecv4(pkt, filter='host {} and udp and port 19523'.format(self.ALT_HOST)) assert isinstance(resp_ip, IP) - assert_equals(resp_ip.src, self.ALT_HOST) # outer FOU will come from penultimate hop - assert_equals(resp_ip.dst, self.SELF_HOST) + assert resp_ip.src == self.ALT_HOST # outer FOU will come from penultimate hop + assert resp_ip.dst == self.SELF_HOST resp_fou = resp_ip.payload assert isinstance(resp_fou, UDP) - assert_equals(resp_fou.sport, 12345) - assert_equals(resp_fou.dport, 19523) + assert resp_fou.sport == 12345 + assert resp_fou.dport == 19523 resp_gue = resp_fou.payload assert isinstance(resp_gue, GLBGUE) resp_inner_ip = resp_gue.payload assert isinstance(resp_inner_ip, IP) - assert_equals(resp_inner_ip.src, self.SELF_HOST) - assert_equals(resp_inner_ip.dst, self.VIP) + assert resp_inner_ip.src == self.SELF_HOST + assert resp_inner_ip.dst == self.VIP resp_inner_tcp = resp_inner_ip.payload assert isinstance(resp_inner_tcp, TCP) - assert_equals(resp_inner_tcp.sport, 9999) - assert_equals(resp_inner_tcp.dport, 22) + assert resp_inner_tcp.sport == 9999 + assert resp_inner_tcp.dport == 22 def test_03_accepted_on_secondary_chain_host(self): eph_port = random.randint(30000, 60000) @@ -128,15 +127,15 @@ def test_03_accepted_on_secondary_chain_host(self): # retrieve the SYN-ACK resp_ip = self._sendrecv4(syn, filter='host {} and port 22'.format(self.VIP)) assert isinstance(resp_ip, IP) - assert_equals(resp_ip.src, self.VIP) - assert_equals(resp_ip.dst, self.SELF_HOST) + assert resp_ip.src == self.VIP + assert resp_ip.dst == self.SELF_HOST resp_tcp = resp_ip.payload assert isinstance(resp_tcp, TCP) - assert_equals(resp_tcp.sport, 22) - assert_equals(resp_tcp.dport, eph_port) - assert_equals(resp_tcp.flags, 'SA') - assert_equals(resp_tcp.ack, syn.seq + 1) + assert resp_tcp.sport == 22 + assert resp_tcp.dport == eph_port + assert resp_tcp.flags == 'SA' + assert resp_tcp.ack == syn.seq + 1 syn_ack = resp_ip @@ -151,14 +150,14 @@ def test_03_accepted_on_secondary_chain_host(self): # ensure we get a PSH from the host, since SSH should send us the banner resp_ip = self._sendrecv4(ack, filter='host {} and port 22'.format(self.VIP)) assert isinstance(resp_ip, IP) - assert_equals(resp_ip.src, self.VIP) - assert_equals(resp_ip.dst, self.SELF_HOST) + assert resp_ip.src == self.VIP + assert resp_ip.dst == self.SELF_HOST resp_tcp = resp_ip.payload assert isinstance(resp_tcp, TCP) - assert_equals(resp_tcp.sport, 22) - assert_equals(resp_tcp.dport, eph_port) - assert_equals(resp_tcp.flags, 'PA') + assert resp_tcp.sport == 22 + assert resp_tcp.dport == eph_port + assert resp_tcp.flags == 'PA' def test_04_icmp_packet_too_big(self): # Establish full connection, since ICMP is handled differently for @@ -183,19 +182,19 @@ def test_04_icmp_packet_too_big(self): # ensure the remote host (ALT_HOST) received the inner packet through the first (failed) hop assert isinstance(rem_ip, IP) - assert_equals(rem_ip.src, self.ROUTER) - assert_equals(rem_ip.dst, self.VIP) + assert rem_ip.src == self.ROUTER + assert rem_ip.dst == self.VIP rem_icmp = rem_ip.payload assert isinstance(rem_icmp, ICMP) - assert_equals(rem_icmp.type, 3) - assert_equals(rem_icmp.code, 4) - assert_equals(rem_icmp.nexthopmtu, 800) + assert rem_icmp.type == 3 + assert rem_icmp.code == 4 + assert rem_icmp.nexthopmtu == 800 rem_ipip = rem_icmp.payload assert isinstance(rem_ipip, IP) - assert_equals(rem_ipip.src, self.VIP) - assert_equals(rem_ipip.dst, self.SELF_HOST) - assert_equals(rem_ipip.sport, 80) - assert_equals(rem_ipip.dport, eph_port) + assert rem_ipip.src == self.VIP + assert rem_ipip.dst == self.SELF_HOST + assert rem_ipip.sport == 80 + assert rem_ipip.dport == eph_port def _establish_conn(self, dport): seq_no = random.randint(0, 2**32-1) diff --git a/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py b/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py index 84c78831..053fa1f1 100644 --- a/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py +++ b/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this project. If not, see . -from nose.tools import assert_equals from scapy.all import IP, IPv6, UDP, TCP, ICMPv6EchoRequest, ICMPv6EchoReply, ICMPv6PacketTooBig, sniff, send, conf, L3RawSocket6 from glb_scapy import GLBGUEChainedRouting, GLBGUE from glb_test_utils import GLBTestHelpers @@ -50,8 +49,8 @@ def test_00_icmp_accepted(self): resp_ip = self._sendrecv6(pkt, lfilter=lambda p: isinstance(p, IPv6) and isinstance(p.payload, ICMPv6EchoReply)) assert isinstance(resp_ip, IPv6) - assert_equals(resp_ip.src, self.V4_TO_V6[dst]) - assert_equals(resp_ip.dst, self.SELF_HOST_V6) + assert resp_ip.src == self.V4_TO_V6[dst] + assert resp_ip.dst == self.SELF_HOST_V6 resp_icmp = resp_ip.payload assert isinstance(resp_icmp, ICMPv6EchoReply) @@ -68,14 +67,14 @@ def test_01_syn_accepted(self): # expect a SYN-ACK back from self.PROXY_HOST (decapsulated) resp_ip = self._sendrecv6(pkt, filter='host {} and port 22'.format(self.V4_TO_V6[self.PROXY_HOST])) assert isinstance(resp_ip, IPv6) - assert_equals(resp_ip.src, self.V4_TO_V6[self.PROXY_HOST]) - assert_equals(resp_ip.dst, self.SELF_HOST_V6) + assert resp_ip.src == self.V4_TO_V6[self.PROXY_HOST] + assert resp_ip.dst == self.SELF_HOST_V6 resp_tcp = resp_ip.payload assert isinstance(resp_tcp, TCP) - assert_equals(resp_tcp.sport, 22) - assert_equals(resp_tcp.dport, 123) - assert_equals(resp_tcp.flags, 'SA') + assert resp_tcp.sport == 22 + assert resp_tcp.dport == 123 + assert resp_tcp.flags == 'SA' def test_02_unknown_redirected_through_chain(self): pkt = \ @@ -89,26 +88,26 @@ def test_02_unknown_redirected_through_chain(self): # should arrive from the last host in the chain that wasn't us. resp_ip = self._sendrecv4(pkt, filter='src host {} and udp and port 19523'.format(self.ALT_HOST)) assert isinstance(resp_ip, IP) - assert_equals(resp_ip.src, self.ALT_HOST) # outer FOU will come from penultimate hop - assert_equals(resp_ip.dst, self.SELF_HOST) + assert resp_ip.src == self.ALT_HOST # outer FOU will come from penultimate hop + assert resp_ip.dst == self.SELF_HOST resp_fou = resp_ip.payload assert isinstance(resp_fou, UDP) - assert_equals(resp_fou.sport, 12345) - assert_equals(resp_fou.dport, 19523) + assert resp_fou.sport == 12345 + assert resp_fou.dport == 19523 resp_gue = resp_fou.payload assert isinstance(resp_gue, GLBGUE) resp_inner_ip = resp_gue.payload assert isinstance(resp_inner_ip, IPv6) - assert_equals(resp_inner_ip.src, self.SELF_HOST_V6) - assert_equals(resp_inner_ip.dst, self.VIP) + assert resp_inner_ip.src == self.SELF_HOST_V6 + assert resp_inner_ip.dst == self.VIP resp_inner_tcp = resp_inner_ip.payload assert isinstance(resp_inner_tcp, TCP) - assert_equals(resp_inner_tcp.sport, 9999) - assert_equals(resp_inner_tcp.dport, 22) + assert resp_inner_tcp.sport == 9999 + assert resp_inner_tcp.dport == 22 def test_03_accepted_on_secondary_chain_host(self): eph_port = random.randint(30000, 60000) @@ -133,15 +132,15 @@ def test_03_accepted_on_secondary_chain_host(self): # retrieve the SYN-ACK resp_ip = self._sendrecv6(syn, filter='ip6 host {} and port 22'.format(self.VIP)) assert isinstance(resp_ip, IPv6) - assert_equals(resp_ip.src, self.VIP) - assert_equals(resp_ip.dst, self.SELF_HOST_V6) + assert resp_ip.src == self.VIP + assert resp_ip.dst == self.SELF_HOST_V6 resp_tcp = resp_ip.payload assert isinstance(resp_tcp, TCP) - assert_equals(resp_tcp.sport, 22) - assert_equals(resp_tcp.dport, eph_port) - assert_equals(resp_tcp.flags, 'SA') - assert_equals(resp_tcp.ack, syn.seq + 1) + assert resp_tcp.sport == 22 + assert resp_tcp.dport == eph_port + assert resp_tcp.flags == 'SA' + assert resp_tcp.ack == syn.seq + 1 syn_ack = resp_ip @@ -156,14 +155,14 @@ def test_03_accepted_on_secondary_chain_host(self): # ensure we get a PSH from the host, since SSH should send us the banner resp_ip = self._sendrecv6(ack, filter='ip6 host {} and port 22'.format(self.VIP)) assert isinstance(resp_ip, IPv6) - assert_equals(resp_ip.src, self.VIP) - assert_equals(resp_ip.dst, self.SELF_HOST_V6) + assert resp_ip.src == self.VIP + assert resp_ip.dst == self.SELF_HOST_V6 resp_tcp = resp_ip.payload assert isinstance(resp_tcp, TCP) - assert_equals(resp_tcp.sport, 22) - assert_equals(resp_tcp.dport, eph_port) - assert_equals(resp_tcp.flags, 'PA') + assert resp_tcp.sport == 22 + assert resp_tcp.dport == eph_port + assert resp_tcp.flags == 'PA' def test_04_icmp_packet_too_big(self): # Establish full connection, since ICMP is handled differently for @@ -188,17 +187,17 @@ def test_04_icmp_packet_too_big(self): # ensure the remote host (ALT_HOST) received the inner packet through the first (failed) hop assert isinstance(rem_ip, IPv6) - assert_equals(rem_ip.src, self.ROUTER) - assert_equals(rem_ip.dst, self.VIP) + assert rem_ip.src == self.ROUTER + assert rem_ip.dst == self.VIP rem_icmp = rem_ip.payload assert isinstance(rem_icmp, ICMPv6PacketTooBig) - assert_equals(rem_icmp.mtu, 1400) + assert rem_icmp.mtu == 1400 rem_ipip = rem_icmp.payload assert isinstance(rem_ipip, IPv6) - assert_equals(rem_ipip.src, self.VIP) - assert_equals(rem_ipip.dst, self.SELF_HOST_V6) - assert_equals(rem_ipip.sport, 80) - assert_equals(rem_ipip.dport, eph_port) + assert rem_ipip.src == self.VIP + assert rem_ipip.dst == self.SELF_HOST_V6 + assert rem_ipip.sport == 80 + assert rem_ipip.dport == eph_port def _establish_conn(self, dport): From d3e6594c05c4ac9c4edf766dc06fd7564aa69b01 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 16:29:20 -0400 Subject: [PATCH 17/38] First draft of Dockerfile.noble --- script/Dockerfile.noble | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 script/Dockerfile.noble diff --git a/script/Dockerfile.noble b/script/Dockerfile.noble new file mode 100644 index 00000000..3477ee04 --- /dev/null +++ b/script/Dockerfile.noble @@ -0,0 +1,72 @@ +FROM --platform=linux/amd64 ubuntu:noble@sha256:c4a8d5503dfb2a3eb8ab5f807da5bc69a85730fb49b5cfca2330194ebcc41c7b + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get -y install curl git +RUN add-apt-repository universe + +# DPDK +RUN apt-get update +RUN apt-get install --assume-yes \ + build-essential \ + dpdk=23.11.4-0ubuntu0.24.04.2 \ + dpdk-dev=23.11.4-0ubuntu0.24.04.2 \ + libdpdk-dev=23.11.4-0ubuntu0.24.04.2 \ + wget \ + pkg-config \ + libjansson-dev \ + libsystemd-dev \ + +# iptables / DKMS +RUN apt-get update +RUN apt-get install --assume-yes --fix-broken \ + wget \ + pkg-config \ + libsystemd-dev \ + dkms \ + dh-dkms \ + dpkg-dev \ + fakeroot \ + debhelper \ + libxtables-dev + +# golang +RUN ARCH=$(dpkg --print-architecture) && wget --quiet https://golang.org/dl/go1.24.5.linux-${ARCH}.tar.gz -O- | tar -C /usr/local -zxvf - +ENV GOROOT /usr/local/go +ENV GOPATH /go +ENV GOFLAGS=-buildvcs=false +ENV PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" + + +# fpm for packaging +RUN apt-get update && apt-get install -y ruby ruby-dev rubygems build-essential + +# See fpm dependency breakage issue: https://github.com/jordansissel/fpm/issues/1918 +RUN gem install --version 2.7.6 dotenv +RUN gem install ffi -f +RUN gem install rake fpm + +# XDP +# linux-libc-dev must be upgraded to get a bpf.h that matches what we use. the rest match what we do in Vagrant for testing. +RUN apt-get update && apt install -y apt-transport-https curl software-properties-common +RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-20 clang-tools-20 + + +# Hack because the kernel headers are not installed in the right place (linuxkit vs generic) +RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname -r) + +# Hack for C99 math +RUN sed -i '1s/^/#define __USE_C99_MATH\n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h +RUN sed -i '2s/^/#include \n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h + + +# Python test dependencies (scapy/nose etc.) used by the test suites. +RUN apt-get update && apt-get install -y python3 python3-pip python3-dev +COPY requirements.txt /tmp/requirements.txt +RUN pip3 install --no-cache-dir -r /tmp/requirements.txt + +# valgrind is required by the glb-director test suite +RUN apt-get update && apt-get install -y valgrind + +# netcat and jq are required by the glb-healthcheck test suite +RUN apt-get update && apt-get install -y netcat jq tcpdump From 7e458d23153035521327a2a4431c6c263dc7b758 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 19 May 2026 18:52:33 -0400 Subject: [PATCH 18/38] Build compatability between focal and noble --- Makefile | 12 +++++ script/Dockerfile.noble | 51 ++++++++++++++++----- script/cibuild-create-packages | 10 +++- script/test-local | 8 +++- src/glb-director-xdp/bpf/Makefile | 5 +- src/glb-director-xdp/main.go | 5 ++ src/glb-director-xdp/script/create-packages | 5 +- src/glb-director/cli/main.c | 6 +++ src/glb-director/log.h | 2 +- src/glb-director/script/create-packages | 23 ++++++++++ src/glb-director/shared_opt.c | 6 +++ 11 files changed, 116 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index c94302a5..7b5fc5dc 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,23 @@ +# Set GLB_SKIP_DPDK_DIRECTOR=1 to skip building the DPDK glb-director package. +# In that mode we still build glb-director-cli (a runtime dep of glb-director-xdp) +# via GLB_CLI_ONLY=1. This is used on distros where DPDK 17 / KNI is unavailable +# (e.g. Ubuntu noble). +GLB_SKIP_DPDK_DIRECTOR ?= + mkdeb: make -C src/glb-redirect mkdeb make -C src/glb-healthcheck mkdeb cd src/glb-director-xdp && script/create-packages +ifeq ($(GLB_SKIP_DPDK_DIRECTOR),) cd src/glb-director && script/create-packages +else + cd src/glb-director && GLB_CLI_ONLY=1 script/create-packages +endif clean: make -C src/glb-redirect clean make -C src/glb-healthcheck clean +ifeq ($(GLB_SKIP_DPDK_DIRECTOR),) make -C src/glb-director clean +endif make -C src/glb-director/cli clean diff --git a/script/Dockerfile.noble b/script/Dockerfile.noble index 3477ee04..db4ab130 100644 --- a/script/Dockerfile.noble +++ b/script/Dockerfile.noble @@ -3,19 +3,16 @@ FROM --platform=linux/amd64 ubuntu:noble@sha256:c4a8d5503dfb2a3eb8ab5f807da5bc69 ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get -y install curl git -RUN add-apt-repository universe -# DPDK -RUN apt-get update -RUN apt-get install --assume-yes \ +# Build deps for glb-director-cli and glb-director-xdp. +# Note: we do NOT install DPDK on noble. The DPDK-based glb-director is only +# built on focal; on noble the supported forwarder is glb-director-xdp. +RUN apt-get update && apt-get install --assume-yes \ build-essential \ - dpdk=23.11.4-0ubuntu0.24.04.2 \ - dpdk-dev=23.11.4-0ubuntu0.24.04.2 \ - libdpdk-dev=23.11.4-0ubuntu0.24.04.2 \ wget \ pkg-config \ libjansson-dev \ - libsystemd-dev \ + libsystemd-dev # iptables / DKMS RUN apt-get update @@ -30,6 +27,23 @@ RUN apt-get install --assume-yes --fix-broken \ debhelper \ libxtables-dev +# `dkms mkdeb` was removed in dkms 3.x (which is what noble ships). The +# glb-redirect Makefile uses the older `template-dkms-mkdeb` flow plus +# `dkms mkdeb --source-only`. Restore both by installing the focal dkms +# package alongside dkms 3.x: focal's /etc/dkms/template-dkms-mkdeb/ and the +# `mkdeb` code path in /usr/sbin/dkms are extracted into a parallel location +# and the script is wrapped so `dkms mkdeb ...` calls the legacy binary. +RUN set -eux; \ + cd /tmp; \ + wget -q http://archive.ubuntu.com/ubuntu/pool/main/d/dkms/dkms_2.8.1-5ubuntu2_all.deb; \ + dpkg-deb -x dkms_2.8.1-5ubuntu2_all.deb /tmp/dkms-focal; \ + cp -r /tmp/dkms-focal/etc/dkms/template-dkms-mkdeb /etc/dkms/template-dkms-mkdeb; \ + install -m 0755 /tmp/dkms-focal/usr/sbin/dkms /usr/sbin/dkms-legacy; \ + rm -rf /tmp/dkms-focal /tmp/dkms_2.8.1-5ubuntu2_all.deb; \ + printf '#!/bin/sh\nif [ "$1" = "mkdeb" ]; then exec /usr/sbin/dkms-legacy "$@"; fi\nexec /usr/sbin/dkms.real "$@"\n' > /usr/local/bin/dkms; \ + chmod 0755 /usr/local/bin/dkms; \ + mv /usr/sbin/dkms /usr/sbin/dkms.real + # golang RUN ARCH=$(dpkg --print-architecture) && wget --quiet https://golang.org/dl/go1.24.5.linux-${ARCH}.tar.gz -O- | tar -C /usr/local -zxvf - ENV GOROOT /usr/local/go @@ -51,6 +65,16 @@ RUN gem install rake fpm RUN apt-get update && apt install -y apt-transport-https curl software-properties-common RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-20 clang-tools-20 +# The xdp bpf/Makefile defaults CLANG/LLC to clang-10/llc-10 (focal). On noble +# we ship clang-20 instead, so steer the BPF build at it. +ENV CLANG=clang-20 +ENV LLC=llc-20 +# Noble's 6.8 kernel headers split rwonce.h into arch/x86/include/generated/asm. +# Add only that one path; pulling in the full arch/x86/include drags in +# asm/alternative.h which uses kernel-private types (s32/u16) that the bpf +# clang invocation doesn't have, and that header isn't needed by focal. +ENV EXTRA_BPF_INCLUDES="-I /usr/src/linux-headers-6.10.14-linuxkit/arch/x86/include/generated" + # Hack because the kernel headers are not installed in the right place (linuxkit vs generic) RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname -r) @@ -59,14 +83,19 @@ RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname RUN sed -i '1s/^/#define __USE_C99_MATH\n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h RUN sed -i '2s/^/#include \n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h +# Newer kernel headers (6.8 in noble) added linux/kcsan-checks.h which uses +# `size_t` without pulling stddef.h, breaking the bpf clang build. Inject a +# stddef.h up front, mirroring the kasan-checks.h hack above. +RUN sed -i '1s|^|#include \n|' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kcsan-checks.h + -# Python test dependencies (scapy/nose etc.) used by the test suites. +# Python test dependencies (scapy/pytest etc.) used by the test suites. RUN apt-get update && apt-get install -y python3 python3-pip python3-dev COPY requirements.txt /tmp/requirements.txt -RUN pip3 install --no-cache-dir -r /tmp/requirements.txt +RUN pip3 install --no-cache-dir --break-system-packages -r /tmp/requirements.txt # valgrind is required by the glb-director test suite RUN apt-get update && apt-get install -y valgrind # netcat and jq are required by the glb-healthcheck test suite -RUN apt-get update && apt-get install -y netcat jq tcpdump +RUN apt-get update && apt-get install -y netcat-openbsd jq tcpdump diff --git a/script/cibuild-create-packages b/script/cibuild-create-packages index 4d1d2cd4..386900eb 100755 --- a/script/cibuild-create-packages +++ b/script/cibuild-create-packages @@ -30,10 +30,18 @@ begin_fold "Building packages" rm -rf tmp/build/ mkdir -p tmp/build/ + # The DPDK glb-director sub-build only works on focal (DPDK 17 / KNI). + # On other distros we skip it but still produce glb-director-cli (a runtime + # dep of glb-director-xdp). + EXTRA_MAKE_ENV="" + if [ "$DISTRO" != "focal" ]; then + EXTRA_MAKE_ENV="GLB_SKIP_DPDK_DIRECTOR=1" + fi + docker run --rm \ --volume "$HOSTPATH":/glb-director \ "glb-director-build-$DISTRO" \ bash -c "cd /glb-director && - make BUILDDIR=/glb-director/tmp/build clean mkdeb" + make BUILDDIR=/glb-director/tmp/build $EXTRA_MAKE_ENV clean mkdeb" ) end_fold \ No newline at end of file diff --git a/script/test-local b/script/test-local index 76fb2ec3..f029cdbb 100755 --- a/script/test-local +++ b/script/test-local @@ -25,7 +25,13 @@ cd "$HOSTPATH" echo "==> Building Docker image for ${DISTRO}..." docker build --platform linux/amd64 --file "${DOCKERFILE}" --tag "${IMAGE}" . -TEST_SUITES=(director director-xdp healthcheck redirect) +# The DPDK-based glb-director only builds on focal (DPDK 17 / KNI). +# On other distros, skip that suite; XDP is the supported forwarder. +if [[ "$DISTRO" == "focal" ]]; then + TEST_SUITES=(director director-xdp healthcheck redirect) +else + TEST_SUITES=(director-xdp healthcheck redirect) +fi for suite in "${TEST_SUITES[@]}"; do echo "" diff --git a/src/glb-director-xdp/bpf/Makefile b/src/glb-director-xdp/bpf/Makefile index fa114682..51e8c556 100644 --- a/src/glb-director-xdp/bpf/Makefile +++ b/src/glb-director-xdp/bpf/Makefile @@ -1,7 +1,7 @@ all: glb_encap.o glb_encap_trace.o passer.o tailcall.o -CLANG=clang-10 -LLC=llc-10 +CLANG?=clang-10 +LLC?=llc-10 ifeq ($(KVER),) KVER=$(shell uname -r) @@ -23,6 +23,7 @@ endif -I /usr/src/linux-headers-$(KVER:-amd64=-common)/include/uapi \ -I /usr/src/linux-headers-$(KVER:-amd64=-common)/include \ -I /usr/src/linux-headers-$(KVER)/include \ + $(EXTRA_BPF_INCLUDES) \ -I include/ \ -I ../.. \ $< > .tmp.ll diff --git a/src/glb-director-xdp/main.go b/src/glb-director-xdp/main.go index 07e3d602..3afca1cb 100644 --- a/src/glb-director-xdp/main.go +++ b/src/glb-director-xdp/main.go @@ -97,6 +97,11 @@ typedef struct { #include "../glb-director/glb_fwd_config.c" +// Single definition of the global debug flag declared (extern) in log.h. +// glb_fwd_config.c -> log.h is the only translation unit in this xdp build +// that references it, so define it once here. +bool debug = false; + // cgo borked zero-size trailing arrays, so we return the pointer instead. struct glb_fwd_config_content_table *_get_tables(struct glb_fwd_config_content *content) { return content->tables; diff --git a/src/glb-director-xdp/script/create-packages b/src/glb-director-xdp/script/create-packages index 8288edea..dd2578dc 100755 --- a/src/glb-director-xdp/script/create-packages +++ b/src/glb-director-xdp/script/create-packages @@ -36,7 +36,10 @@ cd $ROOTDIR . packaging/version.sh -make -C ../glb-director/cli +# Only the cli tool is needed to build forwarding tables; avoid building the +# rest of the cli/ targets (glb-config-check, glb-director-pcap, ...) since +# they link DPDK and won't compile on distros without DPDK 17 (e.g. noble). +make -C ../glb-director/cli glb-director-cli make ../glb-director/cli/glb-director-cli build-config ../glb-director/packaging/forwarding_table.json ../glb-director/packaging/forwarding_table.bin diff --git a/src/glb-director/cli/main.c b/src/glb-director/cli/main.c index b6d539e4..0302ab38 100644 --- a/src/glb-director/cli/main.c +++ b/src/glb-director/cli/main.c @@ -43,6 +43,12 @@ #include "log.h" +/* Standalone definition of the global debug flag declared in log.h. + * The cli/glb-director-cli target compiles only main.c + siphash24.c, so it + * needs to provide its own definition (other targets get it from shared_opt.c + * or, for test-check-config, from tests/test_check_config.c). */ +bool debug = false; + #define GLB_BACKEND_HEALTH_DOWN 0 #define GLB_BACKEND_HEALTH_UP 1 diff --git a/src/glb-director/log.h b/src/glb-director/log.h index 58c4da2c..cd8e8cde 100644 --- a/src/glb-director/log.h +++ b/src/glb-director/log.h @@ -36,7 +36,7 @@ #include #define MAX_MESSAGE_SZ 1024 -bool debug; +extern bool debug; // outputs formatted logs to stdout or stderr diff --git a/src/glb-director/script/create-packages b/src/glb-director/script/create-packages index b3811ef3..dbccb26c 100755 --- a/src/glb-director/script/create-packages +++ b/src/glb-director/script/create-packages @@ -36,6 +36,29 @@ cd $ROOTDIR . packaging/version.sh +# GLB_CLI_ONLY=1 builds and packages only glb-director-cli (no DPDK). +# Used on distros where the DPDK director is not built (e.g. noble), but where +# glb-director-cli is still required as a runtime dep of glb-director-xdp. +if [ "${GLB_CLI_ONLY:-0}" = "1" ]; then + make -C cli clean + make -C cli glb-director-cli + + fpm -f -s dir -t deb \ + -n glb-director-cli \ + -v ${GLB_DIRECTOR_VERSION} \ + -d 'libjansson4' \ + --license 'BSD 3-Clause' \ + --maintainer 'GitHub ' \ + cli/glb-director-cli=/usr/sbin/ + + if [ -d "$BUILDDIR" ]; then + cp glb-director-cli_${GLB_DIRECTOR_VERSION}_amd64.deb $BUILDDIR/ + fi + + make -C cli clean + exit 0 +fi + make clean make -C cli clean diff --git a/src/glb-director/shared_opt.c b/src/glb-director/shared_opt.c index 5070a65b..d60a6448 100644 --- a/src/glb-director/shared_opt.c +++ b/src/glb-director/shared_opt.c @@ -33,6 +33,12 @@ #include "shared_opt.h" #include +/* Single definition of the global debug flag declared in log.h / shared_opt.h. + * Newer GCC defaults to -fno-common, so a tentative definition in the header + * (the original `bool debug;` in log.h) is rejected as a duplicate symbol when + * multiple translation units include it. */ +bool debug; + /* parses --config-file, --forwarding-table, and --debug cli options */ void get_options(char *config_file, char *forwarding_table, int argc, From 419c02fc9147361ea5c1e2a3e68f1e5a1614e147 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 20 May 2026 09:59:04 -0400 Subject: [PATCH 19/38] Improvements from copilot --- src/glb-director/tests/glb_test_utils.py | 2 +- src/glb-healthcheck/test/lib.sh | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index 320ba212..66992cae 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -88,7 +88,7 @@ def setup(self, iface): stderr=subprocess.STDOUT, ) - print(('launched as pid', self.director.pid)) + print('launched as pid', self.director.pid) ip = IPRoute() diff --git a/src/glb-healthcheck/test/lib.sh b/src/glb-healthcheck/test/lib.sh index 7126caf9..7a2fd8e1 100644 --- a/src/glb-healthcheck/test/lib.sh +++ b/src/glb-healthcheck/test/lib.sh @@ -144,12 +144,11 @@ skip_test () { # default forwarding table are reachable on their HTTP healthcheck port. # Used to decide whether to skip tests that depend on real backends being up. proxy_backends_available () { - # quick TCP probe with a short timeout; both proxies must answer on :80 + # quick TCP probe with an explicit short timeout; both proxies must answer on :80 for ip in 192.168.50.10 192.168.50.11; do - if ! (exec 3<>/dev/tcp/$ip/80) 2>/dev/null; then + if ! nc -z -w 1 "$ip" 80 >/dev/null 2>&1; then return 1 fi - exec 3<&- 3>&- 2>/dev/null || true done return 0 } From 8a2426e6b2292b7896b826f84a4c97d495c58d18 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 20 May 2026 10:11:18 -0400 Subject: [PATCH 20/38] Run build and test for noble in github action --- .github/workflows/ci.yml | 2 +- .github/workflows/test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3527e735..27a4ccfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - distro: [focal] + distro: [focal, noble] steps: - uses: actions/checkout@v6 - name: Run package build ${{ matrix.distro }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60cfe3b2..6c97c6bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - distro: [focal] + distro: [focal, noble] steps: - name: Checkout code uses: actions/checkout@v3 @@ -48,7 +48,7 @@ jobs: fail-fast: false matrix: test-suite: [director, director-xdp, healthcheck, redirect] - distro: [focal] + distro: [focal, noble] steps: - name: Checkout code uses: actions/checkout@v3 From 0189ff4288d18b5be1dd4009dd6d922ecec0fd93 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 20 May 2026 13:41:29 -0400 Subject: [PATCH 21/38] BPF compatability in ubuntu noble --- .../xdp-root-shim/xdp-root-shim.c | 58 ++++++++++++++++++- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/glb-director-xdp/xdp-root-shim/xdp-root-shim.c b/src/glb-director-xdp/xdp-root-shim/xdp-root-shim.c index ec66f072..daae90d3 100644 --- a/src/glb-director-xdp/xdp-root-shim/xdp-root-shim.c +++ b/src/glb-director-xdp/xdp-root-shim/xdp-root-shim.c @@ -38,6 +38,18 @@ #include #include +/* libbpf 0.6 introduced bpf/libbpf_version.h with LIBBPF_MAJOR_VERSION. + * Older releases (e.g. libbpf 0.0.6 shipped on Ubuntu focal) don't have it. */ +#if __has_include() +#include +#endif + +#if defined(LIBBPF_MAJOR_VERSION) && LIBBPF_MAJOR_VERSION >= 1 +#define GLB_LIBBPF_MODERN 1 +#else +#define GLB_LIBBPF_MODERN 0 +#endif + int main(int argc, char **argv) { if (argc != 4) { fprintf(stderr, "Usage: %s \n", argv[0]); @@ -70,7 +82,38 @@ int main(int argc, char **argv) { return 1; } - /* load the tailcall bpf */ + /* load the tailcall bpf + * + * libbpf 1.0+ (shipped on ubuntu noble) removed bpf_prog_load_xattr() and + * bpf_set_link_xdp_fd(). Use the modern open-file / load / attach API on + * noble, and fall back to the legacy API on focal (libbpf 0.0.x). + */ +#if GLB_LIBBPF_MODERN + struct bpf_object *shim_obj = bpf_object__open_file(tailcall_elf_path, NULL); + if (!shim_obj || libbpf_get_error(shim_obj)) { + fprintf(stderr, "Could not open '%s'\n", tailcall_elf_path); + return 1; + } + + /* force every program in the object to load as XDP */ + struct bpf_program *prog; + bpf_object__for_each_program(prog, shim_obj) { + bpf_program__set_type(prog, BPF_PROG_TYPE_XDP); + } + + if (bpf_object__load(shim_obj) != 0) { + fprintf(stderr, "Could not load '%s'\n", tailcall_elf_path); + return 1; + } + + /* find the first XDP program in the object to attach to the iface */ + prog = bpf_object__next_program(shim_obj, NULL); + if (!prog) { + fprintf(stderr, "No BPF programs found in '%s'\n", tailcall_elf_path); + return 1; + } + int prog_fd = bpf_program__fd(prog); +#else struct bpf_prog_load_attr prog_load_attr = { .prog_type = BPF_PROG_TYPE_XDP, .file = tailcall_elf_path, @@ -82,18 +125,27 @@ int main(int argc, char **argv) { fprintf(stderr, "Could not load '%s'\n", prog_load_attr.file); return 1; } +#endif - /* pin the map to the */ + /* pin the map to the bpffs */ struct bpf_map *root_array = bpf_object__find_map_by_name(shim_obj, "root_array"); + if (!root_array) { + fprintf(stderr, "Could not find map 'root_array' in '%s'\n", tailcall_elf_path); + return 1; + } unlink(bpffs_path); if (bpf_map__pin(root_array, bpffs_path) != 0) { fprintf(stderr, "Could not pin root array map to '%s'\n", bpffs_path); return 1; } - + /* bind it to the interface with XDP */ +#if GLB_LIBBPF_MODERN + if (bpf_xdp_attach(iface_index, prog_fd, 0, NULL) < 0) { +#else if (bpf_set_link_xdp_fd(iface_index, prog_fd, 0) < 0) { +#endif fprintf(stderr, "Could not attach XDP program to interface '%s'\n", iface_name); return 1; } From 89a776b7720e964ee8cde9e652822afe2a11a400 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 20 May 2026 13:59:23 -0400 Subject: [PATCH 22/38] Fix the makepath for glb-director-cli dependency in glb-director-xdp --- src/glb-director-xdp/script/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/glb-director-xdp/script/test b/src/glb-director-xdp/script/test index 9ad0844a..ea51ad17 100755 --- a/src/glb-director-xdp/script/test +++ b/src/glb-director-xdp/script/test @@ -42,7 +42,7 @@ export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin begin_fold "Building glb-director-xdp for testing" ( - make -j4 -C ../glb-director/cli + make -j4 -C ../glb-director/cli glb-director-cli make # scan-build bricks go ) end_fold From f7a69576b6cc75e40257f89cef64b2c77704a03d Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 20 May 2026 14:24:04 -0400 Subject: [PATCH 23/38] Update the glb-director-xdp makefile to use a kernel agnostic path for compiler.h --- script/Dockerfile.noble | 5 ----- src/glb-director-xdp/bpf/Makefile | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/script/Dockerfile.noble b/script/Dockerfile.noble index db4ab130..75e13dfd 100644 --- a/script/Dockerfile.noble +++ b/script/Dockerfile.noble @@ -69,11 +69,6 @@ RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-20 # we ship clang-20 instead, so steer the BPF build at it. ENV CLANG=clang-20 ENV LLC=llc-20 -# Noble's 6.8 kernel headers split rwonce.h into arch/x86/include/generated/asm. -# Add only that one path; pulling in the full arch/x86/include drags in -# asm/alternative.h which uses kernel-private types (s32/u16) that the bpf -# clang invocation doesn't have, and that header isn't needed by focal. -ENV EXTRA_BPF_INCLUDES="-I /usr/src/linux-headers-6.10.14-linuxkit/arch/x86/include/generated" # Hack because the kernel headers are not installed in the right place (linuxkit vs generic) diff --git a/src/glb-director-xdp/bpf/Makefile b/src/glb-director-xdp/bpf/Makefile index 51e8c556..7bb1f105 100644 --- a/src/glb-director-xdp/bpf/Makefile +++ b/src/glb-director-xdp/bpf/Makefile @@ -23,6 +23,7 @@ endif -I /usr/src/linux-headers-$(KVER:-amd64=-common)/include/uapi \ -I /usr/src/linux-headers-$(KVER:-amd64=-common)/include \ -I /usr/src/linux-headers-$(KVER)/include \ + -I /usr/src/linux-headers-$(KVER)/arch/x86/include/generated \ $(EXTRA_BPF_INCLUDES) \ -I include/ \ -I ../.. \ From 303faee371705ad9dfae72dea7e4a37fb1357f5c Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 20 May 2026 15:05:22 -0400 Subject: [PATCH 24/38] Update the xdp BPF structs for compatability with noble --- src/glb-director-xdp/bpf/Makefile | 2 +- src/glb-director-xdp/bpf/glb_encap.c | 73 ++++++++++++++------------ src/glb-director-xdp/bpf/tailcall.c | 12 ++--- src/glb-director-xdp/bpf/xdpcap_hook.h | 11 +++- 4 files changed, 56 insertions(+), 42 deletions(-) diff --git a/src/glb-director-xdp/bpf/Makefile b/src/glb-director-xdp/bpf/Makefile index 7bb1f105..c0bc48db 100644 --- a/src/glb-director-xdp/bpf/Makefile +++ b/src/glb-director-xdp/bpf/Makefile @@ -8,7 +8,7 @@ KVER=$(shell uname -r) endif %.o: %.c - $(CLANG) -c -O2 -emit-llvm -o - -D__KERNEL__ \ + $(CLANG) -c -O2 -emit-llvm -o - -D__KERNEL__ -g \ -Wall \ -Wno-gnu-variable-sized-type-not-at-end \ -Wno-address-of-packed-member \ diff --git a/src/glb-director-xdp/bpf/glb_encap.c b/src/glb-director-xdp/bpf/glb_encap.c index 50bfc448..1badc98e 100644 --- a/src/glb-director-xdp/bpf/glb_encap.c +++ b/src/glb-director-xdp/bpf/glb_encap.c @@ -52,7 +52,12 @@ typedef struct { /* xdpcap integration */ #include "xdpcap_hook.h" -struct bpf_map_def SEC("maps") xdpcap_hook = XDPCAP_HOOK(); +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); + __uint(max_entries, 5); +} xdpcap_hook SEC(".maps"); typedef struct glb_bind { uint32_t ipv4; @@ -61,11 +66,11 @@ typedef struct glb_bind { uint16_t port; } __attribute__((__packed__)) glb_bind_t; -struct bpf_map_def SEC("maps") config_bits = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(uint32_t), - .value_size = 6, // maximum size stored - .max_entries = 5, +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, sizeof(uint32_t)); + __uint(value_size, 6); // maximum size stored + __uint(max_entries, 5); /* 0: 6 byes of gateway dst MAC @@ -74,36 +79,36 @@ struct bpf_map_def SEC("maps") config_bits = { 3: 4 bytes of glb_director_hash_fields 4: 4 bytes of glb_director_hash_fields (alt) */ -}; - -struct bpf_map_def SEC("maps") glb_binds = { - .type = BPF_MAP_TYPE_HASH, - .key_size = sizeof(struct glb_bind), - .value_size = sizeof(uint32_t), - .max_entries = BPF_MAX_BINDS, -}; - -struct bpf_map_def SEC("maps") glb_tables = { - .type = BPF_MAP_TYPE_ARRAY_OF_MAPS, - .key_size = sizeof(uint32_t), - .max_entries = 4096, -}; - -struct bpf_map_def SEC("maps") glb_table_secrets = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(uint32_t), +} config_bits SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct glb_bind)); + __uint(value_size, sizeof(uint32_t)); + __uint(max_entries, BPF_MAX_BINDS); +} glb_binds SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(key_size, sizeof(uint32_t)); + __uint(max_entries, 4096); +} glb_tables SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, sizeof(uint32_t)); #define GLB_FMT_SECURE_KEY_BYTES 16 - .value_size = GLB_FMT_SECURE_KEY_BYTES, - .max_entries = 4096, -}; - -struct bpf_map_def SEC("maps") glb_global_packet_counters = { - .type = BPF_MAP_TYPE_PERCPU_ARRAY, - .key_size = sizeof(uint32_t), - .value_size = sizeof(struct glb_global_stats), + __uint(value_size, GLB_FMT_SECURE_KEY_BYTES); + __uint(max_entries, 4096); +} glb_table_secrets SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(uint32_t)); + __uint(value_size, sizeof(struct glb_global_stats)); /* we don't actually need an array, but PERCPU_* only has multi-element types */ - .max_entries = 1, -}; + __uint(max_entries, 1); +} glb_global_packet_counters SEC(".maps"); static __always_inline uint16_t compute_ipv4_checksum(void *iph) { uint16_t *iph16 = (uint16_t *)iph; diff --git a/src/glb-director-xdp/bpf/tailcall.c b/src/glb-director-xdp/bpf/tailcall.c index 92b330da..839667f2 100644 --- a/src/glb-director-xdp/bpf/tailcall.c +++ b/src/glb-director-xdp/bpf/tailcall.c @@ -19,12 +19,12 @@ #define ROOT_ARRAY_SIZE 3 -struct bpf_map_def SEC("maps") root_array = { - .type = BPF_MAP_TYPE_PROG_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(__u32), - .max_entries = ROOT_ARRAY_SIZE, -}; +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, ROOT_ARRAY_SIZE); +} root_array SEC(".maps"); SEC("xdp-root") int xdp_root(struct xdp_md *ctx) { diff --git a/src/glb-director-xdp/bpf/xdpcap_hook.h b/src/glb-director-xdp/bpf/xdpcap_hook.h index c190161d..3cd72ac2 100644 --- a/src/glb-director-xdp/bpf/xdpcap_hook.h +++ b/src/glb-director-xdp/bpf/xdpcap_hook.h @@ -38,7 +38,16 @@ * Create a bpf map suitable for use as an xdpcap hook point. * * For example: - * struct bpf_map_def xdpcap_hook = XDPCAP_HOOK(); + * struct { + * __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + * __uint(key_size, sizeof(int)); + * __uint(value_size, sizeof(int)); + * __uint(max_entries, 5); + * } xdpcap_hook SEC(".maps"); + * + * The legacy XDPCAP_HOOK() initializer macro is kept for source compat with + * older callers but new code should declare the map directly using the BTF + * style above (libbpf 1.0+ rejects the legacy "maps" section). */ #define XDPCAP_HOOK() { \ .type = BPF_MAP_TYPE_PROG_ARRAY, \ From e3a0e2c96ec793976ed35603392e10c8ea10713b Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 20 May 2026 15:13:14 -0400 Subject: [PATCH 25/38] Exclude noble director from test --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c97c6bd..585d1056 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,6 +49,13 @@ jobs: matrix: test-suite: [director, director-xdp, healthcheck, redirect] distro: [focal, noble] + exclude: + # The DPDK-based glb-director only builds on focal (DPDK 17 / KNI). + # On other distros the supported forwarder is glb-director-xdp, so + # skip the director suite there. This mirrors the same exclusion in + # script/test-local. + - distro: noble + test-suite: director steps: - name: Checkout code uses: actions/checkout@v3 From ccdfa21a7c4b4a4145cd119bac1dc5cac9c168df Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Tue, 26 May 2026 15:15:09 -0400 Subject: [PATCH 26/38] Add build artifacts to .gitignore, better handling in glb_test_util.py --- .github/workflows/test.yml | 7 ++-- .gitignore | 10 +++++ src/glb-director/tests/glb_test_utils.py | 11 +++-- src/glb-director/tests/test-config.json | 2 +- .../tests/glb_test_remote_snoop.py | 2 +- src/glb-redirect/tests/glb_test_utils.py | 42 +++++++++++++------ 6 files changed, 53 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60cfe3b2..a357ae53 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,7 @@ on: branches: - main - master + workflow_dispatch: permissions: contents: read @@ -22,7 +23,7 @@ jobs: distro: [focal] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -51,7 +52,7 @@ jobs: distro: [focal] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Download image artifact uses: actions/download-artifact@v4 @@ -69,4 +70,4 @@ jobs: --volume $(pwd):/workspace \ --workdir /workspace \ glb-director-build-${{ matrix.distro }}:latest \ - bash -c "cd /workspace/src/glb-${{ matrix.test-suite }} && script/test" \ No newline at end of file + bash -c "cd /workspace/src/glb-${{ matrix.test-suite }} && script/test" diff --git a/.gitignore b/.gitignore index b8d3ee0e..10173c86 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,13 @@ src/glb-redirect/*/*.mod *.symvers *.order *.deb +*.bin + +# Runtime artifacts produced by script/test-local / the test suites +src/glb-director/director-output.txt +src/glb-director/cli/test-check-config +src/glb-director/tests/director-config.json +src/glb-director/tests/test-tables.json +src/glb-director-xdp/glb-director-xdp +src/glb-healthcheck/reload.txt +src/glb-healthcheck/log/* \ No newline at end of file diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index 66992cae..fb244416 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -205,6 +205,11 @@ def __init__(self): err = probe.stderr.decode('utf-8', errors='replace').strip() raise SkipTest("Kernel ({}) cannot attach XDP to veth: {}".format(kernel_release, err)) finally: + # If the attach above succeeded the veth still has XDP loaded; + # on some kernels `ip link del` then returns ENOTSUP. Detach + # first, mirroring the teardown_class workaround below. + subprocess.call(['ip', 'link', 'set', 'dev', probe_a, 'xdp', 'off'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) subprocess.call(['ip', 'link', 'del', 'dev', probe_a], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -253,7 +258,7 @@ def launch_director(self): env=notify_director.updated_env(), ) - print(('launched as pid', self.director.pid)) + print('launched as pid', self.director.pid) notify_director.wait() @@ -480,12 +485,12 @@ def wait_for_packet(self, iface, condition, timeout_seconds=5): # Newer scapy L2ListenSocket no longer exposes `.iff`; fall back to # `.iface` and finally repr() so the print never crashes the test. iface_name = getattr(iface, 'iff', None) or getattr(iface, 'iface', None) or repr(iface) - print(('Waiting for packets on', iface_name, 'with timeout', timeout_seconds)) + print('Waiting for packets on', iface_name, 'with timeout', timeout_seconds) try: with timeout(timeout_seconds): while True: packet = iface.recv(MTU) - print((repr(packet))) + print(repr(packet)) if condition(packet): return packet except: diff --git a/src/glb-director/tests/test-config.json b/src/glb-director/tests/test-config.json index 59b07e6f..6e703b25 100644 --- a/src/glb-director/tests/test-config.json +++ b/src/glb-director/tests/test-config.json @@ -77,4 +77,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/src/glb-redirect/tests/glb_test_remote_snoop.py b/src/glb-redirect/tests/glb_test_remote_snoop.py index c010447d..1bc18a58 100644 --- a/src/glb-redirect/tests/glb_test_remote_snoop.py +++ b/src/glb-redirect/tests/glb_test_remote_snoop.py @@ -49,7 +49,7 @@ def recv(self, recv_filter, timeout=10): pkt_ether = Ether(pkt_raw) pkt = pkt_ether.payload - if self.debug: print(("got packet from {}: {}".format(self.remote_host, repr(pkt)))) + if self.debug: print("got packet from {}: {}".format(self.remote_host, repr(pkt))) if recv_filter(pkt): if self.debug: print(" -> match!") return pkt diff --git a/src/glb-redirect/tests/glb_test_utils.py b/src/glb-redirect/tests/glb_test_utils.py index 95ee2f71..73b2fcfb 100644 --- a/src/glb-redirect/tests/glb_test_utils.py +++ b/src/glb-redirect/tests/glb_test_utils.py @@ -21,20 +21,34 @@ from nose.plugins.skip import SkipTest +def _tcp_probe(host, port, timeout=0.2): + """Return True iff a TCP connect to host:port succeeds within `timeout`.""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + rc = s.connect_ex((host, port)) + s.close() + return rc == 0 + except OSError: + return False + + def _proxy_backends_available(): """Return True iff the Vagrant proxy backends (proxy1/proxy2) used by these tests are reachable. They live in the Vagrant `glb_datacenter_network` - and aren't present when running under script/test-local in Docker.""" + and aren't present when running under script/test-local in Docker. + + Both SSH (22) and the test-snoop helper (9999) must answer: the + multi-host scenarios in test_glb_redirect_v*_on_v*.py use + RemoteSnoop -> tcp/9999 on each proxy, and probing only SSH would let + those tests run (and then hang on the SYNC handshake) on lab hosts + where test-snoop.service hasn't been started.""" for host in ('192.168.50.10', '192.168.50.11'): - try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(0.2) - # port 22 is used as a liveness probe in the actual tests - rc = s.connect_ex((host, 22)) - s.close() - if rc != 0: - return False - except OSError: + # port 22 is used as a liveness probe in the actual tests + if not _tcp_probe(host, 22): + return False + # port 9999 is script/helpers/test-snoop.py, required by RemoteSnoop + if not _tcp_probe(host, 9999): return False return True @@ -46,9 +60,11 @@ def skip_if_no_vagrant_network(): the Docker test image.""" if not _proxy_backends_available(): raise SkipTest( - "Vagrant proxy backends (192.168.50.10/11) not reachable; " - "glb-redirect tests require the Vagrant test network with the " - "glb-redirect iptables module installed on proxy1/proxy2.") + "Vagrant proxy backends (192.168.50.10/11) not reachable on " + "both ssh/22 and test-snoop/9999; glb-redirect tests require " + "the Vagrant test network with the glb-redirect iptables " + "module installed on proxy1/proxy2 and test-snoop.service " + "running.") class GLBTestHelpers(object): From 41ee404a08121bad6547bcf1d0707e6e6e45e431 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 27 May 2026 16:02:40 -0400 Subject: [PATCH 27/38] Add pytest.ini for glb-redirect --- src/glb-redirect/pytest.ini | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/glb-redirect/pytest.ini diff --git a/src/glb-redirect/pytest.ini b/src/glb-redirect/pytest.ini new file mode 100644 index 00000000..d9f7173d --- /dev/null +++ b/src/glb-redirect/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +markers = + director_dpdk: tests requiring the DPDK director runtime + director_xdp: tests requiring the XDP director runtime + +git \ No newline at end of file From 9181172abb8c4073f4757981beebc3553b37aa0f Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 27 May 2026 16:31:40 -0400 Subject: [PATCH 28/38] Remove usused import. Write to devnull to avoid deadlock --- src/glb-director/tests/glb_test_utils.py | 2 +- src/glb-redirect/tests/glb_test_utils.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index fb244416..13dd5ed4 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -470,7 +470,7 @@ def teardown_class(cls): for iface in (cls.IFACE_NAME_DIRECTOR, cls.IFACE_NAME_PY): if len(ip.link_lookup(ifname=iface)) > 0: rc = subprocess.call(['ip', 'link', 'del', 'dev', iface], - stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if rc != 0: # Fall back to pyroute2; surface any error rather than # masking it (matches the previous behaviour). diff --git a/src/glb-redirect/tests/glb_test_utils.py b/src/glb-redirect/tests/glb_test_utils.py index 73b2fcfb..8919f725 100644 --- a/src/glb-redirect/tests/glb_test_utils.py +++ b/src/glb-redirect/tests/glb_test_utils.py @@ -16,7 +16,6 @@ # along with this project. If not, see . from scapy.all import sniff, send, L3RawSocket, L3RawSocket6 -import os import socket from nose.plugins.skip import SkipTest From 45ca6cb28df23a34694c8a6d4cd8f0394a0ba286 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Mon, 1 Jun 2026 11:29:10 -0400 Subject: [PATCH 29/38] Remove conflict markers --- script/Dockerfile.focal | 4 ---- src/glb-director/tests/glb_test_utils.py | 5 ---- src/glb-director/tests/test_cli_tool.py | 23 ------------------- .../tests/test_rendezvous_table.py | 5 ---- src/glb-redirect/tests/glb_test_utils.py | 5 ---- 5 files changed, 42 deletions(-) diff --git a/script/Dockerfile.focal b/script/Dockerfile.focal index 4e25977f..8fae6957 100644 --- a/script/Dockerfile.focal +++ b/script/Dockerfile.focal @@ -44,11 +44,7 @@ RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname RUN sed -i '1s/^/#define __USE_C99_MATH\n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h RUN sed -i '2s/^/#include \n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h -<<<<<<< HEAD -# Python test dependencies (scapy/pytest etc.) used by the test suites. -======= # Python test dependencies (scapy/nose etc.) used by the test suites. ->>>>>>> origin/master RUN apt-get update && apt-get install -y python3 python3-pip python3-dev COPY requirements.txt /tmp/requirements.txt RUN pip3 install --no-cache-dir -r /tmp/requirements.txt diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index 15875cb1..0f16386a 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -21,12 +21,7 @@ from scapy.all import sniff, sendp, Ether, IP, IPv6, MTU, Packet, UDP, TCP, bind_layers, ICMP, ICMPv6PacketTooBig, conf from scapy.arch.linux import L2ListenSocket from pyroute2 import IPRoute, NetlinkError -<<<<<<< HEAD from unittest import SkipTest -======= -from nose.tools import assert_equals -from nose.plugins.skip import SkipTest ->>>>>>> origin/master import subprocess, time import signal from contextlib import contextmanager diff --git a/src/glb-director/tests/test_cli_tool.py b/src/glb-director/tests/test_cli_tool.py index abf52d6c..9e152e23 100644 --- a/src/glb-director/tests/test_cli_tool.py +++ b/src/glb-director/tests/test_cli_tool.py @@ -74,11 +74,7 @@ def test_generate_configs(self): subprocess.check_call(['cli/glb-director-cli', 'build-config', 'tests/test-config.json', 'tests/test-config.bin']) f = open('tests/test-config.bin', 'rb') -<<<<<<< HEAD assert f.read(4) == b'GLBD' -======= - assert_equals(f.read(4), b'GLBD') ->>>>>>> origin/master num_table_entries = 0x10000 max_num_backends = 0x100 @@ -111,17 +107,10 @@ def test_generate_configs(self): assert inet_family == 2 assert inet_addr == socket.inet_pton(socket.AF_INET6, backend['ip']) else: -<<<<<<< HEAD assert inet_family == 1 assert inet_addr == socket.inet_pton(socket.AF_INET, backend['ip']).ljust(16, b'\x00') assert be_state == 1 assert be_health == 1 -======= - assert_equals(inet_family, 1) - assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, backend['ip']).ljust(16, b'\x00')) - assert_equals(be_state, 1) - assert_equals(be_health, 1) ->>>>>>> origin/master # validate binds for this table num_binds, = struct.unpack('>>>>>> origin/master # validate table entries for table_index in range(num_table_entries): diff --git a/src/glb-director/tests/test_rendezvous_table.py b/src/glb-director/tests/test_rendezvous_table.py index 2ffb7829..3ef17c79 100644 --- a/src/glb-director/tests/test_rendezvous_table.py +++ b/src/glb-director/tests/test_rendezvous_table.py @@ -23,13 +23,8 @@ def test_row_seeds(self): forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) -<<<<<<< HEAD assert table.calculate_forwarding_table_row_seed(0x0000).hex() == '491c53a72df4c837' assert table.calculate_forwarding_table_row_seed(0xffff).hex() == 'f223c0cc65161620' -======= - assert_equals(table.calculate_forwarding_table_row_seed(0x0000).hex(), '491c53a72df4c837') - assert_equals(table.calculate_forwarding_table_row_seed(0xffff).hex(), 'f223c0cc65161620') ->>>>>>> origin/master def test_order_hosts_0000(self): """ diff --git a/src/glb-redirect/tests/glb_test_utils.py b/src/glb-redirect/tests/glb_test_utils.py index c8430f88..6e38a522 100644 --- a/src/glb-redirect/tests/glb_test_utils.py +++ b/src/glb-redirect/tests/glb_test_utils.py @@ -17,12 +17,7 @@ from scapy.all import sniff, send, L3RawSocket, L3RawSocket6 import socket -<<<<<<< HEAD from unittest import SkipTest -======= -from nose.plugins.skip import SkipTest ->>>>>>> origin/master - def _tcp_probe(host, port, timeout=0.2): """Return True iff a TCP connect to host:port succeeds within `timeout`.""" From b2a39183fc6f49c016faec1bf02ecba6f80f1aec Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Mon, 1 Jun 2026 11:31:29 -0400 Subject: [PATCH 30/38] Conflict marker --- src/glb-director/tests/glb_test_utils.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index 0f16386a..e0eed766 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -474,13 +474,8 @@ def teardown_class(cls): # Fall back to pyroute2; surface any error rather than # masking it (matches the previous behaviour). ip.link('remove', ifname=iface) -<<<<<<< HEAD assert len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)) == 0 assert len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)) == 0 -======= - assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_PY)), 0) - assert_equals(len(ip.link_lookup(ifname=cls.IFACE_NAME_DIRECTOR)), 0) ->>>>>>> origin/master def sendp(self, *args, **kwargs): sendp(*args, **kwargs) From 1f3a066b0c11a49e9de9c8d24b83252cc4842f03 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Mon, 1 Jun 2026 12:08:43 -0400 Subject: [PATCH 31/38] Code review feedback --- requirements.txt | 2 +- src/glb-director/tests/glb_test_utils.py | 89 +++++++++++++----------- src/glb-redirect/pytest.ini | 4 -- 3 files changed, 51 insertions(+), 44 deletions(-) delete mode 100644 src/glb-redirect/pytest.ini diff --git a/requirements.txt b/requirements.txt index 04b83589..a93022c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pytest>=7.4,<9 +pytest~=8.3.0 pyroute2==0.5.19 scapy==2.7.0rc1 siphash==0.0.1 diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index e0eed766..578f0bcf 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -73,19 +73,22 @@ def __init__(self): self.director = None def setup(self, iface): - # launch the glb director, mocking an eth device with the dpdk end of our veth - self.director = subprocess.Popen( - [ - './build/glb-director', - '--vdev=eth_pcap0,iface=' + iface, - '--', - '--debug', - '--config-file', './tests/director-config.json', - '--forwarding-table', './tests/test-tables.bin' - ], - stdout=open('director-output.txt', 'ab'), - stderr=subprocess.STDOUT, - ) + # launch the glb director, mocking an eth device with the dpdk end of our veth. + # `with open(...)` closes the parent-side fd after Popen has dup2'd it + # into the child, so we don't leak an fd per test class. + with open('director-output.txt', 'ab') as out: + self.director = subprocess.Popen( + [ + './build/glb-director', + '--vdev=eth_pcap0,iface=' + iface, + '--', + '--debug', + '--config-file', './tests/director-config.json', + '--forwarding-table', './tests/test-tables.bin' + ], + stdout=out, + stderr=subprocess.STDOUT, + ) print('launched as pid', self.director.pid) @@ -223,17 +226,21 @@ def setup_pyside(self, iface): def setup(self, iface): notify_shim = SystemdNotify('/tmp/glb-notify-shim.sock') - self.xdp_root = subprocess.Popen( - [ - '../glb-director-xdp/xdp-root-shim/xdp-root-shim', - os.path.abspath('../glb-director-xdp/bpf/tailcall.o'), - '/sys/fs/bpf/root_array@' + iface, - iface, - ], - stdout=open('director-output.txt', 'ab'), - stderr=subprocess.STDOUT, - env=notify_shim.updated_env(), - ) + # `with open(...)` closes the parent-side fd after Popen dup2's it + # into the child; otherwise this fd leaks for the lifetime of the + # test process. + with open('director-output.txt', 'ab') as out: + self.xdp_root = subprocess.Popen( + [ + '../glb-director-xdp/xdp-root-shim/xdp-root-shim', + os.path.abspath('../glb-director-xdp/bpf/tailcall.o'), + '/sys/fs/bpf/root_array@' + iface, + iface, + ], + stdout=out, + stderr=subprocess.STDOUT, + env=notify_shim.updated_env(), + ) notify_shim.wait() self.director_iface = iface @@ -241,21 +248,25 @@ def setup(self, iface): def launch_director(self): notify_director = SystemdNotify('/tmp/glb-notify.sock') - self.director = subprocess.Popen( - [ - # 'strace', - '../glb-director-xdp/glb-director-xdp', - '--pid-file', '/tmp/glb-director-xdp.pid', - '--xdp-root-path=/sys/fs/bpf/root_array@' + self.director_iface, - '--debug', - '--config-file', os.path.abspath('./tests/director-config.json'), - '--forwarding-table', os.path.abspath('./tests/test-tables.bin'), - '--bpf-program', os.path.abspath('../glb-director-xdp/bpf/glb_encap.o'), - ], - stdout=open('director-output.txt', 'ab'), - stderr=subprocess.STDOUT, - env=notify_director.updated_env(), - ) + # `with open(...)` closes the parent-side fd after Popen dup2's it + # into the child; otherwise this fd leaks for the lifetime of the + # test process. + with open('director-output.txt', 'ab') as out: + self.director = subprocess.Popen( + [ + # 'strace', + '../glb-director-xdp/glb-director-xdp', + '--pid-file', '/tmp/glb-director-xdp.pid', + '--xdp-root-path=/sys/fs/bpf/root_array@' + self.director_iface, + '--debug', + '--config-file', os.path.abspath('./tests/director-config.json'), + '--forwarding-table', os.path.abspath('./tests/test-tables.bin'), + '--bpf-program', os.path.abspath('../glb-director-xdp/bpf/glb_encap.o'), + ], + stdout=out, + stderr=subprocess.STDOUT, + env=notify_director.updated_env(), + ) print('launched as pid', self.director.pid) diff --git a/src/glb-redirect/pytest.ini b/src/glb-redirect/pytest.ini deleted file mode 100644 index 3f90ef69..00000000 --- a/src/glb-redirect/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -markers = - director_dpdk: tests requiring the DPDK director runtime - director_xdp: tests requiring the XDP director runtime From 0ed77847f4d0a2f5c48e82541cf09a4a898d44d1 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 3 Jun 2026 11:26:04 -0400 Subject: [PATCH 32/38] Get DKMS via https --- script/Dockerfile.noble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/Dockerfile.noble b/script/Dockerfile.noble index 75e13dfd..0c9f21de 100644 --- a/script/Dockerfile.noble +++ b/script/Dockerfile.noble @@ -35,7 +35,7 @@ RUN apt-get install --assume-yes --fix-broken \ # and the script is wrapped so `dkms mkdeb ...` calls the legacy binary. RUN set -eux; \ cd /tmp; \ - wget -q http://archive.ubuntu.com/ubuntu/pool/main/d/dkms/dkms_2.8.1-5ubuntu2_all.deb; \ + wget -q https://archive.ubuntu.com/ubuntu/pool/main/d/dkms/dkms_2.8.1-5ubuntu2_all.deb; \ dpkg-deb -x dkms_2.8.1-5ubuntu2_all.deb /tmp/dkms-focal; \ cp -r /tmp/dkms-focal/etc/dkms/template-dkms-mkdeb /etc/dkms/template-dkms-mkdeb; \ install -m 0755 /tmp/dkms-focal/usr/sbin/dkms /usr/sbin/dkms-legacy; \ From 4bb36466537805e1ba0a207f40803a734d750a28 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 3 Jun 2026 11:27:24 -0400 Subject: [PATCH 33/38] Close file in cli_tool --- src/glb-director/tests/test_cli_tool.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/glb-director/tests/test_cli_tool.py b/src/glb-director/tests/test_cli_tool.py index 9e152e23..197686d7 100644 --- a/src/glb-director/tests/test_cli_tool.py +++ b/src/glb-director/tests/test_cli_tool.py @@ -147,6 +147,8 @@ def test_generate_configs(self): ] assert actual_first_ips == expected_first_ips[:2] + + f.close() # forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') # table = GLBRendezvousTable(forwarding_table_seed) From c1b6b73b0a049c1c62f41304efcccbd036db4a05 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Wed, 3 Jun 2026 11:42:27 -0400 Subject: [PATCH 34/38] Open test-config.bin with a with so it always closes --- src/glb-director/tests/test_cli_tool.py | 148 ++++++++++++------------ 1 file changed, 73 insertions(+), 75 deletions(-) diff --git a/src/glb-director/tests/test_cli_tool.py b/src/glb-director/tests/test_cli_tool.py index 197686d7..932d2ee8 100644 --- a/src/glb-director/tests/test_cli_tool.py +++ b/src/glb-director/tests/test_cli_tool.py @@ -73,82 +73,80 @@ def test_generate_configs(self): subprocess.check_call(['cli/glb-director-cli', 'build-config', 'tests/test-config.json', 'tests/test-config.bin']) - f = open('tests/test-config.bin', 'rb') - assert f.read(4) == b'GLBD' - - num_table_entries = 0x10000 - max_num_backends = 0x100 - max_num_binds = 0x100 - - file_header = struct.unpack(' Date: Fri, 5 Jun 2026 09:18:08 -0400 Subject: [PATCH 35/38] Use signed package for dkms --- script/Dockerfile.noble | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/script/Dockerfile.noble b/script/Dockerfile.noble index 0c9f21de..6741599d 100644 --- a/script/Dockerfile.noble +++ b/script/Dockerfile.noble @@ -27,19 +27,30 @@ RUN apt-get install --assume-yes --fix-broken \ debhelper \ libxtables-dev -# `dkms mkdeb` was removed in dkms 3.x (which is what noble ships). The -# glb-redirect Makefile uses the older `template-dkms-mkdeb` flow plus -# `dkms mkdeb --source-only`. Restore both by installing the focal dkms -# package alongside dkms 3.x: focal's /etc/dkms/template-dkms-mkdeb/ and the -# `mkdeb` code path in /usr/sbin/dkms are extracted into a parallel location -# and the script is wrapped so `dkms mkdeb ...` calls the legacy binary. +# noble's dkms 3.x dropped `dkms mkdeb`, which the glb-redirect Makefile needs +# (template-dkms-mkdeb + `dkms mkdeb --source-only`). Restore it by extracting +# focal's dkms 2.x alongside 3.x and shimming `dkms mkdeb` to the legacy binary. +# +# We `apt-get download` the focal dkms from a GPG-verified focal source (full +# apt chain of trust, no hardcoded/unverified URL). The focal pockets are +# pinned below noble's so they're only used for this download, never installs. RUN set -eux; \ + keyring=/usr/share/keyrings/ubuntu-archive-keyring.gpg; \ + printf 'deb [signed-by=%s] http://archive.ubuntu.com/ubuntu focal main\ndeb [signed-by=%s] http://archive.ubuntu.com/ubuntu focal-updates main\n' "$keyring" "$keyring" > /etc/apt/sources.list.d/focal-dkms.list; \ + printf 'Package: *\nPin: release n=focal\nPin-Priority: 100\n\nPackage: *\nPin: release n=focal-updates\nPin-Priority: 100\n' > /etc/apt/preferences.d/focal-dkms.pref; \ + apt-get update; \ + # Sanity check: the focal pin must not change the dkms candidate (stays 3.x). + case "$(apt-cache policy dkms | awk '/Candidate:/{print $2}')" in 3.*) : ;; *) echo "ERROR: dkms candidate changed by focal pin" >&2; exit 1 ;; esac; \ cd /tmp; \ - wget -q https://archive.ubuntu.com/ubuntu/pool/main/d/dkms/dkms_2.8.1-5ubuntu2_all.deb; \ - dpkg-deb -x dkms_2.8.1-5ubuntu2_all.deb /tmp/dkms-focal; \ + apt-get download dkms/focal-updates; \ + deb="$(ls dkms_*_all.deb)"; \ + dpkg-deb -x "$deb" /tmp/dkms-focal; \ cp -r /tmp/dkms-focal/etc/dkms/template-dkms-mkdeb /etc/dkms/template-dkms-mkdeb; \ install -m 0755 /tmp/dkms-focal/usr/sbin/dkms /usr/sbin/dkms-legacy; \ - rm -rf /tmp/dkms-focal /tmp/dkms_2.8.1-5ubuntu2_all.deb; \ + rm -rf /tmp/dkms-focal "/tmp/$deb"; \ + # Remove the temporary focal source + pin so they don't linger in the image. + rm -f /etc/apt/sources.list.d/focal-dkms.list /etc/apt/preferences.d/focal-dkms.pref; \ + apt-get update; \ printf '#!/bin/sh\nif [ "$1" = "mkdeb" ]; then exec /usr/sbin/dkms-legacy "$@"; fi\nexec /usr/sbin/dkms.real "$@"\n' > /usr/local/bin/dkms; \ chmod 0755 /usr/local/bin/dkms; \ mv /usr/sbin/dkms /usr/sbin/dkms.real From 1304ee4ee77f3506263fc801ed6c316cb55e6432 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Fri, 5 Jun 2026 09:21:06 -0400 Subject: [PATCH 36/38] Define debug with explicit initializer --- src/glb-director/shared_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/glb-director/shared_opt.c b/src/glb-director/shared_opt.c index d60a6448..7bbaaa57 100644 --- a/src/glb-director/shared_opt.c +++ b/src/glb-director/shared_opt.c @@ -37,7 +37,7 @@ * Newer GCC defaults to -fno-common, so a tentative definition in the header * (the original `bool debug;` in log.h) is rejected as a duplicate symbol when * multiple translation units include it. */ -bool debug; +bool debug = false; /* parses --config-file, --forwarding-table, and --debug cli options */ From ed4e89824fc7be2df9e1f334e229f616fea305e7 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Fri, 5 Jun 2026 10:08:28 -0400 Subject: [PATCH 37/38] Use libbpf for the distro --- src/glb-director-xdp/script/create-packages | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/glb-director-xdp/script/create-packages b/src/glb-director-xdp/script/create-packages index dd2578dc..b7e5d461 100755 --- a/src/glb-director-xdp/script/create-packages +++ b/src/glb-director-xdp/script/create-packages @@ -48,7 +48,7 @@ fpm -f -s dir -t deb \ -n glb-director-xdp \ -v ${GLB_DIRECTOR_XDP_VERSION} \ -d "xdp-root-shim" \ - -d "libbpf0" \ + -d "libbpf1 | libbpf0" \ -d "glb-director-cli >= ${GLB_DIRECTOR_VERSION}" \ --conflicts 'glb-director' \ --license 'BSD 3-Clause' \ @@ -67,7 +67,7 @@ fpm -f -s dir -t deb \ fpm -f -s dir -t deb \ -n xdp-root-shim \ -v ${XDP_ROOT_SHIM_VERSION} \ - -d "libbpf0" \ + -d "libbpf1 | libbpf0" \ --license 'BSD 3-Clause' \ --maintainer 'GitHub ' \ --deb-systemd packaging/xdp-root-shim\@.service \ From f5e54de76af8d796b440e51e87c2c3d08777cf15 Mon Sep 17 00:00:00 2001 From: Mark Penny Date: Fri, 5 Jun 2026 13:43:52 -0400 Subject: [PATCH 38/38] Make GLB_SKIP_DPDK_DIRECTOR checks explicitly verify the value is '1 --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 7b5fc5dc..a69dbf59 100644 --- a/Makefile +++ b/Makefile @@ -8,16 +8,16 @@ mkdeb: make -C src/glb-redirect mkdeb make -C src/glb-healthcheck mkdeb cd src/glb-director-xdp && script/create-packages -ifeq ($(GLB_SKIP_DPDK_DIRECTOR),) - cd src/glb-director && script/create-packages -else +ifeq ($(GLB_SKIP_DPDK_DIRECTOR),1) cd src/glb-director && GLB_CLI_ONLY=1 script/create-packages +else + cd src/glb-director && script/create-packages endif clean: make -C src/glb-redirect clean make -C src/glb-healthcheck clean -ifeq ($(GLB_SKIP_DPDK_DIRECTOR),) +ifneq ($(GLB_SKIP_DPDK_DIRECTOR),1) make -C src/glb-director clean endif make -C src/glb-director/cli clean