diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 94e2c6be..d1f8ecf7 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,13 +1,21 @@ blank_issues_enabled: true contact_links: + - name: 💨 Performance Issues + url: https://github.com/slackhq/nebula/discussions/new/choose + about: 'We ask that you create a discussion instead of an issue for performance-related questions. This allows us to have a more open conversation about the issue and helps us to better understand the problem.' + + - name: 📄 Documentation Issues + url: https://github.com/definednet/nebula-docs + about: "If you've found an issue with the website documentation, please file it in the nebula-docs repository." + + - name: 📱 Mobile Nebula Issues + url: https://github.com/definednet/mobile_nebula + about: "If you're using the mobile Nebula app and have found an issue, please file it in the mobile_nebula repository." + - name: 📘 Documentation url: https://nebula.defined.net/docs/ - about: Review documentation. + about: 'The documentation is the best place to start if you are new to Nebula.' - name: 💁 Support/Chat - url: https://join.slack.com/t/nebulaoss/shared_invite/enQtOTA5MDI4NDg3MTg4LTkwY2EwNTI4NzQyMzc0M2ZlODBjNWI3NTY1MzhiOThiMmZlZjVkMTI0NGY4YTMyNjUwMWEyNzNkZTJmYzQxOGU - about: 'This issue tracker is not for support questions. Join us on Slack for assistance!' - - - name: 📱 Mobile Nebula - url: https://github.com/definednet/mobile_nebula - about: 'This issue tracker is not for mobile support. Try the Mobile Nebula repo instead!' + url: https://join.slack.com/t/nebulaoss/shared_invite/zt-39pk4xopc-CUKlGcb5Z39dQ0cK1v7ehA + about: 'For faster support, join us on Slack for assistance!' diff --git a/.github/actions/code-sign/action.yml b/.github/actions/code-sign/action.yml new file mode 100644 index 00000000..f3956d95 --- /dev/null +++ b/.github/actions/code-sign/action.yml @@ -0,0 +1,116 @@ +name: Code-sign Windows binaries +description: > + Sign every .exe under a given path in place via the DefinedNet code-signer + Lambda. If `role` or `bucket` is empty, logs a notice and skips signing so + forks and dev branches without AWS access still produce usable builds. + +inputs: + path: + description: "Directory whose .exe files should be signed in place" + required: true + role: + description: "IAM role ARN to assume via OIDC; empty disables signing" + required: false + default: "" + bucket: + description: "S3 staging bucket the code-signer Lambda reads from; empty disables signing" + required: false + default: "" + region: + description: "AWS region for the role and Lambda" + required: false + default: "us-east-2" + function-name: + description: "Code-signer Lambda function name" + required: false + default: "code-signer" + key-prefix: + description: "S3 key prefix to write under; defaults to code-signing// of the calling repo" + required: false + default: "" + +runs: + using: composite + steps: + - name: Skip notice + if: inputs.role == '' || inputs.bucket == '' + shell: sh + run: echo "::notice::code-signer role or bucket not set; skipping code signing." + + - name: Configure AWS credentials + if: inputs.role != '' && inputs.bucket != '' + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ inputs.role }} + aws-region: ${{ inputs.region }} + # Default is 12 retries to ride out IAM trust-policy propagation; once + # the role is stable we want a real misconfiguration to fail fast. + retry-max-attempts: 5 + + - name: Sign .exe files + if: inputs.role != '' && inputs.bucket != '' + shell: sh + env: + SIGN_PATH: ${{ inputs.path }} + BUCKET: ${{ inputs.bucket }} + FUNCTION_NAME: ${{ inputs.function-name }} + KEY_PREFIX: ${{ inputs.key-prefix }} + run: | + set -eu + # Default the prefix to this repo so the S3 key attributes the sign correctly. + # nebula-nightly runs this same action but writes under its own repo's prefix. + KEY_PREFIX="${KEY_PREFIX:-code-signing/$GITHUB_REPOSITORY}" + RUN="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + + find "$SIGN_PATH" -name '*.exe' -print | while read -r path + do + rel=${path#"$SIGN_PATH"/} + file=$(basename "$path") + name=${file%.exe} + prefix="${KEY_PREFIX}/${RUN}" + src="${prefix}/unsigned/${rel}" + dst="${prefix}/signed/${rel}" + + echo "::group::Sign ${rel}" + echo "Uploading unsigned to s3://${BUCKET}/${src}" + aws s3 cp --no-progress "$path" "s3://${BUCKET}/${src}" >/dev/null + + echo "Invoking ${FUNCTION_NAME} Lambda" + payload=$(jq -nc \ + --arg s "$src" \ + --arg d "$dst" \ + --arg p "$name" \ + '{source_key: $s, dest_key: $d, program_name: $p}') + meta=$(aws lambda invoke \ + --function-name "$FUNCTION_NAME" \ + --cli-binary-format raw-in-base64-out \ + --payload "$payload" \ + --output json \ + /tmp/sign-resp.json) + if echo "$meta" | jq -e '.FunctionError != null' >/dev/null + then + echo "::endgroup::" + echo "::error::code-signer Lambda failed for ${rel}" + cat /tmp/sign-resp.json >&2 + exit 1 + fi + + echo "Downloading signed back to ${path}" + aws s3 cp --no-progress "s3://${BUCKET}/${dst}" "$path" >/dev/null + + aws s3 rm "s3://${BUCKET}/${src}" >/dev/null 2>&1 || true + aws s3 rm "s3://${BUCKET}/${dst}" >/dev/null 2>&1 || true + + # Sanity-check the bytes we got back actually carry an Authenticode + # signature that this machine can validate end to end. + status=$(powershell -NoProfile -Command "(Get-AuthenticodeSignature -FilePath '$path').Status" | tr -d '\r') + if [ "$status" != "Valid" ] + then + echo "::endgroup::" + echo "::error::${rel} signature status: ${status} (expected Valid)" + exit 1 + fi + + echo "Signed ${rel} (sha256=$(jq -r '.sha256' /tmp/sign-resp.json), status=${status})" + echo "::endgroup::" + done diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..102ddb32 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,11 @@ + diff --git a/.github/workflows/gofmt.yml b/.github/workflows/gofmt.yml deleted file mode 100644 index 399bc95d..00000000 --- a/.github/workflows/gofmt.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: gofmt -on: - push: - branches: - - master - pull_request: - paths: - - '.github/workflows/gofmt.yml' - - '**.go' -jobs: - - gofmt: - name: Run gofmt - runs-on: ubuntu-latest - steps: - - - uses: actions/checkout@v4 - - - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - check-latest: true - - - name: Install goimports - run: | - go install golang.org/x/tools/cmd/goimports@latest - - - name: gofmt - run: | - if [ "$(find . -iname '*.go' | grep -v '\.pb\.go$' | xargs goimports -l)" ] - then - find . -iname '*.go' | grep -v '\.pb\.go$' | xargs goimports -d - exit 1 - fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5b8cedb..b22b3430 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,11 +10,11 @@ jobs: name: Build Linux/BSD All runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v7 with: - go-version-file: 'go.mod' + go-version: '1.26' check-latest: true - name: Build @@ -24,7 +24,7 @@ jobs: mv build/*.tar.gz release - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v7 with: name: linux-latest path: release @@ -32,12 +32,15 @@ jobs: build-windows: name: Build Windows runs-on: windows-latest + permissions: + id-token: write + contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v7 with: - go-version-file: 'go.mod' + go-version: '1.26' check-latest: true - name: Build @@ -54,8 +57,15 @@ jobs: mkdir build\dist\windows mv dist\windows\wintun build\dist\windows\ + - name: Code-sign + uses: ./.github/actions/code-sign + with: + path: build + role: ${{ secrets.DEFINED_CODE_SIGNER_ROLE }} + bucket: ${{ secrets.DEFINED_CODE_SIGNER_BUCKET }} + - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v7 with: name: windows-latest path: build @@ -64,18 +74,18 @@ jobs: name: Build Universal Darwin env: HAS_SIGNING_CREDS: ${{ secrets.AC_USERNAME != '' }} - runs-on: macos-11 + runs-on: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v7 with: - go-version-file: 'go.mod' + go-version: '1.26' check-latest: true - name: Import certificates if: env.HAS_SIGNING_CREDS == 'true' - uses: Apple-Actions/import-codesign-certs@v2 + uses: Apple-Actions/import-codesign-certs@v7 with: p12-file-base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }} p12-password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }} @@ -104,20 +114,69 @@ jobs: fi - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v7 with: name: darwin-latest path: ./release/* + build-docker: + name: Create and Upload Docker Images + # Technically we only need build-linux to succeed, but if any platforms fail we'll + # want to investigate and restart the build + needs: [build-linux, build-darwin, build-windows] + runs-on: ubuntu-latest + env: + HAS_DOCKER_CREDS: ${{ vars.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }} + # XXX It's not possible to write a conditional here, so instead we do it on every step + #if: ${{ env.HAS_DOCKER_CREDS == 'true' }} + steps: + # Be sure to checkout the code before downloading artifacts, or they will + # be overwritten + - name: Checkout code + if: ${{ env.HAS_DOCKER_CREDS == 'true' }} + uses: actions/checkout@v7 + + - name: Download artifacts + if: ${{ env.HAS_DOCKER_CREDS == 'true' }} + uses: actions/download-artifact@v8 + with: + name: linux-latest + path: artifacts + + - name: Login to Docker Hub + if: ${{ env.HAS_DOCKER_CREDS == 'true' }} + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + if: ${{ env.HAS_DOCKER_CREDS == 'true' }} + uses: docker/setup-buildx-action@v4 + + - name: Build and push images + if: ${{ env.HAS_DOCKER_CREDS == 'true' }} + env: + DOCKER_IMAGE_REPO: ${{ vars.DOCKER_IMAGE_REPO || 'nebulaoss/nebula' }} + DOCKER_IMAGE_TAG: ${{ vars.DOCKER_IMAGE_TAG || 'latest' }} + run: | + mkdir -p build/linux-{amd64,arm64} + tar -zxvf artifacts/nebula-linux-amd64.tar.gz -C build/linux-amd64/ + tar -zxvf artifacts/nebula-linux-arm64.tar.gz -C build/linux-arm64/ + docker buildx build . --push -f docker/Dockerfile --platform linux/amd64,linux/arm64 \ + --build-arg VERSION="${GITHUB_REF#refs/tags/v}" \ + --build-arg REVISION="${GITHUB_SHA}" \ + --tag "${DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_TAG}" --tag "${DOCKER_IMAGE_REPO}:${GITHUB_REF#refs/tags/v}" + release: name: Create and Upload Release needs: [build-linux, build-darwin, build-windows] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v8 with: path: artifacts @@ -163,10 +222,11 @@ jobs: id: create_release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REF_NAME: ${{ github.ref_name }} run: | cd artifacts gh release create \ --verify-tag \ - --title "Release ${{ github.ref_name }}" \ - "${{ github.ref_name }}" \ + --title "Release ${GITHUB_REF_NAME}" \ + "${GITHUB_REF_NAME}" \ SHASUM256.txt *-latest/*.zip *-latest/*.tar.gz diff --git a/.github/workflows/smoke-extra.yml b/.github/workflows/smoke-extra.yml new file mode 100644 index 00000000..445b83ed --- /dev/null +++ b/.github/workflows/smoke-extra.yml @@ -0,0 +1,132 @@ +name: smoke-extra +on: + push: + branches: + - master + pull_request: + types: [opened, synchronize, labeled, reopened] + paths: + - '.github/workflows/smoke**' + - '**Makefile' + - '**.go' + - '**.proto' + - 'go.mod' + - 'go.sum' +jobs: + + smoke-extra-libvirt: + if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra') + name: ${{ matrix.target }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: + - freebsd-amd64 + - openbsd-amd64 + - netbsd-amd64 + - linux-amd64-ipv6disable + env: + VAGRANT_DEFAULT_PROVIDER: libvirt + steps: + + - uses: actions/checkout@v7 + + - uses: actions/setup-go@v7 + with: + go-version: '1.26' + check-latest: true + + - name: add hashicorp source + run: wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + + - name: install vagrant and libvirt + run: | + sudo apt-get update && sudo apt-get install -y vagrant libvirt-daemon-system libvirt-dev + sudo chmod 666 /dev/kvm + sudo usermod -aG libvirt $(whoami) + sudo chmod 666 /var/run/libvirt/libvirt-sock + vagrant plugin install vagrant-libvirt + + - name: ${{ matrix.target }} + run: make smoke-vagrant/${{ matrix.target }} + + timeout-minutes: 30 + + # linux-386 needs VirtualBox, which conflicts with KVM/libvirt -- isolated job. + smoke-extra-virtualbox: + if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra') + name: linux-386 + runs-on: ubuntu-latest + env: + VAGRANT_DEFAULT_PROVIDER: virtualbox + steps: + + - uses: actions/checkout@v7 + + - uses: actions/setup-go@v7 + with: + go-version: '1.26' + check-latest: true + + - name: add hashicorp source + run: wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + + - name: install vagrant and virtualbox + run: | + sudo apt-get update && sudo apt-get install -y vagrant virtualbox + sudo rmmod kvm_amd kvm_intel kvm 2>/dev/null || true + + - name: linux-386 + run: make smoke-vagrant/linux-386 + + timeout-minutes: 30 + + smoke-windows: + if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra') + name: Run windows smoke test + runs-on: windows-latest + steps: + + - uses: actions/checkout@v7 + + - uses: actions/setup-go@v7 + with: + go-version: '1.26' + check-latest: true + + # WSL2 + Ubuntu so the smoke can run a real linux peer with its own + # netns. iputils-ping is needed for the in-WSL ping check. WSL1 has no + # real kernel and would lack /dev/net/tun, so we have to force WSL2. + - uses: Vampire/setup-wsl@v3 + with: + distribution: Ubuntu-24.04 + additional-packages: iputils-ping iproute2 + + # Vampire/setup-wsl provisions WSL1 even when the WSL2 platform is present. + # Convert the distro to WSL2 explicitly before we try to use /dev/net/tun. + - name: convert distro to WSL2 + shell: pwsh + run: | + wsl --set-version Ubuntu-24.04 2 + wsl --shutdown + wsl --list --verbose + + - name: build windows nebula + run: make bin-windows + + - name: build linux nebula for WSL + shell: bash + env: + GOOS: linux + GOARCH: amd64 + run: | + mkdir -p build/linux-amd64 + go build -o build/linux-amd64/nebula ./cmd/nebula + + - name: run smoke-windows + shell: pwsh + working-directory: ./.github/workflows/smoke + run: ./smoke-windows.ps1 + + timeout-minutes: 15 diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index f02b1ba2..aa4a41e8 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -18,11 +18,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v7 with: - go-version-file: 'go.mod' + go-version: '1.26' check-latest: true - name: build @@ -36,6 +36,14 @@ jobs: working-directory: ./.github/workflows/smoke run: ./smoke.sh + - name: setup docker image ipv6 + working-directory: ./.github/workflows/smoke + run: SMOKE_OVERLAY_IPV6=1 ./build.sh + + - name: run smoke ipv6 + working-directory: ./.github/workflows/smoke + run: SMOKE_OVERLAY_IPV6=1 ./smoke.sh + - name: setup relay docker image working-directory: ./.github/workflows/smoke run: ./build-relay.sh diff --git a/.github/workflows/smoke/build-relay.sh b/.github/workflows/smoke/build-relay.sh index 70b07f4e..249e6c84 100755 --- a/.github/workflows/smoke/build-relay.sh +++ b/.github/workflows/smoke/build-relay.sh @@ -16,8 +16,10 @@ relay: am_relay: true EOF - export LIGHTHOUSES="192.168.100.1 172.17.0.2:4242" - export REMOTE_ALLOW_LIST='{"172.17.0.4/32": false, "172.17.0.5/32": false}' + # TEST-NET-3 placeholder IPs; smoke-relay.sh seds them to real container IPs. + # Mapping: .2 lighthouse1, .3 host2, .4 host3, .5 host4. + export LIGHTHOUSES="192.168.100.1 203.0.113.2:4242" + export REMOTE_ALLOW_LIST='{"203.0.113.4/32": false, "203.0.113.5/32": false}' HOST="host2" ../genconfig.sh >host2.yml <host3.yml diff --git a/.github/workflows/smoke/build.sh b/.github/workflows/smoke/build.sh index 9cbb2005..fef76098 100755 --- a/.github/workflows/smoke/build.sh +++ b/.github/workflows/smoke/build.sh @@ -5,35 +5,63 @@ set -e -x rm -rf ./build mkdir ./build +if [ "$SMOKE_OVERLAY_IPV6" ] +then + LIGHTHOUSE_NIP="fd00:4242:0:0:0:ffff:c0a8:6401" + HOST2_NIP="fd00:4242:0:0:0:ffff:c0a8:6402" + HOST3_NIP="fd00:4242:0:0:0:ffff:c0a8:6403" + HOST4_NIP="fd00:4242:0:0:0:ffff:c0a8:6404" +else + LIGHTHOUSE_NIP="192.168.100.1" + HOST2_NIP="192.168.100.2" + HOST3_NIP="192.168.100.3" + HOST4_NIP="192.168.100.4" +fi + +# Smoke containers run on a dedicated docker network whose subnet is allocated +# at smoke time, not known at build time. Configs are written with TEST-NET-3 +# placeholder IPs (RFC 5737) and smoke.sh / smoke-vagrant.sh / smoke-relay.sh +# sed the real container IPs in before starting nebula. +# +# Placeholder mapping (last octet == fixed container slot): +# 203.0.113.2 -> lighthouse1, 203.0.113.3 -> host2, +# 203.0.113.4 -> host3, 203.0.113.5 -> host4. +LIGHTHOUSE_IP="203.0.113.2" + ( cd build cp ../../../../build/linux-amd64/nebula . cp ../../../../build/linux-amd64/nebula-cert . + if [ "$1" ] + then + cp "../../../../build/$1/nebula" "$1-nebula" + fi + HOST="lighthouse1" \ AM_LIGHTHOUSE=true \ ../genconfig.sh >lighthouse1.yml HOST="host2" \ - LIGHTHOUSES="192.168.100.1 172.17.0.2:4242" \ + LIGHTHOUSES="$LIGHTHOUSE_NIP $LIGHTHOUSE_IP:4242" \ ../genconfig.sh >host2.yml HOST="host3" \ - LIGHTHOUSES="192.168.100.1 172.17.0.2:4242" \ + LIGHTHOUSES="$LIGHTHOUSE_NIP $LIGHTHOUSE_IP:4242" \ INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \ ../genconfig.sh >host3.yml HOST="host4" \ - LIGHTHOUSES="192.168.100.1 172.17.0.2:4242" \ + LIGHTHOUSES="$LIGHTHOUSE_NIP $LIGHTHOUSE_IP:4242" \ OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \ ../genconfig.sh >host4.yml ../../../../nebula-cert ca -curve "${CURVE:-25519}" -name "Smoke Test" - ../../../../nebula-cert sign -name "lighthouse1" -groups "lighthouse,lighthouse1" -ip "192.168.100.1/24" - ../../../../nebula-cert sign -name "host2" -groups "host,host2" -ip "192.168.100.2/24" - ../../../../nebula-cert sign -name "host3" -groups "host,host3" -ip "192.168.100.3/24" - ../../../../nebula-cert sign -name "host4" -groups "host,host4" -ip "192.168.100.4/24" + ../../../../nebula-cert sign -name "lighthouse1" -groups "lighthouse,lighthouse1" -ip "$LIGHTHOUSE_NIP/24" + ../../../../nebula-cert sign -name "host2" -groups "host,host2" -ip "$HOST2_NIP/24" + ../../../../nebula-cert sign -name "host3" -groups "host,host3" -ip "$HOST3_NIP/24" + ../../../../nebula-cert sign -name "host4" -groups "host,host4" -ip "$HOST4_NIP/24" ) docker build -t "nebula:${NAME:-smoke}" . diff --git a/.github/workflows/smoke/genconfig.sh b/.github/workflows/smoke/genconfig.sh index 373ea5fc..16e768ee 100755 --- a/.github/workflows/smoke/genconfig.sh +++ b/.github/workflows/smoke/genconfig.sh @@ -47,7 +47,7 @@ listen: port: ${LISTEN_PORT:-4242} tun: - dev: ${TUN_DEV:-nebula1} + dev: ${TUN_DEV:-tun0} firewall: inbound_action: reject diff --git a/.github/workflows/smoke/smoke-relay.sh b/.github/workflows/smoke/smoke-relay.sh index 8926091e..aa1cd915 100755 --- a/.github/workflows/smoke/smoke-relay.sh +++ b/.github/workflows/smoke/smoke-relay.sh @@ -6,6 +6,8 @@ set -o pipefail mkdir -p logs +NETWORK="nebula-smoke-relay" + cleanup() { echo echo " *** cleanup" @@ -16,22 +18,53 @@ cleanup() { then docker kill lighthouse1 host2 host3 host4 fi + docker network rm "$NETWORK" >/dev/null 2>&1 } trap cleanup EXIT +# Create a dedicated smoke network with an explicit subnet (required for --ip +# below). Probe a short list of candidates so a locally-used range doesn't +# fail the whole test — we only need one to be free. +docker network rm "$NETWORK" >/dev/null 2>&1 || true +for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do + if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then + break + fi +done +if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then + echo "failed to create $NETWORK: every candidate subnet is in use" >&2 + exit 1 +fi + +# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1, +# .3 host2, .4 host3, .5 host4 — matches the placeholders in build-relay.sh. +SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")" +PREFIX="${SUBNET%/*}" +PREFIX="${PREFIX%.*}" +LIGHTHOUSE_IP="$PREFIX.2" +HOST2_IP="$PREFIX.3" +HOST3_IP="$PREFIX.4" +HOST4_IP="$PREFIX.5" + +# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones. +for f in build/host2.yml build/host3.yml build/host4.yml; do + sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp" + mv "$f.tmp" "$f" +done + docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test -docker run --name host2 --rm nebula:smoke-relay -config host2.yml -test -docker run --name host3 --rm nebula:smoke-relay -config host3.yml -test -docker run --name host4 --rm nebula:smoke-relay -config host4.yml -test +docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" nebula:smoke-relay -config host2.yml -test +docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" nebula:smoke-relay -config host3.yml -test +docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" nebula:smoke-relay -config host4.yml -test -docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' & +docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' & sleep 1 -docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' & +docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' & sleep 1 -docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' & +docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' & sleep 1 -docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' & +docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' & sleep 1 set +x @@ -76,7 +109,13 @@ docker exec host4 sh -c 'kill 1' docker exec host3 sh -c 'kill 1' docker exec host2 sh -c 'kill 1' docker exec lighthouse1 sh -c 'kill 1' -sleep 1 + +# Wait up to 30s for all backgrounded jobs to exit rather than relying on a +# fixed sleep. +for _ in $(seq 1 30); do + [ -z "$(jobs -r)" ] && break + sleep 1 +done if [ "$(jobs -r)" ] then diff --git a/.github/workflows/smoke/smoke-vagrant.sh b/.github/workflows/smoke/smoke-vagrant.sh new file mode 100755 index 00000000..e3863cb5 --- /dev/null +++ b/.github/workflows/smoke/smoke-vagrant.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +set -e -x + +set -o pipefail + +export VAGRANT_CWD="$PWD/vagrant-$1" + +mkdir -p logs + +NETWORK="nebula-smoke" + +cleanup() { + echo + echo " *** cleanup" + echo + + set +e + if [ "$(jobs -r)" ] + then + docker kill lighthouse1 host2 + fi + vagrant destroy -f + docker network rm "$NETWORK" >/dev/null 2>&1 +} + +trap cleanup EXIT + +# Create a dedicated smoke network with an explicit subnet (required for --ip +# below). Probe a short list of candidates so a locally-used range doesn't +# fail the whole test — we only need one to be free. +docker network rm "$NETWORK" >/dev/null 2>&1 || true +for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do + if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then + break + fi +done +if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then + echo "failed to create $NETWORK: every candidate subnet is in use" >&2 + exit 1 +fi + +# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1, +# .3 host2 — matches the placeholders in build.sh. +SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")" +PREFIX="${SUBNET%/*}" +PREFIX="${PREFIX%.*}" +LIGHTHOUSE_IP="$PREFIX.2" +HOST2_IP="$PREFIX.3" + +# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones. +# This must happen before `vagrant up` rsyncs build/ into the VM for host3. +for f in build/host2.yml build/host3.yml; do + sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp" + mv "$f.tmp" "$f" +done + +CONTAINER="nebula:${NAME:-smoke}" + +docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test +docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test + +vagrant up +vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test" -- -T + +docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' & +sleep 1 +docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' & +sleep 1 +vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" 2>&1 -- -T | tee logs/host3 | sed -u 's/^/ [host3] /' & +sleep 15 + +# grab tcpdump pcaps for debugging +docker exec lighthouse1 tcpdump -i nebula1 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap & +docker exec lighthouse1 tcpdump -i eth0 -q -w - -U 2>logs/lighthouse1.outside.log >logs/lighthouse1.outside.pcap & +docker exec host2 tcpdump -i nebula1 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap & +docker exec host2 tcpdump -i eth0 -q -w - -U 2>logs/host2.outside.log >logs/host2.outside.pcap & +# vagrant ssh -c "tcpdump -i nebula1 -q -w - -U" 2>logs/host3.inside.log >logs/host3.inside.pcap & +# vagrant ssh -c "tcpdump -i eth0 -q -w - -U" 2>logs/host3.outside.log >logs/host3.outside.pcap & + +#docker exec host2 ncat -nklv 0.0.0.0 2000 & +#vagrant ssh -c "ncat -nklv 0.0.0.0 2000" & +#docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 0.0.0.0 3000 & +#vagrant ssh -c "ncat -e '/usr/bin/echo host3' -nkluv 0.0.0.0 3000" & + +set +x +echo +echo " *** Testing ping from lighthouse1" +echo +set -x +docker exec lighthouse1 ping -c1 192.168.100.2 +docker exec lighthouse1 ping -c1 192.168.100.3 + +set +x +echo +echo " *** Testing ping from host2" +echo +set -x +docker exec host2 ping -c1 192.168.100.1 +# Should fail because not allowed by host3 inbound firewall +! docker exec host2 ping -c1 192.168.100.3 -w5 || exit 1 + +#set +x +#echo +#echo " *** Testing ncat from host2" +#echo +#set -x +# Should fail because not allowed by host3 inbound firewall +#! docker exec host2 ncat -nzv -w5 192.168.100.3 2000 || exit 1 +#! docker exec host2 ncat -nzuv -w5 192.168.100.3 3000 | grep -q host3 || exit 1 + +set +x +echo +echo " *** Testing ping from host3" +echo +set -x +vagrant ssh -c "ping -c1 192.168.100.1" -- -T +vagrant ssh -c "ping -c1 192.168.100.2" -- -T + +#set +x +#echo +#echo " *** Testing ncat from host3" +#echo +#set -x +#vagrant ssh -c "ncat -nzv -w5 192.168.100.2 2000" +#vagrant ssh -c "ncat -nzuv -w5 192.168.100.2 3000" | grep -q host2 + +vagrant ssh -c "sudo xargs kill &2 + exit 1 +fi diff --git a/.github/workflows/smoke/smoke-windows.ps1 b/.github/workflows/smoke/smoke-windows.ps1 new file mode 100644 index 00000000..0436598d --- /dev/null +++ b/.github/workflows/smoke/smoke-windows.ps1 @@ -0,0 +1,272 @@ +#!/usr/bin/env pwsh +# Windows smoke test for the nebula tun + UDP + NLM code paths. +# +# Topology: +# - lighthouse runs natively on the Windows host (wintun + windows UDP) +# - peer runs inside WSL2 (Linux build of nebula, /dev/net/tun) +# +# WSL2 gives us a real netns boundary so the loopback fast-path on Windows +# does not short-circuit the overlay -- when WSL pings the lighthouse VPN IP, +# Linux has no idea that IP is local to the Windows host, so the packet is +# forced through nebula. Same in reverse. + +$ErrorActionPreference = 'Stop' + +# wsl.exe emits UTF-16 LE by default which PowerShell reads as bytes, mangling +# every captured string. WSL_UTF8 makes wsl.exe emit UTF-8 instead. +$env:WSL_UTF8 = '1' + +$RepoRoot = Resolve-Path "$PSScriptRoot\..\..\.." +$Nebula = Join-Path $RepoRoot 'nebula.exe' +$NebulaCert = Join-Path $RepoRoot 'nebula-cert.exe' +$NebulaLinux = Join-Path $RepoRoot 'build\linux-amd64\nebula' + +if (-not (Test-Path $Nebula)) { throw "missing $Nebula; run 'make bin-windows' first" } +if (-not (Test-Path $NebulaCert)) { throw "missing $NebulaCert; run 'make bin-windows' first" } +if (-not (Test-Path $NebulaLinux)) { throw "missing $NebulaLinux; build the linux nebula first" } + +# Matches the distro installed by Vampire/setup-wsl in smoke-extra.yml. +$Distro = 'Ubuntu-24.04' +$listed = (wsl --list --quiet 2>$null) -join "`n" +if ($listed -notmatch [regex]::Escape($Distro)) { + throw "WSL distro $Distro not registered. Got: $listed" +} +Write-Host "Using WSL distro: $Distro" + +# Windows host as seen from inside WSL: WSL's default-route gateway. We extract +# it with a regex rather than awk fields so PowerShell does not eat any '$N' +# tokens, and tabs/double-spaces in `ip route` output do not confuse a cut. +$ipCmd = 'ip route show default | grep -oE "([0-9]+\.){3}[0-9]+" | head -1' +$WindowsIp = (wsl -d $Distro -- bash -c $ipCmd).Trim() +if (-not $WindowsIp) { throw "could not determine Windows host IP from WSL" } +Write-Host "Windows host IP from WSL: $WindowsIp" + +$WorkDir = Join-Path $env:TEMP 'nebula-smoke-windows' +if (Test-Path $WorkDir) { Remove-Item -Recurse -Force $WorkDir } +New-Item -ItemType Directory -Path $WorkDir | Out-Null + +$WslDir = '/tmp/nebula-smoke' +wsl -d $Distro -- bash -c "rm -rf $WslDir && mkdir -p $WslDir" | Out-Null + +$DevName = 'nebula-smoke' +$Ip1 = '192.168.241.1' +$Ip2 = '192.168.241.2' +$Port = 4242 + +& $NebulaCert ca -name 'smoke-ca' -out-crt "$WorkDir\ca.crt" -out-key "$WorkDir\ca.key" +if ($LASTEXITCODE -ne 0) { throw "nebula-cert ca failed (exit $LASTEXITCODE)" } + +& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key" +if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign lighthouse failed (exit $LASTEXITCODE)" } + +& $NebulaCert sign -name 'peer' -networks "$Ip2/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key" +if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign peer failed (exit $LASTEXITCODE)" } + +# Windows lighthouse config. +@" +pki: + ca: $WorkDir\ca.crt + cert: $WorkDir\lighthouse.crt + key: $WorkDir\lighthouse.key +static_host_map: {} +lighthouse: + am_lighthouse: true + interval: 60 + hosts: [] +listen: + host: 0.0.0.0 + port: $Port +tun: + disabled: false + dev: $DevName + drop_local_broadcast: false + drop_multicast: false + tx_queue: 500 + mtu: 1300 + network_category: private +logging: + level: info + format: text +firewall: + outbound_action: drop + inbound_action: drop + conntrack: + tcp_timeout: 12m + udp_timeout: 3m + default_timeout: 10m + outbound: + - port: any + proto: any + host: any + inbound: + - port: any + proto: any + host: any +"@ | Out-File -FilePath "$WorkDir\lighthouse.yml" -Encoding utf8 + +# WSL peer config (paths are POSIX, deliberately). +@" +pki: + ca: $WslDir/ca.crt + cert: $WslDir/peer.crt + key: $WslDir/peer.key +static_host_map: + "${Ip1}": ["${WindowsIp}:$Port"] +lighthouse: + am_lighthouse: false + interval: 60 + hosts: + - "${Ip1}" +listen: + host: 0.0.0.0 + port: 0 +tun: + disabled: false + dev: nebula1 + drop_local_broadcast: false + drop_multicast: false + tx_queue: 500 + mtu: 1300 +logging: + level: info + format: text +firewall: + outbound_action: drop + inbound_action: drop + conntrack: + tcp_timeout: 12m + udp_timeout: 3m + default_timeout: 10m + outbound: + - port: any + proto: any + host: any + inbound: + - port: any + proto: any + host: any +"@ | Out-File -FilePath "$WorkDir\peer.yml" -Encoding utf8 + +# Stage WSL artifacts. Convert Windows paths to WSL paths ourselves rather than +# calling `wslpath`, because PowerShell's argument-passing to external EXEs +# strips backslashes from path arguments in ways that are hard to escape around. +function ConvertTo-WslPath { + param([string]$WindowsPath) + if ($WindowsPath -notmatch '^([A-Za-z]):\\(.*)$') { + throw "cannot convert path to WSL: $WindowsPath" + } + return "/mnt/$($matches[1].ToLower())/$($matches[2].Replace('\','/'))" +} + +$WslWorkDir = ConvertTo-WslPath $WorkDir +$WslNebulaPath = ConvertTo-WslPath $NebulaLinux +wsl -d $Distro -- bash -c "cp '$WslWorkDir/ca.crt' '$WslWorkDir/peer.crt' '$WslWorkDir/peer.key' '$WslWorkDir/peer.yml' $WslDir/ && cp '$WslNebulaPath' $WslDir/nebula && chmod +x $WslDir/nebula" + +# Make sure WSL has tun support and /dev/net/tun is usable before starting +# nebula. Diagnostics first so a fail here points at the real problem (e.g. +# WSL1 distros do not have a real kernel and will not have tun). +Write-Host '=== WSL diagnostic ===' +wsl --version 2>&1 | Out-Host +wsl --list --verbose 2>&1 | Out-Host +wsl -d $Distro -u root -- uname -a | Out-Host +wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; ls -l /dev/net/tun" +if ($LASTEXITCODE -ne 0) { throw "failed to prepare /dev/net/tun in WSL (TUN support missing?)" } + +# Deliberately no New-NetFirewallRule calls here -- nebula's windows_bypass_wdf +# feature is supposed to install WFP permit filters that let inbound traffic +# through Windows Defender Firewall on its own. If this smoke regresses, that +# feature regressed. + +$lhOut = Join-Path $WorkDir 'lighthouse.out.log' +$lhErr = Join-Path $WorkDir 'lighthouse.err.log' +$lhProc = Start-Process -FilePath $Nebula -ArgumentList @('-config', "$WorkDir\lighthouse.yml") ` + -PassThru -NoNewWindow ` + -RedirectStandardOutput $lhOut ` + -RedirectStandardError $lhErr + +# Run nebula in WSL as root with no sudo + no shell wrapper. PowerShell's +# Start-Process arg quoting mangles `bash -c "..."` strings that contain +# spaces/redirections, so we skip bash entirely and let Start-Process do the +# stdout/stderr capture itself. +$peerOut = Join-Path $WorkDir 'peer.out.log' +$peerErr = Join-Path $WorkDir 'peer.err.log' +$peerProc = Start-Process -FilePath 'wsl' ` + -ArgumentList @('-d', $Distro, '-u', 'root', '--', "$WslDir/nebula", '-config', "$WslDir/peer.yml") ` + -PassThru -NoNewWindow ` + -RedirectStandardOutput $peerOut ` + -RedirectStandardError $peerErr + +function Wait-Until { + param([scriptblock]$Predicate, [int]$TimeoutSec, [string]$What) + $deadline = (Get-Date).AddSeconds($TimeoutSec) + while ((Get-Date) -lt $deadline) { + if (& $Predicate) { return } + Start-Sleep -Milliseconds 500 + } + throw "timed out waiting for: $What" +} + +try { + Wait-Until -TimeoutSec 30 -What "windows wintun adapter $DevName with NetworkCategory=Private" -Predicate { + if ($lhProc.HasExited) { throw "lighthouse exited (code $($lhProc.ExitCode)) before tun was ready" } + $p = Get-NetConnectionProfile -InterfaceAlias $DevName -ErrorAction SilentlyContinue + $p -and ("$($p.NetworkCategory)" -ieq 'Private') + } + Write-Host "OK: $DevName NetworkCategory=Private" + + Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip2" -Predicate { + if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before tun was ready" } + $r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet $Ip2' && echo yes" + ("$r").Trim() -eq 'yes' + } + Write-Host "OK: WSL nebula1 has $Ip2" + + Wait-Until -TimeoutSec 30 -What "ping from WSL peer to windows lighthouse ($Ip1)" -Predicate { + if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before ping succeeded" } + $r = wsl -d $Distro -u root -- bash -c "ping -c1 -W1 $Ip1 >/dev/null 2>&1 && echo OK" + ("$r").Trim() -eq 'OK' + } + Write-Host "OK: WSL peer -> windows lighthouse" + + Wait-Until -TimeoutSec 30 -What "ping from windows lighthouse to WSL peer ($Ip2)" -Predicate { + $null = & ping.exe -n 1 -w 1000 $Ip2 + $LASTEXITCODE -eq 0 + } + Write-Host "OK: windows lighthouse -> WSL peer" + + Write-Host '' + Write-Host 'All smoke checks passed.' +} +catch { + Write-Host '' + Write-Host '=== lighthouse stdout ===' + Get-Content $lhOut -ErrorAction SilentlyContinue | Out-Host + Write-Host '=== lighthouse stderr ===' + Get-Content $lhErr -ErrorAction SilentlyContinue | Out-Host + Write-Host '=== peer stdout ===' + Get-Content $peerOut -ErrorAction SilentlyContinue | Out-Host + Write-Host '=== peer stderr ===' + Get-Content $peerErr -ErrorAction SilentlyContinue | Out-Host + Write-Host '=== nebula WFP filters ===' + # Dump nebula-installed filters so we can verify they got registered with + # the conditions we expect. + $wfpDump = Join-Path $WorkDir 'wfp.xml' + netsh wfp show filters file=$wfpDump 2>&1 | Out-Null + if (Test-Path $wfpDump) { + Select-String -Path $wfpDump -Pattern 'Nebula' -Context 0,80 -ErrorAction SilentlyContinue | Out-Host + } + throw +} +finally { + if (-not $lhProc.HasExited) { + Stop-Process -Id $lhProc.Id -Force -ErrorAction SilentlyContinue + $lhProc.WaitForExit(5000) | Out-Null + } + wsl -d $Distro -u root -- bash -c "pkill -f $WslDir/nebula 2>/dev/null; true" | Out-Null + # pkill returns 1 when no match and wsl propagates that; the smoke is done + # so we don't want it to leak into the script's exit code. + $global:LASTEXITCODE = 0 + if ($peerProc -and -not $peerProc.HasExited) { + Stop-Process -Id $peerProc.Id -Force -ErrorAction SilentlyContinue + } +} diff --git a/.github/workflows/smoke/smoke.sh b/.github/workflows/smoke/smoke.sh index 3177255b..f13ed380 100755 --- a/.github/workflows/smoke/smoke.sh +++ b/.github/workflows/smoke/smoke.sh @@ -6,6 +6,8 @@ set -o pipefail mkdir -p logs +NETWORK="nebula-smoke" + cleanup() { echo echo " *** cleanup" @@ -16,57 +18,103 @@ cleanup() { then docker kill lighthouse1 host2 host3 host4 fi + docker network rm "$NETWORK" >/dev/null 2>&1 } trap cleanup EXIT +# Create a dedicated smoke network with an explicit subnet (required for --ip +# below). Probe a short list of candidates so a locally-used range doesn't +# fail the whole test — we only need one to be free. +docker network rm "$NETWORK" >/dev/null 2>&1 || true +for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do + if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then + break + fi +done +if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then + echo "failed to create $NETWORK: every candidate subnet is in use" >&2 + exit 1 +fi + +# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1, +# .3 host2, .4 host3, .5 host4 — matches the placeholders in build.sh. +SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")" +PREFIX="${SUBNET%/*}" +PREFIX="${PREFIX%.*}" +LIGHTHOUSE_IP="$PREFIX.2" +HOST2_IP="$PREFIX.3" +HOST3_IP="$PREFIX.4" +HOST4_IP="$PREFIX.5" + +if [ "$SMOKE_OVERLAY_IPV6" ] +then + LIGHTHOUSE_NIP="fd00:4242:0:0:0:ffff:c0a8:6401" + HOST2_NIP="fd00:4242:0:0:0:ffff:c0a8:6402" + HOST3_NIP="fd00:4242:0:0:0:ffff:c0a8:6403" + HOST4_NIP="fd00:4242:0:0:0:ffff:c0a8:6404" +else + LIGHTHOUSE_NIP="192.168.100.1" + HOST2_NIP="192.168.100.2" + HOST3_NIP="192.168.100.3" + HOST4_NIP="192.168.100.4" +fi + +# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones. +# build/lighthouse1.yml has no IPs to rewrite so it's skipped. +for f in build/host2.yml build/host3.yml build/host4.yml; do + sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp" + mv "$f.tmp" "$f" +done + CONTAINER="nebula:${NAME:-smoke}" docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test -docker run --name host2 --rm "$CONTAINER" -config host2.yml -test -docker run --name host3 --rm "$CONTAINER" -config host3.yml -test -docker run --name host4 --rm "$CONTAINER" -config host4.yml -test +docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test +docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" "$CONTAINER" -config host3.yml -test +docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" "$CONTAINER" -config host4.yml -test -docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' & +docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' & sleep 1 -docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' & +docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' & sleep 1 -docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' & +docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' & sleep 1 -docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' & +docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' & sleep 1 # grab tcpdump pcaps for debugging -docker exec lighthouse1 tcpdump -i nebula1 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap & +docker exec lighthouse1 tcpdump -i tun0 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap & docker exec lighthouse1 tcpdump -i eth0 -q -w - -U 2>logs/lighthouse1.outside.log >logs/lighthouse1.outside.pcap & -docker exec host2 tcpdump -i nebula1 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap & +docker exec host2 tcpdump -i tun0 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap & docker exec host2 tcpdump -i eth0 -q -w - -U 2>logs/host2.outside.log >logs/host2.outside.pcap & -docker exec host3 tcpdump -i nebula1 -q -w - -U 2>logs/host3.inside.log >logs/host3.inside.pcap & +docker exec host3 tcpdump -i tun0 -q -w - -U 2>logs/host3.inside.log >logs/host3.inside.pcap & docker exec host3 tcpdump -i eth0 -q -w - -U 2>logs/host3.outside.log >logs/host3.outside.pcap & -docker exec host4 tcpdump -i nebula1 -q -w - -U 2>logs/host4.inside.log >logs/host4.inside.pcap & +docker exec host4 tcpdump -i tun0 -q -w - -U 2>logs/host4.inside.log >logs/host4.inside.pcap & docker exec host4 tcpdump -i eth0 -q -w - -U 2>logs/host4.outside.log >logs/host4.outside.pcap & -docker exec host2 ncat -nklv 0.0.0.0 2000 & -docker exec host3 ncat -nklv 0.0.0.0 2000 & -docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 0.0.0.0 3000 & -docker exec host3 ncat -e '/usr/bin/echo host3' -nkluv 0.0.0.0 3000 & +docker exec host2 ncat -nklv 2000 & +docker exec host3 ncat -nklv 2000 & +docker exec host4 ncat -e '/usr/bin/echo helloagainfromhost4' -nkluv 4000 & +docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 3000 & +docker exec host3 ncat -e '/usr/bin/echo host3' -nkluv 3000 & set +x echo echo " *** Testing ping from lighthouse1" echo set -x -docker exec lighthouse1 ping -c1 192.168.100.2 -docker exec lighthouse1 ping -c1 192.168.100.3 +docker exec lighthouse1 ping -c1 $HOST2_NIP +docker exec lighthouse1 ping -c1 $HOST3_NIP set +x echo echo " *** Testing ping from host2" echo set -x -docker exec host2 ping -c1 192.168.100.1 +docker exec host2 ping -c1 $LIGHTHOUSE_NIP # Should fail because not allowed by host3 inbound firewall -! docker exec host2 ping -c1 192.168.100.3 -w5 || exit 1 +! docker exec host2 ping -c1 $HOST3_NIP -w5 || exit 1 set +x echo @@ -74,34 +122,34 @@ echo " *** Testing ncat from host2" echo set -x # Should fail because not allowed by host3 inbound firewall -! docker exec host2 ncat -nzv -w5 192.168.100.3 2000 || exit 1 -! docker exec host2 ncat -nzuv -w5 192.168.100.3 3000 | grep -q host3 || exit 1 +! docker exec host2 ncat -nzv -w5 $HOST3_NIP 2000 || exit 1 +! docker exec host2 ncat -nzuv -w5 $HOST3_NIP 3000 | grep -q host3 || exit 1 set +x echo echo " *** Testing ping from host3" echo set -x -docker exec host3 ping -c1 192.168.100.1 -docker exec host3 ping -c1 192.168.100.2 +docker exec host3 ping -c1 $LIGHTHOUSE_NIP +docker exec host3 ping -c1 $HOST2_NIP set +x echo echo " *** Testing ncat from host3" echo set -x -docker exec host3 ncat -nzv -w5 192.168.100.2 2000 -docker exec host3 ncat -nzuv -w5 192.168.100.2 3000 | grep -q host2 +docker exec host3 ncat -nzv -w5 $HOST2_NIP 2000 +docker exec host3 ncat -nzuv -w5 $HOST2_NIP 3000 | grep -q host2 set +x echo echo " *** Testing ping from host4" echo set -x -docker exec host4 ping -c1 192.168.100.1 +docker exec host4 ping -c1 $LIGHTHOUSE_NIP # Should fail because not allowed by host4 outbound firewall -! docker exec host4 ping -c1 192.168.100.2 -w5 || exit 1 -! docker exec host4 ping -c1 192.168.100.3 -w5 || exit 1 +! docker exec host4 ping -c1 $HOST2_NIP -w5 || exit 1 +! docker exec host4 ping -c1 $HOST3_NIP -w5 || exit 1 set +x echo @@ -109,27 +157,34 @@ echo " *** Testing ncat from host4" echo set -x # Should fail because not allowed by host4 outbound firewall -! docker exec host4 ncat -nzv -w5 192.168.100.2 2000 || exit 1 -! docker exec host4 ncat -nzv -w5 192.168.100.3 2000 || exit 1 -! docker exec host4 ncat -nzuv -w5 192.168.100.2 3000 | grep -q host2 || exit 1 -! docker exec host4 ncat -nzuv -w5 192.168.100.3 3000 | grep -q host3 || exit 1 +! docker exec host4 ncat -nzv -w5 $HOST2_NIP 2000 || exit 1 +! docker exec host4 ncat -nzv -w5 $HOST3_NIP 2000 || exit 1 +! docker exec host4 ncat -nzuv -w5 $HOST2_NIP 3000 | grep -q host2 || exit 1 +! docker exec host4 ncat -nzuv -w5 $HOST3_NIP 3000 | grep -q host3 || exit 1 set +x echo echo " *** Testing conntrack" echo set -x -# host2 can ping host3 now that host3 pinged it first -docker exec host2 ping -c1 192.168.100.3 -# host4 can ping host2 once conntrack established -docker exec host2 ping -c1 192.168.100.4 -docker exec host4 ping -c1 192.168.100.2 + +# host4's outbound firewall only allows ICMP to the lighthouse, so host4 +# cannot initiate UDP to host2. Once host2 initiates a flow to host4:4000, +# conntrack must let host4's listener reply on that flow. If it doesn't, +# the echo back from host4 never reaches host2. +docker exec host2 sh -c "(/usr/bin/echo host2; sleep 2) | ncat -nuv $HOST4_NIP 4000" | grep -q helloagainfromhost4 docker exec host4 sh -c 'kill 1' docker exec host3 sh -c 'kill 1' docker exec host2 sh -c 'kill 1' docker exec lighthouse1 sh -c 'kill 1' -sleep 1 + +# Wait up to 30s for all backgrounded jobs to exit rather than relying on a +# fixed sleep. +for _ in $(seq 1 30); do + [ -z "$(jobs -r)" ] && break + sleep 1 +done if [ "$(jobs -r)" ] then diff --git a/.github/workflows/smoke/vagrant-freebsd-amd64/Vagrantfile b/.github/workflows/smoke/vagrant-freebsd-amd64/Vagrantfile new file mode 100644 index 00000000..c8a4c64a --- /dev/null +++ b/.github/workflows/smoke/vagrant-freebsd-amd64/Vagrantfile @@ -0,0 +1,7 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure("2") do |config| + config.vm.box = "generic/freebsd14" + + config.vm.synced_folder "../build", "/nebula", type: "rsync" +end diff --git a/.github/workflows/smoke/vagrant-linux-386/Vagrantfile b/.github/workflows/smoke/vagrant-linux-386/Vagrantfile new file mode 100644 index 00000000..4b1d0bd0 --- /dev/null +++ b/.github/workflows/smoke/vagrant-linux-386/Vagrantfile @@ -0,0 +1,7 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure("2") do |config| + config.vm.box = "ubuntu/xenial32" + + config.vm.synced_folder "../build", "/nebula" +end diff --git a/.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile b/.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile new file mode 100644 index 00000000..eeb9679e --- /dev/null +++ b/.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile @@ -0,0 +1,16 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure("2") do |config| + config.vm.box = "bento/ubuntu-24.04" + + config.vm.synced_folder "../build", "/nebula" + + config.vm.provision :shell do |shell| + shell.inline = <<-EOF + sed -i 's/GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="ipv6.disable=1"/' /etc/default/grub + update-grub + EOF + shell.privileged = true + shell.reboot = true + end +end diff --git a/.github/workflows/smoke/vagrant-netbsd-amd64/Vagrantfile b/.github/workflows/smoke/vagrant-netbsd-amd64/Vagrantfile new file mode 100644 index 00000000..a3fa7ec2 --- /dev/null +++ b/.github/workflows/smoke/vagrant-netbsd-amd64/Vagrantfile @@ -0,0 +1,7 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure("2") do |config| + config.vm.box = "DefinedNet/netbsd10" + + config.vm.synced_folder "../build", "/nebula", type: "rsync" +end diff --git a/.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile b/.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile new file mode 100644 index 00000000..6dd26373 --- /dev/null +++ b/.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile @@ -0,0 +1,7 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure("2") do |config| + config.vm.box = "DefinedNet/openbsd78" + + config.vm.synced_folder "../build", "/nebula", type: "rsync" +end diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34fe5f32..443279d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,92 +13,138 @@ on: - 'go.sum' jobs: - test-linux: - name: Build all and test on ubuntu-linux + static: + name: Static checks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v7 with: - go-version-file: 'go.mod' + go-version: '1.26' check-latest: true - - name: Build - run: make all + - name: Install goimports + run: go install golang.org/x/tools/cmd/goimports@latest + + - name: gofmt + run: | + if [ "$(find . -iname '*.go' | grep -v '\.pb\.go$' | xargs goimports -l)" ] + then + find . -iname '*.go' | grep -v '\.pb\.go$' | xargs goimports -d + exit 1 + fi - name: Vet run: make vet - - name: Test - run: make test - - - name: End 2 end - run: make e2evv - - - name: Build test mobile - run: make build-test-mobile - - - uses: actions/upload-artifact@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v9 with: - name: e2e packet flow - path: e2e/mermaid/ - if-no-files-found: warn + version: v2.12 - test-linux-boringcrypto: - name: Build and test on linux with boringcrypto - runs-on: ubuntu-latest + test: + name: Test ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: linux + os: ubuntu-latest + build-cmd: go build ./cmd/nebula ./cmd/nebula-cert + test-cmd: make test + e2e-cmd: make e2evv + - name: linux-boringcrypto + os: ubuntu-latest + build-cmd: make bin-boringcrypto + test-cmd: make test-boringcrypto + e2e-cmd: make e2e GOEXPERIMENT=boringcrypto CGO_ENABLED=1 TEST_ENV="TEST_LOGS=1" TEST_FLAGS="-v -ldflags -checklinkname=0" + - name: linux-pkcs11 + os: ubuntu-latest + build-cmd: make bin-pkcs11 + test-cmd: make test-pkcs11 + e2e-cmd: '' + - name: macos + os: macos-latest + build-cmd: go build ./cmd/nebula ./cmd/nebula-cert + test-cmd: make test + e2e-cmd: make e2evv + - name: windows + os: windows-latest + build-cmd: go build ./cmd/nebula ./cmd/nebula-cert + test-cmd: make test + e2e-cmd: make e2evv steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v7 with: - go-version-file: 'go.mod' + go-version: '1.26' check-latest: true - name: Build - run: make bin-boringcrypto + run: ${{ matrix.build-cmd }} + + - name: Cross-build darwin-amd64 + if: matrix.name == 'macos' + run: GOARCH=amd64 go build -o /tmp/nebula-amd64 ./cmd/nebula && GOARCH=amd64 go build -o /tmp/nebula-cert-amd64 ./cmd/nebula-cert - name: Test - run: make test-boringcrypto + run: ${{ matrix.test-cmd }} - name: End 2 end - run: make e2evv GOEXPERIMENT=boringcrypto CGO_ENABLED=1 + if: matrix.e2e-cmd != '' + run: ${{ matrix.e2e-cmd }} - test: - name: Build and test on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + - uses: actions/upload-artifact@v7 + if: matrix.e2e-cmd != '' && always() + with: + name: e2e packet flow ${{ matrix.name }} + path: e2e/mermaid/ + if-no-files-found: warn + + cross-build: + name: Cross-build ${{ matrix.name }} + runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - os: [windows-latest, macos-11] + include: + - {name: linux-arm, make-target: all-cross-linux-arm} + - {name: linux-mips, make-target: all-cross-linux-mips} + - {name: linux-other, make-target: all-cross-linux-other} + - {name: freebsd, make-target: all-freebsd} + - {name: openbsd, make-target: all-openbsd} + - {name: netbsd, make-target: all-netbsd} + - {name: windows, make-target: all-cross-windows} + - {name: mobile, make-target: build-test-mobile} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v7 with: - go-version-file: 'go.mod' + go-version: '1.26' check-latest: true - - name: Build nebula - run: go build ./cmd/nebula - - - name: Build nebula-cert - run: go build ./cmd/nebula-cert - - - name: Vet - run: make vet + - name: Build ${{ matrix.name }} + run: make -j"$(nproc)" ${{ matrix.make-target }} - - name: Test - run: make test + finish: + name: CI status + if: always() + needs: [static, test, cross-build] + runs-on: ubuntu-latest + steps: - - name: End 2 end - run: make e2evv + - name: Fail if any upstream job failed + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "upstream results: ${{ toJSON(needs) }}" + exit 1 - - uses: actions/upload-artifact@v3 - with: - name: e2e packet flow - path: e2e/mermaid/ - if-no-files-found: warn + - name: All upstream jobs passed + run: echo "ok" diff --git a/.gitignore b/.gitignore index 0efb9675..55068f31 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,8 @@ /nebula-darwin /nebula.exe /nebula-cert.exe -/coverage.out +**/coverage.out +**/cover.out /cpu.pprof /build /*.tar.gz @@ -13,5 +14,6 @@ **.crt **.key **.pem +**.pub !/examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.key !/examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.crt diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 00000000..be0513d4 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,37 @@ +version: "2" +linters: + default: none + enable: + - sloglint + - testifylint + settings: + sloglint: + # Enforce key-value pair form for Info/Debug/Warn/Error/Log/With and + # the package-level slog equivalents. Use l.Log(ctx, level, ...) for + # custom levels instead of LogAttrs when you can. + # + # LogAttrs is also flagged by this rule because it takes ...slog.Attr; + # the few legitimate sites (where attrs is built up as a []slog.Attr) + # carry a //nolint:sloglint with rationale. + kv-only: true + # no-mixed-args is on by default: forbids mixing kv and attrs in one call. + # discard-handler is on by default (since Go 1.24): suggests + # slog.DiscardHandler over slog.NewTextHandler(io.Discard, nil). + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/CHANGELOG.md b/CHANGELOG.md index 71c3ed47..b86444f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,318 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.11.0] - 2026-07-23 + +See the [v1.11.0](https://github.com/slackhq/nebula/milestone/25?closed=1) milestone for a complete list of changes. + +### Breaking + +- Logging has switched from logrus to Go's structured `slog`. Log output changes: levels are upper case + (`level=INFO`), trace prints as `level=DEBUG-4`, timestamps are always RFC3339Nano and `logging.timestamp_format` + is ignored, and some messages were reworded. Review any log parsing before upgrading. This is also an API break + for embedders, as constructors now take a `*slog.Logger`. (#1672, #1734, #1621) +- `firewall.inbound_action` and `firewall.outbound_action` (used to set reject vs. drop policy) were each being + applied to the opposite direction, that is now corrected. This only affects how blocked packets are answered, not + which packets the firewall allows or denies. If you set either of these you are getting the behavior of the other + one today and likely want to swap them before upgrading. (#1798) +- On Windows, Nebula now installs WFP PERMIT filters for the nebula adapter and the listener port by default. WFP + sits below Windows Defender Firewall, so any WDF inbound rules you rely on for either will no longer apply. Set + `tun.windows_bypass_wdf` and `listen.windows_bypass_wdf` to false to leave WDF in charge. (#1710) +- On Windows, the nebula device is now set to the `private` network category instead of whatever Windows decided, + which is usually `Public`. This makes the host firewall less restrictive on the overlay. Set + `tun.network_category` to `unset` to keep the old behavior. (#1710) +- Reject packets for non-TCP now use ICMP code 13, communication administratively prohibited, instead of code 3, + port unreachable. Anything keying off the old code needs updating. (#1766, #1768) +- The SSH debug server's profiling commands are now confined to `sshd.sandbox_dir`, which defaults to + `$TMP/nebula-debug`. Relative paths resolve inside it and absolute paths outside it are rejected, so anything + scripting `start-cpu-profile`, `save-heap-profile`, or `save-mutex-profile` with a path elsewhere needs the + directory set. The directory is not created for you. (#1622) + +### Added + +- Sign the Windows release binaries. (#1718) +- Generate IPv6 reject packets, matching the existing IPv4 behavior. (#1766, #1767, #1768) +- Accept `-` in `nebula-cert` to read from stdin or write to stdout. (#1714) +- Search for both `config.yml` and `config.yaml` in service and command line modes. (#1717) +- Add version labels to the Docker/OCI images. (#1772) +- Rebind the listener and re-query lighthouses on macOS when the underlay network changes, so devices moving + between wifi and wired or between networks recover without waiting for dead tunnel detection. Controlled by + `listen.rebind_on_network_change` (default `true`, not reloadable). (#1816) + +### Changed + +- Reload the firewall when the unsafe networks in the certificate change. (#1719) +- Reconfigure, start, and stop the stats listener on a config reload instead of requiring a restart. (#1670) +- Update a static host's addresses when they change on reload. (#1713) +- Don't require a port on ICMP firewall rules. (#1609) +- Connection track ICMP traffic. (#1602) +- Return `NODATA` instead of `NXDOMAIN` from the DNS server for a name that exists but has no record of the + requested type, so clients that query `AAAA` first (busybox/Alpine) fall through to `A`. (#1668) +- Record the local host's details in the DNS server. (#1716) +- Install Windows unsafe routes as link routes. (#1709) +- Reduce relay handshake log spam, and only log a handshake send error at error level when the remote list + changes. (#1733, #1765, #1810) +- Start, stop, and reload subsystems (DNS, stats, conntrack, ssh, punchy) cleanly without leaking goroutines. (#1640, #1654, #1661, #1667, #1669, #1708, #1806, #1815) +- `Control` is now safe to stop and wait on from any lifecycle state, and a new `Control.Wait` blocks until nebula + has fully stopped and returns the first fatal reader error. Failed starts release the udp sockets and tun fd + instead of leaking them. (#1794) +- Trigger an immediate lighthouse update when reconnecting to or adding a lighthouse instead of waiting for the next update tick. (#1645) +- Bring the Darwin and OpenBSD tun implementations in line with the other BSDs. (#1703) +- Update to build against go v1.26. (#1818) +- Various dependency updates. (#1586, #1587, #1604, #1617, #1618, #1627, #1628, #1629, #1652, #1664, #1665, #1697, #1721, #1732, #1742, #1743, #1750, #1763, #1771, #1782, #1800, #1807) + +### Fixed + +- Fix a data race on a host's remote address that could send packets to the wrong address during a roam. (#1773) +- Fix tunnels that could permanently escape connection manager monitoring. (#1752) +- Fix a crash when reloading the SSH server's trusted keys. (#1787) +- Fix hostmap corruption when a host has multiple overlay addresses. Each address now gets its own list instead of + a single shared chain, which also fixes two latent bugs on the add and makePrimary paths. (#1788, #1790) +- Apply `remote_allow_list` IPv4 rules to 4-in-6 mapped addresses. (#1786) +- Don't panic in the DNS server on a short or empty query name. (#1635) +- Advance the replay window on relayed packets so a relay drops replayed frames instead of re-forwarding them. (#1751) +- Fix a race in relay state handling. (#1753) +- Lock replay window updates so concurrent readers can't corrupt it. (#1802) +- Reject malformed handshakes more reliably, including invalid ed25519 key lengths. (#1601, #1756) +- Properly handle `closetunnel` packets. (#1638) +- Fix an IPv6 extension-header length overflow that could make the firewall parse the wrong protocol and ports. (#1789) +- Fix relay re-establishment when a handshake arrives over a relay entry that a one-sided teardown left + `Disestablished`, which silently dropped every send until dead tunnel detection forced a re-handshake. (#1805) +- Don't build new relay state on a tunnel that was just discarded. (#1796) +- Don't delete the wrong pending hostinfo in the handshake manager. (#1811) +- Don't call the packet reader after a UDP error on Darwin. (#1755) +- Open the FreeBSD tun device non blocking. (#1666) + +## [1.10.3] - 2026-02-06 + +### Security + +- Fix an issue where blocklist bypass is possible when using curve P256 since the signature can have 2 valid representations. + Both fingerprint representations will be tested against the blocklist. + Any newly issued P256 based certificates will have their signature clamped to the low-s form. + Nebula will assert the low-s signature form when validating certificates in a future version. [GHSA-69x3-g4r3-p962](https://github.com/slackhq/nebula/security/advisories/GHSA-69x3-g4r3-p962) + +### Changed + +- Improve error reporting if nebula fails to start due to a tun device naming issue. (#1588) + +## [1.10.2] - 2026-01-21 + +### Fixed + +- Fix panic when using `use_system_route_table` that was introduced in v1.10.1. (#1580) + +### Changed + +- Fix some typos in comments. (#1582) +- Dependency updates. (#1581) + +## [1.10.1] - 2026-01-16 + +See the [v1.10.1](https://github.com/slackhq/nebula/milestone/26?closed=1) milestone for a complete list of changes. + +### Fixed + +- Fix a bug where an unsafe route derived from the system route table could be lost on a config reload. (#1573) +- Fix the PEM banner for ECDSA P256 public keys. (#1552) +- Fix a regression on Windows from 1.9.x where nebula could fall back to a less performant UDP listener if + non-critical ioctls failed. (#1568) +- Fix a bug in handshake processing when a peer sends an unexpected public key. (#1566) + +### Added + +- Add a config option to control accepting `recv_error` packets which defaults to `always`. (#1569) + +### Changed + +- Various dependency updates. (#1541, #1549, #1550, #1557, #1558, #1560, #1561, #1570, #1571) + +## [1.10.0] - 2025-12-04 + +See the [v1.10.0](https://github.com/slackhq/nebula/milestone/16?closed=1) milestone for a complete list of changes. + +### Added + +- Support for ipv6 and multiple ipv4/6 addresses in the overlay. + A new v2 ASN.1 based certificate format. + Certificates now have a unified interface for external implementations. + (#1212, #1216, #1345, #1359, #1381, #1419, #1464, #1466, #1451, #1476, #1467, #1481, #1399, #1488, #1492, #1495, #1468, #1521, #1535, #1538) +- Add the ability to mark packets on linux to better target nebula packets in iptables/nftables. (#1331) +- Add ECMP support for `unsafe_routes`. (#1332) +- PKCS11 support for P256 keys when built with `pkcs11` tag (#1153, #1482) + +### Changed + +- **NOTE**: `default_local_cidr_any` now defaults to false, meaning that any firewall rule + intended to target an `unsafe_routes` entry must explicitly declare it via the + `local_cidr` field. This is almost always the intended behavior. This flag is + deprecated and will be removed in a future release. (#1373) +- Improve logging when a relay is in use on an inbound packet. (#1533) +- Avoid fatal errors if `rountines` is > 1 on systems that don't support more than 1 routine. (#1531) +- Log a warning if a firewall rule contains an `any` that negates a more restrictive filter. (#1513) +- Accept encrypted CA passphrase from an environment variable. (#1421) +- Allow handshaking with any trusted remote. (#1509) +- Log only the count of blocklisted certificate fingerprints instead of the entire list. (#1525) +- Don't fatal when the ssh server is unable to be configured successfully. (#1520) +- Update to build against go v1.25. (#1483) +- Allow projects using `nebula` as a library with userspace networking to configure the `logger` and build version. (#1239) +- Upgrade to `yaml.v3`. (#1148, #1371, #1438, #1478) + +### Fixed + +- Fix a potential bug with udp ipv4 only on darwin. (#1532) +- Improve lost packet statistics. (#1441, #1537) +- Honor `remote_allow_list` in hole punch response. (#1186) +- Fix a panic when `tun.use_system_route_table` is `true` and a route lacks a destination. (#1437) +- Fix an issue when `tun.use_system_route_table: true` could result in heavy CPU utilization when many thousands of routes + are present. (#1326) +- Fix tests for 32 bit machines. (#1394) +- Fix a possible 32bit integer underflow in config handling. (#1353) +- Fix moving a udp address from one vpn address to another in the `static_host_map` + which could cause rapid re-handshaking with an incorrect remote. (#1259) +- Improve smoke tests in environments where the docker network is not the default. (#1347) + +## [1.9.7] - 2025-10-10 + +### Security + +- Fix an issue where Nebula could incorrectly accept and process a packet from an erroneous source IP when the sender's + certificate is configured with unsafe_routes (cert v1/v2) or multiple IPs (cert v2). (#1494) + +### Changed + +- Disable sending `recv_error` messages when a packet is received outside the allowable counter window. (#1459) +- Improve error messages and remove some unnecessary fatal conditions in the Windows and generic udp listener. (#1453) + +## [1.9.6] - 2025-7-15 + +### Added + +- Support dropping inactive tunnels. This is disabled by default in this release but can be enabled with `tunnels.drop_inactive`. See example config for more details. (#1413) + +### Fixed + +- Fix Darwin freeze due to presence of some Network Extensions (#1426) +- Ensure the same relay tunnel is always used when multiple relay tunnels are present (#1422) +- Fix Windows freeze due to ICMP error handling (#1412) +- Fix relay migration panic (#1403) + +## [1.9.5] - 2024-12-05 + +### Added + +- Gracefully ignore v2 certificates. (#1282) + +### Fixed + +- Fix relays that refuse to re-establish after one of the remote tunnel pairs breaks. (#1277) + +## [1.9.4] - 2024-09-09 + +### Added + +- Support UDP dialing with gVisor. (#1181) + +### Changed + +- Make some Nebula state programmatically available via control object. (#1188) +- Switch internal representation of IPs to netip, to prepare for IPv6 support + in the overlay. (#1173) +- Minor build and cleanup changes. (#1171, #1164, #1162) +- Various dependency updates. (#1195, #1190, #1174, #1168, #1167, #1161, #1147, #1146) + +### Fixed + +- Fix a bug on big endian hosts, like mips. (#1194) +- Fix a rare panic if a local index collision happens. (#1191) +- Fix integer wraparound in the calculation of handshake timeouts on 32-bit targets. (#1185) + +## [1.9.3] - 2024-06-06 + +### Fixed + +- Initialize messageCounter to 2 instead of verifying later. (#1156) + +## [1.9.2] - 2024-06-03 + +### Fixed + +- Ensure messageCounter is set before handshake is complete. (#1154) + +## [1.9.1] - 2024-05-29 + +### Fixed + +- Fixed a potential deadlock in GetOrHandshake. (#1151) + +## [1.9.0] - 2024-05-07 + +### Deprecated + +- This release adds a new setting `default_local_cidr_any` that defaults to + true to match previous behavior, but will default to false in the next + release (1.10). When set to false, `local_cidr` is matched correctly for + firewall rules on hosts acting as unsafe routers, and should be set for any + firewall rules you want to allow unsafe route hosts to access. See the issue + and example config for more details. (#1071, #1099) + +### Added + +- Nebula now has an official Docker image `nebulaoss/nebula` that is + distroless and contains just the `nebula` and `nebula-cert` binaries. You + can find it here: https://hub.docker.com/r/nebulaoss/nebula (#1037) + +- Experimental binaries for `loong64` are now provided. (#1003) + +- Added example service script for OpenRC. (#711) + +- The SSH daemon now supports inlined host keys. (#1054) + +- The SSH daemon now supports certificates with `sshd.trusted_cas`. (#1098) + +### Changed + +- Config setting `tun.unsafe_routes` is now reloadable. (#1083) + +- Small documentation and internal improvements. (#1065, #1067, #1069, #1108, + #1109, #1111, #1135) + +- Various dependency updates. (#1139, #1138, #1134, #1133, #1126, #1123, #1110, + #1094, #1092, #1087, #1086, #1085, #1072, #1063, #1059, #1055, #1053, #1047, + #1046, #1034, #1022) + +### Removed + +- Support for the deprecated `local_range` option has been removed. Please + change to `preferred_ranges` (which is also now reloadable). (#1043) + +- We are now building with go1.22, which means that for Windows you need at + least Windows 10 or Windows Server 2016. This is because support for earlier + versions was removed in Go 1.21. See https://go.dev/doc/go1.21#windows (#981) + +- Removed vagrant example, as it was unmaintained. (#1129) + +- Removed Fedora and Arch nebula.service files, as they are maintained in the + upstream repos. (#1128, #1132) + +- Remove the TCP round trip tracking metrics, as they never had correct data + and were an experiment to begin with. (#1114) + +### Fixed + +- Fixed a potential deadlock introduced in 1.8.1. (#1112) + +- Fixed support for Linux when IPv6 has been disabled at the OS level. (#787) + +- DNS will return NXDOMAIN now when there are no results. (#845) + +- Allow `::` in `lighthouse.dns.host`. (#1115) + +- Capitalization of `NotAfter` fixed in DNS TXT response. (#1127) + +- Don't log invalid certificates. It is untrusted data and can cause a large + volume of logs. (#1116) + ## [1.8.2] - 2024-01-08 ### Fixed @@ -558,7 +870,19 @@ created.) - Initial public release. -[Unreleased]: https://github.com/slackhq/nebula/compare/v1.8.2...HEAD +[Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.3...HEAD +[1.10.3]: https://github.com/slackhq/nebula/releases/tag/v1.10.3 +[1.10.2]: https://github.com/slackhq/nebula/releases/tag/v1.10.2 +[1.10.1]: https://github.com/slackhq/nebula/releases/tag/v1.10.1 +[1.10.0]: https://github.com/slackhq/nebula/releases/tag/v1.10.0 +[1.9.7]: https://github.com/slackhq/nebula/releases/tag/v1.9.7 +[1.9.6]: https://github.com/slackhq/nebula/releases/tag/v1.9.6 +[1.9.5]: https://github.com/slackhq/nebula/releases/tag/v1.9.5 +[1.9.4]: https://github.com/slackhq/nebula/releases/tag/v1.9.4 +[1.9.3]: https://github.com/slackhq/nebula/releases/tag/v1.9.3 +[1.9.2]: https://github.com/slackhq/nebula/releases/tag/v1.9.2 +[1.9.1]: https://github.com/slackhq/nebula/releases/tag/v1.9.1 +[1.9.0]: https://github.com/slackhq/nebula/releases/tag/v1.9.0 [1.8.2]: https://github.com/slackhq/nebula/releases/tag/v1.8.2 [1.8.1]: https://github.com/slackhq/nebula/releases/tag/v1.8.1 [1.8.0]: https://github.com/slackhq/nebula/releases/tag/v1.8.0 diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..00cd7bd1 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +#ECCN:Open Source diff --git a/LOGGING.md b/LOGGING.md index bd2fdefc..e2508c83 100644 --- a/LOGGING.md +++ b/LOGGING.md @@ -33,6 +33,5 @@ l.WithError(err). WithField("vpnIp", IntIp(hostinfo.hostId)). WithField("udpAddr", addr). WithField("handshake", m{"stage": 1, "style": "ix"}). - WithField("cert", remoteCert). Info("Invalid certificate from host") ``` \ No newline at end of file diff --git a/Makefile b/Makefile index a02a6ec4..24c71459 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,14 @@ -GOMINVERSION = 1.20 NEBULA_CMD_PATH = "./cmd/nebula" -GO111MODULE = on -export GO111MODULE CGO_ENABLED = 0 export CGO_ENABLED # Set up OS specific bits ifeq ($(OS),Windows_NT) - #TODO: we should be able to ditch awk as well - GOVERSION := $(shell go version | awk "{print substr($$3, 3)}") - GOISMIN := $(shell IF "$(GOVERSION)" GEQ "$(GOMINVERSION)" ECHO 1) NEBULA_CMD_SUFFIX = .exe NULL_FILE = nul # RIO on windows does pointer stuff that makes go vet angry VET_FLAGS = -unsafeptr=false else - GOVERSION := $(shell go version | awk '{print substr($$3, 3)}') - GOISMIN := $(shell expr "$(GOVERSION)" ">=" "$(GOMINVERSION)") NEBULA_CMD_SUFFIX = NULL_FILE = /dev/null endif @@ -30,6 +22,9 @@ ifndef BUILD_NUMBER endif endif +DOCKER_IMAGE_REPO ?= nebulaoss/nebula +DOCKER_IMAGE_TAG ?= latest + LDFLAGS = -X main.Build=$(BUILD_NUMBER) ALL_LINUX = linux-amd64 \ @@ -44,7 +39,8 @@ ALL_LINUX = linux-amd64 \ linux-mips64 \ linux-mips64le \ linux-mips-softfloat \ - linux-riscv64 + linux-riscv64 \ + linux-loong64 ALL_FREEBSD = freebsd-amd64 \ freebsd-arm64 @@ -64,10 +60,22 @@ ALL = $(ALL_LINUX) \ windows-amd64 \ windows-arm64 +# Cross-build shards used by .github/workflows/test.yml — same as ALL_* +# but with the arch that has a native CI runner removed, so the cross-build +# job is not duplicating coverage the native test jobs already give. +ALL_CROSS_LINUX = $(filter-out linux-amd64,$(ALL_LINUX)) + +# ALL_CROSS_LINUX further split into family sub-shards so each can run on +# its own CI runner in parallel. Union of the three must equal +# ALL_CROSS_LINUX; adding a new linux arch goes into the matching family. +ALL_CROSS_LINUX_ARM = linux-arm-5 linux-arm-6 linux-arm-7 linux-arm64 +ALL_CROSS_LINUX_MIPS = linux-mips linux-mipsle linux-mips64 linux-mips64le linux-mips-softfloat +ALL_CROSS_LINUX_OTHER = linux-386 linux-ppc64le linux-riscv64 linux-loong64 + e2e: $(TEST_ENV) go test -tags=e2e_testing -count=1 $(TEST_FLAGS) ./e2e -e2ev: TEST_FLAGS = -v +e2ev: TEST_FLAGS += -v e2ev: e2e e2evv: TEST_ENV += TEST_LOGS=1 @@ -82,8 +90,41 @@ e2evvvv: e2ev e2e-bench: TEST_FLAGS = -bench=. -benchmem -run=^$ e2e-bench: e2e +DOCKER_BIN = build/linux-amd64/nebula build/linux-amd64/nebula-cert + all: $(ALL:%=build/%/nebula) $(ALL:%=build/%/nebula-cert) +all-linux: $(ALL_LINUX:%=build/%/nebula) $(ALL_LINUX:%=build/%/nebula-cert) + +all-freebsd: $(ALL_FREEBSD:%=build/%/nebula) $(ALL_FREEBSD:%=build/%/nebula-cert) + +all-openbsd: $(ALL_OPENBSD:%=build/%/nebula) $(ALL_OPENBSD:%=build/%/nebula-cert) + +all-netbsd: $(ALL_NETBSD:%=build/%/nebula) $(ALL_NETBSD:%=build/%/nebula-cert) + +all-darwin: build/darwin-amd64/nebula build/darwin-amd64/nebula-cert build/darwin-arm64/nebula build/darwin-arm64/nebula-cert + +all-windows: build/windows-amd64/nebula.exe build/windows-amd64/nebula-cert.exe build/windows-arm64/nebula.exe build/windows-arm64/nebula-cert.exe + +# CI cross-build shards. darwin-arm64 is covered by the native macos-latest +# job; windows-amd64 is covered by the native windows-latest job; both are +# omitted here to avoid building them a second time. darwin-amd64 stays in +# all-cross-darwin because intel mac is only a labeled/master-time native +# job, so PRs still need cross-build coverage for it. +all-cross-linux: $(ALL_CROSS_LINUX:%=build/%/nebula) $(ALL_CROSS_LINUX:%=build/%/nebula-cert) + +all-cross-linux-arm: $(ALL_CROSS_LINUX_ARM:%=build/%/nebula) $(ALL_CROSS_LINUX_ARM:%=build/%/nebula-cert) + +all-cross-linux-mips: $(ALL_CROSS_LINUX_MIPS:%=build/%/nebula) $(ALL_CROSS_LINUX_MIPS:%=build/%/nebula-cert) + +all-cross-linux-other: $(ALL_CROSS_LINUX_OTHER:%=build/%/nebula) $(ALL_CROSS_LINUX_OTHER:%=build/%/nebula-cert) + +all-cross-darwin: build/darwin-amd64/nebula build/darwin-amd64/nebula-cert + +all-cross-windows: build/windows-arm64/nebula.exe build/windows-arm64/nebula-cert.exe + +docker: docker/linux-$(shell go env GOARCH) + release: $(ALL:%=build/nebula-%.tar.gz) release-linux: $(ALL_LINUX:%=build/nebula-%.tar.gz) @@ -96,7 +137,7 @@ release-netbsd: $(ALL_NETBSD:%=build/nebula-%.tar.gz) release-boringcrypto: build/nebula-linux-$(shell go env GOARCH)-boringcrypto.tar.gz -BUILD_ARGS = -trimpath +BUILD_ARGS += -trimpath bin-windows: build/windows-amd64/nebula.exe build/windows-amd64/nebula-cert.exe mv $? . @@ -116,6 +157,10 @@ bin-freebsd-arm64: build/freebsd-arm64/nebula build/freebsd-arm64/nebula-cert bin-boringcrypto: build/linux-$(shell go env GOARCH)-boringcrypto/nebula build/linux-$(shell go env GOARCH)-boringcrypto/nebula-cert mv $? . +bin-pkcs11: BUILD_ARGS += -tags pkcs11 +bin-pkcs11: CGO_ENABLED = 1 +bin-pkcs11: bin + bin: go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula${NEBULA_CMD_SUFFIX} ${NEBULA_CMD_PATH} go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula-cert${NEBULA_CMD_SUFFIX} ./cmd/nebula-cert @@ -133,6 +178,8 @@ build/linux-mips-softfloat/%: LDFLAGS += -s -w # boringcrypto build/linux-amd64-boringcrypto/%: GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1 build/linux-arm64-boringcrypto/%: GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1 +build/linux-amd64-boringcrypto/%: LDFLAGS += -checklinkname=0 +build/linux-arm64-boringcrypto/%: LDFLAGS += -checklinkname=0 build/%/nebula: .FORCE GOOS=$(firstword $(subst -, , $*)) \ @@ -156,6 +203,9 @@ build/nebula-%.tar.gz: build/%/nebula build/%/nebula-cert build/nebula-%.zip: build/%/nebula.exe build/%/nebula-cert.exe cd build/$* && zip ../nebula-$*.zip nebula.exe nebula-cert.exe +docker/%: build/%/nebula build/%/nebula-cert + docker build . $(DOCKER_BUILD_ARGS) -f docker/Dockerfile --platform "$(subst -,/,$*)" --tag "${DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_TAG}" --tag "${DOCKER_IMAGE_REPO}:$(BUILD_NUMBER)" + vet: go vet $(VET_FLAGS) -v ./... @@ -163,7 +213,10 @@ test: go test -v ./... test-boringcrypto: - GOEXPERIMENT=boringcrypto CGO_ENABLED=1 go test -v ./... + GOEXPERIMENT=boringcrypto CGO_ENABLED=1 go test -ldflags "-checklinkname=0" -v ./... + +test-pkcs11: + CGO_ENABLED=1 go test -v -tags pkcs11 ./... test-cov-html: go test -coverprofile=coverage.out @@ -186,7 +239,7 @@ bench-cpu-long: go test -bench=. -benchtime=60s -cpuprofile=cpu.pprof go tool pprof go-audit.test cpu.pprof -proto: nebula.pb.go cert/cert.pb.go +proto: nebula.pb.go cert/cert_v1.pb.go nebula.pb.go: nebula.proto .FORCE go build github.com/gogo/protobuf/protoc-gen-gogofaster @@ -215,10 +268,17 @@ smoke-relay-docker: bin-docker cd .github/workflows/smoke/ && ./build-relay.sh cd .github/workflows/smoke/ && ./smoke-relay.sh +smoke-docker-ipv6: export SMOKE_OVERLAY_IPV6 = 1 +smoke-docker-ipv6: smoke-docker + smoke-docker-race: BUILD_ARGS = -race smoke-docker-race: CGO_ENABLED = 1 smoke-docker-race: smoke-docker +smoke-vagrant/%: bin-docker build/%/nebula + cd .github/workflows/smoke/ && ./build.sh $* + cd .github/workflows/smoke/ && ./smoke-vagrant.sh $* + .FORCE: -.PHONY: bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html +.PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html smoke-vagrant/% .DEFAULT_GOAL := bin diff --git a/README.md b/README.md index 51e913d5..7cbcb412 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ It lets you seamlessly connect computers anywhere in the world. Nebula is portab It can be used to connect a small number of computers, but is also able to connect tens of thousands of computers. Nebula incorporates a number of existing concepts like encryption, security groups, certificates, -and tunneling, and each of those individual pieces existed before Nebula in various forms. +and tunneling. What makes Nebula different to existing offerings is that it brings all of these ideas together, resulting in a sum that is greater than its individual parts. @@ -12,7 +12,7 @@ Further documentation can be found [here](https://nebula.defined.net/docs/). You can read more about Nebula [here](https://medium.com/p/884110a5579). -You can also join the NebulaOSS Slack group [here](https://join.slack.com/t/nebulaoss/shared_invite/enQtOTA5MDI4NDg3MTg4LTkwY2EwNTI4NzQyMzc0M2ZlODBjNWI3NTY1MzhiOThiMmZlZjVkMTI0NGY4YTMyNjUwMWEyNzNkZTJmYzQxOGU). +You can also join the NebulaOSS Slack group [here](https://join.slack.com/t/nebulaoss/shared_invite/zt-39pk4xopc-CUKlGcb5Z39dQ0cK1v7ehA). ## Supported Platforms @@ -28,41 +28,46 @@ Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for #### Distribution Packages - [Arch Linux](https://archlinux.org/packages/extra/x86_64/nebula/) - ``` - $ sudo pacman -S nebula + ```sh + sudo pacman -S nebula ``` - [Fedora Linux](https://src.fedoraproject.org/rpms/nebula) - ``` - $ sudo dnf install nebula + ```sh + sudo dnf install nebula ``` - [Debian Linux](https://packages.debian.org/source/stable/nebula) - ``` - $ sudo apt install nebula + ```sh + sudo apt install nebula ``` - [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=nebula) - ``` - $ sudo apk add nebula + ```sh + sudo apk add nebula ``` -- [macOS Homebrew](https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/nebula.rb) +- [macOS Homebrew](https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/n/nebula.rb) + ```sh + brew install nebula ``` - $ brew install nebula + +- [Docker](https://hub.docker.com/r/nebulaoss/nebula) + ```sh + docker pull nebulaoss/nebula ``` -#### Mobile +#### Mobile ([source code](https://github.com/DefinedNet/mobile_nebula)) - [iOS](https://apps.apple.com/us/app/mobile-nebula/id1509587936?itsct=apps_box&itscg=30200) - [Android](https://play.google.com/store/apps/details?id=net.defined.mobile_nebula&pcampaignid=pcampaignidMKT-Other-global-all-co-prtnr-py-PartBadge-Mar2515-1) ## Technical Overview -Nebula is a mutually authenticated peer-to-peer software defined network based on the [Noise Protocol Framework](https://noiseprotocol.org/). +Nebula is a mutually authenticated peer-to-peer software-defined network based on the [Noise Protocol Framework](https://noiseprotocol.org/). Nebula uses certificates to assert a node's IP address, name, and membership within user-defined groups. Nebula's user-defined groups allow for provider agnostic traffic filtering between nodes. -Discovery nodes allow individual peers to find each other and optionally use UDP hole punching to establish connections from behind most firewalls or NATs. +Discovery nodes (aka lighthouses) allow individual peers to find each other and optionally use UDP hole punching to establish connections from behind most firewalls or NATs. Users can move data between nodes in any number of cloud service providers, datacenters, and endpoints, without needing to maintain a particular addressing scheme. Nebula uses Elliptic-curve Diffie-Hellman (`ECDH`) key exchange and `AES-256-GCM` in its default configuration. @@ -71,34 +76,42 @@ Nebula was created to provide a mechanism for groups of hosts to communicate sec ## Getting started (quickly) +**Don't want to manage your own PKI and lighthouses?** [Managed Nebula](https://www.defined.net/) from Defined Networking handles all of this for you. + To set up a Nebula network, you'll need: #### 1. The [Nebula binaries](https://github.com/slackhq/nebula/releases) or [Distribution Packages](https://github.com/slackhq/nebula#distribution-packages) for your specific platform. Specifically you'll need `nebula-cert` and the specific nebula binary for each platform you use. #### 2. (Optional, but you really should..) At least one discovery node with a routable IP address, which we call a lighthouse. -Nebula lighthouses allow nodes to find each other, anywhere in the world. A lighthouse is the only node in a Nebula network whose IP should not change. Running a lighthouse requires very few compute resources, and you can easily use the least expensive option from a cloud hosting provider. If you're not sure which provider to use, a number of us have used $5/mo [DigitalOcean](https://digitalocean.com) droplets as lighthouses. - - Once you have launched an instance, ensure that Nebula udp traffic (default port udp/4242) can reach it over the internet. +Nebula lighthouses allow nodes to find each other, anywhere in the world. A lighthouse is the only node in a Nebula network whose IP should not change. Running a lighthouse requires very few compute resources, and you can easily use the least expensive option from a cloud hosting provider. If you're not sure which provider to use, a number of us have used $6/mo [DigitalOcean](https://digitalocean.com) droplets as lighthouses. +Once you have launched an instance, ensure that Nebula udp traffic (default port udp/4242) can reach it over the internet. #### 3. A Nebula certificate authority, which will be the root of trust for a particular Nebula network. - ``` - ./nebula-cert ca -name "Myorganization, Inc" - ``` - This will create files named `ca.key` and `ca.cert` in the current directory. The `ca.key` file is the most sensitive file you'll create, because it is the key used to sign the certificates for individual nebula nodes/hosts. Please store this file somewhere safe, preferably with strong encryption. +```sh +./nebula-cert ca -name "Myorganization, Inc" +``` + +This will create files named `ca.key` and `ca.cert` in the current directory. The `ca.key` file is the most sensitive file you'll create, because it is the key used to sign the certificates for individual nebula nodes/hosts. Please store this file somewhere safe, preferably with strong encryption. + +**Be aware!** By default, certificate authorities have a 1-year lifetime before expiration. See [this guide](https://nebula.defined.net/docs/guides/rotating-certificate-authority/) for details on rotating a CA. #### 4. Nebula host keys and certificates generated from that certificate authority + This assumes you have four nodes, named lighthouse1, laptop, server1, host3. You can name the nodes any way you'd like, including FQDN. You'll also need to choose IP addresses and the associated subnet. In this example, we are creating a nebula network that will use 192.168.100.x/24 as its network range. This example also demonstrates nebula groups, which can later be used to define traffic rules in a nebula network. -``` +```sh ./nebula-cert sign -name "lighthouse1" -ip "192.168.100.1/24" ./nebula-cert sign -name "laptop" -ip "192.168.100.2/24" -groups "laptop,home,ssh" ./nebula-cert sign -name "server1" -ip "192.168.100.9/24" -groups "servers" ./nebula-cert sign -name "host3" -ip "192.168.100.10/24" ``` +By default, host certificates will expire 1 second before the CA expires. Use the `-duration` flag to specify a shorter lifetime. + #### 5. Configuration files for each host + Download a copy of the nebula [example configuration](https://github.com/slackhq/nebula/blob/master/examples/config.yml). * On the lighthouse node, you'll need to ensure `am_lighthouse: true` is set. @@ -113,10 +126,13 @@ For each host, copy the nebula binary to the host, along with `config.yml` from **DO NOT COPY `ca.key` TO INDIVIDUAL NODES.** #### 7. Run nebula on each host -``` + +```sh ./nebula -config /path/to/config.yml ``` +For more detailed instructions, [find the full documentation here](https://nebula.defined.net/docs/). + ## Building Nebula from source Make sure you have [go](https://go.dev/doc/install) installed and clone this repo. Change to the nebula directory. @@ -135,8 +151,10 @@ The default curve used for cryptographic handshakes and signatures is Curve25519 In addition, Nebula can be built using the [BoringCrypto GOEXPERIMENT](https://github.com/golang/go/blob/go1.20/src/crypto/internal/boring/README.md) by running either of the following make targets: - make bin-boringcrypto - make release-boringcrypto +```sh +make bin-boringcrypto +make release-boringcrypto +``` This is not the recommended default deployment, but may be useful based on your compliance requirements. @@ -144,5 +162,3 @@ This is not the recommended default deployment, but may be useful based on your Nebula was created at Slack Technologies, Inc by Nate Brown and Ryan Huber, with contributions from Oliver Fross, Alan Lam, Wade Simmons, and Lining Wang. - - diff --git a/allow_list.go b/allow_list.go index 9186b2fc..cba56fce 100644 --- a/allow_list.go +++ b/allow_list.go @@ -2,17 +2,16 @@ package nebula import ( "fmt" - "net" + "net/netip" "regexp" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" ) type AllowList struct { // The values of this cidrTree are `bool`, signifying allow/deny - cidrTree *cidr.Tree6[bool] + cidrTree *bart.Table[bool] } type RemoteAllowList struct { @@ -20,7 +19,7 @@ type RemoteAllowList struct { // Inside Range Specific, keys of this tree are inside CIDRs and values // are *AllowList - insideAllowLists *cidr.Tree6[*AllowList] + insideAllowLists *bart.Table[*AllowList] } type LocalAllowList struct { @@ -37,7 +36,7 @@ type AllowListNameRule struct { func NewLocalAllowListFromConfig(c *config.C, k string) (*LocalAllowList, error) { var nameRules []AllowListNameRule - handleKey := func(key string, value interface{}) (bool, error) { + handleKey := func(key string, value any) (bool, error) { if key == "interfaces" { var err error nameRules, err = getAllowListInterfaces(k, value) @@ -71,7 +70,7 @@ func NewRemoteAllowListFromConfig(c *config.C, k, rangesKey string) (*RemoteAllo // If the handleKey func returns true, the rest of the parsing is skipped // for this key. This allows parsing of special values like `interfaces`. -func newAllowListFromConfig(c *config.C, k string, handleKey func(key string, value interface{}) (bool, error)) (*AllowList, error) { +func newAllowListFromConfig(c *config.C, k string, handleKey func(key string, value any) (bool, error)) (*AllowList, error) { r := c.Get(k) if r == nil { return nil, nil @@ -82,13 +81,13 @@ func newAllowListFromConfig(c *config.C, k string, handleKey func(key string, va // If the handleKey func returns true, the rest of the parsing is skipped // for this key. This allows parsing of special values like `interfaces`. -func newAllowList(k string, raw interface{}, handleKey func(key string, value interface{}) (bool, error)) (*AllowList, error) { - rawMap, ok := raw.(map[interface{}]interface{}) +func newAllowList(k string, raw any, handleKey func(key string, value any) (bool, error)) (*AllowList, error) { + rawMap, ok := raw.(map[string]any) if !ok { return nil, fmt.Errorf("config `%s` has invalid type: %T", k, raw) } - tree := cidr.NewTree6[bool]() + tree := new(bart.Table[bool]) // Keep track of the rules we have added for both ipv4 and ipv6 type allowListRules struct { @@ -101,12 +100,7 @@ func newAllowList(k string, raw interface{}, handleKey func(key string, value in rules4 := allowListRules{firstValue: true, allValuesMatch: true, defaultSet: false} rules6 := allowListRules{firstValue: true, allValuesMatch: true, defaultSet: false} - for rawKey, rawValue := range rawMap { - rawCIDR, ok := rawKey.(string) - if !ok { - return nil, fmt.Errorf("config `%s` has invalid key (type %T): %v", k, rawKey, rawKey) - } - + for rawCIDR, rawValue := range rawMap { if handleKey != nil { handled, err := handleKey(rawCIDR, rawValue) if err != nil { @@ -117,23 +111,24 @@ func newAllowList(k string, raw interface{}, handleKey func(key string, value in } } - value, ok := rawValue.(bool) + value, ok := config.AsBool(rawValue) if !ok { return nil, fmt.Errorf("config `%s` has invalid value (type %T): %v", k, rawValue, rawValue) } - _, ipNet, err := net.ParseCIDR(rawCIDR) + ipNet, err := netip.ParsePrefix(rawCIDR) if err != nil { - return nil, fmt.Errorf("config `%s` has invalid CIDR: %s", k, rawCIDR) + return nil, fmt.Errorf("config `%s` has invalid CIDR: %s. %w", k, rawCIDR, err) } - // TODO: should we error on duplicate CIDRs in the config? - tree.AddCIDR(ipNet, value) + ipNet = netip.PrefixFrom(ipNet.Addr().Unmap(), ipNet.Bits()) - maskBits, maskSize := ipNet.Mask.Size() + tree.Insert(ipNet, value) + + maskBits := ipNet.Bits() var rules *allowListRules - if maskSize == 32 { + if ipNet.Addr().Is4() { rules = &rules4 } else { rules = &rules6 @@ -156,8 +151,7 @@ func newAllowList(k string, raw interface{}, handleKey func(key string, value in if !rules4.defaultSet { if rules4.allValuesMatch { - _, zeroCIDR, _ := net.ParseCIDR("0.0.0.0/0") - tree.AddCIDR(zeroCIDR, !rules4.allValues) + tree.Insert(netip.PrefixFrom(netip.IPv4Unspecified(), 0), !rules4.allValues) } else { return nil, fmt.Errorf("config `%s` contains both true and false rules, but no default set for 0.0.0.0/0", k) } @@ -165,8 +159,7 @@ func newAllowList(k string, raw interface{}, handleKey func(key string, value in if !rules6.defaultSet { if rules6.allValuesMatch { - _, zeroCIDR, _ := net.ParseCIDR("::/0") - tree.AddCIDR(zeroCIDR, !rules6.allValues) + tree.Insert(netip.PrefixFrom(netip.IPv6Unspecified(), 0), !rules6.allValues) } else { return nil, fmt.Errorf("config `%s` contains both true and false rules, but no default set for ::/0", k) } @@ -175,22 +168,18 @@ func newAllowList(k string, raw interface{}, handleKey func(key string, value in return &AllowList{cidrTree: tree}, nil } -func getAllowListInterfaces(k string, v interface{}) ([]AllowListNameRule, error) { +func getAllowListInterfaces(k string, v any) ([]AllowListNameRule, error) { var nameRules []AllowListNameRule - rawRules, ok := v.(map[interface{}]interface{}) + rawRules, ok := v.(map[string]any) if !ok { return nil, fmt.Errorf("config `%s.interfaces` is invalid (type %T): %v", k, v, v) } firstEntry := true var allValues bool - for rawName, rawAllow := range rawRules { - name, ok := rawName.(string) - if !ok { - return nil, fmt.Errorf("config `%s.interfaces` has invalid key (type %T): %v", k, rawName, rawName) - } - allow, ok := rawAllow.(bool) + for name, rawAllow := range rawRules { + allow, ok := config.AsBool(rawAllow) if !ok { return nil, fmt.Errorf("config `%s.interfaces` has invalid value (type %T): %v", k, rawAllow, rawAllow) } @@ -218,72 +207,49 @@ func getAllowListInterfaces(k string, v interface{}) ([]AllowListNameRule, error return nameRules, nil } -func getRemoteAllowRanges(c *config.C, k string) (*cidr.Tree6[*AllowList], error) { +func getRemoteAllowRanges(c *config.C, k string) (*bart.Table[*AllowList], error) { value := c.Get(k) if value == nil { return nil, nil } - remoteAllowRanges := cidr.NewTree6[*AllowList]() + remoteAllowRanges := new(bart.Table[*AllowList]) - rawMap, ok := value.(map[interface{}]interface{}) + rawMap, ok := value.(map[string]any) if !ok { return nil, fmt.Errorf("config `%s` has invalid type: %T", k, value) } - for rawKey, rawValue := range rawMap { - rawCIDR, ok := rawKey.(string) - if !ok { - return nil, fmt.Errorf("config `%s` has invalid key (type %T): %v", k, rawKey, rawKey) - } - + for rawCIDR, rawValue := range rawMap { allowList, err := newAllowList(fmt.Sprintf("%s.%s", k, rawCIDR), rawValue, nil) if err != nil { return nil, err } - _, ipNet, err := net.ParseCIDR(rawCIDR) + ipNet, err := netip.ParsePrefix(rawCIDR) if err != nil { - return nil, fmt.Errorf("config `%s` has invalid CIDR: %s", k, rawCIDR) + return nil, fmt.Errorf("config `%s` has invalid CIDR: %s. %w", k, rawCIDR, err) } - remoteAllowRanges.AddCIDR(ipNet, allowList) + remoteAllowRanges.Insert(netip.PrefixFrom(ipNet.Addr().Unmap(), ipNet.Bits()), allowList) } return remoteAllowRanges, nil } -func (al *AllowList) Allow(ip net.IP) bool { +func (al *AllowList) Allow(addr netip.Addr) bool { if al == nil { return true } - _, result := al.cidrTree.MostSpecificContains(ip) + result, _ := al.cidrTree.Lookup(addr) return result } -func (al *AllowList) AllowIpV4(ip iputil.VpnIp) bool { +func (al *LocalAllowList) Allow(udpAddr netip.Addr) bool { if al == nil { return true } - - _, result := al.cidrTree.MostSpecificContainsIpV4(ip) - return result -} - -func (al *AllowList) AllowIpV6(hi, lo uint64) bool { - if al == nil { - return true - } - - _, result := al.cidrTree.MostSpecificContainsIpV6(hi, lo) - return result -} - -func (al *LocalAllowList) Allow(ip net.IP) bool { - if al == nil { - return true - } - return al.AllowList.Allow(ip) + return al.AllowList.Allow(udpAddr) } func (al *LocalAllowList) AllowName(name string) bool { @@ -301,43 +267,37 @@ func (al *LocalAllowList) AllowName(name string) bool { return !al.nameRules[0].Allow } -func (al *RemoteAllowList) AllowUnknownVpnIp(ip net.IP) bool { +func (al *RemoteAllowList) AllowUnknownVpnAddr(vpnAddr netip.Addr) bool { if al == nil { return true } - return al.AllowList.Allow(ip) + return al.AllowList.Allow(vpnAddr) } -func (al *RemoteAllowList) Allow(vpnIp iputil.VpnIp, ip net.IP) bool { - if !al.getInsideAllowList(vpnIp).Allow(ip) { +func (al *RemoteAllowList) Allow(vpnAddr netip.Addr, udpAddr netip.Addr) bool { + if !al.getInsideAllowList(vpnAddr).Allow(udpAddr) { return false } - return al.AllowList.Allow(ip) + return al.AllowList.Allow(udpAddr) } -func (al *RemoteAllowList) AllowIpV4(vpnIp iputil.VpnIp, ip iputil.VpnIp) bool { - if al == nil { - return true - } - if !al.getInsideAllowList(vpnIp).AllowIpV4(ip) { +func (al *RemoteAllowList) AllowAll(vpnAddrs []netip.Addr, udpAddr netip.Addr) bool { + if !al.AllowList.Allow(udpAddr) { return false } - return al.AllowList.AllowIpV4(ip) -} -func (al *RemoteAllowList) AllowIpV6(vpnIp iputil.VpnIp, hi, lo uint64) bool { - if al == nil { - return true - } - if !al.getInsideAllowList(vpnIp).AllowIpV6(hi, lo) { - return false + for _, vpnAddr := range vpnAddrs { + if !al.getInsideAllowList(vpnAddr).Allow(udpAddr) { + return false + } } - return al.AllowList.AllowIpV6(hi, lo) + + return true } -func (al *RemoteAllowList) getInsideAllowList(vpnIp iputil.VpnIp) *AllowList { +func (al *RemoteAllowList) getInsideAllowList(vpnAddr netip.Addr) *AllowList { if al.insideAllowLists != nil { - ok, inside := al.insideAllowLists.MostSpecificContainsIpV4(vpnIp) + inside, ok := al.insideAllowLists.Lookup(vpnAddr) if ok { return inside } diff --git a/allow_list_test.go b/allow_list_test.go index 334cb606..6135f36c 100644 --- a/allow_list_test.go +++ b/allow_list_test.go @@ -1,40 +1,41 @@ package nebula import ( - "net" + "net/netip" "regexp" "testing" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNewAllowListFromConfig(t *testing.T) { l := test.NewLogger() c := config.NewC(l) - c.Settings["allowlist"] = map[interface{}]interface{}{ + c.Settings["allowlist"] = map[string]any{ "192.168.0.0": true, } r, err := newAllowListFromConfig(c, "allowlist", nil) - assert.EqualError(t, err, "config `allowlist` has invalid CIDR: 192.168.0.0") + require.EqualError(t, err, "config `allowlist` has invalid CIDR: 192.168.0.0. netip.ParsePrefix(\"192.168.0.0\"): no '/'") assert.Nil(t, r) - c.Settings["allowlist"] = map[interface{}]interface{}{ + c.Settings["allowlist"] = map[string]any{ "192.168.0.0/16": "abc", } r, err = newAllowListFromConfig(c, "allowlist", nil) - assert.EqualError(t, err, "config `allowlist` has invalid value (type string): abc") + require.EqualError(t, err, "config `allowlist` has invalid value (type string): abc") - c.Settings["allowlist"] = map[interface{}]interface{}{ + c.Settings["allowlist"] = map[string]any{ "192.168.0.0/16": true, "10.0.0.0/8": false, } r, err = newAllowListFromConfig(c, "allowlist", nil) - assert.EqualError(t, err, "config `allowlist` contains both true and false rules, but no default set for 0.0.0.0/0") + require.EqualError(t, err, "config `allowlist` contains both true and false rules, but no default set for 0.0.0.0/0") - c.Settings["allowlist"] = map[interface{}]interface{}{ + c.Settings["allowlist"] = map[string]any{ "0.0.0.0/0": true, "10.0.0.0/8": false, "10.42.42.0/24": true, @@ -42,9 +43,9 @@ func TestNewAllowListFromConfig(t *testing.T) { "fd00:fd00::/16": false, } r, err = newAllowListFromConfig(c, "allowlist", nil) - assert.EqualError(t, err, "config `allowlist` contains both true and false rules, but no default set for ::/0") + require.EqualError(t, err, "config `allowlist` contains both true and false rules, but no default set for ::/0") - c.Settings["allowlist"] = map[interface{}]interface{}{ + c.Settings["allowlist"] = map[string]any{ "0.0.0.0/0": true, "10.0.0.0/8": false, "10.42.42.0/24": true, @@ -54,7 +55,7 @@ func TestNewAllowListFromConfig(t *testing.T) { assert.NotNil(t, r) } - c.Settings["allowlist"] = map[interface{}]interface{}{ + c.Settings["allowlist"] = map[string]any{ "0.0.0.0/0": true, "10.0.0.0/8": false, "10.42.42.0/24": true, @@ -69,25 +70,25 @@ func TestNewAllowListFromConfig(t *testing.T) { // Test interface names - c.Settings["allowlist"] = map[interface{}]interface{}{ - "interfaces": map[interface{}]interface{}{ + c.Settings["allowlist"] = map[string]any{ + "interfaces": map[string]any{ `docker.*`: "foo", }, } lr, err := NewLocalAllowListFromConfig(c, "allowlist") - assert.EqualError(t, err, "config `allowlist.interfaces` has invalid value (type string): foo") + require.EqualError(t, err, "config `allowlist.interfaces` has invalid value (type string): foo") - c.Settings["allowlist"] = map[interface{}]interface{}{ - "interfaces": map[interface{}]interface{}{ + c.Settings["allowlist"] = map[string]any{ + "interfaces": map[string]any{ `docker.*`: false, `eth.*`: true, }, } lr, err = NewLocalAllowListFromConfig(c, "allowlist") - assert.EqualError(t, err, "config `allowlist.interfaces` values must all be the same true/false value") + require.EqualError(t, err, "config `allowlist.interfaces` values must all be the same true/false value") - c.Settings["allowlist"] = map[interface{}]interface{}{ - "interfaces": map[interface{}]interface{}{ + c.Settings["allowlist"] = map[string]any{ + "interfaces": map[string]any{ `docker.*`: false, }, } @@ -98,30 +99,30 @@ func TestNewAllowListFromConfig(t *testing.T) { } func TestAllowList_Allow(t *testing.T) { - assert.Equal(t, true, ((*AllowList)(nil)).Allow(net.ParseIP("1.1.1.1"))) - - tree := cidr.NewTree6[bool]() - tree.AddCIDR(cidr.Parse("0.0.0.0/0"), true) - tree.AddCIDR(cidr.Parse("10.0.0.0/8"), false) - tree.AddCIDR(cidr.Parse("10.42.42.42/32"), true) - tree.AddCIDR(cidr.Parse("10.42.0.0/16"), true) - tree.AddCIDR(cidr.Parse("10.42.42.0/24"), true) - tree.AddCIDR(cidr.Parse("10.42.42.0/24"), false) - tree.AddCIDR(cidr.Parse("::1/128"), true) - tree.AddCIDR(cidr.Parse("::2/128"), false) + assert.True(t, ((*AllowList)(nil)).Allow(netip.MustParseAddr("1.1.1.1"))) + + tree := new(bart.Table[bool]) + tree.Insert(netip.MustParsePrefix("0.0.0.0/0"), true) + tree.Insert(netip.MustParsePrefix("10.0.0.0/8"), false) + tree.Insert(netip.MustParsePrefix("10.42.42.42/32"), true) + tree.Insert(netip.MustParsePrefix("10.42.0.0/16"), true) + tree.Insert(netip.MustParsePrefix("10.42.42.0/24"), true) + tree.Insert(netip.MustParsePrefix("10.42.42.0/24"), false) + tree.Insert(netip.MustParsePrefix("::1/128"), true) + tree.Insert(netip.MustParsePrefix("::2/128"), false) al := &AllowList{cidrTree: tree} - assert.Equal(t, true, al.Allow(net.ParseIP("1.1.1.1"))) - assert.Equal(t, false, al.Allow(net.ParseIP("10.0.0.4"))) - assert.Equal(t, true, al.Allow(net.ParseIP("10.42.42.42"))) - assert.Equal(t, false, al.Allow(net.ParseIP("10.42.42.41"))) - assert.Equal(t, true, al.Allow(net.ParseIP("10.42.0.1"))) - assert.Equal(t, true, al.Allow(net.ParseIP("::1"))) - assert.Equal(t, false, al.Allow(net.ParseIP("::2"))) + assert.True(t, al.Allow(netip.MustParseAddr("1.1.1.1"))) + assert.False(t, al.Allow(netip.MustParseAddr("10.0.0.4"))) + assert.True(t, al.Allow(netip.MustParseAddr("10.42.42.42"))) + assert.False(t, al.Allow(netip.MustParseAddr("10.42.42.41"))) + assert.True(t, al.Allow(netip.MustParseAddr("10.42.0.1"))) + assert.True(t, al.Allow(netip.MustParseAddr("::1"))) + assert.False(t, al.Allow(netip.MustParseAddr("::2"))) } func TestLocalAllowList_AllowName(t *testing.T) { - assert.Equal(t, true, ((*LocalAllowList)(nil)).AllowName("docker0")) + assert.True(t, ((*LocalAllowList)(nil)).AllowName("docker0")) rules := []AllowListNameRule{ {Name: regexp.MustCompile("^docker.*$"), Allow: false}, @@ -129,9 +130,9 @@ func TestLocalAllowList_AllowName(t *testing.T) { } al := &LocalAllowList{nameRules: rules} - assert.Equal(t, false, al.AllowName("docker0")) - assert.Equal(t, false, al.AllowName("tun0")) - assert.Equal(t, true, al.AllowName("eth0")) + assert.False(t, al.AllowName("docker0")) + assert.False(t, al.AllowName("tun0")) + assert.True(t, al.AllowName("eth0")) rules = []AllowListNameRule{ {Name: regexp.MustCompile("^eth.*$"), Allow: true}, @@ -139,7 +140,7 @@ func TestLocalAllowList_AllowName(t *testing.T) { } al = &LocalAllowList{nameRules: rules} - assert.Equal(t, false, al.AllowName("docker0")) - assert.Equal(t, true, al.AllowName("eth0")) - assert.Equal(t, true, al.AllowName("ens5")) + assert.False(t, al.AllowName("docker0")) + assert.True(t, al.AllowName("eth0")) + assert.True(t, al.AllowName("ens5")) } diff --git a/bits.go b/bits.go index b4f96c6c..15bafd87 100644 --- a/bits.go +++ b/bits.go @@ -1,157 +1,263 @@ package nebula import ( + "context" + "fmt" + "log/slog" + "math" + mathbits "math/bits" + "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" ) +const bitsPerWord = 64 + +// Bits is a sliding-window anti-replay tracker. The window is stored as a +// circular bitmap packed into uint64 words (8x denser than a []bool), so a +// length-N window costs N/8 bytes. length must be a power of two. type Bits struct { length uint64 + lengthMask uint64 current uint64 - bits []bool - firstSeen bool + bits []uint64 lostCounter metrics.Counter dupeCounter metrics.Counter outOfWindowCounter metrics.Counter } -func NewBits(bits uint64) *Bits { - return &Bits{ - length: bits, - bits: make([]bool, bits, bits), +func NewBits(length uint64) *Bits { + if length == 0 || length&(length-1) != 0 { + panic(fmt.Sprintf("Bits length must be a power of two, got %d", length)) + } + + nWords := length / bitsPerWord + if nWords == 0 { + nWords = 1 + } + b := &Bits{ + length: length, + lengthMask: length - 1, + bits: make([]uint64, nWords), current: 0, lostCounter: metrics.GetOrRegisterCounter("network.packets.lost", nil), dupeCounter: metrics.GetOrRegisterCounter("network.packets.duplicate", nil), outOfWindowCounter: metrics.GetOrRegisterCounter("network.packets.out_of_window", nil), } + + // There is no counter value 0, mark it to avoid counting a lost packet later. + b.bits[0] = 1 + return b } -func (b *Bits) Check(l logrus.FieldLogger, i uint64) bool { - // If i is the next number, return true. - if i > b.current || (i == 0 && b.firstSeen == false && b.current < b.length) { +func (b *Bits) get(i uint64) bool { + pos := i & b.lengthMask + //bit-shifting by 6 because i is a bit index, not a u64 index, and we need to find the u64 without bit in it + return b.bits[pos>>6]&(uint64(1)<<(pos&63)) != 0 +} + +func (b *Bits) set(i uint64) { + pos := i & b.lengthMask + b.bits[pos>>6] |= uint64(1) << (pos & 63) +} + +// clearRange clears `count` bits starting at circular position `startPos` +// (already masked to [0, length)) and returns how many of them were set +// before the clear. count must be in [1, length]. +func (b *Bits) clearRange(startPos, count uint64) uint64 { + wasSet := uint64(0) + if count >= b.length { + for _, w := range b.bits { + wasSet += uint64(mathbits.OnesCount64(w)) + } + clear(b.bits) + return wasSet + } + + pos := startPos + remaining := count + + // handle the potential partial word before pos becomes u64 aligned + word := pos >> 6 + bit := pos & 63 + take := uint64(64) - bit + if take > remaining { + take = remaining + } + if take > b.length-pos { + take = b.length - pos + } + var mask uint64 + if take == 64 { + mask = math.MaxUint64 + } else { + mask = ((uint64(1) << take) - 1) << bit + } + wasSet += uint64(mathbits.OnesCount64(b.bits[word] & mask)) + b.bits[word] &^= mask + remaining -= take + pos = (pos + take) & b.lengthMask + + // Clear whole words, keeping track of the number of set bits + for remaining >= 64 { + word = pos >> 6 + wasSet += uint64(mathbits.OnesCount64(b.bits[word])) + b.bits[word] = 0 + remaining -= 64 + pos = (pos + 64) & b.lengthMask + } + + // Clear the remaining partial word + if remaining > 0 { + word = pos >> 6 + mask = (uint64(1) << remaining) - 1 + wasSet += uint64(mathbits.OnesCount64(b.bits[word] & mask)) + b.bits[word] &^= mask + } + + return wasSet +} + +func (b *Bits) strictlyWithinWindow(i uint64) bool { + // Handle the case where the window hasn't slid yet. This avoids u64 underflow. + inWarmup := b.current < b.length + if i < b.length && inWarmup { return true } - // If i is within the window, check if it's been set already. The first window will fail this check + // Next, if the packet is in-window, see if we've seen it before if i > b.current-b.length { - return !b.bits[i%b.length] + return true } + return false //not within window! +} - // If i is within the first window - if i < b.length { - return !b.bits[i%b.length] +// Check returns true if i is within (or way out in front of) the window, and not a replay +func (b *Bits) Check(l *slog.Logger, i uint64) bool { + // If i is the next number, return true. + if i > b.current { + return true + } + + if b.strictlyWithinWindow(i) { + return !b.get(i) } // Not within the window - l.Debugf("rejected a packet (top) %d %d\n", b.current, i) + if l.Enabled(context.Background(), slog.LevelDebug) { + l.Debug("rejected a packet (top)", "current", b.current, "incoming", i) + } return false } -func (b *Bits) Update(l *logrus.Logger, i uint64) bool { - // If i is the next number, return true and update current. +// Update has three branches: +// - i == b.current+1: fast path; advance the cursor by one and lose-count +// the slot we just stomped (only past warmup; see the i > b.length guard +// below). +// - i > b.current+1: jump path; clear all slots between current and i +// (or up to a full window's worth, whichever is smaller) via clearRange, +// then mark i. Two arms here: a warmup arm that handles the very first +// window before the cursor has slid, and a steady-state arm that treats +// every cleared empty slot as a lost packet. +// - i <= b.current: in-window check for duplicates; out-of-window otherwise. +// +// NewBits seeds bits[0]=1 so counter 0 looks "received" — Update never +// clears that marker during warmup (clearRange skips position 0 when +// startPos=1), and once b.current >= b.length the marker is no longer +// consulted. The marker prevents a fictitious "lost" hit on the first real +// counter. +func (b *Bits) Update(l *slog.Logger, i uint64) bool { + // Fast path: i is the next expected counter. Split out so the function + // stays small and avoids paying for the slow paths' slog argument-build + // stack frame on every call. The bit read/test/write is inlined to + // touch the backing word once. if i == b.current+1 { - // Report missed packets, we can only understand what was missed after the first window has been gone through - if i > b.length && b.bits[i%b.length] == false { + pos := i & b.lengthMask + word := pos >> 6 + mask := uint64(1) << (pos & 63) + w := b.bits[word] + if i > b.length && w&mask == 0 { b.lostCounter.Inc(1) } - b.bits[i%b.length] = true + b.bits[word] = w | mask b.current = i return true } + return b.updateSlow(l, i) +} - // If i packet is greater than current but less than the maximum length of our bitmap, - // flip everything in between to false and move ahead. - if i > b.current && i < b.current+b.length { - // In between current and i need to be zero'd to allow those packets to come in later - for n := b.current + 1; n < i; n++ { - b.bits[n%b.length] = false +// updateSlow handles jumps, in-window backfill, dupes, and out-of-window. +func (b *Bits) updateSlow(l *slog.Logger, i uint64) bool { + // If i is a jump, adjust the window, record lost, update current, and return true + if i > b.current { + end := i + if end > b.current+b.length { + end = b.current + b.length } + count := end - b.current + startPos := (b.current + 1) & b.lengthMask - b.bits[i%b.length] = true - b.current = i - //l.Debugf("missed %d packets between %d and %d\n", i-b.current, i, b.current) - return true - } - - // If i is greater than the delta between current and the total length of our bitmap, - // just flip everything in the map and move ahead. - if i >= b.current+b.length { - // The current window loss will be accounted for later, only record the jump as loss up until then - lost := maxInt64(0, int64(i-b.current-b.length)) - //TODO: explain this - if b.current == 0 { - lost++ + var lost int64 + if b.current >= b.length { + // Steady state: every cleared slot is past warmup, so any unset + // bit we evict is a lost packet from the previous cycle. + wasSet := b.clearRange(startPos, count) + lost = int64(count) - int64(wasSet) + } else { + // Warmup (the very first window). Some cleared slots represent + // packets <= length where eviction is not "lost" in the usual + // sense. This branch is taken at most once per connection so we + // don't bother optimizing it. + for n := b.current + 1; n <= end; n++ { + if !b.get(n) && n > b.length { + lost++ + } + } + b.clearRange(startPos, count) } - for n := range b.bits { - // Don't want to count the first window as a loss - //TODO: this is likely wrong, we are wanting to track only the bit slots that we aren't going to track anymore and this is marking everything as missed - //if b.bits[n] == false { - // lost++ - //} - b.bits[n] = false + // Anything past the new window can never be backfilled, so it's lost. + if i > b.current+b.length { + lost += int64(i - b.current - b.length) } - b.lostCounter.Inc(lost) - if l.Level >= logrus.DebugLevel { - l.WithField("receiveWindow", m{"accepted": true, "currentCounter": b.current, "incomingCounter": i, "reason": "window shifting"}). - Debug("Receive window") - } - b.bits[i%b.length] = true + b.set(i) b.current = i return true } - // Allow for the 0 packet to come in within the first window - if i == 0 && b.firstSeen == false && b.current < b.length { - b.firstSeen = true - b.bits[i%b.length] = true - return true - } - - // If i is within the window of current minus length (the total pat window size), - // allow it and flip to true but to NOT change current. We also have to account for the first window - if ((b.current >= b.length && i > b.current-b.length) || (b.current < b.length && i < b.length)) && i <= b.current { - if b.current == i { - if l.Level >= logrus.DebugLevel { - l.WithField("receiveWindow", m{"accepted": false, "currentCounter": b.current, "incomingCounter": i, "reason": "duplicate"}). - Debug("Receive window") - } - b.dupeCounter.Inc(1) - return false - } - - if b.bits[i%b.length] == true { - if l.Level >= logrus.DebugLevel { - l.WithField("receiveWindow", m{"accepted": false, "currentCounter": b.current, "incomingCounter": i, "reason": "old duplicate"}). - Debug("Receive window") + // If i is within the current window but below the current counter, check to see if it's a duplicate + if b.strictlyWithinWindow(i) { + pos := i & b.lengthMask + word := pos >> 6 + mask := uint64(1) << (pos & 63) + w := b.bits[word] + if b.current == i || w&mask != 0 { + if l.Enabled(context.Background(), slog.LevelDebug) { + l.Debug("Receive window", + "accepted", false, + "currentCounter", b.current, + "incomingCounter", i, + "reason", "duplicate", + ) } b.dupeCounter.Inc(1) return false } - b.bits[i%b.length] = true + b.bits[word] = w | mask return true - } // In all other cases, fail and don't change current. b.outOfWindowCounter.Inc(1) - if l.Level >= logrus.DebugLevel { - l.WithField("accepted", false). - WithField("currentCounter", b.current). - WithField("incomingCounter", i). - WithField("reason", "nonsense"). - Debug("Receive window") + if l.Enabled(context.Background(), slog.LevelDebug) { + l.Debug("Receive window", + "accepted", false, + "currentCounter", b.current, + "incomingCounter", i, + "reason", "nonsense", + ) } return false } - -func maxInt64(a, b int64) int64 { - if a > b { - return a - } - - return b -} diff --git a/bits_test.go b/bits_test.go index 95abe018..da44c92a 100644 --- a/bits_test.go +++ b/bits_test.go @@ -7,77 +7,114 @@ import ( "github.com/stretchr/testify/assert" ) +// snapshot returns the bitmap as a []bool of length b.length, for readable +// test assertions against the now-packed []uint64 storage. +func (b *Bits) snapshot() []bool { + out := make([]bool, b.length) + for i := uint64(0); i < b.length; i++ { + out[i] = b.get(i) + } + return out +} + +func TestBitsRequiresPowerOfTwo(t *testing.T) { + assert.Panics(t, func() { NewBits(10) }) + assert.Panics(t, func() { NewBits(0) }) + assert.NotPanics(t, func() { NewBits(1) }) + assert.NotPanics(t, func() { NewBits(16) }) + assert.NotPanics(t, func() { NewBits(1024) }) + assert.NotPanics(t, func() { NewBits(16384) }) +} + func TestBits(t *testing.T) { l := test.NewLogger() - b := NewBits(10) - - // make sure it is the right size - assert.Len(t, b.bits, 10) + b := NewBits(16) + assert.EqualValues(t, 16, b.length) // This is initialized to zero - receive one. This should work. - assert.True(t, b.Check(l, 1)) - u := b.Update(l, 1) - assert.True(t, u) + assert.True(t, b.Update(l, 1)) assert.EqualValues(t, 1, b.current) - g := []bool{false, true, false, false, false, false, false, false, false, false} - assert.Equal(t, g, b.bits) + g := []bool{true, true, false, false, false, false, false, false, false, false, false, false, false, false, false, false} + assert.Equal(t, g, b.snapshot()) // Receive two assert.True(t, b.Check(l, 2)) - u = b.Update(l, 2) - assert.True(t, u) + assert.True(t, b.Update(l, 2)) assert.EqualValues(t, 2, b.current) - g = []bool{false, true, true, false, false, false, false, false, false, false} - assert.Equal(t, g, b.bits) + g = []bool{true, true, true, false, false, false, false, false, false, false, false, false, false, false, false, false} + assert.Equal(t, g, b.snapshot()) // Receive two again - it will fail assert.False(t, b.Check(l, 2)) - u = b.Update(l, 2) - assert.False(t, u) + assert.False(t, b.Update(l, 2)) assert.EqualValues(t, 2, b.current) - // Jump ahead to 15, which should clear everything and set the 6th element - assert.True(t, b.Check(l, 15)) - u = b.Update(l, 15) - assert.True(t, u) - assert.EqualValues(t, 15, b.current) - g = []bool{false, false, false, false, false, true, false, false, false, false} - assert.Equal(t, g, b.bits) - - // Mark 14, which is allowed because it is in the window - assert.True(t, b.Check(l, 14)) - u = b.Update(l, 14) - assert.True(t, u) - assert.EqualValues(t, 15, b.current) - g = []bool{false, false, false, false, true, true, false, false, false, false} - assert.Equal(t, g, b.bits) - - // Mark 5, which is not allowed because it is not in the window + // Jump ahead to 25, which clears the window and sets slot 25%16 = 9. + assert.True(t, b.Check(l, 25)) + assert.True(t, b.Update(l, 25)) + assert.EqualValues(t, 25, b.current) + g = []bool{false, false, false, false, false, false, false, false, false, true, false, false, false, false, false, false} + assert.Equal(t, g, b.snapshot()) + + // Mark 24, which is in window (current 25, length 16, window covers [10,25]). + assert.True(t, b.Check(l, 24)) + assert.True(t, b.Update(l, 24)) + assert.EqualValues(t, 25, b.current) + g = []bool{false, false, false, false, false, false, false, false, true, true, false, false, false, false, false, false} + assert.Equal(t, g, b.snapshot()) + + // Mark 5, not allowed because 5 <= current-length (25-16=9). assert.False(t, b.Check(l, 5)) - u = b.Update(l, 5) - assert.False(t, u) - assert.EqualValues(t, 15, b.current) - g = []bool{false, false, false, false, true, true, false, false, false, false} - assert.Equal(t, g, b.bits) - - // make sure we handle wrapping around once to the current position - b = NewBits(10) + assert.False(t, b.Update(l, 5)) + assert.EqualValues(t, 25, b.current) + g = []bool{false, false, false, false, false, false, false, false, true, true, false, false, false, false, false, false} + assert.Equal(t, g, b.snapshot()) + + // Make sure we handle wrapping around once to the same slot. With + // length=16, packets 1 and 17 share slot 1. + b = NewBits(16) assert.True(t, b.Update(l, 1)) - assert.True(t, b.Update(l, 11)) - assert.Equal(t, []bool{false, true, false, false, false, false, false, false, false, false}, b.bits) + assert.True(t, b.Update(l, 17)) + assert.Equal(t, []bool{false, true, false, false, false, false, false, false, false, false, false, false, false, false, false, false}, b.snapshot()) // Walk through a few windows in order - b = NewBits(10) - for i := uint64(0); i <= 100; i++ { + b = NewBits(16) + for i := uint64(1); i <= 100; i++ { assert.True(t, b.Check(l, i), "Error while checking %v", i) assert.True(t, b.Update(l, i), "Error while updating %v", i) } + + assert.False(t, b.Check(l, 1), "Out of window check") +} + +func TestBitsLargeJumps(t *testing.T) { + l := test.NewLogger() + + // length=16. Update(55) from current=0: + // warmup, per-bit loop sees no n>16 with unset bits (slot 0 was set by + // NewBits and gets re-evaluated when n=16; n=16 is not strictly > 16), + // so the loop contributes 0. The jump exceeds the window so we record + // 55 - 0 - 16 = 39 packets fell out the back. + b := NewBits(16) + b.lostCounter.Clear() + assert.True(t, b.Update(l, 55)) + assert.Equal(t, int64(39), b.lostCounter.Count()) + + // Update(100): clears 16 slots starting at slot 56%16=8. Only slot 7 (for + // packet 55) was set, so 16 - 1 = 15 evicted slots had unset bits. + // Plus 100 - 55 - 16 = 29 packets fell past the window. Total 44. + assert.True(t, b.Update(l, 100)) + assert.Equal(t, int64(39+44), b.lostCounter.Count()) + + // Update(200): same shape: 16 - 1 = 15 evicted unset, plus 200 - 100 - 16 = 84 past window. Total 99. + assert.True(t, b.Update(l, 200)) + assert.Equal(t, int64(39+44+99), b.lostCounter.Count()) } func TestBitsDupeCounter(t *testing.T) { l := test.NewLogger() - b := NewBits(10) + b := NewBits(16) b.lostCounter.Clear() b.dupeCounter.Clear() b.outOfWindowCounter.Clear() @@ -102,127 +139,300 @@ func TestBitsDupeCounter(t *testing.T) { func TestBitsOutOfWindowCounter(t *testing.T) { l := test.NewLogger() - b := NewBits(10) + b := NewBits(16) b.lostCounter.Clear() b.dupeCounter.Clear() b.outOfWindowCounter.Clear() + // Jump to 20 (warmup branch + 4 past-window packets). assert.True(t, b.Update(l, 20)) assert.Equal(t, int64(0), b.outOfWindowCounter.Count()) - assert.True(t, b.Update(l, 21)) - assert.True(t, b.Update(l, 22)) - assert.True(t, b.Update(l, 23)) - assert.True(t, b.Update(l, 24)) - assert.True(t, b.Update(l, 25)) - assert.True(t, b.Update(l, 26)) - assert.True(t, b.Update(l, 27)) - assert.True(t, b.Update(l, 28)) - assert.True(t, b.Update(l, 29)) + // 9 single-step advances, each evicts a slot whose bit was cleared during + // the jump above and whose value was never seen, so each contributes 1 + // to lostCounter. + for n := uint64(21); n <= 29; n++ { + assert.True(t, b.Update(l, n)) + } assert.Equal(t, int64(0), b.outOfWindowCounter.Count()) + // 0 is below current-length (29-16=13) so it falls outside the window. assert.False(t, b.Update(l, 0)) assert.Equal(t, int64(1), b.outOfWindowCounter.Count()) - //tODO: make sure lostcounter doesn't increase in orderly increment - assert.Equal(t, int64(20), b.lostCounter.Count()) + // 4 from the Update(20) jump + 9 from 21..29. + assert.Equal(t, int64(13), b.lostCounter.Count()) assert.Equal(t, int64(0), b.dupeCounter.Count()) assert.Equal(t, int64(1), b.outOfWindowCounter.Count()) } func TestBitsLostCounter(t *testing.T) { l := test.NewLogger() - b := NewBits(10) + b := NewBits(16) b.lostCounter.Clear() b.dupeCounter.Clear() b.outOfWindowCounter.Clear() - //assert.True(t, b.Update(0)) - assert.True(t, b.Update(l, 0)) - assert.True(t, b.Update(l, 20)) - assert.True(t, b.Update(l, 21)) - assert.True(t, b.Update(l, 22)) - assert.True(t, b.Update(l, 23)) - assert.True(t, b.Update(l, 24)) - assert.True(t, b.Update(l, 25)) - assert.True(t, b.Update(l, 26)) - assert.True(t, b.Update(l, 27)) - assert.True(t, b.Update(l, 28)) - assert.True(t, b.Update(l, 29)) - assert.Equal(t, int64(20), b.lostCounter.Count()) + // Walk 20..29 like the original, just with a bigger window. Same + // reasoning as TestBitsOutOfWindowCounter: 4 past-window from Update(20), + // then 9 more from the unit advances. + for n := uint64(20); n <= 29; n++ { + assert.True(t, b.Update(l, n)) + } + assert.Equal(t, int64(13), b.lostCounter.Count()) assert.Equal(t, int64(0), b.dupeCounter.Count()) assert.Equal(t, int64(0), b.outOfWindowCounter.Count()) - b = NewBits(10) + b = NewBits(16) b.lostCounter.Clear() b.dupeCounter.Clear() b.outOfWindowCounter.Clear() - assert.True(t, b.Update(l, 0)) + // Update(15) clears the warmup window (no lost), sets slot 15. + assert.True(t, b.Update(l, 15)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + + // Update(16): slot 0 was already set (NewBits seeded it), and 16 is not + // strictly > length, so nothing is recorded as lost. + assert.True(t, b.Update(l, 16)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + + // Update(17): we jumped straight from 0 to 15, so slot 1 was cleared + // (and never re-set). 17 > 16 is past warmup, so packet 1 is recorded lost. + assert.True(t, b.Update(l, 17)) + assert.Equal(t, int64(1), b.lostCounter.Count()) + + // Fill in 18..30 in single steps. Each i evicts slot i%16. Slots 2..14 + // were all cleared during Update(15), and we never re-set any of them, + // so each i in 18..30 is a fresh lost packet — 13 more. + for n := uint64(18); n <= 30; n++ { + assert.True(t, b.Update(l, n)) + } + assert.Equal(t, int64(14), b.lostCounter.Count()) + + // Jump ahead by exactly one window size. + assert.True(t, b.Update(l, 46)) + // end = min(46, 30+16) = 46, count = 16, all slots cleared. Before the + // jump every slot 0..15 had been set (Update(15), (16), (17), 18..30), + // so wasSet=16 and 46 == current+length means no past-window slack: + // lost contribution = 0. + assert.Equal(t, int64(14), b.lostCounter.Count()) + + // Walk 47..55. The Update(46) jump cleared every slot, so only slot 14 + // (for packet 46) is set when we start. Each subsequent unit step lands + // on a slot that was cleared and is past warmup, so it counts as lost. + // 9 more = 23. + for n := uint64(47); n <= 55; n++ { + assert.True(t, b.Update(l, n)) + } + assert.Equal(t, int64(23), b.lostCounter.Count()) + + // Jump ahead by two windows: clears the window plus past-window loss. + assert.True(t, b.Update(l, 87)) + // current=55, length=16. end = min(87, 71) = 71. count=16, all slots + // cleared. Slots set before the clear are slots 14,15,0..7 (10 total). + // Lost from clear = 16 - 10 = 6. Past window: 87 - 55 - 16 = 16. +22. + assert.Equal(t, int64(45), b.lostCounter.Count()) + assert.Equal(t, int64(0), b.dupeCounter.Count()) + assert.Equal(t, int64(0), b.outOfWindowCounter.Count()) +} + +func TestBitsLostCounterIssue1(t *testing.T) { + l := test.NewLogger() + b := NewBits(16) + b.lostCounter.Clear() + b.dupeCounter.Clear() + b.outOfWindowCounter.Clear() + + // Receive 4, backfill 1, then 9, 2, 3, 5, 6, 7 (skip 8), 10, 11, 14. + // Then jump to 25 — slot 25%16=9 is being evicted, but it had been set + // (we received packet 9), so no spurious lost increment. The original + // regression was about double-counting a missing packet when its slot + // got cleared on a jump. With the jump path now using clearRange's + // word-level wasSet count, the same semantics hold. + assert.True(t, b.Update(l, 4)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + assert.True(t, b.Update(l, 1)) assert.Equal(t, int64(0), b.lostCounter.Count()) assert.True(t, b.Update(l, 9)) assert.Equal(t, int64(0), b.lostCounter.Count()) - // 10 will set 0 index, 0 was already set, no lost packets + assert.True(t, b.Update(l, 2)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + assert.True(t, b.Update(l, 3)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + assert.True(t, b.Update(l, 5)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + assert.True(t, b.Update(l, 6)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + assert.True(t, b.Update(l, 7)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + // Skip packet 8. assert.True(t, b.Update(l, 10)) assert.Equal(t, int64(0), b.lostCounter.Count()) - // 11 will set 1 index, 1 was missed, we should see 1 packet lost assert.True(t, b.Update(l, 11)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + + assert.True(t, b.Update(l, 14)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + + // Jump to 25. With length=16, slot 25%16=9 corresponds to packet 9 + // (which we DID receive), so its bit is set and no lost++ from that + // eviction. The trace below shows the only loss is packet 8. + assert.True(t, b.Update(l, 25)) + // current was 14, i=25. end=min(25,30)=25. count=11. startPos=15. + // steady? current=14<16, so warmup branch: per-bit n=15..25, count those + // with !get(n) AND n>16. n=17..25 are >16. Among slots 17%16=1..25%16=9 + // did we set slots 1..9 (packets 1..9)? Yes for all but slot 8 (packet 8 + // was skipped). n=24 maps to slot 8 which is FALSE → lost++. All other + // n in 17..25 map to slots that are set. n=16 is not strictly > 16. So + // lost = 1. assert.Equal(t, int64(1), b.lostCounter.Count()) - // Now let's fill in the window, should end up with 8 lost packets + + // Fill in 12, 13, 15, 16. Each is below current=25 (in-window). 16 must + // recheck slot 0 — it was set by NewBits and then cleared by the + // Update(25) jump, so 16 backfills cleanly. assert.True(t, b.Update(l, 12)) + assert.Equal(t, int64(1), b.lostCounter.Count()) assert.True(t, b.Update(l, 13)) - assert.True(t, b.Update(l, 14)) + assert.Equal(t, int64(1), b.lostCounter.Count()) assert.True(t, b.Update(l, 15)) + assert.Equal(t, int64(1), b.lostCounter.Count()) assert.True(t, b.Update(l, 16)) - assert.True(t, b.Update(l, 17)) - assert.True(t, b.Update(l, 18)) - assert.True(t, b.Update(l, 19)) - assert.Equal(t, int64(8), b.lostCounter.Count()) - - // Jump ahead by a window size - assert.True(t, b.Update(l, 29)) - assert.Equal(t, int64(8), b.lostCounter.Count()) - // Now lets walk ahead normally through the window, the missed packets should fill in - assert.True(t, b.Update(l, 30)) - assert.True(t, b.Update(l, 31)) - assert.True(t, b.Update(l, 32)) - assert.True(t, b.Update(l, 33)) - assert.True(t, b.Update(l, 34)) - assert.True(t, b.Update(l, 35)) - assert.True(t, b.Update(l, 36)) - assert.True(t, b.Update(l, 37)) - assert.True(t, b.Update(l, 38)) - // 39 packets tracked, 22 seen, 17 lost - assert.Equal(t, int64(17), b.lostCounter.Count()) - - // Jump ahead by 2 windows, should have recording 1 full window missing - assert.True(t, b.Update(l, 58)) - assert.Equal(t, int64(27), b.lostCounter.Count()) - // Now lets walk ahead normally through the window, the missed packets should fill in from this window - assert.True(t, b.Update(l, 59)) - assert.True(t, b.Update(l, 60)) - assert.True(t, b.Update(l, 61)) - assert.True(t, b.Update(l, 62)) - assert.True(t, b.Update(l, 63)) - assert.True(t, b.Update(l, 64)) - assert.True(t, b.Update(l, 65)) - assert.True(t, b.Update(l, 66)) - assert.True(t, b.Update(l, 67)) - // 68 packets tracked, 32 seen, 36 missed - assert.Equal(t, int64(36), b.lostCounter.Count()) + assert.Equal(t, int64(1), b.lostCounter.Count()) + + // We missed packet 8 above and that loss is still recorded once, never + // double-counted, never zeroed. + assert.Equal(t, int64(1), b.lostCounter.Count()) assert.Equal(t, int64(0), b.dupeCounter.Count()) assert.Equal(t, int64(0), b.outOfWindowCounter.Count()) } -func BenchmarkBits(b *testing.B) { - z := NewBits(10) +// TestBitsWarmupOvershoot exercises the jump path's warmup arm with an +// overshoot past one full window. NewBits leaves current=0 with only slot 0 +// "set" by the marker. Jumping straight to length+k must (a) clear every +// slot the jump straddles, (b) count only past-window slack (not the +// in-window slots, which never had a "lost" tenant during warmup), and +// (c) leave the cursor at the new counter so subsequent unit advances +// count from steady state. The marker bit at slot 0 is irrelevant once +// current >= length. +func TestBitsWarmupOvershoot(t *testing.T) { + l := test.NewLogger() + b := NewBits(16) + b.lostCounter.Clear() + + // Jump from current=0 to i=20 (length=16, overshoot=4). + // Warmup arm: counts slots in [1..16] where bit unset and n>length. + // Only n=16 was unset and >length: but slot 16%16=0 is the marker, + // so b.get(16) reads bits[0]=1 and skips. Result: 0 lost from the loop. + // Past-window: i - current - length = 20 - 0 - 16 = 4 lost. + assert.True(t, b.Update(l, 20)) + assert.Equal(t, int64(4), b.lostCounter.Count()) + assert.Equal(t, uint64(20), b.current) + + // Steady state now (current=20 >= length=16). Unit advance to 21 + // stomps slot 21%16=5, which was cleared by the jump and not reset, + // so this is +1 lost. + assert.True(t, b.Update(l, 21)) + assert.Equal(t, int64(5), b.lostCounter.Count()) +} + +// TestBitsCheckAcrossWarmupBoundary pins the underflow trick in Check's +// in-window clause. While in warmup, b.current-b.length underflows uint64 +// to a huge value so the first OR-clause is always false; the second +// clause (i < length && current < length) carries the in-window check. +// Once current >= length the regimes flip cleanly. +func TestBitsCheckAcrossWarmupBoundary(t *testing.T) { + l := test.NewLogger() + b := NewBits(16) + + // Warmup: current=0. Check(0) must read the marker (set) and return false. + assert.False(t, b.Check(l, 0), "marker slot should look already-received") + // Warmup: any 0 < i < length is in-window and unset → accepted. + for i := uint64(1); i < 16; i++ { + assert.True(t, b.Check(l, i), "warmup in-window i=%d should be accepted", i) + } + // Warmup: i >= length but > current is "next number" so accepted. + assert.True(t, b.Check(l, 16)) + assert.True(t, b.Check(l, 1_000_000)) + + // Cross into steady state. + assert.True(t, b.Update(l, 100)) + // Now current=100, length=16. In-window range is [85..100]. + // 84 is just outside: the underflow clause activates; 84 > 100-16=84 is false. + // And the warmup clause is false (current >= length). So out of window. + assert.False(t, b.Check(l, 84)) + // 85 sits at the boundary. 85 > 84 is true → in window, unset → accept. + assert.True(t, b.Check(l, 85)) + // 100 is current itself; not strictly greater, in-window, but already set. + assert.False(t, b.Check(l, 100)) + // Way out: clearly out of window. + assert.False(t, b.Check(l, 50)) +} + +// TestBitsMarkerInvariant verifies the seeded bits[0]=1 marker behaves +// correctly across warmup and beyond. Update should never clear the marker +// during warmup (clearRange skips position 0 when startPos=1), and once +// current >= length the marker is no longer consulted by Check/Update on +// the live path — but it must still report counter 0 as a duplicate while +// we are in warmup. +func TestBitsMarkerInvariant(t *testing.T) { + l := test.NewLogger() + b := NewBits(8) + + // Counter 0 is the seeded marker; Check sees it as already received. + assert.False(t, b.Check(l, 0)) + // Update(0) at current=0 hits the duplicate branch. + b.dupeCounter.Clear() + assert.False(t, b.Update(l, 0)) + assert.Equal(t, int64(1), b.dupeCounter.Count()) + + // Walk forward through warmup; the marker must remain set. + for n := uint64(1); n <= 7; n++ { + assert.True(t, b.Update(l, n)) + } + // Position 0 (the marker) should still read as set because we never + // cleared it; Update(0) still looks like a duplicate. + assert.False(t, b.Check(l, 0)) + + // Cross into steady state with a unit advance to 8: pos=0, evicts the + // marker bit. The lost-counter guard (i > b.length) is false (8 == 8), + // so this advance does NOT charge a lost packet — exactly what the + // marker is there to prevent. + b.lostCounter.Clear() + assert.True(t, b.Update(l, 8)) + assert.Equal(t, int64(0), b.lostCounter.Count()) + // The slot at pos 0 is now occupied by counter 8. + assert.False(t, b.Check(l, 8)) +} + +// BenchmarkBitsUpdateInOrder is the steady-state hot path: each call is +// i == current+1. +func BenchmarkBitsUpdateInOrder(b *testing.B) { + l := test.NewLogger() + z := NewBits(16384) for n := 0; n < b.N; n++ { - for i := range z.bits { - z.bits[i] = true - } - for i := range z.bits { - z.bits[i] = false - } + z.Update(l, uint64(n)+1) + } +} +// BenchmarkBitsUpdateReorder simulates light reorder within the window: +// every other packet arrives one slot behind its predecessor (forces the +// in-window backfill branch). +func BenchmarkBitsUpdateReorder(b *testing.B) { + l := test.NewLogger() + z := NewBits(16384) + for n := 0; n < b.N; n++ { + base := uint64(n) * 2 + z.Update(l, base+2) + z.Update(l, base+1) + } +} + +// BenchmarkBitsUpdateLargeJumps stresses the clearRange word-level path. +func BenchmarkBitsUpdateLargeJumps(b *testing.B) { + l := test.NewLogger() + z := NewBits(16384) + for n := 0; n < b.N; n++ { + z.Update(l, uint64(n+1)*1000) } } diff --git a/boring.go b/boring.go index 9cd9d37f..abe403fc 100644 --- a/boring.go +++ b/boring.go @@ -1,5 +1,4 @@ //go:build boringcrypto -// +build boringcrypto package nebula diff --git a/calculated_remote.go b/calculated_remote.go index 38f5bea2..0e28bb42 100644 --- a/calculated_remote.go +++ b/calculated_remote.go @@ -1,41 +1,40 @@ package nebula import ( + "encoding/binary" "fmt" "math" "net" + "net/netip" "strconv" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" ) // This allows us to "guess" what the remote might be for a host while we wait // for the lighthouse response. See "lighthouse.calculated_remotes" in the // example config file. type calculatedRemote struct { - ipNet net.IPNet - maskIP iputil.VpnIp - mask iputil.VpnIp - port uint32 + ipNet netip.Prefix + mask netip.Prefix + port uint32 } -func newCalculatedRemote(ipNet *net.IPNet, port int) (*calculatedRemote, error) { - // Ensure this is an IPv4 mask that we expect - ones, bits := ipNet.Mask.Size() - if ones == 0 || bits != 32 { - return nil, fmt.Errorf("invalid mask: %v", ipNet) +func newCalculatedRemote(cidr, maskCidr netip.Prefix, port int) (*calculatedRemote, error) { + if maskCidr.Addr().BitLen() != cidr.Addr().BitLen() { + return nil, fmt.Errorf("invalid mask: %s for cidr: %s", maskCidr, cidr) } + + masked := maskCidr.Masked() if port < 0 || port > math.MaxUint16 { return nil, fmt.Errorf("invalid port: %d", port) } return &calculatedRemote{ - ipNet: *ipNet, - maskIP: iputil.Ip2VpnIp(ipNet.IP), - mask: iputil.Ip2VpnIp(ipNet.Mask), - port: uint32(port), + ipNet: maskCidr, + mask: masked, + port: uint32(port), }, nil } @@ -43,49 +42,70 @@ func (c *calculatedRemote) String() string { return fmt.Sprintf("CalculatedRemote(mask=%v port=%d)", c.ipNet, c.port) } -func (c *calculatedRemote) Apply(ip iputil.VpnIp) *Ip4AndPort { - // Combine the masked bytes of the "mask" IP with the unmasked bytes - // of the overlay IP - masked := (c.maskIP & c.mask) | (ip & ^c.mask) +func (c *calculatedRemote) ApplyV4(addr netip.Addr) *V4AddrPort { + // Combine the masked bytes of the "mask" IP with the unmasked bytes of the overlay IP + maskb := net.CIDRMask(c.mask.Bits(), c.mask.Addr().BitLen()) + mask := binary.BigEndian.Uint32(maskb[:]) + + b := c.mask.Addr().As4() + maskAddr := binary.BigEndian.Uint32(b[:]) + + b = addr.As4() + intAddr := binary.BigEndian.Uint32(b[:]) - return &Ip4AndPort{Ip: uint32(masked), Port: c.port} + return &V4AddrPort{(maskAddr & mask) | (intAddr & ^mask), c.port} } -func NewCalculatedRemotesFromConfig(c *config.C, k string) (*cidr.Tree4[[]*calculatedRemote], error) { +func (c *calculatedRemote) ApplyV6(addr netip.Addr) *V6AddrPort { + mask := net.CIDRMask(c.mask.Bits(), c.mask.Addr().BitLen()) + maskAddr := c.mask.Addr().As16() + calcAddr := addr.As16() + + ap := V6AddrPort{Port: c.port} + + maskb := binary.BigEndian.Uint64(mask[:8]) + maskAddrb := binary.BigEndian.Uint64(maskAddr[:8]) + calcAddrb := binary.BigEndian.Uint64(calcAddr[:8]) + ap.Hi = (maskAddrb & maskb) | (calcAddrb & ^maskb) + + maskb = binary.BigEndian.Uint64(mask[8:]) + maskAddrb = binary.BigEndian.Uint64(maskAddr[8:]) + calcAddrb = binary.BigEndian.Uint64(calcAddr[8:]) + ap.Lo = (maskAddrb & maskb) | (calcAddrb & ^maskb) + + return &ap +} + +func NewCalculatedRemotesFromConfig(c *config.C, k string) (*bart.Table[[]*calculatedRemote], error) { value := c.Get(k) if value == nil { return nil, nil } - calculatedRemotes := cidr.NewTree4[[]*calculatedRemote]() + calculatedRemotes := new(bart.Table[[]*calculatedRemote]) - rawMap, ok := value.(map[any]any) + rawMap, ok := value.(map[string]any) if !ok { return nil, fmt.Errorf("config `%s` has invalid type: %T", k, value) } - for rawKey, rawValue := range rawMap { - rawCIDR, ok := rawKey.(string) - if !ok { - return nil, fmt.Errorf("config `%s` has invalid key (type %T): %v", k, rawKey, rawKey) - } - - _, ipNet, err := net.ParseCIDR(rawCIDR) + for rawCIDR, rawValue := range rawMap { + cidr, err := netip.ParsePrefix(rawCIDR) if err != nil { return nil, fmt.Errorf("config `%s` has invalid CIDR: %s", k, rawCIDR) } - entry, err := newCalculatedRemotesListFromConfig(rawValue) + entry, err := newCalculatedRemotesListFromConfig(cidr, rawValue) if err != nil { return nil, fmt.Errorf("config '%s.%s': %w", k, rawCIDR, err) } - calculatedRemotes.AddCIDR(ipNet, entry) + calculatedRemotes.Insert(cidr, entry) } return calculatedRemotes, nil } -func newCalculatedRemotesListFromConfig(raw any) ([]*calculatedRemote, error) { +func newCalculatedRemotesListFromConfig(cidr netip.Prefix, raw any) ([]*calculatedRemote, error) { rawList, ok := raw.([]any) if !ok { return nil, fmt.Errorf("calculated_remotes entry has invalid type: %T", raw) @@ -93,7 +113,7 @@ func newCalculatedRemotesListFromConfig(raw any) ([]*calculatedRemote, error) { var l []*calculatedRemote for _, e := range rawList { - c, err := newCalculatedRemotesEntryFromConfig(e) + c, err := newCalculatedRemotesEntryFromConfig(cidr, e) if err != nil { return nil, fmt.Errorf("calculated_remotes entry: %w", err) } @@ -103,8 +123,8 @@ func newCalculatedRemotesListFromConfig(raw any) ([]*calculatedRemote, error) { return l, nil } -func newCalculatedRemotesEntryFromConfig(raw any) (*calculatedRemote, error) { - rawMap, ok := raw.(map[any]any) +func newCalculatedRemotesEntryFromConfig(cidr netip.Prefix, raw any) (*calculatedRemote, error) { + rawMap, ok := raw.(map[string]any) if !ok { return nil, fmt.Errorf("invalid type: %T", raw) } @@ -117,7 +137,7 @@ func newCalculatedRemotesEntryFromConfig(raw any) (*calculatedRemote, error) { if !ok { return nil, fmt.Errorf("invalid mask (type %T): %v", rawValue, rawValue) } - _, ipNet, err := net.ParseCIDR(rawMask) + maskCidr, err := netip.ParsePrefix(rawMask) if err != nil { return nil, fmt.Errorf("invalid mask: %s", rawMask) } @@ -139,5 +159,5 @@ func newCalculatedRemotesEntryFromConfig(raw any) (*calculatedRemote, error) { return nil, fmt.Errorf("invalid port (type %T): %v", rawValue, rawValue) } - return newCalculatedRemote(ipNet, port) + return newCalculatedRemote(cidr, maskCidr, port) } diff --git a/calculated_remote_test.go b/calculated_remote_test.go index 2ddebca7..6df893c1 100644 --- a/calculated_remote_test.go +++ b/calculated_remote_test.go @@ -1,27 +1,81 @@ package nebula import ( - "net" + "net/netip" "testing" - "github.com/slackhq/nebula/iputil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestCalculatedRemoteApply(t *testing.T) { - _, ipNet, err := net.ParseCIDR("192.168.1.0/24") + // Test v4 addresses + ipNet := netip.MustParsePrefix("192.168.1.0/24") + c, err := newCalculatedRemote(ipNet, ipNet, 4242) require.NoError(t, err) - c, err := newCalculatedRemote(ipNet, 4242) + input, err := netip.ParseAddr("10.0.10.182") require.NoError(t, err) - input := iputil.Ip2VpnIp([]byte{10, 0, 10, 182}) + expected, err := netip.ParseAddr("192.168.1.182") + require.NoError(t, err) + + assert.Equal(t, netAddrToProtoV4AddrPort(expected, 4242), c.ApplyV4(input)) + + // Test v6 addresses + ipNet = netip.MustParsePrefix("ffff:ffff:ffff:ffff::0/64") + c, err = newCalculatedRemote(ipNet, ipNet, 4242) + require.NoError(t, err) + + input, err = netip.ParseAddr("beef:beef:beef:beef:beef:beef:beef:beef") + require.NoError(t, err) + + expected, err = netip.ParseAddr("ffff:ffff:ffff:ffff:beef:beef:beef:beef") + require.NoError(t, err) + + assert.Equal(t, netAddrToProtoV6AddrPort(expected, 4242), c.ApplyV6(input)) + + // Test v6 addresses part 2 + ipNet = netip.MustParsePrefix("ffff:ffff:ffff:ffff:ffff::0/80") + c, err = newCalculatedRemote(ipNet, ipNet, 4242) + require.NoError(t, err) + + input, err = netip.ParseAddr("beef:beef:beef:beef:beef:beef:beef:beef") + require.NoError(t, err) + + expected, err = netip.ParseAddr("ffff:ffff:ffff:ffff:ffff:beef:beef:beef") + require.NoError(t, err) + + assert.Equal(t, netAddrToProtoV6AddrPort(expected, 4242), c.ApplyV6(input)) + + // Test v6 addresses part 2 + ipNet = netip.MustParsePrefix("ffff:ffff:ffff::0/48") + c, err = newCalculatedRemote(ipNet, ipNet, 4242) + require.NoError(t, err) + + input, err = netip.ParseAddr("beef:beef:beef:beef:beef:beef:beef:beef") + require.NoError(t, err) + + expected, err = netip.ParseAddr("ffff:ffff:ffff:beef:beef:beef:beef:beef") + require.NoError(t, err) + + assert.Equal(t, netAddrToProtoV6AddrPort(expected, 4242), c.ApplyV6(input)) +} + +func Test_newCalculatedRemote(t *testing.T) { + c, err := newCalculatedRemote(netip.MustParsePrefix("1::1/128"), netip.MustParsePrefix("1.0.0.0/32"), 4242) + require.EqualError(t, err, "invalid mask: 1.0.0.0/32 for cidr: 1::1/128") + require.Nil(t, c) - expected := &Ip4AndPort{ - Ip: uint32(iputil.Ip2VpnIp([]byte{192, 168, 1, 182})), - Port: 4242, - } + c, err = newCalculatedRemote(netip.MustParsePrefix("1.0.0.0/32"), netip.MustParsePrefix("1::1/128"), 4242) + require.EqualError(t, err, "invalid mask: 1::1/128 for cidr: 1.0.0.0/32") + require.Nil(t, c) - assert.Equal(t, expected, c.Apply(input)) + c, err = newCalculatedRemote(netip.MustParsePrefix("1.0.0.0/32"), netip.MustParsePrefix("1.0.0.0/32"), 4242) + require.NoError(t, err) + require.NotNil(t, c) + + c, err = newCalculatedRemote(netip.MustParsePrefix("1::1/128"), netip.MustParsePrefix("1::1/128"), 4242) + require.NoError(t, err) + require.NotNil(t, c) } diff --git a/cert/Makefile b/cert/Makefile index 28170b62..311afc24 100644 --- a/cert/Makefile +++ b/cert/Makefile @@ -1,7 +1,7 @@ GO111MODULE = on export GO111MODULE -cert.pb.go: cert.proto .FORCE +cert_v1.pb.go: cert_v1.proto .FORCE go build google.golang.org/protobuf/cmd/protoc-gen-go PATH="$(CURDIR):$(PATH)" protoc --go_out=. --go_opt=paths=source_relative $< rm protoc-gen-go diff --git a/cert/README.md b/cert/README.md index ae19a28f..1e27a6bf 100644 --- a/cert/README.md +++ b/cert/README.md @@ -2,14 +2,25 @@ This is a library for interacting with `nebula` style certificates and authorities. -A `protobuf` definition of the certificate format is also included +There are now 2 versions of `nebula` certificates: -### Compiling the protobuf definition +## v1 -Make sure you have `protoc` installed. +This version is deprecated. + +A `protobuf` definition of the certificate format is included at `cert_v1.proto` + +To compile the definition you will need `protoc` installed. To compile for `go` with the same version of protobuf specified in go.mod: ```bash -make +make proto ``` + +## v2 + +This is the latest version which uses asn.1 DER encoding. It can support ipv4 and ipv6 and tolerate +future certificate changes better than v1. + +`cert_v2.asn1` defines the wire format and can be used to compile marshalers. \ No newline at end of file diff --git a/cert/asn1.go b/cert/asn1.go new file mode 100644 index 00000000..6bf6a8de --- /dev/null +++ b/cert/asn1.go @@ -0,0 +1,52 @@ +package cert + +import ( + "golang.org/x/crypto/cryptobyte" + "golang.org/x/crypto/cryptobyte/asn1" +) + +// readOptionalASN1Boolean reads an asn.1 boolean with a specific tag instead of a asn.1 tag wrapping a boolean with a value +// https://github.com/golang/go/issues/64811#issuecomment-1944446920 +func readOptionalASN1Boolean(b *cryptobyte.String, out *bool, tag asn1.Tag, defaultValue bool) bool { + var present bool + var child cryptobyte.String + if !b.ReadOptionalASN1(&child, &present, tag) { + return false + } + + if !present { + *out = defaultValue + return true + } + + // Ensure we have 1 byte + if len(child) == 1 { + *out = child[0] > 0 + return true + } + + return false +} + +// readOptionalASN1Byte reads an asn.1 uint8 with a specific tag instead of a asn.1 tag wrapping a uint8 with a value +// Similar issue as with readOptionalASN1Boolean +func readOptionalASN1Byte(b *cryptobyte.String, out *byte, tag asn1.Tag, defaultValue byte) bool { + var present bool + var child cryptobyte.String + if !b.ReadOptionalASN1(&child, &present, tag) { + return false + } + + if !present { + *out = defaultValue + return true + } + + // Ensure we have 1 byte + if len(child) == 1 { + *out = child[0] + return true + } + + return false +} diff --git a/cert/ca.go b/cert/ca.go deleted file mode 100644 index 0ffbd879..00000000 --- a/cert/ca.go +++ /dev/null @@ -1,140 +0,0 @@ -package cert - -import ( - "errors" - "fmt" - "strings" - "time" -) - -type NebulaCAPool struct { - CAs map[string]*NebulaCertificate - certBlocklist map[string]struct{} -} - -// NewCAPool creates a CAPool -func NewCAPool() *NebulaCAPool { - ca := NebulaCAPool{ - CAs: make(map[string]*NebulaCertificate), - certBlocklist: make(map[string]struct{}), - } - - return &ca -} - -// NewCAPoolFromBytes will create a new CA pool from the provided -// input bytes, which must be a PEM-encoded set of nebula certificates. -// If the pool contains any expired certificates, an ErrExpired will be -// returned along with the pool. The caller must handle any such errors. -func NewCAPoolFromBytes(caPEMs []byte) (*NebulaCAPool, error) { - pool := NewCAPool() - var err error - var expired bool - for { - caPEMs, err = pool.AddCACertificate(caPEMs) - if errors.Is(err, ErrExpired) { - expired = true - err = nil - } - if err != nil { - return nil, err - } - if len(caPEMs) == 0 || strings.TrimSpace(string(caPEMs)) == "" { - break - } - } - - if expired { - return pool, ErrExpired - } - - return pool, nil -} - -// AddCACertificate verifies a Nebula CA certificate and adds it to the pool -// Only the first pem encoded object will be consumed, any remaining bytes are returned. -// Parsed certificates will be verified and must be a CA -func (ncp *NebulaCAPool) AddCACertificate(pemBytes []byte) ([]byte, error) { - c, pemBytes, err := UnmarshalNebulaCertificateFromPEM(pemBytes) - if err != nil { - return pemBytes, err - } - - if !c.Details.IsCA { - return pemBytes, fmt.Errorf("%s: %w", c.Details.Name, ErrNotCA) - } - - if !c.CheckSignature(c.Details.PublicKey) { - return pemBytes, fmt.Errorf("%s: %w", c.Details.Name, ErrNotSelfSigned) - } - - sum, err := c.Sha256Sum() - if err != nil { - return pemBytes, fmt.Errorf("could not calculate shasum for provided CA; error: %s; %s", err, c.Details.Name) - } - - ncp.CAs[sum] = c - if c.Expired(time.Now()) { - return pemBytes, fmt.Errorf("%s: %w", c.Details.Name, ErrExpired) - } - - return pemBytes, nil -} - -// BlocklistFingerprint adds a cert fingerprint to the blocklist -func (ncp *NebulaCAPool) BlocklistFingerprint(f string) { - ncp.certBlocklist[f] = struct{}{} -} - -// ResetCertBlocklist removes all previously blocklisted cert fingerprints -func (ncp *NebulaCAPool) ResetCertBlocklist() { - ncp.certBlocklist = make(map[string]struct{}) -} - -// NOTE: This uses an internal cache for Sha256Sum() that will not be invalidated -// automatically if you manually change any fields in the NebulaCertificate. -func (ncp *NebulaCAPool) IsBlocklisted(c *NebulaCertificate) bool { - return ncp.isBlocklistedWithCache(c, false) -} - -// IsBlocklisted returns true if the fingerprint fails to generate or has been explicitly blocklisted -func (ncp *NebulaCAPool) isBlocklistedWithCache(c *NebulaCertificate, useCache bool) bool { - h, err := c.sha256SumWithCache(useCache) - if err != nil { - return true - } - - if _, ok := ncp.certBlocklist[h]; ok { - return true - } - - return false -} - -// GetCAForCert attempts to return the signing certificate for the provided certificate. -// No signature validation is performed -func (ncp *NebulaCAPool) GetCAForCert(c *NebulaCertificate) (*NebulaCertificate, error) { - if c.Details.Issuer == "" { - return nil, fmt.Errorf("no issuer in certificate") - } - - signer, ok := ncp.CAs[c.Details.Issuer] - if ok { - return signer, nil - } - - return nil, fmt.Errorf("could not find ca for the certificate") -} - -// GetFingerprints returns an array of trusted CA fingerprints -func (ncp *NebulaCAPool) GetFingerprints() []string { - fp := make([]string, len(ncp.CAs)) - - i := 0 - for k := range ncp.CAs { - fp[i] = k - i++ - } - - return fp -} diff --git a/cert/ca_pool.go b/cert/ca_pool.go new file mode 100644 index 00000000..966f78e3 --- /dev/null +++ b/cert/ca_pool.go @@ -0,0 +1,345 @@ +package cert + +import ( + "bufio" + "bytes" + "encoding/pem" + "errors" + "fmt" + "io" + "net/netip" + "slices" + "time" +) + +type CAPool struct { + CAs map[string]*CachedCertificate + certBlocklist map[string]struct{} +} + +// NewCAPool creates an empty CAPool +func NewCAPool() *CAPool { + ca := CAPool{ + CAs: make(map[string]*CachedCertificate), + certBlocklist: make(map[string]struct{}), + } + + return &ca +} + +// NewCAPoolFromPEM will create a new CA pool from the provided +// input bytes, which must be a PEM-encoded set of nebula certificates. +// If the pool contains any expired certificates, an ErrExpired will be +// returned along with the pool. The caller must handle any such errors. +func NewCAPoolFromPEM(caPEMs []byte) (*CAPool, error) { + return NewCAPoolFromPEMReader(bytes.NewReader(caPEMs)) +} + +// NewCAPoolFromPEMReader will create a new CA pool from the provided reader. +// The reader must contain a PEM-encoded set of nebula certificates. +func NewCAPoolFromPEMReader(r io.Reader) (*CAPool, error) { + pool := NewCAPool() + + var expired bool + + scanner := bufio.NewScanner(r) + scanner.Split(SplitPEM) + + for scanner.Scan() { + pemBytes := scanner.Bytes() + + block, rest := pem.Decode(pemBytes) + if len(bytes.TrimSpace(rest)) > 0 { + return nil, ErrInvalidPEMBlock + } + if block == nil { + return nil, ErrInvalidPEMBlock + } + + c, err := unmarshalCertificateBlock(block) + if err != nil { + return nil, err + } + + err = pool.AddCA(c) + if errors.Is(err, ErrExpired) { + expired = true + continue + } else if err != nil { + return nil, err + } + } + if err := scanner.Err(); err != nil { + return nil, ErrInvalidPEMBlock + } + + if expired { + return pool, ErrExpired + } + + return pool, nil +} + +// AddCAFromPEM verifies a Nebula CA certificate and adds it to the pool. +// Only the first pem encoded object will be consumed, any remaining bytes are returned. +// Parsed certificates will be verified and must be a CA +func (ncp *CAPool) AddCAFromPEM(pemBytes []byte) ([]byte, error) { + c, pemBytes, err := UnmarshalCertificateFromPEM(pemBytes) + if err != nil { + return pemBytes, err + } + + err = ncp.AddCA(c) + if err != nil { + return pemBytes, err + } + + return pemBytes, nil +} + +// AddCA verifies a Nebula CA certificate and adds it to the pool. +func (ncp *CAPool) AddCA(c Certificate) error { + if !c.IsCA() { + return fmt.Errorf("%s: %w", c.Name(), ErrNotCA) + } + + if !c.CheckSignature(c.PublicKey()) { + return fmt.Errorf("%s: %w", c.Name(), ErrNotSelfSigned) + } + + sum, err := c.Fingerprint() + if err != nil { + return fmt.Errorf("could not calculate fingerprint for provided CA; error: %w; %s", err, c.Name()) + } + + cc := &CachedCertificate{ + Certificate: c, + Fingerprint: sum, + InvertedGroups: make(map[string]struct{}), + } + + for _, g := range c.Groups() { + cc.InvertedGroups[g] = struct{}{} + } + + ncp.CAs[sum] = cc + + if c.Expired(time.Now()) { + return fmt.Errorf("%s: %w", c.Name(), ErrExpired) + } + + return nil +} + +// BlocklistFingerprint adds a cert fingerprint to the blocklist +func (ncp *CAPool) BlocklistFingerprint(f string) { + ncp.certBlocklist[f] = struct{}{} +} + +// ResetCertBlocklist removes all previously blocklisted cert fingerprints +func (ncp *CAPool) ResetCertBlocklist() { + ncp.certBlocklist = make(map[string]struct{}) +} + +// IsBlocklisted tests the provided fingerprint against the pools blocklist. +// Returns true if the fingerprint is blocked. +func (ncp *CAPool) IsBlocklisted(fingerprint string) bool { + if _, ok := ncp.certBlocklist[fingerprint]; ok { + return true + } + + return false +} + +// VerifyCertificate verifies the certificate is valid and is signed by a trusted CA in the pool. +// If the certificate is valid then the returned CachedCertificate can be used in subsequent verification attempts +// to increase performance. +func (ncp *CAPool) VerifyCertificate(now time.Time, c Certificate) (*CachedCertificate, error) { + if c == nil { + return nil, fmt.Errorf("no certificate") + } + fp, err := c.Fingerprint() + if err != nil { + return nil, fmt.Errorf("could not calculate fingerprint to verify: %w", err) + } + + signer, err := ncp.verify(c, now, fp, "") + if err != nil { + return nil, err + } + + // Pre nebula v1.10.3 could generate signatures in either high or low s form and validation + // of signatures allowed for either. Nebula v1.10.3 and beyond clamps signature generation to low-s form + // but validation still allows for either. Since a change in the signature bytes affects the fingerprint, we + // need to test both forms until such a time comes that we enforce low-s form on signature validation. + fp2, err := CalculateAlternateFingerprint(c) + if err != nil { + return nil, fmt.Errorf("could not calculate alternate fingerprint to verify: %w", err) + } + if fp2 != "" && ncp.IsBlocklisted(fp2) { + return nil, ErrBlockListed + } + + cc := CachedCertificate{ + Certificate: c, + InvertedGroups: make(map[string]struct{}), + Fingerprint: fp, + fingerprint2: fp2, + signerFingerprint: signer.Fingerprint, + } + + for _, g := range c.Groups() { + cc.InvertedGroups[g] = struct{}{} + } + + return &cc, nil +} + +// VerifyCachedCertificate is the same as VerifyCertificate other than it operates on a pre-verified structure and +// is a cheaper operation to perform as a result. +func (ncp *CAPool) VerifyCachedCertificate(now time.Time, c *CachedCertificate) error { + // Check any available alternate fingerprint forms for this certificate, re P256 high-s/low-s + if c.fingerprint2 != "" && ncp.IsBlocklisted(c.fingerprint2) { + return ErrBlockListed + } + + _, err := ncp.verify(c.Certificate, now, c.Fingerprint, c.signerFingerprint) + return err +} + +func (ncp *CAPool) verify(c Certificate, now time.Time, certFp string, signerFp string) (*CachedCertificate, error) { + if ncp.IsBlocklisted(certFp) { + return nil, ErrBlockListed + } + + signer, err := ncp.GetCAForCert(c) + if err != nil { + return nil, err + } + + if signer.Certificate.Curve() != c.Curve() { + return nil, ErrCurveMismatch + } + + if signer.Certificate.Expired(now) { + return nil, ErrRootExpired + } + + if c.Expired(now) { + return nil, ErrExpired + } + + // If we are checking a cached certificate then we can bail early here + // Either the root is no longer trusted or everything is fine + if len(signerFp) > 0 { + if signerFp != signer.Fingerprint { + return nil, ErrFingerprintMismatch + } + return signer, nil + } + if !c.CheckSignature(signer.Certificate.PublicKey()) { + return nil, ErrSignatureMismatch + } + + err = CheckCAConstraints(signer.Certificate, c) + if err != nil { + return nil, err + } + + return signer, nil +} + +// GetCAForCert attempts to return the signing certificate for the provided certificate. +// No signature validation is performed +func (ncp *CAPool) GetCAForCert(c Certificate) (*CachedCertificate, error) { + issuer := c.Issuer() + if issuer == "" { + return nil, fmt.Errorf("no issuer in certificate") + } + + signer, ok := ncp.CAs[issuer] + if ok { + return signer, nil + } + + return nil, ErrCaNotFound +} + +// GetFingerprints returns an array of trusted CA fingerprints +func (ncp *CAPool) GetFingerprints() []string { + fp := make([]string, len(ncp.CAs)) + + i := 0 + for k := range ncp.CAs { + fp[i] = k + i++ + } + + return fp +} + +// CheckCAConstraints returns an error if the sub certificate violates constraints present in the signer certificate. +func CheckCAConstraints(signer Certificate, sub Certificate) error { + return checkCAConstraints(signer, sub.NotBefore(), sub.NotAfter(), sub.Groups(), sub.Networks(), sub.UnsafeNetworks()) +} + +// checkCAConstraints is a very generic function allowing both Certificates and TBSCertificates to be tested. +func checkCAConstraints(signer Certificate, notBefore, notAfter time.Time, groups []string, networks, unsafeNetworks []netip.Prefix) error { + // Make sure this cert isn't valid after the root + if notAfter.After(signer.NotAfter()) { + return fmt.Errorf("certificate expires after signing certificate") + } + + // Make sure this cert wasn't valid before the root + if notBefore.Before(signer.NotBefore()) { + return fmt.Errorf("certificate is valid before the signing certificate") + } + + // If the signer has a limited set of groups make sure the cert only contains a subset + signerGroups := signer.Groups() + if len(signerGroups) > 0 { + for _, g := range groups { + if !slices.Contains(signerGroups, g) { + return fmt.Errorf("certificate contained a group not present on the signing ca: %s", g) + } + } + } + + // If the signer has a limited set of ip ranges to issue from make sure the cert only contains a subset + signingNetworks := signer.Networks() + if len(signingNetworks) > 0 { + for _, certNetwork := range networks { + found := false + for _, signingNetwork := range signingNetworks { + if signingNetwork.Contains(certNetwork.Addr()) && signingNetwork.Bits() <= certNetwork.Bits() { + found = true + break + } + } + + if !found { + return fmt.Errorf("certificate contained a network assignment outside the limitations of the signing ca: %s", certNetwork.String()) + } + } + } + + // If the signer has a limited set of subnet ranges to issue from make sure the cert only contains a subset + signingUnsafeNetworks := signer.UnsafeNetworks() + if len(signingUnsafeNetworks) > 0 { + for _, certUnsafeNetwork := range unsafeNetworks { + found := false + for _, caNetwork := range signingUnsafeNetworks { + if caNetwork.Contains(certUnsafeNetwork.Addr()) && caNetwork.Bits() <= certUnsafeNetwork.Bits() { + found = true + break + } + } + + if !found { + return fmt.Errorf("certificate contained an unsafe network assignment outside the limitations of the signing ca: %s", certUnsafeNetwork.String()) + } + } + } + + return nil +} diff --git a/cert/ca_pool_test.go b/cert/ca_pool_test.go new file mode 100644 index 00000000..c246e770 --- /dev/null +++ b/cert/ca_pool_test.go @@ -0,0 +1,684 @@ +package cert + +import ( + "bytes" + "io" + "net/netip" + "strings" + "testing" + "time" + + "github.com/slackhq/nebula/cert/p256" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewCAPoolFromBytes(t *testing.T) { + noNewLines := ` +# Current provisional, Remove once everything moves over to the real root. +-----BEGIN NEBULA CERTIFICATE----- +Cj4KDm5lYnVsYSByb290IGNhKM0cMM24zPCvBzogV24YEw5YiqeI/oYo8XXFsoo+ +PBmiOafNJhLacf9rsspAARJAz9OAnh8TKAUKix1kKVMyQU4iM3LsFfZRf6ODWXIf +2qWMpB6fpd3PSoVYziPoOt2bIHIFLlgRLPJz3I3xBEdBCQ== +-----END NEBULA CERTIFICATE----- +# root-ca01 +-----BEGIN NEBULA CERTIFICATE----- +CkEKEW5lYnVsYSByb290IGNhIDAxKM0cMM24zPCvBzogPzbWTxt8ZgXPQEwup7Br +BrtIt1O0q5AuTRT3+t2x1VJAARJAZ+2ib23qBXjdy49oU1YysrwuKkWWKrtJ7Jye +rFBQpDXikOukhQD/mfkloFwJ+Yjsfru7IpTN4ZfjXL+kN/2sCA== +-----END NEBULA CERTIFICATE----- +` + + withNewLines := ` +# Current provisional, Remove once everything moves over to the real root. + +-----BEGIN NEBULA CERTIFICATE----- +Cj4KDm5lYnVsYSByb290IGNhKM0cMM24zPCvBzogV24YEw5YiqeI/oYo8XXFsoo+ +PBmiOafNJhLacf9rsspAARJAz9OAnh8TKAUKix1kKVMyQU4iM3LsFfZRf6ODWXIf +2qWMpB6fpd3PSoVYziPoOt2bIHIFLlgRLPJz3I3xBEdBCQ== +-----END NEBULA CERTIFICATE----- + +# root-ca01 + + +-----BEGIN NEBULA CERTIFICATE----- +CkEKEW5lYnVsYSByb290IGNhIDAxKM0cMM24zPCvBzogPzbWTxt8ZgXPQEwup7Br +BrtIt1O0q5AuTRT3+t2x1VJAARJAZ+2ib23qBXjdy49oU1YysrwuKkWWKrtJ7Jye +rFBQpDXikOukhQD/mfkloFwJ+Yjsfru7IpTN4ZfjXL+kN/2sCA== +-----END NEBULA CERTIFICATE----- + +` + + expired := ` +# expired certificate +-----BEGIN NEBULA CERTIFICATE----- +CjMKB2V4cGlyZWQozRwwzRw6ICJSG94CqX8wn5I65Pwn25V6HftVfWeIySVtp2DA +7TY/QAESQMaAk5iJT5EnQwK524ZaaHGEJLUqqbh5yyOHhboIGiVTWkFeH3HccTW8 +Tq5a8AyWDQdfXbtEZ1FwabeHfH5Asw0= +-----END NEBULA CERTIFICATE----- +` + + p256 := ` +# p256 certificate +-----BEGIN NEBULA CERTIFICATE----- +CmQKEG5lYnVsYSBQMjU2IHRlc3QozRwwzbjM8K8HOkEEdrmmg40zQp44AkMq6DZp +k+coOv04r+zh33ISyhbsafnYduN17p2eD7CmHvHuerguXD9f32gcxo/KsFCKEjMe ++0ABoAYBEkcwRQIgVoTg38L7uWku9xQgsr06kxZ/viQLOO/w1Qj1vFUEnhcCIQCq +75SjTiV92kv/1GcbT3wWpAZQQDBiUHVMVmh1822szA== +-----END NEBULA CERTIFICATE----- +` + + rootCA := certificateV1{ + details: detailsV1{ + name: "nebula root ca", + }, + } + + rootCA01 := certificateV1{ + details: detailsV1{ + name: "nebula root ca 01", + }, + } + + rootCAP256 := certificateV1{ + details: detailsV1{ + name: "nebula P256 test", + }, + } + + p, err := NewCAPoolFromPEM([]byte(noNewLines)) + require.NoError(t, err) + assert.Equal(t, p.CAs["ce4e6c7a596996eb0d82a8875f0f0137a4b53ce22d2421c9fd7150e7a26f6300"].Certificate.Name(), rootCA.details.name) + assert.Equal(t, p.CAs["04c585fcd9a49b276df956a22b7ebea3bf23f1fca5a17c0b56ce2e626631969e"].Certificate.Name(), rootCA01.details.name) + + pp, err := NewCAPoolFromPEM([]byte(withNewLines)) + require.NoError(t, err) + assert.Equal(t, pp.CAs["ce4e6c7a596996eb0d82a8875f0f0137a4b53ce22d2421c9fd7150e7a26f6300"].Certificate.Name(), rootCA.details.name) + assert.Equal(t, pp.CAs["04c585fcd9a49b276df956a22b7ebea3bf23f1fca5a17c0b56ce2e626631969e"].Certificate.Name(), rootCA01.details.name) + + // expired cert, no valid certs + ppp, err := NewCAPoolFromPEM([]byte(expired)) + assert.Equal(t, ErrExpired, err) + assert.Equal(t, "expired", ppp.CAs["c39b35a0e8f246203fe4f32b9aa8bfd155f1ae6a6be9d78370641e43397f48f5"].Certificate.Name()) + + // expired cert, with valid certs + pppp, err := NewCAPoolFromPEM(append([]byte(expired), noNewLines...)) + assert.Equal(t, ErrExpired, err) + assert.Equal(t, pppp.CAs["ce4e6c7a596996eb0d82a8875f0f0137a4b53ce22d2421c9fd7150e7a26f6300"].Certificate.Name(), rootCA.details.name) + assert.Equal(t, pppp.CAs["04c585fcd9a49b276df956a22b7ebea3bf23f1fca5a17c0b56ce2e626631969e"].Certificate.Name(), rootCA01.details.name) + assert.Equal(t, "expired", pppp.CAs["c39b35a0e8f246203fe4f32b9aa8bfd155f1ae6a6be9d78370641e43397f48f5"].Certificate.Name()) + assert.Len(t, pppp.CAs, 3) + + ppppp, err := NewCAPoolFromPEM([]byte(p256)) + require.NoError(t, err) + assert.Equal(t, ppppp.CAs["552bf7d99bec1fc775a0e4c324bf6d8f789b3078f1919c7960d2e5e0c351ee97"].Certificate.Name(), rootCAP256.details.name) + assert.Len(t, ppppp.CAs, 1) +} + +// oneByteReader wraps a reader to return at most 1 byte per Read call, +// exercising the streaming accumulation logic in NewCAPoolFromPEMReader. +type oneByteReader struct { + r io.Reader +} + +func (o *oneByteReader) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + return o.r.Read(p[:1]) +} + +func TestNewCAPoolFromPEMReader_EmptyReader(t *testing.T) { + pool, err := NewCAPoolFromPEMReader(bytes.NewReader(nil)) + require.NoError(t, err) + assert.Empty(t, pool.CAs) + + pool, err = NewCAPoolFromPEMReader(strings.NewReader(" \n\t\n ")) + require.NoError(t, err) + assert.Empty(t, pool.CAs) +} + +func TestNewCAPoolFromPEMReader_OneByteReads(t *testing.T) { + ca1, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil) + ca2, _, _, pem2 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil) + + bundle := append(pem1, pem2...) + pool, err := NewCAPoolFromPEMReader(&oneByteReader{r: bytes.NewReader(bundle)}) + require.NoError(t, err) + assert.Len(t, pool.CAs, 2) + + fp1, err := ca1.Fingerprint() + require.NoError(t, err) + fp2, err := ca2.Fingerprint() + require.NoError(t, err) + + assert.Contains(t, pool.CAs, fp1) + assert.Contains(t, pool.CAs, fp2) +} + +func TestNewCAPoolFromPEMReader_TruncatedPEM(t *testing.T) { + _, err := NewCAPoolFromPEMReader(strings.NewReader("-----BEGIN NEBULA CERTIFICATE-----\npartialdata")) + assert.ErrorIs(t, err, ErrInvalidPEMBlock) +} + +func TestNewCAPoolFromPEMReader_TrailingGarbage(t *testing.T) { + _, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil) + + bundle := append(pem1, []byte("some trailing garbage")...) + _, err := NewCAPoolFromPEMReader(bytes.NewReader(bundle)) + assert.ErrorIs(t, err, ErrInvalidPEMBlock) +} + +func TestCertificateV1_Verify(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil) + c, _, _, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test cert", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil) + + caPool := NewCAPool() + require.NoError(t, caPool.AddCA(ca)) + + f, err := c.Fingerprint() + require.NoError(t, err) + caPool.BlocklistFingerprint(f) + + _, err = caPool.VerifyCertificate(time.Now(), c) + require.EqualError(t, err, "certificate is in the block list") + + caPool.ResetCertBlocklist() + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + _, err = caPool.VerifyCertificate(time.Now().Add(time.Hour*1000), c) + require.EqualError(t, err, "root certificate is expired") + + assert.PanicsWithError(t, "certificate is valid before the signing certificate", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test cert2", time.Time{}, time.Time{}, nil, nil, nil) + }) + + // Test group assertion + ca, _, caKey, _ = NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{"test1", "test2"}) + caPem, err := ca.MarshalPEM() + require.NoError(t, err) + + caPool = NewCAPool() + b, err := caPool.AddCAFromPEM(caPem) + require.NoError(t, err) + assert.Empty(t, b) + + assert.PanicsWithError(t, "certificate contained a group not present on the signing ca: bad", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1", "bad"}) + }) + + c, _, _, _ = NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test2", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) +} + +func TestCertificateV1_VerifyP256(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version1, Curve_P256, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil) + c, _, _, _ := NewTestCert(Version1, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil) + + caPool := NewCAPool() + require.NoError(t, caPool.AddCA(ca)) + + f, err := c.Fingerprint() + require.NoError(t, err) + caPool.BlocklistFingerprint(f) + + _, err = caPool.VerifyCertificate(time.Now(), c) + require.EqualError(t, err, "certificate is in the block list") + + // Create a copy of the cert and swap to the alternate form for the signature + nc := c.Copy() + b, err := p256.Swap(c.Signature()) + require.NoError(t, err) + require.NoError(t, nc.(*certificateV1).setSignature(b)) + + _, err = caPool.VerifyCertificate(time.Now(), nc) + require.EqualError(t, err, "certificate is in the block list") + + caPool.ResetCertBlocklist() + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + _, err = caPool.VerifyCertificate(time.Now().Add(time.Hour*1000), c) + require.EqualError(t, err, "root certificate is expired") + + assert.PanicsWithError(t, "certificate is valid before the signing certificate", func() { + NewTestCert(Version1, Curve_P256, ca, caKey, "test", time.Time{}, time.Time{}, nil, nil, nil) + }) + + // Test group assertion + ca, _, caKey, _ = NewTestCaCert(Version1, Curve_P256, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{"test1", "test2"}) + caPem, err := ca.MarshalPEM() + require.NoError(t, err) + + caPool = NewCAPool() + b, err = caPool.AddCAFromPEM(caPem) + require.NoError(t, err) + assert.Empty(t, b) + + assert.PanicsWithError(t, "certificate contained a group not present on the signing ca: bad", func() { + NewTestCert(Version1, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1", "bad"}) + }) + + c, _, _, _ = NewTestCert(Version1, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"}) + cc, err := caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Reset the blocklist and block the alternate form fingerprint + caPool.ResetCertBlocklist() + caPool.BlocklistFingerprint(cc.fingerprint2) + err = caPool.VerifyCachedCertificate(time.Now(), cc) + require.EqualError(t, err, "certificate is in the block list") + + caPool.ResetCertBlocklist() + err = caPool.VerifyCachedCertificate(time.Now(), cc) + require.NoError(t, err) +} + +func TestCertificateV1_Verify_IPs(t *testing.T) { + caIp1 := mustParsePrefixUnmapped("10.0.0.0/16") + caIp2 := mustParsePrefixUnmapped("192.168.0.0/24") + ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), []netip.Prefix{caIp1, caIp2}, nil, []string{"test"}) + + caPem, err := ca.MarshalPEM() + require.NoError(t, err) + + caPool := NewCAPool() + b, err := caPool.AddCAFromPEM(caPem) + require.NoError(t, err) + assert.Empty(t, b) + + // ip is outside the network + cIp1 := mustParsePrefixUnmapped("10.1.0.0/24") + cIp2 := mustParsePrefixUnmapped("192.168.0.1/16") + assert.PanicsWithError(t, "certificate contained a network assignment outside the limitations of the signing ca: 10.1.0.0/24", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + }) + + // ip is outside the network reversed order of above + cIp1 = mustParsePrefixUnmapped("192.168.0.1/24") + cIp2 = mustParsePrefixUnmapped("10.1.0.0/24") + assert.PanicsWithError(t, "certificate contained a network assignment outside the limitations of the signing ca: 10.1.0.0/24", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + }) + + // ip is within the network but mask is outside + cIp1 = mustParsePrefixUnmapped("10.0.1.0/15") + cIp2 = mustParsePrefixUnmapped("192.168.0.1/24") + assert.PanicsWithError(t, "certificate contained a network assignment outside the limitations of the signing ca: 10.0.1.0/15", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + }) + + // ip is within the network but mask is outside reversed order of above + cIp1 = mustParsePrefixUnmapped("192.168.0.1/24") + cIp2 = mustParsePrefixUnmapped("10.0.1.0/15") + assert.PanicsWithError(t, "certificate contained a network assignment outside the limitations of the signing ca: 10.0.1.0/15", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + }) + + // ip and mask are within the network + cIp1 = mustParsePrefixUnmapped("10.0.1.0/16") + cIp2 = mustParsePrefixUnmapped("192.168.0.1/25") + c, _, _, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches + c, _, _, _ = NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{caIp1, caIp2}, nil, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches reversed + c, _, _, _ = NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{caIp2, caIp1}, nil, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches reversed with just 1 + c, _, _, _ = NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{caIp1}, nil, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) +} + +func TestCertificateV1_Verify_Subnets(t *testing.T) { + caIp1 := mustParsePrefixUnmapped("10.0.0.0/16") + caIp2 := mustParsePrefixUnmapped("192.168.0.0/24") + ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, []netip.Prefix{caIp1, caIp2}, []string{"test"}) + + caPem, err := ca.MarshalPEM() + require.NoError(t, err) + + caPool := NewCAPool() + b, err := caPool.AddCAFromPEM(caPem) + require.NoError(t, err) + assert.Empty(t, b) + + // ip is outside the network + cIp1 := mustParsePrefixUnmapped("10.1.0.0/24") + cIp2 := mustParsePrefixUnmapped("192.168.0.1/16") + assert.PanicsWithError(t, "certificate contained an unsafe network assignment outside the limitations of the signing ca: 10.1.0.0/24", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + }) + + // ip is outside the network reversed order of above + cIp1 = mustParsePrefixUnmapped("192.168.0.1/24") + cIp2 = mustParsePrefixUnmapped("10.1.0.0/24") + assert.PanicsWithError(t, "certificate contained an unsafe network assignment outside the limitations of the signing ca: 10.1.0.0/24", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + }) + + // ip is within the network but mask is outside + cIp1 = mustParsePrefixUnmapped("10.0.1.0/15") + cIp2 = mustParsePrefixUnmapped("192.168.0.1/24") + assert.PanicsWithError(t, "certificate contained an unsafe network assignment outside the limitations of the signing ca: 10.0.1.0/15", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + }) + + // ip is within the network but mask is outside reversed order of above + cIp1 = mustParsePrefixUnmapped("192.168.0.1/24") + cIp2 = mustParsePrefixUnmapped("10.0.1.0/15") + assert.PanicsWithError(t, "certificate contained an unsafe network assignment outside the limitations of the signing ca: 10.0.1.0/15", func() { + NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + }) + + // ip and mask are within the network + cIp1 = mustParsePrefixUnmapped("10.0.1.0/16") + cIp2 = mustParsePrefixUnmapped("192.168.0.1/25") + c, _, _, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches + c, _, _, _ = NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{caIp1, caIp2}, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches reversed + c, _, _, _ = NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{caIp2, caIp1}, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches reversed with just 1 + c, _, _, _ = NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{caIp1}, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) +} + +func TestCertificateV2_Verify(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil) + c, _, _, _ := NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test cert", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil) + + caPool := NewCAPool() + require.NoError(t, caPool.AddCA(ca)) + + f, err := c.Fingerprint() + require.NoError(t, err) + caPool.BlocklistFingerprint(f) + + _, err = caPool.VerifyCertificate(time.Now(), c) + require.EqualError(t, err, "certificate is in the block list") + + caPool.ResetCertBlocklist() + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + _, err = caPool.VerifyCertificate(time.Now().Add(time.Hour*1000), c) + require.EqualError(t, err, "root certificate is expired") + + assert.PanicsWithError(t, "certificate is valid before the signing certificate", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test cert2", time.Time{}, time.Time{}, nil, nil, nil) + }) + + // Test group assertion + ca, _, caKey, _ = NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{"test1", "test2"}) + caPem, err := ca.MarshalPEM() + require.NoError(t, err) + + caPool = NewCAPool() + b, err := caPool.AddCAFromPEM(caPem) + require.NoError(t, err) + assert.Empty(t, b) + + assert.PanicsWithError(t, "certificate contained a group not present on the signing ca: bad", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1", "bad"}) + }) + + c, _, _, _ = NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test2", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) +} + +func TestCertificateV2_VerifyP256(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version2, Curve_P256, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil) + c, _, _, _ := NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil) + + caPool := NewCAPool() + require.NoError(t, caPool.AddCA(ca)) + + f, err := c.Fingerprint() + require.NoError(t, err) + caPool.BlocklistFingerprint(f) + + _, err = caPool.VerifyCertificate(time.Now(), c) + require.EqualError(t, err, "certificate is in the block list") + + // Create a copy of the cert and swap to the alternate form for the signature + nc := c.Copy() + b, err := p256.Swap(c.Signature()) + require.NoError(t, err) + require.NoError(t, nc.(*certificateV2).setSignature(b)) + + _, err = caPool.VerifyCertificate(time.Now(), nc) + require.EqualError(t, err, "certificate is in the block list") + + caPool.ResetCertBlocklist() + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + _, err = caPool.VerifyCertificate(time.Now().Add(time.Hour*1000), c) + require.EqualError(t, err, "root certificate is expired") + + assert.PanicsWithError(t, "certificate is valid before the signing certificate", func() { + NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Time{}, time.Time{}, nil, nil, nil) + }) + + // Test group assertion + ca, _, caKey, _ = NewTestCaCert(Version2, Curve_P256, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{"test1", "test2"}) + caPem, err := ca.MarshalPEM() + require.NoError(t, err) + + caPool = NewCAPool() + b, err = caPool.AddCAFromPEM(caPem) + require.NoError(t, err) + assert.Empty(t, b) + + assert.PanicsWithError(t, "certificate contained a group not present on the signing ca: bad", func() { + NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1", "bad"}) + }) + + c, _, _, _ = NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"}) + cc, err := caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Reset the blocklist and block the alternate form fingerprint + caPool.ResetCertBlocklist() + caPool.BlocklistFingerprint(cc.fingerprint2) + err = caPool.VerifyCachedCertificate(time.Now(), cc) + require.EqualError(t, err, "certificate is in the block list") + + caPool.ResetCertBlocklist() + err = caPool.VerifyCachedCertificate(time.Now(), cc) + require.NoError(t, err) +} + +func TestCertificateV2_Verify_IPs(t *testing.T) { + caIp1 := mustParsePrefixUnmapped("10.0.0.0/16") + caIp2 := mustParsePrefixUnmapped("192.168.0.0/24") + ca, _, caKey, _ := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), []netip.Prefix{caIp1, caIp2}, nil, []string{"test"}) + + caPem, err := ca.MarshalPEM() + require.NoError(t, err) + + caPool := NewCAPool() + b, err := caPool.AddCAFromPEM(caPem) + require.NoError(t, err) + assert.Empty(t, b) + + // ip is outside the network + cIp1 := mustParsePrefixUnmapped("10.1.0.0/24") + cIp2 := mustParsePrefixUnmapped("192.168.0.1/16") + assert.PanicsWithError(t, "certificate contained a network assignment outside the limitations of the signing ca: 10.1.0.0/24", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + }) + + // ip is outside the network reversed order of above + cIp1 = mustParsePrefixUnmapped("192.168.0.1/24") + cIp2 = mustParsePrefixUnmapped("10.1.0.0/24") + assert.PanicsWithError(t, "certificate contained a network assignment outside the limitations of the signing ca: 10.1.0.0/24", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + }) + + // ip is within the network but mask is outside + cIp1 = mustParsePrefixUnmapped("10.0.1.0/15") + cIp2 = mustParsePrefixUnmapped("192.168.0.1/24") + assert.PanicsWithError(t, "certificate contained a network assignment outside the limitations of the signing ca: 10.0.1.0/15", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + }) + + // ip is within the network but mask is outside reversed order of above + cIp1 = mustParsePrefixUnmapped("192.168.0.1/24") + cIp2 = mustParsePrefixUnmapped("10.0.1.0/15") + assert.PanicsWithError(t, "certificate contained a network assignment outside the limitations of the signing ca: 10.0.1.0/15", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + }) + + // ip and mask are within the network + cIp1 = mustParsePrefixUnmapped("10.0.1.0/16") + cIp2 = mustParsePrefixUnmapped("192.168.0.1/25") + c, _, _, _ := NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1, cIp2}, nil, []string{"test"}) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches + c, _, _, _ = NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{caIp1, caIp2}, nil, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches reversed + c, _, _, _ = NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{caIp2, caIp1}, nil, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches reversed with just 1 + c, _, _, _ = NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{caIp1}, nil, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) +} + +func TestCertificateV2_Verify_Subnets(t *testing.T) { + caIp1 := mustParsePrefixUnmapped("10.0.0.0/16") + caIp2 := mustParsePrefixUnmapped("192.168.0.0/24") + ca, _, caKey, _ := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, []netip.Prefix{caIp1, caIp2}, []string{"test"}) + + caPem, err := ca.MarshalPEM() + require.NoError(t, err) + + caPool := NewCAPool() + b, err := caPool.AddCAFromPEM(caPem) + require.NoError(t, err) + assert.Empty(t, b) + + // ip is outside the network + cIp1 := mustParsePrefixUnmapped("10.1.0.0/24") + cIp2 := mustParsePrefixUnmapped("192.168.0.1/16") + assert.PanicsWithError(t, "certificate contained an unsafe network assignment outside the limitations of the signing ca: 10.1.0.0/24", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + }) + + // ip is outside the network reversed order of above + cIp1 = mustParsePrefixUnmapped("192.168.0.1/24") + cIp2 = mustParsePrefixUnmapped("10.1.0.0/24") + assert.PanicsWithError(t, "certificate contained an unsafe network assignment outside the limitations of the signing ca: 10.1.0.0/24", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + }) + + // ip is within the network but mask is outside + cIp1 = mustParsePrefixUnmapped("10.0.1.0/15") + cIp2 = mustParsePrefixUnmapped("192.168.0.1/24") + assert.PanicsWithError(t, "certificate contained an unsafe network assignment outside the limitations of the signing ca: 10.0.1.0/15", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + }) + + // ip is within the network but mask is outside reversed order of above + cIp1 = mustParsePrefixUnmapped("192.168.0.1/24") + cIp2 = mustParsePrefixUnmapped("10.0.1.0/15") + assert.PanicsWithError(t, "certificate contained an unsafe network assignment outside the limitations of the signing ca: 10.0.1.0/15", func() { + NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + }) + + // ip and mask are within the network + cIp1 = mustParsePrefixUnmapped("10.0.1.0/16") + cIp2 = mustParsePrefixUnmapped("192.168.0.1/25") + c, _, _, _ := NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{cIp1, cIp2}, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches + c, _, _, _ = NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{caIp1, caIp2}, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches reversed + c, _, _, _ = NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{caIp2, caIp1}, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) + + // Exact matches reversed with just 1 + c, _, _, _ = NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, []netip.Prefix{caIp1}, []string{"test"}) + require.NoError(t, err) + _, err = caPool.VerifyCertificate(time.Now(), c) + require.NoError(t, err) +} + +func TestCertificateV2_CurveMismatch(t *testing.T) { + caIp1 := mustParsePrefixUnmapped("10.0.0.0/16") + caIp2 := mustParsePrefixUnmapped("192.168.0.0/24") + ca, _, caKey, _ := NewTestCaCert(Version2, Curve_P256, time.Now(), time.Now().Add(10*time.Minute), []netip.Prefix{caIp1, caIp2}, nil, []string{"test"}) + + caPem, err := ca.MarshalPEM() + require.NoError(t, err) + + caPool := NewCAPool() + b, err := caPool.AddCAFromPEM(caPem) + require.NoError(t, err) + assert.Empty(t, b) + + // ip is outside the network + cIp1 := mustParsePrefixUnmapped("10.0.0.1/24") + c, _, _, _ := NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1}, nil, []string{"test"}) + + fp, _ := c.Fingerprint() + _, err = caPool.verify(c, time.Now(), fp, c.Issuer()) + require.NoError(t, err) + // + c2 := c.(*certificateV2) + c2.curve = Curve_CURVE25519 + fp, _ = c.Fingerprint() + _, err = caPool.verify(c, time.Now(), fp, c.Issuer()) + require.Error(t, err) +} diff --git a/cert/cert.go b/cert/cert.go index 4f1b776c..01d775e5 100644 --- a/cert/cert.go +++ b/cert/cert.go @@ -1,1029 +1,187 @@ package cert import ( - "bytes" - "crypto/ecdh" - "crypto/ecdsa" - "crypto/ed25519" - "crypto/elliptic" - "crypto/rand" - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "encoding/json" - "encoding/pem" - "errors" "fmt" - "math" - "math/big" - "net" - "sync/atomic" + "net/netip" "time" - "golang.org/x/crypto/curve25519" - "google.golang.org/protobuf/proto" + "github.com/slackhq/nebula/cert/p256" ) -const publicKeyLen = 32 +type Version uint8 const ( - CertBanner = "NEBULA CERTIFICATE" - X25519PrivateKeyBanner = "NEBULA X25519 PRIVATE KEY" - X25519PublicKeyBanner = "NEBULA X25519 PUBLIC KEY" - EncryptedEd25519PrivateKeyBanner = "NEBULA ED25519 ENCRYPTED PRIVATE KEY" - Ed25519PrivateKeyBanner = "NEBULA ED25519 PRIVATE KEY" - Ed25519PublicKeyBanner = "NEBULA ED25519 PUBLIC KEY" - - P256PrivateKeyBanner = "NEBULA P256 PRIVATE KEY" - P256PublicKeyBanner = "NEBULA P256 PUBLIC KEY" - EncryptedECDSAP256PrivateKeyBanner = "NEBULA ECDSA P256 ENCRYPTED PRIVATE KEY" - ECDSAP256PrivateKeyBanner = "NEBULA ECDSA P256 PRIVATE KEY" + VersionPre1 Version = 0 + Version1 Version = 1 + Version2 Version = 2 ) -type NebulaCertificate struct { - Details NebulaCertificateDetails - Signature []byte +type Certificate interface { + // Version defines the underlying certificate structure and wire protocol version + // Version1 certificates are ipv4 only and uses protobuf serialization + // Version2 certificates are ipv4 or ipv6 and uses asn.1 serialization + Version() Version - // the cached hex string of the calculated sha256sum - // for VerifyWithCache - sha256sum atomic.Pointer[string] + // Name is the human-readable name that identifies this certificate. + Name() string - // the cached public key bytes if they were verified as the signer - // for VerifyWithCache - signatureVerified atomic.Pointer[[]byte] -} + // Networks is a list of ip addresses and network sizes assigned to this certificate. + // If IsCA is true then certificates signed by this CA can only have ip addresses and + // networks that are contained by an entry in this list. + Networks() []netip.Prefix -type NebulaCertificateDetails struct { - Name string - Ips []*net.IPNet - Subnets []*net.IPNet - Groups []string - NotBefore time.Time - NotAfter time.Time - PublicKey []byte - IsCA bool - Issuer string + // UnsafeNetworks is a list of networks that this host can act as an unsafe router for. + // If IsCA is true then certificates signed by this CA can only have networks that are + // contained by an entry in this list. + UnsafeNetworks() []netip.Prefix - // Map of groups for faster lookup - InvertedGroups map[string]struct{} + // Groups is a list of identities that can be used to write more general firewall rule + // definitions. + // If IsCA is true then certificates signed by this CA can only use groups that are + // in this list. + Groups() []string - Curve Curve -} + // IsCA signifies if this is a certificate authority (true) or a host certificate (false). + // It is invalid to use a CA certificate as a host certificate. + IsCA() bool -type NebulaEncryptedData struct { - EncryptionMetadata NebulaEncryptionMetadata - Ciphertext []byte -} + // NotBefore is the time at which this certificate becomes valid. + // If IsCA is true then certificate signed by this CA can not have a time before this. + NotBefore() time.Time -type NebulaEncryptionMetadata struct { - EncryptionAlgorithm string - Argon2Parameters Argon2Parameters -} + // NotAfter is the time at which this certificate becomes invalid. + // If IsCA is true then certificate signed by this CA can not have a time after this. + NotAfter() time.Time -type m map[string]interface{} + // Issuer is the fingerprint of the CA that signed this certificate. + // If IsCA is true then this will be empty. + Issuer() string -// Returned if we try to unmarshal an encrypted private key without a passphrase -var ErrPrivateKeyEncrypted = errors.New("private key must be decrypted") + // PublicKey is the raw bytes to be used in asymmetric cryptographic operations. + PublicKey() []byte -// UnmarshalNebulaCertificate will unmarshal a protobuf byte representation of a nebula cert -func UnmarshalNebulaCertificate(b []byte) (*NebulaCertificate, error) { - if len(b) == 0 { - return nil, fmt.Errorf("nil byte array") - } - var rc RawNebulaCertificate - err := proto.Unmarshal(b, &rc) - if err != nil { - return nil, err - } + // MarshalPublicKeyPEM is the value of PublicKey marshalled to PEM + MarshalPublicKeyPEM() []byte - if rc.Details == nil { - return nil, fmt.Errorf("encoded Details was nil") - } + // Curve identifies which curve was used for the PublicKey and Signature. + Curve() Curve - if len(rc.Details.Ips)%2 != 0 { - return nil, fmt.Errorf("encoded IPs should be in pairs, an odd number was found") - } + // Signature is the cryptographic seal for all the details of this certificate. + // CheckSignature can be used to verify that the details of this certificate are valid. + Signature() []byte - if len(rc.Details.Subnets)%2 != 0 { - return nil, fmt.Errorf("encoded Subnets should be in pairs, an odd number was found") - } + // CheckSignature will check that the certificate Signature() matches the + // computed signature. A true result means this certificate has not been tampered with. + CheckSignature(signingPublicKey []byte) bool - nc := NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: rc.Details.Name, - Groups: make([]string, len(rc.Details.Groups)), - Ips: make([]*net.IPNet, len(rc.Details.Ips)/2), - Subnets: make([]*net.IPNet, len(rc.Details.Subnets)/2), - NotBefore: time.Unix(rc.Details.NotBefore, 0), - NotAfter: time.Unix(rc.Details.NotAfter, 0), - PublicKey: make([]byte, len(rc.Details.PublicKey)), - IsCA: rc.Details.IsCA, - InvertedGroups: make(map[string]struct{}), - Curve: rc.Details.Curve, - }, - Signature: make([]byte, len(rc.Signature)), - } + // Fingerprint returns the hex encoded sha256 sum of the certificate. + // This acts as a unique fingerprint and can be used to blocklist certificates. + Fingerprint() (string, error) - copy(nc.Signature, rc.Signature) - copy(nc.Details.Groups, rc.Details.Groups) - nc.Details.Issuer = hex.EncodeToString(rc.Details.Issuer) + // Expired tests if the certificate is valid for the provided time. + Expired(t time.Time) bool - if len(rc.Details.PublicKey) < publicKeyLen { - return nil, fmt.Errorf("Public key was fewer than 32 bytes; %v", len(rc.Details.PublicKey)) - } - copy(nc.Details.PublicKey, rc.Details.PublicKey) + // VerifyPrivateKey returns an error if the private key is not a pair with the certificates public key. + VerifyPrivateKey(curve Curve, privateKey []byte) error - for i, rawIp := range rc.Details.Ips { - if i%2 == 0 { - nc.Details.Ips[i/2] = &net.IPNet{IP: int2ip(rawIp)} - } else { - nc.Details.Ips[i/2].Mask = net.IPMask(int2ip(rawIp)) - } - } + // Marshal will return the byte representation of this certificate + // This is primarily the format transmitted on the wire. + Marshal() ([]byte, error) - for i, rawIp := range rc.Details.Subnets { - if i%2 == 0 { - nc.Details.Subnets[i/2] = &net.IPNet{IP: int2ip(rawIp)} - } else { - nc.Details.Subnets[i/2].Mask = net.IPMask(int2ip(rawIp)) - } - } + // MarshalForHandshakes prepares the bytes needed to use directly in a handshake + MarshalForHandshakes() ([]byte, error) - for _, g := range rc.Details.Groups { - nc.Details.InvertedGroups[g] = struct{}{} - } + // MarshalPEM will return a PEM encoded representation of this certificate + // This is primarily the format stored on disk + MarshalPEM() ([]byte, error) - return &nc, nil -} + // MarshalJSON will return the json representation of this certificate + MarshalJSON() ([]byte, error) -// UnmarshalNebulaCertificateFromPEM will unmarshal the first pem block in a byte array, returning any non consumed data -// or an error on failure -func UnmarshalNebulaCertificateFromPEM(b []byte) (*NebulaCertificate, []byte, error) { - p, r := pem.Decode(b) - if p == nil { - return nil, r, fmt.Errorf("input did not contain a valid PEM encoded block") - } - if p.Type != CertBanner { - return nil, r, fmt.Errorf("bytes did not contain a proper nebula certificate banner") - } - nc, err := UnmarshalNebulaCertificate(p.Bytes) - return nc, r, err -} + // String will return a human-readable representation of this certificate + String() string -func MarshalPrivateKey(curve Curve, b []byte) []byte { - switch curve { - case Curve_CURVE25519: - return pem.EncodeToMemory(&pem.Block{Type: X25519PrivateKeyBanner, Bytes: b}) - case Curve_P256: - return pem.EncodeToMemory(&pem.Block{Type: P256PrivateKeyBanner, Bytes: b}) - default: - return nil - } + // Copy creates a copy of the certificate + Copy() Certificate } -func MarshalSigningPrivateKey(curve Curve, b []byte) []byte { - switch curve { - case Curve_CURVE25519: - return pem.EncodeToMemory(&pem.Block{Type: Ed25519PrivateKeyBanner, Bytes: b}) - case Curve_P256: - return pem.EncodeToMemory(&pem.Block{Type: ECDSAP256PrivateKeyBanner, Bytes: b}) - default: - return nil - } -} +// CachedCertificate represents a verified certificate with some cached fields to improve +// performance. +type CachedCertificate struct { + Certificate Certificate + InvertedGroups map[string]struct{} + Fingerprint string + signerFingerprint string -// MarshalX25519PrivateKey is a simple helper to PEM encode an X25519 private key -func MarshalX25519PrivateKey(b []byte) []byte { - return pem.EncodeToMemory(&pem.Block{Type: X25519PrivateKeyBanner, Bytes: b}) + // A place to store a 2nd fingerprint if the certificate could have one, such as with P256 + fingerprint2 string } -// MarshalEd25519PrivateKey is a simple helper to PEM encode an Ed25519 private key -func MarshalEd25519PrivateKey(key ed25519.PrivateKey) []byte { - return pem.EncodeToMemory(&pem.Block{Type: Ed25519PrivateKeyBanner, Bytes: key}) +func (cc *CachedCertificate) String() string { + return cc.Certificate.String() } -func UnmarshalPrivateKey(b []byte) ([]byte, []byte, Curve, error) { - k, r := pem.Decode(b) - if k == nil { - return nil, r, 0, fmt.Errorf("input did not contain a valid PEM encoded block") - } - var expectedLen int - var curve Curve - switch k.Type { - case X25519PrivateKeyBanner: - expectedLen = 32 - curve = Curve_CURVE25519 - case P256PrivateKeyBanner: - expectedLen = 32 - curve = Curve_P256 - default: - return nil, r, 0, fmt.Errorf("bytes did not contain a proper nebula private key banner") - } - if len(k.Bytes) != expectedLen { - return nil, r, 0, fmt.Errorf("key was not %d bytes, is invalid %s private key", expectedLen, curve) +// Recombine will attempt to unmarshal a certificate received in a handshake. +// Handshakes save space by placing the peers public key in a different part of the packet, we have to +// reassemble the actual certificate structure with that in mind. +// Implementations MUST assert the public key is not in the raw certificate bytes if the passed in public key is not empty. +func Recombine(v Version, rawCertBytes, publicKey []byte, curve Curve) (Certificate, error) { + if publicKey == nil { + return nil, ErrNoPeerStaticKey } - return k.Bytes, r, curve, nil -} -func UnmarshalSigningPrivateKey(b []byte) ([]byte, []byte, Curve, error) { - k, r := pem.Decode(b) - if k == nil { - return nil, r, 0, fmt.Errorf("input did not contain a valid PEM encoded block") - } - var curve Curve - switch k.Type { - case EncryptedEd25519PrivateKeyBanner: - return nil, nil, Curve_CURVE25519, ErrPrivateKeyEncrypted - case EncryptedECDSAP256PrivateKeyBanner: - return nil, nil, Curve_P256, ErrPrivateKeyEncrypted - case Ed25519PrivateKeyBanner: - curve = Curve_CURVE25519 - if len(k.Bytes) != ed25519.PrivateKeySize { - return nil, r, 0, fmt.Errorf("key was not %d bytes, is invalid Ed25519 private key", ed25519.PrivateKeySize) - } - case ECDSAP256PrivateKeyBanner: - curve = Curve_P256 - if len(k.Bytes) != 32 { - return nil, r, 0, fmt.Errorf("key was not 32 bytes, is invalid ECDSA P256 private key") - } - default: - return nil, r, 0, fmt.Errorf("bytes did not contain a proper nebula Ed25519/ECDSA private key banner") + if rawCertBytes == nil { + return nil, ErrNoPayload } - return k.Bytes, r, curve, nil -} -// EncryptAndMarshalSigningPrivateKey is a simple helper to encrypt and PEM encode a private key -func EncryptAndMarshalSigningPrivateKey(curve Curve, b []byte, passphrase []byte, kdfParams *Argon2Parameters) ([]byte, error) { - ciphertext, err := aes256Encrypt(passphrase, kdfParams, b) - if err != nil { - return nil, err - } - - b, err = proto.Marshal(&RawNebulaEncryptedData{ - EncryptionMetadata: &RawNebulaEncryptionMetadata{ - EncryptionAlgorithm: "AES-256-GCM", - Argon2Parameters: &RawNebulaArgon2Parameters{ - Version: kdfParams.version, - Memory: kdfParams.Memory, - Parallelism: uint32(kdfParams.Parallelism), - Iterations: kdfParams.Iterations, - Salt: kdfParams.salt, - }, - }, - Ciphertext: ciphertext, - }) - if err != nil { - return nil, err - } + var c Certificate + var err error - switch curve { - case Curve_CURVE25519: - return pem.EncodeToMemory(&pem.Block{Type: EncryptedEd25519PrivateKeyBanner, Bytes: b}), nil - case Curve_P256: - return pem.EncodeToMemory(&pem.Block{Type: EncryptedECDSAP256PrivateKeyBanner, Bytes: b}), nil + switch v { + // Implementations must ensure the result is a valid cert! + case VersionPre1, Version1: + c, err = unmarshalCertificateV1(rawCertBytes, publicKey) + case Version2: + c, err = unmarshalCertificateV2(rawCertBytes, publicKey, curve) default: - return nil, fmt.Errorf("invalid curve: %v", curve) - } -} - -// UnmarshalX25519PrivateKey will try to pem decode an X25519 private key, returning any other bytes b -// or an error on failure -func UnmarshalX25519PrivateKey(b []byte) ([]byte, []byte, error) { - k, r := pem.Decode(b) - if k == nil { - return nil, r, fmt.Errorf("input did not contain a valid PEM encoded block") - } - if k.Type != X25519PrivateKeyBanner { - return nil, r, fmt.Errorf("bytes did not contain a proper nebula X25519 private key banner") - } - if len(k.Bytes) != publicKeyLen { - return nil, r, fmt.Errorf("key was not 32 bytes, is invalid X25519 private key") - } - - return k.Bytes, r, nil -} - -// UnmarshalEd25519PrivateKey will try to pem decode an Ed25519 private key, returning any other bytes b -// or an error on failure -func UnmarshalEd25519PrivateKey(b []byte) (ed25519.PrivateKey, []byte, error) { - k, r := pem.Decode(b) - if k == nil { - return nil, r, fmt.Errorf("input did not contain a valid PEM encoded block") + return nil, ErrUnknownVersion } - if k.Type == EncryptedEd25519PrivateKeyBanner { - return nil, r, ErrPrivateKeyEncrypted - } else if k.Type != Ed25519PrivateKeyBanner { - return nil, r, fmt.Errorf("bytes did not contain a proper nebula Ed25519 private key banner") - } - - if len(k.Bytes) != ed25519.PrivateKeySize { - return nil, r, fmt.Errorf("key was not 64 bytes, is invalid ed25519 private key") - } - - return k.Bytes, r, nil -} - -// UnmarshalNebulaCertificate will unmarshal a protobuf byte representation of a nebula cert into its -// protobuf-generated struct. -func UnmarshalNebulaEncryptedData(b []byte) (*NebulaEncryptedData, error) { - if len(b) == 0 { - return nil, fmt.Errorf("nil byte array") - } - var rned RawNebulaEncryptedData - err := proto.Unmarshal(b, &rned) - if err != nil { - return nil, err - } - - if rned.EncryptionMetadata == nil { - return nil, fmt.Errorf("encoded EncryptionMetadata was nil") - } - - if rned.EncryptionMetadata.Argon2Parameters == nil { - return nil, fmt.Errorf("encoded Argon2Parameters was nil") - } - - params, err := unmarshalArgon2Parameters(rned.EncryptionMetadata.Argon2Parameters) if err != nil { return nil, err } - ned := NebulaEncryptedData{ - EncryptionMetadata: NebulaEncryptionMetadata{ - EncryptionAlgorithm: rned.EncryptionMetadata.EncryptionAlgorithm, - Argon2Parameters: *params, - }, - Ciphertext: rned.Ciphertext, - } - - return &ned, nil -} - -func unmarshalArgon2Parameters(params *RawNebulaArgon2Parameters) (*Argon2Parameters, error) { - if params.Version < math.MinInt32 || params.Version > math.MaxInt32 { - return nil, fmt.Errorf("Argon2Parameters Version must be at least %d and no more than %d", math.MinInt32, math.MaxInt32) - } - if params.Memory <= 0 || params.Memory > math.MaxUint32 { - return nil, fmt.Errorf("Argon2Parameters Memory must be be greater than 0 and no more than %d KiB", uint32(math.MaxUint32)) - } - if params.Parallelism <= 0 || params.Parallelism > math.MaxUint8 { - return nil, fmt.Errorf("Argon2Parameters Parallelism must be be greater than 0 and no more than %d", math.MaxUint8) - } - if params.Iterations <= 0 || params.Iterations > math.MaxUint32 { - return nil, fmt.Errorf("-argon-iterations must be be greater than 0 and no more than %d", uint32(math.MaxUint32)) - } - - return &Argon2Parameters{ - version: rune(params.Version), - Memory: uint32(params.Memory), - Parallelism: uint8(params.Parallelism), - Iterations: uint32(params.Iterations), - salt: params.Salt, - }, nil - -} - -// DecryptAndUnmarshalSigningPrivateKey will try to pem decode and decrypt an Ed25519/ECDSA private key with -// the given passphrase, returning any other bytes b or an error on failure -func DecryptAndUnmarshalSigningPrivateKey(passphrase, b []byte) (Curve, []byte, []byte, error) { - var curve Curve - - k, r := pem.Decode(b) - if k == nil { - return curve, nil, r, fmt.Errorf("input did not contain a valid PEM encoded block") - } - - switch k.Type { - case EncryptedEd25519PrivateKeyBanner: - curve = Curve_CURVE25519 - case EncryptedECDSAP256PrivateKeyBanner: - curve = Curve_P256 - default: - return curve, nil, r, fmt.Errorf("bytes did not contain a proper nebula encrypted Ed25519/ECDSA private key banner") - } - - ned, err := UnmarshalNebulaEncryptedData(k.Bytes) - if err != nil { - return curve, nil, r, err - } - - var bytes []byte - switch ned.EncryptionMetadata.EncryptionAlgorithm { - case "AES-256-GCM": - bytes, err = aes256Decrypt(passphrase, &ned.EncryptionMetadata.Argon2Parameters, ned.Ciphertext) - if err != nil { - return curve, nil, r, err - } - default: - return curve, nil, r, fmt.Errorf("unsupported encryption algorithm: %s", ned.EncryptionMetadata.EncryptionAlgorithm) - } - - switch curve { - case Curve_CURVE25519: - if len(bytes) != ed25519.PrivateKeySize { - return curve, nil, r, fmt.Errorf("key was not %d bytes, is invalid ed25519 private key", ed25519.PrivateKeySize) - } - case Curve_P256: - if len(bytes) != 32 { - return curve, nil, r, fmt.Errorf("key was not 32 bytes, is invalid ECDSA P256 private key") - } - } - - return curve, bytes, r, nil -} - -func MarshalPublicKey(curve Curve, b []byte) []byte { - switch curve { - case Curve_CURVE25519: - return pem.EncodeToMemory(&pem.Block{Type: X25519PublicKeyBanner, Bytes: b}) - case Curve_P256: - return pem.EncodeToMemory(&pem.Block{Type: P256PublicKeyBanner, Bytes: b}) - default: - return nil - } -} - -// MarshalX25519PublicKey is a simple helper to PEM encode an X25519 public key -func MarshalX25519PublicKey(b []byte) []byte { - return pem.EncodeToMemory(&pem.Block{Type: X25519PublicKeyBanner, Bytes: b}) -} - -// MarshalEd25519PublicKey is a simple helper to PEM encode an Ed25519 public key -func MarshalEd25519PublicKey(key ed25519.PublicKey) []byte { - return pem.EncodeToMemory(&pem.Block{Type: Ed25519PublicKeyBanner, Bytes: key}) -} - -func UnmarshalPublicKey(b []byte) ([]byte, []byte, Curve, error) { - k, r := pem.Decode(b) - if k == nil { - return nil, r, 0, fmt.Errorf("input did not contain a valid PEM encoded block") - } - var expectedLen int - var curve Curve - switch k.Type { - case X25519PublicKeyBanner: - expectedLen = 32 - curve = Curve_CURVE25519 - case P256PublicKeyBanner: - // Uncompressed - expectedLen = 65 - curve = Curve_P256 - default: - return nil, r, 0, fmt.Errorf("bytes did not contain a proper nebula public key banner") - } - if len(k.Bytes) != expectedLen { - return nil, r, 0, fmt.Errorf("key was not %d bytes, is invalid %s public key", expectedLen, curve) - } - return k.Bytes, r, curve, nil -} - -// UnmarshalX25519PublicKey will try to pem decode an X25519 public key, returning any other bytes b -// or an error on failure -func UnmarshalX25519PublicKey(b []byte) ([]byte, []byte, error) { - k, r := pem.Decode(b) - if k == nil { - return nil, r, fmt.Errorf("input did not contain a valid PEM encoded block") - } - if k.Type != X25519PublicKeyBanner { - return nil, r, fmt.Errorf("bytes did not contain a proper nebula X25519 public key banner") - } - if len(k.Bytes) != publicKeyLen { - return nil, r, fmt.Errorf("key was not 32 bytes, is invalid X25519 public key") - } - - return k.Bytes, r, nil -} - -// UnmarshalEd25519PublicKey will try to pem decode an Ed25519 public key, returning any other bytes b -// or an error on failure -func UnmarshalEd25519PublicKey(b []byte) (ed25519.PublicKey, []byte, error) { - k, r := pem.Decode(b) - if k == nil { - return nil, r, fmt.Errorf("input did not contain a valid PEM encoded block") - } - if k.Type != Ed25519PublicKeyBanner { - return nil, r, fmt.Errorf("bytes did not contain a proper nebula Ed25519 public key banner") - } - if len(k.Bytes) != ed25519.PublicKeySize { - return nil, r, fmt.Errorf("key was not 32 bytes, is invalid ed25519 public key") - } - - return k.Bytes, r, nil -} - -// Sign signs a nebula cert with the provided private key -func (nc *NebulaCertificate) Sign(curve Curve, key []byte) error { - if curve != nc.Details.Curve { - return fmt.Errorf("curve in cert and private key supplied don't match") - } - - b, err := proto.Marshal(nc.getRawDetails()) - if err != nil { - return err - } - - var sig []byte - - switch curve { - case Curve_CURVE25519: - signer := ed25519.PrivateKey(key) - sig = ed25519.Sign(signer, b) - case Curve_P256: - signer := &ecdsa.PrivateKey{ - PublicKey: ecdsa.PublicKey{ - Curve: elliptic.P256(), - }, - // ref: https://github.com/golang/go/blob/go1.19/src/crypto/x509/sec1.go#L95 - D: new(big.Int).SetBytes(key), - } - // ref: https://github.com/golang/go/blob/go1.19/src/crypto/x509/sec1.go#L119 - signer.X, signer.Y = signer.Curve.ScalarBaseMult(key) - - // We need to hash first for ECDSA - // - https://pkg.go.dev/crypto/ecdsa#SignASN1 - hashed := sha256.Sum256(b) - sig, err = ecdsa.SignASN1(rand.Reader, signer, hashed[:]) - if err != nil { - return err - } - default: - return fmt.Errorf("invalid curve: %s", nc.Details.Curve) - } - - nc.Signature = sig - return nil -} - -// CheckSignature verifies the signature against the provided public key -func (nc *NebulaCertificate) CheckSignature(key []byte) bool { - b, err := proto.Marshal(nc.getRawDetails()) - if err != nil { - return false - } - switch nc.Details.Curve { - case Curve_CURVE25519: - return ed25519.Verify(ed25519.PublicKey(key), b, nc.Signature) - case Curve_P256: - x, y := elliptic.Unmarshal(elliptic.P256(), key) - pubKey := &ecdsa.PublicKey{Curve: elliptic.P256(), X: x, Y: y} - hashed := sha256.Sum256(b) - return ecdsa.VerifyASN1(pubKey, hashed[:], nc.Signature) - default: - return false - } -} - -// NOTE: This uses an internal cache that will not be invalidated automatically -// if you manually change any fields in the NebulaCertificate. -func (nc *NebulaCertificate) checkSignatureWithCache(key []byte, useCache bool) bool { - if !useCache { - return nc.CheckSignature(key) - } - - if v := nc.signatureVerified.Load(); v != nil { - return bytes.Equal(*v, key) + if c.Curve() != curve { + return nil, fmt.Errorf("certificate curve %s does not match expected %s", c.Curve().String(), curve.String()) } - verified := nc.CheckSignature(key) - if verified { - keyCopy := make([]byte, len(key)) - copy(keyCopy, key) - nc.signatureVerified.Store(&keyCopy) - } - - return verified -} - -// Expired will return true if the nebula cert is too young or too old compared to the provided time, otherwise false -func (nc *NebulaCertificate) Expired(t time.Time) bool { - return nc.Details.NotBefore.After(t) || nc.Details.NotAfter.Before(t) -} - -// Verify will ensure a certificate is good in all respects (expiry, group membership, signature, cert blocklist, etc) -func (nc *NebulaCertificate) Verify(t time.Time, ncp *NebulaCAPool) (bool, error) { - return nc.verify(t, ncp, false) -} - -// VerifyWithCache will ensure a certificate is good in all respects (expiry, group membership, signature, cert blocklist, etc) -// -// NOTE: This uses an internal cache that will not be invalidated automatically -// if you manually change any fields in the NebulaCertificate. -func (nc *NebulaCertificate) VerifyWithCache(t time.Time, ncp *NebulaCAPool) (bool, error) { - return nc.verify(t, ncp, true) + return c, nil } -// ResetCache resets the cache used by VerifyWithCache. -func (nc *NebulaCertificate) ResetCache() { - nc.sha256sum.Store(nil) - nc.signatureVerified.Store(nil) -} - -// Verify will ensure a certificate is good in all respects (expiry, group membership, signature, cert blocklist, etc) -func (nc *NebulaCertificate) verify(t time.Time, ncp *NebulaCAPool, useCache bool) (bool, error) { - if ncp.isBlocklistedWithCache(nc, useCache) { - return false, ErrBlockListed +// CalculateAlternateFingerprint calculates a 2nd fingerprint representation for P256 certificates +// CAPool blocklist testing through `VerifyCertificate` and `VerifyCachedCertificate` automatically performs this step. +func CalculateAlternateFingerprint(c Certificate) (string, error) { + if c.Curve() != Curve_P256 { + return "", nil } - signer, err := ncp.GetCAForCert(nc) + nc := c.Copy() + b, err := p256.Swap(nc.Signature()) if err != nil { - return false, err - } - - if signer.Expired(t) { - return false, ErrRootExpired - } - - if nc.Expired(t) { - return false, ErrExpired - } - - if !nc.checkSignatureWithCache(signer.Details.PublicKey, useCache) { - return false, ErrSignatureMismatch - } - - if err := nc.CheckRootConstrains(signer); err != nil { - return false, err - } - - return true, nil -} - -// CheckRootConstrains returns an error if the certificate violates constraints set on the root (groups, ips, subnets) -func (nc *NebulaCertificate) CheckRootConstrains(signer *NebulaCertificate) error { - // Make sure this cert wasn't valid before the root - if signer.Details.NotAfter.Before(nc.Details.NotAfter) { - return fmt.Errorf("certificate expires after signing certificate") - } - - // Make sure this cert isn't valid after the root - if signer.Details.NotBefore.After(nc.Details.NotBefore) { - return fmt.Errorf("certificate is valid before the signing certificate") - } - - // If the signer has a limited set of groups make sure the cert only contains a subset - if len(signer.Details.InvertedGroups) > 0 { - for _, g := range nc.Details.Groups { - if _, ok := signer.Details.InvertedGroups[g]; !ok { - return fmt.Errorf("certificate contained a group not present on the signing ca: %s", g) - } - } - } - - // If the signer has a limited set of ip ranges to issue from make sure the cert only contains a subset - if len(signer.Details.Ips) > 0 { - for _, ip := range nc.Details.Ips { - if !netMatch(ip, signer.Details.Ips) { - return fmt.Errorf("certificate contained an ip assignment outside the limitations of the signing ca: %s", ip.String()) - } - } - } - - // If the signer has a limited set of subnet ranges to issue from make sure the cert only contains a subset - if len(signer.Details.Subnets) > 0 { - for _, subnet := range nc.Details.Subnets { - if !netMatch(subnet, signer.Details.Subnets) { - return fmt.Errorf("certificate contained a subnet assignment outside the limitations of the signing ca: %s", subnet) - } - } - } - - return nil -} - -// VerifyPrivateKey checks that the public key in the Nebula certificate and a supplied private key match -func (nc *NebulaCertificate) VerifyPrivateKey(curve Curve, key []byte) error { - if curve != nc.Details.Curve { - return fmt.Errorf("curve in cert and private key supplied don't match") - } - if nc.Details.IsCA { - switch curve { - case Curve_CURVE25519: - // the call to PublicKey below will panic slice bounds out of range otherwise - if len(key) != ed25519.PrivateKeySize { - return fmt.Errorf("key was not 64 bytes, is invalid ed25519 private key") - } - - if !ed25519.PublicKey(nc.Details.PublicKey).Equal(ed25519.PrivateKey(key).Public()) { - return fmt.Errorf("public key in cert and private key supplied don't match") - } - case Curve_P256: - privkey, err := ecdh.P256().NewPrivateKey(key) - if err != nil { - return fmt.Errorf("cannot parse private key as P256") - } - pub := privkey.PublicKey().Bytes() - if !bytes.Equal(pub, nc.Details.PublicKey) { - return fmt.Errorf("public key in cert and private key supplied don't match") - } - default: - return fmt.Errorf("invalid curve: %s", curve) - } - return nil + return "", err } - var pub []byte - switch curve { - case Curve_CURVE25519: - var err error - pub, err = curve25519.X25519(key, curve25519.Basepoint) - if err != nil { - return err - } - case Curve_P256: - privkey, err := ecdh.P256().NewPrivateKey(key) - if err != nil { - return err - } - pub = privkey.PublicKey().Bytes() + switch v := nc.(type) { + case *certificateV1: + err = v.setSignature(b) + case *certificateV2: + err = v.setSignature(b) default: - return fmt.Errorf("invalid curve: %s", curve) - } - if !bytes.Equal(pub, nc.Details.PublicKey) { - return fmt.Errorf("public key in cert and private key supplied don't match") - } - - return nil -} - -// String will return a pretty printed representation of a nebula cert -func (nc *NebulaCertificate) String() string { - if nc == nil { - return "NebulaCertificate {}\n" - } - - s := "NebulaCertificate {\n" - s += "\tDetails {\n" - s += fmt.Sprintf("\t\tName: %v\n", nc.Details.Name) - - if len(nc.Details.Ips) > 0 { - s += "\t\tIps: [\n" - for _, ip := range nc.Details.Ips { - s += fmt.Sprintf("\t\t\t%v\n", ip.String()) - } - s += "\t\t]\n" - } else { - s += "\t\tIps: []\n" - } - - if len(nc.Details.Subnets) > 0 { - s += "\t\tSubnets: [\n" - for _, ip := range nc.Details.Subnets { - s += fmt.Sprintf("\t\t\t%v\n", ip.String()) - } - s += "\t\t]\n" - } else { - s += "\t\tSubnets: []\n" - } - - if len(nc.Details.Groups) > 0 { - s += "\t\tGroups: [\n" - for _, g := range nc.Details.Groups { - s += fmt.Sprintf("\t\t\t\"%v\"\n", g) - } - s += "\t\t]\n" - } else { - s += "\t\tGroups: []\n" - } - - s += fmt.Sprintf("\t\tNot before: %v\n", nc.Details.NotBefore) - s += fmt.Sprintf("\t\tNot After: %v\n", nc.Details.NotAfter) - s += fmt.Sprintf("\t\tIs CA: %v\n", nc.Details.IsCA) - s += fmt.Sprintf("\t\tIssuer: %s\n", nc.Details.Issuer) - s += fmt.Sprintf("\t\tPublic key: %x\n", nc.Details.PublicKey) - s += fmt.Sprintf("\t\tCurve: %s\n", nc.Details.Curve) - s += "\t}\n" - fp, err := nc.Sha256Sum() - if err == nil { - s += fmt.Sprintf("\tFingerprint: %s\n", fp) - } - s += fmt.Sprintf("\tSignature: %x\n", nc.Signature) - s += "}" - - return s -} - -// getRawDetails marshals the raw details into protobuf ready struct -func (nc *NebulaCertificate) getRawDetails() *RawNebulaCertificateDetails { - rd := &RawNebulaCertificateDetails{ - Name: nc.Details.Name, - Groups: nc.Details.Groups, - NotBefore: nc.Details.NotBefore.Unix(), - NotAfter: nc.Details.NotAfter.Unix(), - PublicKey: make([]byte, len(nc.Details.PublicKey)), - IsCA: nc.Details.IsCA, - Curve: nc.Details.Curve, - } - - for _, ipNet := range nc.Details.Ips { - rd.Ips = append(rd.Ips, ip2int(ipNet.IP), ip2int(ipNet.Mask)) + return "", ErrUnknownVersion } - for _, ipNet := range nc.Details.Subnets { - rd.Subnets = append(rd.Subnets, ip2int(ipNet.IP), ip2int(ipNet.Mask)) - } - - copy(rd.PublicKey, nc.Details.PublicKey[:]) - - // I know, this is terrible - rd.Issuer, _ = hex.DecodeString(nc.Details.Issuer) - - return rd -} - -// Marshal will marshal a nebula cert into a protobuf byte array -func (nc *NebulaCertificate) Marshal() ([]byte, error) { - rc := RawNebulaCertificate{ - Details: nc.getRawDetails(), - Signature: nc.Signature, - } - - return proto.Marshal(&rc) -} - -// MarshalToPEM will marshal a nebula cert into a protobuf byte array and pem encode the result -func (nc *NebulaCertificate) MarshalToPEM() ([]byte, error) { - b, err := nc.Marshal() - if err != nil { - return nil, err - } - return pem.EncodeToMemory(&pem.Block{Type: CertBanner, Bytes: b}), nil -} - -// Sha256Sum calculates a sha-256 sum of the marshaled certificate -func (nc *NebulaCertificate) Sha256Sum() (string, error) { - b, err := nc.Marshal() if err != nil { return "", err } - - sum := sha256.Sum256(b) - return hex.EncodeToString(sum[:]), nil -} - -// NOTE: This uses an internal cache that will not be invalidated automatically -// if you manually change any fields in the NebulaCertificate. -func (nc *NebulaCertificate) sha256SumWithCache(useCache bool) (string, error) { - if !useCache { - return nc.Sha256Sum() - } - - if s := nc.sha256sum.Load(); s != nil { - return *s, nil - } - s, err := nc.Sha256Sum() - if err != nil { - return s, err - } - - nc.sha256sum.Store(&s) - return s, nil -} - -func (nc *NebulaCertificate) MarshalJSON() ([]byte, error) { - toString := func(ips []*net.IPNet) []string { - s := []string{} - for _, ip := range ips { - s = append(s, ip.String()) - } - return s - } - - fp, _ := nc.Sha256Sum() - jc := m{ - "details": m{ - "name": nc.Details.Name, - "ips": toString(nc.Details.Ips), - "subnets": toString(nc.Details.Subnets), - "groups": nc.Details.Groups, - "notBefore": nc.Details.NotBefore, - "notAfter": nc.Details.NotAfter, - "publicKey": fmt.Sprintf("%x", nc.Details.PublicKey), - "isCa": nc.Details.IsCA, - "issuer": nc.Details.Issuer, - "curve": nc.Details.Curve.String(), - }, - "fingerprint": fp, - "signature": fmt.Sprintf("%x", nc.Signature), - } - return json.Marshal(jc) -} - -//func (nc *NebulaCertificate) Copy() *NebulaCertificate { -// r, err := nc.Marshal() -// if err != nil { -// //TODO -// return nil -// } -// -// c, err := UnmarshalNebulaCertificate(r) -// return c -//} - -func (nc *NebulaCertificate) Copy() *NebulaCertificate { - c := &NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: nc.Details.Name, - Groups: make([]string, len(nc.Details.Groups)), - Ips: make([]*net.IPNet, len(nc.Details.Ips)), - Subnets: make([]*net.IPNet, len(nc.Details.Subnets)), - NotBefore: nc.Details.NotBefore, - NotAfter: nc.Details.NotAfter, - PublicKey: make([]byte, len(nc.Details.PublicKey)), - IsCA: nc.Details.IsCA, - Issuer: nc.Details.Issuer, - InvertedGroups: make(map[string]struct{}, len(nc.Details.InvertedGroups)), - }, - Signature: make([]byte, len(nc.Signature)), - } - - copy(c.Signature, nc.Signature) - copy(c.Details.Groups, nc.Details.Groups) - copy(c.Details.PublicKey, nc.Details.PublicKey) - - for i, p := range nc.Details.Ips { - c.Details.Ips[i] = &net.IPNet{ - IP: make(net.IP, len(p.IP)), - Mask: make(net.IPMask, len(p.Mask)), - } - copy(c.Details.Ips[i].IP, p.IP) - copy(c.Details.Ips[i].Mask, p.Mask) - } - - for i, p := range nc.Details.Subnets { - c.Details.Subnets[i] = &net.IPNet{ - IP: make(net.IP, len(p.IP)), - Mask: make(net.IPMask, len(p.Mask)), - } - copy(c.Details.Subnets[i].IP, p.IP) - copy(c.Details.Subnets[i].Mask, p.Mask) - } - - for g := range nc.Details.InvertedGroups { - c.Details.InvertedGroups[g] = struct{}{} - } - - return c -} - -func netMatch(certIp *net.IPNet, rootIps []*net.IPNet) bool { - for _, net := range rootIps { - if net.Contains(certIp.IP) && maskContains(net.Mask, certIp.Mask) { - return true - } - } - - return false -} - -func maskContains(caMask, certMask net.IPMask) bool { - caM := maskTo4(caMask) - cM := maskTo4(certMask) - // Make sure forcing to ipv4 didn't nuke us - if caM == nil || cM == nil { - return false - } - - // Make sure the cert mask is not greater than the ca mask - for i := 0; i < len(caMask); i++ { - if caM[i] > cM[i] { - return false - } - } - - return true -} - -func maskTo4(ip net.IPMask) net.IPMask { - if len(ip) == net.IPv4len { - return ip - } - - if len(ip) == net.IPv6len && isZeros(ip[0:10]) && ip[10] == 0xff && ip[11] == 0xff { - return ip[12:16] - } - - return nil -} - -func isZeros(b []byte) bool { - for i := 0; i < len(b); i++ { - if b[i] != 0 { - return false - } - } - return true -} - -func ip2int(ip []byte) uint32 { - if len(ip) == 16 { - return binary.BigEndian.Uint32(ip[12:16]) - } - return binary.BigEndian.Uint32(ip) -} - -func int2ip(nn uint32) net.IP { - ip := make(net.IP, net.IPv4len) - binary.BigEndian.PutUint32(ip, nn) - return ip + return nc.Fingerprint() } diff --git a/cert/cert_test.go b/cert/cert_test.go deleted file mode 100644 index 30e99eca..00000000 --- a/cert/cert_test.go +++ /dev/null @@ -1,1230 +0,0 @@ -package cert - -import ( - "crypto/ecdh" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "fmt" - "io" - "net" - "testing" - "time" - - "github.com/slackhq/nebula/test" - "github.com/stretchr/testify/assert" - "golang.org/x/crypto/curve25519" - "golang.org/x/crypto/ed25519" - "google.golang.org/protobuf/proto" -) - -func TestMarshalingNebulaCertificate(t *testing.T) { - before := time.Now().Add(time.Second * -60).Round(time.Second) - after := time.Now().Add(time.Second * 60).Round(time.Second) - pubKey := []byte("1234567890abcedfghij1234567890ab") - - nc := NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "testing", - Ips: []*net.IPNet{ - {IP: net.ParseIP("10.1.1.1"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("10.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - {IP: net.ParseIP("10.1.1.3"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - }, - Subnets: []*net.IPNet{ - {IP: net.ParseIP("9.1.1.1"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - {IP: net.ParseIP("9.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("9.1.1.3"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - }, - Groups: []string{"test-group1", "test-group2", "test-group3"}, - NotBefore: before, - NotAfter: after, - PublicKey: pubKey, - IsCA: false, - Issuer: "1234567890abcedfghij1234567890ab", - }, - Signature: []byte("1234567890abcedfghij1234567890ab"), - } - - b, err := nc.Marshal() - assert.Nil(t, err) - //t.Log("Cert size:", len(b)) - - nc2, err := UnmarshalNebulaCertificate(b) - assert.Nil(t, err) - - assert.Equal(t, nc.Signature, nc2.Signature) - assert.Equal(t, nc.Details.Name, nc2.Details.Name) - assert.Equal(t, nc.Details.NotBefore, nc2.Details.NotBefore) - assert.Equal(t, nc.Details.NotAfter, nc2.Details.NotAfter) - assert.Equal(t, nc.Details.PublicKey, nc2.Details.PublicKey) - assert.Equal(t, nc.Details.IsCA, nc2.Details.IsCA) - - // IP byte arrays can be 4 or 16 in length so we have to go this route - assert.Equal(t, len(nc.Details.Ips), len(nc2.Details.Ips)) - for i, wIp := range nc.Details.Ips { - assert.Equal(t, wIp.String(), nc2.Details.Ips[i].String()) - } - - assert.Equal(t, len(nc.Details.Subnets), len(nc2.Details.Subnets)) - for i, wIp := range nc.Details.Subnets { - assert.Equal(t, wIp.String(), nc2.Details.Subnets[i].String()) - } - - assert.EqualValues(t, nc.Details.Groups, nc2.Details.Groups) -} - -func TestNebulaCertificate_Sign(t *testing.T) { - before := time.Now().Add(time.Second * -60).Round(time.Second) - after := time.Now().Add(time.Second * 60).Round(time.Second) - pubKey := []byte("1234567890abcedfghij1234567890ab") - - nc := NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "testing", - Ips: []*net.IPNet{ - {IP: net.ParseIP("10.1.1.1"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("10.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - {IP: net.ParseIP("10.1.1.3"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - }, - Subnets: []*net.IPNet{ - {IP: net.ParseIP("9.1.1.1"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - {IP: net.ParseIP("9.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("9.1.1.3"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - }, - Groups: []string{"test-group1", "test-group2", "test-group3"}, - NotBefore: before, - NotAfter: after, - PublicKey: pubKey, - IsCA: false, - Issuer: "1234567890abcedfghij1234567890ab", - }, - } - - pub, priv, err := ed25519.GenerateKey(rand.Reader) - assert.Nil(t, err) - assert.False(t, nc.CheckSignature(pub)) - assert.Nil(t, nc.Sign(Curve_CURVE25519, priv)) - assert.True(t, nc.CheckSignature(pub)) - - _, err = nc.Marshal() - assert.Nil(t, err) - //t.Log("Cert size:", len(b)) -} - -func TestNebulaCertificate_SignP256(t *testing.T) { - before := time.Now().Add(time.Second * -60).Round(time.Second) - after := time.Now().Add(time.Second * 60).Round(time.Second) - pubKey := []byte("01234567890abcedfghij1234567890ab1234567890abcedfghij1234567890ab") - - nc := NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "testing", - Ips: []*net.IPNet{ - {IP: net.ParseIP("10.1.1.1"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("10.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - {IP: net.ParseIP("10.1.1.3"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - }, - Subnets: []*net.IPNet{ - {IP: net.ParseIP("9.1.1.1"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - {IP: net.ParseIP("9.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("9.1.1.3"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - }, - Groups: []string{"test-group1", "test-group2", "test-group3"}, - NotBefore: before, - NotAfter: after, - PublicKey: pubKey, - IsCA: false, - Curve: Curve_P256, - Issuer: "1234567890abcedfghij1234567890ab", - }, - } - - priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - pub := elliptic.Marshal(elliptic.P256(), priv.PublicKey.X, priv.PublicKey.Y) - rawPriv := priv.D.FillBytes(make([]byte, 32)) - - assert.Nil(t, err) - assert.False(t, nc.CheckSignature(pub)) - assert.Nil(t, nc.Sign(Curve_P256, rawPriv)) - assert.True(t, nc.CheckSignature(pub)) - - _, err = nc.Marshal() - assert.Nil(t, err) - //t.Log("Cert size:", len(b)) -} - -func TestNebulaCertificate_Expired(t *testing.T) { - nc := NebulaCertificate{ - Details: NebulaCertificateDetails{ - NotBefore: time.Now().Add(time.Second * -60).Round(time.Second), - NotAfter: time.Now().Add(time.Second * 60).Round(time.Second), - }, - } - - assert.True(t, nc.Expired(time.Now().Add(time.Hour))) - assert.True(t, nc.Expired(time.Now().Add(-time.Hour))) - assert.False(t, nc.Expired(time.Now())) -} - -func TestNebulaCertificate_MarshalJSON(t *testing.T) { - time.Local = time.UTC - pubKey := []byte("1234567890abcedfghij1234567890ab") - - nc := NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "testing", - Ips: []*net.IPNet{ - {IP: net.ParseIP("10.1.1.1"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("10.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - {IP: net.ParseIP("10.1.1.3"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - }, - Subnets: []*net.IPNet{ - {IP: net.ParseIP("9.1.1.1"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - {IP: net.ParseIP("9.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("9.1.1.3"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - }, - Groups: []string{"test-group1", "test-group2", "test-group3"}, - NotBefore: time.Date(1, 0, 0, 1, 0, 0, 0, time.UTC), - NotAfter: time.Date(1, 0, 0, 2, 0, 0, 0, time.UTC), - PublicKey: pubKey, - IsCA: false, - Issuer: "1234567890abcedfghij1234567890ab", - }, - Signature: []byte("1234567890abcedfghij1234567890ab"), - } - - b, err := nc.MarshalJSON() - assert.Nil(t, err) - assert.Equal( - t, - "{\"details\":{\"curve\":\"CURVE25519\",\"groups\":[\"test-group1\",\"test-group2\",\"test-group3\"],\"ips\":[\"10.1.1.1/24\",\"10.1.1.2/16\",\"10.1.1.3/ff00ff00\"],\"isCa\":false,\"issuer\":\"1234567890abcedfghij1234567890ab\",\"name\":\"testing\",\"notAfter\":\"0000-11-30T02:00:00Z\",\"notBefore\":\"0000-11-30T01:00:00Z\",\"publicKey\":\"313233343536373839306162636564666768696a313233343536373839306162\",\"subnets\":[\"9.1.1.1/ff00ff00\",\"9.1.1.2/24\",\"9.1.1.3/16\"]},\"fingerprint\":\"26cb1c30ad7872c804c166b5150fa372f437aa3856b04edb4334b4470ec728e4\",\"signature\":\"313233343536373839306162636564666768696a313233343536373839306162\"}", - string(b), - ) -} - -func TestNebulaCertificate_Verify(t *testing.T) { - ca, _, caKey, err := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - - c, _, _, err := newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - - h, err := ca.Sha256Sum() - assert.Nil(t, err) - - caPool := NewCAPool() - caPool.CAs[h] = ca - - f, err := c.Sha256Sum() - assert.Nil(t, err) - caPool.BlocklistFingerprint(f) - - v, err := c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate is in the block list") - - caPool.ResetCertBlocklist() - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) - - v, err = c.Verify(time.Now().Add(time.Hour*1000), caPool) - assert.False(t, v) - assert.EqualError(t, err, "root certificate is expired") - - c, _, _, err = newTestCert(ca, caKey, time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - v, err = c.Verify(time.Now().Add(time.Minute*6), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate is expired") - - // Test group assertion - ca, _, caKey, err = newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{"test1", "test2"}) - assert.Nil(t, err) - - caPem, err := ca.MarshalToPEM() - assert.Nil(t, err) - - caPool = NewCAPool() - caPool.AddCACertificate(caPem) - - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{"test1", "bad"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained a group not present on the signing ca: bad") - - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{"test1"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) -} - -func TestNebulaCertificate_VerifyP256(t *testing.T) { - ca, _, caKey, err := newTestCaCertP256(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - - c, _, _, err := newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - - h, err := ca.Sha256Sum() - assert.Nil(t, err) - - caPool := NewCAPool() - caPool.CAs[h] = ca - - f, err := c.Sha256Sum() - assert.Nil(t, err) - caPool.BlocklistFingerprint(f) - - v, err := c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate is in the block list") - - caPool.ResetCertBlocklist() - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) - - v, err = c.Verify(time.Now().Add(time.Hour*1000), caPool) - assert.False(t, v) - assert.EqualError(t, err, "root certificate is expired") - - c, _, _, err = newTestCert(ca, caKey, time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - v, err = c.Verify(time.Now().Add(time.Minute*6), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate is expired") - - // Test group assertion - ca, _, caKey, err = newTestCaCertP256(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{"test1", "test2"}) - assert.Nil(t, err) - - caPem, err := ca.MarshalToPEM() - assert.Nil(t, err) - - caPool = NewCAPool() - caPool.AddCACertificate(caPem) - - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{"test1", "bad"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained a group not present on the signing ca: bad") - - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{"test1"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) -} - -func TestNebulaCertificate_Verify_IPs(t *testing.T) { - _, caIp1, _ := net.ParseCIDR("10.0.0.0/16") - _, caIp2, _ := net.ParseCIDR("192.168.0.0/24") - ca, _, caKey, err := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{caIp1, caIp2}, []*net.IPNet{}, []string{"test"}) - assert.Nil(t, err) - - caPem, err := ca.MarshalToPEM() - assert.Nil(t, err) - - caPool := NewCAPool() - caPool.AddCACertificate(caPem) - - // ip is outside the network - cIp1 := &net.IPNet{IP: net.ParseIP("10.1.0.0"), Mask: []byte{255, 255, 255, 0}} - cIp2 := &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 0, 0}} - c, _, _, err := newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"}) - assert.Nil(t, err) - v, err := c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained an ip assignment outside the limitations of the signing ca: 10.1.0.0/24") - - // ip is outside the network reversed order of above - cIp1 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}} - cIp2 = &net.IPNet{IP: net.ParseIP("10.1.0.0"), Mask: []byte{255, 255, 255, 0}} - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained an ip assignment outside the limitations of the signing ca: 10.1.0.0/24") - - // ip is within the network but mask is outside - cIp1 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 254, 0, 0}} - cIp2 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}} - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained an ip assignment outside the limitations of the signing ca: 10.0.1.0/15") - - // ip is within the network but mask is outside reversed order of above - cIp1 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}} - cIp2 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 254, 0, 0}} - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained an ip assignment outside the limitations of the signing ca: 10.0.1.0/15") - - // ip and mask are within the network - cIp1 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 255, 0, 0}} - cIp2 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 128}} - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{cIp1, cIp2}, []*net.IPNet{}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) - - // Exact matches - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{caIp1, caIp2}, []*net.IPNet{}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) - - // Exact matches reversed - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{caIp2, caIp1}, []*net.IPNet{}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) - - // Exact matches reversed with just 1 - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{caIp1}, []*net.IPNet{}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) -} - -func TestNebulaCertificate_Verify_Subnets(t *testing.T) { - _, caIp1, _ := net.ParseCIDR("10.0.0.0/16") - _, caIp2, _ := net.ParseCIDR("192.168.0.0/24") - ca, _, caKey, err := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{caIp1, caIp2}, []string{"test"}) - assert.Nil(t, err) - - caPem, err := ca.MarshalToPEM() - assert.Nil(t, err) - - caPool := NewCAPool() - caPool.AddCACertificate(caPem) - - // ip is outside the network - cIp1 := &net.IPNet{IP: net.ParseIP("10.1.0.0"), Mask: []byte{255, 255, 255, 0}} - cIp2 := &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 0, 0}} - c, _, _, err := newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"}) - assert.Nil(t, err) - v, err := c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained a subnet assignment outside the limitations of the signing ca: 10.1.0.0/24") - - // ip is outside the network reversed order of above - cIp1 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}} - cIp2 = &net.IPNet{IP: net.ParseIP("10.1.0.0"), Mask: []byte{255, 255, 255, 0}} - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained a subnet assignment outside the limitations of the signing ca: 10.1.0.0/24") - - // ip is within the network but mask is outside - cIp1 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 254, 0, 0}} - cIp2 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}} - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained a subnet assignment outside the limitations of the signing ca: 10.0.1.0/15") - - // ip is within the network but mask is outside reversed order of above - cIp1 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 0}} - cIp2 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 254, 0, 0}} - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.False(t, v) - assert.EqualError(t, err, "certificate contained a subnet assignment outside the limitations of the signing ca: 10.0.1.0/15") - - // ip and mask are within the network - cIp1 = &net.IPNet{IP: net.ParseIP("10.0.1.0"), Mask: []byte{255, 255, 0, 0}} - cIp2 = &net.IPNet{IP: net.ParseIP("192.168.0.1"), Mask: []byte{255, 255, 255, 128}} - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{cIp1, cIp2}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) - - // Exact matches - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{caIp1, caIp2}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) - - // Exact matches reversed - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{caIp2, caIp1}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) - - // Exact matches reversed with just 1 - c, _, _, err = newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{caIp1}, []string{"test"}) - assert.Nil(t, err) - v, err = c.Verify(time.Now(), caPool) - assert.True(t, v) - assert.Nil(t, err) -} - -func TestNebulaCertificate_VerifyPrivateKey(t *testing.T) { - ca, _, caKey, err := newTestCaCert(time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - err = ca.VerifyPrivateKey(Curve_CURVE25519, caKey) - assert.Nil(t, err) - - _, _, caKey2, err := newTestCaCert(time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - err = ca.VerifyPrivateKey(Curve_CURVE25519, caKey2) - assert.NotNil(t, err) - - c, _, priv, err := newTestCert(ca, caKey, time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{}) - err = c.VerifyPrivateKey(Curve_CURVE25519, priv) - assert.Nil(t, err) - - _, priv2 := x25519Keypair() - err = c.VerifyPrivateKey(Curve_CURVE25519, priv2) - assert.NotNil(t, err) -} - -func TestNebulaCertificate_VerifyPrivateKeyP256(t *testing.T) { - ca, _, caKey, err := newTestCaCertP256(time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - err = ca.VerifyPrivateKey(Curve_P256, caKey) - assert.Nil(t, err) - - _, _, caKey2, err := newTestCaCertP256(time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - err = ca.VerifyPrivateKey(Curve_P256, caKey2) - assert.NotNil(t, err) - - c, _, priv, err := newTestCert(ca, caKey, time.Time{}, time.Time{}, []*net.IPNet{}, []*net.IPNet{}, []string{}) - err = c.VerifyPrivateKey(Curve_P256, priv) - assert.Nil(t, err) - - _, priv2 := p256Keypair() - err = c.VerifyPrivateKey(Curve_P256, priv2) - assert.NotNil(t, err) -} - -func TestNewCAPoolFromBytes(t *testing.T) { - noNewLines := ` -# Current provisional, Remove once everything moves over to the real root. ------BEGIN NEBULA CERTIFICATE----- -CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL -vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv -bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB ------END NEBULA CERTIFICATE----- -# root-ca01 ------BEGIN NEBULA CERTIFICATE----- -CkMKEW5lYnVsYSByb290IGNhIDAxKJL2u9EFMJL86+cGOiDPXMH4oU6HZTk/CqTG -BVG+oJpAoqokUBbI4U0N8CSfpUABEkB/Pm5A2xyH/nc8mg/wvGUWG3pZ7nHzaDMf -8/phAUt+FLzqTECzQKisYswKvE3pl9mbEYKbOdIHrxdIp95mo4sF ------END NEBULA CERTIFICATE----- -` - - withNewLines := ` -# Current provisional, Remove once everything moves over to the real root. - ------BEGIN NEBULA CERTIFICATE----- -CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL -vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv -bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB ------END NEBULA CERTIFICATE----- - -# root-ca01 - - ------BEGIN NEBULA CERTIFICATE----- -CkMKEW5lYnVsYSByb290IGNhIDAxKJL2u9EFMJL86+cGOiDPXMH4oU6HZTk/CqTG -BVG+oJpAoqokUBbI4U0N8CSfpUABEkB/Pm5A2xyH/nc8mg/wvGUWG3pZ7nHzaDMf -8/phAUt+FLzqTECzQKisYswKvE3pl9mbEYKbOdIHrxdIp95mo4sF ------END NEBULA CERTIFICATE----- - -` - - expired := ` -# expired certificate ------BEGIN NEBULA CERTIFICATE----- -CjkKB2V4cGlyZWQouPmWjQYwufmWjQY6ILCRaoCkJlqHgv5jfDN4lzLHBvDzaQm4 -vZxfu144hmgjQAESQG4qlnZi8DncvD/LDZnLgJHOaX1DWCHHEh59epVsC+BNgTie -WH1M9n4O7cFtGlM6sJJOS+rCVVEJ3ABS7+MPdQs= ------END NEBULA CERTIFICATE----- -` - - p256 := ` -# p256 certificate ------BEGIN NEBULA CERTIFICATE----- -CmYKEG5lYnVsYSBQMjU2IHRlc3Qo4s+7mgYw4tXrsAc6QQRkaW2jFmllYvN4+/k2 -6tctO9sPT3jOx8ES6M1nIqOhpTmZeabF/4rELDqPV4aH5jfJut798DUXql0FlF8H -76gvQAGgBgESRzBFAiEAib0/te6eMiZOKD8gdDeloMTS0wGuX2t0C7TFdUhAQzgC -IBNWYMep3ysx9zCgknfG5dKtwGTaqF++BWKDYdyl34KX ------END NEBULA CERTIFICATE----- -` - - rootCA := NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "nebula root ca", - }, - } - - rootCA01 := NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "nebula root ca 01", - }, - } - - rootCAP256 := NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "nebula P256 test", - }, - } - - p, err := NewCAPoolFromBytes([]byte(noNewLines)) - assert.Nil(t, err) - assert.Equal(t, p.CAs[string("c9bfaf7ce8e84b2eeda2e27b469f4b9617bde192efd214b68891ecda6ed49522")].Details.Name, rootCA.Details.Name) - assert.Equal(t, p.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name) - - pp, err := NewCAPoolFromBytes([]byte(withNewLines)) - assert.Nil(t, err) - assert.Equal(t, pp.CAs[string("c9bfaf7ce8e84b2eeda2e27b469f4b9617bde192efd214b68891ecda6ed49522")].Details.Name, rootCA.Details.Name) - assert.Equal(t, pp.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name) - - // expired cert, no valid certs - ppp, err := NewCAPoolFromBytes([]byte(expired)) - assert.Equal(t, ErrExpired, err) - assert.Equal(t, ppp.CAs[string("152070be6bb19bc9e3bde4c2f0e7d8f4ff5448b4c9856b8eccb314fade0229b0")].Details.Name, "expired") - - // expired cert, with valid certs - pppp, err := NewCAPoolFromBytes(append([]byte(expired), noNewLines...)) - assert.Equal(t, ErrExpired, err) - assert.Equal(t, pppp.CAs[string("c9bfaf7ce8e84b2eeda2e27b469f4b9617bde192efd214b68891ecda6ed49522")].Details.Name, rootCA.Details.Name) - assert.Equal(t, pppp.CAs[string("5c9c3f23e7ee7fe97637cbd3a0a5b854154d1d9aaaf7b566a51f4a88f76b64cd")].Details.Name, rootCA01.Details.Name) - assert.Equal(t, pppp.CAs[string("152070be6bb19bc9e3bde4c2f0e7d8f4ff5448b4c9856b8eccb314fade0229b0")].Details.Name, "expired") - assert.Equal(t, len(pppp.CAs), 3) - - ppppp, err := NewCAPoolFromBytes([]byte(p256)) - assert.Nil(t, err) - assert.Equal(t, ppppp.CAs[string("a7938893ec8c4ef769b06d7f425e5e46f7a7f5ffa49c3bcf4a86b608caba9159")].Details.Name, rootCAP256.Details.Name) - assert.Equal(t, len(ppppp.CAs), 1) -} - -func appendByteSlices(b ...[]byte) []byte { - retSlice := []byte{} - for _, v := range b { - retSlice = append(retSlice, v...) - } - return retSlice -} - -func TestUnmrshalCertPEM(t *testing.T) { - goodCert := []byte(` -# A good cert ------BEGIN NEBULA CERTIFICATE----- -CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL -vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv -bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB ------END NEBULA CERTIFICATE----- -`) - badBanner := []byte(`# A bad banner ------BEGIN NOT A NEBULA CERTIFICATE----- -CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL -vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv -bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB ------END NOT A NEBULA CERTIFICATE----- -`) - invalidPem := []byte(`# Not a valid PEM format --BEGIN NEBULA CERTIFICATE----- -CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL -vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv -bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB --END NEBULA CERTIFICATE----`) - - certBundle := appendByteSlices(goodCert, badBanner, invalidPem) - - // Success test case - cert, rest, err := UnmarshalNebulaCertificateFromPEM(certBundle) - assert.NotNil(t, cert) - assert.Equal(t, rest, append(badBanner, invalidPem...)) - assert.Nil(t, err) - - // Fail due to invalid banner. - cert, rest, err = UnmarshalNebulaCertificateFromPEM(rest) - assert.Nil(t, cert) - assert.Equal(t, rest, invalidPem) - assert.EqualError(t, err, "bytes did not contain a proper nebula certificate banner") - - // Fail due to ivalid PEM format, because - // it's missing the requisite pre-encapsulation boundary. - cert, rest, err = UnmarshalNebulaCertificateFromPEM(rest) - assert.Nil(t, cert) - assert.Equal(t, rest, invalidPem) - assert.EqualError(t, err, "input did not contain a valid PEM encoded block") -} - -func TestUnmarshalSigningPrivateKey(t *testing.T) { - privKey := []byte(`# A good key ------BEGIN NEBULA ED25519 PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== ------END NEBULA ED25519 PRIVATE KEY----- -`) - privP256Key := []byte(`# A good key ------BEGIN NEBULA ECDSA P256 PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= ------END NEBULA ECDSA P256 PRIVATE KEY----- -`) - shortKey := []byte(`# A short key ------BEGIN NEBULA ED25519 PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA ------END NEBULA ED25519 PRIVATE KEY----- -`) - invalidBanner := []byte(`# Invalid banner ------BEGIN NOT A NEBULA PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== ------END NOT A NEBULA PRIVATE KEY----- -`) - invalidPem := []byte(`# Not a valid PEM format --BEGIN NEBULA ED25519 PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== --END NEBULA ED25519 PRIVATE KEY-----`) - - keyBundle := appendByteSlices(privKey, privP256Key, shortKey, invalidBanner, invalidPem) - - // Success test case - k, rest, curve, err := UnmarshalSigningPrivateKey(keyBundle) - assert.Len(t, k, 64) - assert.Equal(t, rest, appendByteSlices(privP256Key, shortKey, invalidBanner, invalidPem)) - assert.Equal(t, Curve_CURVE25519, curve) - assert.Nil(t, err) - - // Success test case - k, rest, curve, err = UnmarshalSigningPrivateKey(rest) - assert.Len(t, k, 32) - assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) - assert.Equal(t, Curve_P256, curve) - assert.Nil(t, err) - - // Fail due to short key - k, rest, curve, err = UnmarshalSigningPrivateKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) - assert.EqualError(t, err, "key was not 64 bytes, is invalid Ed25519 private key") - - // Fail due to invalid banner - k, rest, curve, err = UnmarshalSigningPrivateKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, invalidPem) - assert.EqualError(t, err, "bytes did not contain a proper nebula Ed25519/ECDSA private key banner") - - // Fail due to ivalid PEM format, because - // it's missing the requisite pre-encapsulation boundary. - k, rest, curve, err = UnmarshalSigningPrivateKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, invalidPem) - assert.EqualError(t, err, "input did not contain a valid PEM encoded block") -} - -func TestDecryptAndUnmarshalSigningPrivateKey(t *testing.T) { - passphrase := []byte("DO NOT USE THIS KEY") - privKey := []byte(`# A good key ------BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY----- -CjwKC0FFUy0yNTYtR0NNEi0IExCAgIABGAEgBCognnjujd67Vsv99p22wfAjQaDT -oCMW1mdjkU3gACKNW4MSXOWR9Sts4C81yk1RUku2gvGKs3TB9LYoklLsIizSYOLl -+Vs//O1T0I1Xbml2XBAROsb/VSoDln/6LMqR4B6fn6B3GOsLBBqRI8daDl9lRMPB -qrlJ69wer3ZUHFXA ------END NEBULA ED25519 ENCRYPTED PRIVATE KEY----- -`) - shortKey := []byte(`# A key which, once decrypted, is too short ------BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY----- -CjwKC0FFUy0yNTYtR0NNEi0IExCAgIABGAEgBCoga5h8owMEBWRSMMJKzuUvWce7 -k0qlBkQmCxiuLh80MuASW70YcKt8jeEIS2axo2V6zAKA9TSMcCsJW1kDDXEtL/xe -GLF5T7sDl5COp4LU3pGxpV+KoeQ/S3gQCAAcnaOtnJQX+aSDnbO3jCHyP7U9CHbs -rQr3bdH3Oy/WiYU= ------END NEBULA ED25519 ENCRYPTED PRIVATE KEY----- -`) - invalidBanner := []byte(`# Invalid banner (not encrypted) ------BEGIN NEBULA ED25519 PRIVATE KEY----- -bWRp2CTVFhW9HD/qCd28ltDgK3w8VXSeaEYczDWos8sMUBqDb9jP3+NYwcS4lURG -XgLvodMXZJuaFPssp+WwtA== ------END NEBULA ED25519 PRIVATE KEY----- -`) - invalidPem := []byte(`# Not a valid PEM format --BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY----- -CjwKC0FFUy0yNTYtR0NNEi0IExCAgIABGAEgBCognnjujd67Vsv99p22wfAjQaDT -oCMW1mdjkU3gACKNW4MSXOWR9Sts4C81yk1RUku2gvGKs3TB9LYoklLsIizSYOLl -+Vs//O1T0I1Xbml2XBAROsb/VSoDln/6LMqR4B6fn6B3GOsLBBqRI8daDl9lRMPB -qrlJ69wer3ZUHFXA --END NEBULA ED25519 ENCRYPTED PRIVATE KEY----- -`) - - keyBundle := appendByteSlices(privKey, shortKey, invalidBanner, invalidPem) - - // Success test case - curve, k, rest, err := DecryptAndUnmarshalSigningPrivateKey(passphrase, keyBundle) - assert.Nil(t, err) - assert.Equal(t, Curve_CURVE25519, curve) - assert.Len(t, k, 64) - assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) - - // Fail due to short key - curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey(passphrase, rest) - assert.EqualError(t, err, "key was not 64 bytes, is invalid ed25519 private key") - assert.Nil(t, k) - assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) - - // Fail due to invalid banner - curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey(passphrase, rest) - assert.EqualError(t, err, "bytes did not contain a proper nebula encrypted Ed25519/ECDSA private key banner") - assert.Nil(t, k) - assert.Equal(t, rest, invalidPem) - - // Fail due to ivalid PEM format, because - // it's missing the requisite pre-encapsulation boundary. - curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey(passphrase, rest) - assert.EqualError(t, err, "input did not contain a valid PEM encoded block") - assert.Nil(t, k) - assert.Equal(t, rest, invalidPem) - - // Fail due to invalid passphrase - curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey([]byte("invalid passphrase"), privKey) - assert.EqualError(t, err, "invalid passphrase or corrupt private key") - assert.Nil(t, k) - assert.Equal(t, rest, []byte{}) -} - -func TestEncryptAndMarshalSigningPrivateKey(t *testing.T) { - // Having proved that decryption works correctly above, we can test the - // encryption function produces a value which can be decrypted - passphrase := []byte("passphrase") - bytes := []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") - kdfParams := NewArgon2Parameters(64*1024, 4, 3) - key, err := EncryptAndMarshalSigningPrivateKey(Curve_CURVE25519, bytes, passphrase, kdfParams) - assert.Nil(t, err) - - // Verify the "key" can be decrypted successfully - curve, k, rest, err := DecryptAndUnmarshalSigningPrivateKey(passphrase, key) - assert.Len(t, k, 64) - assert.Equal(t, Curve_CURVE25519, curve) - assert.Equal(t, rest, []byte{}) - assert.Nil(t, err) - - // EncryptAndMarshalEd25519PrivateKey does not create any errors itself -} - -func TestUnmarshalPrivateKey(t *testing.T) { - privKey := []byte(`# A good key ------BEGIN NEBULA X25519 PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= ------END NEBULA X25519 PRIVATE KEY----- -`) - privP256Key := []byte(`# A good key ------BEGIN NEBULA P256 PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= ------END NEBULA P256 PRIVATE KEY----- -`) - shortKey := []byte(`# A short key ------BEGIN NEBULA X25519 PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== ------END NEBULA X25519 PRIVATE KEY----- -`) - invalidBanner := []byte(`# Invalid banner ------BEGIN NOT A NEBULA PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= ------END NOT A NEBULA PRIVATE KEY----- -`) - invalidPem := []byte(`# Not a valid PEM format --BEGIN NEBULA X25519 PRIVATE KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= --END NEBULA X25519 PRIVATE KEY-----`) - - keyBundle := appendByteSlices(privKey, privP256Key, shortKey, invalidBanner, invalidPem) - - // Success test case - k, rest, curve, err := UnmarshalPrivateKey(keyBundle) - assert.Len(t, k, 32) - assert.Equal(t, rest, appendByteSlices(privP256Key, shortKey, invalidBanner, invalidPem)) - assert.Equal(t, Curve_CURVE25519, curve) - assert.Nil(t, err) - - // Success test case - k, rest, curve, err = UnmarshalPrivateKey(rest) - assert.Len(t, k, 32) - assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) - assert.Equal(t, Curve_P256, curve) - assert.Nil(t, err) - - // Fail due to short key - k, rest, curve, err = UnmarshalPrivateKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) - assert.EqualError(t, err, "key was not 32 bytes, is invalid CURVE25519 private key") - - // Fail due to invalid banner - k, rest, curve, err = UnmarshalPrivateKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, invalidPem) - assert.EqualError(t, err, "bytes did not contain a proper nebula private key banner") - - // Fail due to ivalid PEM format, because - // it's missing the requisite pre-encapsulation boundary. - k, rest, curve, err = UnmarshalPrivateKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, invalidPem) - assert.EqualError(t, err, "input did not contain a valid PEM encoded block") -} - -func TestUnmarshalEd25519PublicKey(t *testing.T) { - pubKey := []byte(`# A good key ------BEGIN NEBULA ED25519 PUBLIC KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= ------END NEBULA ED25519 PUBLIC KEY----- -`) - shortKey := []byte(`# A short key ------BEGIN NEBULA ED25519 PUBLIC KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== ------END NEBULA ED25519 PUBLIC KEY----- -`) - invalidBanner := []byte(`# Invalid banner ------BEGIN NOT A NEBULA PUBLIC KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= ------END NOT A NEBULA PUBLIC KEY----- -`) - invalidPem := []byte(`# Not a valid PEM format --BEGIN NEBULA ED25519 PUBLIC KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= --END NEBULA ED25519 PUBLIC KEY-----`) - - keyBundle := appendByteSlices(pubKey, shortKey, invalidBanner, invalidPem) - - // Success test case - k, rest, err := UnmarshalEd25519PublicKey(keyBundle) - assert.Equal(t, len(k), 32) - assert.Nil(t, err) - assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) - - // Fail due to short key - k, rest, err = UnmarshalEd25519PublicKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) - assert.EqualError(t, err, "key was not 32 bytes, is invalid ed25519 public key") - - // Fail due to invalid banner - k, rest, err = UnmarshalEd25519PublicKey(rest) - assert.Nil(t, k) - assert.EqualError(t, err, "bytes did not contain a proper nebula Ed25519 public key banner") - assert.Equal(t, rest, invalidPem) - - // Fail due to ivalid PEM format, because - // it's missing the requisite pre-encapsulation boundary. - k, rest, err = UnmarshalEd25519PublicKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, invalidPem) - assert.EqualError(t, err, "input did not contain a valid PEM encoded block") -} - -func TestUnmarshalX25519PublicKey(t *testing.T) { - pubKey := []byte(`# A good key ------BEGIN NEBULA X25519 PUBLIC KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= ------END NEBULA X25519 PUBLIC KEY----- -`) - pubP256Key := []byte(`# A good key ------BEGIN NEBULA P256 PUBLIC KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAA= ------END NEBULA P256 PUBLIC KEY----- -`) - shortKey := []byte(`# A short key ------BEGIN NEBULA X25519 PUBLIC KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== ------END NEBULA X25519 PUBLIC KEY----- -`) - invalidBanner := []byte(`# Invalid banner ------BEGIN NOT A NEBULA PUBLIC KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= ------END NOT A NEBULA PUBLIC KEY----- -`) - invalidPem := []byte(`# Not a valid PEM format --BEGIN NEBULA X25519 PUBLIC KEY----- -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= --END NEBULA X25519 PUBLIC KEY-----`) - - keyBundle := appendByteSlices(pubKey, pubP256Key, shortKey, invalidBanner, invalidPem) - - // Success test case - k, rest, curve, err := UnmarshalPublicKey(keyBundle) - assert.Equal(t, len(k), 32) - assert.Nil(t, err) - assert.Equal(t, rest, appendByteSlices(pubP256Key, shortKey, invalidBanner, invalidPem)) - assert.Equal(t, Curve_CURVE25519, curve) - - // Success test case - k, rest, curve, err = UnmarshalPublicKey(rest) - assert.Equal(t, len(k), 65) - assert.Nil(t, err) - assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) - assert.Equal(t, Curve_P256, curve) - - // Fail due to short key - k, rest, curve, err = UnmarshalPublicKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) - assert.EqualError(t, err, "key was not 32 bytes, is invalid CURVE25519 public key") - - // Fail due to invalid banner - k, rest, curve, err = UnmarshalPublicKey(rest) - assert.Nil(t, k) - assert.EqualError(t, err, "bytes did not contain a proper nebula public key banner") - assert.Equal(t, rest, invalidPem) - - // Fail due to ivalid PEM format, because - // it's missing the requisite pre-encapsulation boundary. - k, rest, curve, err = UnmarshalPublicKey(rest) - assert.Nil(t, k) - assert.Equal(t, rest, invalidPem) - assert.EqualError(t, err, "input did not contain a valid PEM encoded block") -} - -// Ensure that upgrading the protobuf library does not change how certificates -// are marshalled, since this would break signature verification -func TestMarshalingNebulaCertificateConsistency(t *testing.T) { - before := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - after := time.Date(2017, time.January, 18, 28, 40, 0, 0, time.UTC) - pubKey := []byte("1234567890abcedfghij1234567890ab") - - nc := NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "testing", - Ips: []*net.IPNet{ - {IP: net.ParseIP("10.1.1.1"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("10.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - {IP: net.ParseIP("10.1.1.3"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - }, - Subnets: []*net.IPNet{ - {IP: net.ParseIP("9.1.1.1"), Mask: net.IPMask(net.ParseIP("255.0.255.0"))}, - {IP: net.ParseIP("9.1.1.2"), Mask: net.IPMask(net.ParseIP("255.255.255.0"))}, - {IP: net.ParseIP("9.1.1.3"), Mask: net.IPMask(net.ParseIP("255.255.0.0"))}, - }, - Groups: []string{"test-group1", "test-group2", "test-group3"}, - NotBefore: before, - NotAfter: after, - PublicKey: pubKey, - IsCA: false, - Issuer: "1234567890abcedfghij1234567890ab", - }, - Signature: []byte("1234567890abcedfghij1234567890ab"), - } - - b, err := nc.Marshal() - assert.Nil(t, err) - //t.Log("Cert size:", len(b)) - assert.Equal(t, "0aa2010a0774657374696e67121b8182845080feffff0f828284508080fcff0f8382845080fe83f80f1a1b8182844880fe83f80f8282844880feffff0f838284488080fcff0f220b746573742d67726f757031220b746573742d67726f757032220b746573742d67726f75703328f0e0e7d70430a08681c4053a20313233343536373839306162636564666768696a3132333435363738393061624a081234567890abcedf1220313233343536373839306162636564666768696a313233343536373839306162", fmt.Sprintf("%x", b)) - - b, err = proto.Marshal(nc.getRawDetails()) - assert.Nil(t, err) - //t.Log("Raw cert size:", len(b)) - assert.Equal(t, "0a0774657374696e67121b8182845080feffff0f828284508080fcff0f8382845080fe83f80f1a1b8182844880fe83f80f8282844880feffff0f838284488080fcff0f220b746573742d67726f757031220b746573742d67726f757032220b746573742d67726f75703328f0e0e7d70430a08681c4053a20313233343536373839306162636564666768696a3132333435363738393061624a081234567890abcedf", fmt.Sprintf("%x", b)) -} - -func TestNebulaCertificate_Copy(t *testing.T) { - ca, _, caKey, err := newTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - - c, _, _, err := newTestCert(ca, caKey, time.Now(), time.Now().Add(5*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - assert.Nil(t, err) - cc := c.Copy() - - test.AssertDeepCopyEqual(t, c, cc) -} - -func TestUnmarshalNebulaCertificate(t *testing.T) { - // Test that we don't panic with an invalid certificate (#332) - data := []byte("\x98\x00\x00") - _, err := UnmarshalNebulaCertificate(data) - assert.EqualError(t, err, "encoded Details was nil") -} - -func newTestCaCert(before, after time.Time, ips, subnets []*net.IPNet, groups []string) (*NebulaCertificate, []byte, []byte, error) { - pub, priv, err := ed25519.GenerateKey(rand.Reader) - if before.IsZero() { - before = time.Now().Add(time.Second * -60).Round(time.Second) - } - if after.IsZero() { - after = time.Now().Add(time.Second * 60).Round(time.Second) - } - - nc := &NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "test ca", - NotBefore: time.Unix(before.Unix(), 0), - NotAfter: time.Unix(after.Unix(), 0), - PublicKey: pub, - IsCA: true, - InvertedGroups: make(map[string]struct{}), - }, - } - - if len(ips) > 0 { - nc.Details.Ips = ips - } - - if len(subnets) > 0 { - nc.Details.Subnets = subnets - } - - if len(groups) > 0 { - nc.Details.Groups = groups - } - - err = nc.Sign(Curve_CURVE25519, priv) - if err != nil { - return nil, nil, nil, err - } - return nc, pub, priv, nil -} - -func newTestCaCertP256(before, after time.Time, ips, subnets []*net.IPNet, groups []string) (*NebulaCertificate, []byte, []byte, error) { - priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - pub := elliptic.Marshal(elliptic.P256(), priv.PublicKey.X, priv.PublicKey.Y) - rawPriv := priv.D.FillBytes(make([]byte, 32)) - - if before.IsZero() { - before = time.Now().Add(time.Second * -60).Round(time.Second) - } - if after.IsZero() { - after = time.Now().Add(time.Second * 60).Round(time.Second) - } - - nc := &NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "test ca", - NotBefore: time.Unix(before.Unix(), 0), - NotAfter: time.Unix(after.Unix(), 0), - PublicKey: pub, - IsCA: true, - Curve: Curve_P256, - InvertedGroups: make(map[string]struct{}), - }, - } - - if len(ips) > 0 { - nc.Details.Ips = ips - } - - if len(subnets) > 0 { - nc.Details.Subnets = subnets - } - - if len(groups) > 0 { - nc.Details.Groups = groups - } - - err = nc.Sign(Curve_P256, rawPriv) - if err != nil { - return nil, nil, nil, err - } - return nc, pub, rawPriv, nil -} - -func newTestCert(ca *NebulaCertificate, key []byte, before, after time.Time, ips, subnets []*net.IPNet, groups []string) (*NebulaCertificate, []byte, []byte, error) { - issuer, err := ca.Sha256Sum() - if err != nil { - return nil, nil, nil, err - } - - if before.IsZero() { - before = time.Now().Add(time.Second * -60).Round(time.Second) - } - if after.IsZero() { - after = time.Now().Add(time.Second * 60).Round(time.Second) - } - - if len(groups) == 0 { - groups = []string{"test-group1", "test-group2", "test-group3"} - } - - if len(ips) == 0 { - ips = []*net.IPNet{ - {IP: net.ParseIP("10.1.1.1").To4(), Mask: net.IPMask(net.ParseIP("255.255.255.0").To4())}, - {IP: net.ParseIP("10.1.1.2").To4(), Mask: net.IPMask(net.ParseIP("255.255.0.0").To4())}, - {IP: net.ParseIP("10.1.1.3").To4(), Mask: net.IPMask(net.ParseIP("255.0.255.0").To4())}, - } - } - - if len(subnets) == 0 { - subnets = []*net.IPNet{ - {IP: net.ParseIP("9.1.1.1").To4(), Mask: net.IPMask(net.ParseIP("255.0.255.0").To4())}, - {IP: net.ParseIP("9.1.1.2").To4(), Mask: net.IPMask(net.ParseIP("255.255.255.0").To4())}, - {IP: net.ParseIP("9.1.1.3").To4(), Mask: net.IPMask(net.ParseIP("255.255.0.0").To4())}, - } - } - - var pub, rawPriv []byte - - switch ca.Details.Curve { - case Curve_CURVE25519: - pub, rawPriv = x25519Keypair() - case Curve_P256: - pub, rawPriv = p256Keypair() - default: - return nil, nil, nil, fmt.Errorf("unknown curve: %v", ca.Details.Curve) - } - - nc := &NebulaCertificate{ - Details: NebulaCertificateDetails{ - Name: "testing", - Ips: ips, - Subnets: subnets, - Groups: groups, - NotBefore: time.Unix(before.Unix(), 0), - NotAfter: time.Unix(after.Unix(), 0), - PublicKey: pub, - IsCA: false, - Curve: ca.Details.Curve, - Issuer: issuer, - InvertedGroups: make(map[string]struct{}), - }, - } - - err = nc.Sign(ca.Details.Curve, key) - if err != nil { - return nil, nil, nil, err - } - - return nc, pub, rawPriv, nil -} - -func x25519Keypair() ([]byte, []byte) { - privkey := make([]byte, 32) - if _, err := io.ReadFull(rand.Reader, privkey); err != nil { - panic(err) - } - - pubkey, err := curve25519.X25519(privkey, curve25519.Basepoint) - if err != nil { - panic(err) - } - - return pubkey, privkey -} - -func p256Keypair() ([]byte, []byte) { - privkey, err := ecdh.P256().GenerateKey(rand.Reader) - if err != nil { - panic(err) - } - pubkey := privkey.PublicKey() - return pubkey.Bytes(), privkey.Bytes() -} diff --git a/cert/cert_v1.go b/cert/cert_v1.go new file mode 100644 index 00000000..4df30032 --- /dev/null +++ b/cert/cert_v1.go @@ -0,0 +1,505 @@ +package cert + +import ( + "bytes" + "crypto/ecdh" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "net" + "net/netip" + "time" + + "golang.org/x/crypto/curve25519" + "google.golang.org/protobuf/proto" +) + +const publicKeyLen = 32 + +type certificateV1 struct { + details detailsV1 + signature []byte +} + +type detailsV1 struct { + name string + networks []netip.Prefix + unsafeNetworks []netip.Prefix + groups []string + notBefore time.Time + notAfter time.Time + publicKey []byte + isCA bool + issuer string + + curve Curve +} + +type m = map[string]any + +func (c *certificateV1) Version() Version { + return Version1 +} + +func (c *certificateV1) Curve() Curve { + return c.details.curve +} + +func (c *certificateV1) Groups() []string { + return c.details.groups +} + +func (c *certificateV1) IsCA() bool { + return c.details.isCA +} + +func (c *certificateV1) Issuer() string { + return c.details.issuer +} + +func (c *certificateV1) Name() string { + return c.details.name +} + +func (c *certificateV1) Networks() []netip.Prefix { + return c.details.networks +} + +func (c *certificateV1) NotAfter() time.Time { + return c.details.notAfter +} + +func (c *certificateV1) NotBefore() time.Time { + return c.details.notBefore +} + +func (c *certificateV1) PublicKey() []byte { + return c.details.publicKey +} + +func (c *certificateV1) MarshalPublicKeyPEM() []byte { + return marshalCertPublicKeyToPEM(c) +} + +func (c *certificateV1) Signature() []byte { + return c.signature +} + +func (c *certificateV1) UnsafeNetworks() []netip.Prefix { + return c.details.unsafeNetworks +} + +func (c *certificateV1) Fingerprint() (string, error) { + b, err := c.Marshal() + if err != nil { + return "", err + } + + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]), nil +} + +func (c *certificateV1) CheckSignature(key []byte) bool { + b, err := proto.Marshal(c.getRawDetails()) + if err != nil { + return false + } + switch c.details.curve { + case Curve_CURVE25519: + if len(key) != ed25519.PublicKeySize { + return false //avoids a panic internal to ed25519 + } + return ed25519.Verify(key, b, c.signature) + case Curve_P256: + pubKey, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), key) + if err != nil { + return false + } + hashed := sha256.Sum256(b) + return ecdsa.VerifyASN1(pubKey, hashed[:], c.signature) + default: + return false + } +} + +func (c *certificateV1) Expired(t time.Time) bool { + return c.details.notBefore.After(t) || c.details.notAfter.Before(t) +} + +func (c *certificateV1) VerifyPrivateKey(curve Curve, key []byte) error { + if curve != c.details.curve { + return fmt.Errorf("curve in cert and private key supplied don't match") + } + if c.details.isCA { + switch curve { + case Curve_CURVE25519: + // the call to PublicKey below will panic slice bounds out of range otherwise + if len(key) != ed25519.PrivateKeySize { + return fmt.Errorf("key was not 64 bytes, is invalid ed25519 private key") + } + + if !ed25519.PublicKey(c.details.publicKey).Equal(ed25519.PrivateKey(key).Public()) { + return fmt.Errorf("public key in cert and private key supplied don't match") + } + case Curve_P256: + privkey, err := ecdh.P256().NewPrivateKey(key) + if err != nil { + return fmt.Errorf("cannot parse private key as P256: %w", err) + } + pub := privkey.PublicKey().Bytes() + if !bytes.Equal(pub, c.details.publicKey) { + return fmt.Errorf("public key in cert and private key supplied don't match") + } + default: + return fmt.Errorf("invalid curve: %s", curve) + } + return nil + } + + var pub []byte + switch curve { + case Curve_CURVE25519: + var err error + pub, err = curve25519.X25519(key, curve25519.Basepoint) + if err != nil { + return err + } + case Curve_P256: + privkey, err := ecdh.P256().NewPrivateKey(key) + if err != nil { + return err + } + pub = privkey.PublicKey().Bytes() + default: + return fmt.Errorf("invalid curve: %s", curve) + } + if !bytes.Equal(pub, c.details.publicKey) { + return fmt.Errorf("public key in cert and private key supplied don't match") + } + + return nil +} + +// getRawDetails marshals the raw details into protobuf ready struct +func (c *certificateV1) getRawDetails() *RawNebulaCertificateDetails { + rd := &RawNebulaCertificateDetails{ + Name: c.details.name, + Groups: c.details.groups, + NotBefore: c.details.notBefore.Unix(), + NotAfter: c.details.notAfter.Unix(), + PublicKey: make([]byte, len(c.details.publicKey)), + IsCA: c.details.isCA, + Curve: c.details.curve, + } + + for _, ipNet := range c.details.networks { + mask := net.CIDRMask(ipNet.Bits(), ipNet.Addr().BitLen()) + rd.Ips = append(rd.Ips, addr2int(ipNet.Addr()), ip2int(mask)) + } + + for _, ipNet := range c.details.unsafeNetworks { + mask := net.CIDRMask(ipNet.Bits(), ipNet.Addr().BitLen()) + rd.Subnets = append(rd.Subnets, addr2int(ipNet.Addr()), ip2int(mask)) + } + + copy(rd.PublicKey, c.details.publicKey[:]) + + // I know, this is terrible + rd.Issuer, _ = hex.DecodeString(c.details.issuer) + + return rd +} + +func (c *certificateV1) String() string { + b, err := json.MarshalIndent(c.marshalJSON(), "", "\t") + if err != nil { + return fmt.Sprintf("", err) + } + return string(b) +} + +func (c *certificateV1) MarshalForHandshakes() ([]byte, error) { + pubKey := c.details.publicKey + c.details.publicKey = nil + rawCertNoKey, err := c.Marshal() + if err != nil { + return nil, err + } + c.details.publicKey = pubKey + return rawCertNoKey, nil +} + +func (c *certificateV1) Marshal() ([]byte, error) { + rc := RawNebulaCertificate{ + Details: c.getRawDetails(), + Signature: c.signature, + } + + return proto.Marshal(&rc) +} + +func (c *certificateV1) MarshalPEM() ([]byte, error) { + b, err := c.Marshal() + if err != nil { + return nil, err + } + return pem.EncodeToMemory(&pem.Block{Type: CertificateBanner, Bytes: b}), nil +} + +func (c *certificateV1) MarshalJSON() ([]byte, error) { + return json.Marshal(c.marshalJSON()) +} + +func (c *certificateV1) marshalJSON() m { + fp, _ := c.Fingerprint() + return m{ + "version": Version1, + "details": m{ + "name": c.details.name, + "networks": c.details.networks, + "unsafeNetworks": c.details.unsafeNetworks, + "groups": c.details.groups, + "notBefore": c.details.notBefore, + "notAfter": c.details.notAfter, + "publicKey": fmt.Sprintf("%x", c.details.publicKey), + "isCa": c.details.isCA, + "issuer": c.details.issuer, + "curve": c.details.curve.String(), + }, + "fingerprint": fp, + "signature": fmt.Sprintf("%x", c.Signature()), + } +} + +func (c *certificateV1) Copy() Certificate { + nc := &certificateV1{ + details: detailsV1{ + name: c.details.name, + notBefore: c.details.notBefore, + notAfter: c.details.notAfter, + publicKey: make([]byte, len(c.details.publicKey)), + isCA: c.details.isCA, + issuer: c.details.issuer, + curve: c.details.curve, + }, + signature: make([]byte, len(c.signature)), + } + + if c.details.groups != nil { + nc.details.groups = make([]string, len(c.details.groups)) + copy(nc.details.groups, c.details.groups) + } + + if c.details.networks != nil { + nc.details.networks = make([]netip.Prefix, len(c.details.networks)) + copy(nc.details.networks, c.details.networks) + } + + if c.details.unsafeNetworks != nil { + nc.details.unsafeNetworks = make([]netip.Prefix, len(c.details.unsafeNetworks)) + copy(nc.details.unsafeNetworks, c.details.unsafeNetworks) + } + + copy(nc.signature, c.signature) + copy(nc.details.publicKey, c.details.publicKey) + + return nc +} + +func (c *certificateV1) fromTBSCertificate(t *TBSCertificate) error { + c.details = detailsV1{ + name: t.Name, + networks: t.Networks, + unsafeNetworks: t.UnsafeNetworks, + groups: t.Groups, + notBefore: t.NotBefore, + notAfter: t.NotAfter, + publicKey: t.PublicKey, + isCA: t.IsCA, + curve: t.Curve, + issuer: t.issuer, + } + + return c.validate() +} + +func (c *certificateV1) validate() error { + // Empty names are allowed + + if len(c.details.publicKey) == 0 { + return ErrInvalidPublicKey + } + + // Original v1 rules allowed multiple networks to be present but ignored all but the first one. + // Continue to allow this behavior + if !c.details.isCA && len(c.details.networks) == 0 { + return NewErrInvalidCertificateProperties("non-CA certificates must contain exactly one network") + } + + for _, network := range c.details.networks { + if !network.IsValid() || !network.Addr().IsValid() { + return NewErrInvalidCertificateProperties("invalid network: %s", network) + } + + if network.Addr().Is6() { + return NewErrInvalidCertificateProperties("certificate may not contain IPv6 networks: %v", network) + } + + if network.Addr().IsUnspecified() { + return NewErrInvalidCertificateProperties("non-CA certificates must not use the zero address as a network: %s", network) + } + + if network.Addr().Zone() != "" { + return NewErrInvalidCertificateProperties("networks may not contain zones: %s", network) + } + } + + for _, network := range c.details.unsafeNetworks { + if !network.IsValid() || !network.Addr().IsValid() { + return NewErrInvalidCertificateProperties("invalid unsafe network: %s", network) + } + + if network.Addr().Is6() { + return NewErrInvalidCertificateProperties("certificate may not contain IPv6 unsafe networks: %v", network) + } + + if network.Addr().Zone() != "" { + return NewErrInvalidCertificateProperties("unsafe networks may not contain zones: %s", network) + } + } + + // v1 doesn't bother with sort order or uniqueness of networks or unsafe networks. + // We can't modify the unmarshalled data because verification requires re-marshalling and a re-ordered + // unsafe networks would result in a different signature. + + return nil +} + +func (c *certificateV1) marshalForSigning() ([]byte, error) { + b, err := proto.Marshal(c.getRawDetails()) + if err != nil { + return nil, err + } + return b, nil +} + +func (c *certificateV1) setSignature(b []byte) error { + if len(b) == 0 { + return ErrEmptySignature + } + c.signature = b + return nil +} + +// unmarshalCertificateV1 will unmarshal a protobuf byte representation of a nebula cert +// if the publicKey is provided here then it is not required to be present in `b` +func unmarshalCertificateV1(b []byte, publicKey []byte) (*certificateV1, error) { + if len(b) == 0 { + return nil, fmt.Errorf("nil byte array") + } + var rc RawNebulaCertificate + err := proto.Unmarshal(b, &rc) + if err != nil { + return nil, err + } + + if rc.Details == nil { + return nil, fmt.Errorf("encoded Details was nil") + } + + if len(rc.Details.Ips)%2 != 0 { + return nil, fmt.Errorf("encoded IPs should be in pairs, an odd number was found") + } + + if len(rc.Details.Subnets)%2 != 0 { + return nil, fmt.Errorf("encoded Subnets should be in pairs, an odd number was found") + } + + nc := certificateV1{ + details: detailsV1{ + name: rc.Details.Name, + groups: make([]string, len(rc.Details.Groups)), + networks: make([]netip.Prefix, len(rc.Details.Ips)/2), + unsafeNetworks: make([]netip.Prefix, len(rc.Details.Subnets)/2), + notBefore: time.Unix(rc.Details.NotBefore, 0), + notAfter: time.Unix(rc.Details.NotAfter, 0), + publicKey: nil, + isCA: rc.Details.IsCA, + curve: rc.Details.Curve, + }, + signature: make([]byte, len(rc.Signature)), + } + + copy(nc.signature, rc.Signature) + copy(nc.details.groups, rc.Details.Groups) + nc.details.issuer = hex.EncodeToString(rc.Details.Issuer) + + // If a public key is passed in as an argument, the certificate pubkey must be empty + // and the passed-in pubkey copied into the cert. + if len(publicKey) > 0 { + if len(rc.Details.PublicKey) != 0 { + return nil, ErrCertPubkeyPresent + } + nc.details.publicKey = make([]byte, len(publicKey)) + copy(nc.details.publicKey, publicKey) + } else { + nc.details.publicKey = make([]byte, len(rc.Details.PublicKey)) + copy(nc.details.publicKey, rc.Details.PublicKey) + } + + var ip netip.Addr + for i, rawIp := range rc.Details.Ips { + if i%2 == 0 { + ip = int2addr(rawIp) + } else { + ones, _ := net.IPMask(int2ip(rawIp)).Size() + nc.details.networks[i/2] = netip.PrefixFrom(ip, ones) + } + } + + for i, rawIp := range rc.Details.Subnets { + if i%2 == 0 { + ip = int2addr(rawIp) + } else { + ones, _ := net.IPMask(int2ip(rawIp)).Size() + nc.details.unsafeNetworks[i/2] = netip.PrefixFrom(ip, ones) + } + } + + err = nc.validate() + if err != nil { + return nil, err + } + + return &nc, nil +} + +func ip2int(ip []byte) uint32 { + if len(ip) == 16 { + return binary.BigEndian.Uint32(ip[12:16]) + } + return binary.BigEndian.Uint32(ip) +} + +func int2ip(nn uint32) net.IP { + ip := make(net.IP, net.IPv4len) + binary.BigEndian.PutUint32(ip, nn) + return ip +} + +func addr2int(addr netip.Addr) uint32 { + b := addr.Unmap().As4() + return binary.BigEndian.Uint32(b[:]) +} + +func int2addr(nn uint32) netip.Addr { + ip := [4]byte{} + binary.BigEndian.PutUint32(ip[:], nn) + return netip.AddrFrom4(ip).Unmap() +} diff --git a/cert/cert.pb.go b/cert/cert_v1.pb.go similarity index 62% rename from cert/cert.pb.go rename to cert/cert_v1.pb.go index 3570e075..32de1a0d 100644 --- a/cert/cert.pb.go +++ b/cert/cert_v1.pb.go @@ -1,8 +1,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 +// protoc-gen-go v1.34.2 // protoc v3.21.5 -// source: cert.proto +// source: cert_v1.proto package cert @@ -50,11 +50,11 @@ func (x Curve) String() string { } func (Curve) Descriptor() protoreflect.EnumDescriptor { - return file_cert_proto_enumTypes[0].Descriptor() + return file_cert_v1_proto_enumTypes[0].Descriptor() } func (Curve) Type() protoreflect.EnumType { - return &file_cert_proto_enumTypes[0] + return &file_cert_v1_proto_enumTypes[0] } func (x Curve) Number() protoreflect.EnumNumber { @@ -63,7 +63,7 @@ func (x Curve) Number() protoreflect.EnumNumber { // Deprecated: Use Curve.Descriptor instead. func (Curve) EnumDescriptor() ([]byte, []int) { - return file_cert_proto_rawDescGZIP(), []int{0} + return file_cert_v1_proto_rawDescGZIP(), []int{0} } type RawNebulaCertificate struct { @@ -78,7 +78,7 @@ type RawNebulaCertificate struct { func (x *RawNebulaCertificate) Reset() { *x = RawNebulaCertificate{} if protoimpl.UnsafeEnabled { - mi := &file_cert_proto_msgTypes[0] + mi := &file_cert_v1_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -91,7 +91,7 @@ func (x *RawNebulaCertificate) String() string { func (*RawNebulaCertificate) ProtoMessage() {} func (x *RawNebulaCertificate) ProtoReflect() protoreflect.Message { - mi := &file_cert_proto_msgTypes[0] + mi := &file_cert_v1_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -104,7 +104,7 @@ func (x *RawNebulaCertificate) ProtoReflect() protoreflect.Message { // Deprecated: Use RawNebulaCertificate.ProtoReflect.Descriptor instead. func (*RawNebulaCertificate) Descriptor() ([]byte, []int) { - return file_cert_proto_rawDescGZIP(), []int{0} + return file_cert_v1_proto_rawDescGZIP(), []int{0} } func (x *RawNebulaCertificate) GetDetails() *RawNebulaCertificateDetails { @@ -143,7 +143,7 @@ type RawNebulaCertificateDetails struct { func (x *RawNebulaCertificateDetails) Reset() { *x = RawNebulaCertificateDetails{} if protoimpl.UnsafeEnabled { - mi := &file_cert_proto_msgTypes[1] + mi := &file_cert_v1_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -156,7 +156,7 @@ func (x *RawNebulaCertificateDetails) String() string { func (*RawNebulaCertificateDetails) ProtoMessage() {} func (x *RawNebulaCertificateDetails) ProtoReflect() protoreflect.Message { - mi := &file_cert_proto_msgTypes[1] + mi := &file_cert_v1_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -169,7 +169,7 @@ func (x *RawNebulaCertificateDetails) ProtoReflect() protoreflect.Message { // Deprecated: Use RawNebulaCertificateDetails.ProtoReflect.Descriptor instead. func (*RawNebulaCertificateDetails) Descriptor() ([]byte, []int) { - return file_cert_proto_rawDescGZIP(), []int{1} + return file_cert_v1_proto_rawDescGZIP(), []int{1} } func (x *RawNebulaCertificateDetails) GetName() string { @@ -254,7 +254,7 @@ type RawNebulaEncryptedData struct { func (x *RawNebulaEncryptedData) Reset() { *x = RawNebulaEncryptedData{} if protoimpl.UnsafeEnabled { - mi := &file_cert_proto_msgTypes[2] + mi := &file_cert_v1_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -267,7 +267,7 @@ func (x *RawNebulaEncryptedData) String() string { func (*RawNebulaEncryptedData) ProtoMessage() {} func (x *RawNebulaEncryptedData) ProtoReflect() protoreflect.Message { - mi := &file_cert_proto_msgTypes[2] + mi := &file_cert_v1_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -280,7 +280,7 @@ func (x *RawNebulaEncryptedData) ProtoReflect() protoreflect.Message { // Deprecated: Use RawNebulaEncryptedData.ProtoReflect.Descriptor instead. func (*RawNebulaEncryptedData) Descriptor() ([]byte, []int) { - return file_cert_proto_rawDescGZIP(), []int{2} + return file_cert_v1_proto_rawDescGZIP(), []int{2} } func (x *RawNebulaEncryptedData) GetEncryptionMetadata() *RawNebulaEncryptionMetadata { @@ -309,7 +309,7 @@ type RawNebulaEncryptionMetadata struct { func (x *RawNebulaEncryptionMetadata) Reset() { *x = RawNebulaEncryptionMetadata{} if protoimpl.UnsafeEnabled { - mi := &file_cert_proto_msgTypes[3] + mi := &file_cert_v1_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -322,7 +322,7 @@ func (x *RawNebulaEncryptionMetadata) String() string { func (*RawNebulaEncryptionMetadata) ProtoMessage() {} func (x *RawNebulaEncryptionMetadata) ProtoReflect() protoreflect.Message { - mi := &file_cert_proto_msgTypes[3] + mi := &file_cert_v1_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -335,7 +335,7 @@ func (x *RawNebulaEncryptionMetadata) ProtoReflect() protoreflect.Message { // Deprecated: Use RawNebulaEncryptionMetadata.ProtoReflect.Descriptor instead. func (*RawNebulaEncryptionMetadata) Descriptor() ([]byte, []int) { - return file_cert_proto_rawDescGZIP(), []int{3} + return file_cert_v1_proto_rawDescGZIP(), []int{3} } func (x *RawNebulaEncryptionMetadata) GetEncryptionAlgorithm() string { @@ -367,7 +367,7 @@ type RawNebulaArgon2Parameters struct { func (x *RawNebulaArgon2Parameters) Reset() { *x = RawNebulaArgon2Parameters{} if protoimpl.UnsafeEnabled { - mi := &file_cert_proto_msgTypes[4] + mi := &file_cert_v1_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -380,7 +380,7 @@ func (x *RawNebulaArgon2Parameters) String() string { func (*RawNebulaArgon2Parameters) ProtoMessage() {} func (x *RawNebulaArgon2Parameters) ProtoReflect() protoreflect.Message { - mi := &file_cert_proto_msgTypes[4] + mi := &file_cert_v1_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -393,7 +393,7 @@ func (x *RawNebulaArgon2Parameters) ProtoReflect() protoreflect.Message { // Deprecated: Use RawNebulaArgon2Parameters.ProtoReflect.Descriptor instead. func (*RawNebulaArgon2Parameters) Descriptor() ([]byte, []int) { - return file_cert_proto_rawDescGZIP(), []int{4} + return file_cert_v1_proto_rawDescGZIP(), []int{4} } func (x *RawNebulaArgon2Parameters) GetVersion() int32 { @@ -431,87 +431,87 @@ func (x *RawNebulaArgon2Parameters) GetSalt() []byte { return nil } -var File_cert_proto protoreflect.FileDescriptor - -var file_cert_proto_rawDesc = []byte{ - 0x0a, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x63, 0x65, - 0x72, 0x74, 0x22, 0x71, 0x0a, 0x14, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x43, - 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x3b, 0x0a, 0x07, 0x44, 0x65, - 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x65, - 0x72, 0x74, 0x2e, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x43, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x07, - 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x9c, 0x02, 0x0a, 0x1b, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, - 0x75, 0x6c, 0x61, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x44, 0x65, - 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x49, 0x70, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x03, 0x49, 0x70, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x53, - 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x53, 0x75, - 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x1c, 0x0a, - 0x09, 0x4e, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x09, 0x4e, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x4e, - 0x6f, 0x74, 0x41, 0x66, 0x74, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x4e, - 0x6f, 0x74, 0x41, 0x66, 0x74, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x75, 0x62, 0x6c, 0x69, - 0x63, 0x4b, 0x65, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x50, 0x75, 0x62, 0x6c, - 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x73, 0x43, 0x41, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x73, 0x43, 0x41, 0x12, 0x16, 0x0a, 0x06, 0x49, 0x73, 0x73, - 0x75, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x49, 0x73, 0x73, 0x75, 0x65, - 0x72, 0x12, 0x21, 0x0a, 0x05, 0x63, 0x75, 0x72, 0x76, 0x65, 0x18, 0x64, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x0b, 0x2e, 0x63, 0x65, 0x72, 0x74, 0x2e, 0x43, 0x75, 0x72, 0x76, 0x65, 0x52, 0x05, 0x63, - 0x75, 0x72, 0x76, 0x65, 0x22, 0x8b, 0x01, 0x0a, 0x16, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, - 0x6c, 0x61, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x12, - 0x51, 0x0a, 0x12, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x65, - 0x72, 0x74, 0x2e, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x45, 0x6e, 0x63, 0x72, - 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x12, - 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, 0x78, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, 0x65, - 0x78, 0x74, 0x22, 0x9c, 0x01, 0x0a, 0x1b, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, - 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x12, 0x30, 0x0a, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, 0x67, 0x6f, 0x72, - 0x69, 0x74, 0x68, 0x6d, 0x12, 0x4b, 0x0a, 0x10, 0x41, 0x72, 0x67, 0x6f, 0x6e, 0x32, 0x50, 0x61, - 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, - 0x2e, 0x63, 0x65, 0x72, 0x74, 0x2e, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x41, - 0x72, 0x67, 0x6f, 0x6e, 0x32, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, - 0x10, 0x41, 0x72, 0x67, 0x6f, 0x6e, 0x32, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, - 0x73, 0x22, 0xa3, 0x01, 0x0a, 0x19, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x41, - 0x72, 0x67, 0x6f, 0x6e, 0x32, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, - 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x6d, - 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, - 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, - 0x69, 0x73, 0x6d, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x2a, 0x21, 0x0a, 0x05, 0x43, 0x75, 0x72, 0x76, 0x65, - 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x55, 0x52, 0x56, 0x45, 0x32, 0x35, 0x35, 0x31, 0x39, 0x10, 0x00, - 0x12, 0x08, 0x0a, 0x04, 0x50, 0x32, 0x35, 0x36, 0x10, 0x01, 0x42, 0x20, 0x5a, 0x1e, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x68, 0x71, - 0x2f, 0x6e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x2f, 0x63, 0x65, 0x72, 0x74, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, +var File_cert_v1_proto protoreflect.FileDescriptor + +var file_cert_v1_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x76, 0x31, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x04, 0x63, 0x65, 0x72, 0x74, 0x22, 0x71, 0x0a, 0x14, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, + 0x6c, 0x61, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x3b, 0x0a, + 0x07, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x2e, 0x63, 0x65, 0x72, 0x74, 0x2e, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x43, + 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x73, 0x52, 0x07, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x9c, 0x02, 0x0a, 0x1b, 0x52, 0x61, 0x77, + 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x49, 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x03, 0x49, 0x70, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, + 0x07, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, + 0x12, 0x1c, 0x0a, 0x09, 0x4e, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x09, 0x4e, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x12, 0x1a, + 0x0a, 0x08, 0x4e, 0x6f, 0x74, 0x41, 0x66, 0x74, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x08, 0x4e, 0x6f, 0x74, 0x41, 0x66, 0x74, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x75, + 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x50, + 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x73, 0x43, 0x41, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x73, 0x43, 0x41, 0x12, 0x16, 0x0a, 0x06, + 0x49, 0x73, 0x73, 0x75, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x49, 0x73, + 0x73, 0x75, 0x65, 0x72, 0x12, 0x21, 0x0a, 0x05, 0x63, 0x75, 0x72, 0x76, 0x65, 0x18, 0x64, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x63, 0x65, 0x72, 0x74, 0x2e, 0x43, 0x75, 0x72, 0x76, 0x65, + 0x52, 0x05, 0x63, 0x75, 0x72, 0x76, 0x65, 0x22, 0x8b, 0x01, 0x0a, 0x16, 0x52, 0x61, 0x77, 0x4e, + 0x65, 0x62, 0x75, 0x6c, 0x61, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x44, 0x61, + 0x74, 0x61, 0x12, 0x51, 0x0a, 0x12, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x2e, 0x63, 0x65, 0x72, 0x74, 0x2e, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x45, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x52, 0x12, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x74, + 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x43, 0x69, 0x70, 0x68, 0x65, + 0x72, 0x74, 0x65, 0x78, 0x74, 0x22, 0x9c, 0x01, 0x0a, 0x1b, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, + 0x75, 0x6c, 0x61, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x30, 0x0a, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, + 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x4b, 0x0a, 0x10, 0x41, 0x72, 0x67, 0x6f, 0x6e, + 0x32, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x65, 0x72, 0x74, 0x2e, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, + 0x6c, 0x61, 0x41, 0x72, 0x67, 0x6f, 0x6e, 0x32, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, + 0x72, 0x73, 0x52, 0x10, 0x41, 0x72, 0x67, 0x6f, 0x6e, 0x32, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, + 0x74, 0x65, 0x72, 0x73, 0x22, 0xa3, 0x01, 0x0a, 0x19, 0x52, 0x61, 0x77, 0x4e, 0x65, 0x62, 0x75, + 0x6c, 0x61, 0x41, 0x72, 0x67, 0x6f, 0x6e, 0x32, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, + 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, + 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6d, 0x65, + 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, + 0x69, 0x73, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x61, 0x6c, + 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x74, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x69, 0x74, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x2a, 0x21, 0x0a, 0x05, 0x43, 0x75, + 0x72, 0x76, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x55, 0x52, 0x56, 0x45, 0x32, 0x35, 0x35, 0x31, + 0x39, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x50, 0x32, 0x35, 0x36, 0x10, 0x01, 0x42, 0x20, 0x5a, + 0x1e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x6c, 0x61, 0x63, + 0x6b, 0x68, 0x71, 0x2f, 0x6e, 0x65, 0x62, 0x75, 0x6c, 0x61, 0x2f, 0x63, 0x65, 0x72, 0x74, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( - file_cert_proto_rawDescOnce sync.Once - file_cert_proto_rawDescData = file_cert_proto_rawDesc + file_cert_v1_proto_rawDescOnce sync.Once + file_cert_v1_proto_rawDescData = file_cert_v1_proto_rawDesc ) -func file_cert_proto_rawDescGZIP() []byte { - file_cert_proto_rawDescOnce.Do(func() { - file_cert_proto_rawDescData = protoimpl.X.CompressGZIP(file_cert_proto_rawDescData) +func file_cert_v1_proto_rawDescGZIP() []byte { + file_cert_v1_proto_rawDescOnce.Do(func() { + file_cert_v1_proto_rawDescData = protoimpl.X.CompressGZIP(file_cert_v1_proto_rawDescData) }) - return file_cert_proto_rawDescData + return file_cert_v1_proto_rawDescData } -var file_cert_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_cert_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_cert_proto_goTypes = []interface{}{ +var file_cert_v1_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_cert_v1_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_cert_v1_proto_goTypes = []any{ (Curve)(0), // 0: cert.Curve (*RawNebulaCertificate)(nil), // 1: cert.RawNebulaCertificate (*RawNebulaCertificateDetails)(nil), // 2: cert.RawNebulaCertificateDetails @@ -519,7 +519,7 @@ var file_cert_proto_goTypes = []interface{}{ (*RawNebulaEncryptionMetadata)(nil), // 4: cert.RawNebulaEncryptionMetadata (*RawNebulaArgon2Parameters)(nil), // 5: cert.RawNebulaArgon2Parameters } -var file_cert_proto_depIdxs = []int32{ +var file_cert_v1_proto_depIdxs = []int32{ 2, // 0: cert.RawNebulaCertificate.Details:type_name -> cert.RawNebulaCertificateDetails 0, // 1: cert.RawNebulaCertificateDetails.curve:type_name -> cert.Curve 4, // 2: cert.RawNebulaEncryptedData.EncryptionMetadata:type_name -> cert.RawNebulaEncryptionMetadata @@ -531,13 +531,13 @@ var file_cert_proto_depIdxs = []int32{ 0, // [0:4] is the sub-list for field type_name } -func init() { file_cert_proto_init() } -func file_cert_proto_init() { - if File_cert_proto != nil { +func init() { file_cert_v1_proto_init() } +func file_cert_v1_proto_init() { + if File_cert_v1_proto != nil { return } if !protoimpl.UnsafeEnabled { - file_cert_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_cert_v1_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*RawNebulaCertificate); i { case 0: return &v.state @@ -549,7 +549,7 @@ func file_cert_proto_init() { return nil } } - file_cert_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_cert_v1_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*RawNebulaCertificateDetails); i { case 0: return &v.state @@ -561,7 +561,7 @@ func file_cert_proto_init() { return nil } } - file_cert_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_cert_v1_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*RawNebulaEncryptedData); i { case 0: return &v.state @@ -573,7 +573,7 @@ func file_cert_proto_init() { return nil } } - file_cert_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_cert_v1_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*RawNebulaEncryptionMetadata); i { case 0: return &v.state @@ -585,7 +585,7 @@ func file_cert_proto_init() { return nil } } - file_cert_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_cert_v1_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*RawNebulaArgon2Parameters); i { case 0: return &v.state @@ -602,19 +602,19 @@ func file_cert_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_cert_proto_rawDesc, + RawDescriptor: file_cert_v1_proto_rawDesc, NumEnums: 1, NumMessages: 5, NumExtensions: 0, NumServices: 0, }, - GoTypes: file_cert_proto_goTypes, - DependencyIndexes: file_cert_proto_depIdxs, - EnumInfos: file_cert_proto_enumTypes, - MessageInfos: file_cert_proto_msgTypes, + GoTypes: file_cert_v1_proto_goTypes, + DependencyIndexes: file_cert_v1_proto_depIdxs, + EnumInfos: file_cert_v1_proto_enumTypes, + MessageInfos: file_cert_v1_proto_msgTypes, }.Build() - File_cert_proto = out.File - file_cert_proto_rawDesc = nil - file_cert_proto_goTypes = nil - file_cert_proto_depIdxs = nil + File_cert_v1_proto = out.File + file_cert_v1_proto_rawDesc = nil + file_cert_v1_proto_goTypes = nil + file_cert_v1_proto_depIdxs = nil } diff --git a/cert/cert.proto b/cert/cert_v1.proto similarity index 100% rename from cert/cert.proto rename to cert/cert_v1.proto diff --git a/cert/cert_v1_test.go b/cert/cert_v1_test.go new file mode 100644 index 00000000..6e374634 --- /dev/null +++ b/cert/cert_v1_test.go @@ -0,0 +1,334 @@ +package cert + +import ( + "crypto/ed25519" + "fmt" + "net/netip" + "testing" + "time" + + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestCertificateV1_Marshal(t *testing.T) { + t.Parallel() + before := time.Now().Add(time.Second * -60).Round(time.Second) + after := time.Now().Add(time.Second * 60).Round(time.Second) + pubKey := []byte("1234567890abcedfghij1234567890ab") + + nc := certificateV1{ + details: detailsV1{ + name: "testing", + networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.1/24"), + mustParsePrefixUnmapped("10.1.1.2/16"), + }, + unsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.2/24"), + mustParsePrefixUnmapped("9.1.1.3/16"), + }, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: before, + notAfter: after, + publicKey: pubKey, + isCA: false, + issuer: "1234567890abcedfghij1234567890ab", + }, + signature: []byte("1234567890abcedfghij1234567890ab"), + } + + b, err := nc.Marshal() + require.NoError(t, err) + //t.Log("Cert size:", len(b)) + + nc2, err := unmarshalCertificateV1(b, nil) + require.NoError(t, err) + + assert.Equal(t, Version1, nc.Version()) + assert.Equal(t, Curve_CURVE25519, nc.Curve()) + assert.Equal(t, nc.Signature(), nc2.Signature()) + assert.Equal(t, nc.Name(), nc2.Name()) + assert.Equal(t, nc.NotBefore(), nc2.NotBefore()) + assert.Equal(t, nc.NotAfter(), nc2.NotAfter()) + assert.Equal(t, nc.PublicKey(), nc2.PublicKey()) + assert.Equal(t, nc.IsCA(), nc2.IsCA()) + + assert.Equal(t, nc.Networks(), nc2.Networks()) + assert.Equal(t, nc.UnsafeNetworks(), nc2.UnsafeNetworks()) + + assert.Equal(t, nc.Groups(), nc2.Groups()) +} + +func TestCertificateV1_Unmarshal(t *testing.T) { + t.Parallel() + before := time.Now().Add(time.Second * -60).Round(time.Second) + after := time.Now().Add(time.Second * 60).Round(time.Second) + pubKey := []byte("1234567890abcedfghij1234567890ab") + invalidPubkey := []byte("00000000000000000000000000000000") + + nc := certificateV1{ + details: detailsV1{ + name: "testing", + networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.1/24"), + mustParsePrefixUnmapped("10.1.1.2/16"), + }, + unsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.2/24"), + mustParsePrefixUnmapped("9.1.1.3/16"), + }, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: before, + notAfter: after, + publicKey: pubKey, + isCA: false, + issuer: "1234567890abcedfghij1234567890ab", + }, + signature: []byte("1234567890abcedfghij1234567890ab"), + } + + // This certificate has a pubkey included + certWithPubkey, err := nc.Marshal() + require.NoError(t, err) + + // This certificate is missing the pubkey section + certWithoutPubkey, err := nc.MarshalForHandshakes() + require.NoError(t, err) + + // Cert has no pubkey and no pubkey passed in must fail to validate + isNil, err := unmarshalCertificateV1(certWithoutPubkey, nil) + require.Error(t, err) + + // Cert has different pubkey than one passed in must fail + isNil, err = unmarshalCertificateV1(certWithPubkey, invalidPubkey) + require.Nil(t, isNil) + require.Error(t, err) + + // Cert has pubkey and no pubkey argument works ok + _, err = unmarshalCertificateV1(certWithPubkey, nil) + require.NoError(t, err) + + // Cert has no pubkey and valid, correctly signed pubkey passed in + nc2, err := unmarshalCertificateV1(certWithoutPubkey, pubKey) + require.NoError(t, err) + + assert.Equal(t, pubKey, nc2.PublicKey()) +} + +func TestCertificateV1_PublicKeyPem(t *testing.T) { + t.Parallel() + before := time.Now().Add(time.Second * -60).Round(time.Second) + after := time.Now().Add(time.Second * 60).Round(time.Second) + pubKey := ed25519.PublicKey("1234567890abcedfghij1234567890ab") + + nc := certificateV1{ + details: detailsV1{ + name: "testing", + networks: []netip.Prefix{}, + unsafeNetworks: []netip.Prefix{}, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: before, + notAfter: after, + publicKey: pubKey, + isCA: false, + issuer: "1234567890abcedfghij1234567890ab", + }, + signature: []byte("1234567890abcedfghij1234567890ab"), + } + + assert.Equal(t, Version1, nc.Version()) + assert.Equal(t, Curve_CURVE25519, nc.Curve()) + pubPem := "-----BEGIN NEBULA X25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA X25519 PUBLIC KEY-----\n" + assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem) + assert.False(t, nc.IsCA()) + + nc.details.isCA = true + assert.Equal(t, Curve_CURVE25519, nc.Curve()) + pubPem = "-----BEGIN NEBULA ED25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA ED25519 PUBLIC KEY-----\n" + assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem) + assert.True(t, nc.IsCA()) + + pubP256KeyPem := []byte(`-----BEGIN NEBULA P256 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA P256 PUBLIC KEY----- +`) + + pubP256KeyPemCA := []byte(`-----BEGIN NEBULA ECDSA P256 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA ECDSA P256 PUBLIC KEY----- +`) + pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem) + require.NoError(t, err) + nc.details.curve = Curve_P256 + nc.details.publicKey = pubP256Key + assert.Equal(t, Curve_P256, nc.Curve()) + assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPemCA)) + assert.True(t, nc.IsCA()) + + nc.details.isCA = false + assert.Equal(t, Curve_P256, nc.Curve()) + assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem)) + assert.False(t, nc.IsCA()) +} + +func TestCertificateV1_Expired(t *testing.T) { + nc := certificateV1{ + details: detailsV1{ + notBefore: time.Now().Add(time.Second * -60).Round(time.Second), + notAfter: time.Now().Add(time.Second * 60).Round(time.Second), + }, + } + + assert.True(t, nc.Expired(time.Now().Add(time.Hour))) + assert.True(t, nc.Expired(time.Now().Add(-time.Hour))) + assert.False(t, nc.Expired(time.Now())) +} + +func TestCertificateV1_MarshalJSON(t *testing.T) { + time.Local = time.UTC + pubKey := []byte("1234567890abcedfghij1234567890ab") + + nc := certificateV1{ + details: detailsV1{ + name: "testing", + networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.1/24"), + mustParsePrefixUnmapped("10.1.1.2/16"), + }, + unsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.2/24"), + mustParsePrefixUnmapped("9.1.1.3/16"), + }, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: time.Date(1, 0, 0, 1, 0, 0, 0, time.UTC), + notAfter: time.Date(1, 0, 0, 2, 0, 0, 0, time.UTC), + publicKey: pubKey, + isCA: false, + issuer: "1234567890abcedfghij1234567890ab", + }, + signature: []byte("1234567890abcedfghij1234567890ab"), + } + + b, err := nc.MarshalJSON() + require.NoError(t, err) + assert.JSONEq( + t, + "{\"details\":{\"curve\":\"CURVE25519\",\"groups\":[\"test-group1\",\"test-group2\",\"test-group3\"],\"isCa\":false,\"issuer\":\"1234567890abcedfghij1234567890ab\",\"name\":\"testing\",\"networks\":[\"10.1.1.1/24\",\"10.1.1.2/16\"],\"notAfter\":\"0000-11-30T02:00:00Z\",\"notBefore\":\"0000-11-30T01:00:00Z\",\"publicKey\":\"313233343536373839306162636564666768696a313233343536373839306162\",\"unsafeNetworks\":[\"9.1.1.2/24\",\"9.1.1.3/16\"]},\"fingerprint\":\"3944c53d4267a229295b56cb2d27d459164c010ac97d655063ba421e0670f4ba\",\"signature\":\"313233343536373839306162636564666768696a313233343536373839306162\",\"version\":1}", + string(b), + ) +} + +func TestCertificateV1_VerifyPrivateKey(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil) + err := ca.VerifyPrivateKey(Curve_CURVE25519, caKey) + require.NoError(t, err) + + _, _, caKey2, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil) + require.NoError(t, err) + err = ca.VerifyPrivateKey(Curve_CURVE25519, caKey2) + require.Error(t, err) + + c, _, priv, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Time{}, time.Time{}, nil, nil, nil) + rawPriv, b, curve, err := UnmarshalPrivateKeyFromPEM(priv) + require.NoError(t, err) + assert.Empty(t, b) + assert.Equal(t, Curve_CURVE25519, curve) + err = c.VerifyPrivateKey(Curve_CURVE25519, rawPriv) + require.NoError(t, err) + + _, priv2 := X25519Keypair() + err = c.VerifyPrivateKey(Curve_CURVE25519, priv2) + require.Error(t, err) +} + +func TestCertificateV1_VerifyPrivateKeyP256(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version1, Curve_P256, time.Time{}, time.Time{}, nil, nil, nil) + err := ca.VerifyPrivateKey(Curve_P256, caKey) + require.NoError(t, err) + + _, _, caKey2, _ := NewTestCaCert(Version1, Curve_P256, time.Time{}, time.Time{}, nil, nil, nil) + require.NoError(t, err) + err = ca.VerifyPrivateKey(Curve_P256, caKey2) + require.Error(t, err) + + c, _, priv, _ := NewTestCert(Version1, Curve_P256, ca, caKey, "test", time.Time{}, time.Time{}, nil, nil, nil) + rawPriv, b, curve, err := UnmarshalPrivateKeyFromPEM(priv) + require.NoError(t, err) + assert.Empty(t, b) + assert.Equal(t, Curve_P256, curve) + err = c.VerifyPrivateKey(Curve_P256, rawPriv) + require.NoError(t, err) + + _, priv2 := P256Keypair() + err = c.VerifyPrivateKey(Curve_P256, priv2) + require.Error(t, err) +} + +// Ensure that upgrading the protobuf library does not change how certificates +// are marshalled, since this would break signature verification +func TestMarshalingCertificateV1Consistency(t *testing.T) { + before := time.Date(1970, time.January, 1, 1, 1, 1, 1, time.UTC) + after := time.Date(9999, time.January, 1, 1, 1, 1, 1, time.UTC) + pubKey := []byte("1234567890abcedfghij1234567890ab") + + nc := certificateV1{ + details: detailsV1{ + name: "testing", + networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.2/16"), + mustParsePrefixUnmapped("10.1.1.1/24"), + }, + unsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.3/16"), + mustParsePrefixUnmapped("9.1.1.2/24"), + }, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: before, + notAfter: after, + publicKey: pubKey, + isCA: false, + issuer: "1234567890abcedfghij1234567890ab", + }, + signature: []byte("1234567890abcedfghij1234567890ab"), + } + + b, err := nc.Marshal() + require.NoError(t, err) + assert.Equal(t, "0a8e010a0774657374696e671212828284508080fcff0f8182845080feffff0f1a12838284488080fcff0f8282844880feffff0f220b746573742d67726f757031220b746573742d67726f757032220b746573742d67726f75703328cd1c30cdb8ccf0af073a20313233343536373839306162636564666768696a3132333435363738393061624a081234567890abcedf1220313233343536373839306162636564666768696a313233343536373839306162", fmt.Sprintf("%x", b)) + + b, err = proto.Marshal(nc.getRawDetails()) + require.NoError(t, err) + assert.Equal(t, "0a0774657374696e671212828284508080fcff0f8182845080feffff0f1a12838284488080fcff0f8282844880feffff0f220b746573742d67726f757031220b746573742d67726f757032220b746573742d67726f75703328cd1c30cdb8ccf0af073a20313233343536373839306162636564666768696a3132333435363738393061624a081234567890abcedf", fmt.Sprintf("%x", b)) +} + +func TestCertificateV1_Copy(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil) + c, _, _, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil) + cc := c.Copy() + test.AssertDeepCopyEqual(t, c, cc) +} + +func TestUnmarshalCertificateV1(t *testing.T) { + // Test that we don't panic with an invalid certificate (#332) + data := []byte("\x98\x00\x00") + _, err := unmarshalCertificateV1(data, nil) + require.EqualError(t, err, "encoded Details was nil") +} + +func appendByteSlices(b ...[]byte) []byte { + retSlice := []byte{} + for _, v := range b { + retSlice = append(retSlice, v...) + } + return retSlice +} + +func mustParsePrefixUnmapped(s string) netip.Prefix { + prefix := netip.MustParsePrefix(s) + return netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()) +} diff --git a/cert/cert_v2.asn1 b/cert/cert_v2.asn1 new file mode 100644 index 00000000..f863133f --- /dev/null +++ b/cert/cert_v2.asn1 @@ -0,0 +1,37 @@ +Nebula DEFINITIONS AUTOMATIC TAGS ::= BEGIN + +Name ::= UTF8String (SIZE (1..253)) +Time ::= INTEGER (0..18446744073709551615) -- Seconds since unix epoch, uint64 maximum +Network ::= OCTET STRING (SIZE (5,17)) -- IP addresses are 4 or 16 bytes + 1 byte for the prefix length +Curve ::= ENUMERATED { + curve25519 (0), + p256 (1) +} + +-- The maximum size of a certificate must not exceed 65536 bytes +Certificate ::= SEQUENCE { + details OCTET STRING, + curve Curve DEFAULT curve25519, + publicKey OCTET STRING, + -- signature(details + curve + publicKey) using the appropriate method for curve + signature OCTET STRING +} + +Details ::= SEQUENCE { + name Name, + + -- At least 1 ipv4 or ipv6 address must be present if isCA is false + networks SEQUENCE OF Network OPTIONAL, + unsafeNetworks SEQUENCE OF Network OPTIONAL, + groups SEQUENCE OF Name OPTIONAL, + isCA BOOLEAN DEFAULT false, + notBefore Time, + notAfter Time, + + -- issuer is only required if isCA is false, if isCA is true then it must not be present + issuer OCTET STRING OPTIONAL, + ... + -- New fields can be added below here +} + +END \ No newline at end of file diff --git a/cert/cert_v2.go b/cert/cert_v2.go new file mode 100644 index 00000000..c2b43a69 --- /dev/null +++ b/cert/cert_v2.go @@ -0,0 +1,745 @@ +package cert + +import ( + "bytes" + "crypto/ecdh" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "net/netip" + "slices" + "time" + + "golang.org/x/crypto/cryptobyte" + "golang.org/x/crypto/cryptobyte/asn1" + "golang.org/x/crypto/curve25519" +) + +const ( + classConstructed = 0x20 + classContextSpecific = 0x80 + + TagCertDetails = 0 | classConstructed | classContextSpecific + TagCertCurve = 1 | classContextSpecific + TagCertPublicKey = 2 | classContextSpecific + TagCertSignature = 3 | classContextSpecific + + TagDetailsName = 0 | classContextSpecific + TagDetailsNetworks = 1 | classConstructed | classContextSpecific + TagDetailsUnsafeNetworks = 2 | classConstructed | classContextSpecific + TagDetailsGroups = 3 | classConstructed | classContextSpecific + TagDetailsIsCA = 4 | classContextSpecific + TagDetailsNotBefore = 5 | classContextSpecific + TagDetailsNotAfter = 6 | classContextSpecific + TagDetailsIssuer = 7 | classContextSpecific +) + +const ( + // MaxCertificateSize is the maximum length a valid certificate can be + MaxCertificateSize = 65536 + + // MaxNameLength is limited to a maximum realistic DNS domain name to help facilitate DNS systems + MaxNameLength = 253 + + // MaxNetworkLength is the maximum length a network value can be. + // 16 bytes for an ipv6 address + 1 byte for the prefix length + MaxNetworkLength = 17 +) + +type certificateV2 struct { + details detailsV2 + + // RawDetails contains the entire asn.1 DER encoded Details struct + // This is to benefit forwards compatibility in signature checking. + // signature(RawDetails + Curve + PublicKey) == Signature + rawDetails []byte + curve Curve + publicKey []byte + signature []byte +} + +type detailsV2 struct { + name string + networks []netip.Prefix // MUST BE SORTED + unsafeNetworks []netip.Prefix // MUST BE SORTED + groups []string + isCA bool + notBefore time.Time + notAfter time.Time + issuer string +} + +func (c *certificateV2) Version() Version { + return Version2 +} + +func (c *certificateV2) Curve() Curve { + return c.curve +} + +func (c *certificateV2) Groups() []string { + return c.details.groups +} + +func (c *certificateV2) IsCA() bool { + return c.details.isCA +} + +func (c *certificateV2) Issuer() string { + return c.details.issuer +} + +func (c *certificateV2) Name() string { + return c.details.name +} + +func (c *certificateV2) Networks() []netip.Prefix { + return c.details.networks +} + +func (c *certificateV2) NotAfter() time.Time { + return c.details.notAfter +} + +func (c *certificateV2) NotBefore() time.Time { + return c.details.notBefore +} + +func (c *certificateV2) PublicKey() []byte { + return c.publicKey +} + +func (c *certificateV2) MarshalPublicKeyPEM() []byte { + return marshalCertPublicKeyToPEM(c) +} + +func (c *certificateV2) Signature() []byte { + return c.signature +} + +func (c *certificateV2) UnsafeNetworks() []netip.Prefix { + return c.details.unsafeNetworks +} + +func (c *certificateV2) Fingerprint() (string, error) { + if len(c.rawDetails) == 0 { + return "", ErrMissingDetails + } + + b := make([]byte, len(c.rawDetails)+1+len(c.publicKey)+len(c.signature)) + copy(b, c.rawDetails) + b[len(c.rawDetails)] = byte(c.curve) + copy(b[len(c.rawDetails)+1:], c.publicKey) + copy(b[len(c.rawDetails)+1+len(c.publicKey):], c.signature) + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]), nil +} + +func (c *certificateV2) CheckSignature(key []byte) bool { + if len(c.rawDetails) == 0 { + return false + } + b := make([]byte, len(c.rawDetails)+1+len(c.publicKey)) + copy(b, c.rawDetails) + b[len(c.rawDetails)] = byte(c.curve) + copy(b[len(c.rawDetails)+1:], c.publicKey) + + switch c.curve { + case Curve_CURVE25519: + if len(key) != ed25519.PublicKeySize { + return false //avoids a panic internal to ed25519 + } + return ed25519.Verify(key, b, c.signature) + case Curve_P256: + pubKey, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), key) + if err != nil { + return false + } + hashed := sha256.Sum256(b) + return ecdsa.VerifyASN1(pubKey, hashed[:], c.signature) + default: + return false + } +} + +func (c *certificateV2) Expired(t time.Time) bool { + return c.details.notBefore.After(t) || c.details.notAfter.Before(t) +} + +func (c *certificateV2) VerifyPrivateKey(curve Curve, key []byte) error { + if curve != c.curve { + return ErrPublicPrivateCurveMismatch + } + if c.details.isCA { + switch curve { + case Curve_CURVE25519: + // the call to PublicKey below will panic slice bounds out of range otherwise + if len(key) != ed25519.PrivateKeySize { + return ErrInvalidPrivateKey + } + + if !ed25519.PublicKey(c.publicKey).Equal(ed25519.PrivateKey(key).Public()) { + return ErrPublicPrivateKeyMismatch + } + case Curve_P256: + privkey, err := ecdh.P256().NewPrivateKey(key) + if err != nil { + return ErrInvalidPrivateKey + } + pub := privkey.PublicKey().Bytes() + if !bytes.Equal(pub, c.publicKey) { + return ErrPublicPrivateKeyMismatch + } + default: + return fmt.Errorf("invalid curve: %s", curve) + } + return nil + } + + var pub []byte + switch curve { + case Curve_CURVE25519: + var err error + pub, err = curve25519.X25519(key, curve25519.Basepoint) + if err != nil { + return ErrInvalidPrivateKey + } + case Curve_P256: + privkey, err := ecdh.P256().NewPrivateKey(key) + if err != nil { + return ErrInvalidPrivateKey + } + pub = privkey.PublicKey().Bytes() + default: + return fmt.Errorf("invalid curve: %s", curve) + } + if !bytes.Equal(pub, c.publicKey) { + return ErrPublicPrivateKeyMismatch + } + + return nil +} + +func (c *certificateV2) String() string { + mb, err := c.marshalJSON() + if err != nil { + return fmt.Sprintf("", err) + } + + b, err := json.MarshalIndent(mb, "", "\t") + if err != nil { + return fmt.Sprintf("", err) + } + return string(b) +} + +func (c *certificateV2) MarshalForHandshakes() ([]byte, error) { + if c.rawDetails == nil { + return nil, ErrEmptyRawDetails + } + var b cryptobyte.Builder + // Outermost certificate + b.AddASN1(asn1.SEQUENCE, func(b *cryptobyte.Builder) { + + // Add the cert details which is already marshalled + b.AddBytes(c.rawDetails) + + // Skipping the curve and public key since those come across in a different part of the handshake + + // Add the signature + b.AddASN1(TagCertSignature, func(b *cryptobyte.Builder) { + b.AddBytes(c.signature) + }) + }) + + return b.Bytes() +} + +func (c *certificateV2) Marshal() ([]byte, error) { + if c.rawDetails == nil { + return nil, ErrEmptyRawDetails + } + var b cryptobyte.Builder + // Outermost certificate + b.AddASN1(asn1.SEQUENCE, func(b *cryptobyte.Builder) { + + // Add the cert details which is already marshalled + b.AddBytes(c.rawDetails) + + // Add the curve only if its not the default value + if c.curve != Curve_CURVE25519 { + b.AddASN1(TagCertCurve, func(b *cryptobyte.Builder) { + b.AddBytes([]byte{byte(c.curve)}) + }) + } + + // Add the public key if it is not empty + if c.publicKey != nil { + b.AddASN1(TagCertPublicKey, func(b *cryptobyte.Builder) { + b.AddBytes(c.publicKey) + }) + } + + // Add the signature + b.AddASN1(TagCertSignature, func(b *cryptobyte.Builder) { + b.AddBytes(c.signature) + }) + }) + + return b.Bytes() +} + +func (c *certificateV2) MarshalPEM() ([]byte, error) { + b, err := c.Marshal() + if err != nil { + return nil, err + } + return pem.EncodeToMemory(&pem.Block{Type: CertificateV2Banner, Bytes: b}), nil +} + +func (c *certificateV2) MarshalJSON() ([]byte, error) { + b, err := c.marshalJSON() + if err != nil { + return nil, err + } + return json.Marshal(b) +} + +func (c *certificateV2) marshalJSON() (m, error) { + fp, err := c.Fingerprint() + if err != nil { + return nil, err + } + + return m{ + "details": m{ + "name": c.details.name, + "networks": c.details.networks, + "unsafeNetworks": c.details.unsafeNetworks, + "groups": c.details.groups, + "notBefore": c.details.notBefore, + "notAfter": c.details.notAfter, + "isCa": c.details.isCA, + "issuer": c.details.issuer, + }, + "version": Version2, + "publicKey": fmt.Sprintf("%x", c.publicKey), + "curve": c.curve.String(), + "fingerprint": fp, + "signature": fmt.Sprintf("%x", c.Signature()), + }, nil +} + +func (c *certificateV2) Copy() Certificate { + nc := &certificateV2{ + details: detailsV2{ + name: c.details.name, + notBefore: c.details.notBefore, + notAfter: c.details.notAfter, + isCA: c.details.isCA, + issuer: c.details.issuer, + }, + curve: c.curve, + publicKey: make([]byte, len(c.publicKey)), + signature: make([]byte, len(c.signature)), + rawDetails: make([]byte, len(c.rawDetails)), + } + + if c.details.groups != nil { + nc.details.groups = make([]string, len(c.details.groups)) + copy(nc.details.groups, c.details.groups) + } + + if c.details.networks != nil { + nc.details.networks = make([]netip.Prefix, len(c.details.networks)) + copy(nc.details.networks, c.details.networks) + } + + if c.details.unsafeNetworks != nil { + nc.details.unsafeNetworks = make([]netip.Prefix, len(c.details.unsafeNetworks)) + copy(nc.details.unsafeNetworks, c.details.unsafeNetworks) + } + + copy(nc.rawDetails, c.rawDetails) + copy(nc.signature, c.signature) + copy(nc.publicKey, c.publicKey) + + return nc +} + +func (c *certificateV2) fromTBSCertificate(t *TBSCertificate) error { + c.details = detailsV2{ + name: t.Name, + networks: t.Networks, + unsafeNetworks: t.UnsafeNetworks, + groups: t.Groups, + isCA: t.IsCA, + notBefore: t.NotBefore, + notAfter: t.NotAfter, + issuer: t.issuer, + } + c.curve = t.Curve + c.publicKey = t.PublicKey + return c.validate() +} + +func (c *certificateV2) validate() error { + // Empty names are allowed + + if len(c.publicKey) == 0 { + return ErrInvalidPublicKey + } + + if !c.details.isCA && len(c.details.networks) == 0 { + return NewErrInvalidCertificateProperties("non-CA certificate must contain at least 1 network") + } + + hasV4Networks := false + hasV6Networks := false + for _, network := range c.details.networks { + if !network.IsValid() || !network.Addr().IsValid() { + return NewErrInvalidCertificateProperties("invalid network: %s", network) + } + + if network.Addr().IsUnspecified() { + return NewErrInvalidCertificateProperties("non-CA certificates must not use the zero address as a network: %s", network) + } + + if network.Addr().Zone() != "" { + return NewErrInvalidCertificateProperties("networks may not contain zones: %s", network) + } + + if network.Addr().Is4In6() { + return NewErrInvalidCertificateProperties("4in6 networks are not allowed: %s", network) + } + + hasV4Networks = hasV4Networks || network.Addr().Is4() + hasV6Networks = hasV6Networks || network.Addr().Is6() + } + + slices.SortFunc(c.details.networks, comparePrefix) + err := findDuplicatePrefix(c.details.networks) + if err != nil { + return err + } + + for _, network := range c.details.unsafeNetworks { + if !network.IsValid() || !network.Addr().IsValid() { + return NewErrInvalidCertificateProperties("invalid unsafe network: %s", network) + } + + if network.Addr().Zone() != "" { + return NewErrInvalidCertificateProperties("unsafe networks may not contain zones: %s", network) + } + + if !c.details.isCA { + if network.Addr().Is6() { + if !hasV6Networks { + return NewErrInvalidCertificateProperties("IPv6 unsafe networks require an IPv6 address assignment: %s", network) + } + } else if network.Addr().Is4() { + if !hasV4Networks { + return NewErrInvalidCertificateProperties("IPv4 unsafe networks require an IPv4 address assignment: %s", network) + } + } + } + } + + slices.SortFunc(c.details.unsafeNetworks, comparePrefix) + err = findDuplicatePrefix(c.details.unsafeNetworks) + if err != nil { + return err + } + + return nil +} + +func (c *certificateV2) marshalForSigning() ([]byte, error) { + d, err := c.details.Marshal() + if err != nil { + return nil, fmt.Errorf("marshalling certificate details failed: %w", err) + } + c.rawDetails = d + + b := make([]byte, len(c.rawDetails)+1+len(c.publicKey)) + copy(b, c.rawDetails) + b[len(c.rawDetails)] = byte(c.curve) + copy(b[len(c.rawDetails)+1:], c.publicKey) + return b, nil +} + +func (c *certificateV2) setSignature(b []byte) error { + if len(b) == 0 { + return ErrEmptySignature + } + c.signature = b + return nil +} + +func (d *detailsV2) Marshal() ([]byte, error) { + var b cryptobyte.Builder + var err error + + // Details are a structure + b.AddASN1(TagCertDetails, func(b *cryptobyte.Builder) { + + // Add the name + b.AddASN1(TagDetailsName, func(b *cryptobyte.Builder) { + b.AddBytes([]byte(d.name)) + }) + + // Add the networks if any exist + if len(d.networks) > 0 { + b.AddASN1(TagDetailsNetworks, func(b *cryptobyte.Builder) { + for _, n := range d.networks { + sb, innerErr := n.MarshalBinary() + if innerErr != nil { + // MarshalBinary never returns an error + err = fmt.Errorf("unable to marshal network: %w", innerErr) + return + } + b.AddASN1OctetString(sb) + } + }) + } + + // Add the unsafe networks if any exist + if len(d.unsafeNetworks) > 0 { + b.AddASN1(TagDetailsUnsafeNetworks, func(b *cryptobyte.Builder) { + for _, n := range d.unsafeNetworks { + sb, innerErr := n.MarshalBinary() + if innerErr != nil { + // MarshalBinary never returns an error + err = fmt.Errorf("unable to marshal unsafe network: %w", innerErr) + return + } + b.AddASN1OctetString(sb) + } + }) + } + + // Add groups if any exist + if len(d.groups) > 0 { + b.AddASN1(TagDetailsGroups, func(b *cryptobyte.Builder) { + for _, group := range d.groups { + b.AddASN1(asn1.UTF8String, func(b *cryptobyte.Builder) { + b.AddBytes([]byte(group)) + }) + } + }) + } + + // Add IsCA only if true + if d.isCA { + b.AddASN1(TagDetailsIsCA, func(b *cryptobyte.Builder) { + b.AddUint8(0xff) + }) + } + + // Add not before + b.AddASN1Int64WithTag(d.notBefore.Unix(), TagDetailsNotBefore) + + // Add not after + b.AddASN1Int64WithTag(d.notAfter.Unix(), TagDetailsNotAfter) + + // Add the issuer if present + if d.issuer != "" { + issuerBytes, innerErr := hex.DecodeString(d.issuer) + if innerErr != nil { + err = fmt.Errorf("failed to decode issuer: %w", innerErr) + return + } + b.AddASN1(TagDetailsIssuer, func(b *cryptobyte.Builder) { + b.AddBytes(issuerBytes) + }) + } + }) + + if err != nil { + return nil, err + } + + return b.Bytes() +} + +func unmarshalCertificateV2(b []byte, publicKey []byte, curve Curve) (*certificateV2, error) { + l := len(b) + if l == 0 || l > MaxCertificateSize { + return nil, ErrBadFormat + } + + input := cryptobyte.String(b) + // Open the envelope + if !input.ReadASN1(&input, asn1.SEQUENCE) || input.Empty() { + return nil, ErrBadFormat + } + + // Grab the cert details, we need to preserve the tag and length + var rawDetails cryptobyte.String + if !input.ReadASN1Element(&rawDetails, TagCertDetails) || rawDetails.Empty() { + return nil, ErrBadFormat + } + + //Maybe grab the curve + var rawCurve byte + if !readOptionalASN1Byte(&input, &rawCurve, TagCertCurve, byte(curve)) { + return nil, ErrBadFormat + } + curve = Curve(rawCurve) + + // Maybe grab the public key + var rawPublicKey cryptobyte.String + if len(publicKey) > 0 { + // If a public key is passed in, then the handshake certificate must + // not have a public key present + if input.PeekASN1Tag(TagCertPublicKey) { + return nil, ErrCertPubkeyPresent + } + rawPublicKey = make(cryptobyte.String, len(publicKey)) + copy(rawPublicKey, publicKey) + } else if !input.ReadOptionalASN1(&rawPublicKey, nil, TagCertPublicKey) { + return nil, ErrBadFormat + } + + if len(rawPublicKey) == 0 { + return nil, ErrBadFormat + } + + // Grab the signature + var rawSignature cryptobyte.String + if !input.ReadASN1(&rawSignature, TagCertSignature) || rawSignature.Empty() { + return nil, ErrBadFormat + } + + // Finally unmarshal the details + details, err := unmarshalDetails(rawDetails) + if err != nil { + return nil, err + } + + c := &certificateV2{ + details: details, + rawDetails: rawDetails, + curve: curve, + publicKey: rawPublicKey, + signature: rawSignature, + } + + err = c.validate() + if err != nil { + return nil, err + } + + return c, nil +} + +func unmarshalDetails(b cryptobyte.String) (detailsV2, error) { + // Open the envelope + if !b.ReadASN1(&b, TagCertDetails) || b.Empty() { + return detailsV2{}, ErrBadFormat + } + + // Read the name + var name cryptobyte.String + if !b.ReadASN1(&name, TagDetailsName) || name.Empty() || len(name) > MaxNameLength { + return detailsV2{}, ErrBadFormat + } + + // Read the network addresses + var subString cryptobyte.String + var found bool + + if !b.ReadOptionalASN1(&subString, &found, TagDetailsNetworks) { + return detailsV2{}, ErrBadFormat + } + + var networks []netip.Prefix + var val cryptobyte.String + if found { + for !subString.Empty() { + if !subString.ReadASN1(&val, asn1.OCTET_STRING) || val.Empty() || len(val) > MaxNetworkLength { + return detailsV2{}, ErrBadFormat + } + + var n netip.Prefix + if err := n.UnmarshalBinary(val); err != nil { + return detailsV2{}, ErrBadFormat + } + networks = append(networks, n) + } + } + + // Read out any unsafe networks + if !b.ReadOptionalASN1(&subString, &found, TagDetailsUnsafeNetworks) { + return detailsV2{}, ErrBadFormat + } + + var unsafeNetworks []netip.Prefix + if found { + for !subString.Empty() { + if !subString.ReadASN1(&val, asn1.OCTET_STRING) || val.Empty() || len(val) > MaxNetworkLength { + return detailsV2{}, ErrBadFormat + } + + var n netip.Prefix + if err := n.UnmarshalBinary(val); err != nil { + return detailsV2{}, ErrBadFormat + } + unsafeNetworks = append(unsafeNetworks, n) + } + } + + // Read out any groups + if !b.ReadOptionalASN1(&subString, &found, TagDetailsGroups) { + return detailsV2{}, ErrBadFormat + } + + var groups []string + if found { + for !subString.Empty() { + if !subString.ReadASN1(&val, asn1.UTF8String) || val.Empty() { + return detailsV2{}, ErrBadFormat + } + groups = append(groups, string(val)) + } + } + + // Read out IsCA + var isCa bool + if !readOptionalASN1Boolean(&b, &isCa, TagDetailsIsCA, false) { + return detailsV2{}, ErrBadFormat + } + + // Read not before and not after + var notBefore int64 + if !b.ReadASN1Int64WithTag(¬Before, TagDetailsNotBefore) { + return detailsV2{}, ErrBadFormat + } + + var notAfter int64 + if !b.ReadASN1Int64WithTag(¬After, TagDetailsNotAfter) { + return detailsV2{}, ErrBadFormat + } + + // Read issuer + var issuer cryptobyte.String + if !b.ReadOptionalASN1(&issuer, nil, TagDetailsIssuer) { + return detailsV2{}, ErrBadFormat + } + + return detailsV2{ + name: string(name), + networks: networks, + unsafeNetworks: unsafeNetworks, + groups: groups, + isCA: isCa, + notBefore: time.Unix(notBefore, 0), + notAfter: time.Unix(notAfter, 0), + issuer: hex.EncodeToString(issuer), + }, nil +} diff --git a/cert/cert_v2_test.go b/cert/cert_v2_test.go new file mode 100644 index 00000000..e2a6b98d --- /dev/null +++ b/cert/cert_v2_test.go @@ -0,0 +1,379 @@ +package cert + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "net/netip" + "slices" + "testing" + "time" + + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCertificateV2_Marshal(t *testing.T) { + t.Parallel() + before := time.Now().Add(time.Second * -60).Round(time.Second) + after := time.Now().Add(time.Second * 60).Round(time.Second) + pubKey := []byte("1234567890abcedfghij1234567890ab") + + nc := certificateV2{ + details: detailsV2{ + name: "testing", + networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.2/16"), + mustParsePrefixUnmapped("10.1.1.1/24"), + }, + unsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.3/16"), + mustParsePrefixUnmapped("9.1.1.2/24"), + }, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: before, + notAfter: after, + isCA: false, + issuer: "1234567890abcdef1234567890abcdef", + }, + signature: []byte("1234567890abcdef1234567890abcdef"), + publicKey: pubKey, + } + + db, err := nc.details.Marshal() + require.NoError(t, err) + nc.rawDetails = db + + b, err := nc.Marshal() + require.NoError(t, err) + //t.Log("Cert size:", len(b)) + + nc2, err := unmarshalCertificateV2(b, nil, Curve_CURVE25519) + require.NoError(t, err) + + assert.Equal(t, Version2, nc.Version()) + assert.Equal(t, Curve_CURVE25519, nc.Curve()) + assert.Equal(t, nc.Signature(), nc2.Signature()) + assert.Equal(t, nc.Name(), nc2.Name()) + assert.Equal(t, nc.NotBefore(), nc2.NotBefore()) + assert.Equal(t, nc.NotAfter(), nc2.NotAfter()) + assert.Equal(t, nc.PublicKey(), nc2.PublicKey()) + assert.Equal(t, nc.IsCA(), nc2.IsCA()) + assert.Equal(t, nc.Issuer(), nc2.Issuer()) + + // unmarshalling will sort networks and unsafeNetworks, we need to do the same + // but first make sure it fails + assert.NotEqual(t, nc.Networks(), nc2.Networks()) + assert.NotEqual(t, nc.UnsafeNetworks(), nc2.UnsafeNetworks()) + + slices.SortFunc(nc.details.networks, comparePrefix) + slices.SortFunc(nc.details.unsafeNetworks, comparePrefix) + + assert.Equal(t, nc.Networks(), nc2.Networks()) + assert.Equal(t, nc.UnsafeNetworks(), nc2.UnsafeNetworks()) + + assert.Equal(t, nc.Groups(), nc2.Groups()) +} + +func TestCertificateV2_Unmarshal(t *testing.T) { + t.Parallel() + before := time.Now().Add(time.Second * -60).Round(time.Second) + after := time.Now().Add(time.Second * 60).Round(time.Second) + pubKey := []byte("1234567890abcedfghij1234567890ab") + + nc := certificateV2{ + details: detailsV2{ + name: "testing", + networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.2/16"), + mustParsePrefixUnmapped("10.1.1.1/24"), + }, + unsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.3/16"), + mustParsePrefixUnmapped("9.1.1.2/24"), + }, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: before, + notAfter: after, + isCA: false, + issuer: "1234567890abcdef1234567890abcdef", + }, + signature: []byte("1234567890abcdef1234567890abcdef"), + publicKey: pubKey, + } + + db, err := nc.details.Marshal() + require.NoError(t, err) + nc.rawDetails = db + + certWithPubkey, err := nc.Marshal() + require.NoError(t, err) + //t.Log("Cert size:", len(b)) + certWithoutPubkey, err := nc.MarshalForHandshakes() + require.NoError(t, err) + + // Cert must not have a pubkey if one is passed in as an argument + _, err = unmarshalCertificateV2(certWithPubkey, pubKey, Curve_CURVE25519) + require.ErrorIs(t, err, ErrCertPubkeyPresent) + + // Certs must have pubkeys + _, err = unmarshalCertificateV2(certWithoutPubkey, nil, Curve_CURVE25519) + require.ErrorIs(t, err, ErrBadFormat) + + // Ensure proper unmarshal if a pubkey is passed in + nc2, err := unmarshalCertificateV2(certWithoutPubkey, pubKey, Curve_CURVE25519) + require.NoError(t, err) + + assert.Equal(t, nc.PublicKey(), nc2.PublicKey()) +} + +func TestCertificateV2_PublicKeyPem(t *testing.T) { + t.Parallel() + before := time.Now().Add(time.Second * -60).Round(time.Second) + after := time.Now().Add(time.Second * 60).Round(time.Second) + pubKey := ed25519.PublicKey("1234567890abcedfghij1234567890ab") + + nc := certificateV2{ + details: detailsV2{ + name: "testing", + networks: []netip.Prefix{}, + unsafeNetworks: []netip.Prefix{}, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: before, + notAfter: after, + isCA: false, + issuer: "1234567890abcedfghij1234567890ab", + }, + publicKey: pubKey, + signature: []byte("1234567890abcedfghij1234567890ab"), + } + + assert.Equal(t, Version2, nc.Version()) + assert.Equal(t, Curve_CURVE25519, nc.Curve()) + pubPem := "-----BEGIN NEBULA X25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA X25519 PUBLIC KEY-----\n" + assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem) + assert.False(t, nc.IsCA()) + + nc.details.isCA = true + assert.Equal(t, Curve_CURVE25519, nc.Curve()) + pubPem = "-----BEGIN NEBULA ED25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA ED25519 PUBLIC KEY-----\n" + assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem) + assert.True(t, nc.IsCA()) + + pubP256KeyPem := []byte(`-----BEGIN NEBULA P256 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA P256 PUBLIC KEY----- +`) + + pubP256KeyPemCA := []byte(`-----BEGIN NEBULA ECDSA P256 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA ECDSA P256 PUBLIC KEY----- +`) + + pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem) + require.NoError(t, err) + nc.curve = Curve_P256 + nc.publicKey = pubP256Key + assert.Equal(t, Curve_P256, nc.Curve()) + assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPemCA)) + assert.True(t, nc.IsCA()) + + nc.details.isCA = false + assert.Equal(t, Curve_P256, nc.Curve()) + assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem)) + assert.False(t, nc.IsCA()) +} + +func TestCertificateV2_Expired(t *testing.T) { + nc := certificateV2{ + details: detailsV2{ + notBefore: time.Now().Add(time.Second * -60).Round(time.Second), + notAfter: time.Now().Add(time.Second * 60).Round(time.Second), + }, + } + + assert.True(t, nc.Expired(time.Now().Add(time.Hour))) + assert.True(t, nc.Expired(time.Now().Add(-time.Hour))) + assert.False(t, nc.Expired(time.Now())) +} + +func TestCertificateV2_MarshalJSON(t *testing.T) { + time.Local = time.UTC + pubKey := []byte("1234567890abcedf1234567890abcedf") + + nc := certificateV2{ + details: detailsV2{ + name: "testing", + networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.1/24"), + mustParsePrefixUnmapped("10.1.1.2/16"), + }, + unsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.2/24"), + mustParsePrefixUnmapped("9.1.1.3/16"), + }, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: time.Date(1, 0, 0, 1, 0, 0, 0, time.UTC), + notAfter: time.Date(1, 0, 0, 2, 0, 0, 0, time.UTC), + isCA: false, + issuer: "1234567890abcedf1234567890abcedf", + }, + publicKey: pubKey, + signature: []byte("1234567890abcedf1234567890abcedf1234567890abcedf1234567890abcedf"), + } + + b, err := nc.MarshalJSON() + require.ErrorIs(t, err, ErrMissingDetails) + + rd, err := nc.details.Marshal() + require.NoError(t, err) + + nc.rawDetails = rd + b, err = nc.MarshalJSON() + require.NoError(t, err) + assert.JSONEq( + t, + "{\"curve\":\"CURVE25519\",\"details\":{\"groups\":[\"test-group1\",\"test-group2\",\"test-group3\"],\"isCa\":false,\"issuer\":\"1234567890abcedf1234567890abcedf\",\"name\":\"testing\",\"networks\":[\"10.1.1.1/24\",\"10.1.1.2/16\"],\"notAfter\":\"0000-11-30T02:00:00Z\",\"notBefore\":\"0000-11-30T01:00:00Z\",\"unsafeNetworks\":[\"9.1.1.2/24\",\"9.1.1.3/16\"]},\"fingerprint\":\"152d9a7400c1e001cb76cffd035215ebb351f69eeb797f7f847dd086e15e56dd\",\"publicKey\":\"3132333435363738393061626365646631323334353637383930616263656466\",\"signature\":\"31323334353637383930616263656466313233343536373839306162636564663132333435363738393061626365646631323334353637383930616263656466\",\"version\":2}", + string(b), + ) +} + +func TestCertificateV2_VerifyPrivateKey(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version2, Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil) + err := ca.VerifyPrivateKey(Curve_CURVE25519, caKey) + require.NoError(t, err) + + err = ca.VerifyPrivateKey(Curve_CURVE25519, caKey[:16]) + require.ErrorIs(t, err, ErrInvalidPrivateKey) + + _, caKey2, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + err = ca.VerifyPrivateKey(Curve_CURVE25519, caKey2) + require.ErrorIs(t, err, ErrPublicPrivateKeyMismatch) + + c, _, priv, _ := NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Time{}, time.Time{}, nil, nil, nil) + rawPriv, b, curve, err := UnmarshalPrivateKeyFromPEM(priv) + require.NoError(t, err) + assert.Empty(t, b) + assert.Equal(t, Curve_CURVE25519, curve) + err = c.VerifyPrivateKey(Curve_CURVE25519, rawPriv) + require.NoError(t, err) + + _, priv2 := X25519Keypair() + err = c.VerifyPrivateKey(Curve_P256, priv2) + require.ErrorIs(t, err, ErrPublicPrivateCurveMismatch) + + err = c.VerifyPrivateKey(Curve_CURVE25519, priv2) + require.ErrorIs(t, err, ErrPublicPrivateKeyMismatch) + + err = c.VerifyPrivateKey(Curve_CURVE25519, priv2[:16]) + require.ErrorIs(t, err, ErrInvalidPrivateKey) + + ac, ok := c.(*certificateV2) + require.True(t, ok) + ac.curve = Curve(99) + err = c.VerifyPrivateKey(Curve(99), priv2) + require.EqualError(t, err, "invalid curve: 99") + + ca2, _, caKey2, _ := NewTestCaCert(Version2, Curve_P256, time.Time{}, time.Time{}, nil, nil, nil) + err = ca.VerifyPrivateKey(Curve_CURVE25519, caKey) + require.NoError(t, err) + + err = ca2.VerifyPrivateKey(Curve_P256, caKey2[:16]) + require.ErrorIs(t, err, ErrInvalidPrivateKey) + + c, _, priv, _ = NewTestCert(Version2, Curve_P256, ca2, caKey2, "test", time.Time{}, time.Time{}, nil, nil, nil) + rawPriv, b, curve, err = UnmarshalPrivateKeyFromPEM(priv) + + err = c.VerifyPrivateKey(Curve_P256, priv[:16]) + require.ErrorIs(t, err, ErrInvalidPrivateKey) + + err = c.VerifyPrivateKey(Curve_P256, priv) + require.ErrorIs(t, err, ErrInvalidPrivateKey) + + aCa, ok := ca2.(*certificateV2) + require.True(t, ok) + aCa.curve = Curve(99) + err = aCa.VerifyPrivateKey(Curve(99), priv2) + require.EqualError(t, err, "invalid curve: 99") + +} + +func TestCertificateV2_VerifyPrivateKeyP256(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version2, Curve_P256, time.Time{}, time.Time{}, nil, nil, nil) + err := ca.VerifyPrivateKey(Curve_P256, caKey) + require.NoError(t, err) + + _, _, caKey2, _ := NewTestCaCert(Version2, Curve_P256, time.Time{}, time.Time{}, nil, nil, nil) + require.NoError(t, err) + err = ca.VerifyPrivateKey(Curve_P256, caKey2) + require.Error(t, err) + + c, _, priv, _ := NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Time{}, time.Time{}, nil, nil, nil) + rawPriv, b, curve, err := UnmarshalPrivateKeyFromPEM(priv) + require.NoError(t, err) + assert.Empty(t, b) + assert.Equal(t, Curve_P256, curve) + err = c.VerifyPrivateKey(Curve_P256, rawPriv) + require.NoError(t, err) + + _, priv2 := P256Keypair() + err = c.VerifyPrivateKey(Curve_P256, priv2) + require.Error(t, err) +} + +func TestCertificateV2_Copy(t *testing.T) { + ca, _, caKey, _ := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil) + c, _, _, _ := NewTestCert(Version2, Curve_CURVE25519, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil) + cc := c.Copy() + test.AssertDeepCopyEqual(t, c, cc) +} + +func TestUnmarshalCertificateV2(t *testing.T) { + data := []byte("\x98\x00\x00") + _, err := unmarshalCertificateV2(data, nil, Curve_CURVE25519) + require.EqualError(t, err, "bad wire format") +} + +func TestCertificateV2_marshalForSigningStability(t *testing.T) { + before := time.Date(1996, time.May, 5, 0, 0, 0, 0, time.UTC) + after := before.Add(time.Second * 60).Round(time.Second) + pubKey := []byte("1234567890abcedfghij1234567890ab") + + nc := certificateV2{ + details: detailsV2{ + name: "testing", + networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.2/16"), + mustParsePrefixUnmapped("10.1.1.1/24"), + }, + unsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.3/16"), + mustParsePrefixUnmapped("9.1.1.2/24"), + }, + groups: []string{"test-group1", "test-group2", "test-group3"}, + notBefore: before, + notAfter: after, + isCA: false, + issuer: "1234567890abcdef1234567890abcdef", + }, + signature: []byte("1234567890abcdef1234567890abcdef"), + publicKey: pubKey, + } + + const expectedRawDetailsStr = "a070800774657374696e67a10e04050a0101021004050a01010118a20e0405090101031004050901010218a3270c0b746573742d67726f7570310c0b746573742d67726f7570320c0b746573742d67726f7570338504318bef808604318befbc87101234567890abcdef1234567890abcdef" + expectedRawDetails, err := hex.DecodeString(expectedRawDetailsStr) + require.NoError(t, err) + + db, err := nc.details.Marshal() + require.NoError(t, err) + assert.Equal(t, expectedRawDetails, db) + + expectedForSigning, err := hex.DecodeString(expectedRawDetailsStr + "00313233343536373839306162636564666768696a313233343536373839306162") + b, err := nc.marshalForSigning() + require.NoError(t, err) + assert.Equal(t, expectedForSigning, b) +} diff --git a/cert/crypto.go b/cert/crypto.go index 3558e1a5..4c236aee 100644 --- a/cert/crypto.go +++ b/cert/crypto.go @@ -3,14 +3,28 @@ package cert import ( "crypto/aes" "crypto/cipher" + "crypto/ed25519" "crypto/rand" + "encoding/pem" "fmt" "io" + "math" "golang.org/x/crypto/argon2" + "google.golang.org/protobuf/proto" ) -// KDF factors +type NebulaEncryptedData struct { + EncryptionMetadata NebulaEncryptionMetadata + Ciphertext []byte +} + +type NebulaEncryptionMetadata struct { + EncryptionAlgorithm string + Argon2Parameters Argon2Parameters +} + +// Argon2Parameters KDF factors type Argon2Parameters struct { version rune Memory uint32 // KiB @@ -19,7 +33,7 @@ type Argon2Parameters struct { salt []byte } -// Returns a new Argon2Parameters object with current version set +// NewArgon2Parameters Returns a new Argon2Parameters object with current version set func NewArgon2Parameters(memory uint32, parallelism uint8, iterations uint32) *Argon2Parameters { return &Argon2Parameters{ version: argon2.Version, @@ -141,3 +155,146 @@ func splitNonceCiphertext(blob []byte, nonceSize int) ([]byte, []byte, error) { return blob[:nonceSize], blob[nonceSize:], nil } + +// EncryptAndMarshalSigningPrivateKey is a simple helper to encrypt and PEM encode a private key +func EncryptAndMarshalSigningPrivateKey(curve Curve, b []byte, passphrase []byte, kdfParams *Argon2Parameters) ([]byte, error) { + ciphertext, err := aes256Encrypt(passphrase, kdfParams, b) + if err != nil { + return nil, err + } + + b, err = proto.Marshal(&RawNebulaEncryptedData{ + EncryptionMetadata: &RawNebulaEncryptionMetadata{ + EncryptionAlgorithm: "AES-256-GCM", + Argon2Parameters: &RawNebulaArgon2Parameters{ + Version: kdfParams.version, + Memory: kdfParams.Memory, + Parallelism: uint32(kdfParams.Parallelism), + Iterations: kdfParams.Iterations, + Salt: kdfParams.salt, + }, + }, + Ciphertext: ciphertext, + }) + if err != nil { + return nil, err + } + + switch curve { + case Curve_CURVE25519: + return pem.EncodeToMemory(&pem.Block{Type: EncryptedEd25519PrivateKeyBanner, Bytes: b}), nil + case Curve_P256: + return pem.EncodeToMemory(&pem.Block{Type: EncryptedECDSAP256PrivateKeyBanner, Bytes: b}), nil + default: + return nil, fmt.Errorf("invalid curve: %v", curve) + } +} + +// UnmarshalNebulaEncryptedData will unmarshal a protobuf byte representation of a nebula cert into its +// protobuf-generated struct. +func UnmarshalNebulaEncryptedData(b []byte) (*NebulaEncryptedData, error) { + if len(b) == 0 { + return nil, fmt.Errorf("nil byte array") + } + var rned RawNebulaEncryptedData + err := proto.Unmarshal(b, &rned) + if err != nil { + return nil, err + } + + if rned.EncryptionMetadata == nil { + return nil, fmt.Errorf("encoded EncryptionMetadata was nil") + } + + if rned.EncryptionMetadata.Argon2Parameters == nil { + return nil, fmt.Errorf("encoded Argon2Parameters was nil") + } + + params, err := unmarshalArgon2Parameters(rned.EncryptionMetadata.Argon2Parameters) + if err != nil { + return nil, err + } + + ned := NebulaEncryptedData{ + EncryptionMetadata: NebulaEncryptionMetadata{ + EncryptionAlgorithm: rned.EncryptionMetadata.EncryptionAlgorithm, + Argon2Parameters: *params, + }, + Ciphertext: rned.Ciphertext, + } + + return &ned, nil +} + +func unmarshalArgon2Parameters(params *RawNebulaArgon2Parameters) (*Argon2Parameters, error) { + if params.Version < math.MinInt32 || params.Version > math.MaxInt32 { + return nil, fmt.Errorf("Argon2Parameters Version must be at least %d and no more than %d", math.MinInt32, math.MaxInt32) + } + if params.Memory <= 0 || params.Memory > math.MaxUint32 { + return nil, fmt.Errorf("Argon2Parameters Memory must be be greater than 0 and no more than %d KiB", uint32(math.MaxUint32)) + } + if params.Parallelism <= 0 || params.Parallelism > math.MaxUint8 { + return nil, fmt.Errorf("Argon2Parameters Parallelism must be be greater than 0 and no more than %d", math.MaxUint8) + } + if params.Iterations <= 0 || params.Iterations > math.MaxUint32 { + return nil, fmt.Errorf("-argon-iterations must be be greater than 0 and no more than %d", uint32(math.MaxUint32)) + } + + return &Argon2Parameters{ + version: params.Version, + Memory: params.Memory, + Parallelism: uint8(params.Parallelism), + Iterations: params.Iterations, + salt: params.Salt, + }, nil + +} + +// DecryptAndUnmarshalSigningPrivateKey will try to pem decode and decrypt an Ed25519/ECDSA private key with +// the given passphrase, returning any other bytes b or an error on failure +func DecryptAndUnmarshalSigningPrivateKey(passphrase, b []byte) (Curve, []byte, []byte, error) { + var curve Curve + + k, r := pem.Decode(b) + if k == nil { + return curve, nil, r, fmt.Errorf("input did not contain a valid PEM encoded block") + } + + switch k.Type { + case EncryptedEd25519PrivateKeyBanner: + curve = Curve_CURVE25519 + case EncryptedECDSAP256PrivateKeyBanner: + curve = Curve_P256 + default: + return curve, nil, r, fmt.Errorf("bytes did not contain a proper nebula encrypted Ed25519/ECDSA private key banner") + } + + ned, err := UnmarshalNebulaEncryptedData(k.Bytes) + if err != nil { + return curve, nil, r, err + } + + var bytes []byte + switch ned.EncryptionMetadata.EncryptionAlgorithm { + case "AES-256-GCM": + bytes, err = aes256Decrypt(passphrase, &ned.EncryptionMetadata.Argon2Parameters, ned.Ciphertext) + if err != nil { + return curve, nil, r, err + } + default: + return curve, nil, r, fmt.Errorf("unsupported encryption algorithm: %s", ned.EncryptionMetadata.EncryptionAlgorithm) + } + + switch curve { + case Curve_CURVE25519: + if len(bytes) != ed25519.PrivateKeySize { + return curve, nil, r, fmt.Errorf("key was not %d bytes, is invalid ed25519 private key", ed25519.PrivateKeySize) + } + case Curve_P256: + if len(bytes) != 32 { + return curve, nil, r, fmt.Errorf("key was not 32 bytes, is invalid ECDSA P256 private key") + } + } + + return curve, bytes, r, nil +} diff --git a/cert/crypto_test.go b/cert/crypto_test.go index c2e61df0..bf6e0654 100644 --- a/cert/crypto_test.go +++ b/cert/crypto_test.go @@ -4,22 +4,110 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/crypto/argon2" ) func TestNewArgon2Parameters(t *testing.T) { p := NewArgon2Parameters(64*1024, 4, 3) - assert.EqualValues(t, &Argon2Parameters{ + assert.Equal(t, &Argon2Parameters{ version: argon2.Version, Memory: 64 * 1024, Parallelism: 4, Iterations: 3, }, p) p = NewArgon2Parameters(2*1024*1024, 2, 1) - assert.EqualValues(t, &Argon2Parameters{ + assert.Equal(t, &Argon2Parameters{ version: argon2.Version, Memory: 2 * 1024 * 1024, Parallelism: 2, Iterations: 1, }, p) } + +func TestDecryptAndUnmarshalSigningPrivateKey(t *testing.T) { + passphrase := []byte("DO NOT USE") + privKey := []byte(`# A good key +-----BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY----- +CjsKC0FFUy0yNTYtR0NNEiwIExCAgAQYAyAEKiCPoDfGQiosxNPTbPn5EsMlc2MI +c0Bt4oz6gTrFQhX3aBJcimhHKeAuhyTGvllD0Z19fe+DFPcLH3h5VrdjVfIAajg0 +KrbV3n9UHif/Au5skWmquNJzoW1E4MTdRbvpti6o+WdQ49DxjBFhx0YH8LBqrbPU +0BGkUHmIO7daP24= +-----END NEBULA ED25519 ENCRYPTED PRIVATE KEY----- +`) + shortKey := []byte(`# A key which, once decrypted, is too short +-----BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY----- +CjsKC0FFUy0yNTYtR0NNEiwIExCAgAQYAyAEKiAVJwdfl3r+eqi/vF6S7OMdpjfo +hAzmTCRnr58Su4AqmBJbCv3zleYCEKYJP6UI3S8ekLMGISsgO4hm5leukCCyqT0Z +cQ76yrberpzkJKoPLGisX8f+xdy4aXSZl7oEYWQte1+vqbtl/eY9PGZhxUQdcyq7 +hqzIyrRqfUgVuA== +-----END NEBULA ED25519 ENCRYPTED PRIVATE KEY----- +`) + invalidBanner := []byte(`# Invalid banner (not encrypted) +-----BEGIN NEBULA ED25519 PRIVATE KEY----- +bWRp2CTVFhW9HD/qCd28ltDgK3w8VXSeaEYczDWos8sMUBqDb9jP3+NYwcS4lURG +XgLvodMXZJuaFPssp+WwtA== +-----END NEBULA ED25519 PRIVATE KEY----- +`) + invalidPem := []byte(`# Not a valid PEM format +-BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY----- +CjwKC0FFUy0yNTYtR0NNEi0IExCAgIABGAEgBCognnjujd67Vsv99p22wfAjQaDT +oCMW1mdjkU3gACKNW4MSXOWR9Sts4C81yk1RUku2gvGKs3TB9LYoklLsIizSYOLl ++Vs//O1T0I1Xbml2XBAROsb/VSoDln/6LMqR4B6fn6B3GOsLBBqRI8daDl9lRMPB +qrlJ69wer3ZUHFXA +-END NEBULA ED25519 ENCRYPTED PRIVATE KEY----- +`) + + keyBundle := appendByteSlices(privKey, shortKey, invalidBanner, invalidPem) + + // Success test case + curve, k, rest, err := DecryptAndUnmarshalSigningPrivateKey(passphrase, keyBundle) + require.NoError(t, err) + assert.Equal(t, Curve_CURVE25519, curve) + assert.Len(t, k, 64) + assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) + + // Fail due to short key + curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey(passphrase, rest) + require.EqualError(t, err, "key was not 64 bytes, is invalid ed25519 private key") + assert.Nil(t, k) + assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) + + // Fail due to invalid banner + curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey(passphrase, rest) + require.EqualError(t, err, "bytes did not contain a proper nebula encrypted Ed25519/ECDSA private key banner") + assert.Nil(t, k) + assert.Equal(t, rest, invalidPem) + + // Fail due to invalid PEM format, because + // it's missing the requisite pre-encapsulation boundary. + curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey(passphrase, rest) + require.EqualError(t, err, "input did not contain a valid PEM encoded block") + assert.Nil(t, k) + assert.Equal(t, rest, invalidPem) + + // Fail due to invalid passphrase + curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey([]byte("invalid passphrase"), privKey) + require.EqualError(t, err, "invalid passphrase or corrupt private key") + assert.Nil(t, k) + assert.Equal(t, []byte{}, rest) +} + +func TestEncryptAndMarshalSigningPrivateKey(t *testing.T) { + // Having proved that decryption works correctly above, we can test the + // encryption function produces a value which can be decrypted + passphrase := []byte("passphrase") + bytes := []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + kdfParams := NewArgon2Parameters(64*1024, 4, 3) + key, err := EncryptAndMarshalSigningPrivateKey(Curve_CURVE25519, bytes, passphrase, kdfParams) + require.NoError(t, err) + + // Verify the "key" can be decrypted successfully + curve, k, rest, err := DecryptAndUnmarshalSigningPrivateKey(passphrase, key) + assert.Len(t, k, 64) + assert.Equal(t, Curve_CURVE25519, curve) + assert.Equal(t, []byte{}, rest) + require.NoError(t, err) + + // EncryptAndMarshalEd25519PrivateKey does not create any errors itself +} diff --git a/cert/errors.go b/cert/errors.go index 05b42d10..596cfe19 100644 --- a/cert/errors.go +++ b/cert/errors.go @@ -2,13 +2,51 @@ package cert import ( "errors" + "fmt" ) var ( - ErrRootExpired = errors.New("root certificate is expired") - ErrExpired = errors.New("certificate is expired") - ErrNotCA = errors.New("certificate is not a CA") - ErrNotSelfSigned = errors.New("certificate is not self-signed") - ErrBlockListed = errors.New("certificate is in the block list") - ErrSignatureMismatch = errors.New("certificate signature did not match") + ErrBadFormat = errors.New("bad wire format") + ErrRootExpired = errors.New("root certificate is expired") + ErrExpired = errors.New("certificate is expired") + ErrNotCA = errors.New("certificate is not a CA") + ErrNotSelfSigned = errors.New("certificate is not self-signed") + ErrBlockListed = errors.New("certificate is in the block list") + ErrFingerprintMismatch = errors.New("certificate fingerprint did not match") + ErrSignatureMismatch = errors.New("certificate signature did not match") + ErrInvalidPublicKey = errors.New("invalid public key") + ErrInvalidPrivateKey = errors.New("invalid private key") + ErrPublicPrivateCurveMismatch = errors.New("public key does not match private key curve") + ErrPublicPrivateKeyMismatch = errors.New("public key and private key are not a pair") + ErrPrivateKeyEncrypted = errors.New("private key must be decrypted") + ErrCaNotFound = errors.New("could not find ca for the certificate") + ErrUnknownVersion = errors.New("certificate version unrecognized") + ErrCertPubkeyPresent = errors.New("certificate has unexpected pubkey present") + ErrCurveMismatch = errors.New("certificate curve does not match CA") + + ErrInvalidPEMBlock = errors.New("input did not contain a valid PEM encoded block") + ErrInvalidPEMCertificateBanner = errors.New("bytes did not contain a proper certificate banner") + ErrInvalidPEMX25519PublicKeyBanner = errors.New("bytes did not contain a proper X25519 public key banner") + ErrInvalidPEMX25519PrivateKeyBanner = errors.New("bytes did not contain a proper X25519 private key banner") + ErrInvalidPEMEd25519PublicKeyBanner = errors.New("bytes did not contain a proper Ed25519 public key banner") + ErrInvalidPEMEd25519PrivateKeyBanner = errors.New("bytes did not contain a proper Ed25519 private key banner") + + ErrNoPeerStaticKey = errors.New("no peer static key was present") + ErrNoPayload = errors.New("provided payload was empty") + + ErrMissingDetails = errors.New("certificate did not contain details") + ErrEmptySignature = errors.New("empty signature") + ErrEmptyRawDetails = errors.New("empty rawDetails not allowed") ) + +type ErrInvalidCertificateProperties struct { + str string +} + +func NewErrInvalidCertificateProperties(format string, a ...any) error { + return &ErrInvalidCertificateProperties{fmt.Sprintf(format, a...)} +} + +func (e *ErrInvalidCertificateProperties) Error() string { + return e.str +} diff --git a/cert/helper_test.go b/cert/helper_test.go new file mode 100644 index 00000000..9becfa5c --- /dev/null +++ b/cert/helper_test.go @@ -0,0 +1,147 @@ +package cert + +import ( + "crypto/ecdh" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "io" + "net/netip" + "time" + + "golang.org/x/crypto/curve25519" + "golang.org/x/crypto/ed25519" +) + +// testCertNow is the reference "now" used to derive default before/after times +// in NewTestCaCert and NewTestCert. Holding it fixed for the lifetime of the +// test binary keeps CA and leaf defaults aligned at the same second, so a leaf +// signed with default times can never expire after its CA on a rounding race. +var testCertNow = time.Now().Round(time.Second) + +// NewTestCaCert will create a new ca certificate +func NewTestCaCert(version Version, curve Curve, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (Certificate, []byte, []byte, []byte) { + var err error + var pub, priv []byte + + switch curve { + case Curve_CURVE25519: + pub, priv, err = ed25519.GenerateKey(rand.Reader) + case Curve_P256: + privk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + panic(err) + } + + pub = elliptic.Marshal(elliptic.P256(), privk.PublicKey.X, privk.PublicKey.Y) + priv = privk.D.FillBytes(make([]byte, 32)) + default: + // There is no default to allow the underlying lib to respond with an error + } + + if before.IsZero() { + before = testCertNow.Add(time.Second * -60) + } + if after.IsZero() { + after = testCertNow.Add(time.Second * 60) + } + + t := &TBSCertificate{ + Curve: curve, + Version: version, + Name: "test ca", + NotBefore: time.Unix(before.Unix(), 0), + NotAfter: time.Unix(after.Unix(), 0), + PublicKey: pub, + Networks: networks, + UnsafeNetworks: unsafeNetworks, + Groups: groups, + IsCA: true, + } + + c, err := t.Sign(nil, curve, priv) + if err != nil { + panic(err) + } + + pem, err := c.MarshalPEM() + if err != nil { + panic(err) + } + + return c, pub, priv, pem +} + +// NewTestCert will generate a signed certificate with the provided details. +// Expiry times are defaulted if you do not pass them in +func NewTestCert(v Version, curve Curve, ca Certificate, key []byte, name string, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (Certificate, []byte, []byte, []byte) { + if before.IsZero() { + before = testCertNow.Add(time.Second * -60) + } + + if after.IsZero() { + after = testCertNow.Add(time.Second * 60) + } + + if len(networks) == 0 { + networks = []netip.Prefix{netip.MustParsePrefix("10.0.0.123/8")} + } + + var pub, priv []byte + switch curve { + case Curve_CURVE25519: + pub, priv = X25519Keypair() + case Curve_P256: + pub, priv = P256Keypair() + default: + panic("unknown curve") + } + + nc := &TBSCertificate{ + Version: v, + Curve: curve, + Name: name, + Networks: networks, + UnsafeNetworks: unsafeNetworks, + Groups: groups, + NotBefore: time.Unix(before.Unix(), 0), + NotAfter: time.Unix(after.Unix(), 0), + PublicKey: pub, + IsCA: false, + } + + c, err := nc.Sign(ca, ca.Curve(), key) + if err != nil { + panic(err) + } + + pem, err := c.MarshalPEM() + if err != nil { + panic(err) + } + + return c, pub, MarshalPrivateKeyToPEM(curve, priv), pem +} + +func X25519Keypair() ([]byte, []byte) { + privkey := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, privkey); err != nil { + panic(err) + } + + pubkey, err := curve25519.X25519(privkey, curve25519.Basepoint) + if err != nil { + panic(err) + } + + return pubkey, privkey +} + +func P256Keypair() ([]byte, []byte) { + privkey, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + panic(err) + } + pubkey := privkey.PublicKey() + return pubkey.Bytes(), privkey.Bytes() +} diff --git a/cert/p256/p256.go b/cert/p256/p256.go new file mode 100644 index 00000000..dc609a35 --- /dev/null +++ b/cert/p256/p256.go @@ -0,0 +1,127 @@ +package p256 + +import ( + "crypto/elliptic" + "errors" + "math/big" + + "filippo.io/bigmod" + + "golang.org/x/crypto/cryptobyte" + "golang.org/x/crypto/cryptobyte/asn1" +) + +var halfN = new(big.Int).Rsh(elliptic.P256().Params().N, 1) +var nMod *bigmod.Modulus + +func init() { + n, err := bigmod.NewModulus(elliptic.P256().Params().N.Bytes()) + if err != nil { + panic(err) + } + nMod = n +} + +func IsNormalized(sig []byte) (bool, error) { + r, s, err := parseSignature(sig) + if err != nil { + return false, err + } + return checkLowS(r, s), nil +} + +func checkLowS(_, s []byte) bool { + bigS := new(big.Int).SetBytes(s) + // Check if S <= (N/2), because we want to include the midpoint in the set of low-s + return bigS.Cmp(halfN) <= 0 +} + +func swap(r, s []byte) ([]byte, []byte, error) { + var err error + bigS, err := bigmod.NewNat().SetBytes(s, nMod) + if err != nil { + return nil, nil, err + } + sNormalized := nMod.Nat().Sub(bigS, nMod) + + result := sNormalized.Bytes(nMod) + for len(result) > 1 && result[0] == 0 { + result = result[1:] + } + + return r, result, nil +} + +func Normalize(sig []byte) ([]byte, error) { + r, s, err := parseSignature(sig) + if err != nil { + return nil, err + } + + if checkLowS(r, s) { + return sig, nil + } + + newR, newS, err := swap(r, s) + if err != nil { + return nil, err + } + + return encodeSignature(newR, newS) +} + +// Swap will change sig between its current form to the opposite high or low form. +func Swap(sig []byte) ([]byte, error) { + r, s, err := parseSignature(sig) + if err != nil { + return nil, err + } + + newR, newS, err := swap(r, s) + if err != nil { + return nil, err + } + + return encodeSignature(newR, newS) +} + +// parseSignature taken exactly from crypto/ecdsa/ecdsa.go +func parseSignature(sig []byte) (r, s []byte, err error) { + var inner cryptobyte.String + input := cryptobyte.String(sig) + if !input.ReadASN1(&inner, asn1.SEQUENCE) || + !input.Empty() || + !inner.ReadASN1Integer(&r) || + !inner.ReadASN1Integer(&s) || + !inner.Empty() { + return nil, nil, errors.New("invalid ASN.1") + } + return r, s, nil +} + +func encodeSignature(r, s []byte) ([]byte, error) { + var b cryptobyte.Builder + b.AddASN1(asn1.SEQUENCE, func(b *cryptobyte.Builder) { + addASN1IntBytes(b, r) + addASN1IntBytes(b, s) + }) + return b.Bytes() +} + +// addASN1IntBytes encodes in ASN.1 a positive integer represented as +// a big-endian byte slice with zero or more leading zeroes. +func addASN1IntBytes(b *cryptobyte.Builder, bytes []byte) { + for len(bytes) > 0 && bytes[0] == 0 { + bytes = bytes[1:] + } + if len(bytes) == 0 { + b.SetError(errors.New("invalid integer")) + return + } + b.AddASN1(asn1.INTEGER, func(c *cryptobyte.Builder) { + if bytes[0]&0x80 != 0 { + c.AddUint8(0) + } + c.AddBytes(bytes) + }) +} diff --git a/cert/p256/p256_test.go b/cert/p256/p256_test.go new file mode 100644 index 00000000..486a7242 --- /dev/null +++ b/cert/p256/p256_test.go @@ -0,0 +1,28 @@ +package p256 + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFlipping(t *testing.T) { + priv, err1 := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err1) + + out, err := ecdsa.SignASN1(rand.Reader, priv, []byte("big chungus")) + require.NoError(t, err) + + r, s, err := parseSignature(out) + require.NoError(t, err) + + r, s1, err := swap(r, s) + require.NoError(t, err) + r, s2, err := swap(r, s1) + require.NoError(t, err) + require.Equal(t, s, s2) + require.NotEqual(t, s, s1) +} diff --git a/cert/pem.go b/cert/pem.go new file mode 100644 index 00000000..caa19b11 --- /dev/null +++ b/cert/pem.go @@ -0,0 +1,277 @@ +package cert + +import ( + "bytes" + "encoding/pem" + "errors" + "fmt" + + "golang.org/x/crypto/ed25519" +) + +var ErrTruncatedPEMBlock = errors.New("truncated PEM block") + +// SplitPEM is a split function for bufio.Scanner that returns each PEM block. +func SplitPEM(data []byte, atEOF bool) (advance int, token []byte, err error) { + // Look for the start of a PEM block + start := bytes.Index(data, []byte("-----BEGIN ")) + if start == -1 { + if atEOF && len(bytes.TrimSpace(data)) > 0 { + // Non-whitespace content with no PEM block + return 0, nil, ErrTruncatedPEMBlock + } + if atEOF { + return len(data), nil, nil + } + // Request more data + return 0, nil, nil + } + + // Look for the end marker + endMarkerStart := bytes.Index(data[start:], []byte("-----END ")) + if endMarkerStart == -1 { + if atEOF { + // Incomplete PEM block at EOF + return 0, nil, ErrTruncatedPEMBlock + } + // Need more data to find the end + return 0, nil, nil + } + + // Find the actual end of the END line (after the newline) + endMarkerStart += start + endLineEnd := bytes.IndexByte(data[endMarkerStart:], '\n') + var end int + if endLineEnd == -1 { + if atEOF { + // END marker without newline at EOF - take it anyway + end = len(data) + } else { + // Need more data + return 0, nil, nil + } + } else { + end = endMarkerStart + endLineEnd + 1 + } + + // Extract the PEM block + pemBlock := data[start:end] + + // Return the valid PEM block + return end, pemBlock, nil +} + +const ( //cert banners + CertificateBanner = "NEBULA CERTIFICATE" + CertificateV2Banner = "NEBULA CERTIFICATE V2" +) + +const ( //key-agreement-key banners + X25519PrivateKeyBanner = "NEBULA X25519 PRIVATE KEY" + X25519PublicKeyBanner = "NEBULA X25519 PUBLIC KEY" + P256PrivateKeyBanner = "NEBULA P256 PRIVATE KEY" + P256PublicKeyBanner = "NEBULA P256 PUBLIC KEY" +) + +/* including "ECDSA" in the P256 banners is a clue that these keys should be used only for signing */ +const ( //signing key banners + EncryptedECDSAP256PrivateKeyBanner = "NEBULA ECDSA P256 ENCRYPTED PRIVATE KEY" + ECDSAP256PrivateKeyBanner = "NEBULA ECDSA P256 PRIVATE KEY" + ECDSAP256PublicKeyBanner = "NEBULA ECDSA P256 PUBLIC KEY" + EncryptedEd25519PrivateKeyBanner = "NEBULA ED25519 ENCRYPTED PRIVATE KEY" + Ed25519PrivateKeyBanner = "NEBULA ED25519 PRIVATE KEY" + Ed25519PublicKeyBanner = "NEBULA ED25519 PUBLIC KEY" +) + +// UnmarshalCertificateFromPEM will try to unmarshal the first pem block in a byte array, returning any non consumed +// data or an error on failure +func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) { + p, r := pem.Decode(b) + if p == nil { + return nil, r, ErrInvalidPEMBlock + } + + c, err := unmarshalCertificateBlock(p) + if err != nil { + return nil, r, err + } + + return c, r, nil + +} + +// unmarshalCertificateBlock decodes a single PEM block into a certificate. +// It expects a Nebula certificate banner and returns ErrInvalidPEMCertificateBanner otherwise. +func unmarshalCertificateBlock(block *pem.Block) (Certificate, error) { + switch block.Type { + // Implementations must validate the resulting certificate contains valid information + case CertificateBanner: + return unmarshalCertificateV1(block.Bytes, nil) + case CertificateV2Banner: + return unmarshalCertificateV2(block.Bytes, nil, Curve_CURVE25519) + default: + return nil, ErrInvalidPEMCertificateBanner + } +} + +func marshalCertPublicKeyToPEM(c Certificate) []byte { + if c.IsCA() { + return MarshalSigningPublicKeyToPEM(c.Curve(), c.PublicKey()) + } else { + return MarshalPublicKeyToPEM(c.Curve(), c.PublicKey()) + } +} + +// MarshalPublicKeyToPEM returns a PEM representation of a public key used for ECDH. +// if your public key came from a certificate, prefer Certificate.PublicKeyPEM() if possible, to avoid mistakes! +func MarshalPublicKeyToPEM(curve Curve, b []byte) []byte { + switch curve { + case Curve_CURVE25519: + return pem.EncodeToMemory(&pem.Block{Type: X25519PublicKeyBanner, Bytes: b}) + case Curve_P256: + return pem.EncodeToMemory(&pem.Block{Type: P256PublicKeyBanner, Bytes: b}) + default: + return nil + } +} + +// MarshalSigningPublicKeyToPEM returns a PEM representation of a public key used for signing. +// if your public key came from a certificate, prefer Certificate.PublicKeyPEM() if possible, to avoid mistakes! +func MarshalSigningPublicKeyToPEM(curve Curve, b []byte) []byte { + switch curve { + case Curve_CURVE25519: + return pem.EncodeToMemory(&pem.Block{Type: Ed25519PublicKeyBanner, Bytes: b}) + case Curve_P256: + return pem.EncodeToMemory(&pem.Block{Type: ECDSAP256PublicKeyBanner, Bytes: b}) + default: + return nil + } +} + +// UnmarshalPublicKeyFromPEM will try to unmarshal the first pem block in a byte array, returning any non +// consumed data or an error on failure. Only key-agreement (ECDH) public key banners are accepted. +// Use UnmarshalSigningPublicKeyFromPEM for Ed25519/ECDSA banners. +func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) { + k, r := pem.Decode(b) + if k == nil { + return nil, r, 0, fmt.Errorf("input did not contain a valid PEM encoded block") + } + var expectedLen int + var curve Curve + switch k.Type { + case X25519PublicKeyBanner: + expectedLen = 32 + curve = Curve_CURVE25519 + case P256PublicKeyBanner: + // Uncompressed + expectedLen = 65 + curve = Curve_P256 + default: + return nil, r, 0, fmt.Errorf("bytes did not contain a proper public key banner") + } + if len(k.Bytes) != expectedLen { + return nil, r, 0, fmt.Errorf("key was not %d bytes, is invalid %s public key", expectedLen, curve) + } + return k.Bytes, r, curve, nil +} + +// UnmarshalSigningPublicKeyFromPEM will try to unmarshal the first pem block in a byte array, returning any non +// consumed data or an error on failure. Only Ed25519/ECDSA public key banners are accepted. +// Use UnmarshalPublicKeyFromPEM for X25519/P256 (ECDH) banners. +func UnmarshalSigningPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) { + k, r := pem.Decode(b) + if k == nil { + return nil, r, 0, fmt.Errorf("input did not contain a valid PEM encoded block") + } + var expectedLen int + var curve Curve + switch k.Type { + case Ed25519PublicKeyBanner: + expectedLen = 32 + curve = Curve_CURVE25519 + case ECDSAP256PublicKeyBanner: + // Uncompressed + expectedLen = 65 + curve = Curve_P256 + default: + return nil, r, 0, fmt.Errorf("bytes did not contain a proper Ed25519/ECDSA public key banner") + } + if len(k.Bytes) != expectedLen { + return nil, r, 0, fmt.Errorf("key was not %d bytes, is invalid %s public key", expectedLen, curve) + } + return k.Bytes, r, curve, nil +} + +func MarshalPrivateKeyToPEM(curve Curve, b []byte) []byte { + switch curve { + case Curve_CURVE25519: + return pem.EncodeToMemory(&pem.Block{Type: X25519PrivateKeyBanner, Bytes: b}) + case Curve_P256: + return pem.EncodeToMemory(&pem.Block{Type: P256PrivateKeyBanner, Bytes: b}) + default: + return nil + } +} + +func MarshalSigningPrivateKeyToPEM(curve Curve, b []byte) []byte { + switch curve { + case Curve_CURVE25519: + return pem.EncodeToMemory(&pem.Block{Type: Ed25519PrivateKeyBanner, Bytes: b}) + case Curve_P256: + return pem.EncodeToMemory(&pem.Block{Type: ECDSAP256PrivateKeyBanner, Bytes: b}) + default: + return nil + } +} + +// UnmarshalPrivateKeyFromPEM will try to unmarshal the first pem block in a byte array, returning any non +// consumed data or an error on failure +func UnmarshalPrivateKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) { + k, r := pem.Decode(b) + if k == nil { + return nil, r, 0, fmt.Errorf("input did not contain a valid PEM encoded block") + } + var expectedLen int + var curve Curve + switch k.Type { + case X25519PrivateKeyBanner: + expectedLen = 32 + curve = Curve_CURVE25519 + case P256PrivateKeyBanner: + expectedLen = 32 + curve = Curve_P256 + default: + return nil, r, 0, fmt.Errorf("bytes did not contain a proper private key banner") + } + if len(k.Bytes) != expectedLen { + return nil, r, 0, fmt.Errorf("key was not %d bytes, is invalid %s private key", expectedLen, curve) + } + return k.Bytes, r, curve, nil +} + +func UnmarshalSigningPrivateKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) { + k, r := pem.Decode(b) + if k == nil { + return nil, r, 0, fmt.Errorf("input did not contain a valid PEM encoded block") + } + var curve Curve + switch k.Type { + case EncryptedEd25519PrivateKeyBanner: + return nil, nil, Curve_CURVE25519, ErrPrivateKeyEncrypted + case EncryptedECDSAP256PrivateKeyBanner: + return nil, nil, Curve_P256, ErrPrivateKeyEncrypted + case Ed25519PrivateKeyBanner: + curve = Curve_CURVE25519 + if len(k.Bytes) != ed25519.PrivateKeySize { + return nil, r, 0, fmt.Errorf("key was not %d bytes, is invalid Ed25519 private key", ed25519.PrivateKeySize) + } + case ECDSAP256PrivateKeyBanner: + curve = Curve_P256 + if len(k.Bytes) != 32 { + return nil, r, 0, fmt.Errorf("key was not 32 bytes, is invalid ECDSA P256 private key") + } + default: + return nil, r, 0, fmt.Errorf("bytes did not contain a proper Ed25519/ECDSA private key banner") + } + return k.Bytes, r, curve, nil +} diff --git a/cert/pem_test.go b/cert/pem_test.go new file mode 100644 index 00000000..6012dab3 --- /dev/null +++ b/cert/pem_test.go @@ -0,0 +1,404 @@ +package cert + +import ( + "bufio" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func scanAll(t *testing.T, input string) ([]string, error) { + t.Helper() + scanner := bufio.NewScanner(strings.NewReader(input)) + scanner.Split(SplitPEM) + var blocks []string + for scanner.Scan() { + blocks = append(blocks, scanner.Text()) + } + return blocks, scanner.Err() +} + +func TestSplitPEM_Single(t *testing.T) { + input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\n" + blocks, err := scanAll(t, input) + require.NoError(t, err) + require.Len(t, blocks, 1) + require.Equal(t, input, blocks[0]) +} + +func TestSplitPEM_Multiple(t *testing.T) { + block1 := "-----BEGIN TEST-----\naaa\n-----END TEST-----\n" + block2 := "-----BEGIN TEST-----\nbbb\n-----END TEST-----\n" + blocks, err := scanAll(t, block1+block2) + require.NoError(t, err) + require.Len(t, blocks, 2) + require.Equal(t, block1, blocks[0]) + require.Equal(t, block2, blocks[1]) +} + +func TestSplitPEM_CommentsAndWhitespaceBetweenBlocks(t *testing.T) { + input := "# comment\n\n-----BEGIN TEST-----\naaa\n-----END TEST-----\n\n# another comment\n\n-----BEGIN TEST-----\nbbb\n-----END TEST-----\n" + blocks, err := scanAll(t, input) + require.NoError(t, err) + require.Len(t, blocks, 2) +} + +func TestSplitPEM_Empty(t *testing.T) { + blocks, err := scanAll(t, "") + require.NoError(t, err) + require.Empty(t, blocks) +} + +func TestSplitPEM_WhitespaceOnly(t *testing.T) { + blocks, err := scanAll(t, " \n\t\n ") + require.NoError(t, err) + require.Empty(t, blocks) +} + +func TestSplitPEM_TrailingGarbage(t *testing.T) { + input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\ngarbage" + blocks, err := scanAll(t, input) + require.ErrorIs(t, err, ErrTruncatedPEMBlock) + require.Len(t, blocks, 1) +} + +func TestSplitPEM_TruncatedBlock(t *testing.T) { + input := "-----BEGIN TEST-----\npartial data with no end" + _, err := scanAll(t, input) + require.ErrorIs(t, err, ErrTruncatedPEMBlock) +} + +func TestSplitPEM_NoEndNewline(t *testing.T) { + input := "-----BEGIN TEST-----\ndata\n-----END TEST-----" + blocks, err := scanAll(t, input) + require.NoError(t, err) + require.Len(t, blocks, 1) + require.Equal(t, input, blocks[0]) +} + +func TestSplitPEM_GarbageOnly(t *testing.T) { + _, err := scanAll(t, "this is not PEM data") + require.ErrorIs(t, err, ErrTruncatedPEMBlock) +} + +func TestUnmarshalCertificateFromPEM(t *testing.T) { + goodCert := []byte(` +# A good cert +-----BEGIN NEBULA CERTIFICATE----- +CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL +vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv +bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB +-----END NEBULA CERTIFICATE----- +`) + badBanner := []byte(`# A bad banner +-----BEGIN NOT A NEBULA CERTIFICATE----- +CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL +vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv +bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB +-----END NOT A NEBULA CERTIFICATE----- +`) + invalidPem := []byte(`# Not a valid PEM format +-BEGIN NEBULA CERTIFICATE----- +CkAKDm5lYnVsYSByb290IGNhKJfap9AFMJfg1+YGOiCUQGByMuNRhIlQBOyzXWbL +vcKBwDhov900phEfJ5DN3kABEkDCq5R8qBiu8sl54yVfgRcQXEDt3cHr8UTSLszv +bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB +-END NEBULA CERTIFICATE----`) + + certBundle := appendByteSlices(goodCert, badBanner, invalidPem) + + // Success test case + cert, rest, err := UnmarshalCertificateFromPEM(certBundle) + assert.NotNil(t, cert) + assert.Equal(t, rest, append(badBanner, invalidPem...)) + require.NoError(t, err) + + // Fail due to invalid banner. + cert, rest, err = UnmarshalCertificateFromPEM(rest) + assert.Nil(t, cert) + assert.Equal(t, rest, invalidPem) + require.EqualError(t, err, "bytes did not contain a proper certificate banner") + + // Fail due to invalid PEM format, because + // it's missing the requisite pre-encapsulation boundary. + cert, rest, err = UnmarshalCertificateFromPEM(rest) + assert.Nil(t, cert) + assert.Equal(t, rest, invalidPem) + require.EqualError(t, err, "input did not contain a valid PEM encoded block") +} + +func TestUnmarshalSigningPrivateKeyFromPEM(t *testing.T) { + privKey := []byte(`# A good key +-----BEGIN NEBULA ED25519 PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +-----END NEBULA ED25519 PRIVATE KEY----- +`) + privP256Key := []byte(`# A good key +-----BEGIN NEBULA ECDSA P256 PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA ECDSA P256 PRIVATE KEY----- +`) + shortKey := []byte(`# A short key +-----BEGIN NEBULA ED25519 PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +-----END NEBULA ED25519 PRIVATE KEY----- +`) + invalidBanner := []byte(`# Invalid banner +-----BEGIN NOT A NEBULA PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +-----END NOT A NEBULA PRIVATE KEY----- +`) + invalidPem := []byte(`# Not a valid PEM format +-BEGIN NEBULA ED25519 PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +-END NEBULA ED25519 PRIVATE KEY-----`) + + keyBundle := appendByteSlices(privKey, privP256Key, shortKey, invalidBanner, invalidPem) + + // Success test case + k, rest, curve, err := UnmarshalSigningPrivateKeyFromPEM(keyBundle) + assert.Len(t, k, 64) + assert.Equal(t, rest, appendByteSlices(privP256Key, shortKey, invalidBanner, invalidPem)) + assert.Equal(t, Curve_CURVE25519, curve) + require.NoError(t, err) + + // Success test case + k, rest, curve, err = UnmarshalSigningPrivateKeyFromPEM(rest) + assert.Len(t, k, 32) + assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) + assert.Equal(t, Curve_P256, curve) + require.NoError(t, err) + + // Fail due to short key + k, rest, curve, err = UnmarshalSigningPrivateKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) + require.EqualError(t, err, "key was not 64 bytes, is invalid Ed25519 private key") + + // Fail due to invalid banner + k, rest, curve, err = UnmarshalSigningPrivateKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, invalidPem) + require.EqualError(t, err, "bytes did not contain a proper Ed25519/ECDSA private key banner") + + // Fail due to invalid PEM format, because + // it's missing the requisite pre-encapsulation boundary. + k, rest, curve, err = UnmarshalSigningPrivateKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, invalidPem) + require.EqualError(t, err, "input did not contain a valid PEM encoded block") +} + +func TestUnmarshalPrivateKeyFromPEM(t *testing.T) { + privKey := []byte(`# A good key +-----BEGIN NEBULA X25519 PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA X25519 PRIVATE KEY----- +`) + privP256Key := []byte(`# A good key +-----BEGIN NEBULA P256 PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA P256 PRIVATE KEY----- +`) + shortKey := []byte(`# A short key +-----BEGIN NEBULA X25519 PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +-----END NEBULA X25519 PRIVATE KEY----- +`) + invalidBanner := []byte(`# Invalid banner +-----BEGIN NOT A NEBULA PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-----END NOT A NEBULA PRIVATE KEY----- +`) + invalidPem := []byte(`# Not a valid PEM format +-BEGIN NEBULA X25519 PRIVATE KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-END NEBULA X25519 PRIVATE KEY-----`) + + keyBundle := appendByteSlices(privKey, privP256Key, shortKey, invalidBanner, invalidPem) + + // Success test case + k, rest, curve, err := UnmarshalPrivateKeyFromPEM(keyBundle) + assert.Len(t, k, 32) + assert.Equal(t, rest, appendByteSlices(privP256Key, shortKey, invalidBanner, invalidPem)) + assert.Equal(t, Curve_CURVE25519, curve) + require.NoError(t, err) + + // Success test case + k, rest, curve, err = UnmarshalPrivateKeyFromPEM(rest) + assert.Len(t, k, 32) + assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) + assert.Equal(t, Curve_P256, curve) + require.NoError(t, err) + + // Fail due to short key + k, rest, curve, err = UnmarshalPrivateKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) + require.EqualError(t, err, "key was not 32 bytes, is invalid CURVE25519 private key") + + // Fail due to invalid banner + k, rest, curve, err = UnmarshalPrivateKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, invalidPem) + require.EqualError(t, err, "bytes did not contain a proper private key banner") + + // Fail due to invalid PEM format, because + // it's missing the requisite pre-encapsulation boundary. + k, rest, curve, err = UnmarshalPrivateKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, invalidPem) + require.EqualError(t, err, "input did not contain a valid PEM encoded block") +} + +func TestUnmarshalPublicKeyFromPEM(t *testing.T) { + t.Parallel() + pubKey := []byte(`# A good key +-----BEGIN NEBULA X25519 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA X25519 PUBLIC KEY----- +`) + pubP256Key := []byte(`# A good key +-----BEGIN NEBULA P256 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA P256 PUBLIC KEY----- +`) + signingKey := []byte(`# A signing key has the wrong scope for this function +-----BEGIN NEBULA ECDSA P256 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA ECDSA P256 PUBLIC KEY----- +`) + shortKey := []byte(`# A short key +-----BEGIN NEBULA X25519 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +-----END NEBULA X25519 PUBLIC KEY----- +`) + invalidBanner := []byte(`# Invalid banner +-----BEGIN NOT A NEBULA PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-----END NOT A NEBULA PUBLIC KEY----- +`) + invalidPem := []byte(`# Not a valid PEM format +-BEGIN NEBULA X25519 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-END NEBULA X25519 PUBLIC KEY-----`) + + keyBundle := appendByteSlices(pubKey, pubP256Key, signingKey, shortKey, invalidBanner, invalidPem) + + // X25519 key + k, rest, curve, err := UnmarshalPublicKeyFromPEM(keyBundle) + assert.Len(t, k, 32) + require.NoError(t, err) + assert.Equal(t, rest, appendByteSlices(pubP256Key, signingKey, shortKey, invalidBanner, invalidPem)) + assert.Equal(t, Curve_CURVE25519, curve) + + // P256 key + k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest) + assert.Len(t, k, 65) + require.NoError(t, err) + assert.Equal(t, rest, appendByteSlices(signingKey, shortKey, invalidBanner, invalidPem)) + assert.Equal(t, Curve_P256, curve) + + // Reject a signing public key (Ed25519/ECDSA banner) + k, rest, _, err = UnmarshalPublicKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) + require.EqualError(t, err, "bytes did not contain a proper public key banner") + + // Fail due to short key + k, rest, _, err = UnmarshalPublicKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) + require.EqualError(t, err, "key was not 32 bytes, is invalid CURVE25519 public key") + + // Fail due to invalid banner + k, rest, _, err = UnmarshalPublicKeyFromPEM(rest) + assert.Nil(t, k) + require.EqualError(t, err, "bytes did not contain a proper public key banner") + assert.Equal(t, rest, invalidPem) + + // Fail due to invalid PEM format, because + // it's missing the requisite pre-encapsulation boundary. + k, rest, _, err = UnmarshalPublicKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, invalidPem) + require.EqualError(t, err, "input did not contain a valid PEM encoded block") +} + +func TestUnmarshalSigningPublicKeyFromPEM(t *testing.T) { + t.Parallel() + pubKey := []byte(`# A good key +-----BEGIN NEBULA ED25519 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA ED25519 PUBLIC KEY----- +`) + pubP256Key := []byte(`# A good key +-----BEGIN NEBULA ECDSA P256 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA ECDSA P256 PUBLIC KEY----- +`) + ecdhKey := []byte(`# A key-agreement key has the wrong scope for this function +-----BEGIN NEBULA X25519 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-----END NEBULA X25519 PUBLIC KEY----- +`) + shortKey := []byte(`# A short key +-----BEGIN NEBULA ED25519 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +-----END NEBULA ED25519 PUBLIC KEY----- +`) + invalidBanner := []byte(`# Invalid banner +-----BEGIN NOT A NEBULA PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-----END NOT A NEBULA PUBLIC KEY----- +`) + invalidPem := []byte(`# Not a valid PEM format +-BEGIN NEBULA ED25519 PUBLIC KEY----- +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +-END NEBULA ED25519 PUBLIC KEY-----`) + + keyBundle := appendByteSlices(pubKey, pubP256Key, ecdhKey, shortKey, invalidBanner, invalidPem) + + // Ed25519 key + k, rest, curve, err := UnmarshalSigningPublicKeyFromPEM(keyBundle) + assert.Len(t, k, 32) + require.NoError(t, err) + assert.Equal(t, rest, appendByteSlices(pubP256Key, ecdhKey, shortKey, invalidBanner, invalidPem)) + assert.Equal(t, Curve_CURVE25519, curve) + + // ECDSA P256 key + k, rest, curve, err = UnmarshalSigningPublicKeyFromPEM(rest) + assert.Len(t, k, 65) + require.NoError(t, err) + assert.Equal(t, rest, appendByteSlices(ecdhKey, shortKey, invalidBanner, invalidPem)) + assert.Equal(t, Curve_P256, curve) + + // Reject a key-agreement public key (X25519/P256 banner) + k, rest, _, err = UnmarshalSigningPublicKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem)) + require.EqualError(t, err, "bytes did not contain a proper Ed25519/ECDSA public key banner") + + // Fail due to short key + k, rest, _, err = UnmarshalSigningPublicKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem)) + require.EqualError(t, err, "key was not 32 bytes, is invalid CURVE25519 public key") + + // Fail due to invalid banner + k, rest, _, err = UnmarshalSigningPublicKeyFromPEM(rest) + assert.Nil(t, k) + require.EqualError(t, err, "bytes did not contain a proper Ed25519/ECDSA public key banner") + assert.Equal(t, rest, invalidPem) + + // Fail due to invalid PEM format, because + // it's missing the requisite pre-encapsulation boundary. + k, rest, _, err = UnmarshalSigningPublicKeyFromPEM(rest) + assert.Nil(t, k) + assert.Equal(t, rest, invalidPem) + require.EqualError(t, err, "input did not contain a valid PEM encoded block") +} diff --git a/cert/sign.go b/cert/sign.go new file mode 100644 index 00000000..fbfffe4e --- /dev/null +++ b/cert/sign.go @@ -0,0 +1,170 @@ +package cert + +import ( + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "fmt" + "net/netip" + "time" + + "github.com/slackhq/nebula/cert/p256" +) + +// TBSCertificate represents a certificate intended to be signed. +// It is invalid to use this structure as a Certificate. +type TBSCertificate struct { + Version Version + Name string + Networks []netip.Prefix + UnsafeNetworks []netip.Prefix + Groups []string + IsCA bool + NotBefore time.Time + NotAfter time.Time + PublicKey []byte + Curve Curve + issuer string +} + +type beingSignedCertificate interface { + // fromTBSCertificate copies the values from the TBSCertificate to this versions internal representation + // Implementations must validate the resulting certificate contains valid information + fromTBSCertificate(*TBSCertificate) error + + // marshalForSigning returns the bytes that should be signed + marshalForSigning() ([]byte, error) + + // setSignature sets the signature for the certificate that has just been signed. The signature must not be blank. + setSignature([]byte) error +} + +type SignerLambda func(certBytes []byte) ([]byte, error) + +// Sign will create a sealed certificate using details provided by the TBSCertificate as long as those +// details do not violate constraints of the signing certificate. +// If the TBSCertificate is a CA then signer must be nil. +func (t *TBSCertificate) Sign(signer Certificate, curve Curve, key []byte) (Certificate, error) { + switch t.Curve { + case Curve_CURVE25519: + pk := ed25519.PrivateKey(key) + sp := func(certBytes []byte) ([]byte, error) { + sig := ed25519.Sign(pk, certBytes) + return sig, nil + } + return t.SignWith(signer, curve, sp) + case Curve_P256: + pk, err := ecdsa.ParseRawPrivateKey(elliptic.P256(), key) + if err != nil { + return nil, err + } + sp := func(certBytes []byte) ([]byte, error) { + // We need to hash first for ECDSA + // - https://pkg.go.dev/crypto/ecdsa#SignASN1 + hashed := sha256.Sum256(certBytes) + return ecdsa.SignASN1(rand.Reader, pk, hashed[:]) + } + return t.SignWith(signer, curve, sp) + default: + return nil, fmt.Errorf("invalid curve: %s", t.Curve) + } +} + +// SignWith does the same thing as sign, but uses the function in `sp` to calculate the signature. +// You should only use SignWith if you do not have direct access to your private key. +func (t *TBSCertificate) SignWith(signer Certificate, curve Curve, sp SignerLambda) (Certificate, error) { + if curve != t.Curve { + return nil, fmt.Errorf("curve in cert and private key supplied don't match") + } + + if signer != nil { + if t.IsCA { + return nil, fmt.Errorf("can not sign a CA certificate with another") + } + + err := checkCAConstraints(signer, t.NotBefore, t.NotAfter, t.Groups, t.Networks, t.UnsafeNetworks) + if err != nil { + return nil, err + } + + issuer, err := signer.Fingerprint() + if err != nil { + return nil, fmt.Errorf("error computing issuer: %v", err) + } + t.issuer = issuer + } else { + if !t.IsCA { + return nil, fmt.Errorf("self signed certificates must have IsCA set to true") + } + } + + var c beingSignedCertificate + switch t.Version { + case Version1: + c = &certificateV1{} + err := c.fromTBSCertificate(t) + if err != nil { + return nil, err + } + case Version2: + c = &certificateV2{} + err := c.fromTBSCertificate(t) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown cert version %d", t.Version) + } + + certBytes, err := c.marshalForSigning() + if err != nil { + return nil, err + } + + sig, err := sp(certBytes) + if err != nil { + return nil, err + } + + if curve == Curve_P256 { + sig, err = p256.Normalize(sig) + if err != nil { + return nil, err + } + } + + err = c.setSignature(sig) + if err != nil { + return nil, err + } + + sc, ok := c.(Certificate) + if !ok { + return nil, fmt.Errorf("invalid certificate") + } + + return sc, nil +} + +func comparePrefix(a, b netip.Prefix) int { + addr := a.Addr().Compare(b.Addr()) + if addr == 0 { + return a.Bits() - b.Bits() + } + return addr +} + +// findDuplicatePrefix returns an error if there is a duplicate prefix in the pre-sorted input slice sortedPrefixes +func findDuplicatePrefix(sortedPrefixes []netip.Prefix) error { + if len(sortedPrefixes) < 2 { + return nil + } + for i := 1; i < len(sortedPrefixes); i++ { + if comparePrefix(sortedPrefixes[i], sortedPrefixes[i-1]) == 0 { + return NewErrInvalidCertificateProperties("duplicate network detected: %v", sortedPrefixes[i]) + } + } + return nil +} diff --git a/cert/sign_test.go b/cert/sign_test.go new file mode 100644 index 00000000..bf4c9c0d --- /dev/null +++ b/cert/sign_test.go @@ -0,0 +1,137 @@ +package cert + +import ( + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "net/netip" + "testing" + "time" + + "github.com/slackhq/nebula/cert/p256" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCertificateV1_Sign(t *testing.T) { + before := time.Now().Add(time.Second * -60).Round(time.Second) + after := time.Now().Add(time.Second * 60).Round(time.Second) + pubKey := []byte("1234567890abcedfghij1234567890ab") + + tbs := TBSCertificate{ + Version: Version1, + Name: "testing", + Networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.1/24"), + mustParsePrefixUnmapped("10.1.1.2/16"), + }, + UnsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.2/24"), + mustParsePrefixUnmapped("9.1.1.3/24"), + }, + Groups: []string{"test-group1", "test-group2", "test-group3"}, + NotBefore: before, + NotAfter: after, + PublicKey: pubKey, + IsCA: false, + } + + pub, priv, err := ed25519.GenerateKey(rand.Reader) + c, err := tbs.Sign(&certificateV1{details: detailsV1{notBefore: before, notAfter: after}}, Curve_CURVE25519, priv) + require.NoError(t, err) + assert.NotNil(t, c) + assert.True(t, c.CheckSignature(pub)) + + b, err := c.Marshal() + require.NoError(t, err) + uc, err := unmarshalCertificateV1(b, nil) + require.NoError(t, err) + assert.NotNil(t, uc) +} + +func TestCertificateV1_SignP256(t *testing.T) { + before := time.Now().Add(time.Second * -60).Round(time.Second) + after := time.Now().Add(time.Second * 60).Round(time.Second) + pubKey := []byte("01234567890abcedfghij1234567890ab1234567890abcedfghij1234567890ab") + + tbs := TBSCertificate{ + Version: Version1, + Name: "testing", + Networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.1/24"), + mustParsePrefixUnmapped("10.1.1.2/16"), + }, + UnsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.2/24"), + mustParsePrefixUnmapped("9.1.1.3/16"), + }, + Groups: []string{"test-group1", "test-group2", "test-group3"}, + NotBefore: before, + NotAfter: after, + PublicKey: pubKey, + IsCA: false, + Curve: Curve_P256, + } + + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + pub := elliptic.Marshal(elliptic.P256(), priv.PublicKey.X, priv.PublicKey.Y) + rawPriv := priv.D.FillBytes(make([]byte, 32)) + + c, err := tbs.Sign(&certificateV1{details: detailsV1{notBefore: before, notAfter: after}}, Curve_P256, rawPriv) + require.NoError(t, err) + assert.NotNil(t, c) + assert.True(t, c.CheckSignature(pub)) + + b, err := c.Marshal() + require.NoError(t, err) + uc, err := unmarshalCertificateV1(b, nil) + require.NoError(t, err) + assert.NotNil(t, uc) +} + +func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) { + before := time.Now().Add(time.Second * -60).Round(time.Second) + after := time.Now().Add(time.Second * 60).Round(time.Second) + pubKey := []byte("01234567890abcedfghij1234567890ab1234567890abcedfghij1234567890ab") + + tbs := TBSCertificate{ + Version: Version1, + Name: "testing", + Networks: []netip.Prefix{ + mustParsePrefixUnmapped("10.1.1.1/24"), + mustParsePrefixUnmapped("10.1.1.2/16"), + }, + UnsafeNetworks: []netip.Prefix{ + mustParsePrefixUnmapped("9.1.1.2/24"), + mustParsePrefixUnmapped("9.1.1.3/16"), + }, + Groups: []string{"test-group1", "test-group2", "test-group3"}, + NotBefore: before, + NotAfter: after, + PublicKey: pubKey, + IsCA: true, + Curve: Curve_P256, + } + + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + pub := elliptic.Marshal(elliptic.P256(), priv.PublicKey.X, priv.PublicKey.Y) + rawPriv := priv.D.FillBytes(make([]byte, 32)) + + for i := 0; i < 1000; i++ { + if i&1 == 1 { + tbs.Version = Version1 + } else { + tbs.Version = Version2 + } + c, err := tbs.Sign(nil, Curve_P256, rawPriv) + require.NoError(t, err) + assert.NotNil(t, c) + assert.True(t, c.CheckSignature(pub)) + normie, err := p256.IsNormalized(c.Signature()) + require.NoError(t, err) + assert.True(t, normie) + } +} diff --git a/cert_test/cert.go b/cert_test/cert.go new file mode 100644 index 00000000..4c440aff --- /dev/null +++ b/cert_test/cert.go @@ -0,0 +1,223 @@ +package cert_test + +import ( + "crypto/ecdh" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "io" + "net/netip" + "time" + + "github.com/slackhq/nebula/cert" + "golang.org/x/crypto/curve25519" + "golang.org/x/crypto/ed25519" +) + +// testCertNow is the reference "now" used to derive default before/after times +// in NewTestCaCert and NewTestCert. Holding it fixed for the lifetime of the +// test binary keeps CA and leaf defaults aligned at the same second, so a leaf +// signed with default times can never expire after its CA on a rounding race. +var testCertNow = time.Now().Round(time.Second) + +// NewTestCaCert will create a new ca certificate +func NewTestCaCert(version cert.Version, curve cert.Curve, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (cert.Certificate, []byte, []byte, []byte) { + var err error + var pub, priv []byte + + switch curve { + case cert.Curve_CURVE25519: + pub, priv, err = ed25519.GenerateKey(rand.Reader) + case cert.Curve_P256: + privk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + panic(err) + } + + pub = elliptic.Marshal(elliptic.P256(), privk.PublicKey.X, privk.PublicKey.Y) + priv = privk.D.FillBytes(make([]byte, 32)) + default: + // There is no default to allow the underlying lib to respond with an error + } + + if before.IsZero() { + before = testCertNow.Add(time.Second * -60) + } + if after.IsZero() { + after = testCertNow.Add(time.Second * 60) + } + + t := &cert.TBSCertificate{ + Curve: curve, + Version: version, + Name: "test ca", + NotBefore: time.Unix(before.Unix(), 0), + NotAfter: time.Unix(after.Unix(), 0), + PublicKey: pub, + Networks: networks, + UnsafeNetworks: unsafeNetworks, + Groups: groups, + IsCA: true, + } + + c, err := t.Sign(nil, curve, priv) + if err != nil { + panic(err) + } + + pem, err := c.MarshalPEM() + if err != nil { + panic(err) + } + + return c, pub, priv, pem +} + +// NewTestCert will generate a signed certificate with the provided details. +// Expiry times are defaulted if you do not pass them in +func NewTestCert(v cert.Version, curve cert.Curve, ca cert.Certificate, key []byte, name string, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (cert.Certificate, []byte, []byte, []byte) { + if before.IsZero() { + before = testCertNow.Add(time.Second * -60) + } + + if after.IsZero() { + after = testCertNow.Add(time.Second * 60) + } + + var pub, priv []byte + switch curve { + case cert.Curve_CURVE25519: + pub, priv = X25519Keypair() + case cert.Curve_P256: + pub, priv = P256Keypair() + default: + panic("unknown curve") + } + + nc := &cert.TBSCertificate{ + Version: v, + Curve: curve, + Name: name, + Networks: networks, + UnsafeNetworks: unsafeNetworks, + Groups: groups, + NotBefore: time.Unix(before.Unix(), 0), + NotAfter: time.Unix(after.Unix(), 0), + PublicKey: pub, + IsCA: false, + } + + c, err := nc.Sign(ca, ca.Curve(), key) + if err != nil { + panic(err) + } + + pem, err := c.MarshalPEM() + if err != nil { + panic(err) + } + + return c, pub, cert.MarshalPrivateKeyToPEM(curve, priv), pem +} + +func NewTestCertDifferentVersion(c cert.Certificate, v cert.Version, ca cert.Certificate, key []byte) (cert.Certificate, []byte) { + nc := &cert.TBSCertificate{ + Version: v, + Curve: c.Curve(), + Name: c.Name(), + Networks: c.Networks(), + UnsafeNetworks: c.UnsafeNetworks(), + Groups: c.Groups(), + NotBefore: time.Unix(c.NotBefore().Unix(), 0), + NotAfter: time.Unix(c.NotAfter().Unix(), 0), + PublicKey: c.PublicKey(), + IsCA: false, + } + + c, err := nc.Sign(ca, ca.Curve(), key) + if err != nil { + panic(err) + } + + pem, err := c.MarshalPEM() + if err != nil { + panic(err) + } + + return c, pem +} + +func X25519Keypair() ([]byte, []byte) { + privkey := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, privkey); err != nil { + panic(err) + } + + pubkey, err := curve25519.X25519(privkey, curve25519.Basepoint) + if err != nil { + panic(err) + } + + return pubkey, privkey +} + +func P256Keypair() ([]byte, []byte) { + privkey, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + panic(err) + } + pubkey := privkey.PublicKey() + return pubkey.Bytes(), privkey.Bytes() +} + +// DummyCert is a minimal cert.Certificate implementation for testing error paths. +type DummyCert struct { + Version_ cert.Version + Curve_ cert.Curve + Groups_ []string + IsCA_ bool + Issuer_ string + Name_ string + Networks_ []netip.Prefix + NotAfter_ time.Time + NotBefore_ time.Time + PublicKey_ []byte + Signature_ []byte + UnsafeNetworks_ []netip.Prefix +} + +func (d *DummyCert) Version() cert.Version { return d.Version_ } +func (d *DummyCert) Curve() cert.Curve { return d.Curve_ } +func (d *DummyCert) Groups() []string { return d.Groups_ } +func (d *DummyCert) IsCA() bool { return d.IsCA_ } +func (d *DummyCert) Issuer() string { return d.Issuer_ } +func (d *DummyCert) Name() string { return d.Name_ } +func (d *DummyCert) Networks() []netip.Prefix { return d.Networks_ } +func (d *DummyCert) NotAfter() time.Time { return d.NotAfter_ } +func (d *DummyCert) NotBefore() time.Time { return d.NotBefore_ } +func (d *DummyCert) PublicKey() []byte { return d.PublicKey_ } +func (d *DummyCert) Signature() []byte { return d.Signature_ } +func (d *DummyCert) UnsafeNetworks() []netip.Prefix { return d.UnsafeNetworks_ } +func (d *DummyCert) Fingerprint() (string, error) { return "", nil } +func (d *DummyCert) CheckSignature(key []byte) bool { return false } +func (d *DummyCert) MarshalForHandshakes() ([]byte, error) { return nil, nil } +func (d *DummyCert) MarshalPEM() ([]byte, error) { return nil, nil } +func (d *DummyCert) MarshalJSON() ([]byte, error) { return nil, nil } +func (d *DummyCert) Marshal() ([]byte, error) { return nil, nil } +func (d *DummyCert) String() string { return "dummy" } +func (d *DummyCert) Copy() cert.Certificate { return d } +func (d *DummyCert) VerifyPrivateKey(c cert.Curve, k []byte) error { return nil } +func (d *DummyCert) Expired(time.Time) bool { return false } +func (d *DummyCert) MarshalPublicKeyPEM() []byte { return nil } +func (d *DummyCert) PublicKeyPEM() []byte { return nil } + +// NewTestCAPool creates a CAPool from the given CA certificates, panicking on error. +func NewTestCAPool(cas ...cert.Certificate) *cert.CAPool { + pool := cert.NewCAPool() + for _, ca := range cas { + if err := pool.AddCA(ca); err != nil { + panic(err) + } + } + return pool +} diff --git a/cidr/parse.go b/cidr/parse.go deleted file mode 100644 index 74367f6e..00000000 --- a/cidr/parse.go +++ /dev/null @@ -1,10 +0,0 @@ -package cidr - -import "net" - -// Parse is a convenience function that returns only the IPNet -// This function ignores errors since it is primarily a test helper, the result could be nil -func Parse(s string) *net.IPNet { - _, c, _ := net.ParseCIDR(s) - return c -} diff --git a/cidr/tree4.go b/cidr/tree4.go deleted file mode 100644 index c5ebe54a..00000000 --- a/cidr/tree4.go +++ /dev/null @@ -1,203 +0,0 @@ -package cidr - -import ( - "net" - - "github.com/slackhq/nebula/iputil" -) - -type Node[T any] struct { - left *Node[T] - right *Node[T] - parent *Node[T] - hasValue bool - value T -} - -type entry[T any] struct { - CIDR *net.IPNet - Value T -} - -type Tree4[T any] struct { - root *Node[T] - list []entry[T] -} - -const ( - startbit = iputil.VpnIp(0x80000000) -) - -func NewTree4[T any]() *Tree4[T] { - tree := new(Tree4[T]) - tree.root = &Node[T]{} - tree.list = []entry[T]{} - return tree -} - -func (tree *Tree4[T]) AddCIDR(cidr *net.IPNet, val T) { - bit := startbit - node := tree.root - next := tree.root - - ip := iputil.Ip2VpnIp(cidr.IP) - mask := iputil.Ip2VpnIp(cidr.Mask) - - // Find our last ancestor in the tree - for bit&mask != 0 { - if ip&bit != 0 { - next = node.right - } else { - next = node.left - } - - if next == nil { - break - } - - bit = bit >> 1 - node = next - } - - // We already have this range so update the value - if next != nil { - addCIDR := cidr.String() - for i, v := range tree.list { - if addCIDR == v.CIDR.String() { - tree.list = append(tree.list[:i], tree.list[i+1:]...) - break - } - } - - tree.list = append(tree.list, entry[T]{CIDR: cidr, Value: val}) - node.value = val - node.hasValue = true - return - } - - // Build up the rest of the tree we don't already have - for bit&mask != 0 { - next = &Node[T]{} - next.parent = node - - if ip&bit != 0 { - node.right = next - } else { - node.left = next - } - - bit >>= 1 - node = next - } - - // Final node marks our cidr, set the value - node.value = val - node.hasValue = true - tree.list = append(tree.list, entry[T]{CIDR: cidr, Value: val}) -} - -// Contains finds the first match, which may be the least specific -func (tree *Tree4[T]) Contains(ip iputil.VpnIp) (ok bool, value T) { - bit := startbit - node := tree.root - - for node != nil { - if node.hasValue { - return true, node.value - } - - if ip&bit != 0 { - node = node.right - } else { - node = node.left - } - - bit >>= 1 - - } - - return false, value -} - -// MostSpecificContains finds the most specific match -func (tree *Tree4[T]) MostSpecificContains(ip iputil.VpnIp) (ok bool, value T) { - bit := startbit - node := tree.root - - for node != nil { - if node.hasValue { - value = node.value - ok = true - } - - if ip&bit != 0 { - node = node.right - } else { - node = node.left - } - - bit >>= 1 - } - - return ok, value -} - -type eachFunc[T any] func(T) bool - -// EachContains will call a function, passing the value, for each entry until the function returns true or the search is complete -// The final return value will be true if the provided function returned true -func (tree *Tree4[T]) EachContains(ip iputil.VpnIp, each eachFunc[T]) bool { - bit := startbit - node := tree.root - - for node != nil { - if node.hasValue { - // If the each func returns true then we can exit the loop - if each(node.value) { - return true - } - } - - if ip&bit != 0 { - node = node.right - } else { - node = node.left - } - - bit >>= 1 - } - - return false -} - -// GetCIDR returns the entry added by the most recent matching AddCIDR call -func (tree *Tree4[T]) GetCIDR(cidr *net.IPNet) (ok bool, value T) { - bit := startbit - node := tree.root - - ip := iputil.Ip2VpnIp(cidr.IP) - mask := iputil.Ip2VpnIp(cidr.Mask) - - // Find our last ancestor in the tree - for node != nil && bit&mask != 0 { - if ip&bit != 0 { - node = node.right - } else { - node = node.left - } - - bit = bit >> 1 - } - - if bit&mask == 0 && node != nil { - value = node.value - ok = node.hasValue - } - - return ok, value -} - -// List will return all CIDRs and their current values. Do not modify the contents! -func (tree *Tree4[T]) List() []entry[T] { - return tree.list -} diff --git a/cidr/tree4_test.go b/cidr/tree4_test.go deleted file mode 100644 index cd17be4d..00000000 --- a/cidr/tree4_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package cidr - -import ( - "net" - "testing" - - "github.com/slackhq/nebula/iputil" - "github.com/stretchr/testify/assert" -) - -func TestCIDRTree_List(t *testing.T) { - tree := NewTree4[string]() - tree.AddCIDR(Parse("1.0.0.0/16"), "1") - tree.AddCIDR(Parse("1.0.0.0/8"), "2") - tree.AddCIDR(Parse("1.0.0.0/16"), "3") - tree.AddCIDR(Parse("1.0.0.0/16"), "4") - list := tree.List() - assert.Len(t, list, 2) - assert.Equal(t, "1.0.0.0/8", list[0].CIDR.String()) - assert.Equal(t, "2", list[0].Value) - assert.Equal(t, "1.0.0.0/16", list[1].CIDR.String()) - assert.Equal(t, "4", list[1].Value) -} - -func TestCIDRTree_Contains(t *testing.T) { - tree := NewTree4[string]() - tree.AddCIDR(Parse("1.0.0.0/8"), "1") - tree.AddCIDR(Parse("2.1.0.0/16"), "2") - tree.AddCIDR(Parse("3.1.1.0/24"), "3") - tree.AddCIDR(Parse("4.1.1.0/24"), "4a") - tree.AddCIDR(Parse("4.1.1.1/32"), "4b") - tree.AddCIDR(Parse("4.1.2.1/32"), "4c") - tree.AddCIDR(Parse("254.0.0.0/4"), "5") - - tests := []struct { - Found bool - Result interface{} - IP string - }{ - {true, "1", "1.0.0.0"}, - {true, "1", "1.255.255.255"}, - {true, "2", "2.1.0.0"}, - {true, "2", "2.1.255.255"}, - {true, "3", "3.1.1.0"}, - {true, "3", "3.1.1.255"}, - {true, "4a", "4.1.1.255"}, - {true, "4a", "4.1.1.1"}, - {true, "5", "240.0.0.0"}, - {true, "5", "255.255.255.255"}, - {false, "", "239.0.0.0"}, - {false, "", "4.1.2.2"}, - } - - for _, tt := range tests { - ok, r := tree.Contains(iputil.Ip2VpnIp(net.ParseIP(tt.IP))) - assert.Equal(t, tt.Found, ok) - assert.Equal(t, tt.Result, r) - } - - tree = NewTree4[string]() - tree.AddCIDR(Parse("1.1.1.1/0"), "cool") - ok, r := tree.Contains(iputil.Ip2VpnIp(net.ParseIP("0.0.0.0"))) - assert.True(t, ok) - assert.Equal(t, "cool", r) - - ok, r = tree.Contains(iputil.Ip2VpnIp(net.ParseIP("255.255.255.255"))) - assert.True(t, ok) - assert.Equal(t, "cool", r) -} - -func TestCIDRTree_MostSpecificContains(t *testing.T) { - tree := NewTree4[string]() - tree.AddCIDR(Parse("1.0.0.0/8"), "1") - tree.AddCIDR(Parse("2.1.0.0/16"), "2") - tree.AddCIDR(Parse("3.1.1.0/24"), "3") - tree.AddCIDR(Parse("4.1.1.0/24"), "4a") - tree.AddCIDR(Parse("4.1.1.0/30"), "4b") - tree.AddCIDR(Parse("4.1.1.1/32"), "4c") - tree.AddCIDR(Parse("254.0.0.0/4"), "5") - - tests := []struct { - Found bool - Result interface{} - IP string - }{ - {true, "1", "1.0.0.0"}, - {true, "1", "1.255.255.255"}, - {true, "2", "2.1.0.0"}, - {true, "2", "2.1.255.255"}, - {true, "3", "3.1.1.0"}, - {true, "3", "3.1.1.255"}, - {true, "4a", "4.1.1.255"}, - {true, "4b", "4.1.1.2"}, - {true, "4c", "4.1.1.1"}, - {true, "5", "240.0.0.0"}, - {true, "5", "255.255.255.255"}, - {false, "", "239.0.0.0"}, - {false, "", "4.1.2.2"}, - } - - for _, tt := range tests { - ok, r := tree.MostSpecificContains(iputil.Ip2VpnIp(net.ParseIP(tt.IP))) - assert.Equal(t, tt.Found, ok) - assert.Equal(t, tt.Result, r) - } - - tree = NewTree4[string]() - tree.AddCIDR(Parse("1.1.1.1/0"), "cool") - ok, r := tree.MostSpecificContains(iputil.Ip2VpnIp(net.ParseIP("0.0.0.0"))) - assert.True(t, ok) - assert.Equal(t, "cool", r) - - ok, r = tree.MostSpecificContains(iputil.Ip2VpnIp(net.ParseIP("255.255.255.255"))) - assert.True(t, ok) - assert.Equal(t, "cool", r) -} - -func TestTree4_GetCIDR(t *testing.T) { - tree := NewTree4[string]() - tree.AddCIDR(Parse("1.0.0.0/8"), "1") - tree.AddCIDR(Parse("2.1.0.0/16"), "2") - tree.AddCIDR(Parse("3.1.1.0/24"), "3") - tree.AddCIDR(Parse("4.1.1.0/24"), "4a") - tree.AddCIDR(Parse("4.1.1.1/32"), "4b") - tree.AddCIDR(Parse("4.1.2.1/32"), "4c") - tree.AddCIDR(Parse("254.0.0.0/4"), "5") - - tests := []struct { - Found bool - Result interface{} - IPNet *net.IPNet - }{ - {true, "1", Parse("1.0.0.0/8")}, - {true, "2", Parse("2.1.0.0/16")}, - {true, "3", Parse("3.1.1.0/24")}, - {true, "4a", Parse("4.1.1.0/24")}, - {true, "4b", Parse("4.1.1.1/32")}, - {true, "4c", Parse("4.1.2.1/32")}, - {true, "5", Parse("254.0.0.0/4")}, - {false, "", Parse("2.0.0.0/8")}, - } - - for _, tt := range tests { - ok, r := tree.GetCIDR(tt.IPNet) - assert.Equal(t, tt.Found, ok) - assert.Equal(t, tt.Result, r) - } -} - -func BenchmarkCIDRTree_Contains(b *testing.B) { - tree := NewTree4[string]() - tree.AddCIDR(Parse("1.1.0.0/16"), "1") - tree.AddCIDR(Parse("1.2.1.1/32"), "1") - tree.AddCIDR(Parse("192.2.1.1/32"), "1") - tree.AddCIDR(Parse("172.2.1.1/32"), "1") - - ip := iputil.Ip2VpnIp(net.ParseIP("1.2.1.1")) - b.Run("found", func(b *testing.B) { - for i := 0; i < b.N; i++ { - tree.Contains(ip) - } - }) - - ip = iputil.Ip2VpnIp(net.ParseIP("1.2.1.255")) - b.Run("not found", func(b *testing.B) { - for i := 0; i < b.N; i++ { - tree.Contains(ip) - } - }) -} diff --git a/cidr/tree6.go b/cidr/tree6.go deleted file mode 100644 index 3f2cd2a4..00000000 --- a/cidr/tree6.go +++ /dev/null @@ -1,189 +0,0 @@ -package cidr - -import ( - "net" - - "github.com/slackhq/nebula/iputil" -) - -const startbit6 = uint64(1 << 63) - -type Tree6[T any] struct { - root4 *Node[T] - root6 *Node[T] -} - -func NewTree6[T any]() *Tree6[T] { - tree := new(Tree6[T]) - tree.root4 = &Node[T]{} - tree.root6 = &Node[T]{} - return tree -} - -func (tree *Tree6[T]) AddCIDR(cidr *net.IPNet, val T) { - var node, next *Node[T] - - cidrIP, ipv4 := isIPV4(cidr.IP) - if ipv4 { - node = tree.root4 - next = tree.root4 - - } else { - node = tree.root6 - next = tree.root6 - } - - for i := 0; i < len(cidrIP); i += 4 { - ip := iputil.Ip2VpnIp(cidrIP[i : i+4]) - mask := iputil.Ip2VpnIp(cidr.Mask[i : i+4]) - bit := startbit - - // Find our last ancestor in the tree - for bit&mask != 0 { - if ip&bit != 0 { - next = node.right - } else { - next = node.left - } - - if next == nil { - break - } - - bit = bit >> 1 - node = next - } - - // Build up the rest of the tree we don't already have - for bit&mask != 0 { - next = &Node[T]{} - next.parent = node - - if ip&bit != 0 { - node.right = next - } else { - node.left = next - } - - bit >>= 1 - node = next - } - } - - // Final node marks our cidr, set the value - node.value = val - node.hasValue = true -} - -// Finds the most specific match -func (tree *Tree6[T]) MostSpecificContains(ip net.IP) (ok bool, value T) { - var node *Node[T] - - wholeIP, ipv4 := isIPV4(ip) - if ipv4 { - node = tree.root4 - } else { - node = tree.root6 - } - - for i := 0; i < len(wholeIP); i += 4 { - ip := iputil.Ip2VpnIp(wholeIP[i : i+4]) - bit := startbit - - for node != nil { - if node.hasValue { - value = node.value - ok = true - } - - if bit == 0 { - break - } - - if ip&bit != 0 { - node = node.right - } else { - node = node.left - } - - bit >>= 1 - } - } - - return ok, value -} - -func (tree *Tree6[T]) MostSpecificContainsIpV4(ip iputil.VpnIp) (ok bool, value T) { - bit := startbit - node := tree.root4 - - for node != nil { - if node.hasValue { - value = node.value - ok = true - } - - if ip&bit != 0 { - node = node.right - } else { - node = node.left - } - - bit >>= 1 - } - - return ok, value -} - -func (tree *Tree6[T]) MostSpecificContainsIpV6(hi, lo uint64) (ok bool, value T) { - ip := hi - node := tree.root6 - - for i := 0; i < 2; i++ { - bit := startbit6 - - for node != nil { - if node.hasValue { - value = node.value - ok = true - } - - if bit == 0 { - break - } - - if ip&bit != 0 { - node = node.right - } else { - node = node.left - } - - bit >>= 1 - } - - ip = lo - } - - return ok, value -} - -func isIPV4(ip net.IP) (net.IP, bool) { - if len(ip) == net.IPv4len { - return ip, true - } - - if len(ip) == net.IPv6len && isZeros(ip[0:10]) && ip[10] == 0xff && ip[11] == 0xff { - return ip[12:16], true - } - - return ip, false -} - -func isZeros(p net.IP) bool { - for i := 0; i < len(p); i++ { - if p[i] != 0 { - return false - } - } - return true -} diff --git a/cidr/tree6_test.go b/cidr/tree6_test.go deleted file mode 100644 index eb159ec7..00000000 --- a/cidr/tree6_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package cidr - -import ( - "encoding/binary" - "net" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCIDR6Tree_MostSpecificContains(t *testing.T) { - tree := NewTree6[string]() - tree.AddCIDR(Parse("1.0.0.0/8"), "1") - tree.AddCIDR(Parse("2.1.0.0/16"), "2") - tree.AddCIDR(Parse("3.1.1.0/24"), "3") - tree.AddCIDR(Parse("4.1.1.1/24"), "4a") - tree.AddCIDR(Parse("4.1.1.1/30"), "4b") - tree.AddCIDR(Parse("4.1.1.1/32"), "4c") - tree.AddCIDR(Parse("254.0.0.0/4"), "5") - tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/64"), "6a") - tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/80"), "6b") - tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/96"), "6c") - - tests := []struct { - Found bool - Result interface{} - IP string - }{ - {true, "1", "1.0.0.0"}, - {true, "1", "1.255.255.255"}, - {true, "2", "2.1.0.0"}, - {true, "2", "2.1.255.255"}, - {true, "3", "3.1.1.0"}, - {true, "3", "3.1.1.255"}, - {true, "4a", "4.1.1.255"}, - {true, "4b", "4.1.1.2"}, - {true, "4c", "4.1.1.1"}, - {true, "5", "240.0.0.0"}, - {true, "5", "255.255.255.255"}, - {true, "6a", "1:2:0:4:1:1:1:1"}, - {true, "6b", "1:2:0:4:5:1:1:1"}, - {true, "6c", "1:2:0:4:5:0:0:0"}, - {false, "", "239.0.0.0"}, - {false, "", "4.1.2.2"}, - } - - for _, tt := range tests { - ok, r := tree.MostSpecificContains(net.ParseIP(tt.IP)) - assert.Equal(t, tt.Found, ok) - assert.Equal(t, tt.Result, r) - } - - tree = NewTree6[string]() - tree.AddCIDR(Parse("1.1.1.1/0"), "cool") - tree.AddCIDR(Parse("::/0"), "cool6") - ok, r := tree.MostSpecificContains(net.ParseIP("0.0.0.0")) - assert.True(t, ok) - assert.Equal(t, "cool", r) - - ok, r = tree.MostSpecificContains(net.ParseIP("255.255.255.255")) - assert.True(t, ok) - assert.Equal(t, "cool", r) - - ok, r = tree.MostSpecificContains(net.ParseIP("::")) - assert.True(t, ok) - assert.Equal(t, "cool6", r) - - ok, r = tree.MostSpecificContains(net.ParseIP("1:2:3:4:5:6:7:8")) - assert.True(t, ok) - assert.Equal(t, "cool6", r) -} - -func TestCIDR6Tree_MostSpecificContainsIpV6(t *testing.T) { - tree := NewTree6[string]() - tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/64"), "6a") - tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/80"), "6b") - tree.AddCIDR(Parse("1:2:0:4:5:0:0:0/96"), "6c") - - tests := []struct { - Found bool - Result interface{} - IP string - }{ - {true, "6a", "1:2:0:4:1:1:1:1"}, - {true, "6b", "1:2:0:4:5:1:1:1"}, - {true, "6c", "1:2:0:4:5:0:0:0"}, - } - - for _, tt := range tests { - ip := net.ParseIP(tt.IP) - hi := binary.BigEndian.Uint64(ip[:8]) - lo := binary.BigEndian.Uint64(ip[8:]) - - ok, r := tree.MostSpecificContainsIpV6(hi, lo) - assert.Equal(t, tt.Found, ok) - assert.Equal(t, tt.Result, r) - } -} diff --git a/cmd/nebula-cert/ca.go b/cmd/nebula-cert/ca.go index 69df4ab0..3145f445 100644 --- a/cmd/nebula-cert/ca.go +++ b/cmd/nebula-cert/ca.go @@ -8,13 +8,14 @@ import ( "fmt" "io" "math" - "net" + "net/netip" "os" "strings" "time" "github.com/skip2/go-qrcode" "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/pkclient" "golang.org/x/crypto/ed25519" ) @@ -26,32 +27,43 @@ type caFlags struct { outCertPath *string outQRPath *string groups *string - ips *string - subnets *string + networks *string + unsafeNetworks *string argonMemory *uint argonIterations *uint argonParallelism *uint encryption *bool + version *uint - curve *string + curve *string + p11url *string + + // Deprecated options + ips *string + subnets *string } func newCaFlags() *caFlags { cf := caFlags{set: flag.NewFlagSet("ca", flag.ContinueOnError)} cf.set.Usage = func() {} cf.name = cf.set.String("name", "", "Required: name of the certificate authority") + cf.version = cf.set.Uint("version", uint(cert.Version2), "Optional: version of the certificate format to use") cf.duration = cf.set.Duration("duration", time.Duration(time.Hour*8760), "Optional: amount of time the certificate should be valid for. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\"") cf.outKeyPath = cf.set.String("out-key", "ca.key", "Optional: path to write the private key to") cf.outCertPath = cf.set.String("out-crt", "ca.crt", "Optional: path to write the certificate to") cf.outQRPath = cf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate") cf.groups = cf.set.String("groups", "", "Optional: comma separated list of groups. This will limit which groups subordinate certs can use") - cf.ips = cf.set.String("ips", "", "Optional: comma separated list of ipv4 address and network in CIDR notation. This will limit which ipv4 addresses and networks subordinate certs can use for ip addresses") - cf.subnets = cf.set.String("subnets", "", "Optional: comma separated list of ipv4 address and network in CIDR notation. This will limit which ipv4 addresses and networks subordinate certs can use in subnets") + cf.networks = cf.set.String("networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in networks") + cf.unsafeNetworks = cf.set.String("unsafe-networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in unsafe networks") cf.argonMemory = cf.set.Uint("argon-memory", 2*1024*1024, "Optional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase") cf.argonParallelism = cf.set.Uint("argon-parallelism", 4, "Optional: Argon2 parallelism parameter used for encrypted private key passphrase") cf.argonIterations = cf.set.Uint("argon-iterations", 1, "Optional: Argon2 iterations parameter used for encrypted private key passphrase") cf.encryption = cf.set.Bool("encrypt", false, "Optional: prompt for passphrase and write out-key in an encrypted format") cf.curve = cf.set.String("curve", "25519", "EdDSA/ECDSA Curve (25519, P256)") + cf.p11url = p11Flag(cf.set) + + cf.ips = cf.set.String("ips", "", "Deprecated, see -networks") + cf.subnets = cf.set.String("subnets", "", "Deprecated, see -unsafe-networks") return &cf } @@ -76,17 +88,34 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error return err } + isP11 := len(*cf.p11url) > 0 + if err := mustFlagString("name", cf.name); err != nil { return err } - if err := mustFlagString("out-key", cf.outKeyPath); err != nil { - return err + if !isP11 { + if err = mustFlagString("out-key", cf.outKeyPath); err != nil { + return err + } + } else { + // out-key is meaningless under PKCS#11 because the private key never + // leaves the HSM; reject it so we never silently accept or claim a + // stdout slot for it. + outKeySet := false + cf.set.Visit(func(f *flag.Flag) { + if f.Name == "out-key" { + outKeySet = true + } + }) + if outKeySet { + return newHelpErrorf("cannot set -out-key with -pkcs11") + } } if err := mustFlagString("out-crt", cf.outCertPath); err != nil { return err } var kdfParams *cert.Argon2Parameters - if *cf.encryption { + if !isP11 && *cf.encryption { if kdfParams, err = parseArgonParameters(*cf.argonMemory, *cf.argonParallelism, *cf.argonIterations); err != nil { return err } @@ -106,133 +135,201 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error } } - var ips []*net.IPNet - if *cf.ips != "" { - for _, rs := range strings.Split(*cf.ips, ",") { + version := cert.Version(*cf.version) + if version != cert.Version1 && version != cert.Version2 { + return newHelpErrorf("-version must be either %v or %v", cert.Version1, cert.Version2) + } + + var networks []netip.Prefix + if *cf.networks == "" && *cf.ips != "" { + // Pull up deprecated -ips flag if needed + *cf.networks = *cf.ips + } + + if *cf.networks != "" { + for _, rs := range strings.Split(*cf.networks, ",") { rs := strings.Trim(rs, " ") if rs != "" { - ip, ipNet, err := net.ParseCIDR(rs) + n, err := netip.ParsePrefix(rs) if err != nil { - return newHelpErrorf("invalid ip definition: %s", err) + return newHelpErrorf("invalid -networks definition: %s", rs) } - if ip.To4() == nil { - return newHelpErrorf("invalid ip definition: can only be ipv4, have %s", rs) + if version == cert.Version1 && !n.Addr().Is4() { + return newHelpErrorf("invalid -networks definition: v1 certificates can only be ipv4, have %s", rs) } - - ipNet.IP = ip - ips = append(ips, ipNet) + networks = append(networks, n) } } } - var subnets []*net.IPNet - if *cf.subnets != "" { - for _, rs := range strings.Split(*cf.subnets, ",") { + var unsafeNetworks []netip.Prefix + if *cf.unsafeNetworks == "" && *cf.subnets != "" { + // Pull up deprecated -subnets flag if needed + *cf.unsafeNetworks = *cf.subnets + } + + if *cf.unsafeNetworks != "" { + for _, rs := range strings.Split(*cf.unsafeNetworks, ",") { rs := strings.Trim(rs, " ") if rs != "" { - _, s, err := net.ParseCIDR(rs) + n, err := netip.ParsePrefix(rs) if err != nil { - return newHelpErrorf("invalid subnet definition: %s", err) + return newHelpErrorf("invalid -unsafe-networks definition: %s", rs) } - if s.IP.To4() == nil { - return newHelpErrorf("invalid subnet definition: can only be ipv4, have %s", rs) + if version == cert.Version1 && !n.Addr().Is4() { + return newHelpErrorf("invalid -unsafe-networks definition: v1 certificates can only be ipv4, have %s", rs) } - subnets = append(subnets, s) + unsafeNetworks = append(unsafeNetworks, n) } } } + var claims ioClaims + if err := reserveOutputs(&claims, + "out-key", *cf.outKeyPath, + "out-crt", *cf.outCertPath, + "out-qr", *cf.outQRPath, + ); err != nil { + return err + } + var passphrase []byte - if *cf.encryption { - for i := 0; i < 5; i++ { - out.Write([]byte("Enter passphrase: ")) - passphrase, err = pr.ReadPassword() - - if err == ErrNoTerminal { - return fmt.Errorf("out-key must be encrypted interactively") - } else if err != nil { - return fmt.Errorf("error reading passphrase: %s", err) - } + if !isP11 && *cf.encryption { + passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE")) + if len(passphrase) == 0 { + for i := 0; i < 5; i++ { + errOut.Write([]byte("Enter passphrase: ")) + passphrase, err = pr.ReadPassword() + + if err == ErrNoTerminal { + return fmt.Errorf("out-key must be encrypted interactively") + } else if err != nil { + return fmt.Errorf("error reading passphrase: %s", err) + } - if len(passphrase) > 0 { - break + if len(passphrase) > 0 { + break + } } - } - if len(passphrase) == 0 { - return fmt.Errorf("no passphrase specified, remove -encrypt flag to write out-key in plaintext") + if len(passphrase) == 0 { + return fmt.Errorf("no passphrase specified, remove -encrypt flag to write out-key in plaintext") + } } } var curve cert.Curve var pub, rawPriv []byte - switch *cf.curve { - case "25519", "X25519", "Curve25519", "CURVE25519": - curve = cert.Curve_CURVE25519 - pub, rawPriv, err = ed25519.GenerateKey(rand.Reader) + var p11Client *pkclient.PKClient + + if isP11 { + switch *cf.curve { + case "P256": + curve = cert.Curve_P256 + default: + return fmt.Errorf("invalid curve for PKCS#11: %s", *cf.curve) + } + + p11Client, err = pkclient.FromUrl(*cf.p11url) if err != nil { - return fmt.Errorf("error while generating ed25519 keys: %s", err) + return fmt.Errorf("error while creating PKCS#11 client: %w", err) } - case "P256": - var key *ecdsa.PrivateKey - curve = cert.Curve_P256 - key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + defer func(client *pkclient.PKClient) { + _ = client.Close() + }(p11Client) + pub, err = p11Client.GetPubKey() if err != nil { - return fmt.Errorf("error while generating ecdsa keys: %s", err) + return fmt.Errorf("error while getting public key with PKCS#11: %w", err) } - // ref: https://github.com/golang/go/blob/go1.19/src/crypto/x509/sec1.go#L60 - rawPriv = key.D.FillBytes(make([]byte, 32)) - pub = elliptic.Marshal(elliptic.P256(), key.X, key.Y) - } + } else { + switch *cf.curve { + case "25519", "X25519", "Curve25519", "CURVE25519": + curve = cert.Curve_CURVE25519 + pub, rawPriv, err = ed25519.GenerateKey(rand.Reader) + if err != nil { + return fmt.Errorf("error while generating ed25519 keys: %s", err) + } + case "P256": + var key *ecdsa.PrivateKey + curve = cert.Curve_P256 + key, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return fmt.Errorf("error while generating ecdsa keys: %s", err) + } - nc := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: *cf.name, - Groups: groups, - Ips: ips, - Subnets: subnets, - NotBefore: time.Now(), - NotAfter: time.Now().Add(*cf.duration), - PublicKey: pub, - IsCA: true, - Curve: curve, - }, + // ecdh.PrivateKey lets us get at the encoded bytes, even though + // we aren't using ECDH here. + eKey, err := key.ECDH() + if err != nil { + return fmt.Errorf("error while converting ecdsa key: %s", err) + } + rawPriv = eKey.Bytes() + pub = eKey.PublicKey().Bytes() + default: + return fmt.Errorf("invalid curve: %s", *cf.curve) + } } - if _, err := os.Stat(*cf.outKeyPath); err == nil { - return fmt.Errorf("refusing to overwrite existing CA key: %s", *cf.outKeyPath) + t := &cert.TBSCertificate{ + Version: version, + Name: *cf.name, + Groups: groups, + Networks: networks, + UnsafeNetworks: unsafeNetworks, + NotBefore: time.Now(), + NotAfter: time.Now().Add(*cf.duration), + PublicKey: pub, + IsCA: true, + Curve: curve, } - if _, err := os.Stat(*cf.outCertPath); err == nil { - return fmt.Errorf("refusing to overwrite existing CA cert: %s", *cf.outCertPath) + if !isP11 && !isStdio(*cf.outKeyPath) { + if _, err := os.Stat(*cf.outKeyPath); err == nil { + return fmt.Errorf("refusing to overwrite existing CA key: %s", *cf.outKeyPath) + } } - err = nc.Sign(curve, rawPriv) - if err != nil { - return fmt.Errorf("error while signing: %s", err) + if !isStdio(*cf.outCertPath) { + if _, err := os.Stat(*cf.outCertPath); err == nil { + return fmt.Errorf("refusing to overwrite existing CA cert: %s", *cf.outCertPath) + } } + var c cert.Certificate var b []byte - if *cf.encryption { - b, err = cert.EncryptAndMarshalSigningPrivateKey(curve, rawPriv, passphrase, kdfParams) + + if isP11 { + c, err = t.SignWith(nil, curve, p11Client.SignASN1) if err != nil { - return fmt.Errorf("error while encrypting out-key: %s", err) + return fmt.Errorf("error while signing with PKCS#11: %w", err) } } else { - b = cert.MarshalSigningPrivateKey(curve, rawPriv) - } + c, err = t.Sign(nil, curve, rawPriv) + if err != nil { + return fmt.Errorf("error while signing: %s", err) + } - err = os.WriteFile(*cf.outKeyPath, b, 0600) - if err != nil { - return fmt.Errorf("error while writing out-key: %s", err) + if *cf.encryption { + b, err = cert.EncryptAndMarshalSigningPrivateKey(curve, rawPriv, passphrase, kdfParams) + if err != nil { + return fmt.Errorf("error while encrypting out-key: %s", err) + } + } else { + b = cert.MarshalSigningPrivateKeyToPEM(curve, rawPriv) + } + + err = writeOutput(*cf.outKeyPath, b, 0600, out) + if err != nil { + return fmt.Errorf("error while writing out-key: %s", err) + } } - b, err = nc.MarshalToPEM() + b, err = c.MarshalPEM() if err != nil { return fmt.Errorf("error while marshalling certificate: %s", err) } - err = os.WriteFile(*cf.outCertPath, b, 0600) + err = writeOutput(*cf.outCertPath, b, 0600, out) if err != nil { return fmt.Errorf("error while writing out-crt: %s", err) } @@ -243,7 +340,7 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error return fmt.Errorf("error while generating qr code: %s", err) } - err = os.WriteFile(*cf.outQRPath, b, 0600) + err = writeOutput(*cf.outQRPath, b, 0600, out) if err != nil { return fmt.Errorf("error while writing out-qr: %s", err) } @@ -259,6 +356,7 @@ func caSummary() string { func caHelp(out io.Writer) { cf := newCaFlags() out.Write([]byte("Usage of " + os.Args[0] + " " + caSummary() + "\n")) + out.Write([]byte(stdioHelpText)) cf.set.SetOutput(out) cf.set.PrintDefaults() } diff --git a/cmd/nebula-cert/ca_test.go b/cmd/nebula-cert/ca_test.go index 3a534051..ce0113b6 100644 --- a/cmd/nebula-cert/ca_test.go +++ b/cmd/nebula-cert/ca_test.go @@ -14,10 +14,9 @@ import ( "github.com/slackhq/nebula/cert" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -//TODO: test file permissions - func Test_caSummary(t *testing.T) { assert.Equal(t, "ca : create a self signed certificate authority", caSummary()) } @@ -28,6 +27,7 @@ func Test_caHelp(t *testing.T) { assert.Equal( t, "Usage of "+os.Args[0]+" ca : create a self signed certificate authority\n"+ + " Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+ " -argon-iterations uint\n"+ " \tOptional: Argon2 iterations parameter used for encrypted private key passphrase (default 1)\n"+ " -argon-memory uint\n"+ @@ -43,17 +43,24 @@ func Test_caHelp(t *testing.T) { " -groups string\n"+ " \tOptional: comma separated list of groups. This will limit which groups subordinate certs can use\n"+ " -ips string\n"+ - " \tOptional: comma separated list of ipv4 address and network in CIDR notation. This will limit which ipv4 addresses and networks subordinate certs can use for ip addresses\n"+ + " Deprecated, see -networks\n"+ " -name string\n"+ " \tRequired: name of the certificate authority\n"+ + " -networks string\n"+ + " \tOptional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in networks\n"+ " -out-crt string\n"+ " \tOptional: path to write the certificate to (default \"ca.crt\")\n"+ " -out-key string\n"+ " \tOptional: path to write the private key to (default \"ca.key\")\n"+ " -out-qr string\n"+ " \tOptional: output a qr code image (png) of the certificate\n"+ + optionalPkcs11String(" -pkcs11 string\n \tOptional: PKCS#11 URI to an existing private key\n")+ " -subnets string\n"+ - " \tOptional: comma separated list of ipv4 address and network in CIDR notation. This will limit which ipv4 addresses and networks subordinate certs can use in subnets\n", + " \tDeprecated, see -unsafe-networks\n"+ + " -unsafe-networks string\n"+ + " \tOptional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in unsafe networks\n"+ + " -version uint\n"+ + " \tOptional: version of the certificate format to use (default 2)\n", ob.String(), ) } @@ -78,97 +85,109 @@ func Test_ca(t *testing.T) { err: nil, } - pwPromptOb := "Enter passphrase: " + pwPromptEB := "Enter passphrase: " // required args assertHelpError(t, ca( - []string{"-out-key", "nope", "-out-crt", "nope", "duration", "100m"}, ob, eb, nopw, + []string{"-version", "1", "-out-key", "nope", "-out-crt", "nope", "duration", "100m"}, ob, eb, nopw, ), "-name is required") - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // ipv4 only ips - assertHelpError(t, ca([]string{"-name", "ipv6", "-ips", "100::100/100"}, ob, eb, nopw), "invalid ip definition: can only be ipv4, have 100::100/100") - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + assertHelpError(t, ca([]string{"-version", "1", "-name", "ipv6", "-ips", "100::100/100"}, ob, eb, nopw), "invalid -networks definition: v1 certificates can only be ipv4, have 100::100/100") + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // ipv4 only subnets - assertHelpError(t, ca([]string{"-name", "ipv6", "-subnets", "100::100/100"}, ob, eb, nopw), "invalid subnet definition: can only be ipv4, have 100::100/100") - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + assertHelpError(t, ca([]string{"-version", "1", "-name", "ipv6", "-subnets", "100::100/100"}, ob, eb, nopw), "invalid -unsafe-networks definition: v1 certificates can only be ipv4, have 100::100/100") + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // failed key write ob.Reset() eb.Reset() - args := []string{"-name", "test", "-duration", "100m", "-out-crt", "/do/not/write/pleasecrt", "-out-key", "/do/not/write/pleasekey"} - assert.EqualError(t, ca(args, ob, eb, nopw), "error while writing out-key: open /do/not/write/pleasekey: "+NoSuchDirError) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + args := []string{"-version", "1", "-name", "test", "-duration", "100m", "-out-crt", "/do/not/write/pleasecrt", "-out-key", "/do/not/write/pleasekey"} + require.EqualError(t, ca(args, ob, eb, nopw), "error while writing out-key: open /do/not/write/pleasekey: "+NoSuchDirError) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // create temp key file keyF, err := os.CreateTemp("", "test.key") - assert.Nil(t, err) - os.Remove(keyF.Name()) + require.NoError(t, err) + require.NoError(t, os.Remove(keyF.Name())) // failed cert write ob.Reset() eb.Reset() - args = []string{"-name", "test", "-duration", "100m", "-out-crt", "/do/not/write/pleasecrt", "-out-key", keyF.Name()} - assert.EqualError(t, ca(args, ob, eb, nopw), "error while writing out-crt: open /do/not/write/pleasecrt: "+NoSuchDirError) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + args = []string{"-version", "1", "-name", "test", "-duration", "100m", "-out-crt", "/do/not/write/pleasecrt", "-out-key", keyF.Name()} + require.EqualError(t, ca(args, ob, eb, nopw), "error while writing out-crt: open /do/not/write/pleasecrt: "+NoSuchDirError) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // create temp cert file crtF, err := os.CreateTemp("", "test.crt") - assert.Nil(t, err) - os.Remove(crtF.Name()) - os.Remove(keyF.Name()) + require.NoError(t, err) + require.NoError(t, os.Remove(crtF.Name())) + require.NoError(t, os.Remove(keyF.Name())) // test proper cert with removed empty groups and subnets ob.Reset() eb.Reset() - args = []string{"-name", "test", "-duration", "100m", "-groups", "1,, 2 , ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} - assert.Nil(t, ca(args, ob, eb, nopw)) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + args = []string{"-version", "1", "-name", "test", "-duration", "100m", "-groups", "1,, 2 , ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} + require.NoError(t, ca(args, ob, eb, nopw)) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // read cert and key files rb, _ := os.ReadFile(keyF.Name()) - lKey, b, err := cert.UnmarshalEd25519PrivateKey(rb) - assert.Len(t, b, 0) - assert.Nil(t, err) + lKey, b, c, err := cert.UnmarshalSigningPrivateKeyFromPEM(rb) + assert.Equal(t, cert.Curve_CURVE25519, c) + assert.Empty(t, b) + require.NoError(t, err) assert.Len(t, lKey, 64) rb, _ = os.ReadFile(crtF.Name()) - lCrt, b, err := cert.UnmarshalNebulaCertificateFromPEM(rb) - assert.Len(t, b, 0) - assert.Nil(t, err) - - assert.Equal(t, "test", lCrt.Details.Name) - assert.Len(t, lCrt.Details.Ips, 0) - assert.True(t, lCrt.Details.IsCA) - assert.Equal(t, []string{"1", "2", "3", "4", "5"}, lCrt.Details.Groups) - assert.Len(t, lCrt.Details.Subnets, 0) - assert.Len(t, lCrt.Details.PublicKey, 32) - assert.Equal(t, time.Duration(time.Minute*100), lCrt.Details.NotAfter.Sub(lCrt.Details.NotBefore)) - assert.Equal(t, "", lCrt.Details.Issuer) - assert.True(t, lCrt.CheckSignature(lCrt.Details.PublicKey)) + lCrt, b, err := cert.UnmarshalCertificateFromPEM(rb) + assert.Empty(t, b) + require.NoError(t, err) + + assert.Equal(t, "test", lCrt.Name()) + assert.Empty(t, lCrt.Networks()) + assert.True(t, lCrt.IsCA()) + assert.Equal(t, []string{"1", "2", "3", "4", "5"}, lCrt.Groups()) + assert.Empty(t, lCrt.UnsafeNetworks()) + assert.Len(t, lCrt.PublicKey(), 32) + assert.Equal(t, time.Duration(time.Minute*100), lCrt.NotAfter().Sub(lCrt.NotBefore())) + assert.Empty(t, lCrt.Issuer()) + assert.True(t, lCrt.CheckSignature(lCrt.PublicKey())) // test encrypted key os.Remove(keyF.Name()) os.Remove(crtF.Name()) ob.Reset() eb.Reset() - args = []string{"-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} - assert.Nil(t, ca(args, ob, eb, testpw)) - assert.Equal(t, pwPromptOb, ob.String()) - assert.Equal(t, "", eb.String()) + args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} + require.NoError(t, ca(args, ob, eb, testpw)) + assert.Empty(t, ob.String()) + assert.Equal(t, pwPromptEB, eb.String()) + + // test encrypted key with passphrase environment variable + os.Remove(keyF.Name()) + os.Remove(crtF.Name()) + ob.Reset() + eb.Reset() + args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} + os.Setenv("NEBULA_CA_PASSPHRASE", string(passphrase)) + require.NoError(t, ca(args, ob, eb, testpw)) + assert.Empty(t, eb.String()) + os.Setenv("NEBULA_CA_PASSPHRASE", "") // read encrypted key file and verify default params rb, _ = os.ReadFile(keyF.Name()) k, _ := pem.Decode(rb) ned, err := cert.UnmarshalNebulaEncryptedData(k.Bytes) - assert.Nil(t, err) + require.NoError(t, err) // we won't know salt in advance, so just check start of string assert.Equal(t, uint32(2*1024*1024), ned.EncryptionMetadata.Argon2Parameters.Memory) assert.Equal(t, uint8(4), ned.EncryptionMetadata.Argon2Parameters.Parallelism) @@ -178,54 +197,118 @@ func Test_ca(t *testing.T) { var curve cert.Curve curve, lKey, b, err = cert.DecryptAndUnmarshalSigningPrivateKey(passphrase, rb) assert.Equal(t, cert.Curve_CURVE25519, curve) - assert.Nil(t, err) - assert.Len(t, b, 0) + require.NoError(t, err) + assert.Empty(t, b) assert.Len(t, lKey, 64) - // test when reading passsword results in an error + // test when reading password results in an error os.Remove(keyF.Name()) os.Remove(crtF.Name()) ob.Reset() eb.Reset() - args = []string{"-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} - assert.Error(t, ca(args, ob, eb, errpw)) - assert.Equal(t, pwPromptOb, ob.String()) - assert.Equal(t, "", eb.String()) + args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} + require.Error(t, ca(args, ob, eb, errpw)) + assert.Empty(t, ob.String()) + assert.Equal(t, pwPromptEB, eb.String()) // test when user fails to enter a password os.Remove(keyF.Name()) os.Remove(crtF.Name()) ob.Reset() eb.Reset() - args = []string{"-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} - assert.EqualError(t, ca(args, ob, eb, nopw), "no passphrase specified, remove -encrypt flag to write out-key in plaintext") - assert.Equal(t, strings.Repeat(pwPromptOb, 5), ob.String()) // prompts 5 times before giving up - assert.Equal(t, "", eb.String()) + args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} + require.EqualError(t, ca(args, ob, eb, nopw), "no passphrase specified, remove -encrypt flag to write out-key in plaintext") + assert.Empty(t, ob.String()) + assert.Equal(t, strings.Repeat(pwPromptEB, 5), eb.String()) // prompts 5 times before giving up // create valid cert/key for overwrite tests os.Remove(keyF.Name()) os.Remove(crtF.Name()) ob.Reset() eb.Reset() - args = []string{"-name", "test", "-duration", "100m", "-groups", "1,, 2 , ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} - assert.Nil(t, ca(args, ob, eb, nopw)) + args = []string{"-version", "1", "-name", "test", "-duration", "100m", "-groups", "1,, 2 , ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} + require.NoError(t, ca(args, ob, eb, nopw)) // test that we won't overwrite existing certificate file ob.Reset() eb.Reset() - args = []string{"-name", "test", "-duration", "100m", "-groups", "1,, 2 , ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} - assert.EqualError(t, ca(args, ob, eb, nopw), "refusing to overwrite existing CA key: "+keyF.Name()) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + args = []string{"-version", "1", "-name", "test", "-duration", "100m", "-groups", "1,, 2 , ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} + require.EqualError(t, ca(args, ob, eb, nopw), "refusing to overwrite existing CA key: "+keyF.Name()) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // test that we won't overwrite existing key file os.Remove(keyF.Name()) ob.Reset() eb.Reset() - args = []string{"-name", "test", "-duration", "100m", "-groups", "1,, 2 , ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} - assert.EqualError(t, ca(args, ob, eb, nopw), "refusing to overwrite existing CA cert: "+crtF.Name()) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + args = []string{"-version", "1", "-name", "test", "-duration", "100m", "-groups", "1,, 2 , ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()} + require.EqualError(t, ca(args, ob, eb, nopw), "refusing to overwrite existing CA cert: "+crtF.Name()) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) os.Remove(keyF.Name()) } + +func Test_ca_stdio(t *testing.T) { + nopw := &StubPasswordReader{} + + keyF, err := os.CreateTemp("", "ca.key") + require.NoError(t, err) + os.Remove(keyF.Name()) + defer os.Remove(keyF.Name()) + + crtF, err := os.CreateTemp("", "ca.crt") + require.NoError(t, err) + os.Remove(crtF.Name()) + defer os.Remove(crtF.Name()) + + // out-crt on stdout, out-key on disk + ob := &bytes.Buffer{} + eb := &bytes.Buffer{} + require.NoError(t, ca([]string{"-name", "test-ca", "-duration", "1h", "-out-crt", "-", "-out-key", keyF.Name()}, ob, eb, nopw)) + assert.Empty(t, eb.String()) + c, _, err := cert.UnmarshalCertificateFromPEM(ob.Bytes()) + require.NoError(t, err) + assert.True(t, c.IsCA()) + assert.Equal(t, "test-ca", c.Name()) + + // out-key on stdout, out-crt on disk + os.Remove(keyF.Name()) + ob.Reset() + eb.Reset() + require.NoError(t, ca([]string{"-name", "test-ca", "-duration", "1h", "-out-crt", crtF.Name(), "-out-key", "-"}, ob, eb, nopw)) + assert.Empty(t, eb.String()) + _, _, curve, err := cert.UnmarshalSigningPrivateKeyFromPEM(ob.Bytes()) + require.NoError(t, err) + assert.Equal(t, cert.Curve_CURVE25519, curve) + + // dual stdout is rejected up front + os.Remove(crtF.Name()) + ob.Reset() + eb.Reset() + require.EqualError(t, + ca([]string{"-name", "test-ca", "-duration", "1h", "-out-crt", "-", "-out-key", "-"}, ob, eb, nopw), + `-out-key and -out-crt both set to "-", only one output may write to stdout`) + assert.Empty(t, ob.String()) + + // an output conflict combined with -encrypt must error BEFORE prompting + // for a passphrase; pr would record any read attempt + tracker := &trackingPasswordReader{} + ob.Reset() + eb.Reset() + require.EqualError(t, + ca([]string{"-name", "test-ca", "-duration", "1h", "-encrypt", "-out-crt", "-", "-out-key", "-"}, ob, eb, tracker), + `-out-key and -out-crt both set to "-", only one output may write to stdout`) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + assert.Zero(t, tracker.calls, "passphrase prompt should not have been called") +} + +type trackingPasswordReader struct { + calls int +} + +func (pr *trackingPasswordReader) ReadPassword() ([]byte, error) { + pr.calls++ + return []byte(""), nil +} diff --git a/cmd/nebula-cert/keygen.go b/cmd/nebula-cert/keygen.go index d94cbf14..dea6c4af 100644 --- a/cmd/nebula-cert/keygen.go +++ b/cmd/nebula-cert/keygen.go @@ -6,6 +6,8 @@ import ( "io" "os" + "github.com/slackhq/nebula/pkclient" + "github.com/slackhq/nebula/cert" ) @@ -13,8 +15,8 @@ type keygenFlags struct { set *flag.FlagSet outKeyPath *string outPubPath *string - - curve *string + curve *string + p11url *string } func newKeygenFlags() *keygenFlags { @@ -23,6 +25,7 @@ func newKeygenFlags() *keygenFlags { cf.outPubPath = cf.set.String("out-pub", "", "Required: path to write the public key to") cf.outKeyPath = cf.set.String("out-key", "", "Required: path to write the private key to") cf.curve = cf.set.String("curve", "25519", "ECDH Curve (25519, P256)") + cf.p11url = p11Flag(cf.set) return &cf } @@ -33,32 +36,68 @@ func keygen(args []string, out io.Writer, errOut io.Writer) error { return err } - if err := mustFlagString("out-key", cf.outKeyPath); err != nil { - return err + isP11 := len(*cf.p11url) > 0 + + if !isP11 { + if err = mustFlagString("out-key", cf.outKeyPath); err != nil { + return err + } + } else if *cf.outKeyPath != "" { + return newHelpErrorf("cannot set -out-key with -pkcs11") } - if err := mustFlagString("out-pub", cf.outPubPath); err != nil { + if err = mustFlagString("out-pub", cf.outPubPath); err != nil { return err } var pub, rawPriv []byte var curve cert.Curve - switch *cf.curve { - case "25519", "X25519", "Curve25519", "CURVE25519": - pub, rawPriv = x25519Keypair() - curve = cert.Curve_CURVE25519 - case "P256": - pub, rawPriv = p256Keypair() - curve = cert.Curve_P256 - default: - return fmt.Errorf("invalid curve: %s", *cf.curve) + if isP11 { + switch *cf.curve { + case "P256": + curve = cert.Curve_P256 + default: + return fmt.Errorf("invalid curve for PKCS#11: %s", *cf.curve) + } + } else { + switch *cf.curve { + case "25519", "X25519", "Curve25519", "CURVE25519": + pub, rawPriv = x25519Keypair() + curve = cert.Curve_CURVE25519 + case "P256": + pub, rawPriv = p256Keypair() + curve = cert.Curve_P256 + default: + return fmt.Errorf("invalid curve: %s", *cf.curve) + } } - err = os.WriteFile(*cf.outKeyPath, cert.MarshalPrivateKey(curve, rawPriv), 0600) - if err != nil { - return fmt.Errorf("error while writing out-key: %s", err) + var claims ioClaims + if err := reserveOutputs(&claims, + "out-key", *cf.outKeyPath, + "out-pub", *cf.outPubPath, + ); err != nil { + return err } - err = os.WriteFile(*cf.outPubPath, cert.MarshalPublicKey(curve, pub), 0600) + if isP11 { + p11Client, err := pkclient.FromUrl(*cf.p11url) + if err != nil { + return fmt.Errorf("error while creating PKCS#11 client: %w", err) + } + defer func(client *pkclient.PKClient) { + _ = client.Close() + }(p11Client) + pub, err = p11Client.GetPubKey() + if err != nil { + return fmt.Errorf("error while getting public key: %w", err) + } + } else { + err = writeOutput(*cf.outKeyPath, cert.MarshalPrivateKeyToPEM(curve, rawPriv), 0600, out) + if err != nil { + return fmt.Errorf("error while writing out-key: %s", err) + } + } + err = writeOutput(*cf.outPubPath, cert.MarshalPublicKeyToPEM(curve, pub), 0600, out) if err != nil { return fmt.Errorf("error while writing out-pub: %s", err) } @@ -72,7 +111,8 @@ func keygenSummary() string { func keygenHelp(out io.Writer) { cf := newKeygenFlags() - out.Write([]byte("Usage of " + os.Args[0] + " " + keygenSummary() + "\n")) + _, _ = out.Write([]byte("Usage of " + os.Args[0] + " " + keygenSummary() + "\n")) + _, _ = out.Write([]byte(stdioHelpText)) cf.set.SetOutput(out) cf.set.PrintDefaults() } diff --git a/cmd/nebula-cert/keygen_test.go b/cmd/nebula-cert/keygen_test.go index 9a3b3f3b..98c4c456 100644 --- a/cmd/nebula-cert/keygen_test.go +++ b/cmd/nebula-cert/keygen_test.go @@ -7,10 +7,9 @@ import ( "github.com/slackhq/nebula/cert" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -//TODO: test file permissions - func Test_keygenSummary(t *testing.T) { assert.Equal(t, "keygen : create a public/private key pair. the public key can be passed to `nebula-cert sign`", keygenSummary()) } @@ -21,12 +20,14 @@ func Test_keygenHelp(t *testing.T) { assert.Equal( t, "Usage of "+os.Args[0]+" keygen : create a public/private key pair. the public key can be passed to `nebula-cert sign`\n"+ + " Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+ " -curve string\n"+ " \tECDH Curve (25519, P256) (default \"25519\")\n"+ " -out-key string\n"+ " \tRequired: path to write the private key to\n"+ " -out-pub string\n"+ - " \tRequired: path to write the public key to\n", + " \tRequired: path to write the public key to\n"+ + optionalPkcs11String(" -pkcs11 string\n \tOptional: PKCS#11 URI to an existing private key\n"), ob.String(), ) } @@ -37,57 +38,99 @@ func Test_keygen(t *testing.T) { // required args assertHelpError(t, keygen([]string{"-out-pub", "nope"}, ob, eb), "-out-key is required") - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) assertHelpError(t, keygen([]string{"-out-key", "nope"}, ob, eb), "-out-pub is required") - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // failed key write ob.Reset() eb.Reset() args := []string{"-out-pub", "/do/not/write/pleasepub", "-out-key", "/do/not/write/pleasekey"} - assert.EqualError(t, keygen(args, ob, eb), "error while writing out-key: open /do/not/write/pleasekey: "+NoSuchDirError) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + require.EqualError(t, keygen(args, ob, eb), "error while writing out-key: open /do/not/write/pleasekey: "+NoSuchDirError) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // create temp key file keyF, err := os.CreateTemp("", "test.key") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(keyF.Name()) // failed pub write ob.Reset() eb.Reset() args = []string{"-out-pub", "/do/not/write/pleasepub", "-out-key", keyF.Name()} - assert.EqualError(t, keygen(args, ob, eb), "error while writing out-pub: open /do/not/write/pleasepub: "+NoSuchDirError) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + require.EqualError(t, keygen(args, ob, eb), "error while writing out-pub: open /do/not/write/pleasepub: "+NoSuchDirError) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // create temp pub file pubF, err := os.CreateTemp("", "test.pub") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(pubF.Name()) // test proper keygen ob.Reset() eb.Reset() args = []string{"-out-pub", pubF.Name(), "-out-key", keyF.Name()} - assert.Nil(t, keygen(args, ob, eb)) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + require.NoError(t, keygen(args, ob, eb)) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // read cert and key files rb, _ := os.ReadFile(keyF.Name()) - lKey, b, err := cert.UnmarshalX25519PrivateKey(rb) - assert.Len(t, b, 0) - assert.Nil(t, err) + lKey, b, curve, err := cert.UnmarshalPrivateKeyFromPEM(rb) + assert.Equal(t, cert.Curve_CURVE25519, curve) + assert.Empty(t, b) + require.NoError(t, err) assert.Len(t, lKey, 32) rb, _ = os.ReadFile(pubF.Name()) - lPub, b, err := cert.UnmarshalX25519PublicKey(rb) - assert.Len(t, b, 0) - assert.Nil(t, err) + lPub, b, curve, err := cert.UnmarshalPublicKeyFromPEM(rb) + assert.Equal(t, cert.Curve_CURVE25519, curve) + assert.Empty(t, b) + require.NoError(t, err) assert.Len(t, lPub, 32) } + +func Test_keygen_stdio(t *testing.T) { + keyF, err := os.CreateTemp("", "test.key") + require.NoError(t, err) + os.Remove(keyF.Name()) + defer os.Remove(keyF.Name()) + + pubF, err := os.CreateTemp("", "test.pub") + require.NoError(t, err) + os.Remove(pubF.Name()) + defer os.Remove(pubF.Name()) + + // out-pub on stdout, out-key on disk + ob := &bytes.Buffer{} + eb := &bytes.Buffer{} + require.NoError(t, keygen([]string{"-out-pub", "-", "-out-key", keyF.Name()}, ob, eb)) + assert.Empty(t, eb.String()) + lPub, _, curve, err := cert.UnmarshalPublicKeyFromPEM(ob.Bytes()) + require.NoError(t, err) + assert.Equal(t, cert.Curve_CURVE25519, curve) + assert.Len(t, lPub, 32) + + // out-key on stdout, out-pub on disk + os.Remove(keyF.Name()) + ob.Reset() + eb.Reset() + require.NoError(t, keygen([]string{"-out-pub", pubF.Name(), "-out-key", "-"}, ob, eb)) + assert.Empty(t, eb.String()) + lKey, _, curve, err := cert.UnmarshalPrivateKeyFromPEM(ob.Bytes()) + require.NoError(t, err) + assert.Equal(t, cert.Curve_CURVE25519, curve) + assert.Len(t, lKey, 32) + + // both on stdout is a conflict caught up front + ob.Reset() + eb.Reset() + require.EqualError(t, keygen([]string{"-out-pub", "-", "-out-key", "-"}, ob, eb), + `-out-key and -out-pub both set to "-", only one output may write to stdout`) + assert.Empty(t, ob.String()) +} diff --git a/cmd/nebula-cert/main.go b/cmd/nebula-cert/main.go index b803d30a..82ad2fee 100644 --- a/cmd/nebula-cert/main.go +++ b/cmd/nebula-cert/main.go @@ -5,10 +5,28 @@ import ( "fmt" "io" "os" + "runtime/debug" + "strings" ) +// A version string that can be set with +// +// -ldflags "-X main.Build=SOMEVERSION" +// +// at compile-time. var Build string +func init() { + if Build == "" { + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + + Build = strings.TrimPrefix(info.Main.Version, "v") + } +} + type helpError struct { s string } @@ -17,7 +35,7 @@ func (he *helpError) Error() string { return he.s } -func newHelpErrorf(s string, v ...interface{}) error { +func newHelpErrorf(s string, v ...any) error { return &helpError{s: fmt.Sprintf(s, v...)} } diff --git a/cmd/nebula-cert/main_test.go b/cmd/nebula-cert/main_test.go index 3d0fa1b6..2e92e7e2 100644 --- a/cmd/nebula-cert/main_test.go +++ b/cmd/nebula-cert/main_test.go @@ -3,15 +3,15 @@ package main import ( "bytes" "errors" + "fmt" "io" "os" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -//TODO: all flag parsing continueOnError will print to stderr on its own currently - func Test_help(t *testing.T) { expected := "Usage of " + os.Args[0] + " :\n" + " Global flags:\n" + @@ -77,8 +77,16 @@ func assertHelpError(t *testing.T, err error, msg string) { case *helpError: // good default: - t.Fatal("err was not a helpError") + t.Fatal(fmt.Sprintf("err was not a helpError: %q, expected %q", err, msg)) } - assert.EqualError(t, err, msg) + require.EqualError(t, err, msg) +} + +func optionalPkcs11String(msg string) string { + if p11Supported() { + return msg + } else { + return "" + } } diff --git a/cmd/nebula-cert/p11_cgo.go b/cmd/nebula-cert/p11_cgo.go new file mode 100644 index 00000000..f1f1ec62 --- /dev/null +++ b/cmd/nebula-cert/p11_cgo.go @@ -0,0 +1,15 @@ +//go:build cgo && pkcs11 + +package main + +import ( + "flag" +) + +func p11Supported() bool { + return true +} + +func p11Flag(set *flag.FlagSet) *string { + return set.String("pkcs11", "", "Optional: PKCS#11 URI to an existing private key") +} diff --git a/cmd/nebula-cert/p11_stub.go b/cmd/nebula-cert/p11_stub.go new file mode 100644 index 00000000..5afeaeab --- /dev/null +++ b/cmd/nebula-cert/p11_stub.go @@ -0,0 +1,16 @@ +//go:build !cgo || !pkcs11 + +package main + +import ( + "flag" +) + +func p11Supported() bool { + return false +} + +func p11Flag(set *flag.FlagSet) *string { + var ret = "" + return &ret +} diff --git a/cmd/nebula-cert/passwords.go b/cmd/nebula-cert/passwords.go index 8129560e..0aa2115d 100644 --- a/cmd/nebula-cert/passwords.go +++ b/cmd/nebula-cert/passwords.go @@ -22,7 +22,9 @@ func (pr StdinPasswordReader) ReadPassword() ([]byte, error) { } password, err := term.ReadPassword(int(os.Stdin.Fd())) - fmt.Println() + // Terminal echo is off while reading, so the user's Enter key does not + // produce a visible newline. Emit one on stderr to match the prompt. + fmt.Fprintln(os.Stderr) return password, err } diff --git a/cmd/nebula-cert/print.go b/cmd/nebula-cert/print.go index 746d6a3a..3ba0571e 100644 --- a/cmd/nebula-cert/print.go +++ b/cmd/nebula-cert/print.go @@ -40,33 +40,46 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error { return err } - rawCert, err := os.ReadFile(*pf.path) + var claims ioClaims + if err := reserveInputs(&claims, "path", *pf.path); err != nil { + return err + } + if err := reserveOutputs(&claims, "out-qr", *pf.outQRPath); err != nil { + return err + } + + rawCert, err := readInput("path", *pf.path, &claims) if err != nil { return fmt.Errorf("unable to read cert; %s", err) } - var c *cert.NebulaCertificate + // When the QR is going to stdout, suppress the human-readable text/json + // output so the binary stream is not contaminated. + qrToStdout := isStdio(*pf.outQRPath) + + var c cert.Certificate var qrBytes []byte part := 0 + var jsonCerts []cert.Certificate + for { - c, rawCert, err = cert.UnmarshalNebulaCertificateFromPEM(rawCert) + c, rawCert, err = cert.UnmarshalCertificateFromPEM(rawCert) if err != nil { return fmt.Errorf("error while unmarshaling cert: %s", err) } - if *pf.json { - b, _ := json.Marshal(c) - out.Write(b) - out.Write([]byte("\n")) - - } else { - out.Write([]byte(c.String())) - out.Write([]byte("\n")) + if !qrToStdout { + if *pf.json { + jsonCerts = append(jsonCerts, c) + } else { + _, _ = out.Write([]byte(c.String())) + _, _ = out.Write([]byte("\n")) + } } if *pf.outQRPath != "" { - b, err := c.MarshalToPEM() + b, err := c.MarshalPEM() if err != nil { return fmt.Errorf("error while marshalling cert to PEM: %s", err) } @@ -80,13 +93,19 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error { part++ } + if *pf.json && !qrToStdout { + b, _ := json.Marshal(jsonCerts) + _, _ = out.Write(b) + _, _ = out.Write([]byte("\n")) + } + if *pf.outQRPath != "" { b, err := qrcode.Encode(string(qrBytes), qrcode.Medium, -5) if err != nil { return fmt.Errorf("error while generating qr code: %s", err) } - err = os.WriteFile(*pf.outQRPath, b, 0600) + err = writeOutput(*pf.outQRPath, b, 0600, out) if err != nil { return fmt.Errorf("error while writing out-qr: %s", err) } @@ -102,6 +121,7 @@ func printSummary() string { func printHelp(out io.Writer) { pf := newPrintFlags() out.Write([]byte("Usage of " + os.Args[0] + " " + printSummary() + "\n")) + out.Write([]byte(stdioHelpText)) pf.set.SetOutput(out) pf.set.PrintDefaults() } diff --git a/cmd/nebula-cert/print_test.go b/cmd/nebula-cert/print_test.go index 9fa8a549..8d5d31be 100644 --- a/cmd/nebula-cert/print_test.go +++ b/cmd/nebula-cert/print_test.go @@ -2,12 +2,17 @@ package main import ( "bytes" + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "net/netip" "os" "testing" "time" "github.com/slackhq/nebula/cert" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_printSummary(t *testing.T) { @@ -20,6 +25,7 @@ func Test_printHelp(t *testing.T) { assert.Equal( t, "Usage of "+os.Args[0]+" print : prints details about a certificate\n"+ + " Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+ " -json\n"+ " \tOptional: outputs certificates in json format\n"+ " -out-qr string\n"+ @@ -38,84 +44,241 @@ func Test_printCert(t *testing.T) { // no path err := printCert([]string{}, ob, eb) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) assertHelpError(t, err, "-path is required") // no cert at path ob.Reset() eb.Reset() err = printCert([]string{"-path", "does_not_exist"}, ob, eb) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) - assert.EqualError(t, err, "unable to read cert; open does_not_exist: "+NoSuchFileError) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + require.EqualError(t, err, "unable to read cert; open does_not_exist: "+NoSuchFileError) // invalid cert at path ob.Reset() eb.Reset() tf, err := os.CreateTemp("", "print-cert") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(tf.Name()) tf.WriteString("-----BEGIN NOPE-----") err = printCert([]string{"-path", tf.Name()}, ob, eb) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) - assert.EqualError(t, err, "error while unmarshaling cert: input did not contain a valid PEM encoded block") + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + require.EqualError(t, err, "error while unmarshaling cert: input did not contain a valid PEM encoded block") // test multiple certs ob.Reset() eb.Reset() tf.Truncate(0) tf.Seek(0, 0) - c := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "test", - Groups: []string{"hi"}, - PublicKey: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2}, - }, - Signature: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2}, - } + ca, caKey := NewTestCaCert("test ca", nil, nil, time.Time{}, time.Time{}, nil, nil, nil) + c, _ := NewTestCert(ca, caKey, "test", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.0.0.123/8")}, nil, []string{"hi"}) - p, _ := c.MarshalToPEM() + p, _ := c.MarshalPEM() tf.Write(p) tf.Write(p) tf.Write(p) err = printCert([]string{"-path", tf.Name()}, ob, eb) - assert.Nil(t, err) + fp, _ := c.Fingerprint() + pk := hex.EncodeToString(c.PublicKey()) + sig := hex.EncodeToString(c.Signature()) + require.NoError(t, err) assert.Equal( t, - "NebulaCertificate {\n\tDetails {\n\t\tName: test\n\t\tIps: []\n\t\tSubnets: []\n\t\tGroups: [\n\t\t\t\"hi\"\n\t\t]\n\t\tNot before: 0001-01-01 00:00:00 +0000 UTC\n\t\tNot After: 0001-01-01 00:00:00 +0000 UTC\n\t\tIs CA: false\n\t\tIssuer: \n\t\tPublic key: 0102030405060708090001020304050607080900010203040506070809000102\n\t\tCurve: CURVE25519\n\t}\n\tFingerprint: cc3492c0e9c48f17547f5987ea807462ebb3451e622590a10bb3763c344c82bd\n\tSignature: 0102030405060708090001020304050607080900010203040506070809000102\n}\nNebulaCertificate {\n\tDetails {\n\t\tName: test\n\t\tIps: []\n\t\tSubnets: []\n\t\tGroups: [\n\t\t\t\"hi\"\n\t\t]\n\t\tNot before: 0001-01-01 00:00:00 +0000 UTC\n\t\tNot After: 0001-01-01 00:00:00 +0000 UTC\n\t\tIs CA: false\n\t\tIssuer: \n\t\tPublic key: 0102030405060708090001020304050607080900010203040506070809000102\n\t\tCurve: CURVE25519\n\t}\n\tFingerprint: cc3492c0e9c48f17547f5987ea807462ebb3451e622590a10bb3763c344c82bd\n\tSignature: 0102030405060708090001020304050607080900010203040506070809000102\n}\nNebulaCertificate {\n\tDetails {\n\t\tName: test\n\t\tIps: []\n\t\tSubnets: []\n\t\tGroups: [\n\t\t\t\"hi\"\n\t\t]\n\t\tNot before: 0001-01-01 00:00:00 +0000 UTC\n\t\tNot After: 0001-01-01 00:00:00 +0000 UTC\n\t\tIs CA: false\n\t\tIssuer: \n\t\tPublic key: 0102030405060708090001020304050607080900010203040506070809000102\n\t\tCurve: CURVE25519\n\t}\n\tFingerprint: cc3492c0e9c48f17547f5987ea807462ebb3451e622590a10bb3763c344c82bd\n\tSignature: 0102030405060708090001020304050607080900010203040506070809000102\n}\n", + //"NebulaCertificate {\n\tDetails {\n\t\tName: test\n\t\tIps: []\n\t\tSubnets: []\n\t\tGroups: [\n\t\t\t\"hi\"\n\t\t]\n\t\tNot before: 0001-01-01 00:00:00 +0000 UTC\n\t\tNot After: 0001-01-01 00:00:00 +0000 UTC\n\t\tIs CA: false\n\t\tIssuer: "+c.Issuer()+"\n\t\tPublic key: "+pk+"\n\t\tCurve: CURVE25519\n\t}\n\tFingerprint: "+fp+"\n\tSignature: "+sig+"\n}\nNebulaCertificate {\n\tDetails {\n\t\tName: test\n\t\tIps: []\n\t\tSubnets: []\n\t\tGroups: [\n\t\t\t\"hi\"\n\t\t]\n\t\tNot before: 0001-01-01 00:00:00 +0000 UTC\n\t\tNot After: 0001-01-01 00:00:00 +0000 UTC\n\t\tIs CA: false\n\t\tIssuer: "+c.Issuer()+"\n\t\tPublic key: "+pk+"\n\t\tCurve: CURVE25519\n\t}\n\tFingerprint: "+fp+"\n\tSignature: "+sig+"\n}\nNebulaCertificate {\n\tDetails {\n\t\tName: test\n\t\tIps: []\n\t\tSubnets: []\n\t\tGroups: [\n\t\t\t\"hi\"\n\t\t]\n\t\tNot before: 0001-01-01 00:00:00 +0000 UTC\n\t\tNot After: 0001-01-01 00:00:00 +0000 UTC\n\t\tIs CA: false\n\t\tIssuer: "+c.Issuer()+"\n\t\tPublic key: "+pk+"\n\t\tCurve: CURVE25519\n\t}\n\tFingerprint: "+fp+"\n\tSignature: "+sig+"\n}\n", + `{ + "details": { + "curve": "CURVE25519", + "groups": [ + "hi" + ], + "isCa": false, + "issuer": "`+c.Issuer()+`", + "name": "test", + "networks": [ + "10.0.0.123/8" + ], + "notAfter": "0001-01-01T00:00:00Z", + "notBefore": "0001-01-01T00:00:00Z", + "publicKey": "`+pk+`", + "unsafeNetworks": [] + }, + "fingerprint": "`+fp+`", + "signature": "`+sig+`", + "version": 1 +} +{ + "details": { + "curve": "CURVE25519", + "groups": [ + "hi" + ], + "isCa": false, + "issuer": "`+c.Issuer()+`", + "name": "test", + "networks": [ + "10.0.0.123/8" + ], + "notAfter": "0001-01-01T00:00:00Z", + "notBefore": "0001-01-01T00:00:00Z", + "publicKey": "`+pk+`", + "unsafeNetworks": [] + }, + "fingerprint": "`+fp+`", + "signature": "`+sig+`", + "version": 1 +} +{ + "details": { + "curve": "CURVE25519", + "groups": [ + "hi" + ], + "isCa": false, + "issuer": "`+c.Issuer()+`", + "name": "test", + "networks": [ + "10.0.0.123/8" + ], + "notAfter": "0001-01-01T00:00:00Z", + "notBefore": "0001-01-01T00:00:00Z", + "publicKey": "`+pk+`", + "unsafeNetworks": [] + }, + "fingerprint": "`+fp+`", + "signature": "`+sig+`", + "version": 1 +} +`, ob.String(), ) - assert.Equal(t, "", eb.String()) + assert.Empty(t, eb.String()) // test json ob.Reset() eb.Reset() tf.Truncate(0) tf.Seek(0, 0) - c = cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "test", - Groups: []string{"hi"}, - PublicKey: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2}, - }, - Signature: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2}, - } - - p, _ = c.MarshalToPEM() tf.Write(p) tf.Write(p) tf.Write(p) err = printCert([]string{"-json", "-path", tf.Name()}, ob, eb) - assert.Nil(t, err) + fp, _ = c.Fingerprint() + pk = hex.EncodeToString(c.PublicKey()) + sig = hex.EncodeToString(c.Signature()) + require.NoError(t, err) + assert.Equal( + t, + `[{"details":{"curve":"CURVE25519","groups":["hi"],"isCa":false,"issuer":"`+c.Issuer()+`","name":"test","networks":["10.0.0.123/8"],"notAfter":"0001-01-01T00:00:00Z","notBefore":"0001-01-01T00:00:00Z","publicKey":"`+pk+`","unsafeNetworks":[]},"fingerprint":"`+fp+`","signature":"`+sig+`","version":1},{"details":{"curve":"CURVE25519","groups":["hi"],"isCa":false,"issuer":"`+c.Issuer()+`","name":"test","networks":["10.0.0.123/8"],"notAfter":"0001-01-01T00:00:00Z","notBefore":"0001-01-01T00:00:00Z","publicKey":"`+pk+`","unsafeNetworks":[]},"fingerprint":"`+fp+`","signature":"`+sig+`","version":1},{"details":{"curve":"CURVE25519","groups":["hi"],"isCa":false,"issuer":"`+c.Issuer()+`","name":"test","networks":["10.0.0.123/8"],"notAfter":"0001-01-01T00:00:00Z","notBefore":"0001-01-01T00:00:00Z","publicKey":"`+pk+`","unsafeNetworks":[]},"fingerprint":"`+fp+`","signature":"`+sig+`","version":1}] +`, + ob.String(), + ) + assert.Empty(t, eb.String()) + + // read cert from stdin + ob.Reset() + eb.Reset() + withStdin(t, bytes.NewReader(p)) + err = printCert([]string{"-json", "-path", "-"}, ob, eb) + require.NoError(t, err) assert.Equal( t, - "{\"details\":{\"curve\":\"CURVE25519\",\"groups\":[\"hi\"],\"ips\":[],\"isCa\":false,\"issuer\":\"\",\"name\":\"test\",\"notAfter\":\"0001-01-01T00:00:00Z\",\"notBefore\":\"0001-01-01T00:00:00Z\",\"publicKey\":\"0102030405060708090001020304050607080900010203040506070809000102\",\"subnets\":[]},\"fingerprint\":\"cc3492c0e9c48f17547f5987ea807462ebb3451e622590a10bb3763c344c82bd\",\"signature\":\"0102030405060708090001020304050607080900010203040506070809000102\"}\n{\"details\":{\"curve\":\"CURVE25519\",\"groups\":[\"hi\"],\"ips\":[],\"isCa\":false,\"issuer\":\"\",\"name\":\"test\",\"notAfter\":\"0001-01-01T00:00:00Z\",\"notBefore\":\"0001-01-01T00:00:00Z\",\"publicKey\":\"0102030405060708090001020304050607080900010203040506070809000102\",\"subnets\":[]},\"fingerprint\":\"cc3492c0e9c48f17547f5987ea807462ebb3451e622590a10bb3763c344c82bd\",\"signature\":\"0102030405060708090001020304050607080900010203040506070809000102\"}\n{\"details\":{\"curve\":\"CURVE25519\",\"groups\":[\"hi\"],\"ips\":[],\"isCa\":false,\"issuer\":\"\",\"name\":\"test\",\"notAfter\":\"0001-01-01T00:00:00Z\",\"notBefore\":\"0001-01-01T00:00:00Z\",\"publicKey\":\"0102030405060708090001020304050607080900010203040506070809000102\",\"subnets\":[]},\"fingerprint\":\"cc3492c0e9c48f17547f5987ea807462ebb3451e622590a10bb3763c344c82bd\",\"signature\":\"0102030405060708090001020304050607080900010203040506070809000102\"}\n", + `[{"details":{"curve":"CURVE25519","groups":["hi"],"isCa":false,"issuer":"`+c.Issuer()+`","name":"test","networks":["10.0.0.123/8"],"notAfter":"0001-01-01T00:00:00Z","notBefore":"0001-01-01T00:00:00Z","publicKey":"`+pk+`","unsafeNetworks":[]},"fingerprint":"`+fp+`","signature":"`+sig+`","version":1}] +`, ob.String(), ) - assert.Equal(t, "", eb.String()) + assert.Empty(t, eb.String()) + + // -out-qr - sends only the PNG to stdout, suppressing the cert dump + ob.Reset() + eb.Reset() + withStdin(t, bytes.NewReader(p)) + err = printCert([]string{"-path", "-", "-out-qr", "-"}, ob, eb) + require.NoError(t, err) + assert.Empty(t, eb.String()) + stdout := ob.Bytes() + require.NotEmpty(t, stdout) + // PNG magic, no PEM/JSON noise prepended + assert.Equal(t, []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, stdout[:8]) + assert.NotContains(t, string(stdout), "NebulaCertificate") + assert.NotContains(t, string(stdout), `"details"`) + + // json + out-qr - still suppresses json + ob.Reset() + eb.Reset() + withStdin(t, bytes.NewReader(p)) + err = printCert([]string{"-json", "-path", "-", "-out-qr", "-"}, ob, eb) + require.NoError(t, err) + assert.Empty(t, eb.String()) + assert.Equal(t, []byte{0x89, 'P', 'N', 'G'}, ob.Bytes()[:4]) + assert.NotContains(t, ob.String(), `"details"`) +} + +// NewTestCaCert will generate a CA cert +func NewTestCaCert(name string, pubKey, privKey []byte, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (cert.Certificate, []byte) { + var err error + if pubKey == nil || privKey == nil { + pubKey, privKey, err = ed25519.GenerateKey(rand.Reader) + if err != nil { + panic(err) + } + } + + t := &cert.TBSCertificate{ + Version: cert.Version1, + Name: name, + NotBefore: time.Unix(before.Unix(), 0), + NotAfter: time.Unix(after.Unix(), 0), + PublicKey: pubKey, + Networks: networks, + UnsafeNetworks: unsafeNetworks, + Groups: groups, + IsCA: true, + } + + c, err := t.Sign(nil, cert.Curve_CURVE25519, privKey) + if err != nil { + panic(err) + } + + return c, privKey +} + +func NewTestCert(ca cert.Certificate, signerKey []byte, name string, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (cert.Certificate, []byte) { + if before.IsZero() { + before = ca.NotBefore() + } + + if after.IsZero() { + after = ca.NotAfter() + } + + if len(networks) == 0 { + networks = []netip.Prefix{netip.MustParsePrefix("10.0.0.123/8")} + } + + pub, rawPriv := x25519Keypair() + nc := &cert.TBSCertificate{ + Version: cert.Version1, + Name: name, + Networks: networks, + UnsafeNetworks: unsafeNetworks, + Groups: groups, + NotBefore: time.Unix(before.Unix(), 0), + NotAfter: time.Unix(after.Unix(), 0), + PublicKey: pub, + IsCA: false, + } + + c, err := nc.Sign(ca, ca.Curve(), signerKey) + if err != nil { + panic(err) + } + + return c, rawPriv } diff --git a/cmd/nebula-cert/sign.go b/cmd/nebula-cert/sign.go index 35d64468..9b57c4fe 100644 --- a/cmd/nebula-cert/sign.go +++ b/cmd/nebula-cert/sign.go @@ -3,50 +3,63 @@ package main import ( "crypto/ecdh" "crypto/rand" + "errors" "flag" "fmt" "io" - "net" + "net/netip" "os" "strings" "time" "github.com/skip2/go-qrcode" "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/pkclient" "golang.org/x/crypto/curve25519" ) type signFlags struct { - set *flag.FlagSet - caKeyPath *string - caCertPath *string - name *string - ip *string - duration *time.Duration - inPubPath *string - outKeyPath *string - outCertPath *string - outQRPath *string - groups *string - subnets *string + set *flag.FlagSet + version *uint + caKeyPath *string + caCertPath *string + name *string + networks *string + unsafeNetworks *string + duration *time.Duration + inPubPath *string + outKeyPath *string + outCertPath *string + outQRPath *string + groups *string + + p11url *string + + // Deprecated options + ip *string + subnets *string } func newSignFlags() *signFlags { sf := signFlags{set: flag.NewFlagSet("sign", flag.ContinueOnError)} sf.set.Usage = func() {} + sf.version = sf.set.Uint("version", 0, "Optional: version of the certificate format to use. The default is to match the version of the signing CA") sf.caKeyPath = sf.set.String("ca-key", "ca.key", "Optional: path to the signing CA key") sf.caCertPath = sf.set.String("ca-crt", "ca.crt", "Optional: path to the signing CA cert") sf.name = sf.set.String("name", "", "Required: name of the cert, usually a hostname") - sf.ip = sf.set.String("ip", "", "Required: ipv4 address and network in CIDR notation to assign the cert") + sf.networks = sf.set.String("networks", "", "Required: comma separated list of ip address and network in CIDR notation to assign to this cert") + sf.unsafeNetworks = sf.set.String("unsafe-networks", "", "Optional: comma separated list of ip address and network in CIDR notation. Unsafe networks this cert can route for") sf.duration = sf.set.Duration("duration", 0, "Optional: how long the cert should be valid for. The default is 1 second before the signing cert expires. Valid time units are seconds: \"s\", minutes: \"m\", hours: \"h\"") sf.inPubPath = sf.set.String("in-pub", "", "Optional (if out-key not set): path to read a previously generated public key") sf.outKeyPath = sf.set.String("out-key", "", "Optional (if in-pub not set): path to write the private key to") sf.outCertPath = sf.set.String("out-crt", "", "Optional: path to write the certificate to") sf.outQRPath = sf.set.String("out-qr", "", "Optional: output a qr code image (png) of the certificate") sf.groups = sf.set.String("groups", "", "Optional: comma separated list of groups") - sf.subnets = sf.set.String("subnets", "", "Optional: comma separated list of ipv4 address and network in CIDR notation. Subnets this cert can serve for") - return &sf + sf.p11url = p11Flag(sf.set) + sf.ip = sf.set.String("ip", "", "Deprecated, see -networks") + sf.subnets = sf.set.String("subnets", "", "Deprecated, see -unsafe-networks") + return &sf } func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error { @@ -56,8 +69,12 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) return err } - if err := mustFlagString("ca-key", sf.caKeyPath); err != nil { - return err + isP11 := len(*sf.p11url) > 0 + + if !isP11 { + if err := mustFlagString("ca-key", sf.caKeyPath); err != nil { + return err + } } if err := mustFlagString("ca-crt", sf.caCertPath); err != nil { return err @@ -65,187 +82,325 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) if err := mustFlagString("name", sf.name); err != nil { return err } - if err := mustFlagString("ip", sf.ip); err != nil { - return err - } - if *sf.inPubPath != "" && *sf.outKeyPath != "" { + if !isP11 && *sf.inPubPath != "" && *sf.outKeyPath != "" { return newHelpErrorf("cannot set both -in-pub and -out-key") } + if isP11 && *sf.outKeyPath != "" { + return newHelpErrorf("cannot set -out-key with -pkcs11") + } - rawCAKey, err := os.ReadFile(*sf.caKeyPath) - if err != nil { - return fmt.Errorf("error while reading ca-key: %s", err) + var v4Networks []netip.Prefix + var v6Networks []netip.Prefix + if *sf.networks == "" && *sf.ip != "" { + // Pull up deprecated -ip flag if needed + *sf.networks = *sf.ip + } + + if len(*sf.networks) == 0 { + return newHelpErrorf("-networks is required") + } + + version := cert.Version(*sf.version) + if version != 0 && version != cert.Version1 && version != cert.Version2 { + return newHelpErrorf("-version must be either %v or %v", cert.Version1, cert.Version2) + } + + if *sf.outKeyPath == "" { + *sf.outKeyPath = *sf.name + ".key" + } + if *sf.outCertPath == "" { + *sf.outCertPath = *sf.name + ".crt" + } + + var claims ioClaims + if err := reserveInputs(&claims, + "ca-key", *sf.caKeyPath, + "ca-crt", *sf.caCertPath, + "in-pub", *sf.inPubPath, + ); err != nil { + return err + } + if err := reserveOutputs(&claims, + "out-key", *sf.outKeyPath, + "out-crt", *sf.outCertPath, + "out-qr", *sf.outQRPath, + ); err != nil { + return err } var curve cert.Curve var caKey []byte - // naively attempt to decode the private key as though it is not encrypted - caKey, _, curve, err = cert.UnmarshalSigningPrivateKey(rawCAKey) - if err == cert.ErrPrivateKeyEncrypted { - // ask for a passphrase until we get one - var passphrase []byte - for i := 0; i < 5; i++ { - out.Write([]byte("Enter passphrase: ")) - passphrase, err = pr.ReadPassword() - - if err == ErrNoTerminal { - return fmt.Errorf("ca-key is encrypted and must be decrypted interactively") - } else if err != nil { - return fmt.Errorf("error reading password: %s", err) - } - - if len(passphrase) > 0 { - break - } - } - if len(passphrase) == 0 { - return fmt.Errorf("cannot open encrypted ca-key without passphrase") + if !isP11 { + var rawCAKey []byte + rawCAKey, err = readInput("ca-key", *sf.caKeyPath, &claims) + if err != nil { + return fmt.Errorf("error while reading ca-key: %s", err) } - curve, caKey, _, err = cert.DecryptAndUnmarshalSigningPrivateKey(passphrase, rawCAKey) - if err != nil { - return fmt.Errorf("error while parsing encrypted ca-key: %s", err) + // naively attempt to decode the private key as though it is not encrypted + caKey, _, curve, err = cert.UnmarshalSigningPrivateKeyFromPEM(rawCAKey) + if errors.Is(err, cert.ErrPrivateKeyEncrypted) { + var passphrase []byte + passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE")) + if len(passphrase) == 0 { + // ask for a passphrase until we get one + for i := 0; i < 5; i++ { + errOut.Write([]byte("Enter passphrase: ")) + passphrase, err = pr.ReadPassword() + + if errors.Is(err, ErrNoTerminal) { + return fmt.Errorf("ca-key is encrypted and must be decrypted interactively") + } else if err != nil { + return fmt.Errorf("error reading password: %s", err) + } + + if len(passphrase) > 0 { + break + } + } + if len(passphrase) == 0 { + return fmt.Errorf("cannot open encrypted ca-key without passphrase") + } + } + curve, caKey, _, err = cert.DecryptAndUnmarshalSigningPrivateKey(passphrase, rawCAKey) + if err != nil { + return fmt.Errorf("error while parsing encrypted ca-key: %s", err) + } + } else if err != nil { + return fmt.Errorf("error while parsing ca-key: %s", err) } - } else if err != nil { - return fmt.Errorf("error while parsing ca-key: %s", err) } - rawCACert, err := os.ReadFile(*sf.caCertPath) + rawCACert, err := readInput("ca-crt", *sf.caCertPath, &claims) if err != nil { return fmt.Errorf("error while reading ca-crt: %s", err) } - caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM(rawCACert) + caCert, _, err := cert.UnmarshalCertificateFromPEM(rawCACert) if err != nil { return fmt.Errorf("error while parsing ca-crt: %s", err) } - if err := caCert.VerifyPrivateKey(curve, caKey); err != nil { - return fmt.Errorf("refusing to sign, root certificate does not match private key") - } - - issuer, err := caCert.Sha256Sum() - if err != nil { - return fmt.Errorf("error while getting -ca-crt fingerprint: %s", err) + if !isP11 { + if err := caCert.VerifyPrivateKey(curve, caKey); err != nil { + return fmt.Errorf("refusing to sign, root certificate does not match private key") + } } if caCert.Expired(time.Now()) { return fmt.Errorf("ca certificate is expired") } + if version == 0 { + version = caCert.Version() + } + // if no duration is given, expire one second before the root expires if *sf.duration <= 0 { - *sf.duration = time.Until(caCert.Details.NotAfter) - time.Second*1 + *sf.duration = time.Until(caCert.NotAfter()) - time.Second*1 } - ip, ipNet, err := net.ParseCIDR(*sf.ip) - if err != nil { - return newHelpErrorf("invalid ip definition: %s", err) - } - if ip.To4() == nil { - return newHelpErrorf("invalid ip definition: can only be ipv4, have %s", *sf.ip) - } - ipNet.IP = ip + if *sf.networks != "" { + for _, rs := range strings.Split(*sf.networks, ",") { + rs := strings.Trim(rs, " ") + if rs != "" { + n, err := netip.ParsePrefix(rs) + if err != nil { + return newHelpErrorf("invalid -networks definition: %s", rs) + } - groups := []string{} - if *sf.groups != "" { - for _, rg := range strings.Split(*sf.groups, ",") { - g := strings.TrimSpace(rg) - if g != "" { - groups = append(groups, g) + if n.Addr().Is4() { + v4Networks = append(v4Networks, n) + } else { + v6Networks = append(v6Networks, n) + } } } } - subnets := []*net.IPNet{} - if *sf.subnets != "" { - for _, rs := range strings.Split(*sf.subnets, ",") { + var v4UnsafeNetworks []netip.Prefix + var v6UnsafeNetworks []netip.Prefix + if *sf.unsafeNetworks == "" && *sf.subnets != "" { + // Pull up deprecated -subnets flag if needed + *sf.unsafeNetworks = *sf.subnets + } + + if *sf.unsafeNetworks != "" { + for _, rs := range strings.Split(*sf.unsafeNetworks, ",") { rs := strings.Trim(rs, " ") if rs != "" { - _, s, err := net.ParseCIDR(rs) + n, err := netip.ParsePrefix(rs) if err != nil { - return newHelpErrorf("invalid subnet definition: %s", err) + return newHelpErrorf("invalid -unsafe-networks definition: %s", rs) } - if s.IP.To4() == nil { - return newHelpErrorf("invalid subnet definition: can only be ipv4, have %s", rs) + + if n.Addr().Is4() { + v4UnsafeNetworks = append(v4UnsafeNetworks, n) + } else { + v6UnsafeNetworks = append(v6UnsafeNetworks, n) } - subnets = append(subnets, s) + } + } + } + + var groups []string + if *sf.groups != "" { + for _, rg := range strings.Split(*sf.groups, ",") { + g := strings.TrimSpace(rg) + if g != "" { + groups = append(groups, g) } } } var pub, rawPriv []byte + var p11Client *pkclient.PKClient + + if isP11 { + curve = cert.Curve_P256 + p11Client, err = pkclient.FromUrl(*sf.p11url) + if err != nil { + return fmt.Errorf("error while creating PKCS#11 client: %w", err) + } + defer func(client *pkclient.PKClient) { + _ = client.Close() + }(p11Client) + } + if *sf.inPubPath != "" { - rawPub, err := os.ReadFile(*sf.inPubPath) + var pubCurve cert.Curve + rawPub, err := readInput("in-pub", *sf.inPubPath, &claims) if err != nil { return fmt.Errorf("error while reading in-pub: %s", err) } - var pubCurve cert.Curve - pub, _, pubCurve, err = cert.UnmarshalPublicKey(rawPub) + + pub, _, pubCurve, err = cert.UnmarshalPublicKeyFromPEM(rawPub) if err != nil { return fmt.Errorf("error while parsing in-pub: %s", err) } if pubCurve != curve { return fmt.Errorf("curve of in-pub does not match ca") } + } else if isP11 { + pub, err = p11Client.GetPubKey() + if err != nil { + return fmt.Errorf("error while getting public key with PKCS#11: %w", err) + } } else { pub, rawPriv = newKeypair(curve) } - nc := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: *sf.name, - Ips: []*net.IPNet{ipNet}, - Groups: groups, - Subnets: subnets, - NotBefore: time.Now(), - NotAfter: time.Now().Add(*sf.duration), - PublicKey: pub, - IsCA: false, - Issuer: issuer, - Curve: curve, - }, + if !isStdio(*sf.outCertPath) { + if _, err := os.Stat(*sf.outCertPath); err == nil { + return fmt.Errorf("refusing to overwrite existing cert: %s", *sf.outCertPath) + } } - if err := nc.CheckRootConstrains(caCert); err != nil { - return fmt.Errorf("refusing to sign, root certificate constraints violated: %s", err) - } + var crts []cert.Certificate - if *sf.outKeyPath == "" { - *sf.outKeyPath = *sf.name + ".key" - } + notBefore := time.Now() + notAfter := notBefore.Add(*sf.duration) - if *sf.outCertPath == "" { - *sf.outCertPath = *sf.name + ".crt" - } + switch version { + case cert.Version1: + // Make sure we have only one ipv4 address + if len(v4Networks) != 1 { + return newHelpErrorf("invalid -networks definition: v1 certificates can only have a single ipv4 address") + } - if _, err := os.Stat(*sf.outCertPath); err == nil { - return fmt.Errorf("refusing to overwrite existing cert: %s", *sf.outCertPath) - } + if len(v6Networks) > 0 { + return newHelpErrorf("invalid -networks definition: v1 certificates can only contain ipv4 addresses") + } - err = nc.Sign(curve, caKey) - if err != nil { - return fmt.Errorf("error while signing: %s", err) + if len(v6UnsafeNetworks) > 0 { + return newHelpErrorf("invalid -unsafe-networks definition: v1 certificates can only contain ipv4 addresses") + } + + t := &cert.TBSCertificate{ + Version: cert.Version1, + Name: *sf.name, + Networks: []netip.Prefix{v4Networks[0]}, + Groups: groups, + UnsafeNetworks: v4UnsafeNetworks, + NotBefore: notBefore, + NotAfter: notAfter, + PublicKey: pub, + IsCA: false, + Curve: curve, + } + + var nc cert.Certificate + if p11Client == nil { + nc, err = t.Sign(caCert, curve, caKey) + if err != nil { + return fmt.Errorf("error while signing: %w", err) + } + } else { + nc, err = t.SignWith(caCert, curve, p11Client.SignASN1) + if err != nil { + return fmt.Errorf("error while signing with PKCS#11: %w", err) + } + } + + crts = append(crts, nc) + + case cert.Version2: + t := &cert.TBSCertificate{ + Version: cert.Version2, + Name: *sf.name, + Networks: append(v4Networks, v6Networks...), + Groups: groups, + UnsafeNetworks: append(v4UnsafeNetworks, v6UnsafeNetworks...), + NotBefore: notBefore, + NotAfter: notAfter, + PublicKey: pub, + IsCA: false, + Curve: curve, + } + + var nc cert.Certificate + if p11Client == nil { + nc, err = t.Sign(caCert, curve, caKey) + if err != nil { + return fmt.Errorf("error while signing: %w", err) + } + } else { + nc, err = t.SignWith(caCert, curve, p11Client.SignASN1) + if err != nil { + return fmt.Errorf("error while signing with PKCS#11: %w", err) + } + } + + crts = append(crts, nc) + default: + // this should be unreachable + return fmt.Errorf("invalid version: %d", version) } - if *sf.inPubPath == "" { - if _, err := os.Stat(*sf.outKeyPath); err == nil { - return fmt.Errorf("refusing to overwrite existing key: %s", *sf.outKeyPath) + if !isP11 && *sf.inPubPath == "" { + if !isStdio(*sf.outKeyPath) { + if _, err := os.Stat(*sf.outKeyPath); err == nil { + return fmt.Errorf("refusing to overwrite existing key: %s", *sf.outKeyPath) + } } - err = os.WriteFile(*sf.outKeyPath, cert.MarshalPrivateKey(curve, rawPriv), 0600) + err = writeOutput(*sf.outKeyPath, cert.MarshalPrivateKeyToPEM(curve, rawPriv), 0600, out) if err != nil { return fmt.Errorf("error while writing out-key: %s", err) } } - b, err := nc.MarshalToPEM() - if err != nil { - return fmt.Errorf("error while marshalling certificate: %s", err) + var b []byte + for _, c := range crts { + sb, err := c.MarshalPEM() + if err != nil { + return fmt.Errorf("error while marshalling certificate: %s", err) + } + b = append(b, sb...) } - err = os.WriteFile(*sf.outCertPath, b, 0600) + err = writeOutput(*sf.outCertPath, b, 0600, out) if err != nil { return fmt.Errorf("error while writing out-crt: %s", err) } @@ -256,7 +411,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) return fmt.Errorf("error while generating qr code: %s", err) } - err = os.WriteFile(*sf.outQRPath, b, 0600) + err = writeOutput(*sf.outQRPath, b, 0600, out) if err != nil { return fmt.Errorf("error while writing out-qr: %s", err) } @@ -306,6 +461,7 @@ func signSummary() string { func signHelp(out io.Writer) { sf := newSignFlags() out.Write([]byte("Usage of " + os.Args[0] + " " + signSummary() + "\n")) + out.Write([]byte(stdioHelpText)) sf.set.SetOutput(out) sf.set.PrintDefaults() } diff --git a/cmd/nebula-cert/sign_test.go b/cmd/nebula-cert/sign_test.go index adf83a26..64d5c7d9 100644 --- a/cmd/nebula-cert/sign_test.go +++ b/cmd/nebula-cert/sign_test.go @@ -13,11 +13,10 @@ import ( "github.com/slackhq/nebula/cert" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/crypto/ed25519" ) -//TODO: test file permissions - func Test_signSummary(t *testing.T) { assert.Equal(t, "sign : create and sign a certificate", signSummary()) } @@ -28,6 +27,7 @@ func Test_signHelp(t *testing.T) { assert.Equal( t, "Usage of "+os.Args[0]+" sign : create and sign a certificate\n"+ + " Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+ " -ca-crt string\n"+ " \tOptional: path to the signing CA cert (default \"ca.crt\")\n"+ " -ca-key string\n"+ @@ -39,17 +39,24 @@ func Test_signHelp(t *testing.T) { " -in-pub string\n"+ " \tOptional (if out-key not set): path to read a previously generated public key\n"+ " -ip string\n"+ - " \tRequired: ipv4 address and network in CIDR notation to assign the cert\n"+ + " \tDeprecated, see -networks\n"+ " -name string\n"+ " \tRequired: name of the cert, usually a hostname\n"+ + " -networks string\n"+ + " \tRequired: comma separated list of ip address and network in CIDR notation to assign to this cert\n"+ " -out-crt string\n"+ " \tOptional: path to write the certificate to\n"+ " -out-key string\n"+ " \tOptional (if in-pub not set): path to write the private key to\n"+ " -out-qr string\n"+ " \tOptional: output a qr code image (png) of the certificate\n"+ + optionalPkcs11String(" -pkcs11 string\n \tOptional: PKCS#11 URI to an existing private key\n")+ " -subnets string\n"+ - " \tOptional: comma separated list of ipv4 address and network in CIDR notation. Subnets this cert can serve for\n", + " \tDeprecated, see -unsafe-networks\n"+ + " -unsafe-networks string\n"+ + " \tOptional: comma separated list of ip address and network in CIDR notation. Unsafe networks this cert can route for\n"+ + " -version uint\n"+ + " \tOptional: version of the certificate format to use. The default is to match the version of the signing CA\n", ob.String(), ) } @@ -76,20 +83,20 @@ func Test_signCert(t *testing.T) { // required args assertHelpError(t, signCert( - []string{"-ca-crt", "./nope", "-ca-key", "./nope", "-ip", "1.1.1.1/24", "-out-key", "nope", "-out-crt", "nope"}, ob, eb, nopw, + []string{"-version", "1", "-ca-crt", "./nope", "-ca-key", "./nope", "-ip", "1.1.1.1/24", "-out-key", "nope", "-out-crt", "nope"}, ob, eb, nopw, ), "-name is required") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) assertHelpError(t, signCert( - []string{"-ca-crt", "./nope", "-ca-key", "./nope", "-name", "test", "-out-key", "nope", "-out-crt", "nope"}, ob, eb, nopw, - ), "-ip is required") + []string{"-version", "1", "-ca-crt", "./nope", "-ca-key", "./nope", "-name", "test", "-out-key", "nope", "-out-crt", "nope"}, ob, eb, nopw, + ), "-networks is required") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // cannot set -in-pub and -out-key assertHelpError(t, signCert( - []string{"-ca-crt", "./nope", "-ca-key", "./nope", "-name", "test", "-in-pub", "nope", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope"}, ob, eb, nopw, + []string{"-version", "1", "-ca-crt", "./nope", "-ca-key", "./nope", "-name", "test", "-in-pub", "nope", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope"}, ob, eb, nopw, ), "cannot set both -in-pub and -out-key") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) @@ -97,18 +104,18 @@ func Test_signCert(t *testing.T) { // failed to read key ob.Reset() eb.Reset() - args := []string{"-ca-crt", "./nope", "-ca-key", "./nope", "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "error while reading ca-key: open ./nope: "+NoSuchFileError) + args := []string{"-version", "1", "-ca-crt", "./nope", "-ca-key", "./nope", "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while reading ca-key: open ./nope: "+NoSuchFileError) // failed to unmarshal key ob.Reset() eb.Reset() caKeyF, err := os.CreateTemp("", "sign-cert.key") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(caKeyF.Name()) - args = []string{"-ca-crt", "./nope", "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing ca-key: input did not contain a valid PEM encoded block") + args = []string{"-version", "1", "-ca-crt", "./nope", "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing ca-key: input did not contain a valid PEM encoded block") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) @@ -116,11 +123,11 @@ func Test_signCert(t *testing.T) { ob.Reset() eb.Reset() caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader) - caKeyF.Write(cert.MarshalEd25519PrivateKey(caPriv)) + caKeyF.Write(cert.MarshalSigningPrivateKeyToPEM(cert.Curve_CURVE25519, caPriv)) // failed to read cert - args = []string{"-ca-crt", "./nope", "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "error while reading ca-crt: open ./nope: "+NoSuchFileError) + args = []string{"-version", "1", "-ca-crt", "./nope", "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while reading ca-crt: open ./nope: "+NoSuchFileError) assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) @@ -128,30 +135,22 @@ func Test_signCert(t *testing.T) { ob.Reset() eb.Reset() caCrtF, err := os.CreateTemp("", "sign-cert.crt") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(caCrtF.Name()) - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing ca-crt: input did not contain a valid PEM encoded block") + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing ca-crt: input did not contain a valid PEM encoded block") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // write a proper ca cert for later - ca := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "ca", - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Minute * 200), - PublicKey: caPub, - IsCA: true, - }, - } - b, _ := ca.MarshalToPEM() + ca, _ := NewTestCaCert("ca", caPub, caPriv, time.Now(), time.Now().Add(time.Minute*200), nil, nil, nil) + b, _ := ca.MarshalPEM() caCrtF.Write(b) // failed to read pub - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-in-pub", "./nope", "-duration", "100m"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "error while reading in-pub: open ./nope: "+NoSuchFileError) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-in-pub", "./nope", "-duration", "100m"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while reading in-pub: open ./nope: "+NoSuchFileError) assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) @@ -159,11 +158,11 @@ func Test_signCert(t *testing.T) { ob.Reset() eb.Reset() inPubF, err := os.CreateTemp("", "in.pub") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(inPubF.Name()) - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-in-pub", inPubF.Name(), "-duration", "100m"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing in-pub: input did not contain a valid PEM encoded block") + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-in-pub", inPubF.Name(), "-duration", "100m"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing in-pub: input did not contain a valid PEM encoded block") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) @@ -171,116 +170,124 @@ func Test_signCert(t *testing.T) { ob.Reset() eb.Reset() inPub, _ := x25519Keypair() - inPubF.Write(cert.MarshalX25519PublicKey(inPub)) + inPubF.Write(cert.MarshalPublicKeyToPEM(cert.Curve_CURVE25519, inPub)) // bad ip cidr ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "a1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} - assertHelpError(t, signCert(args, ob, eb, nopw), "invalid ip definition: invalid CIDR address: a1.1.1.1/24") + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "a1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} + assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -networks definition: a1.1.1.1/24") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "100::100/100", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} - assertHelpError(t, signCert(args, ob, eb, nopw), "invalid ip definition: can only be ipv4, have 100::100/100") + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "100::100/100", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} + assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -networks definition: v1 certificates can only have a single ipv4 address") + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + + ob.Reset() + eb.Reset() + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24,1.1.1.2/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} + assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -networks definition: v1 certificates can only have a single ipv4 address") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // bad subnet cidr ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "a"} - assertHelpError(t, signCert(args, ob, eb, nopw), "invalid subnet definition: invalid CIDR address: a") + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "a"} + assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -unsafe-networks definition: a") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "100::100/100"} - assertHelpError(t, signCert(args, ob, eb, nopw), "invalid subnet definition: can only be ipv4, have 100::100/100") + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "100::100/100"} + assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -unsafe-networks definition: v1 certificates can only contain ipv4 addresses") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // mismatched ca key _, caPriv2, _ := ed25519.GenerateKey(rand.Reader) caKeyF2, err := os.CreateTemp("", "sign-cert-2.key") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(caKeyF2.Name()) - caKeyF2.Write(cert.MarshalEd25519PrivateKey(caPriv2)) + caKeyF2.Write(cert.MarshalSigningPrivateKeyToPEM(cert.Curve_CURVE25519, caPriv2)) ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF2.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "a"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "refusing to sign, root certificate does not match private key") + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF2.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "a"} + require.EqualError(t, signCert(args, ob, eb, nopw), "refusing to sign, root certificate does not match private key") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // failed key write ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "/do/not/write/pleasecrt", "-out-key", "/do/not/write/pleasekey", "-duration", "100m", "-subnets", "10.1.1.1/32"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "error while writing out-key: open /do/not/write/pleasekey: "+NoSuchDirError) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "/do/not/write/pleasecrt", "-out-key", "/do/not/write/pleasekey", "-duration", "100m", "-subnets", "10.1.1.1/32"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while writing out-key: open /do/not/write/pleasekey: "+NoSuchDirError) assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // create temp key file keyF, err := os.CreateTemp("", "test.key") - assert.Nil(t, err) + require.NoError(t, err) os.Remove(keyF.Name()) // failed cert write ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "/do/not/write/pleasecrt", "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "error while writing out-crt: open /do/not/write/pleasecrt: "+NoSuchDirError) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "/do/not/write/pleasecrt", "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while writing out-crt: open /do/not/write/pleasecrt: "+NoSuchDirError) assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) os.Remove(keyF.Name()) // create temp cert file crtF, err := os.CreateTemp("", "test.crt") - assert.Nil(t, err) + require.NoError(t, err) os.Remove(crtF.Name()) // test proper cert with removed empty groups and subnets ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.Nil(t, signCert(args, ob, eb, nopw)) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.NoError(t, signCert(args, ob, eb, nopw)) assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // read cert and key files rb, _ := os.ReadFile(keyF.Name()) - lKey, b, err := cert.UnmarshalX25519PrivateKey(rb) - assert.Len(t, b, 0) - assert.Nil(t, err) + lKey, b, curve, err := cert.UnmarshalPrivateKeyFromPEM(rb) + assert.Equal(t, cert.Curve_CURVE25519, curve) + assert.Empty(t, b) + require.NoError(t, err) assert.Len(t, lKey, 32) rb, _ = os.ReadFile(crtF.Name()) - lCrt, b, err := cert.UnmarshalNebulaCertificateFromPEM(rb) - assert.Len(t, b, 0) - assert.Nil(t, err) - - assert.Equal(t, "test", lCrt.Details.Name) - assert.Equal(t, "1.1.1.1/24", lCrt.Details.Ips[0].String()) - assert.Len(t, lCrt.Details.Ips, 1) - assert.False(t, lCrt.Details.IsCA) - assert.Equal(t, []string{"1", "2", "3", "4", "5"}, lCrt.Details.Groups) - assert.Len(t, lCrt.Details.Subnets, 3) - assert.Len(t, lCrt.Details.PublicKey, 32) - assert.Equal(t, time.Duration(time.Minute*100), lCrt.Details.NotAfter.Sub(lCrt.Details.NotBefore)) + lCrt, b, err := cert.UnmarshalCertificateFromPEM(rb) + assert.Empty(t, b) + require.NoError(t, err) + + assert.Equal(t, "test", lCrt.Name()) + assert.Equal(t, "1.1.1.1/24", lCrt.Networks()[0].String()) + assert.Len(t, lCrt.Networks(), 1) + assert.False(t, lCrt.IsCA()) + assert.Equal(t, []string{"1", "2", "3", "4", "5"}, lCrt.Groups()) + assert.Len(t, lCrt.UnsafeNetworks(), 3) + assert.Len(t, lCrt.PublicKey(), 32) + assert.Equal(t, time.Duration(time.Minute*100), lCrt.NotAfter().Sub(lCrt.NotBefore())) sns := []string{} - for _, sn := range lCrt.Details.Subnets { + for _, sn := range lCrt.UnsafeNetworks() { sns = append(sns, sn.String()) } assert.Equal(t, []string{"10.1.1.1/32", "10.2.2.2/32", "10.5.5.5/32"}, sns) - issuer, _ := ca.Sha256Sum() - assert.Equal(t, issuer, lCrt.Details.Issuer) + issuer, _ := ca.Fingerprint() + assert.Equal(t, issuer, lCrt.Issuer()) assert.True(t, lCrt.CheckSignature(caPub)) @@ -289,53 +296,55 @@ func Test_signCert(t *testing.T) { os.Remove(crtF.Name()) ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-in-pub", inPubF.Name(), "-duration", "100m", "-groups", "1"} - assert.Nil(t, signCert(args, ob, eb, nopw)) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-in-pub", inPubF.Name(), "-duration", "100m", "-groups", "1"} + require.NoError(t, signCert(args, ob, eb, nopw)) assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // read cert file and check pub key matches in-pub rb, _ = os.ReadFile(crtF.Name()) - lCrt, b, err = cert.UnmarshalNebulaCertificateFromPEM(rb) - assert.Len(t, b, 0) - assert.Nil(t, err) - assert.Equal(t, lCrt.Details.PublicKey, inPub) + lCrt, b, err = cert.UnmarshalCertificateFromPEM(rb) + assert.Empty(t, b) + require.NoError(t, err) + assert.Equal(t, lCrt.PublicKey(), inPub) // test refuse to sign cert with duration beyond root ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "1000m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "refusing to sign, root certificate constraints violated: certificate expires after signing certificate") + os.Remove(keyF.Name()) + os.Remove(crtF.Name()) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "1000m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while signing: certificate expires after signing certificate") assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // create valid cert/key for overwrite tests os.Remove(keyF.Name()) os.Remove(crtF.Name()) - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.Nil(t, signCert(args, ob, eb, nopw)) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.NoError(t, signCert(args, ob, eb, nopw)) // test that we won't overwrite existing key file os.Remove(crtF.Name()) ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "refusing to overwrite existing key: "+keyF.Name()) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.EqualError(t, signCert(args, ob, eb, nopw), "refusing to overwrite existing key: "+keyF.Name()) assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) // create valid cert/key for overwrite tests os.Remove(keyF.Name()) os.Remove(crtF.Name()) - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.Nil(t, signCert(args, ob, eb, nopw)) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.NoError(t, signCert(args, ob, eb, nopw)) // test that we won't overwrite existing certificate file os.Remove(keyF.Name()) ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.EqualError(t, signCert(args, ob, eb, nopw), "refusing to overwrite existing cert: "+crtF.Name()) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.EqualError(t, signCert(args, ob, eb, nopw), "refusing to overwrite existing cert: "+crtF.Name()) assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) @@ -348,11 +357,11 @@ func Test_signCert(t *testing.T) { eb.Reset() caKeyF, err = os.CreateTemp("", "sign-cert.key") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(caKeyF.Name()) caCrtF, err = os.CreateTemp("", "sign-cert.crt") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(caCrtF.Name()) // generate the encrypted key @@ -361,50 +370,165 @@ func Test_signCert(t *testing.T) { b, _ = cert.EncryptAndMarshalSigningPrivateKey(cert.Curve_CURVE25519, caPriv, passphrase, kdfParams) caKeyF.Write(b) - ca = cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "ca", - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Minute * 200), - PublicKey: caPub, - IsCA: true, - }, - } - b, _ = ca.MarshalToPEM() + ca, _ = NewTestCaCert("ca", caPub, caPriv, time.Now(), time.Now().Add(time.Minute*200), nil, nil, nil) + b, _ = ca.MarshalPEM() caCrtF.Write(b) // test with the proper password - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.Nil(t, signCert(args, ob, eb, testpw)) - assert.Equal(t, "Enter passphrase: ", ob.String()) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.NoError(t, signCert(args, ob, eb, testpw)) + assert.Empty(t, ob.String()) + assert.Equal(t, "Enter passphrase: ", eb.String()) + + // test with the proper password in the environment + os.Remove(crtF.Name()) + os.Remove(keyF.Name()) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + os.Setenv("NEBULA_CA_PASSPHRASE", string(passphrase)) + ob.Reset() + eb.Reset() + require.NoError(t, signCert(args, ob, eb, testpw)) + assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) + os.Setenv("NEBULA_CA_PASSPHRASE", "") // test with the wrong password ob.Reset() eb.Reset() testpw.password = []byte("invalid password") - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.Error(t, signCert(args, ob, eb, testpw)) - assert.Equal(t, "Enter passphrase: ", ob.String()) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.Error(t, signCert(args, ob, eb, testpw)) + assert.Empty(t, ob.String()) + assert.Equal(t, "Enter passphrase: ", eb.String()) + + // test with the wrong password in environment + ob.Reset() + eb.Reset() + + os.Setenv("NEBULA_CA_PASSPHRASE", "invalid password") + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing encrypted ca-key: invalid passphrase or corrupt private key") + assert.Empty(t, ob.String()) assert.Empty(t, eb.String()) + os.Setenv("NEBULA_CA_PASSPHRASE", "") // test with the user not entering a password ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.Error(t, signCert(args, ob, eb, nopw)) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.Error(t, signCert(args, ob, eb, nopw)) // normally the user hitting enter on the prompt would add newlines between these - assert.Equal(t, "Enter passphrase: Enter passphrase: Enter passphrase: Enter passphrase: Enter passphrase: ", ob.String()) - assert.Empty(t, eb.String()) + assert.Empty(t, ob.String()) + assert.Equal(t, "Enter passphrase: Enter passphrase: Enter passphrase: Enter passphrase: Enter passphrase: ", eb.String()) // test an error condition ob.Reset() eb.Reset() - args = []string{"-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} - assert.Error(t, signCert(args, ob, eb, errpw)) - assert.Equal(t, "Enter passphrase: ", ob.String()) + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, , 10.2.2.2/32 , , ,, 10.5.5.5/32", "-groups", "1,, 2 , ,,,3,4,5"} + require.Error(t, signCert(args, ob, eb, errpw)) + assert.Empty(t, ob.String()) + assert.Equal(t, "Enter passphrase: ", eb.String()) +} + +func Test_signCert_stdio(t *testing.T) { + nopw := &StubPasswordReader{ + password: []byte(""), + err: nil, + } + + caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader) + rawCAKey := cert.MarshalSigningPrivateKeyToPEM(cert.Curve_CURVE25519, caPriv) + + ca, _ := NewTestCaCert("ca", caPub, caPriv, time.Now(), time.Now().Add(time.Minute*200), nil, nil, nil) + rawCACrt, _ := ca.MarshalPEM() + + caCrtF, err := os.CreateTemp("", "sign-cert.crt") + require.NoError(t, err) + defer os.Remove(caCrtF.Name()) + caCrtF.Write(rawCACrt) + + caKeyF, err := os.CreateTemp("", "sign-cert.key") + require.NoError(t, err) + defer os.Remove(caKeyF.Name()) + caKeyF.Write(rawCAKey) + + keyF, err := os.CreateTemp("", "sign.key") + require.NoError(t, err) + os.Remove(keyF.Name()) + defer os.Remove(keyF.Name()) + + // ca-key on stdin, cert to stdout + withStdin(t, bytes.NewReader(rawCAKey)) + ob := &bytes.Buffer{} + eb := &bytes.Buffer{} + args := []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", "-", "-name", "stdin-test", "-ip", "1.1.1.1/24", "-out-crt", "-", "-out-key", keyF.Name(), "-duration", "100m"} + require.NoError(t, signCert(args, ob, eb, nopw)) + assert.Empty(t, eb.String()) + + lCrt, _, err := cert.UnmarshalCertificateFromPEM(ob.Bytes()) + require.NoError(t, err) + assert.Equal(t, "stdin-test", lCrt.Name()) + assert.True(t, lCrt.CheckSignature(caPub)) + + // two flags reading from stdin should error before any read attempt; + // otherwise an interactive shell would hang on io.ReadAll + stdinIn := bytes.NewReader(rawCAKey) + withStdin(t, stdinIn) + ob.Reset() + eb.Reset() + args = []string{"-version", "1", "-ca-crt", "-", "-ca-key", "-", "-name", "stdin-test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"} + require.EqualError(t, signCert(args, ob, eb, nopw), + `-ca-key and -ca-crt both set to "-", only one input may read from stdin`) + assert.Equal(t, len(rawCAKey), stdinIn.Len(), "stdin should be untouched when conflict is caught up front") + + // two flags writing to stdout should error before any output is written + // AND before stdin is consumed + stdinR := bytes.NewReader(rawCAKey) + withStdin(t, stdinR) + ob.Reset() + eb.Reset() + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", "-", "-name", "stdin-test", "-ip", "1.1.1.1/24", "-out-crt", "-", "-out-key", "-", "-duration", "100m"} + require.EqualError(t, signCert(args, ob, eb, nopw), + `-out-key and -out-crt both set to "-", only one output may write to stdout`) + assert.Empty(t, ob.String()) + // stdin should be untouched because the conflict was caught up front + assert.Equal(t, len(rawCAKey), stdinR.Len()) + + // out-key on stdout, cert on disk + keyF2, err := os.CreateTemp("", "sign.key") + require.NoError(t, err) + os.Remove(keyF2.Name()) + defer os.Remove(keyF2.Name()) + crtF, err := os.CreateTemp("", "sign.crt") + require.NoError(t, err) + os.Remove(crtF.Name()) + defer os.Remove(crtF.Name()) + + ob.Reset() + eb.Reset() + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "stdin-test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", "-", "-duration", "100m"} + require.NoError(t, signCert(args, ob, eb, nopw)) + assert.Empty(t, eb.String()) + _, _, curve, err := cert.UnmarshalPrivateKeyFromPEM(ob.Bytes()) + require.NoError(t, err) + assert.Equal(t, cert.Curve_CURVE25519, curve) + + // in-pub on stdin (caller already has a keypair, only the cert is generated) + inPub, _ := x25519Keypair() + rawInPub := cert.MarshalPublicKeyToPEM(cert.Curve_CURVE25519, inPub) + + withStdin(t, bytes.NewReader(rawInPub)) + os.Remove(crtF.Name()) + ob.Reset() + eb.Reset() + args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "in-pub-test", "-ip", "1.1.1.1/24", "-in-pub", "-", "-out-crt", "-", "-duration", "100m"} + require.NoError(t, signCert(args, ob, eb, nopw)) assert.Empty(t, eb.String()) + stdinCrt, _, err := cert.UnmarshalCertificateFromPEM(ob.Bytes()) + require.NoError(t, err) + assert.Equal(t, "in-pub-test", stdinCrt.Name()) + assert.Equal(t, inPub, stdinCrt.PublicKey()) } diff --git a/cmd/nebula-cert/stdio.go b/cmd/nebula-cert/stdio.go new file mode 100644 index 00000000..3f71d52f --- /dev/null +++ b/cmd/nebula-cert/stdio.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "io" + "os" +) + +// stdioPath is the special path value that selects stdin (for inputs) or +// stdout (for outputs) instead of a file on disk. +const stdioPath = "-" + +// stdioHelpText is rendered just under the Usage line of each subcommand +// help so the - convention is documented once instead of on every flag. +const stdioHelpText = " Pass \"-\" to any path flag to read from stdin or write to stdout.\n" + +// stdinReader is the source used when an input flag is set to "-". +// It is a package level var so tests can swap in a deterministic reader. +// Tests that mutate stdinReader cannot run with t.Parallel(). +var stdinReader io.Reader = os.Stdin + +// ioClaims tracks which flags have claimed stdin and stdout during a single +// command invocation so we can refuse a second flag asking for the same +// stream. +type ioClaims struct { + in string + out string +} + +func (c *ioClaims) claimIn(flagName string) error { + if c.in != "" && c.in != flagName { + return fmt.Errorf("-%s and -%s both set to %q, only one input may read from stdin", c.in, flagName, stdioPath) + } + c.in = flagName + return nil +} + +func (c *ioClaims) claimOut(flagName string) error { + if c.out != "" && c.out != flagName { + return fmt.Errorf("-%s and -%s both set to %q, only one output may write to stdout", c.out, flagName, stdioPath) + } + c.out = flagName + return nil +} + +// reserveInputs walks alternating (flagName, path) pairs and claims stdin +// for any path equal to stdioPath. It must be called before any input is +// read so a conflict can be reported immediately instead of blocking on +// io.ReadAll while waiting for input that will never arrive. +func reserveInputs(claims *ioClaims, pairs ...string) error { + return reserveStdio(claims, "reserveInputs", (*ioClaims).claimIn, pairs) +} + +// reserveOutputs walks alternating (flagName, path) pairs and claims stdout +// for any path equal to stdioPath. It must be called before any output is +// written so a conflict cannot leave one stream half written before the +// second flag fails. +func reserveOutputs(claims *ioClaims, pairs ...string) error { + return reserveStdio(claims, "reserveOutputs", (*ioClaims).claimOut, pairs) +} + +func reserveStdio(claims *ioClaims, who string, claim func(*ioClaims, string) error, pairs []string) error { + if len(pairs)%2 != 0 { + panic(who + " requires alternating name, path pairs") + } + for i := 0; i < len(pairs); i += 2 { + name, path := pairs[i], pairs[i+1] + if path != stdioPath { + continue + } + if err := claim(claims, name); err != nil { + return err + } + } + return nil +} + +// readInput returns the bytes referenced by path, reading from stdin when +// path is stdioPath. +func readInput(flagName, path string, claims *ioClaims) ([]byte, error) { + if path == stdioPath { + if err := claims.claimIn(flagName); err != nil { + return nil, err + } + return io.ReadAll(stdinReader) + } + return os.ReadFile(path) +} + +// openInput returns a reader for path. When path is stdioPath the returned +// reader wraps stdin and Close is a no-op. +func openInput(flagName, path string, claims *ioClaims) (io.ReadCloser, error) { + if path == stdioPath { + if err := claims.claimIn(flagName); err != nil { + return nil, err + } + return io.NopCloser(stdinReader), nil + } + return os.Open(path) +} + +// writeOutput writes data to path, or to stdout when path is stdioPath. perm +// is only used for file output. The caller must have already claimed stdout +// via reserveOutputs before invoking with stdioPath. +func writeOutput(path string, data []byte, perm os.FileMode, stdout io.Writer) error { + if path == stdioPath { + _, err := stdout.Write(data) + return err + } + return os.WriteFile(path, data, perm) +} + +// isStdio reports whether path is the stdio sentinel and so should skip +// existence checks like "refuse to overwrite". +func isStdio(path string) bool { + return path == stdioPath +} diff --git a/cmd/nebula-cert/stdio_test.go b/cmd/nebula-cert/stdio_test.go new file mode 100644 index 00000000..dc87a597 --- /dev/null +++ b/cmd/nebula-cert/stdio_test.go @@ -0,0 +1,167 @@ +package main + +import ( + "bytes" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// withStdin temporarily replaces stdinReader for the duration of t. +func withStdin(t *testing.T, r io.Reader) { + t.Helper() + prev := stdinReader + stdinReader = r + t.Cleanup(func() { stdinReader = prev }) +} + +func Test_readInput_stdin(t *testing.T) { + withStdin(t, bytes.NewBufferString("hello")) + var claims ioClaims + + got, err := readInput("path", "-", &claims) + require.NoError(t, err) + assert.Equal(t, []byte("hello"), got) + assert.Equal(t, "path", claims.in) +} + +func Test_readInput_file(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "f") + require.NoError(t, os.WriteFile(p, []byte("file"), 0600)) + var claims ioClaims + + got, err := readInput("path", p, &claims) + require.NoError(t, err) + assert.Equal(t, []byte("file"), got) + assert.Empty(t, claims.in) +} + +func Test_readInput_doubleStdinErrors(t *testing.T) { + withStdin(t, bytes.NewBufferString("hello")) + var claims ioClaims + + _, err := readInput("ca-key", "-", &claims) + require.NoError(t, err) + + _, err = readInput("ca-crt", "-", &claims) + require.EqualError(t, err, `-ca-key and -ca-crt both set to "-", only one input may read from stdin`) +} + +func Test_openInput_stdin(t *testing.T) { + withStdin(t, bytes.NewBufferString("hi")) + var claims ioClaims + + r, err := openInput("ca", "-", &claims) + require.NoError(t, err) + defer r.Close() + b, err := io.ReadAll(r) + require.NoError(t, err) + assert.Equal(t, []byte("hi"), b) +} + +func Test_openInput_doubleStdinErrors(t *testing.T) { + withStdin(t, bytes.NewBufferString("hi")) + var claims ioClaims + + r, err := openInput("ca", "-", &claims) + require.NoError(t, err) + r.Close() + + _, err = openInput("crt", "-", &claims) + require.EqualError(t, err, `-ca and -crt both set to "-", only one input may read from stdin`) +} + +func Test_writeOutput_stdout(t *testing.T) { + out := &bytes.Buffer{} + + err := writeOutput("-", []byte("payload"), 0600, out) + require.NoError(t, err) + assert.Equal(t, "payload", out.String()) +} + +func Test_writeOutput_file(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "f") + out := &bytes.Buffer{} + + err := writeOutput(p, []byte("payload"), 0600, out) + require.NoError(t, err) + assert.Empty(t, out.String()) + got, err := os.ReadFile(p) + require.NoError(t, err) + assert.Equal(t, []byte("payload"), got) +} + +func Test_reserveOutputs_noConflict(t *testing.T) { + var claims ioClaims + require.NoError(t, reserveOutputs(&claims, + "out-key", "/tmp/key", + "out-crt", "-", + "out-qr", "", + )) + assert.Equal(t, "out-crt", claims.out) +} + +func Test_reserveOutputs_conflict(t *testing.T) { + var claims ioClaims + err := reserveOutputs(&claims, + "out-key", "-", + "out-crt", "-", + ) + require.EqualError(t, err, `-out-key and -out-crt both set to "-", only one output may write to stdout`) +} + +func Test_reserveOutputs_panicsOnOddPairs(t *testing.T) { + defer func() { + r := recover() + require.NotNil(t, r) + }() + var claims ioClaims + _ = reserveOutputs(&claims, "out-key") +} + +func Test_reserveInputs_noConflict(t *testing.T) { + var claims ioClaims + require.NoError(t, reserveInputs(&claims, + "ca-key", "/tmp/ca.key", + "ca-crt", "-", + "in-pub", "", + )) + assert.Equal(t, "ca-crt", claims.in) +} + +func Test_reserveInputs_conflict(t *testing.T) { + var claims ioClaims + err := reserveInputs(&claims, + "ca-key", "-", + "ca-crt", "-", + ) + require.EqualError(t, err, `-ca-key and -ca-crt both set to "-", only one input may read from stdin`) +} + +func Test_claimIn_idempotent(t *testing.T) { + // pre-claim then a lazy re-claim of the same flag should be a no-op + var claims ioClaims + require.NoError(t, claims.claimIn("ca-key")) + require.NoError(t, claims.claimIn("ca-key")) + assert.Equal(t, "ca-key", claims.in) +} + +func Test_claimOut_idempotent(t *testing.T) { + var claims ioClaims + require.NoError(t, claims.claimOut("out-crt")) + require.NoError(t, claims.claimOut("out-crt")) + assert.Equal(t, "out-crt", claims.out) +} + +func Test_isStdio(t *testing.T) { + assert.True(t, isStdio("-")) + assert.False(t, isStdio("")) + assert.False(t, isStdio("./-")) + assert.False(t, isStdio("foo")) +} diff --git a/cmd/nebula-cert/verify.go b/cmd/nebula-cert/verify.go index c9559136..76d3dbe6 100644 --- a/cmd/nebula-cert/verify.go +++ b/cmd/nebula-cert/verify.go @@ -1,11 +1,11 @@ package main import ( + "errors" "flag" "fmt" "io" "os" - "strings" "time" "github.com/slackhq/nebula/cert" @@ -39,39 +39,51 @@ func verify(args []string, out io.Writer, errOut io.Writer) error { return err } - rawCACert, err := os.ReadFile(*vf.caPath) - if err != nil { - return fmt.Errorf("error while reading ca: %s", err) + var claims ioClaims + if err := reserveInputs(&claims, + "ca", *vf.caPath, + "crt", *vf.certPath, + ); err != nil { + return err } - caPool := cert.NewCAPool() - for { - rawCACert, err = caPool.AddCACertificate(rawCACert) - if err != nil { - return fmt.Errorf("error while adding ca cert to pool: %s", err) - } - - if rawCACert == nil || len(rawCACert) == 0 || strings.TrimSpace(string(rawCACert)) == "" { - break - } + caReader, err := openInput("ca", *vf.caPath, &claims) + if err != nil { + return fmt.Errorf("error while reading ca: %w", err) } + defer caReader.Close() - rawCert, err := os.ReadFile(*vf.certPath) - if err != nil { - return fmt.Errorf("unable to read crt; %s", err) + caPool, err := cert.NewCAPoolFromPEMReader(caReader) + if err != nil && !errors.Is(err, cert.ErrExpired) { + return fmt.Errorf("error while adding ca cert to pool: %w", err) } - c, _, err := cert.UnmarshalNebulaCertificateFromPEM(rawCert) + rawCert, err := readInput("crt", *vf.certPath, &claims) if err != nil { - return fmt.Errorf("error while parsing crt: %s", err) + return fmt.Errorf("unable to read crt: %w", err) } - - good, err := c.Verify(time.Now(), caPool) - if !good { - return err + var errs []error + for { + if len(rawCert) == 0 { + break + } + c, extra, err := cert.UnmarshalCertificateFromPEM(rawCert) + if err != nil { + return fmt.Errorf("error while parsing crt: %w", err) + } + rawCert = extra + _, err = caPool.VerifyCertificate(time.Now(), c) + if err != nil { + switch { + case errors.Is(err, cert.ErrCaNotFound): + errs = append(errs, fmt.Errorf("error while verifying certificate v%d %s with issuer %s: %w", c.Version(), c.Name(), c.Issuer(), err)) + default: + errs = append(errs, fmt.Errorf("error while verifying certificate %+v: %w", c, err)) + } + } } - return nil + return errors.Join(errs...) } func verifySummary() string { @@ -80,7 +92,8 @@ func verifySummary() string { func verifyHelp(out io.Writer) { vf := newVerifyFlags() - out.Write([]byte("Usage of " + os.Args[0] + " " + verifySummary() + "\n")) + _, _ = out.Write([]byte("Usage of " + os.Args[0] + " " + verifySummary() + "\n")) + _, _ = out.Write([]byte(stdioHelpText)) vf.set.SetOutput(out) vf.set.PrintDefaults() } diff --git a/cmd/nebula-cert/verify_test.go b/cmd/nebula-cert/verify_test.go index f0f4c78d..aa089d0e 100644 --- a/cmd/nebula-cert/verify_test.go +++ b/cmd/nebula-cert/verify_test.go @@ -9,6 +9,7 @@ import ( "github.com/slackhq/nebula/cert" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/crypto/ed25519" ) @@ -22,6 +23,7 @@ func Test_verifyHelp(t *testing.T) { assert.Equal( t, "Usage of "+os.Args[0]+" verify : verifies a certificate isn't expired and was signed by a trusted authority.\n"+ + " Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+ " -ca string\n"+ " \tRequired: path to a file containing one or more ca certificates\n"+ " -crt string\n"+ @@ -37,105 +39,130 @@ func Test_verify(t *testing.T) { // required args assertHelpError(t, verify([]string{"-ca", "derp"}, ob, eb), "-crt is required") - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) assertHelpError(t, verify([]string{"-crt", "derp"}, ob, eb), "-ca is required") - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) // no ca at path ob.Reset() eb.Reset() err := verify([]string{"-ca", "does_not_exist", "-crt", "does_not_exist"}, ob, eb) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) - assert.EqualError(t, err, "error while reading ca: open does_not_exist: "+NoSuchFileError) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + require.EqualError(t, err, "error while reading ca: open does_not_exist: "+NoSuchFileError) // invalid ca at path ob.Reset() eb.Reset() caFile, err := os.CreateTemp("", "verify-ca") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(caFile.Name()) caFile.WriteString("-----BEGIN NOPE-----") err = verify([]string{"-ca", caFile.Name(), "-crt", "does_not_exist"}, ob, eb) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) - assert.EqualError(t, err, "error while adding ca cert to pool: input did not contain a valid PEM encoded block") + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + require.ErrorIs(t, err, cert.ErrInvalidPEMBlock) // make a ca for later caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader) - ca := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "test-ca", - NotBefore: time.Now().Add(time.Hour * -1), - NotAfter: time.Now().Add(time.Hour * 2), - PublicKey: caPub, - IsCA: true, - }, - } - ca.Sign(cert.Curve_CURVE25519, caPriv) - b, _ := ca.MarshalToPEM() + ca, _ := NewTestCaCert("test-ca", caPub, caPriv, time.Now().Add(time.Hour*-1), time.Now().Add(time.Hour*2), nil, nil, nil) + b, _ := ca.MarshalPEM() caFile.Truncate(0) caFile.Seek(0, 0) caFile.Write(b) // no crt at path err = verify([]string{"-ca", caFile.Name(), "-crt", "does_not_exist"}, ob, eb) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) - assert.EqualError(t, err, "unable to read crt; open does_not_exist: "+NoSuchFileError) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + require.EqualError(t, err, "unable to read crt: open does_not_exist: "+NoSuchFileError) // invalid crt at path ob.Reset() eb.Reset() certFile, err := os.CreateTemp("", "verify-cert") - assert.Nil(t, err) + require.NoError(t, err) defer os.Remove(certFile.Name()) certFile.WriteString("-----BEGIN NOPE-----") err = verify([]string{"-ca", caFile.Name(), "-crt", certFile.Name()}, ob, eb) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) - assert.EqualError(t, err, "error while parsing crt: input did not contain a valid PEM encoded block") + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + require.EqualError(t, err, "error while parsing crt: input did not contain a valid PEM encoded block") // unverifiable cert at path - _, badPriv, _ := ed25519.GenerateKey(rand.Reader) - certPub, _ := x25519Keypair() - signer, _ := ca.Sha256Sum() - crt := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "test-cert", - NotBefore: time.Now().Add(time.Hour * -1), - NotAfter: time.Now().Add(time.Hour), - PublicKey: certPub, - IsCA: false, - Issuer: signer, - }, + crt, _ := NewTestCert(ca, caPriv, "test-cert", time.Now().Add(time.Hour*-1), time.Now().Add(time.Hour), nil, nil, nil) + // Slightly evil hack to modify the certificate after it was sealed to generate an invalid signature + pub := crt.PublicKey() + for i, _ := range pub { + pub[i] = 0 } - - crt.Sign(cert.Curve_CURVE25519, badPriv) - b, _ = crt.MarshalToPEM() + b, _ = crt.MarshalPEM() certFile.Truncate(0) certFile.Seek(0, 0) certFile.Write(b) err = verify([]string{"-ca", caFile.Name(), "-crt", certFile.Name()}, ob, eb) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) - assert.EqualError(t, err, "certificate signature did not match") + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + require.ErrorIs(t, err, cert.ErrSignatureMismatch) // verified cert at path - crt.Sign(cert.Curve_CURVE25519, caPriv) - b, _ = crt.MarshalToPEM() + crt, _ = NewTestCert(ca, caPriv, "test-cert", time.Now().Add(time.Hour*-1), time.Now().Add(time.Hour), nil, nil, nil) + b, _ = crt.MarshalPEM() certFile.Truncate(0) certFile.Seek(0, 0) certFile.Write(b) err = verify([]string{"-ca", caFile.Name(), "-crt", certFile.Name()}, ob, eb) - assert.Equal(t, "", ob.String()) - assert.Equal(t, "", eb.String()) - assert.Nil(t, err) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + require.NoError(t, err) +} + +func Test_verify_stdio(t *testing.T) { + ob := &bytes.Buffer{} + eb := &bytes.Buffer{} + + caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader) + ca, _ := NewTestCaCert("test-ca", caPub, caPriv, time.Now().Add(time.Hour*-1), time.Now().Add(time.Hour*2), nil, nil, nil) + caPEM, _ := ca.MarshalPEM() + + crt, _ := NewTestCert(ca, caPriv, "test-cert", time.Now().Add(time.Hour*-1), time.Now().Add(time.Hour), nil, nil, nil) + crtPEM, _ := crt.MarshalPEM() + + caFile, err := os.CreateTemp("", "verify-ca") + require.NoError(t, err) + defer os.Remove(caFile.Name()) + caFile.Write(caPEM) + + // crt on stdin, ca on disk + withStdin(t, bytes.NewReader(crtPEM)) + require.NoError(t, verify([]string{"-ca", caFile.Name(), "-crt", "-"}, ob, eb)) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + + // ca on stdin, crt on disk + certFile, err := os.CreateTemp("", "verify-cert") + require.NoError(t, err) + defer os.Remove(certFile.Name()) + certFile.Write(crtPEM) + + withStdin(t, bytes.NewReader(caPEM)) + ob.Reset() + eb.Reset() + require.NoError(t, verify([]string{"-ca", "-", "-crt", certFile.Name()}, ob, eb)) + assert.Empty(t, ob.String()) + assert.Empty(t, eb.String()) + + // both flags on stdin should error + withStdin(t, bytes.NewReader(caPEM)) + ob.Reset() + eb.Reset() + require.EqualError(t, verify([]string{"-ca", "-", "-crt", "-"}, ob, eb), + `-ca and -crt both set to "-", only one input may read from stdin`) } diff --git a/cmd/nebula-service/logs_generic.go b/cmd/nebula-service/logs_generic.go index 3b7cdd1c..cc06b4c5 100644 --- a/cmd/nebula-service/logs_generic.go +++ b/cmd/nebula-service/logs_generic.go @@ -3,8 +3,15 @@ package main -import "github.com/sirupsen/logrus" +import ( + "log/slog" + "os" -func HookLogger(l *logrus.Logger) { - // Do nothing, let the logs flow to stdout/stderr + "github.com/slackhq/nebula/logging" +) + +// newPlatformLogger returns a *slog.Logger that writes to stdout. Non-Windows +// platforms have no special sink to integrate with. +func newPlatformLogger() *slog.Logger { + return logging.NewLogger(os.Stdout) } diff --git a/cmd/nebula-service/logs_windows.go b/cmd/nebula-service/logs_windows.go index af6480ef..ca0a55c5 100644 --- a/cmd/nebula-service/logs_windows.go +++ b/cmd/nebula-service/logs_windows.go @@ -1,54 +1,86 @@ package main import ( - "fmt" - "io/ioutil" - "os" + "context" + "log/slog" + "strings" + "sync" - "github.com/kardianos/service" - "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/logging" ) -// HookLogger routes the logrus logs through the service logger so that they end up in the Windows Event Viewer -// logrus output will be discarded -func HookLogger(l *logrus.Logger) { - l.AddHook(newLogHook(logger)) - l.SetOutput(ioutil.Discard) +// newPlatformLogger returns a *slog.Logger that routes every log record +// through the Windows service logger so records end up in the Windows +// Event Log. All the heavy lifting (level management, format swap, +// timestamp toggle, WithAttrs/WithGroup) comes from logging.NewHandler; +// this file only contributes: +// +// - an io.Writer that forwards each formatted line to the service +// logger at the current record's Event Log severity, and +// - a thin severityTag that embeds *logging.Handler and overrides +// only Handle / WithAttrs / WithGroup, so Event Viewer's severity +// column and severity-based filters keep working the way they did +// before the slog migration. +// +// Format (text vs json) is carried by the embedded *logging.Handler, so +// logging.format: json in config still produces JSON lines in Event +// Viewer, same as the pre-slog logrus setup. +func newPlatformLogger() *slog.Logger { + w := &eventLogWriter{} + return slog.New(&severityTag{Handler: logging.NewHandler(w), w: w}) } -type logHook struct { - sl service.Logger +// eventLogWriter forwards slog-formatted lines to the Windows service +// logger at the severity most recently stashed by severityTag.Handle. +// The mutex serializes the stash + inner.Handle + Write cycle per record +// across all concurrent goroutines; slog's builtin text/json handlers +// each hold their own mutex around Write, but that only protects the +// Write call itself, not our stash-then-handle sequence. +type eventLogWriter struct { + mu sync.Mutex + level slog.Level } -func newLogHook(sl service.Logger) *logHook { - return &logHook{sl: sl} +func (w *eventLogWriter) Write(p []byte) (int, error) { + line := strings.TrimRight(string(p), "\n") + switch { + case w.level >= slog.LevelError: + return len(p), logger.Error(line) + case w.level >= slog.LevelWarn: + return len(p), logger.Warning(line) + default: + return len(p), logger.Info(line) + } } -func (h *logHook) Fire(entry *logrus.Entry) error { - line, err := entry.String() - if err != nil { - fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err) - return err - } +// severityTag embeds *logging.Handler to pick up everything it does for +// free (Enabled, SetLevel, GetLevel, SetFormat, GetFormat, +// SetDisableTimestamp) and overrides only Handle / WithAttrs / WithGroup +// so each record's slog.Level is stashed on the writer before formatting +// and so derived handlers stay wrapped as severityTag rather than +// downgrading to bare *logging.Handler. +type severityTag struct { + *logging.Handler + w *eventLogWriter +} - switch entry.Level { - case logrus.PanicLevel: - return h.sl.Error(line) - case logrus.FatalLevel: - return h.sl.Error(line) - case logrus.ErrorLevel: - return h.sl.Error(line) - case logrus.WarnLevel: - return h.sl.Warning(line) - case logrus.InfoLevel: - return h.sl.Info(line) - case logrus.DebugLevel: - return h.sl.Info(line) - default: - return nil +func (s *severityTag) Handle(ctx context.Context, r slog.Record) error { + s.w.mu.Lock() + defer s.w.mu.Unlock() + s.w.level = r.Level + return s.Handler.Handle(ctx, r) +} + +func (s *severityTag) WithAttrs(attrs []slog.Attr) slog.Handler { + if len(attrs) == 0 { + return s } + return &severityTag{Handler: s.Handler.WithAttrs(attrs).(*logging.Handler), w: s.w} } -func (h *logHook) Levels() []logrus.Level { - return logrus.AllLevels +func (s *severityTag) WithGroup(name string) slog.Handler { + if name == "" { + return s + } + return &severityTag{Handler: s.Handler.WithGroup(name).(*logging.Handler), w: s.w} } diff --git a/cmd/nebula-service/main.go b/cmd/nebula-service/main.go index 8d0eaa1d..e0b335f5 100644 --- a/cmd/nebula-service/main.go +++ b/cmd/nebula-service/main.go @@ -4,10 +4,12 @@ import ( "flag" "fmt" "os" + "runtime/debug" + "strings" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/util" ) @@ -18,6 +20,17 @@ import ( // at compile-time. var Build string +func init() { + if Build == "" { + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + + Build = strings.TrimPrefix(info.Main.Version, "v") + } +} + func main() { serviceFlag := flag.String("service", "", "Control the system service.") configPath := flag.String("config", "", "Path to either a file or directory to load configuration from") @@ -37,20 +50,30 @@ func main() { os.Exit(0) } + l := logging.NewLogger(os.Stdout) + if *serviceFlag != "" { - doService(configPath, configTest, Build, serviceFlag) - os.Exit(1) + if *configTest { + fmt.Println("-test is not supported with -service, run the config test without -service") + os.Exit(1) + } + + if err := doService(configPath, Build, serviceFlag); err != nil { + l.Error("Service command failed", "error", err) + os.Exit(1) + } + return } if *configPath == "" { - fmt.Println("-config flag must be set") - flag.Usage() - os.Exit(1) + p, err := config.DefaultPath() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + *configPath = p } - l := logrus.New() - l.Out = os.Stdout - c := config.NewC(l) err := c.Load(*configPath) if err != nil { @@ -58,6 +81,16 @@ func main() { os.Exit(1) } + if err := logging.ApplyConfig(l, c); err != nil { + fmt.Printf("failed to apply logging config: %s", err) + os.Exit(1) + } + c.RegisterReloadCallback(func(c *config.C) { + if err := logging.ApplyConfig(l, c); err != nil { + l.Error("Failed to reconfigure logger on reload", "error", err) + } + }) + ctrl, err := nebula.Main(c, *configTest, Build, l, nil) if err != nil { util.LogWithContextIfNeeded("Failed to start", err, l) @@ -65,8 +98,19 @@ func main() { } if !*configTest { - ctrl.Start() - ctrl.ShutdownBlock() + if err := ctrl.Start(); err != nil { + util.LogWithContextIfNeeded("Error while running", err, l) + os.Exit(1) + } + + go ctrl.ShutdownBlock() + + if err := ctrl.Wait(); err != nil { + l.Error("Nebula stopped due to fatal error", "error", err) + os.Exit(2) + } + + l.Info("Goodbye") } os.Exit(0) diff --git a/cmd/nebula-service/service.go b/cmd/nebula-service/service.go index a54fb0f3..abe9abe0 100644 --- a/cmd/nebula-service/service.go +++ b/cmd/nebula-service/service.go @@ -4,19 +4,17 @@ import ( "fmt" "log" "os" - "path/filepath" "github.com/kardianos/service" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/logging" ) var logger service.Logger type program struct { configPath *string - configTest *bool build string control *nebula.Control } @@ -25,8 +23,7 @@ func (p *program) Start(s service.Service) error { // Start should not block. logger.Info("Nebula service starting.") - l := logrus.New() - HookLogger(l) + l := newPlatformLogger() c := config.NewC(l) err := c.Load(*p.configPath) @@ -34,39 +31,56 @@ func (p *program) Start(s service.Service) error { return fmt.Errorf("failed to load config: %s", err) } - p.control, err = nebula.Main(c, *p.configTest, Build, l, nil) + if err := logging.ApplyConfig(l, c); err != nil { + return fmt.Errorf("failed to apply logging config: %s", err) + } + c.RegisterReloadCallback(func(c *config.C) { + if err := logging.ApplyConfig(l, c); err != nil { + l.Error("Failed to reconfigure logger on reload", "error", err) + } + }) + + p.control, err = nebula.Main(c, false, Build, l, nil) if err != nil { return err } - p.control.Start() + if err := p.control.Start(); err != nil { + return err + } + + // Nebula can stop itself on a fatal packet reader error, make sure to log it if it happens. + go func() { + if err := p.control.Wait(); err != nil { + logger.Error(fmt.Sprintf("Nebula stopped due to fatal error: %v", err)) + os.Exit(2) + } + }() + return nil } func (p *program) Stop(s service.Service) error { logger.Info("Nebula service stopping.") + if p.control == nil { + return nil + } + p.control.Stop() - return nil -} -func fileExists(filename string) bool { - _, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return true + // block until nebula has fully drained before reporting stopped. + // error logging is handled by Start. + _ = p.control.Wait() + return nil } -func doService(configPath *string, configTest *bool, build string, serviceFlag *string) { +func doService(configPath *string, build string, serviceFlag *string) error { if *configPath == "" { - ex, err := os.Executable() + p, err := config.DefaultPath() if err != nil { - panic(err) - } - *configPath = filepath.Dir(ex) + "/config.yaml" - if !fileExists(*configPath) { - *configPath = filepath.Dir(ex) + "/config.yml" + return err } + *configPath = p } svcConfig := &service.Config{ @@ -78,23 +92,22 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag * prg := &program{ configPath: configPath, - configTest: configTest, build: build, } // Here are what the different loggers are doing: // - `log` is the standard go log utility, meant to be used while the process is still attached to stdout/stderr // - `logger` is the service log utility that may be attached to a special place depending on OS (Windows will have it attached to the event log) - // - above, in `Run` we create a `logrus.Logger` which is what nebula expects to use + // - in program.Start we build a *slog.Logger via newPlatformLogger; on non-Windows that is a stdout-backed slog logger, on Windows it routes records through the service logger s, err := service.New(prg, svcConfig) if err != nil { - log.Fatal(err) + return err } errs := make(chan error, 5) logger, err = s.Logger(errs) if err != nil { - log.Fatal(err) + return err } go func() { @@ -109,18 +122,17 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag * switch *serviceFlag { case "run": - err = s.Run() - if err != nil { - // Route any errors to the system logger + if err := s.Run(); err != nil { + // Route any errors to the system logger and report the failure logger.Error(err) + return err } default: - err := service.Control(s, *serviceFlag) - if err != nil { + if err := service.Control(s, *serviceFlag); err != nil { log.Printf("Valid actions: %q\n", service.ControlAction) - log.Fatal(err) + return err } - return } + return nil } diff --git a/cmd/nebula/close_on_timer_test.go b/cmd/nebula/close_on_timer_test.go new file mode 100644 index 00000000..07138c15 --- /dev/null +++ b/cmd/nebula/close_on_timer_test.go @@ -0,0 +1,96 @@ +//go:build linux && !android && !e2e_testing + +package main + +import ( + "fmt" + "net/netip" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/slackhq/nebula" + "github.com/slackhq/nebula/cert" + cert_test "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/require" +) + +// TestControlStopClosesOnTimer reproduces the dnclient lifecycle: nebula runs as +// a library, and on a config update dnclient calls Stop() in-process to tear the +// old instance down before starting a new one. This boots a real nebula (real +// blocking UDP sockets, tun disabled), lets it run, then Stop()s it on a timer +// and asserts it actually closes. If the reader goroutines parked in recvmmsg +// don't wake on Close(), Wait() blocks forever and this fails with a goroutine +// dump instead of relying on a process signal to unstick them. +func TestControlStopClosesOnTimer(t *testing.T) { + l := test.NewLogger() + dir := t.TempDir() + + before := time.Now().Add(-time.Hour) + after := time.Now().Add(time.Hour) + ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil) + networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")} + _, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "close-on-timer", before, after, networks, nil, nil) + + caPath := filepath.Join(dir, "ca.pem") + certPath := filepath.Join(dir, "cert.pem") + keyPath := filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) + require.NoError(t, os.WriteFile(certPath, certPEM, 0o600)) + require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600)) + + // tun disabled so no device/root is needed; routines: 2 so we exercise the + // multi-socket (SO_REUSEPORT) teardown, which is where dnclient runs. + configBody := fmt.Sprintf(` +pki: + ca: %s + cert: %s + key: %s +listen: + host: 127.0.0.1 + port: 0 +tun: + disabled: true +firewall: + outbound: + - port: any + proto: any + host: any + inbound: + - port: any + proto: any + host: any +routines: 2 +`, caPath, certPath, keyPath) + require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600)) + + c := config.NewC(l) + require.NoError(t, c.Load(dir)) + + ctrl, err := nebula.Main(c, false, "close-on-timer", l, nil) + require.NoError(t, err) + require.NoError(t, ctrl.Start()) + + // Run like a live nebula, then close on a timer, exactly as dnclient does. + <-time.NewTimer(5 * time.Second).C + + stopped := make(chan struct{}) + go func() { + ctrl.Stop() // closes the udp sockets (shutdown(2)) and the tun + ctrl.Wait() // blocks until every reader goroutine has returned + close(stopped) + }() + + select { + case <-stopped: + t.Log("nebula closed cleanly on timer") + case <-time.After(10 * time.Second): + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + t.Fatalf("nebula did NOT close within 10s of Stop(): a blocking reader never woke\n%s", buf[:n]) + } +} diff --git a/cmd/nebula/main.go b/cmd/nebula/main.go index 5cf0a028..3c786b84 100644 --- a/cmd/nebula/main.go +++ b/cmd/nebula/main.go @@ -4,10 +4,12 @@ import ( "flag" "fmt" "os" + "runtime/debug" + "strings" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/util" ) @@ -18,6 +20,17 @@ import ( // at compile-time. var Build string +func init() { + if Build == "" { + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + + Build = strings.TrimPrefix(info.Main.Version, "v") + } +} + func main() { configPath := flag.String("config", "", "Path to either a file or directory to load configuration from") configTest := flag.Bool("test", false, "Test the config and print the end result. Non zero exit indicates a faulty config") @@ -37,13 +50,15 @@ func main() { } if *configPath == "" { - fmt.Println("-config flag must be set") - flag.Usage() - os.Exit(1) + p, err := config.DefaultPath() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + *configPath = p } - l := logrus.New() - l.Out = os.Stdout + l := logging.NewLogger(os.Stdout) c := config.NewC(l) err := c.Load(*configPath) @@ -52,6 +67,16 @@ func main() { os.Exit(1) } + if err := logging.ApplyConfig(l, c); err != nil { + fmt.Printf("failed to apply logging config: %s", err) + os.Exit(1) + } + c.RegisterReloadCallback(func(c *config.C) { + if err := logging.ApplyConfig(l, c); err != nil { + l.Error("Failed to reconfigure logger on reload", "error", err) + } + }) + ctrl, err := nebula.Main(c, *configTest, Build, l, nil) if err != nil { util.LogWithContextIfNeeded("Failed to start", err, l) @@ -59,9 +84,20 @@ func main() { } if !*configTest { - ctrl.Start() + if err := ctrl.Start(); err != nil { + util.LogWithContextIfNeeded("Error while running", err, l) + os.Exit(1) + } + + go ctrl.ShutdownBlock() notifyReady(l) - ctrl.ShutdownBlock() + + if err := ctrl.Wait(); err != nil { + l.Error("Nebula stopped due to fatal error", "error", err) + os.Exit(2) + } + + l.Info("Goodbye") } os.Exit(0) diff --git a/cmd/nebula/notify_linux.go b/cmd/nebula/notify_linux.go index 8c3dca55..965986a9 100644 --- a/cmd/nebula/notify_linux.go +++ b/cmd/nebula/notify_linux.go @@ -1,11 +1,10 @@ package main import ( + "log/slog" "net" "os" "time" - - "github.com/sirupsen/logrus" ) // SdNotifyReady tells systemd the service is ready and dependent services can now be started @@ -13,30 +12,30 @@ import ( // https://www.freedesktop.org/software/systemd/man/systemd.service.html const SdNotifyReady = "READY=1" -func notifyReady(l *logrus.Logger) { +func notifyReady(l *slog.Logger) { sockName := os.Getenv("NOTIFY_SOCKET") if sockName == "" { - l.Debugln("NOTIFY_SOCKET systemd env var not set, not sending ready signal") + l.Debug("NOTIFY_SOCKET systemd env var not set, not sending ready signal") return } conn, err := net.DialTimeout("unixgram", sockName, time.Second) if err != nil { - l.WithError(err).Error("failed to connect to systemd notification socket") + l.Error("failed to connect to systemd notification socket", "error", err) return } defer conn.Close() err = conn.SetWriteDeadline(time.Now().Add(time.Second)) if err != nil { - l.WithError(err).Error("failed to set the write deadline for the systemd notification socket") + l.Error("failed to set the write deadline for the systemd notification socket", "error", err) return } if _, err = conn.Write([]byte(SdNotifyReady)); err != nil { - l.WithError(err).Error("failed to signal the systemd notification socket") + l.Error("failed to signal the systemd notification socket", "error", err) return } - l.Debugln("notified systemd the service is ready") + l.Debug("notified systemd the service is ready") } diff --git a/cmd/nebula/notify_notlinux.go b/cmd/nebula/notify_notlinux.go index e7758e09..48cfe949 100644 --- a/cmd/nebula/notify_notlinux.go +++ b/cmd/nebula/notify_notlinux.go @@ -3,8 +3,8 @@ package main -import "github.com/sirupsen/logrus" +import "log/slog" -func notifyReady(_ *logrus.Logger) { +func notifyReady(_ *slog.Logger) { // No init service to notify } diff --git a/config/config.go b/config/config.go index 1aea8327..5bf994a1 100644 --- a/config/config.go +++ b/config/config.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log/slog" "math" "os" "os/signal" @@ -16,23 +17,22 @@ import ( "time" "dario.cat/mergo" - "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v3" ) type C struct { path string files []string - Settings map[interface{}]interface{} - oldSettings map[interface{}]interface{} + Settings map[string]any + oldSettings map[string]any callbacks []func(*C) - l *logrus.Logger + l *slog.Logger reloadLock sync.Mutex } -func NewC(l *logrus.Logger) *C { +func NewC(l *slog.Logger) *C { return &C{ - Settings: make(map[interface{}]interface{}), + Settings: make(map[string]any), l: l, } } @@ -92,8 +92,8 @@ func (c *C) HasChanged(k string) bool { } var ( - nv interface{} - ov interface{} + nv any + ov any ) if k == "" { @@ -107,12 +107,18 @@ func (c *C) HasChanged(k string) bool { newVals, err := yaml.Marshal(nv) if err != nil { - c.l.WithField("config_path", k).WithError(err).Error("Error while marshaling new config") + c.l.Error("Error while marshaling new config", + "config_path", k, + "error", err, + ) } oldVals, err := yaml.Marshal(ov) if err != nil { - c.l.WithField("config_path", k).WithError(err).Error("Error while marshaling old config") + c.l.Error("Error while marshaling old config", + "config_path", k, + "error", err, + ) } return string(newVals) != string(oldVals) @@ -147,14 +153,17 @@ func (c *C) ReloadConfig() { c.reloadLock.Lock() defer c.reloadLock.Unlock() - c.oldSettings = make(map[interface{}]interface{}) + c.oldSettings = make(map[string]any) for k, v := range c.Settings { c.oldSettings[k] = v } err := c.Load(c.path) if err != nil { - c.l.WithField("config_path", c.path).WithError(err).Error("Error occurred while reloading config") + c.l.Error("Error occurred while reloading config", + "config_path", c.path, + "error", err, + ) return } @@ -167,7 +176,7 @@ func (c *C) ReloadConfigString(raw string) error { c.reloadLock.Lock() defer c.reloadLock.Unlock() - c.oldSettings = make(map[interface{}]interface{}) + c.oldSettings = make(map[string]any) for k, v := range c.Settings { c.oldSettings[k] = v } @@ -201,7 +210,7 @@ func (c *C) GetStringSlice(k string, d []string) []string { return d } - rv, ok := r.([]interface{}) + rv, ok := r.([]any) if !ok { return d } @@ -215,13 +224,13 @@ func (c *C) GetStringSlice(k string, d []string) []string { } // GetMap will get the map for k or return the default d if not found or invalid -func (c *C) GetMap(k string, d map[interface{}]interface{}) map[interface{}]interface{} { +func (c *C) GetMap(k string, d map[string]any) map[string]any { r := c.Get(k) if r == nil { return d } - v, ok := r.(map[interface{}]interface{}) + v, ok := r.(map[string]any) if !ok { return d } @@ -243,7 +252,7 @@ func (c *C) GetInt(k string, d int) int { // GetUint32 will get the uint32 for k or return the default d if not found or invalid func (c *C) GetUint32(k string, d uint32) uint32 { r := c.GetInt(k, int(d)) - if uint64(r) > uint64(math.MaxUint32) { + if r < 0 || uint64(r) > uint64(math.MaxUint32) { return d } return uint32(r) @@ -266,6 +275,22 @@ func (c *C) GetBool(k string, d bool) bool { return v } +func AsBool(v any) (value bool, ok bool) { + switch x := v.(type) { + case bool: + return x, true + case string: + switch x { + case "y", "yes": + return true, true + case "n", "no": + return false, true + } + } + + return false, false +} + // GetDuration will get the duration for k or return the default d if not found or invalid func (c *C) GetDuration(k string, d time.Duration) time.Duration { r := c.GetString(k, "") @@ -276,7 +301,7 @@ func (c *C) GetDuration(k string, d time.Duration) time.Duration { return v } -func (c *C) Get(k string) interface{} { +func (c *C) Get(k string) any { return c.get(k, c.Settings) } @@ -284,10 +309,10 @@ func (c *C) IsSet(k string) bool { return c.get(k, c.Settings) != nil } -func (c *C) get(k string, v interface{}) interface{} { +func (c *C) get(k string, v any) any { parts := strings.Split(k, ".") for _, p := range parts { - m, ok := v.(map[interface{}]interface{}) + m, ok := v.(map[string]any) if !ok { return nil } @@ -346,7 +371,7 @@ func (c *C) addFile(path string, direct bool) error { } func (c *C) parseRaw(b []byte) error { - var m map[interface{}]interface{} + var m map[string]any err := yaml.Unmarshal(b, &m) if err != nil { @@ -358,7 +383,7 @@ func (c *C) parseRaw(b []byte) error { } func (c *C) parse() error { - var m map[interface{}]interface{} + var m map[string]any for _, path := range c.files { b, err := os.ReadFile(path) @@ -366,7 +391,7 @@ func (c *C) parse() error { return err } - var nm map[interface{}]interface{} + var nm map[string]any err = yaml.Unmarshal(b, &nm) if err != nil { return err diff --git a/config/config_test.go b/config/config_test.go index fa943930..aba7b2a8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -10,7 +10,7 @@ import ( "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v3" ) func TestConfig_Load(t *testing.T) { @@ -19,40 +19,37 @@ func TestConfig_Load(t *testing.T) { // invalid yaml c := NewC(l) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644) - assert.EqualError(t, c.Load(dir), "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[interface {}]interface {}") + require.EqualError(t, c.Load(dir), "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}") // simple multi config merge c = NewC(l) os.RemoveAll(dir) os.Mkdir(dir, 0755) - assert.Nil(t, err) + require.NoError(t, err) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n inner: hi"), 0644) os.WriteFile(filepath.Join(dir, "02.yml"), []byte("outer:\n inner: override\nnew: hi"), 0644) - assert.Nil(t, c.Load(dir)) - expected := map[interface{}]interface{}{ - "outer": map[interface{}]interface{}{ + require.NoError(t, c.Load(dir)) + expected := map[string]any{ + "outer": map[string]any{ "inner": "override", }, "new": "hi", } assert.Equal(t, expected, c.Settings) - - //TODO: test symlinked file - //TODO: test symlinked directory } func TestConfig_Get(t *testing.T) { l := test.NewLogger() // test simple type c := NewC(l) - c.Settings["firewall"] = map[interface{}]interface{}{"outbound": "hi"} + c.Settings["firewall"] = map[string]any{"outbound": "hi"} assert.Equal(t, "hi", c.Get("firewall.outbound")) // test complex type - inner := []map[interface{}]interface{}{{"port": "1", "code": "2"}} - c.Settings["firewall"] = map[interface{}]interface{}{"outbound": inner} + inner := []map[string]any{{"port": "1", "code": "2"}} + c.Settings["firewall"] = map[string]any{"outbound": inner} assert.EqualValues(t, inner, c.Get("firewall.outbound")) // test missing @@ -62,7 +59,7 @@ func TestConfig_Get(t *testing.T) { func TestConfig_GetStringSlice(t *testing.T) { l := test.NewLogger() c := NewC(l) - c.Settings["slice"] = []interface{}{"one", "two"} + c.Settings["slice"] = []any{"one", "two"} assert.Equal(t, []string{"one", "two"}, c.GetStringSlice("slice", []string{})) } @@ -70,28 +67,28 @@ func TestConfig_GetBool(t *testing.T) { l := test.NewLogger() c := NewC(l) c.Settings["bool"] = true - assert.Equal(t, true, c.GetBool("bool", false)) + assert.True(t, c.GetBool("bool", false)) c.Settings["bool"] = "true" - assert.Equal(t, true, c.GetBool("bool", false)) + assert.True(t, c.GetBool("bool", false)) c.Settings["bool"] = false - assert.Equal(t, false, c.GetBool("bool", true)) + assert.False(t, c.GetBool("bool", true)) c.Settings["bool"] = "false" - assert.Equal(t, false, c.GetBool("bool", true)) + assert.False(t, c.GetBool("bool", true)) c.Settings["bool"] = "Y" - assert.Equal(t, true, c.GetBool("bool", false)) + assert.True(t, c.GetBool("bool", false)) c.Settings["bool"] = "yEs" - assert.Equal(t, true, c.GetBool("bool", false)) + assert.True(t, c.GetBool("bool", false)) c.Settings["bool"] = "N" - assert.Equal(t, false, c.GetBool("bool", true)) + assert.False(t, c.GetBool("bool", true)) c.Settings["bool"] = "nO" - assert.Equal(t, false, c.GetBool("bool", true)) + assert.False(t, c.GetBool("bool", true)) } func TestConfig_HasChanged(t *testing.T) { @@ -104,14 +101,14 @@ func TestConfig_HasChanged(t *testing.T) { // Test key change c = NewC(l) c.Settings["test"] = "hi" - c.oldSettings = map[interface{}]interface{}{"test": "no"} + c.oldSettings = map[string]any{"test": "no"} assert.True(t, c.HasChanged("test")) assert.True(t, c.HasChanged("")) // No key change c = NewC(l) c.Settings["test"] = "hi" - c.oldSettings = map[interface{}]interface{}{"test": "hi"} + c.oldSettings = map[string]any{"test": "hi"} assert.False(t, c.HasChanged("test")) assert.False(t, c.HasChanged("")) } @@ -120,11 +117,11 @@ func TestConfig_ReloadConfig(t *testing.T) { l := test.NewLogger() done := make(chan bool, 1) dir, err := os.MkdirTemp("", "config-test") - assert.Nil(t, err) + require.NoError(t, err) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n inner: hi"), 0644) c := NewC(l) - assert.Nil(t, c.Load(dir)) + require.NoError(t, c.Load(dir)) assert.False(t, c.HasChanged("outer.inner")) assert.False(t, c.HasChanged("outer")) @@ -187,11 +184,11 @@ firewall: `), } - var m map[any]any + var m map[string]any // merge the same way config.parse() merges for _, b := range configs { - var nm map[any]any + var nm map[string]any err := yaml.Unmarshal(b, &nm) require.NoError(t, err) @@ -208,15 +205,15 @@ firewall: t.Logf("Merged Config as YAML:\n%s", mYaml) // If a bug is present, some items might be replaced instead of merged like we expect - expected := map[any]any{ - "firewall": map[any]any{ + expected := map[string]any{ + "firewall": map[string]any{ "inbound": []any{ - map[any]any{"host": "any", "port": "any", "proto": "icmp"}, - map[any]any{"groups": []any{"server"}, "port": 443, "proto": "tcp"}, - map[any]any{"groups": []any{"webapp"}, "port": 443, "proto": "tcp"}}, + map[string]any{"host": "any", "port": "any", "proto": "icmp"}, + map[string]any{"groups": []any{"server"}, "port": 443, "proto": "tcp"}, + map[string]any{"groups": []any{"webapp"}, "port": 443, "proto": "tcp"}}, "outbound": []any{ - map[any]any{"host": "any", "port": "any", "proto": "any"}}}, - "listen": map[any]any{ + map[string]any{"host": "any", "port": "any", "proto": "any"}}}, + "listen": map[string]any{ "host": "0.0.0.0", "port": 4242, }, diff --git a/config/default.go b/config/default.go new file mode 100644 index 00000000..9494c655 --- /dev/null +++ b/config/default.go @@ -0,0 +1,29 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" +) + +// DefaultPath returns a path to a config file alongside the running executable, preferring config.yaml over config.yml. +// If neither file exists an error is returned that names both paths checked. +func DefaultPath() (string, error) { + ex, err := os.Executable() + if err != nil { + return "", err + } + return defaultPathInDir(filepath.Dir(ex)) +} + +func defaultPathInDir(dir string) (string, error) { + yamlPath := filepath.Join(dir, "config.yaml") + if _, err := os.Stat(yamlPath); err == nil { + return yamlPath, nil + } + ymlPath := filepath.Join(dir, "config.yml") + if _, err := os.Stat(ymlPath); err == nil { + return ymlPath, nil + } + return "", fmt.Errorf("no default config found at %s or %s", yamlPath, ymlPath) +} diff --git a/config/default_test.go b/config/default_test.go new file mode 100644 index 00000000..a4d56f59 --- /dev/null +++ b/config/default_test.go @@ -0,0 +1,67 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDefaultPathInDir(t *testing.T) { + t.Run("prefers config.yaml when both exist", func(t *testing.T) { + dir := t.TempDir() + want := filepath.Join(dir, "config.yaml") + other := filepath.Join(dir, "config.yml") + require.NoError(t, os.WriteFile(want, []byte("a: 1"), 0644)) + require.NoError(t, os.WriteFile(other, []byte("a: 2"), 0644)) + + got, err := defaultPathInDir(dir) + require.NoError(t, err) + assert.Equal(t, want, got) + }) + + t.Run("returns config.yaml when only it exists", func(t *testing.T) { + dir := t.TempDir() + want := filepath.Join(dir, "config.yaml") + require.NoError(t, os.WriteFile(want, []byte("a: 1"), 0644)) + + got, err := defaultPathInDir(dir) + require.NoError(t, err) + assert.Equal(t, want, got) + }) + + t.Run("falls back to config.yml when only it exists", func(t *testing.T) { + dir := t.TempDir() + want := filepath.Join(dir, "config.yml") + require.NoError(t, os.WriteFile(want, []byte("a: 1"), 0644)) + + got, err := defaultPathInDir(dir) + require.NoError(t, err) + assert.Equal(t, want, got) + }) + + t.Run("errors when neither exists and names both paths", func(t *testing.T) { + dir := t.TempDir() + got, err := defaultPathInDir(dir) + assert.Empty(t, got) + require.Error(t, err) + assert.Contains(t, err.Error(), filepath.Join(dir, "config.yaml")) + assert.Contains(t, err.Error(), filepath.Join(dir, "config.yml")) + }) +} + +func TestDefaultPath(t *testing.T) { + got, err := DefaultPath() + if err != nil { + ex, exErr := os.Executable() + require.NoError(t, exErr) + assert.Contains(t, err.Error(), filepath.Dir(ex)) + return + } + ex, err := os.Executable() + require.NoError(t, err) + assert.Equal(t, filepath.Dir(ex), filepath.Dir(got)) + assert.Contains(t, []string{"config.yaml", "config.yml"}, filepath.Base(got)) +} diff --git a/connection_manager.go b/connection_manager.go index f5dd5942..88f31321 100644 --- a/connection_manager.go +++ b/connection_manager.go @@ -3,15 +3,17 @@ package nebula import ( "bytes" "context" + "encoding/binary" + "fmt" + "log/slog" + "net/netip" "sync" + "sync/atomic" "time" - "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" - "github.com/slackhq/nebula/udp" ) type trafficDecision int @@ -27,130 +29,115 @@ const ( ) type connectionManager struct { - in map[uint32]struct{} - inLock *sync.RWMutex - - out map[uint32]struct{} - outLock *sync.RWMutex - // relayUsed holds which relay localIndexs are in use relayUsed map[uint32]struct{} relayUsedLock *sync.RWMutex - hostMap *HostMap - trafficTimer *LockingTimerWheel[uint32] - intf *Interface - pendingDeletion map[uint32]struct{} - punchy *Punchy + hostMap *HostMap + trafficTimer *LockingTimerWheel[uint32] + intf *Interface + punchy *Punchy + + // Configuration settings checkInterval time.Duration pendingDeletionInterval time.Duration - metricsTxPunchy metrics.Counter + inactivityTimeout atomic.Int64 + dropInactive atomic.Bool - l *logrus.Logger + l *slog.Logger } -func newConnectionManager(ctx context.Context, l *logrus.Logger, intf *Interface, checkInterval, pendingDeletionInterval time.Duration, punchy *Punchy) *connectionManager { - var max time.Duration - if checkInterval < pendingDeletionInterval { - max = pendingDeletionInterval - } else { - max = checkInterval +func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p *Punchy) *connectionManager { + cm := &connectionManager{ + hostMap: hm, + l: l, + punchy: p, + relayUsed: make(map[uint32]struct{}), + relayUsedLock: &sync.RWMutex{}, } - nc := &connectionManager{ - hostMap: intf.hostMap, - in: make(map[uint32]struct{}), - inLock: &sync.RWMutex{}, - out: make(map[uint32]struct{}), - outLock: &sync.RWMutex{}, - relayUsed: make(map[uint32]struct{}), - relayUsedLock: &sync.RWMutex{}, - trafficTimer: NewLockingTimerWheel[uint32](time.Millisecond*500, max), - intf: intf, - pendingDeletion: make(map[uint32]struct{}), - checkInterval: checkInterval, - pendingDeletionInterval: pendingDeletionInterval, - punchy: punchy, - metricsTxPunchy: metrics.GetOrRegisterCounter("messages.tx.punchy", nil), - l: l, - } + cm.reload(c, true) + c.RegisterReloadCallback(func(c *config.C) { + cm.reload(c, false) + }) - nc.Start(ctx) - return nc + return cm } -func (n *connectionManager) In(localIndex uint32) { - n.inLock.RLock() - // If this already exists, return - if _, ok := n.in[localIndex]; ok { - n.inLock.RUnlock() - return +func (cm *connectionManager) reload(c *config.C, initial bool) { + if initial { + cm.checkInterval = time.Duration(c.GetInt("timers.connection_alive_interval", 5)) * time.Second + cm.pendingDeletionInterval = time.Duration(c.GetInt("timers.pending_deletion_interval", 10)) * time.Second + + // We want at least a minimum resolution of 500ms per tick so that we can hit these intervals + // pretty close to their configured duration. + // The inactivity duration is checked each time a hostinfo ticks through so we don't need the wheel to contain it. + minDuration := min(time.Millisecond*500, cm.checkInterval, cm.pendingDeletionInterval) + maxDuration := max(cm.checkInterval, cm.pendingDeletionInterval) + cm.trafficTimer = NewLockingTimerWheel[uint32](minDuration, maxDuration) } - n.inLock.RUnlock() - n.inLock.Lock() - n.in[localIndex] = struct{}{} - n.inLock.Unlock() -} -func (n *connectionManager) Out(localIndex uint32) { - n.outLock.RLock() - // If this already exists, return - if _, ok := n.out[localIndex]; ok { - n.outLock.RUnlock() - return + if initial || c.HasChanged("tunnels.inactivity_timeout") { + old := cm.getInactivityTimeout() + cm.inactivityTimeout.Store((int64)(c.GetDuration("tunnels.inactivity_timeout", 10*time.Minute))) + if !initial { + cm.l.Info("Inactivity timeout has changed", + "oldDuration", old, + "newDuration", cm.getInactivityTimeout(), + ) + } } - n.outLock.RUnlock() - n.outLock.Lock() - n.out[localIndex] = struct{}{} - n.outLock.Unlock() -} -func (n *connectionManager) RelayUsed(localIndex uint32) { - n.relayUsedLock.RLock() - // If this already exists, return - if _, ok := n.relayUsed[localIndex]; ok { - n.relayUsedLock.RUnlock() - return + if initial || c.HasChanged("tunnels.drop_inactive") { + old := cm.dropInactive.Load() + cm.dropInactive.Store(c.GetBool("tunnels.drop_inactive", false)) + if !initial { + cm.l.Info("Drop inactive setting has changed", + "oldBool", old, + "newBool", cm.dropInactive.Load(), + ) + } } - n.relayUsedLock.RUnlock() - n.relayUsedLock.Lock() - n.relayUsed[localIndex] = struct{}{} - n.relayUsedLock.Unlock() } -// getAndResetTrafficCheck returns if there was any inbound or outbound traffic within the last tick and -// resets the state for this local index -func (n *connectionManager) getAndResetTrafficCheck(localIndex uint32) (bool, bool) { - n.inLock.Lock() - n.outLock.Lock() - _, in := n.in[localIndex] - _, out := n.out[localIndex] - delete(n.in, localIndex) - delete(n.out, localIndex) - n.inLock.Unlock() - n.outLock.Unlock() - return in, out +func (cm *connectionManager) getInactivityTimeout() time.Duration { + return (time.Duration)(cm.inactivityTimeout.Load()) } -func (n *connectionManager) AddTrafficWatch(localIndex uint32) { - // Use a write lock directly because it should be incredibly rare that we are ever already tracking this index - n.outLock.Lock() - if _, ok := n.out[localIndex]; ok { - n.outLock.Unlock() +func (cm *connectionManager) In(h *HostInfo) { + h.in.Store(true) +} + +func (cm *connectionManager) Out(h *HostInfo) { + h.out.Store(true) +} + +func (cm *connectionManager) RelayUsed(localIndex uint32) { + cm.relayUsedLock.RLock() + // If this already exists, return + if _, ok := cm.relayUsed[localIndex]; ok { + cm.relayUsedLock.RUnlock() return } - n.out[localIndex] = struct{}{} - n.trafficTimer.Add(localIndex, n.checkInterval) - n.outLock.Unlock() + cm.relayUsedLock.RUnlock() + cm.relayUsedLock.Lock() + cm.relayUsed[localIndex] = struct{}{} + cm.relayUsedLock.Unlock() } -func (n *connectionManager) Start(ctx context.Context) { - go n.Run(ctx) +// getAndResetTrafficCheck returns if there was any inbound or outbound traffic within the last tick and +// resets the state for this local index +func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time) (bool, bool) { + in := h.in.Swap(false) + out := h.out.Swap(false) + if in || out { + h.lastUsed = now + } + return in, out } -func (n *connectionManager) Run(ctx context.Context) { - //TODO: this tick should be based on the min wheel tick? Check firewall - clockSource := time.NewTicker(500 * time.Millisecond) +func (cm *connectionManager) Start(ctx context.Context) { + clockSource := time.NewTicker(cm.trafficTimer.t.tickDuration) defer clockSource.Stop() p := []byte("") @@ -163,110 +150,115 @@ func (n *connectionManager) Run(ctx context.Context) { return case now := <-clockSource.C: - n.trafficTimer.Advance(now) + cm.trafficTimer.Advance(now) for { - localIndex, has := n.trafficTimer.Purge() + localIndex, has := cm.trafficTimer.Purge() if !has { break } - n.doTrafficCheck(localIndex, p, nb, out, now) + cm.doTrafficCheck(localIndex, p, nb, out, now) } } } } -func (n *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) { - decision, hostinfo, primary := n.makeTrafficDecision(localIndex, now) +func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) { + decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now) switch decision { case deleteTunnel: - if n.hostMap.DeleteHostInfo(hostinfo) { + if cm.hostMap.DeleteHostInfo(hostinfo) { // Only clearing the lighthouse cache if this is the last hostinfo for this vpn ip in the hostmap - n.intf.lightHouse.DeleteVpnIp(hostinfo.vpnIp) + cm.intf.lightHouse.DeleteVpnAddrs(hostinfo.vpnAddrs) } case closeTunnel: - n.intf.sendCloseTunnel(hostinfo) - n.intf.closeTunnel(hostinfo) + cm.intf.sendCloseTunnel(hostinfo) + cm.intf.closeTunnel(hostinfo) case swapPrimary: - n.swapPrimary(hostinfo, primary) + cm.swapPrimary(hostinfo, primary) case migrateRelays: - n.migrateRelayUsed(hostinfo, primary) + cm.migrateRelayUsed(hostinfo, primary) case tryRehandshake: - n.tryRehandshake(hostinfo) + cm.tryRehandshake(hostinfo) case sendTestPacket: - n.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, nb, out) + cm.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, nb, out) } - n.resetRelayTrafficCheck(hostinfo) + cm.resetRelayTrafficCheck(hostinfo) } -func (n *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) { +func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) { if hostinfo != nil { - n.relayUsedLock.Lock() - defer n.relayUsedLock.Unlock() + cm.relayUsedLock.Lock() + defer cm.relayUsedLock.Unlock() // No need to migrate any relays, delete usage info now. for _, idx := range hostinfo.relayState.CopyRelayForIdxs() { - delete(n.relayUsed, idx) + delete(cm.relayUsed, idx) } } } -func (n *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo) { +func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo) { relayFor := oldhostinfo.relayState.CopyAllRelayFor() for _, r := range relayFor { - existing, ok := newhostinfo.relayState.QueryRelayForByIp(r.PeerIp) + existing, ok := newhostinfo.relayState.QueryRelayForByIp(r.PeerAddr) var index uint32 - var relayFrom iputil.VpnIp - var relayTo iputil.VpnIp + var relayFrom netip.Addr + var relayTo netip.Addr switch { - case ok && existing.State == Established: - // This relay already exists in newhostinfo, then do nothing. - continue - case ok && existing.State == Requested: - // The relay exists in a Requested state; re-send the request - index = existing.LocalIndex - switch r.Type { - case TerminalType: - relayFrom = n.intf.myVpnIp - relayTo = existing.PeerIp - case ForwardingType: - relayFrom = existing.PeerIp - relayTo = newhostinfo.vpnIp - default: - // should never happen + case ok: + switch existing.State { + case Established, PeerRequested, Disestablished: + // This relay already exists in newhostinfo, then do nothing. + continue + case Requested: + // The relay exists in a Requested state; re-send the request + index = existing.LocalIndex + switch r.Type { + case TerminalType: + relayFrom = cm.intf.myVpnAddrs[0] + relayTo = existing.PeerAddr + case ForwardingType: + relayFrom = existing.PeerAddr + relayTo = newhostinfo.vpnAddrs[0] + default: + // should never happen + panic(fmt.Sprintf("Migrating unknown relay type: %v", r.Type)) + } } case !ok: - n.relayUsedLock.RLock() - if _, relayUsed := n.relayUsed[r.LocalIndex]; !relayUsed { + cm.relayUsedLock.RLock() + if _, relayUsed := cm.relayUsed[r.LocalIndex]; !relayUsed { // The relay hasn't been used; don't migrate it. - n.relayUsedLock.RUnlock() + cm.relayUsedLock.RUnlock() continue } - n.relayUsedLock.RUnlock() + cm.relayUsedLock.RUnlock() // The relay doesn't exist at all; create some relay state and send the request. var err error - index, err = AddRelay(n.l, newhostinfo, n.hostMap, r.PeerIp, nil, r.Type, Requested) + index, err = AddRelay(cm.l, newhostinfo, cm.hostMap, r.PeerAddr, nil, r.Type, Requested) if err != nil { - n.l.WithError(err).Error("failed to migrate relay to new hostinfo") + cm.l.Error("failed to migrate relay to new hostinfo", "error", err) continue } switch r.Type { case TerminalType: - relayFrom = n.intf.myVpnIp - relayTo = r.PeerIp + relayFrom = cm.intf.myVpnAddrs[0] + relayTo = r.PeerAddr case ForwardingType: - relayFrom = r.PeerIp - relayTo = newhostinfo.vpnIp + relayFrom = r.PeerAddr + relayTo = newhostinfo.vpnAddrs[0] default: // should never happen + panic(fmt.Sprintf("Migrating unknown relay type: %v", r.Type)) } } @@ -274,65 +266,86 @@ func (n *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo) req := NebulaControl{ Type: NebulaControl_CreateRelayRequest, InitiatorRelayIndex: index, - RelayFromIp: uint32(relayFrom), - RelayToIp: uint32(relayTo), } + + switch newhostinfo.GetCert().Certificate.Version() { + case cert.Version1: + if !relayFrom.Is4() { + cm.l.Error("can not migrate v1 relay with a v6 network because the relay is not running a current nebula version") + continue + } + + if !relayTo.Is4() { + cm.l.Error("can not migrate v1 relay with a v6 remote network because the relay is not running a current nebula version") + continue + } + + b := relayFrom.As4() + req.OldRelayFromAddr = binary.BigEndian.Uint32(b[:]) + b = relayTo.As4() + req.OldRelayToAddr = binary.BigEndian.Uint32(b[:]) + case cert.Version2: + req.RelayFromAddr = netAddrToProtoAddr(relayFrom) + req.RelayToAddr = netAddrToProtoAddr(relayTo) + default: + newhostinfo.logger(cm.l).Error("Unknown certificate version found while attempting to migrate relay") + continue + } + msg, err := req.Marshal() if err != nil { - n.l.WithError(err).Error("failed to marshal Control message to migrate relay") + cm.l.Error("failed to marshal Control message to migrate relay", "error", err) } else { - n.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, make([]byte, 12), make([]byte, mtu)) - n.l.WithFields(logrus.Fields{ - "relayFrom": iputil.VpnIp(req.RelayFromIp), - "relayTo": iputil.VpnIp(req.RelayToIp), - "initiatorRelayIndex": req.InitiatorRelayIndex, - "responderRelayIndex": req.ResponderRelayIndex, - "vpnIp": newhostinfo.vpnIp}). - Info("send CreateRelayRequest") + cm.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, make([]byte, 12), make([]byte, mtu)) + cm.l.Info("send CreateRelayRequest", + "relayFrom", relayFrom, + "relayTo", relayTo, + "initiatorRelayIndex", req.InitiatorRelayIndex, + "responderRelayIndex", req.ResponderRelayIndex, + "vpnAddrs", newhostinfo.vpnAddrs, + ) } } } -func (n *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) { - n.hostMap.RLock() - defer n.hostMap.RUnlock() +func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) { + // Read lock the main hostmap to order decisions based on tunnels being the primary tunnel + cm.hostMap.RLock() + defer cm.hostMap.RUnlock() - hostinfo := n.hostMap.Indexes[localIndex] + hostinfo := cm.hostMap.Indexes[localIndex] if hostinfo == nil { - n.l.WithField("localIndex", localIndex).Debugf("Not found in hostmap") - delete(n.pendingDeletion, localIndex) + cm.l.Debug("Not found in hostmap", "localIndex", localIndex) return doNothing, nil, nil } - if n.isInvalidCertificate(now, hostinfo) { - delete(n.pendingDeletion, hostinfo.localIndexId) + if cm.isInvalidCertificate(now, hostinfo) { return closeTunnel, hostinfo, nil } - primary := n.hostMap.Hosts[hostinfo.vpnIp] + primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]] mainHostInfo := true if primary != nil && primary != hostinfo { mainHostInfo = false } // Check for traffic on this hostinfo - inTraffic, outTraffic := n.getAndResetTrafficCheck(localIndex) + inTraffic, outTraffic := cm.getAndResetTrafficCheck(hostinfo, now) // A hostinfo is determined alive if there is incoming traffic if inTraffic { decision := doNothing - if n.l.Level >= logrus.DebugLevel { - hostinfo.logger(n.l). - WithField("tunnelCheck", m{"state": "alive", "method": "passive"}). - Debug("Tunnel status") + if cm.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(cm.l).Debug("Tunnel status", + "tunnelCheck", m{"state": "alive", "method": "passive"}, + ) } - delete(n.pendingDeletion, hostinfo.localIndexId) + hostinfo.pendingDeletion.Store(false) if mainHostInfo { decision = tryRehandshake - } else { - if n.shouldSwapPrimary(hostinfo, primary) { + if cm.shouldSwapPrimary(hostinfo) { decision = swapPrimary } else { // migrate the relays to the primary, if in use. @@ -340,143 +353,198 @@ func (n *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time } } - n.trafficTimer.Add(hostinfo.localIndexId, n.checkInterval) + cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval) if !outTraffic { // Send a punch packet to keep the NAT state alive - n.sendPunch(hostinfo) + cm.punchy.SendPunch(hostinfo) } return decision, hostinfo, primary } - if _, ok := n.pendingDeletion[hostinfo.localIndexId]; ok { + if hostinfo.pendingDeletion.Load() { // We have already sent a test packet and nothing was returned, this hostinfo is dead - hostinfo.logger(n.l). - WithField("tunnelCheck", m{"state": "dead", "method": "active"}). - Info("Tunnel status") + hostinfo.logger(cm.l).Info("Tunnel status", + "tunnelCheck", m{"state": "dead", "method": "active"}, + ) - delete(n.pendingDeletion, hostinfo.localIndexId) return deleteTunnel, hostinfo, nil } decision := doNothing if hostinfo != nil && hostinfo.ConnectionState != nil && mainHostInfo { if !outTraffic { + inactiveFor, isInactive := cm.isInactive(hostinfo, now) + if isInactive { + // Tunnel is inactive, tear it down + hostinfo.logger(cm.l).Info("Dropping tunnel due to inactivity", + "inactiveDuration", inactiveFor, + "primary", mainHostInfo, + ) + + return closeTunnel, hostinfo, primary + } + // If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel. // Just maintain NAT state if configured to do so. - n.sendPunch(hostinfo) - n.trafficTimer.Add(hostinfo.localIndexId, n.checkInterval) + cm.punchy.SendPunch(hostinfo) + cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval) return doNothing, nil, nil - } - if n.punchy.GetTargetEverything() { - // This is similar to the old punchy behavior with a slight optimization. - // We aren't receiving traffic but we are sending it, punch on all known - // ips in case we need to re-prime NAT state - n.sendPunch(hostinfo) - } + // We aren't receiving traffic but we are sending it. The outbound + // traffic itself refreshes the primary remote's NAT state; this + // fans out to non-primary remotes, but only if target_all_remotes + // is configured. + cm.punchy.SendPunchToAll(hostinfo) - if n.l.Level >= logrus.DebugLevel { - hostinfo.logger(n.l). - WithField("tunnelCheck", m{"state": "testing", "method": "active"}). - Debug("Tunnel status") + if cm.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(cm.l).Debug("Tunnel status", + "tunnelCheck", m{"state": "testing", "method": "active"}, + ) } // Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues decision = sendTestPacket } else { - if n.l.Level >= logrus.DebugLevel { - hostinfo.logger(n.l).Debugf("Hostinfo sadness") + if cm.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(cm.l).Debug("Hostinfo sadness") } } - n.pendingDeletion[hostinfo.localIndexId] = struct{}{} - n.trafficTimer.Add(hostinfo.localIndexId, n.pendingDeletionInterval) + hostinfo.pendingDeletion.Store(true) + cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval) return decision, hostinfo, nil } -func (n *connectionManager) shouldSwapPrimary(current, primary *HostInfo) bool { +func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) { + if cm.dropInactive.Load() == false { + // We aren't configured to drop inactive tunnels + return 0, false + } + + inactiveDuration := now.Sub(hostinfo.lastUsed) + if inactiveDuration < cm.getInactivityTimeout() { + // It's not considered inactive + return inactiveDuration, false + } + + // The tunnel is inactive + return inactiveDuration, true +} + +func (cm *connectionManager) shouldSwapPrimary(current *HostInfo) bool { // The primary tunnel is the most recent handshake to complete locally and should work entirely fine. // If we are here then we have multiple tunnels for a host pair and neither side believes the same tunnel is primary. // Let's sort this out. - if current.vpnIp < n.intf.myVpnIp { - // Only one side should flip primary because if both flip then we may never resolve to a single tunnel. - // vpn ip is static across all tunnels for this host pair so lets use that to determine who is flipping. - // The remotes vpn ip is lower than mine. I will not flip. + // Only one side should swap because if both swap then we may never resolve to a single tunnel. + // vpn addr is static across all tunnels for this host pair so lets + // use that to determine if we should consider swapping. + if current.vpnAddrs[0].Compare(cm.intf.myVpnAddrs[0]) < 0 { + // Their primary vpn addr is less than mine. Do not swap. return false } - certState := n.intf.pki.GetCertState() - return bytes.Equal(current.ConnectionState.myCert.Signature, certState.Certificate.Signature) + crt := cm.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version()) + if crt == nil { + //my cert was reloaded away. We should definitely swap from this tunnel + return true + } + // If this tunnel is using the latest certificate then we should swap it to primary for a bit and see if things + // settle down. + return bytes.Equal(current.ConnectionState.myCert.Signature(), crt.Signature()) } -func (n *connectionManager) swapPrimary(current, primary *HostInfo) { - n.hostMap.Lock() +func (cm *connectionManager) swapPrimary(current, primary *HostInfo) { + cm.hostMap.Lock() // Make sure the primary is still the same after the write lock. This avoids a race with a rehandshake. - if n.hostMap.Hosts[current.vpnIp] == primary { - n.hostMap.unlockedMakePrimary(current) + if cm.hostMap.Hosts[current.vpnAddrs[0]] == primary { + cm.hostMap.unlockedMakePrimary(current) } - n.hostMap.Unlock() + cm.hostMap.Unlock() } -// isInvalidCertificate will check if we should destroy a tunnel if pki.disconnect_invalid is true and -// the certificate is no longer valid. Block listed certificates will skip the pki.disconnect_invalid -// check and return true. -func (n *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostInfo) bool { +// isInvalidCertificate decides if we should destroy a tunnel. +// returns true if pki.disconnect_invalid is true and the certificate is no longer valid. +// Blocklisted certificates will skip the pki.disconnect_invalid check and return true. +func (cm *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostInfo) bool { remoteCert := hostinfo.GetCert() if remoteCert == nil { - return false + return false //don't tear down tunnels for handshakes in progress } - valid, err := remoteCert.VerifyWithCache(now, n.intf.pki.GetCAPool()) - if valid { - return false - } - - if !n.intf.disconnectInvalid.Load() && err != cert.ErrBlockListed { + caPool := cm.intf.pki.GetCAPool() + err := caPool.VerifyCachedCertificate(now, remoteCert) + if err == nil { + return false //cert is still valid! yay! + } else if err == cert.ErrBlockListed { //avoiding errors.Is for speed // Block listed certificates should always be disconnected + hostinfo.logger(cm.l).Info("Remote certificate is blocked, tearing down the tunnel", + "error", err, + "fingerprint", remoteCert.Fingerprint, + ) + return true + } else if cm.intf.disconnectInvalid.Load() { + hostinfo.logger(cm.l).Info("Remote certificate is no longer valid, tearing down the tunnel", + "error", err, + "fingerprint", remoteCert.Fingerprint, + ) + return true + } else { + //if we reach here, the cert is no longer valid, but we're configured to keep tunnels from now-invalid certs open return false } - - fingerprint, _ := remoteCert.Sha256Sum() - hostinfo.logger(n.l).WithError(err). - WithField("fingerprint", fingerprint). - Info("Remote certificate is no longer valid, tearing down the tunnel") - - return true } -func (n *connectionManager) sendPunch(hostinfo *HostInfo) { - if !n.punchy.GetPunch() { - // Punching is disabled +func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) { + cs := cm.intf.pki.getCertState() + curCrt := hostinfo.ConnectionState.myCert + curCrtVersion := curCrt.Version() + myCrt := cs.getCertificate(curCrtVersion) + if myCrt == nil { + cm.l.Info("Re-handshaking with remote", + "vpnAddrs", hostinfo.vpnAddrs, + "version", curCrtVersion, + "reason", "local certificate removed", + ) + cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil) return } - - if n.punchy.GetTargetEverything() { - hostinfo.remotes.ForEach(n.hostMap.preferredRanges, func(addr *udp.Addr, preferred bool) { - n.metricsTxPunchy.Inc(1) - n.intf.outside.WriteTo([]byte{1}, addr) - }) - - } else if hostinfo.remote != nil { - n.metricsTxPunchy.Inc(1) - n.intf.outside.WriteTo([]byte{1}, hostinfo.remote) + peerCrt := hostinfo.ConnectionState.peerCert + if peerCrt != nil && curCrtVersion < peerCrt.Certificate.Version() { + // if our certificate version is less than theirs, and we have a matching version available, rehandshake? + if cs.getCertificate(peerCrt.Certificate.Version()) != nil { + cm.l.Info("Re-handshaking with remote", + "vpnAddrs", hostinfo.vpnAddrs, + "version", curCrtVersion, + "peerVersion", peerCrt.Certificate.Version(), + "reason", "local certificate version lower than peer, attempting to correct", + ) + cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(hh *HandshakeHostInfo) { + hh.initiatingVersionOverride = peerCrt.Certificate.Version() + }) + return + } } -} + if !bytes.Equal(curCrt.Signature(), myCrt.Signature()) { + cm.l.Info("Re-handshaking with remote", + "vpnAddrs", hostinfo.vpnAddrs, + "reason", "local certificate is not current", + ) -func (n *connectionManager) tryRehandshake(hostinfo *HostInfo) { - certState := n.intf.pki.GetCertState() - if bytes.Equal(hostinfo.ConnectionState.myCert.Signature, certState.Certificate.Signature) { + cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil) return } + if curCrtVersion < cs.initiatingVersion { + cm.l.Info("Re-handshaking with remote", + "vpnAddrs", hostinfo.vpnAddrs, + "reason", "current cert version < pki.initiatingVersion", + ) - n.l.WithField("vpnIp", hostinfo.vpnIp). - WithField("reason", "local certificate is not current"). - Info("Re-handshaking with remote") - - n.intf.handshakeManager.StartHandshake(hostinfo.vpnIp, nil) + cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil) + return + } } diff --git a/connection_manager_test.go b/connection_manager_test.go index a2607a2b..25637c25 100644 --- a/connection_manager_test.go +++ b/connection_manager_test.go @@ -1,33 +1,31 @@ package nebula import ( - "context" "crypto/ed25519" "crypto/rand" - "net" + "net/netip" "testing" "time" - "github.com/flynn/noise" "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/overlay/overlaytest" "github.com/slackhq/nebula/test" "github.com/slackhq/nebula/udp" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -var vpnIp iputil.VpnIp - func newTestLighthouse() *LightHouse { lh := &LightHouse{ l: test.NewLogger(), - addrMap: map[iputil.VpnIp]*RemoteList{}, - queryChan: make(chan iputil.VpnIp, 10), + addrMap: map[netip.Addr]*RemoteList{}, + queryChan: make(chan netip.Addr, 10), } - lighthouses := map[iputil.VpnIp]struct{}{} - staticList := map[iputil.VpnIp]struct{}{} + lighthouses := []netip.Addr{} + staticList := map[netip.Addr]struct{}{} + lh.localAddrsFn = func(*LocalAllowList) []netip.Addr { return nil } lh.lighthouses.Store(&lighthouses) lh.staticList.Store(&staticList) @@ -37,24 +35,25 @@ func newTestLighthouse() *LightHouse { func Test_NewConnectionManagerTest(t *testing.T) { l := test.NewLogger() //_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24") - _, vpncidr, _ := net.ParseCIDR("172.1.1.1/24") - _, localrange, _ := net.ParseCIDR("10.1.1.1/24") - vpnIp = iputil.Ip2VpnIp(net.ParseIP("172.1.1.2")) - preferredRanges := []*net.IPNet{localrange} + localrange := netip.MustParsePrefix("10.1.1.1/24") + vpnIp := netip.MustParseAddr("172.1.1.2") + preferredRanges := []netip.Prefix{localrange} // Very incomplete mock objects - hostMap := NewHostMap(l, vpncidr, preferredRanges) + hostMap := newHostMap(l) + hostMap.preferredRanges.Store(&preferredRanges) + cs := &CertState{ - RawCertificate: []byte{}, - PrivateKey: []byte{}, - Certificate: &cert.NebulaCertificate{}, - RawCertificateNoKey: []byte{}, + initiatingVersion: cert.Version1, + privateKey: []byte{}, + v1Cert: &dummyCert{version: cert.Version1}, + v1Credential: nil, } lh := newTestLighthouse() ifce := &Interface{ hostMap: hostMap, - inside: &test.NoopTun{}, + inside: &overlaytest.NoopTun{}, outside: &udp.NoopConn{}, firewall: &Firewall{}, lightHouse: lh, @@ -65,76 +64,78 @@ func Test_NewConnectionManagerTest(t *testing.T) { ifce.pki.cs.Store(cs) // Create manager - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - punchy := NewPunchyFromConfig(l, config.NewC(l)) - nc := newConnectionManager(ctx, l, ifce, 5, 10, punchy) + conf := config.NewC(test.NewLogger()) + punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil) + nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy) + nc.intf = ifce p := []byte("") nb := make([]byte, 12, 12) out := make([]byte, mtu) // Add an ip we have established a connection w/ to hostmap hostinfo := &HostInfo{ - vpnIp: vpnIp, + vpnAddrs: []netip.Addr{vpnIp}, localIndexId: 1099, remoteIndexId: 9901, } hostinfo.ConnectionState = &ConnectionState{ - myCert: &cert.NebulaCertificate{}, - H: &noise.HandshakeState{}, + myCert: &dummyCert{version: cert.Version1}, } nc.hostMap.unlockedAddHostInfo(hostinfo, ifce) // We saw traffic out to vpnIp - nc.Out(hostinfo.localIndexId) - nc.In(hostinfo.localIndexId) - assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId) - assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp) + nc.Out(hostinfo) + nc.In(hostinfo) + assert.False(t, hostinfo.pendingDeletion.Load()) + assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) - assert.Contains(t, nc.out, hostinfo.localIndexId) + assert.True(t, hostinfo.out.Load()) + assert.True(t, hostinfo.in.Load()) // Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) - assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId) - assert.NotContains(t, nc.out, hostinfo.localIndexId) - assert.NotContains(t, nc.in, hostinfo.localIndexId) + assert.False(t, hostinfo.pendingDeletion.Load()) + assert.False(t, hostinfo.out.Load()) + assert.False(t, hostinfo.in.Load()) // Do another traffic check tick, this host should be pending deletion now - nc.Out(hostinfo.localIndexId) + nc.Out(hostinfo) + assert.True(t, hostinfo.out.Load()) nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) - assert.Contains(t, nc.pendingDeletion, hostinfo.localIndexId) - assert.NotContains(t, nc.out, hostinfo.localIndexId) - assert.NotContains(t, nc.in, hostinfo.localIndexId) + assert.True(t, hostinfo.pendingDeletion.Load()) + assert.False(t, hostinfo.out.Load()) + assert.False(t, hostinfo.in.Load()) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) - assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp) + assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) // Do a final traffic check tick, the host should now be removed nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) - assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId) - assert.NotContains(t, nc.hostMap.Hosts, hostinfo.vpnIp) + assert.NotContains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs) assert.NotContains(t, nc.hostMap.Indexes, hostinfo.localIndexId) } func Test_NewConnectionManagerTest2(t *testing.T) { l := test.NewLogger() //_, tuncidr, _ := net.ParseCIDR("1.1.1.1/24") - _, vpncidr, _ := net.ParseCIDR("172.1.1.1/24") - _, localrange, _ := net.ParseCIDR("10.1.1.1/24") - preferredRanges := []*net.IPNet{localrange} + localrange := netip.MustParsePrefix("10.1.1.1/24") + vpnIp := netip.MustParseAddr("172.1.1.2") + preferredRanges := []netip.Prefix{localrange} // Very incomplete mock objects - hostMap := NewHostMap(l, vpncidr, preferredRanges) + hostMap := newHostMap(l) + hostMap.preferredRanges.Store(&preferredRanges) + cs := &CertState{ - RawCertificate: []byte{}, - PrivateKey: []byte{}, - Certificate: &cert.NebulaCertificate{}, - RawCertificateNoKey: []byte{}, + initiatingVersion: cert.Version1, + privateKey: []byte{}, + v1Cert: &dummyCert{version: cert.Version1}, + v1Credential: nil, } lh := newTestLighthouse() ifce := &Interface{ hostMap: hostMap, - inside: &test.NoopTun{}, + inside: &overlaytest.NoopTun{}, outside: &udp.NoopConn{}, firewall: &Firewall{}, lightHouse: lh, @@ -145,56 +146,150 @@ func Test_NewConnectionManagerTest2(t *testing.T) { ifce.pki.cs.Store(cs) // Create manager - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - punchy := NewPunchyFromConfig(l, config.NewC(l)) - nc := newConnectionManager(ctx, l, ifce, 5, 10, punchy) + conf := config.NewC(test.NewLogger()) + punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil) + nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy) + nc.intf = ifce p := []byte("") nb := make([]byte, 12, 12) out := make([]byte, mtu) // Add an ip we have established a connection w/ to hostmap hostinfo := &HostInfo{ - vpnIp: vpnIp, + vpnAddrs: []netip.Addr{vpnIp}, localIndexId: 1099, remoteIndexId: 9901, } hostinfo.ConnectionState = &ConnectionState{ - myCert: &cert.NebulaCertificate{}, - H: &noise.HandshakeState{}, + myCert: &dummyCert{version: cert.Version1}, } nc.hostMap.unlockedAddHostInfo(hostinfo, ifce) // We saw traffic out to vpnIp - nc.Out(hostinfo.localIndexId) - nc.In(hostinfo.localIndexId) - assert.NotContains(t, nc.pendingDeletion, hostinfo.vpnIp) - assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp) + nc.Out(hostinfo) + nc.In(hostinfo) + assert.True(t, hostinfo.in.Load()) + assert.True(t, hostinfo.out.Load()) + assert.False(t, hostinfo.pendingDeletion.Load()) + assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) // Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) - assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId) - assert.NotContains(t, nc.out, hostinfo.localIndexId) - assert.NotContains(t, nc.in, hostinfo.localIndexId) + assert.False(t, hostinfo.pendingDeletion.Load()) + assert.False(t, hostinfo.out.Load()) + assert.False(t, hostinfo.in.Load()) // Do another traffic check tick, this host should be pending deletion now - nc.Out(hostinfo.localIndexId) + nc.Out(hostinfo) nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) - assert.Contains(t, nc.pendingDeletion, hostinfo.localIndexId) - assert.NotContains(t, nc.out, hostinfo.localIndexId) - assert.NotContains(t, nc.in, hostinfo.localIndexId) + assert.True(t, hostinfo.pendingDeletion.Load()) + assert.False(t, hostinfo.out.Load()) + assert.False(t, hostinfo.in.Load()) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) - assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp) + assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) // We saw traffic, should no longer be pending deletion - nc.In(hostinfo.localIndexId) + nc.In(hostinfo) nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now()) - assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId) - assert.NotContains(t, nc.out, hostinfo.localIndexId) - assert.NotContains(t, nc.in, hostinfo.localIndexId) + assert.False(t, hostinfo.pendingDeletion.Load()) + assert.False(t, hostinfo.out.Load()) + assert.False(t, hostinfo.in.Load()) assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) - assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnIp) + assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) +} + +func Test_NewConnectionManager_DisconnectInactive(t *testing.T) { + l := test.NewLogger() + localrange := netip.MustParsePrefix("10.1.1.1/24") + vpnAddrs := []netip.Addr{netip.MustParseAddr("172.1.1.2")} + preferredRanges := []netip.Prefix{localrange} + + // Very incomplete mock objects + hostMap := newHostMap(l) + hostMap.preferredRanges.Store(&preferredRanges) + + cs := &CertState{ + initiatingVersion: cert.Version1, + privateKey: []byte{}, + v1Cert: &dummyCert{version: cert.Version1}, + v1Credential: nil, + } + + lh := newTestLighthouse() + ifce := &Interface{ + hostMap: hostMap, + inside: &overlaytest.NoopTun{}, + outside: &udp.NoopConn{}, + firewall: &Firewall{}, + lightHouse: lh, + pki: &PKI{}, + handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig), + l: l, + } + ifce.pki.cs.Store(cs) + + // Create manager + conf := config.NewC(test.NewLogger()) + conf.Settings["tunnels"] = map[string]any{ + "drop_inactive": true, + } + punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil) + nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy) + assert.True(t, nc.dropInactive.Load()) + nc.intf = ifce + + // Add an ip we have established a connection w/ to hostmap + hostinfo := &HostInfo{ + vpnAddrs: vpnAddrs, + localIndexId: 1099, + remoteIndexId: 9901, + } + hostinfo.ConnectionState = &ConnectionState{ + myCert: &dummyCert{version: cert.Version1}, + } + nc.hostMap.unlockedAddHostInfo(hostinfo, ifce) + + // Do a traffic check tick, in and out should be cleared but should not be pending deletion + nc.Out(hostinfo) + nc.In(hostinfo) + assert.True(t, hostinfo.out.Load()) + assert.True(t, hostinfo.in.Load()) + + now := time.Now() + decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now) + assert.Equal(t, tryRehandshake, decision) + assert.Equal(t, now, hostinfo.lastUsed) + assert.False(t, hostinfo.pendingDeletion.Load()) + assert.False(t, hostinfo.out.Load()) + assert.False(t, hostinfo.in.Load()) + + decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5)) + assert.Equal(t, doNothing, decision) + assert.Equal(t, now, hostinfo.lastUsed) + assert.False(t, hostinfo.pendingDeletion.Load()) + assert.False(t, hostinfo.out.Load()) + assert.False(t, hostinfo.in.Load()) + + // Do another traffic check tick, should still not be pending deletion + decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10)) + assert.Equal(t, doNothing, decision) + assert.Equal(t, now, hostinfo.lastUsed) + assert.False(t, hostinfo.pendingDeletion.Load()) + assert.False(t, hostinfo.out.Load()) + assert.False(t, hostinfo.in.Load()) + assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) + assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) + + // Finally advance beyond the inactivity timeout + decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10)) + assert.Equal(t, closeTunnel, decision) + assert.Equal(t, now, hostinfo.lastUsed) + assert.False(t, hostinfo.pendingDeletion.Load()) + assert.False(t, hostinfo.out.Load()) + assert.False(t, hostinfo.in.Load()) + assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId) + assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) } // Check if we can disconnect the peer. @@ -203,59 +298,54 @@ func Test_NewConnectionManagerTest2(t *testing.T) { func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) { now := time.Now() l := test.NewLogger() - ipNet := net.IPNet{ - IP: net.IPv4(172, 1, 1, 2), - Mask: net.IPMask{255, 255, 255, 0}, - } - _, vpncidr, _ := net.ParseCIDR("172.1.1.1/24") - _, localrange, _ := net.ParseCIDR("10.1.1.1/24") - preferredRanges := []*net.IPNet{localrange} - hostMap := NewHostMap(l, vpncidr, preferredRanges) + + vpncidr := netip.MustParsePrefix("172.1.1.1/24") + localrange := netip.MustParsePrefix("10.1.1.1/24") + vpnIp := netip.MustParseAddr("172.1.1.2") + preferredRanges := []netip.Prefix{localrange} + hostMap := newHostMap(l) + hostMap.preferredRanges.Store(&preferredRanges) // Generate keys for CA and peer's cert. pubCA, privCA, _ := ed25519.GenerateKey(rand.Reader) - caCert := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "ca", - NotBefore: now, - NotAfter: now.Add(1 * time.Hour), - IsCA: true, - PublicKey: pubCA, - }, + tbs := &cert.TBSCertificate{ + Version: 1, + Name: "ca", + IsCA: true, + NotBefore: now, + NotAfter: now.Add(1 * time.Hour), + PublicKey: pubCA, } - assert.NoError(t, caCert.Sign(cert.Curve_CURVE25519, privCA)) - ncp := &cert.NebulaCAPool{ - CAs: cert.NewCAPool().CAs, - } - ncp.CAs["ca"] = &caCert + caCert, err := tbs.Sign(nil, cert.Curve_CURVE25519, privCA) + require.NoError(t, err) + ncp := cert.NewCAPool() + require.NoError(t, ncp.AddCA(caCert)) pubCrt, _, _ := ed25519.GenerateKey(rand.Reader) - peerCert := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "host", - Ips: []*net.IPNet{&ipNet}, - Subnets: []*net.IPNet{}, - NotBefore: now, - NotAfter: now.Add(60 * time.Second), - PublicKey: pubCrt, - IsCA: false, - Issuer: "ca", - }, + tbs = &cert.TBSCertificate{ + Version: 1, + Name: "host", + Networks: []netip.Prefix{vpncidr}, + NotBefore: now, + NotAfter: now.Add(60 * time.Second), + PublicKey: pubCrt, } - assert.NoError(t, peerCert.Sign(cert.Curve_CURVE25519, privCA)) + peerCert, err := tbs.Sign(caCert, cert.Curve_CURVE25519, privCA) + require.NoError(t, err) + + cachedPeerCert, err := ncp.VerifyCertificate(now.Add(time.Second), peerCert) cs := &CertState{ - RawCertificate: []byte{}, - PrivateKey: []byte{}, - Certificate: &cert.NebulaCertificate{}, - RawCertificateNoKey: []byte{}, + privateKey: []byte{}, + v1Cert: &dummyCert{}, + v1Credential: nil, } lh := newTestLighthouse() ifce := &Interface{ hostMap: hostMap, - inside: &test.NoopTun{}, + inside: &overlaytest.NoopTun{}, outside: &udp.NoopConn{}, firewall: &Firewall{}, lightHouse: lh, @@ -268,18 +358,17 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) { ifce.disconnectInvalid.Store(true) // Create manager - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - punchy := NewPunchyFromConfig(l, config.NewC(l)) - nc := newConnectionManager(ctx, l, ifce, 5, 10, punchy) + conf := config.NewC(test.NewLogger()) + punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil) + nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy) + nc.intf = ifce ifce.connectionManager = nc hostinfo := &HostInfo{ - vpnIp: vpnIp, + vpnAddrs: []netip.Addr{vpnIp}, ConnectionState: &ConnectionState{ - myCert: &cert.NebulaCertificate{}, - peerCert: &peerCert, - H: &noise.HandshakeState{}, + myCert: &dummyCert{}, + peerCert: cachedPeerCert, }, } nc.hostMap.unlockedAddHostInfo(hostinfo, ifce) @@ -298,3 +387,118 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) { invalid = nc.isInvalidCertificate(nextTick, hostinfo) assert.True(t, invalid) } + +type dummyCert struct { + version cert.Version + curve cert.Curve + groups []string + isCa bool + issuer string + name string + networks []netip.Prefix + notAfter time.Time + notBefore time.Time + publicKey []byte + signature []byte + unsafeNetworks []netip.Prefix +} + +func (d *dummyCert) Version() cert.Version { + return d.version +} + +func (d *dummyCert) Curve() cert.Curve { + return d.curve +} + +func (d *dummyCert) Groups() []string { + return d.groups +} + +func (d *dummyCert) IsCA() bool { + return d.isCa +} + +func (d *dummyCert) Issuer() string { + return d.issuer +} + +func (d *dummyCert) Name() string { + return d.name +} + +func (d *dummyCert) Networks() []netip.Prefix { + return d.networks +} + +func (d *dummyCert) NotAfter() time.Time { + return d.notAfter +} + +func (d *dummyCert) NotBefore() time.Time { + return d.notBefore +} + +func (d *dummyCert) PublicKey() []byte { + return d.publicKey +} + +func (d *dummyCert) MarshalPublicKeyPEM() []byte { + return cert.MarshalPublicKeyToPEM(d.curve, d.publicKey) +} + +func (d *dummyCert) Signature() []byte { + return d.signature +} + +func (d *dummyCert) UnsafeNetworks() []netip.Prefix { + return d.unsafeNetworks +} + +func (d *dummyCert) MarshalForHandshakes() ([]byte, error) { + return nil, nil +} + +func (d *dummyCert) Sign(curve cert.Curve, key []byte) error { + return nil +} + +func (d *dummyCert) CheckSignature(key []byte) bool { + return true +} + +func (d *dummyCert) Expired(t time.Time) bool { + return false +} + +func (d *dummyCert) CheckRootConstraints(signer cert.Certificate) error { + return nil +} + +func (d *dummyCert) VerifyPrivateKey(curve cert.Curve, key []byte) error { + return nil +} + +func (d *dummyCert) String() string { + return "" +} + +func (d *dummyCert) Marshal() ([]byte, error) { + return nil, nil +} + +func (d *dummyCert) MarshalPEM() ([]byte, error) { + return nil, nil +} + +func (d *dummyCert) Fingerprint() (string, error) { + return "", nil +} + +func (d *dummyCert) MarshalJSON() ([]byte, error) { + return nil, nil +} + +func (d *dummyCert) Copy() cert.Certificate { + return d +} diff --git a/connection_state.go b/connection_state.go index 8ef8b3a2..0d6f59e1 100644 --- a/connection_state.go +++ b/connection_state.go @@ -1,78 +1,48 @@ package nebula import ( - "crypto/rand" "encoding/json" + "log/slog" "sync" "sync/atomic" - "github.com/flynn/noise" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/handshake" + "github.com/slackhq/nebula/header" "github.com/slackhq/nebula/noiseutil" ) const ReplayWindow = 1024 type ConnectionState struct { - eKey *NebulaCipherState - dKey *NebulaCipherState - H *noise.HandshakeState - myCert *cert.NebulaCertificate - peerCert *cert.NebulaCertificate + eKey noiseutil.CipherState + dKey noiseutil.CipherState + myCert cert.Certificate + peerCert *cert.CachedCertificate initiator bool messageCounter atomic.Uint64 window *Bits + decryptLock sync.Mutex writeLock sync.Mutex } -func NewConnectionState(l *logrus.Logger, cipher string, certState *CertState, initiator bool, pattern noise.HandshakePattern, psk []byte, pskStage int) *ConnectionState { - var dhFunc noise.DHFunc - switch certState.Certificate.Details.Curve { - case cert.Curve_CURVE25519: - dhFunc = noise.DH25519 - case cert.Curve_P256: - dhFunc = noiseutil.DHP256 - default: - l.Errorf("invalid curve: %s", certState.Certificate.Details.Curve) - return nil - } - - var cs noise.CipherSuite - if cipher == "chachapoly" { - cs = noise.NewCipherSuite(dhFunc, noise.CipherChaChaPoly, noise.HashSHA256) - } else { - cs = noise.NewCipherSuite(dhFunc, noiseutil.CipherAESGCM, noise.HashSHA256) - } - - static := noise.DHKey{Private: certState.PrivateKey, Public: certState.PublicKey} - - b := NewBits(ReplayWindow) - // Clear out bit 0, we never transmit it and we don't want it showing as packet loss - b.Update(l, 0) - - hs, err := noise.NewHandshakeState(noise.Config{ - CipherSuite: cs, - Random: rand.Reader, - Pattern: pattern, - Initiator: initiator, - StaticKeypair: static, - PresharedKey: psk, - PresharedKeyPlacement: pskStage, - }) - if err != nil { - return nil - } - - // The queue and ready params prevent a counter race that would happen when - // sending stored packets and simultaneously accepting new traffic. +// newConnectionStateFromResult builds a fully-populated ConnectionState from a +// completed handshake.Result. It seeds messageCounter and the replay window so +// that the post-handshake message indices already used on the wire don't count +// as missed traffic in the data plane. +func newConnectionStateFromResult(r *handshake.Result) *ConnectionState { ci := &ConnectionState{ - H: hs, - initiator: initiator, - window: b, - myCert: certState.Certificate, + myCert: r.MyCert, + initiator: r.Initiator, + peerCert: r.RemoteCert, + eKey: noiseutil.NewCipherState(r.EKey, r.Cipher), + dKey: noiseutil.NewCipherState(r.DKey, r.Cipher), + window: NewBits(ReplayWindow), + } + ci.messageCounter.Add(r.MessageIndex) + for i := uint64(1); i <= r.MessageIndex; i++ { + ci.window.Update(nil, i) } - return ci } @@ -83,3 +53,56 @@ func (cs *ConnectionState) MarshalJSON() ([]byte, error) { "message_counter": cs.messageCounter.Load(), }) } + +func (cs *ConnectionState) Curve() cert.Curve { + return cs.myCert.Curve() +} + +func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, out []byte, packet []byte, nb []byte) ([]byte, error) { + var err error + cs.decryptLock.Lock() + result := cs.window.Check(l, messageCounter) + cs.decryptLock.Unlock() + if !result { + return nil, ErrAlreadySeen + } + + out, err = cs.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb) + if err != nil { + return nil, err + } + + cs.decryptLock.Lock() + result = cs.window.Update(l, messageCounter) + cs.decryptLock.Unlock() + if !result { + return nil, ErrAlreadySeen + } + return out, nil +} + +// VerifyRelay verifies AEAD protected (but not encrypted) relay frames. packet must be length-checked by the caller. +func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) error { + cs.decryptLock.Lock() + result := cs.window.Check(l, messageCounter) + cs.decryptLock.Unlock() + if !result { + return ErrAlreadySeen + } + + signedPayload := packet[:len(packet)-cs.dKey.Overhead()] + signatureValue := packet[len(packet)-cs.dKey.Overhead():] + _, err := cs.dKey.DecryptDanger(nil, signedPayload, signatureValue, messageCounter, nb) + if err != nil { + return err + } + + cs.decryptLock.Lock() + result = cs.window.Update(l, messageCounter) + cs.decryptLock.Unlock() + if !result { + return ErrAlreadySeen + } + + return nil +} diff --git a/connection_state_test.go b/connection_state_test.go new file mode 100644 index 00000000..dea60d39 --- /dev/null +++ b/connection_state_test.go @@ -0,0 +1,114 @@ +package nebula + +import ( + "net/netip" + "testing" + "time" + + "github.com/flynn/noise" + "github.com/slackhq/nebula/cert" + ct "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/handshake" + "github.com/slackhq/nebula/header" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// runTestHandshake runs a complete IX handshake between two freshly-built +// peers and returns the initiator and responder Results. Used to produce +// real cipher states for tests that need to exercise post-handshake glue. +func runTestHandshake(t *testing.T) (initR, respR *handshake.Result) { + t.Helper() + + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + + makeCreds := func(name string, networks []netip.Prefix) handshake.GetCredentialFunc { + c, _, rawKey, _ := ct.NewTestCert( + cert.Version2, cert.Curve_CURVE25519, ca, caKey, + name, ca.NotBefore(), ca.NotAfter(), networks, nil, nil, + ) + priv, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawKey) + require.NoError(t, err) + hsBytes, err := c.MarshalForHandshakes() + require.NoError(t, err) + ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256) + cred := handshake.NewCredential(c, hsBytes, priv, ncs) + return func(v cert.Version) *handshake.Credential { + if v == cert.Version2 { + return cred + } + return nil + } + } + + verifier := func(c cert.Certificate) (*cert.CachedCertificate, error) { + return caPool.VerifyCertificate(time.Now(), c) + } + + initCreds := makeCreds("initiator", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + respCreds := makeCreds("responder", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}) + + initM, err := handshake.NewMachine( + cert.Version2, initCreds, verifier, + func() (uint32, error) { return 1000, nil }, + true, header.HandshakeIXPSK0, + ) + require.NoError(t, err) + + respM, err := handshake.NewMachine( + cert.Version2, respCreds, verifier, + func() (uint32, error) { return 2000, nil }, + false, header.HandshakeIXPSK0, + ) + require.NoError(t, err) + + msg1, err := initM.Initiate(nil) + require.NoError(t, err) + + resp, respR, err := respM.ProcessPacket(nil, msg1) + require.NoError(t, err) + require.NotNil(t, respR) + + _, initR, err = initM.ProcessPacket(nil, resp) + require.NoError(t, err) + require.NotNil(t, initR) + + return initR, respR +} + +func TestNewConnectionStateFromResult(t *testing.T) { + initR, respR := runTestHandshake(t) + + t.Run("initiator", func(t *testing.T) { + ci := newConnectionStateFromResult(initR) + assert.True(t, ci.initiator) + assert.Equal(t, initR.MyCert, ci.myCert) + assert.Equal(t, initR.RemoteCert, ci.peerCert) + assert.NotNil(t, ci.eKey) + assert.NotNil(t, ci.dKey) + + // IX has 2 handshake messages; the next data-plane send is counter=3. + assert.Equal(t, uint64(2), ci.messageCounter.Load(), + "messageCounter must equal Result.MessageIndex so the next send is N+1") + + // Both handshake counters must be marked seen so they don't appear lost. + // Check returns false if an index has already been recorded. + assert.False(t, ci.window.Check(nil, 1), "counter 1 must already be seen") + assert.False(t, ci.window.Check(nil, 2), "counter 2 must already be seen") + // Counter 3 is the next data-plane message and must NOT be pre-marked. + assert.True(t, ci.window.Check(nil, 3), "counter 3 must not be pre-seeded") + }) + + t.Run("responder", func(t *testing.T) { + ci := newConnectionStateFromResult(respR) + assert.False(t, ci.initiator) + assert.Equal(t, respR.MyCert, ci.myCert) + assert.Equal(t, respR.RemoteCert, ci.peerCert) + assert.NotNil(t, ci.eKey) + assert.NotNil(t, ci.dKey) + assert.Equal(t, uint64(2), ci.messageCounter.Load()) + }) +} diff --git a/control.go b/control.go index 1e27b0fa..7df5a09e 100644 --- a/control.go +++ b/control.go @@ -2,58 +2,98 @@ package nebula import ( "context" - "net" + "errors" + "log/slog" + "net/netip" "os" "os/signal" + "sync" "syscall" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/overlay" - "github.com/slackhq/nebula/udp" ) +type RunState int + +const ( + StateUnknown RunState = iota + StateReady + StateStarted + StateStopping + StateStopped +) + +var ErrAlreadyStarted = errors.New("nebula is already started") +var ErrAlreadyStopped = errors.New("nebula cannot be restarted") +var ErrUnknownState = errors.New("nebula state is invalid") + // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc type controlEach func(h *HostInfo) type controlHostLister interface { - QueryVpnIp(vpnIp iputil.VpnIp) *HostInfo + QueryVpnAddr(vpnAddr netip.Addr) *HostInfo ForEachIndex(each controlEach) - ForEachVpnIp(each controlEach) - GetPreferredRanges() []*net.IPNet + ForEachVpnAddr(each controlEach) + GetPreferredRanges() []netip.Prefix } type Control struct { - f *Interface - l *logrus.Logger - ctx context.Context - cancel context.CancelFunc - sshStart func() - statsStart func() - dnsStart func() - lighthouseStart func() + stateLock sync.Mutex + state RunState + + f *Interface + l *slog.Logger + ctx context.Context + cancel context.CancelFunc + sshStart func() + statsStart func() + dnsStart func() + lighthouseStart func() + networkChangeStart func(rebind func()) + connectionManagerStart func(context.Context) } type ControlHostInfo struct { - VpnIp net.IP `json:"vpnIp"` - LocalIndex uint32 `json:"localIndex"` - RemoteIndex uint32 `json:"remoteIndex"` - RemoteAddrs []*udp.Addr `json:"remoteAddrs"` - Cert *cert.NebulaCertificate `json:"cert"` - MessageCounter uint64 `json:"messageCounter"` - CurrentRemote *udp.Addr `json:"currentRemote"` - CurrentRelaysToMe []iputil.VpnIp `json:"currentRelaysToMe"` - CurrentRelaysThroughMe []iputil.VpnIp `json:"currentRelaysThroughMe"` + VpnAddrs []netip.Addr `json:"vpnAddrs"` + LocalIndex uint32 `json:"localIndex"` + RemoteIndex uint32 `json:"remoteIndex"` + RemoteAddrs []netip.AddrPort `json:"remoteAddrs"` + Cert cert.Certificate `json:"cert"` + MessageCounter uint64 `json:"messageCounter"` + CurrentRemote netip.AddrPort `json:"currentRemote"` + CurrentRelaysToMe []netip.Addr `json:"currentRelaysToMe"` + CurrentRelaysThroughMe []netip.Addr `json:"currentRelaysThroughMe"` } -// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock() -func (c *Control) Start() { +// Start actually runs nebula, this is a nonblocking call. +// Use Wait to block until nebula has fully stopped and to learn whether a fatal reader error caused the shutdown. +func (c *Control) Start() error { + c.stateLock.Lock() + defer c.stateLock.Unlock() + switch c.state { + case StateReady: + //yay! + case StateStopped, StateStopping: + return ErrAlreadyStopped + case StateStarted: + return ErrAlreadyStarted + default: + return ErrUnknownState + } + // Activate the interface - c.f.activate() + err := c.f.activate() + if err != nil { + // Cancel before Close so a caller returning from Wait always observes a dead Context + c.cancel() + _ = c.f.Close() + c.state = StateStopped + return err + } // Call all the delayed funcs that waited patiently for the interface to be created. if c.sshStart != nil { @@ -65,29 +105,80 @@ func (c *Control) Start() { if c.dnsStart != nil { go c.dnsStart() } + if c.networkChangeStart != nil { + go c.networkChangeStart(c.RebindUDPServer) + } + if c.connectionManagerStart != nil { + go c.connectionManagerStart(c.ctx) + } if c.lighthouseStart != nil { c.lighthouseStart() } + c.f.triggerShutdown = c.Stop + // Start reading packets. c.f.run() + c.state = StateStarted + return nil +} + +func (c *Control) State() RunState { + c.stateLock.Lock() + defer c.stateLock.Unlock() + return c.state } func (c *Control) Context() context.Context { return c.ctx } -// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete +// Stop tears nebula down, closing all tunnels and releasing everything it holds. +// Use Wait to block until the shutdown has completed. +// A Control that has been stopped cannot be started again, Start will return ErrAlreadyStopped. func (c *Control) Stop() { - // Stop the handshakeManager (and other services), to prevent new tunnels from - // being created while we're shutting them all down. - c.cancel() + c.stateLock.Lock() + switch c.state { + case StateStarted: + // Fall through to the full teardown below + + case StateReady: + // Never started + c.cancel() + c.state = StateStopped + if err := c.f.Close(); err != nil { + c.l.Error("Close interface failed", "error", err) + } + c.stateLock.Unlock() + return + + default: + c.stateLock.Unlock() + // We are stopping or stopped already + return + } + + c.state = StateStopping + c.stateLock.Unlock() + // Closing tunnels can be slow with a large hostmap, don't hold the lock for it + c.cancel() c.CloseAllTunnels(false) + + c.stateLock.Lock() + c.state = StateStopped if err := c.f.Close(); err != nil { - c.l.WithError(err).Error("Close interface failed") + c.l.Error("Close interface failed", "error", err) } - c.l.Info("Goodbye") + c.stateLock.Unlock() +} + +// Wait blocks until nebula has fully stopped, either via Stop or an internal fatal error, +// and returns the first fatal packet reader error if there was one. +// It is safe to call from multiple goroutines and at any point in the lifecycle, +// but a Wait on a Control that is never started and never stopped will block forever. +func (c *Control) Wait() error { + return c.f.wait() } // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled @@ -98,13 +189,24 @@ func (c *Control) ShutdownBlock() { rawSig := <-sigChan sig := rawSig.String() - c.l.WithField("signal", sig).Info("Caught signal, shutting down") + c.l.Info("Caught signal, shutting down", "signal", sig) c.Stop() } -// RebindUDPServer asks the UDP listener to rebind it's listener. Mainly used on mobile clients when interfaces change +// RebindUDPServer asks the UDP listener to rebind it's listener. Mainly used on mobile clients when interfaces change. func (c *Control) RebindUDPServer() { - _ = c.f.outside.Rebind() + c.stateLock.Lock() + defer c.stateLock.Unlock() + + if c.state != StateStarted { + return + } + + // A failure here means we are likely still pinned to the interface we came up on, so the rest of this is + // unlikely to help. Say so instead of silently carrying on as if we rebound. + if err := c.f.outside.Rebind(); err != nil { + c.l.Error("Failed to rebind udp socket", "error", err) + } // Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0 c.f.lightHouse.SendUpdate() @@ -131,8 +233,47 @@ func (c *Control) ListHostmapIndexes(pendingMap bool) []ControlHostInfo { } } -// GetHostInfoByVpnIp returns a single tunnels hostInfo, or nil if not found -func (c *Control) GetHostInfoByVpnIp(vpnIp iputil.VpnIp, pending bool) *ControlHostInfo { +// GetCertByVpnIp returns the authenticated certificate of the given vpn IP, or nil if not found +func (c *Control) GetCertByVpnIp(vpnIp netip.Addr) cert.Certificate { + if c.f.myVpnAddrsTable.Contains(vpnIp) { + // Only returning the default certificate since its impossible + // for any other host but ourselves to have more than 1 + return c.f.pki.getCertState().GetDefaultCertificate().Copy() + } + hi := c.f.hostMap.QueryVpnAddr(vpnIp) + if hi == nil { + return nil + } + return hi.GetCert().Certificate.Copy() +} + +// CreateTunnel creates a new tunnel to the given vpn ip. +func (c *Control) CreateTunnel(vpnIp netip.Addr) { + c.f.handshakeManager.StartHandshake(vpnIp, nil) +} + +// PrintTunnel creates a new tunnel to the given vpn ip. +func (c *Control) PrintTunnel(vpnIp netip.Addr) *ControlHostInfo { + hi := c.f.hostMap.QueryVpnAddr(vpnIp) + if hi == nil { + return nil + } + chi := copyHostInfo(hi, c.f.hostMap.GetPreferredRanges()) + return &chi +} + +// QueryLighthouse queries the lighthouse. +func (c *Control) QueryLighthouse(vpnIp netip.Addr) *CacheMap { + hi := c.f.lightHouse.Query(vpnIp) + if hi == nil { + return nil + } + return hi.CopyCache() +} + +// GetHostInfoByVpnAddr returns a single tunnels hostInfo, or nil if not found +// Caller should take care to Unmap() any 4in6 addresses prior to calling. +func (c *Control) GetHostInfoByVpnAddr(vpnAddr netip.Addr, pending bool) *ControlHostInfo { var hl controlHostLister if pending { hl = c.f.handshakeManager @@ -140,30 +281,32 @@ func (c *Control) GetHostInfoByVpnIp(vpnIp iputil.VpnIp, pending bool) *ControlH hl = c.f.hostMap } - h := hl.QueryVpnIp(vpnIp) + h := hl.QueryVpnAddr(vpnAddr) if h == nil { return nil } - ch := copyHostInfo(h, c.f.hostMap.preferredRanges) + ch := copyHostInfo(h, c.f.hostMap.GetPreferredRanges()) return &ch } // SetRemoteForTunnel forces a tunnel to use a specific remote -func (c *Control) SetRemoteForTunnel(vpnIp iputil.VpnIp, addr udp.Addr) *ControlHostInfo { - hostInfo := c.f.hostMap.QueryVpnIp(vpnIp) +// Caller should take care to Unmap() any 4in6 addresses prior to calling. +func (c *Control) SetRemoteForTunnel(vpnIp netip.Addr, addr netip.AddrPort) *ControlHostInfo { + hostInfo := c.f.hostMap.QueryVpnAddr(vpnIp) if hostInfo == nil { return nil } - hostInfo.SetRemote(addr.Copy()) - ch := copyHostInfo(hostInfo, c.f.hostMap.preferredRanges) + hostInfo.SetRemote(addr) + ch := copyHostInfo(hostInfo, c.f.hostMap.GetPreferredRanges()) return &ch } // CloseTunnel closes a fully established tunnel. If localOnly is false it will notify the remote end as well. -func (c *Control) CloseTunnel(vpnIp iputil.VpnIp, localOnly bool) bool { - hostInfo := c.f.hostMap.QueryVpnIp(vpnIp) +// Caller should take care to Unmap() any 4in6 addresses prior to calling. +func (c *Control) CloseTunnel(vpnIp netip.Addr, localOnly bool) bool { + hostInfo := c.f.hostMap.QueryVpnAddr(vpnIp) if hostInfo == nil { return false } @@ -187,29 +330,26 @@ func (c *Control) CloseTunnel(vpnIp iputil.VpnIp, localOnly bool) bool { // CloseAllTunnels is just like CloseTunnel except it goes through and shuts them all down, optionally you can avoid shutting down lighthouse tunnels // the int returned is a count of tunnels closed func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) { - //TODO: this is probably better as a function in ConnectionManager or HostMap directly - lighthouses := c.f.lightHouse.GetLighthouses() - shutdown := func(h *HostInfo) { - if excludeLighthouses { - if _, ok := lighthouses[h.vpnIp]; ok { - return - } + if excludeLighthouses && c.f.lightHouse.IsAnyLighthouseAddr(h.vpnAddrs) { + return } c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu)) c.f.closeTunnel(h) - c.l.WithField("vpnIp", h.vpnIp).WithField("udpAddr", h.remote). - Debug("Sending close tunnel message") + c.l.Debug("Sending close tunnel message", + "vpnAddrs", h.vpnAddrs, + "udpAddr", h.GetRemote(), + ) closed++ } // Learn which hosts are being used as relays, so we can shut them down last. - relayingHosts := map[iputil.VpnIp]*HostInfo{} + relayingHosts := map[netip.Addr]*HostInfo{} // Grab the hostMap lock to access the Relays map c.f.hostMap.Lock() for _, relayingHost := range c.f.hostMap.Relays { - relayingHosts[relayingHost.vpnIp] = relayingHost + relayingHosts[relayingHost.vpnAddrs[0]] = relayingHost } c.f.hostMap.Unlock() @@ -217,7 +357,7 @@ func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) { // Grab the hostMap lock to access the Hosts map c.f.hostMap.Lock() for _, relayHost := range c.f.hostMap.Indexes { - if _, ok := relayingHosts[relayHost.vpnIp]; !ok { + if _, ok := relayingHosts[relayHost.vpnAddrs[0]]; !ok { hostInfos = append(hostInfos, relayHost) } } @@ -236,15 +376,19 @@ func (c *Control) Device() overlay.Device { return c.f.inside } -func copyHostInfo(h *HostInfo, preferredRanges []*net.IPNet) ControlHostInfo { - +func copyHostInfo(h *HostInfo, preferredRanges []netip.Prefix) ControlHostInfo { chi := ControlHostInfo{ - VpnIp: h.vpnIp.ToIP(), + VpnAddrs: make([]netip.Addr, len(h.vpnAddrs)), LocalIndex: h.localIndexId, RemoteIndex: h.remoteIndexId, RemoteAddrs: h.remotes.CopyAddrs(preferredRanges), CurrentRelaysToMe: h.relayState.CopyRelayIps(), CurrentRelaysThroughMe: h.relayState.CopyRelayForIps(), + CurrentRemote: h.GetRemote(), + } + + for i, a := range h.vpnAddrs { + chi.VpnAddrs[i] = a } if h.ConnectionState != nil { @@ -252,11 +396,7 @@ func copyHostInfo(h *HostInfo, preferredRanges []*net.IPNet) ControlHostInfo { } if c := h.GetCert(); c != nil { - chi.Cert = c.Copy() - } - - if h.remote != nil { - chi.CurrentRemote = h.remote.Copy() + chi.Cert = c.Certificate.Copy() } return chi @@ -265,7 +405,7 @@ func copyHostInfo(h *HostInfo, preferredRanges []*net.IPNet) ControlHostInfo { func listHostMapHosts(hl controlHostLister) []ControlHostInfo { hosts := make([]ControlHostInfo, 0) pr := hl.GetPreferredRanges() - hl.ForEachVpnIp(func(hostinfo *HostInfo) { + hl.ForEachVpnAddr(func(hostinfo *HostInfo) { hosts = append(hosts, copyHostInfo(hostinfo, pr)) }) return hosts diff --git a/control_lifecycle_test.go b/control_lifecycle_test.go new file mode 100644 index 00000000..0b5d106d --- /dev/null +++ b/control_lifecycle_test.go @@ -0,0 +1,292 @@ +package nebula + +import ( + "context" + "errors" + "io" + "net/netip" + "sync" + "testing" + "time" + + "github.com/gaissmai/bart" + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/routing" + "github.com/slackhq/nebula/test" + "github.com/slackhq/nebula/udp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeDevice struct { + closeOnce sync.Once + closedCh chan struct{} + closed bool +} + +func newFakeDevice() *fakeDevice { + return &fakeDevice{closedCh: make(chan struct{})} +} + +// Read blocks until Close like a real tun with no traffic, then reports EOF +// the same way a closed device does +func (d *fakeDevice) Read(p []byte) (int, error) { + <-d.closedCh + return 0, io.EOF +} + +func (d *fakeDevice) Write(p []byte) (int, error) { return len(p), nil } + +func (d *fakeDevice) Close() error { + d.closeOnce.Do(func() { + d.closed = true + close(d.closedCh) + }) + return nil +} + +func (d *fakeDevice) Activate() error { return nil } +func (d *fakeDevice) Networks() []netip.Prefix { return nil } +func (d *fakeDevice) Name() string { return "fake" } +func (d *fakeDevice) RoutesFor(netip.Addr) routing.Gateways { return nil } +func (d *fakeDevice) SupportsMultiqueue() bool { return false } +func (d *fakeDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) { + return nil, errors.New("unsupported") +} + +// newReadyControl hand-builds the minimum Control that Main would have +// produced right before Start, including the construction token NewInterface +// takes so waiters block until Close releases the resources +func newReadyControl(t *testing.T) (*Control, *fakeDevice, *fakeConn) { + l := test.NewLogger() + dev := newFakeDevice() + conn := &fakeConn{} + ctx, cancel := context.WithCancel(context.Background()) + + myVpnNet := netip.MustParsePrefix("10.128.0.1/16") + nt := new(bart.Lite) + nt.Insert(myVpnNet) + cs := &CertState{ + myVpnNetworks: []netip.Prefix{myVpnNet}, + myVpnNetworksTable: nt, + } + lh, err := NewLightHouseFromConfig(ctx, l, config.NewC(l), cs, nil, nil) + require.NoError(t, err) + + f := &Interface{ + ctx: ctx, + inside: dev, + outside: conn, + writers: []udp.Conn{conn}, + readers: make([]io.ReadWriteCloser, 1), + routines: 1, + hostMap: newHostMap(l), + lightHouse: lh, + l: l, + } + f.wg.Add(1) + + return &Control{ + state: StateReady, + f: f, + l: l, + ctx: ctx, + cancel: cancel, + }, dev, conn +} + +func TestControl_StopBeforeStart(t *testing.T) { + c, dev, conn := newReadyControl(t) + + // A Stop on a never started control must release everything Main acquired + c.Stop() + assert.Equal(t, StateStopped, c.State()) + assert.True(t, dev.closed, "the tun device should have been closed") + assert.True(t, conn.closed, "the udp socket should have been closed") + require.ErrorIs(t, c.ctx.Err(), context.Canceled, "the service context should have been cancelled") + + // Wait must return promptly now that the resources are released + require.NoError(t, c.Wait()) + + // A stopped control can never be started + require.ErrorIs(t, c.Start(), ErrAlreadyStopped) + + // A second Stop is a harmless no-op + c.Stop() + assert.Equal(t, StateStopped, c.State()) + require.NoError(t, c.Wait()) +} + +func TestControl_WaitBlocksUntilStop(t *testing.T) { + c, _, _ := newReadyControl(t) + + done := make(chan error, 1) + go func() { done <- c.Wait() }() + + select { + case <-done: + t.Fatal("Wait returned before Stop") + case <-time.After(50 * time.Millisecond): + } + + c.Stop() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("Wait did not return after Stop") + } +} + +type fakeConn struct { + closed bool + rebinds int +} + +func (c *fakeConn) Rebind() error { c.rebinds++; return nil } +func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil } +func (c *fakeConn) ListenOut(_ udp.EncReader) error { return nil } +func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil } +func (c *fakeConn) ReloadConfig(_ *config.C) {} +func (c *fakeConn) SupportsMultipleReaders() bool { return true } +func (c *fakeConn) Close() error { c.closed = true; return nil } + +type multiqueueDevice struct { + *fakeDevice +} + +func (d *multiqueueDevice) SupportsMultiqueue() bool { return true } + +func TestControl_StartMultiqueueFailureReleases(t *testing.T) { + dev := &multiqueueDevice{fakeDevice: newFakeDevice()} + conn := &fakeConn{} + ctx, cancel := context.WithCancel(context.Background()) + f := &Interface{ + ctx: ctx, + inside: dev, + outside: conn, + writers: []udp.Conn{conn}, + readers: make([]io.ReadWriteCloser, 2), + routines: 2, + l: test.NewLogger(), + } + f.wg.Add(1) + + c := &Control{ + state: StateReady, + f: f, + l: test.NewLogger(), + ctx: ctx, + cancel: cancel, + } + + // The second reader fails to open, everything must be released + require.Error(t, c.Start()) + assert.Equal(t, StateStopped, c.State()) + assert.True(t, dev.closed, "the tun device should have been closed") + assert.True(t, conn.closed, "the udp socket should have been closed") + require.ErrorIs(t, c.ctx.Err(), context.Canceled) + + // And Wait must not hang on the construction token + require.NoError(t, c.Wait()) +} + +func TestInterface_CloseIsIdempotent(t *testing.T) { + dev := newFakeDevice() + f := &Interface{ + inside: dev, + l: test.NewLogger(), + } + f.wg.Add(1) + + require.NoError(t, f.Close()) + assert.True(t, dev.closed) + + // A second Close must not double release the wg token or the device + require.NoError(t, f.Close()) + require.NoError(t, f.wait()) +} + +func TestControl_FatalErrorReportsThroughWait(t *testing.T) { + c, dev, conn := newReadyControl(t) + + // Mirror what Start wires up, without needing real packet readers + c.f.triggerShutdown = c.Stop + c.state = StateStarted + + boom := errors.New("boom") + c.f.onFatal(boom) + + require.ErrorIs(t, c.Wait(), boom) + assert.Equal(t, StateStopped, c.State()) + assert.True(t, dev.closed) + assert.True(t, conn.closed) + + // A second fatal error must not fire the shutdown again or replace the first + c.f.onFatal(errors.New("later")) + require.ErrorIs(t, c.Wait(), boom) + + // Wait stays factual, a Stop after the death does not mask the error + c.Stop() + require.ErrorIs(t, c.Wait(), boom) +} + +func TestControl_ConcurrentStopAndStart(t *testing.T) { + c, _, _ := newReadyControl(t) + + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Go(func() { c.Stop() }) + } + wg.Go(func() { _ = c.Start() }) + wg.Go(func() { + _ = c.Wait() + // A returned Wait must always observe the final state, no matter how + // the race resolved + assert.Equal(t, StateStopped, c.State()) + }) + wg.Wait() + + // However the race resolves, the control must end fully stopped with no + // panic and Wait must observe the final state + require.NoError(t, c.Wait()) + assert.Equal(t, StateStopped, c.State()) + require.ErrorIs(t, c.Start(), ErrAlreadyStopped) +} + +func TestControl_StartStopLifecycle(t *testing.T) { + c, dev, conn := newReadyControl(t) + + require.NoError(t, c.Start()) + assert.Equal(t, StateStarted, c.State()) + require.ErrorIs(t, c.Start(), ErrAlreadyStarted) + + // Stop must unpark the reader blocked in the device and release everything + c.Stop() + assert.Equal(t, StateStopped, c.State()) + assert.True(t, dev.closed, "the tun device should have been closed") + assert.True(t, conn.closed, "the udp socket should have been closed") + require.ErrorIs(t, c.ctx.Err(), context.Canceled) + + // The reader drained off a closed device, that is not a fatal error + require.NoError(t, c.Wait()) + require.ErrorIs(t, c.Start(), ErrAlreadyStopped) +} + +func TestControl_RebindIsGatedByState(t *testing.T) { + c, _, conn := newReadyControl(t) + + // A rebind before Start reaches nothing, the interface is not up + c.RebindUDPServer() + assert.Equal(t, 0, conn.rebinds, "rebind before start must be a no-op") + + require.NoError(t, c.Start()) + c.RebindUDPServer() + assert.Equal(t, 1, conn.rebinds, "rebind while started must reach the conn") + + // A rebind racing a completed stop must not touch the closed conn + c.Stop() + require.NoError(t, c.Wait()) + c.RebindUDPServer() + assert.Equal(t, 1, conn.rebinds, "rebind after stop must be a no-op") +} diff --git a/control_test.go b/control_test.go index 847332b4..94ee4ee3 100644 --- a/control_test.go +++ b/control_test.go @@ -1,119 +1,121 @@ package nebula import ( + "bytes" + "log/slog" "net" + "net/netip" "reflect" "testing" - "time" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cert" - "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/test" - "github.com/slackhq/nebula/udp" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestControl_GetHostInfoByVpnIp(t *testing.T) { + //TODO: CERT-V2 with multiple certificate versions we have a problem with this test + // Some certs versions have different characteristics and each version implements their own Copy() func + // which means this is not a good place to test for exposing memory l := test.NewLogger() // Special care must be taken to re-use all objects provided to the hostmap and certificate in the expectedInfo object // To properly ensure we are not exposing core memory to the caller - hm := NewHostMap(l, &net.IPNet{}, make([]*net.IPNet, 0)) - remote1 := udp.NewAddr(net.ParseIP("0.0.0.100"), 4444) - remote2 := udp.NewAddr(net.ParseIP("1:2:3:4:5:6:7:8"), 4444) + hm := newHostMap(l) + hm.preferredRanges.Store(&[]netip.Prefix{}) + + remote1 := netip.MustParseAddrPort("0.0.0.100:4444") + remote2 := netip.MustParseAddrPort("[1:2:3:4:5:6:7:8]:4444") + ipNet := net.IPNet{ - IP: net.IPv4(1, 2, 3, 4), + IP: remote1.Addr().AsSlice(), Mask: net.IPMask{255, 255, 255, 0}, } ipNet2 := net.IPNet{ - IP: net.ParseIP("1:2:3:4:5:6:7:8"), + IP: remote2.Addr().AsSlice(), Mask: net.IPMask{255, 255, 255, 0}, } - crt := &cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "test", - Ips: []*net.IPNet{&ipNet}, - Subnets: []*net.IPNet{}, - Groups: []string{"default-group"}, - NotBefore: time.Unix(1, 0), - NotAfter: time.Unix(2, 0), - PublicKey: []byte{5, 6, 7, 8}, - IsCA: false, - Issuer: "the-issuer", - InvertedGroups: map[string]struct{}{"default-group": {}}, - }, - Signature: []byte{1, 2, 1, 2, 1, 3}, - } + remotes := NewRemoteList([]netip.Addr{netip.IPv4Unspecified()}, nil) + remotes.unlockedPrependV4(netip.IPv4Unspecified(), netAddrToProtoV4AddrPort(remote1.Addr(), remote1.Port())) + remotes.unlockedPrependV6(netip.IPv4Unspecified(), netAddrToProtoV6AddrPort(remote2.Addr(), remote2.Port())) + + vpnIp, ok := netip.AddrFromSlice(ipNet.IP) + assert.True(t, ok) - remotes := NewRemoteList(nil) - remotes.unlockedPrependV4(0, NewIp4AndPort(remote1.IP, uint32(remote1.Port))) - remotes.unlockedPrependV6(0, NewIp6AndPort(remote2.IP, uint32(remote2.Port))) - hm.unlockedAddHostInfo(&HostInfo{ - remote: remote1, + crt := &dummyCert{} + hi := &HostInfo{ remotes: remotes, ConnectionState: &ConnectionState{ - peerCert: crt, + peerCert: &cert.CachedCertificate{Certificate: crt}, }, remoteIndexId: 200, localIndexId: 201, - vpnIp: iputil.Ip2VpnIp(ipNet.IP), + vpnAddrs: []netip.Addr{vpnIp}, relayState: RelayState{ - relays: map[iputil.VpnIp]struct{}{}, - relayForByIp: map[iputil.VpnIp]*Relay{}, - relayForByIdx: map[uint32]*Relay{}, + relays: nil, + relayForByAddr: map[netip.Addr]*Relay{}, + relayForByIdx: map[uint32]*Relay{}, }, - }, &Interface{}) + } + hi.remote.Store(&remote1) + hm.unlockedAddHostInfo(hi, &Interface{}) - hm.unlockedAddHostInfo(&HostInfo{ - remote: remote1, + vpnIp2, ok := netip.AddrFromSlice(ipNet2.IP) + assert.True(t, ok) + + hi2 := &HostInfo{ remotes: remotes, ConnectionState: &ConnectionState{ peerCert: nil, }, remoteIndexId: 200, localIndexId: 201, - vpnIp: iputil.Ip2VpnIp(ipNet2.IP), + vpnAddrs: []netip.Addr{vpnIp2}, relayState: RelayState{ - relays: map[iputil.VpnIp]struct{}{}, - relayForByIp: map[iputil.VpnIp]*Relay{}, - relayForByIdx: map[uint32]*Relay{}, + relays: nil, + relayForByAddr: map[netip.Addr]*Relay{}, + relayForByIdx: map[uint32]*Relay{}, }, - }, &Interface{}) + } + hi2.remote.Store(&remote1) + hm.unlockedAddHostInfo(hi2, &Interface{}) c := Control{ + state: StateReady, f: &Interface{ hostMap: hm, }, - l: logrus.New(), + l: test.NewLogger(), } - thi := c.GetHostInfoByVpnIp(iputil.Ip2VpnIp(ipNet.IP), false) + thi := c.GetHostInfoByVpnAddr(vpnIp, false) expectedInfo := ControlHostInfo{ - VpnIp: net.IPv4(1, 2, 3, 4).To4(), + VpnAddrs: []netip.Addr{vpnIp}, LocalIndex: 201, RemoteIndex: 200, - RemoteAddrs: []*udp.Addr{remote2, remote1}, + RemoteAddrs: []netip.AddrPort{remote2, remote1}, Cert: crt.Copy(), MessageCounter: 0, - CurrentRemote: udp.NewAddr(net.ParseIP("0.0.0.100"), 4444), - CurrentRelaysToMe: []iputil.VpnIp{}, - CurrentRelaysThroughMe: []iputil.VpnIp{}, + CurrentRemote: remote1, + CurrentRelaysToMe: []netip.Addr{}, + CurrentRelaysThroughMe: []netip.Addr{}, } // Make sure we don't have any unexpected fields - assertFields(t, []string{"VpnIp", "LocalIndex", "RemoteIndex", "RemoteAddrs", "Cert", "MessageCounter", "CurrentRemote", "CurrentRelaysToMe", "CurrentRelaysThroughMe"}, thi) + assertFields(t, []string{"VpnAddrs", "LocalIndex", "RemoteIndex", "RemoteAddrs", "Cert", "MessageCounter", "CurrentRemote", "CurrentRelaysToMe", "CurrentRelaysThroughMe"}, thi) + assert.Equal(t, &expectedInfo, thi) test.AssertDeepCopyEqual(t, &expectedInfo, thi) // Make sure we don't panic if the host info doesn't have a cert yet assert.NotPanics(t, func() { - thi = c.GetHostInfoByVpnIp(iputil.Ip2VpnIp(ipNet2.IP), false) + thi = c.GetHostInfoByVpnAddr(vpnIp2, false) }) } -func assertFields(t *testing.T, expected []string, actualStruct interface{}) { +func assertFields(t *testing.T, expected []string, actualStruct any) { val := reflect.ValueOf(actualStruct).Elem() fields := make([]string, val.NumField()) for i := 0; i < val.NumField(); i++ { @@ -122,3 +124,153 @@ func assertFields(t *testing.T, expected []string, actualStruct interface{}) { assert.Equal(t, expected, fields) } + +// alwaysAllowV4/V6 are check funcs that accept every entry (including nil pointers), +// letting us inject a nil *V4AddrPort/*V6AddrPort into a RemoteList's reported cache +// the same way a malformed proto message off the wire could. +func alwaysAllowV4(netip.Addr, *V4AddrPort) bool { return true } +func alwaysAllowV6(netip.Addr, *V6AddrPort) bool { return true } + +// TestGetRelays_SkipsNilRelayAddrs proves GetRelays tolerates nil entries in the +// RelayVpnAddrs proto slice (which protoAddrToNetAddr would nil-deref on) and still +// returns the valid relays, including the legacy OldRelayVpnAddrs. +func TestGetRelays_SkipsNilRelayAddrs(t *testing.T) { + good := netip.MustParseAddr("10.0.0.9") + + d := &NebulaMetaDetails{ + OldRelayVpnAddrs: []uint32{0x0a000001}, // 10.0.0.1 + RelayVpnAddrs: []*Addr{ + nil, + netAddrToProtoAddr(good), + nil, + }, + } + + var relays []netip.Addr + require.NotPanics(t, func() { relays = d.GetRelays() }) + + assert.Equal(t, []netip.Addr{ + netip.MustParseAddr("10.0.0.1"), + good, + }, relays) +} + +// TestGetRelays_AllNil ensures an all-nil RelayVpnAddrs slice yields no relays and no panic. +func TestGetRelays_AllNil(t *testing.T) { + d := &NebulaMetaDetails{RelayVpnAddrs: []*Addr{nil, nil}} + var relays []netip.Addr + require.NotPanics(t, func() { relays = d.GetRelays() }) + assert.Empty(t, relays) +} + +// TestRemoteList_CopyCache_SkipsNilReported proves CopyCache skips nil reported +// pointers (v4 and v6) instead of nil-dereferencing them in protoV*AddrPortToNetAddrPort. +func TestRemoteList_CopyCache_SkipsNilReported(t *testing.T) { + owner := netip.MustParseAddr("10.0.0.1") + rl := NewRemoteList([]netip.Addr{owner}, nil) + + rl.unlockedSetV4(owner, owner, []*V4AddrPort{ + nil, + newIp4AndPortFromString("1.2.3.4:5"), + nil, + }, alwaysAllowV4) + + rl.unlockedSetV6(owner, owner, []*V6AddrPort{ + nil, + newIp6AndPortFromString("[1::1]:6"), + nil, + }, alwaysAllowV6) + + var cm *CacheMap + require.NotPanics(t, func() { cm = rl.CopyCache() }) + + c := (*cm)[owner.String()] + require.NotNil(t, c) + assert.ElementsMatch(t, []netip.AddrPort{ + netip.MustParseAddrPort("1.2.3.4:5"), + netip.MustParseAddrPort("[1::1]:6"), + }, c.Reported) +} + +// TestRemoteList_Rebuild_SkipsNilReported drives unlockedCollect (via Rebuild) with +// nil reported entries and confirms only the valid addresses survive, with no panic. +func TestRemoteList_Rebuild_SkipsNilReported(t *testing.T) { + owner := netip.MustParseAddr("10.0.0.1") + rl := NewRemoteList([]netip.Addr{owner}, nil) + + rl.unlockedSetV4(owner, owner, []*V4AddrPort{ + nil, + newIp4AndPortFromString("1.2.3.4:5"), + }, alwaysAllowV4) + rl.unlockedSetV6(owner, owner, []*V6AddrPort{ + newIp6AndPortFromString("[1::1]:6"), + nil, + }, alwaysAllowV6) + + require.NotPanics(t, func() { rl.Rebuild([]netip.Prefix{}) }) + + assert.ElementsMatch(t, []netip.AddrPort{ + netip.MustParseAddrPort("1.2.3.4:5"), + netip.MustParseAddrPort("[1::1]:6"), + }, rl.addrs) +} + +// newRelayControl marshals a NebulaControl the way it arrives on the wire so we can feed +// it through HandleControlMsg's unmarshal + validate path. +func newRelayControl(t *testing.T, typ NebulaControl_MessageType, from, to *Addr) []byte { + t.Helper() + msg := &NebulaControl{ + Type: typ, + RelayFromAddr: from, + RelayToAddr: to, + } + b, err := msg.Marshal() + require.NoError(t, err) + return b +} + +// TestRelayManager_HandleControlMsg_NilRelayAddrs verifies the validation block added to +// HandleControlMsg: CreateRelay{Request,Response} carrying a nil RelayFromAddr or +// RelayToAddr are dropped with a debug log rather than nil-dereferencing downstream. +func TestRelayManager_HandleControlMsg_NilRelayAddrs(t *testing.T) { + good := netAddrToProtoAddr(netip.MustParseAddr("10.0.0.9")) + + cases := []struct { + name string + typ NebulaControl_MessageType + from *Addr + to *Addr + wantLog string // debug substring expected, "" == expect no drop log + }{ + {"request nil from", NebulaControl_CreateRelayRequest, nil, good, "nil RelayFromAddr"}, + {"request nil to", NebulaControl_CreateRelayRequest, good, nil, "nil RelayToAddr"}, + {"request both nil", NebulaControl_CreateRelayRequest, nil, nil, "nil RelayFromAddr"}, + {"response nil from", NebulaControl_CreateRelayResponse, nil, good, "nil RelayFromAddr"}, + {"response nil to", NebulaControl_CreateRelayResponse, good, nil, "nil RelayToAddr"}, + // A non-relay control type is not subject to the relay-addr validation and must + // pass through it untouched (the final switch simply no-ops on it). + {"unrelated type nil addrs", NebulaControl_None, nil, nil, ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + l := test.NewLoggerWithOutputAndLevel(&buf, slog.LevelDebug) + rm := &relayManager{l: l, hostmap: newHostMap(l)} + rm.useRelays.Store(true) + + f := &Interface{l: l} + h := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}, localIndexId: 1} + + d := newRelayControl(t, tc.typ, tc.from, tc.to) + + require.NotPanics(t, func() { rm.HandleControlMsg(h, d, f) }) + + if tc.wantLog == "" { + assert.NotContains(t, buf.String(), "nil Relay") + } else { + assert.Contains(t, buf.String(), tc.wantLog) + } + }) + } +} diff --git a/control_tester.go b/control_tester.go index b786ba38..546b9e87 100644 --- a/control_tester.go +++ b/control_tester.go @@ -1,17 +1,11 @@ //go:build e2e_testing -// +build e2e_testing package nebula import ( - "net" + "net/netip" - "github.com/slackhq/nebula/cert" - - "github.com/google/gopacket" - "github.com/google/gopacket/layers" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/udp" ) @@ -26,7 +20,9 @@ func (c *Control) WaitForType(msgType header.MessageType, subType header.Message panic(err) } pipeTo.InjectUDPPacket(p) - if h.Type == msgType && h.Subtype == subType { + match := h.Type == msgType && h.Subtype == subType + p.Release() + if match { return } } @@ -42,7 +38,9 @@ func (c *Control) WaitForTypeByIndex(toIndex uint32, msgType header.MessageType, panic(err) } pipeTo.InjectUDPPacket(p) - if h.RemoteIndex == toIndex && h.Type == msgType && h.Subtype == subType { + match := h.RemoteIndex == toIndex && h.Type == msgType && h.Subtype == subType + p.Release() + if match { return } } @@ -50,37 +48,30 @@ func (c *Control) WaitForTypeByIndex(toIndex uint32, msgType header.MessageType, // InjectLightHouseAddr will push toAddr into the local lighthouse cache for the vpnIp // This is necessary if you did not configure static hosts or are not running a lighthouse -func (c *Control) InjectLightHouseAddr(vpnIp net.IP, toAddr *net.UDPAddr) { +func (c *Control) InjectLightHouseAddr(vpnIp netip.Addr, toAddr netip.AddrPort) { c.f.lightHouse.Lock() - remoteList := c.f.lightHouse.unlockedGetRemoteList(iputil.Ip2VpnIp(vpnIp)) + remoteList := c.f.lightHouse.unlockedGetRemoteList([]netip.Addr{vpnIp}) remoteList.Lock() defer remoteList.Unlock() c.f.lightHouse.Unlock() - iVpnIp := iputil.Ip2VpnIp(vpnIp) - if v4 := toAddr.IP.To4(); v4 != nil { - remoteList.unlockedPrependV4(iVpnIp, NewIp4AndPort(v4, uint32(toAddr.Port))) + if toAddr.Addr().Is4() { + remoteList.unlockedPrependV4(vpnIp, netAddrToProtoV4AddrPort(toAddr.Addr(), toAddr.Port())) } else { - remoteList.unlockedPrependV6(iVpnIp, NewIp6AndPort(toAddr.IP, uint32(toAddr.Port))) + remoteList.unlockedPrependV6(vpnIp, netAddrToProtoV6AddrPort(toAddr.Addr(), toAddr.Port())) } } // InjectRelays will push relayVpnIps into the local lighthouse cache for the vpnIp // This is necessary to inform an initiator of possible relays for communicating with a responder -func (c *Control) InjectRelays(vpnIp net.IP, relayVpnIps []net.IP) { +func (c *Control) InjectRelays(vpnIp netip.Addr, relayVpnIps []netip.Addr) { c.f.lightHouse.Lock() - remoteList := c.f.lightHouse.unlockedGetRemoteList(iputil.Ip2VpnIp(vpnIp)) + remoteList := c.f.lightHouse.unlockedGetRemoteList([]netip.Addr{vpnIp}) remoteList.Lock() defer remoteList.Unlock() c.f.lightHouse.Unlock() - iVpnIp := iputil.Ip2VpnIp(vpnIp) - uVpnIp := []uint32{} - for _, rVPnIp := range relayVpnIps { - uVpnIp = append(uVpnIp, uint32(iputil.Ip2VpnIp(rVPnIp))) - } - - remoteList.unlockedSetRelay(iVpnIp, iVpnIp, uVpnIp) + remoteList.unlockedSetRelay(vpnIp, relayVpnIps) } // GetFromTun will pull a packet off the tun side of nebula @@ -101,53 +92,39 @@ func (c *Control) GetTunTxChan() <-chan []byte { return c.f.inside.(*overlay.TestTun).TxPackets } -// InjectUDPPacket will inject a packet into the udp side of nebula +// InjectUDPPacket injects a packet into the udp side. We copy internally so the caller keeps ownership of p. +// The copy comes from the freelist so steady-state alloc is zero. func (c *Control) InjectUDPPacket(p *udp.Packet) { - c.f.outside.(*udp.TesterConn).Send(p) + c.f.outside.(*udp.TesterConn).Send(p.Copy()) } -// InjectTunUDPPacket puts a udp packet on the tun interface. Using UDP here because it's a simpler protocol -func (c *Control) InjectTunUDPPacket(toIp net.IP, toPort uint16, fromPort uint16, data []byte) { - ip := layers.IPv4{ - Version: 4, - TTL: 64, - Protocol: layers.IPProtocolUDP, - SrcIP: c.f.inside.Cidr().IP, - DstIP: toIp, - } - - udp := layers.UDP{ - SrcPort: layers.UDPPort(fromPort), - DstPort: layers.UDPPort(toPort), - } - err := udp.SetNetworkLayerForChecksum(&ip) - if err != nil { - panic(err) - } +// InjectTunPacket pushes an IP packet onto the tun interface. +func (c *Control) InjectTunPacket(packet []byte) { + c.f.inside.(*overlay.TestTun).Send(packet) +} - buffer := gopacket.NewSerializeBuffer() - opt := gopacket.SerializeOptions{ - ComputeChecksums: true, - FixLengths: true, - } - err = gopacket.SerializeLayers(buffer, opt, &ip, &udp, gopacket.Payload(data)) - if err != nil { - panic(err) - } +func (c *Control) GetVpnAddrs() []netip.Addr { + return c.f.myVpnAddrs +} - c.f.inside.(*overlay.TestTun).Send(buffer.Bytes()) +func (c *Control) GetUDPAddr() netip.AddrPort { + return c.f.outside.(*udp.TesterConn).GetAddr() } -func (c *Control) GetVpnIp() iputil.VpnIp { - return c.f.myVpnIp +// SetUDPAddr moves this node to a new underlay address, standing in for a laptop waking up on a different +// network. Register the new address with the router as well or nothing will route back. +func (c *Control) SetUDPAddr(addr netip.AddrPort) { + c.f.outside.(*udp.TesterConn).SetAddr(addr) } -func (c *Control) GetUDPAddr() string { - return c.f.outside.(*udp.TesterConn).Addr.String() +// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of +// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker. +func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) { + c.f.lightHouse.localAddrsFn = fn } -func (c *Control) KillPendingTunnel(vpnIp net.IP) bool { - hostinfo := c.f.handshakeManager.QueryVpnIp(iputil.Ip2VpnIp(vpnIp)) +func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool { + hostinfo := c.f.handshakeManager.QueryVpnAddr(vpnIp) if hostinfo == nil { return false } @@ -160,10 +137,22 @@ func (c *Control) GetHostmap() *HostMap { return c.f.hostMap } -func (c *Control) GetCert() *cert.NebulaCertificate { - return c.f.pki.GetCertState().Certificate +// GetHostmapIndexCount returns the number of entries in the main hostmap Indexes table, holding +// the hostmap read lock so tests can poll it while connection manager churns tunnels. +func (c *Control) GetHostmapIndexCount() int { + c.f.hostMap.RLock() + defer c.f.hostMap.RUnlock() + return len(c.f.hostMap.Indexes) +} + +func (c *Control) GetF() *Interface { + return c.f +} + +func (c *Control) GetCertState() *CertState { + return c.f.pki.getCertState() } -func (c *Control) ReHandshake(vpnIp iputil.VpnIp) { +func (c *Control) ReHandshake(vpnIp netip.Addr) { c.f.handshakeManager.StartHandshake(vpnIp, nil) } diff --git a/dist/arch/nebula.service b/dist/arch/nebula.service deleted file mode 100644 index 831c71a5..00000000 --- a/dist/arch/nebula.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Nebula overlay networking tool -Wants=basic.target network-online.target nss-lookup.target time-sync.target -After=basic.target network.target network-online.target - -[Service] -Type=notify -NotifyAccess=main -SyslogIdentifier=nebula -ExecReload=/bin/kill -HUP $MAINPID -ExecStart=/usr/bin/nebula -config /etc/nebula/config.yml -Restart=always - -[Install] -WantedBy=multi-user.target diff --git a/dist/fedora/nebula.service b/dist/fedora/nebula.service deleted file mode 100644 index 0f947ead..00000000 --- a/dist/fedora/nebula.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Nebula overlay networking tool -Wants=basic.target network-online.target nss-lookup.target time-sync.target -After=basic.target network.target network-online.target -Before=sshd.service - -[Service] -Type=notify -NotifyAccess=main -SyslogIdentifier=nebula -ExecReload=/bin/kill -HUP $MAINPID -ExecStart=/usr/bin/nebula -config /etc/nebula/config.yml -Restart=always - -[Install] -WantedBy=multi-user.target diff --git a/dist/wireshark/nebula.lua b/dist/wireshark/nebula.lua index ddc808f9..5c7c17f1 100644 --- a/dist/wireshark/nebula.lua +++ b/dist/wireshark/nebula.lua @@ -62,7 +62,7 @@ function nebula.dissector(tvbuf, pktinfo, root) tree:add(pf_version, tvbuf:range(0,1)) local type = tree:add(pf_type, tvbuf:range(0,1)) - local nebula_type = bit32.band(tvbuf:range(0,1):uint(), 0x0F) + local nebula_type = bit.band(tvbuf:range(0,1):uint(), 0x0F) if nebula_type == 0 then local stage = tvbuf(8,8):uint64() tree:add(pf_subtype_handshake, tvbuf:range(1,1)) @@ -84,30 +84,24 @@ end function nebula.prefs_changed() if default_settings.all_ports == nebula.prefs.all_ports and default_settings.port == nebula.prefs.port then - -- Nothing changed, bail return end - -- Remove our old dissector + -- Remove all existing registrations DissectorTable.get("udp.port"):remove_all(nebula) - if nebula.prefs.all_ports and default_settings.all_ports ~= nebula.prefs.all_ports then - default_settings.all_port = nebula.prefs.all_ports - + if nebula.prefs.all_ports then + -- Register on every port for hole punch capture for i=0, 65535 do DissectorTable.get("udp.port"):add(i, nebula) end - - -- no need to establish again on specific ports - return + else + -- Register on the configured port only + DissectorTable.get("udp.port"):add(nebula.prefs.port, nebula) end - - if default_settings.all_ports ~= nebula.prefs.all_ports then - -- Add our new port dissector - default_settings.port = nebula.prefs.port - DissectorTable.get("udp.port"):add(default_settings.port, nebula) - end + default_settings.all_ports = nebula.prefs.all_ports + default_settings.port = nebula.prefs.port end DissectorTable.get("udp.port"):add(default_settings.port, nebula) diff --git a/dns_server.go b/dns_server.go index 3109b4cf..9339b068 100644 --- a/dns_server.go +++ b/dns_server.go @@ -1,93 +1,426 @@ package nebula import ( + "context" "fmt" + "log/slog" "net" + "net/netip" "strconv" "strings" "sync" + "sync/atomic" "github.com/miekg/dns" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" ) -// This whole thing should be rewritten to use context - -var dnsR *dnsRecords -var dnsServer *dns.Server -var dnsAddr string - -type dnsRecords struct { +type dnsServer struct { sync.RWMutex - dnsMap map[string]string + l *slog.Logger + ctx context.Context + dnsMap4 map[string]netip.Addr + dnsMap6 map[string]netip.Addr hostMap *HostMap + pki *PKI + + // selfHost is the cached FQDN we last seeded for ourselves + selfHost string + + mux *dns.ServeMux + + // enabled mirrors `lighthouse.serve_dns && lighthouse.am_lighthouse`. + // Start, Add, and reload consult it so callers don't need to know the + // gating rules. When it toggles off via reload, accumulated records are + // cleared so a later re-enable starts with a fresh map populated from + // new handshakes. + enabled atomic.Bool + + serverMu sync.Mutex + server *dns.Server + // started is closed once `server` has finished binding (or after + // ListenAndServe returns on a bind failure). Stop waits on it before + // calling Shutdown to avoid the miekg/dns "server not started" race + // where a Shutdown that arrives before bind completes is silently + // ignored, leaving the listener running forever. + started chan struct{} + addr string } -func newDnsRecords(hostMap *HostMap) *dnsRecords { - return &dnsRecords{ - dnsMap: make(map[string]string), +// newDnsServerFromConfig builds a dnsServer, applies the initial config, and +// registers a reload callback. The reload callback is registered before the +// initial config is applied, so a SIGHUP can later enable, fix, or disable +// DNS even if the initial application failed. +// +// The dnsServer internally gates on `lighthouse.serve_dns && +// lighthouse.am_lighthouse`. Start and Add are safe to call unconditionally, +// they no-op when DNS isn't enabled. Each Start invocation owns a ctx-cancel +// watcher that tears the listener down on nebula shutdown. The returned +// pointer is always non-nil, even on error. +func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostMap *HostMap, c *config.C) (*dnsServer, error) { + ds := &dnsServer{ + l: l, + ctx: ctx, + dnsMap4: make(map[string]netip.Addr), + dnsMap6: make(map[string]netip.Addr), hostMap: hostMap, + pki: pki, + } + ds.mux = dns.NewServeMux() + ds.mux.HandleFunc(".", ds.handleDnsRequest) + + c.RegisterReloadCallback(func(c *config.C) { + if err := ds.reload(c, false); err != nil { + ds.l.Error("Failed to reload DNS responder from config", "error", err) + } + }) + + if err := ds.reload(c, true); err != nil { + return ds, err + } + ds.seedSelf() + return ds, nil +} + +// reload applies the latest config and reconciles the running state with it: +// - enabled toggled on -> spawn a runner +// - enabled toggled off -> stop the runner +// - listen address changed (while running) -> restart on the new address +// - everything else -> no-op +// +// On the initial call it only records configuration; Control.Start is what +// launches the first runner via dnsStart. +func (d *dnsServer) reload(c *config.C, initial bool) error { + wantsDns := c.GetBool("lighthouse.serve_dns", false) + amLighthouse := c.GetBool("lighthouse.am_lighthouse", false) + enabled := wantsDns && amLighthouse + newAddr := getDnsServerAddr(c) + + d.serverMu.Lock() + running := d.server != nil + sameAddr := d.addr == newAddr + d.addr = newAddr + d.enabled.Store(enabled) + d.serverMu.Unlock() + + if initial { + if wantsDns && !amLighthouse { + d.l.Warn("DNS server refusing to run because this host is not a lighthouse.") + } + return nil + } + + if !enabled { + if running { + d.Stop() + } + // Drop any records that accumulated while enabled; a later re-enable + // will repopulate from fresh handshakes and a fresh seedSelf. + d.clearRecords() + return nil + } + + if !running { + // Was disabled (or never started); bring it up now. + go d.Start() + } else if !sameAddr { + // Stop clears the slot before shutting down, otherwise the Start below can find the dying server and refuse + d.Stop() + go d.Start() + } + + // Refresh the self entry every enabled reload so cert renewals that change our name or VPN addresses are picked up. + d.seedSelf() + return nil +} + +// shutdownServer waits for the server to finish binding (so Shutdown actually +// stops it rather than no-oping) and then shuts it down. +func (d *dnsServer) shutdownServer(srv *dns.Server, started chan struct{}, reason string) { + if srv == nil { + return + } + if started != nil { + <-started + } + if err := srv.Shutdown(); err != nil { + d.l.Warn("Failed to shut down the DNS responder", "reason", reason, "error", err) + } +} + +// Start binds and serves the DNS responder. Blocks until Stop is called or +// the listener errors. Safe to call when DNS is disabled (returns +// immediately). This is what Control.dnsStart points at. +// +// Must be invoked after the tun device is active so that lighthouse.dns.host +// may bind to a nebula IP. +func (d *dnsServer) Start() { + if !d.enabled.Load() { + return + } + + started := make(chan struct{}) + d.serverMu.Lock() + // Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too. + // Two reloads in quick succession can both spawn a Start, the loser would orphan the live listener past Stop + if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() { + d.serverMu.Unlock() + return + } + addr := d.addr + server := &dns.Server{ + Addr: addr, + Net: "udp", + Handler: d.mux, + NotifyStartedFunc: func() { close(started) }, + } + d.server = server + d.started = started + d.serverMu.Unlock() + + // Per-invocation ctx watcher. Exits when Start does, so we don't leak a + // watcher per reload-driven restart. + done := make(chan struct{}) + go func() { + select { + case <-d.ctx.Done(): + d.shutdownServer(server, started, "shutdown") + case <-done: + } + }() + + d.l.Info("Starting DNS responder", "dnsListener", addr) + err := server.ListenAndServe() + close(done) + + // If the listener never bound (bind error) NotifyStartedFunc never fires, + // so close started here to release any Stop caller waiting on it. + select { + case <-started: + default: + close(started) } + + // Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start + d.serverMu.Lock() + if d.server == server { + d.server = nil + d.started = nil + } + d.serverMu.Unlock() + + if err != nil { + d.l.Warn("Failed to run the DNS responder", "error", err) + } +} + +// Stop shuts down the active server, if any. Idempotent. +func (d *dnsServer) Stop() { + d.serverMu.Lock() + srv := d.server + started := d.started + d.server = nil + d.started = nil + d.serverMu.Unlock() + d.shutdownServer(srv, started, "stop") } -func (d *dnsRecords) Query(data string) string { +// Query returns the address for the given name and query type. The second +// return value reports whether the name is known at all (in either A or AAAA), +// which lets callers distinguish NODATA from NXDOMAIN. +func (d *dnsServer) Query(q uint16, data string) (netip.Addr, bool) { + data = strings.ToLower(data) d.RLock() defer d.RUnlock() - if r, ok := d.dnsMap[strings.ToLower(data)]; ok { - return r + addr4, haveV4 := d.dnsMap4[data] + addr6, haveV6 := d.dnsMap6[data] + nameExists := haveV4 || haveV6 + switch q { + case dns.TypeA: + if haveV4 { + return addr4, nameExists + } + case dns.TypeAAAA: + if haveV6 { + return addr6, nameExists + } } - return "" + + return netip.Addr{}, nameExists } -func (d *dnsRecords) QueryCert(data string) string { - ip := net.ParseIP(data[:len(data)-1]) - if ip == nil { +func (d *dnsServer) QueryCert(data string) string { + if len(data) < 2 { return "" } - iip := iputil.Ip2VpnIp(ip) - hostinfo := d.hostMap.QueryVpnIp(iip) + ip, err := netip.ParseAddr(data[:len(data)-1]) + if err != nil { + return "" + } + + // The hostmap only ever contains peers we have handshaked with, so it never carries an entry for ourselves. + // Answer self lookups straight from the local cert state. + if cs := d.certState(); cs != nil && cs.myVpnAddrsTable != nil && cs.myVpnAddrsTable.Contains(ip) { + c := cs.GetDefaultCertificate() + if c == nil { + return "" + } + b, err := c.MarshalJSON() + if err != nil { + return "" + } + return string(b) + } + + hostinfo := d.hostMap.QueryVpnAddr(ip) if hostinfo == nil { return "" } + q := hostinfo.GetCert() if q == nil { return "" } - cert := q.Details - c := fmt.Sprintf("\"Name: %s\" \"Ips: %s\" \"Subnets %s\" \"Groups %s\" \"NotBefore %s\" \"NotAFter %s\" \"PublicKey %x\" \"IsCA %t\" \"Issuer %s\"", cert.Name, cert.Ips, cert.Subnets, cert.Groups, cert.NotBefore, cert.NotAfter, cert.PublicKey, cert.IsCA, cert.Issuer) - return c + + b, err := q.Certificate.MarshalJSON() + if err != nil { + return "" + } + return string(b) +} + +// clearRecords drops all DNS records, including the self entry. +func (d *dnsServer) clearRecords() { + d.Lock() + defer d.Unlock() + clear(d.dnsMap4) + clear(d.dnsMap6) + d.selfHost = "" } -func (d *dnsRecords) Add(host, data string) { +// seedSelf inserts (or refreshes) a record for our own cert name pointing at our VPN addresses, +// so a single-lighthouse network can resolve the lighthouse's own hostname without the two-process workaround. +func (d *dnsServer) seedSelf() { + if !d.enabled.Load() { + return + } + cs := d.certState() + if cs == nil { + return + } + c := cs.GetDefaultCertificate() + if c == nil { + return + } + newHost := strings.ToLower(c.Name()) + "." + d.Lock() defer d.Unlock() - d.dnsMap[strings.ToLower(host)] = data + if d.selfHost != "" && d.selfHost != newHost { + delete(d.dnsMap4, d.selfHost) + delete(d.dnsMap6, d.selfHost) + } + d.selfHost = newHost + delete(d.dnsMap4, newHost) + delete(d.dnsMap6, newHost) + haveV4, haveV6 := false, false + for _, addr := range cs.myVpnAddrs { + if addr.Is4() && !haveV4 { + d.dnsMap4[newHost] = addr + haveV4 = true + } else if addr.Is6() && !haveV6 { + d.dnsMap6[newHost] = addr + haveV6 = true + } + if haveV4 && haveV6 { + break + } + } +} + +func (d *dnsServer) certState() *CertState { + if d.pki == nil { + return nil + } + return d.pki.getCertState() +} + +// Add adds the first IPv4 and IPv6 address that appears in `addresses` as the record for `host` +func (d *dnsServer) Add(host string, addresses []netip.Addr) { + if !d.enabled.Load() { + return + } + host = strings.ToLower(host) + d.Lock() + defer d.Unlock() + haveV4 := false + haveV6 := false + for _, addr := range addresses { + if addr.Is4() && !haveV4 { + d.dnsMap4[host] = addr + haveV4 = true + } else if addr.Is6() && !haveV6 { + d.dnsMap6[host] = addr + haveV6 = true + } + if haveV4 && haveV6 { + break + } + } } -func parseQuery(l *logrus.Logger, m *dns.Msg, w dns.ResponseWriter) { +func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool { + a, _, _ := net.SplitHostPort(addr) + b, err := netip.ParseAddr(a) + if err != nil { + return false + } + + if b.IsLoopback() { + return true + } + + cs := d.certState() + if cs == nil || cs.myVpnAddrsTable == nil { + return false + } + //if we found it in this table, it's good + return cs.myVpnAddrsTable.Contains(b) +} + +func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) { + debugEnabled := d.l.Enabled(context.Background(), slog.LevelDebug) + // Per RFC 2308 §2.2, a name that exists but has no record of the requested + // type must be answered with NOERROR and an empty answer section (NODATA), + // not NXDOMAIN (RFC 2308 §2.1), which is reserved for names that do not + // exist at all. + anyNameExists := false for _, q := range m.Question { switch q.Qtype { - case dns.TypeA: - l.Debugf("Query for A %s", q.Name) - ip := dnsR.Query(q.Name) - if ip != "" { - rr, err := dns.NewRR(fmt.Sprintf("%s A %s", q.Name, ip)) + case dns.TypeA, dns.TypeAAAA: + qType := dns.TypeToString[q.Qtype] + if debugEnabled { + d.l.Debug("DNS query", "type", qType, "name", q.Name) + } + ip, nameExists := d.Query(q.Qtype, q.Name) + if nameExists { + anyNameExists = true + } + if ip.IsValid() { + rr, err := dns.NewRR(fmt.Sprintf("%s %s %s", q.Name, qType, ip)) if err == nil { m.Answer = append(m.Answer, rr) } } case dns.TypeTXT: - a, _, _ := net.SplitHostPort(w.RemoteAddr().String()) - b := net.ParseIP(a) - // We don't answer these queries from non nebula nodes or localhost - //l.Debugf("Does %s contain %s", b, dnsR.hostMap.vpnCIDR) - if !dnsR.hostMap.vpnCIDR.Contains(b) && a != "127.0.0.1" { + // We only answer these queries from nebula nodes or localhost + if !d.isSelfNebulaOrLocalhost(w.RemoteAddr().String()) { return } - l.Debugf("Query for TXT %s", q.Name) - ip := dnsR.QueryCert(q.Name) + if debugEnabled { + d.l.Debug("DNS query", "type", "TXT", "name", q.Name) + } + ip := d.QueryCert(q.Name) if ip != "" { rr, err := dns.NewRR(fmt.Sprintf("%s TXT %s", q.Name, ip)) if err == nil { @@ -96,60 +429,30 @@ func parseQuery(l *logrus.Logger, m *dns.Msg, w dns.ResponseWriter) { } } } + + if len(m.Answer) == 0 && !anyNameExists { + m.Rcode = dns.RcodeNameError + } } -func handleDnsRequest(l *logrus.Logger, w dns.ResponseWriter, r *dns.Msg) { +func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) { m := new(dns.Msg) m.SetReply(r) m.Compress = false switch r.Opcode { case dns.OpcodeQuery: - parseQuery(l, m, w) + d.parseQuery(m, w) } w.WriteMsg(m) } -func dnsMain(l *logrus.Logger, hostMap *HostMap, c *config.C) func() { - dnsR = newDnsRecords(hostMap) - - // attach request handler func - dns.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { - handleDnsRequest(l, w, r) - }) - - c.RegisterReloadCallback(func(c *config.C) { - reloadDns(l, c) - }) - - return func() { - startDns(l, c) - } -} - func getDnsServerAddr(c *config.C) string { - return c.GetString("lighthouse.dns.host", "") + ":" + strconv.Itoa(c.GetInt("lighthouse.dns.port", 53)) -} - -func startDns(l *logrus.Logger, c *config.C) { - dnsAddr = getDnsServerAddr(c) - dnsServer = &dns.Server{Addr: dnsAddr, Net: "udp"} - l.WithField("dnsListener", dnsAddr).Info("Starting DNS responder") - err := dnsServer.ListenAndServe() - defer dnsServer.Shutdown() - if err != nil { - l.Errorf("Failed to start server: %s\n ", err.Error()) + dnsHost := strings.TrimSpace(c.GetString("lighthouse.dns.host", "")) + // Old guidance was to provide the literal `[::]` in `lighthouse.dns.host` but that won't resolve. + if dnsHost == "[::]" { + dnsHost = "::" } -} - -func reloadDns(l *logrus.Logger, c *config.C) { - if dnsAddr == getDnsServerAddr(c) { - l.Debug("No DNS server config change detected") - return - } - - l.Debug("Restarting DNS server") - dnsServer.Shutdown() - go startDns(l, c) + return net.JoinHostPort(dnsHost, strconv.Itoa(c.GetInt("lighthouse.dns.port", 53))) } diff --git a/dns_server_test.go b/dns_server_test.go index 830dc8a8..73267db2 100644 --- a/dns_server_test.go +++ b/dns_server_test.go @@ -1,19 +1,631 @@ package nebula import ( + "context" + "log/slog" + "net" + "net/netip" + "strconv" "testing" + "time" + "github.com/gaissmai/bart" "github.com/miekg/dns" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +type stubDNSWriter struct{} + +func (stubDNSWriter) LocalAddr() net.Addr { return &net.UDPAddr{} } +func (stubDNSWriter) RemoteAddr() net.Addr { + return &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 5353} +} +func (stubDNSWriter) Write([]byte) (int, error) { return 0, nil } +func (stubDNSWriter) WriteMsg(*dns.Msg) error { return nil } +func (stubDNSWriter) Close() error { return nil } +func (stubDNSWriter) TsigStatus() error { return nil } +func (stubDNSWriter) TsigTimersOnly(bool) {} +func (stubDNSWriter) Hijack() {} + func TestParsequery(t *testing.T) { - //TODO: This test is basically pointless + l := slog.New(slog.DiscardHandler) hostMap := &HostMap{} - ds := newDnsRecords(hostMap) - ds.Add("test.com.com", "1.2.3.4") + ds := &dnsServer{ + l: l, + dnsMap4: make(map[string]netip.Addr), + dnsMap6: make(map[string]netip.Addr), + hostMap: hostMap, + } + ds.enabled.Store(true) + addrs := []netip.Addr{ + netip.MustParseAddr("1.2.3.4"), + netip.MustParseAddr("1.2.3.5"), + netip.MustParseAddr("fd01::24"), + netip.MustParseAddr("fd01::25"), + } + ds.Add("test.com.com", addrs) + ds.Add("v4only.com.com", []netip.Addr{netip.MustParseAddr("1.2.3.6")}) + ds.Add("v6only.com.com", []netip.Addr{netip.MustParseAddr("fd01::26")}) - m := new(dns.Msg) + m := &dns.Msg{} m.SetQuestion("test.com.com", dns.TypeA) + ds.parseQuery(m, nil) + assert.NotNil(t, m.Answer) + assert.Equal(t, "1.2.3.4", m.Answer[0].(*dns.A).A.String()) + assert.Equal(t, dns.RcodeSuccess, m.Rcode) + + m = &dns.Msg{} + m.SetQuestion("test.com.com", dns.TypeAAAA) + ds.parseQuery(m, nil) + assert.NotNil(t, m.Answer) + assert.Equal(t, "fd01::24", m.Answer[0].(*dns.AAAA).AAAA.String()) + assert.Equal(t, dns.RcodeSuccess, m.Rcode) + + // A known name with no record of the requested type should return NODATA + // (NOERROR with empty answer), not NXDOMAIN. + m = &dns.Msg{} + m.SetQuestion("v4only.com.com", dns.TypeAAAA) + ds.parseQuery(m, nil) + assert.Empty(t, m.Answer) + assert.Equal(t, dns.RcodeSuccess, m.Rcode) + + m = &dns.Msg{} + m.SetQuestion("v6only.com.com", dns.TypeA) + ds.parseQuery(m, nil) + assert.Empty(t, m.Answer) + assert.Equal(t, dns.RcodeSuccess, m.Rcode) + + // An unknown name should still return NXDOMAIN. + m = &dns.Msg{} + m.SetQuestion("unknown.com.com", dns.TypeA) + ds.parseQuery(m, nil) + assert.Empty(t, m.Answer) + assert.Equal(t, dns.RcodeNameError, m.Rcode) + + // short lookups should not fail + m = &dns.Msg{} + m.Question = []dns.Question{{Name: "", Qtype: dns.TypeTXT, Qclass: dns.ClassINET}} + ds.parseQuery(m, stubDNSWriter{}) + assert.Empty(t, m.Answer) + assert.Equal(t, dns.RcodeNameError, m.Rcode) + + m = &dns.Msg{} + m.Question = []dns.Question{{Name: ".", Qtype: dns.TypeTXT, Qclass: dns.ClassINET}} + ds.parseQuery(m, stubDNSWriter{}) + assert.Empty(t, m.Answer) + assert.Equal(t, dns.RcodeNameError, m.Rcode) +} + +func Test_getDnsServerAddr(t *testing.T) { + c := config.NewC(nil) + + c.Settings["lighthouse"] = map[string]any{ + "dns": map[string]any{ + "host": "0.0.0.0", + "port": "1", + }, + } + assert.Equal(t, "0.0.0.0:1", getDnsServerAddr(c)) + + c.Settings["lighthouse"] = map[string]any{ + "dns": map[string]any{ + "host": "::", + "port": "1", + }, + } + assert.Equal(t, "[::]:1", getDnsServerAddr(c)) + + c.Settings["lighthouse"] = map[string]any{ + "dns": map[string]any{ + "host": "[::]", + "port": "1", + }, + } + assert.Equal(t, "[::]:1", getDnsServerAddr(c)) + + // Make sure whitespace doesn't mess us up + c.Settings["lighthouse"] = map[string]any{ + "dns": map[string]any{ + "host": "[::] ", + "port": "1", + }, + } + assert.Equal(t, "[::]:1", getDnsServerAddr(c)) +} + +func newTestDnsServer(t *testing.T) (*dnsServer, *config.C) { + t.Helper() + sl := slog.New(slog.DiscardHandler) + ds := &dnsServer{ + l: sl, + ctx: context.Background(), + dnsMap4: make(map[string]netip.Addr), + dnsMap6: make(map[string]netip.Addr), + hostMap: &HostMap{}, + } + ds.mux = dns.NewServeMux() + ds.mux.HandleFunc(".", ds.handleDnsRequest) + return ds, config.NewC(nil) +} + +func setDnsConfig(c *config.C, host string, port string, amLighthouse, serveDns bool) { + c.Settings["lighthouse"] = map[string]any{ + "am_lighthouse": amLighthouse, + "serve_dns": serveDns, + "dns": map[string]any{ + "host": host, + "port": port, + }, + } +} + +func TestDnsServer_reload_initial_disabled(t *testing.T) { + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", "0", true, false) + + require.NoError(t, ds.reload(c, true)) + assert.False(t, ds.enabled.Load()) + assert.Equal(t, "127.0.0.1:0", ds.addr) + assert.Nil(t, ds.server) +} + +func TestDnsServer_reload_initial_enabled(t *testing.T) { + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", "0", true, true) + + require.NoError(t, ds.reload(c, true)) + assert.True(t, ds.enabled.Load()) + assert.Equal(t, "127.0.0.1:0", ds.addr) + // initial never starts a runner; that's Control.Start's job + assert.Nil(t, ds.server) +} + +func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) { + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", "0", false, true) + + require.NoError(t, ds.reload(c, true)) + // Wants DNS but isn't a lighthouse: gated off, no runner. + assert.False(t, ds.enabled.Load()) +} + +func TestDnsServer_reload_sameAddr_noOp(t *testing.T) { + port := freeUDPPort(t) + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", port, true, true) + require.NoError(t, ds.reload(c, true)) + + go ds.Start() + waitForBind(t, ds) + + ds.serverMu.Lock() + before := ds.server + ds.serverMu.Unlock() + require.NotNil(t, before) + + // Same address, so the running listener must be left alone rather than rebuilt under live queries + require.NoError(t, ds.reload(c, false)) + assert.True(t, ds.enabled.Load()) + + ds.serverMu.Lock() + after := ds.server + ds.serverMu.Unlock() + assert.Same(t, before, after, "a same-address reload must not restart the listener") + + ds.Stop() +} + +// The branch the old sameAddr test was accidentally hitting: enabled with nothing running means reload starts it. +func TestDnsServer_reload_whenNotRunning_starts(t *testing.T) { + port := freeUDPPort(t) + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", port, true, true) + + // initial only records config, it never starts anything + require.NoError(t, ds.reload(c, true)) + ds.serverMu.Lock() + assert.Nil(t, ds.server, "the initial reload must not start a listener") + ds.serverMu.Unlock() + + require.NoError(t, ds.reload(c, false)) + waitForBind(t, ds) + + ds.serverMu.Lock() + assert.NotNil(t, ds.server, "a reload with nothing running should bring DNS up") + ds.serverMu.Unlock() + + ds.Stop() +} + +func TestDnsServer_StartStop_lifecycle(t *testing.T) { + // Bind to a real (random) UDP port so we exercise the actual + // ListenAndServe + Shutdown plumbing including the started-chan race fix. + port := freeUDPPort(t) + + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", port, true, true) + require.NoError(t, ds.reload(c, true)) + + done := make(chan struct{}) + go func() { + ds.Start() + close(done) + }() + + waitFor(t, func() bool { + ds.serverMu.Lock() + started := ds.started + ds.serverMu.Unlock() + if started == nil { + return false + } + select { + case <-started: + return true + default: + return false + } + }) + + ds.Stop() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Start did not return after Stop") + } +} + +func TestDnsServer_Stop_beforeBind_doesNotHang(t *testing.T) { + // Stop called immediately after Start should not deadlock even if bind + // hasn't completed yet. This exercises the started-chan close-on-bind-fail + // path: by binding to an obviously bad port (privileged) we get a fast + // bind error before NotifyStartedFunc fires. + ds, c := newTestDnsServer(t) + // Use a port that should fail to bind (negative would be invalid, use a + // host that won't resolve to ensure listenUDP fails quickly). + setDnsConfig(c, "256.256.256.256", "53", true, true) + require.NoError(t, ds.reload(c, true)) + + done := make(chan struct{}) + go func() { + ds.Start() + close(done) + }() + + // Give Start a moment to attempt the bind and fail. + select { + case <-done: + // Bind failed and Start returned; Stop should be a no-op. + case <-time.After(time.Second): + t.Fatal("Start did not return after a bad bind") + } + + stopped := make(chan struct{}) + go func() { + ds.Stop() + close(stopped) + }() + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("Stop hung after a failed bind") + } +} + +// newTestPKI builds a minimal *PKI with a single v1 cert whose name and +// VPN addresses are caller-provided, suitable for exercising seedSelf and +// QueryCert self handling. +func newTestPKI(t *testing.T, name string, addrs []netip.Addr) *PKI { + t.Helper() + networks := make([]netip.Prefix, 0, len(addrs)) + for _, a := range addrs { + bits := 32 + if a.Is6() { + bits = 128 + } + networks = append(networks, netip.PrefixFrom(a, bits)) + } + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil) + c, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, name, time.Time{}, time.Time{}, networks, nil, nil) + + addrsTable := new(bart.Lite) + for _, a := range addrs { + addrsTable.Insert(netip.PrefixFrom(a, a.BitLen())) + } + + cs := &CertState{ + v2Cert: c, + initiatingVersion: cert.Version2, + myVpnAddrs: addrs, + myVpnAddrsTable: addrsTable, + } + pki := &PKI{} + pki.cs.Store(cs) + return pki +} + +func TestDnsServer_seedSelf_addsOwnRecord(t *testing.T) { + ds, c := newTestDnsServer(t) + myV4 := netip.MustParseAddr("10.0.0.1") + myV6 := netip.MustParseAddr("fd00::1") + ds.pki = newTestPKI(t, "lighthouse", []netip.Addr{myV4, myV6}) + setDnsConfig(c, "127.0.0.1", "0", true, true) + require.NoError(t, ds.reload(c, true)) + + ds.seedSelf() + got4, exists := ds.Query(dns.TypeA, "lighthouse.") + assert.True(t, exists) + assert.Equal(t, myV4, got4) + got6, exists := ds.Query(dns.TypeAAAA, "lighthouse.") + assert.True(t, exists) + assert.Equal(t, myV6, got6) +} + +func TestDnsServer_seedSelf_disabled_noOp(t *testing.T) { + ds, c := newTestDnsServer(t) + ds.pki = newTestPKI(t, "lighthouse", []netip.Addr{netip.MustParseAddr("10.0.0.1")}) + setDnsConfig(c, "127.0.0.1", "0", true, false) + require.NoError(t, ds.reload(c, true)) + + ds.seedSelf() + _, exists := ds.Query(dns.TypeA, "lighthouse.") + assert.False(t, exists) +} + +func TestDnsServer_clearRecords_dropsSelfHost(t *testing.T) { + ds, c := newTestDnsServer(t) + ds.pki = newTestPKI(t, "lighthouse", []netip.Addr{netip.MustParseAddr("10.0.0.1")}) + setDnsConfig(c, "127.0.0.1", "0", true, true) + require.NoError(t, ds.reload(c, true)) + ds.seedSelf() + require.NotEmpty(t, ds.selfHost) + + ds.clearRecords() + assert.Empty(t, ds.selfHost) + _, exists := ds.Query(dns.TypeA, "lighthouse.") + assert.False(t, exists) +} + +func TestDnsServer_QueryCert_returnsOwnCert(t *testing.T) { + ds, _ := newTestDnsServer(t) + myV4 := netip.MustParseAddr("10.0.0.1") + ds.pki = newTestPKI(t, "lighthouse", []netip.Addr{myV4}) + + got := ds.QueryCert(myV4.String() + ".") + assert.NotEmpty(t, got, "TXT lookup of our own VPN address should return our cert") + + other := netip.MustParseAddr("10.0.0.99") + assert.Empty(t, ds.QueryCert(other.String()+"."), "unknown peer IP should return nothing") +} + +func TestDnsServer_reload_disable_stopsRunningServer(t *testing.T) { + port := freeUDPPort(t) + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", port, true, true) + require.NoError(t, ds.reload(c, true)) + + startReturned := make(chan struct{}) + go func() { + ds.Start() + close(startReturned) + }() + waitForBind(t, ds) + + // Toggle serve_dns off; reload should shut the running server down. + setDnsConfig(c, "127.0.0.1", port, true, false) + require.NoError(t, ds.reload(c, false)) + select { + case <-startReturned: + case <-time.After(5 * time.Second): + t.Fatal("Start did not return after reload disabled DNS") + } + assert.False(t, ds.enabled.Load()) +} + +func freeUDPPort(t *testing.T) string { + t.Helper() + conn, err := net.ListenPacket("udp", "127.0.0.1:0") + require.NoError(t, err) + port := conn.LocalAddr().(*net.UDPAddr).Port + require.NoError(t, conn.Close()) + return strconv.Itoa(port) +} + +func waitForBind(t *testing.T, ds *dnsServer) { + t.Helper() + waitFor(t, func() bool { + ds.serverMu.Lock() + started := ds.started + ds.serverMu.Unlock() + if started == nil { + return false + } + select { + case <-started: + return true + default: + return false + } + }) +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("timed out waiting for condition") +} + +// Two reloads in quick succession, or a HUP before Control.Start, can race two Starts at the same listener. +func TestDnsServer_Start_isIdempotent(t *testing.T) { + port := freeUDPPort(t) + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", port, true, true) + require.NoError(t, ds.reload(c, true)) + + go ds.Start() + waitForBind(t, ds) + + ds.serverMu.Lock() + first := ds.server + ds.serverMu.Unlock() + require.NotNil(t, first) + + // If the second Start replaces the tracked server, Stop kills the wrong one and the port leaks + done := make(chan struct{}) + go func() { + ds.Start() + close(done) + }() + select { + case <-done: + case <-time.After(time.Second * 5): + t.Fatal("second Start never returned") + } + + ds.serverMu.Lock() + second := ds.server + ds.serverMu.Unlock() + assert.Same(t, first, second, "a second Start must not replace the running server") + + // The real proof, after Stop the port must actually be free + ds.Stop() + waitFor(t, func() bool { + pc, err := net.ListenPacket("udp", "127.0.0.1:"+port) + if err != nil { + return false + } + _ = pc.Close() + return true + }) +} + +// An address change must actually end up listening on the new port. Start's guard refuses when a server is already +// installed, so reload has to clear the slot before shutting the old one down. +func TestDnsServer_reload_addrChange_restarts(t *testing.T) { + first := freeUDPPort(t) + second := freeUDPPort(t) + + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", first, true, true) + require.NoError(t, ds.reload(c, true)) + + go ds.Start() + waitForBind(t, ds) + + // Cycle a few times, the failure this guards against depends on which goroutine wins serverMu + for i := range 8 { + want := second + if i%2 == 1 { + want = first + } + setDnsConfig(c, "127.0.0.1", want, true, true) + require.NoError(t, ds.reload(c, false)) + waitForBind(t, ds) + + ds.serverMu.Lock() + srv := ds.server + ds.serverMu.Unlock() + require.NotNil(t, srv, "reload left DNS down instead of restarting it") + require.Equal(t, "127.0.0.1:"+want, srv.Addr, "reload should be serving the new address") + } + + // Land back on second so the port assertions below are meaningful + setDnsConfig(c, "127.0.0.1", second, true, true) + require.NoError(t, ds.reload(c, false)) + waitForBind(t, ds) + + // The old port must be released and the new one actually held + waitFor(t, func() bool { + pc, err := net.ListenPacket("udp", "127.0.0.1:"+first) + if err != nil { + return false + } + _ = pc.Close() + return true + }) + _, err := net.ListenPacket("udp", "127.0.0.1:"+second) + require.Error(t, err, "the new address should be bound by the DNS responder") + + ds.Stop() +} + +// A listener that dies on its own must release the slot, or a later same-addr reload sees it as running and no-ops. +func TestDnsServer_Start_bindFailure_releasesSlot(t *testing.T) { + port := freeUDPPort(t) + blocker, err := net.ListenPacket("udp", "127.0.0.1:"+port) + require.NoError(t, err) + + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", port, true, true) + require.NoError(t, ds.reload(c, true)) + + ds.Start() // returns once the bind fails + + ds.serverMu.Lock() + assert.Nil(t, ds.server, "a listener that failed to bind must not stay parked in the slot") + ds.serverMu.Unlock() + + // With the slot released, a reload can retry once the port frees up + require.NoError(t, blocker.Close()) + require.NoError(t, ds.reload(c, false)) + waitForBind(t, ds) + + ds.serverMu.Lock() + assert.NotNil(t, ds.server, "a same-addr reload should retry after a failed bind") + ds.serverMu.Unlock() + + ds.Stop() +} + +// A disable that lands while Start is between its unlocked check and the guard must not leave a listener behind. +func TestDnsServer_Start_refusesWhenDisabledUnderLock(t *testing.T) { + port := freeUDPPort(t) + ds, c := newTestDnsServer(t) + setDnsConfig(c, "127.0.0.1", port, true, true) + require.NoError(t, ds.reload(c, true)) + require.True(t, ds.enabled.Load()) + + // Holding serverMu parks Start on the lock, the only way to land the disable in that window on purpose + ds.serverMu.Lock() + + done := make(chan struct{}) + go func() { + ds.Start() + close(done) + }() + + select { + case <-done: + ds.serverMu.Unlock() + t.Fatal("Start returned early, the test never exercised the window") + case <-time.After(time.Millisecond * 100): + } + + // The disable reload's critical section. It sees nothing running, so it never calls Stop. + ds.enabled.Store(false) + ds.serverMu.Unlock() + + select { + case <-done: + case <-time.After(time.Second * 5): + t.Fatal("Start never returned") + } + + ds.serverMu.Lock() + assert.Nil(t, ds.server, "Start must not install a listener a disable already cancelled") + ds.serverMu.Unlock() - //parseQuery(m) + pc, err := net.ListenPacket("udp", "127.0.0.1:"+port) + require.NoError(t, err, "an orphaned listener is still holding the port") + _ = pc.Close() } diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..d705fce3 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,21 @@ +FROM gcr.io/distroless/static:latest + +ARG TARGETOS TARGETARCH + +ARG VERSION=dev +ARG REVISION=unknown +LABEL org.opencontainers.image.title="nebula" \ + org.opencontainers.image.description="A scalable overlay networking tool with a focus on performance, simplicity and security" \ + org.opencontainers.image.vendor="Nebula OSS" \ + org.opencontainers.image.source="https://github.com/slackhq/nebula" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.revision="${REVISION}" + +COPY build/$TARGETOS-$TARGETARCH/nebula /nebula +COPY build/$TARGETOS-$TARGETARCH/nebula-cert /nebula-cert + +VOLUME ["/config"] + +ENTRYPOINT ["/nebula"] +# Allow users to override the args passed to nebula +CMD ["-config", "/config/config.yml"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..129744fd --- /dev/null +++ b/docker/README.md @@ -0,0 +1,24 @@ +# NebulaOSS/nebula Docker Image + +## Building + +From the root of the repository, run `make docker`. + +## Running + +To run the built image, use the following command: + +``` +docker run \ + --name nebula \ + --network host \ + --cap-add NET_ADMIN \ + --volume ./config:/config \ + --rm \ + nebulaoss/nebula +``` + +A few notes: + +- The `NET_ADMIN` capability is necessary to create the tun adapter on the host (this is unnecessary if the tun device is disabled.) +- `--volume ./config:/config` should point to a directory that contains your `config.yml` and any other necessary files. diff --git a/e2e/echo_test.go b/e2e/echo_test.go new file mode 100644 index 00000000..5e1299b2 --- /dev/null +++ b/e2e/echo_test.go @@ -0,0 +1,85 @@ +//go:build e2e_testing +// +build e2e_testing + +package e2e + +import ( + "testing" + "time" + + "github.com/slackhq/nebula" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/e2e/router" + "github.com/slackhq/nebula/header" + "github.com/slackhq/nebula/udp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func assertTestRequestEchoed(t *testing.T, cipher string) { + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + over := m{"cipher": cipher} + a, aNet, aUdp, _ := newSimpleServer(cert.Version1, ca, caKey, "a", "10.128.0.1/24", over) + b, bNet, bUdp, _ := newSimpleServer(cert.Version1, ca, caKey, "b", "10.128.0.2/24", over) + + a.InjectLightHouseAddr(bNet[0].Addr(), bUdp) + b.InjectLightHouseAddr(aNet[0].Addr(), aUdp) + a.Start() + b.Start() + t.Cleanup(func() { a.Stop(); b.Stop() }) + r := router.NewR(t, a, b) + defer r.RenderFlow() + + assertTunnel(t, aNet[0].Addr(), bNet[0].Addr(), a, b, r) + drainUDPTx(a) + drainUDPTx(b) + + payload := []byte("a test payload well over sixteen bytes long, wow it's so very long long long!") + require.Greater(t, len(payload), header.Len) + a.GetF().SendMessageToVpnAddr(header.Test, header.TestRequest, bNet[0].Addr(), payload, make([]byte, 12, 12), make([]byte, udp.MTU)) + + // Deliver A's request to B; B must echo a reply back + b.InjectUDPPacket(a.GetFromUDP(true)) + reply := nextUDPTxOfType(t, b, header.Test, header.TestReply, 2*time.Second) + + assert.Equal(t, aUdp, reply.To, "the reply must go back to the requester") + // header + echoed payload + 16-byte AEAD tag: proves the whole payload + // round-tripped rather than being dropped or truncated. + assert.Equal(t, header.Len+len(payload)+16, len(reply.Data), "the full payload must be echoed back") +} + +func TestTestRequestEchoesLongPayloadAES(t *testing.T) { + assertTestRequestEchoed(t, "aes") +} + +func TestTestRequestEchoesLongPayloadChaChaPoly(t *testing.T) { + assertTestRequestEchoed(t, "chachapoly") +} + +// drainUDPTx empties a control's UDP tx queue without blocking. +func drainUDPTx(c *nebula.Control) { + for c.GetFromUDP(false) != nil { + } +} + +// nextUDPTxOfType returns the next packet a control transmits whose nebula +// header matches (wantType, wantSub), skipping unrelated packets. +// It fails the test if none arrives within the timeout. +func nextUDPTxOfType(t *testing.T, c *nebula.Control, wantType header.MessageType, wantSub header.MessageSubType, within time.Duration) *udp.Packet { + t.Helper() + ch := c.GetUDPTxChan() + timeout := time.After(within) + for { + select { + case p := <-ch: + var h header.H + if err := h.Parse(p.Data); err == nil && h.Type == wantType && h.Subtype == wantSub { + return p + } + case <-timeout: + t.Fatalf("timed out waiting for a %v/%v packet on the udp tx queue", wantType, wantSub) + return nil + } + } +} diff --git a/e2e/handshake_manager_test.go b/e2e/handshake_manager_test.go new file mode 100644 index 00000000..b06564d1 --- /dev/null +++ b/e2e/handshake_manager_test.go @@ -0,0 +1,577 @@ +//go:build e2e_testing +// +build e2e_testing + +package e2e + +import ( + "net/netip" + "testing" + "time" + + "github.com/slackhq/nebula" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/e2e/router" + "github.com/slackhq/nebula/header" + "github.com/slackhq/nebula/udp" + "github.com/stretchr/testify/assert" +) + +// makeHandshakePacket creates a handshake packet with the given parameters. +func makeHandshakePacket(from, to netip.AddrPort, subtype header.MessageSubType, remoteIndex uint32, counter uint64) *udp.Packet { + data := make([]byte, 200) + header.Encode(data, header.Version, header.Handshake, subtype, remoteIndex, counter) + for i := header.Len; i < len(data); i++ { + data[i] = byte(i) + } + return &udp.Packet{To: to, From: from, Data: data} +} + +func TestHandshakeRetransmitDuplicate(t *testing.T) { + t.Parallel() + // Verify the responder correctly handles receiving the same msg1 multiple times + // (retransmission). The duplicate goes through CheckAndComplete -> ErrAlreadySeen + // and the cached response is resent. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + + t.Log("Trigger handshake from me to them") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))) + + t.Log("Grab my msg1") + msg1 := myControl.GetFromUDP(true) + + t.Log("Inject msg1 into them, first time") + theirControl.InjectUDPPacket(msg1) + _ = theirControl.GetFromUDP(true) + + t.Log("Inject the SAME msg1 again, tests ErrAlreadySeen path") + theirControl.InjectUDPPacket(msg1) + resp2 := theirControl.GetFromUDP(true) + assert.NotNil(t, resp2, "should get cached response on duplicate msg1") + + t.Log("Complete handshake with cached response") + myControl.InjectUDPPacket(resp2) + myControl.WaitForType(1, 0, theirControl) + + t.Log("Drain cached packet and verify tunnel works") + cachedPacket := theirControl.GetFromTun(true) + assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + t.Log("Verify only one tunnel exists on each side") + assert.Len(t, myControl.ListHostmapHosts(false), 1) + assert.Len(t, theirControl.ListHostmapHosts(false), 1) + + myControl.Stop() + theirControl.Stop() +} + +func TestHandshakeTruncatedPacketRecovery(t *testing.T) { + t.Parallel() + // Verify that a truncated handshake packet is ignored and the real + // packet can still complete the handshake. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + + t.Log("Trigger handshake") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))) + + t.Log("Get msg1 and deliver to responder") + msg1 := myControl.GetFromUDP(true) + theirControl.InjectUDPPacket(msg1) + + t.Log("Get the real response") + realResp := theirControl.GetFromUDP(true) + + t.Log("Truncate the response and inject, should be ignored") + truncResp := realResp.Copy() + truncResp.Data = truncResp.Data[:header.Len] + myControl.InjectUDPPacket(truncResp) + + t.Log("Verify pending handshake survived the truncated packet") + assert.NotEmpty(t, myControl.ListHostmapHosts(true), "pending handshake should still exist") + + t.Log("Inject real response, should complete handshake") + myControl.InjectUDPPacket(realResp) + myControl.WaitForType(1, 0, theirControl) + + t.Log("Drain and verify tunnel") + cachedPacket := theirControl.GetFromTun(true) + assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + myControl.Stop() + theirControl.Stop() +} + +func TestHandshakeOrphanedMsg2Dropped(t *testing.T) { + t.Parallel() + // A msg2 arriving with no matching pending index should be silently dropped + // with no response sent and no state changes. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + + t.Log("Complete a normal handshake") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))) + r.RouteForAllUntilTxTun(theirControl) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + t.Log("Record hostmap state") + myIndexes := len(myControl.ListHostmapIndexes(false)) + + t.Log("Inject a fake msg2 with unknown RemoteIndex") + myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0xDEADBEEF, 2)) + + t.Log("Verify no new indexes created") + assert.Equal(t, myIndexes, len(myControl.ListHostmapIndexes(false))) + + t.Log("Verify no UDP response was sent") + time.Sleep(100 * time.Millisecond) + assert.Nil(t, myControl.GetFromUDP(false), "should not send a response to orphaned msg2") + + t.Log("Verify existing tunnel still works") + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + myControl.Stop() + theirControl.Stop() +} + +func TestHandshakeUnknownMessageCounter(t *testing.T) { + t.Parallel() + // A handshake packet with an unexpected message counter should be silently + // dropped with no side effects and no UDP response. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + myControl.Start() + theirControl.Start() + + t.Log("Inject handshake with MessageCounter=3") + myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 3)) + + t.Log("Inject handshake with MessageCounter=99") + myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 99)) + + t.Log("Verify no tunnels or pending handshakes") + assert.Empty(t, myControl.ListHostmapHosts(false)) + assert.Empty(t, myControl.ListHostmapHosts(true)) + + t.Log("Verify no UDP response was sent") + time.Sleep(100 * time.Millisecond) + assert.Nil(t, myControl.GetFromUDP(false)) + + myControl.Stop() + theirControl.Stop() +} + +func TestHandshakeUnknownSubtype(t *testing.T) { + t.Parallel() + // A handshake packet with an unknown subtype should be silently dropped. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.Start() + theirControl.Start() + + t.Log("Inject handshake with unknown subtype 99") + myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.MessageSubType(99), 0, 1)) + + t.Log("Verify no tunnels or pending handshakes") + assert.Empty(t, myControl.ListHostmapHosts(false)) + assert.Empty(t, myControl.ListHostmapHosts(true)) + + t.Log("Verify no UDP response was sent") + time.Sleep(100 * time.Millisecond) + assert.Nil(t, myControl.GetFromUDP(false)) + + myControl.Stop() + theirControl.Stop() +} + +func TestHandshakeLateResponse(t *testing.T) { + t.Parallel() + // After a handshake times out, a late response should be silently ignored + // with no new tunnels created. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{ + "handshakes": m{ + "try_interval": "200ms", + "retries": 2, + }, + }) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + myControl.Start() + theirControl.Start() + + t.Log("Trigger handshake from me") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))) + + t.Log("Grab msg1 but don't deliver") + msg1 := myControl.GetFromUDP(true) + + t.Log("Wait for handshake to time out") + for i := 0; i < 5; i++ { + time.Sleep(300 * time.Millisecond) + myControl.GetFromUDP(false) + } + + t.Log("Confirm no pending handshakes remain") + assert.Empty(t, myControl.ListHostmapHosts(true)) + + t.Log("Deliver old msg1 to them, they create a tunnel") + theirControl.InjectUDPPacket(msg1) + resp := theirControl.GetFromUDP(true) + assert.NotNil(t, resp) + + t.Log("Inject late response into me, should be ignored") + myControl.InjectUDPPacket(resp) + + t.Log("No tunnel should exist on my side") + assert.Empty(t, myControl.ListHostmapHosts(false)) + assert.Empty(t, myControl.ListHostmapHosts(true)) + + myControl.Stop() + theirControl.Stop() +} + +func TestHandshakeSelfConnectionRejected(t *testing.T) { + t.Parallel() + // Verify that a node rejects a handshake containing its own VPN IP in the + // peer cert. We do this by sending the initiator's own msg1 back to itself. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + + // Need a lighthouse entry to trigger a handshake + myControl.InjectLightHouseAddr(netip.MustParseAddr("10.128.0.2"), netip.MustParseAddrPort("10.0.0.2:4242")) + + myControl.Start() + + t.Log("Trigger handshake from me") + myControl.InjectTunPacket(BuildTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))) + msg1 := myControl.GetFromUDP(true) + + t.Log("Drain any handshake retransmits before injecting") + time.Sleep(100 * time.Millisecond) + for myControl.GetFromUDP(false) != nil { + } + + t.Log("Feed my own msg1 back to me as if it came from someone else") + selfMsg := msg1.Copy() + selfMsg.From = netip.MustParseAddrPort("10.0.0.99:4242") + selfMsg.To = myUdpAddr + myControl.InjectUDPPacket(selfMsg) + + t.Log("Verify no response was sent (self-connection rejected)") + time.Sleep(100 * time.Millisecond) + // Drain any further retransmits from the original handshake, then check + // that none of them are a handshake response (MessageCounter=2) + h := &header.H{} + for { + p := myControl.GetFromUDP(false) + if p == nil { + break + } + _ = h.Parse(p.Data) + assert.NotEqual(t, uint64(2), h.MessageCounter, + "should not send a stage 2 response to self-connection") + } + + t.Log("Verify no tunnel to myself was created") + assert.Nil(t, myControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)) + + myControl.Stop() +} + +func TestHandshakeMessageCounter0Dropped(t *testing.T) { + t.Parallel() + // MessageCounter=0 is not a valid handshake message and should be dropped. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + _, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.Start() + + t.Log("Inject handshake with MessageCounter=0") + myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 0)) + + time.Sleep(100 * time.Millisecond) + assert.Empty(t, myControl.ListHostmapHosts(false)) + assert.Empty(t, myControl.ListHostmapHosts(true)) + assert.Nil(t, myControl.GetFromUDP(false)) + + myControl.Stop() +} + +func TestHandshakeRemoteAllowList(t *testing.T) { + t.Parallel() + // Verify that a handshake from a blocked underlay IP is dropped with no + // response and no state changes. Then verify the same packet from an + // allowed IP succeeds. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{ + "lighthouse": m{ + "remote_allow_list": m{ + "10.0.0.0/8": true, + "0.0.0.0/0": false, + }, + }, + }) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + + t.Log("Trigger handshake from them") + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi"))) + msg1 := theirControl.GetFromUDP(true) + + t.Log("Rewrite the source to a blocked IP and inject") + blockedMsg := msg1.Copy() + blockedMsg.From = netip.MustParseAddrPort("192.168.1.1:4242") + myControl.InjectUDPPacket(blockedMsg) + + t.Log("Verify no tunnel, no pending, no response from blocked source") + time.Sleep(100 * time.Millisecond) + assert.Empty(t, myControl.ListHostmapHosts(false)) + assert.Empty(t, myControl.ListHostmapHosts(true)) + assert.Nil(t, myControl.GetFromUDP(false), "should not respond to blocked source") + + t.Log("Now inject the real packet from the allowed source") + myControl.InjectUDPPacket(msg1) + + t.Log("Verify handshake completes from allowed source") + resp := myControl.GetFromUDP(true) + assert.NotNil(t, resp) + theirControl.InjectUDPPacket(resp) + theirControl.WaitForType(1, 0, myControl) + + t.Log("Drain cached packet and verify tunnel works") + cachedPacket := myControl.GetFromTun(true) + assertUdpPacket(t, []byte("Hi"), cachedPacket, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + myControl.Stop() + theirControl.Stop() +} + +func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) { + t.Parallel() + // When a duplicate msg1 arrives via ErrAlreadySeen, verify the tunnel + // remains functional and hostmap index count is stable. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + + t.Log("Complete a normal handshake via the router") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))) + r.RouteForAllUntilTxTun(theirControl) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + t.Log("Record hostmap state") + theirIndexes := len(theirControl.ListHostmapIndexes(false)) + hi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + assert.NotNil(t, hi) + originalRemote := hi.CurrentRemote + + t.Log("Re-trigger traffic to cause a new handshake attempt (ErrAlreadySeen)") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam"))) + r.RouteForAllUntilTxTun(theirControl) + + t.Log("Verify tunnel still works") + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + t.Log("Verify remote is still valid and index count is stable") + hi2 := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + assert.NotNil(t, hi2) + assert.Equal(t, originalRemote, hi2.CurrentRemote) + assert.Equal(t, theirIndexes, len(theirControl.ListHostmapIndexes(false)), + "no extra indexes should be created from ErrAlreadySeen") + + myControl.Stop() + theirControl.Stop() +} + +func TestHandshakeWrongResponderPacketStore(t *testing.T) { + t.Parallel() + // Verify that when the wrong host responds, the cached packets are + // transferred to the new handshake, the evil tunnel is closed, evil's + // address is blocked, and the correct tunnel is eventually established. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.100/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.99/24", nil) + evilControl, evilVpnIpNet, evilUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "evil", "10.128.0.2/24", nil) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), evilUdpAddr) + + r := router.NewR(t, myControl, theirControl, evilControl) + defer r.RenderFlow() + + myControl.Start() + theirControl.Start() + evilControl.Start() + + t.Log("Send multiple packets to them (cached during handshake)") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1"))) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2"))) + + t.Log("Route until evil tunnel is closed") + h := &header.H{} + r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType { + if err := h.Parse(p.Data); err != nil { + panic(err) + } + if h.Type == header.CloseTunnel && p.To == evilUdpAddr { + return router.RouteAndExit + } + return router.KeepRouting + }) + + t.Log("Verify evil's address is blocked in the new pending handshake") + pendingHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), true) + if pendingHI != nil { + assert.NotContains(t, pendingHI.RemoteAddrs, evilUdpAddr, + "evil's address should be blocked") + } + + t.Log("Inject correct lighthouse addr for them") + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + t.Log("Route until cached packets arrive at the real them") + p := r.RouteForAllUntilTxTun(theirControl) + assert.NotNil(t, p, "a cached packet should be delivered to the correct host") + + t.Log("Verify the correct host has a tunnel") + assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl) + + t.Log("Verify no hostinfo artifacts from evil remain") + assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), true), + "no pending hostinfo for evil") + assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), false), + "no main hostinfo for evil") + + myControl.Stop() + theirControl.Stop() + evilControl.Stop() +} + +func TestHandshakeRelayComplete(t *testing.T) { + t.Parallel() + // Verify that a relay handshake completes correctly and relay state is + // properly maintained on all three nodes. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay", "10.128.0.128/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) + + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + r := router.NewR(t, myControl, relayControl, theirControl) + defer r.RenderFlow() + + myControl.Start() + relayControl.Start() + theirControl.Start() + + t.Log("Trigger handshake via relay") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay"))) + + p := r.RouteForAllUntilTxTun(theirControl) + assertUdpPacket(t, []byte("Hi via relay"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) + + t.Log("Verify bidirectional tunnel via relay") + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + t.Log("Verify relay state on my side shows relay-to-me") + myHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) + assert.NotNil(t, myHI) + assert.NotEmpty(t, myHI.CurrentRelaysToMe, "should have relay-to-me for them") + + t.Log("Verify relay state on their side shows relay-to-me") + theirHI := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + assert.NotNil(t, theirHI) + assert.NotEmpty(t, theirHI.CurrentRelaysToMe, "should have relay-to-me for me") + + t.Log("Verify relay node shows through-me relays") + relayHI := relayControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + assert.NotNil(t, relayHI) + + myControl.Stop() + relayControl.Stop() + theirControl.Stop() +} + +// NOTE: Relay V1 cert + IPv6 rejection is not tested here because +// BuildTunUDPPacket from a V4 node to a V6 address panics in the test +// framework. The check is in handshake_manager.go handleOutbound relay +// logic (lines ~304-313): if the relay host has a V1 cert and either +// address is IPv6, the relay is skipped. + +// NOTE: Relay reestablishment (Disestablished state transition) is covered +// by the existing TestReestablishRelays in handshakes_test.go. diff --git a/e2e/handshakes_test.go b/e2e/handshakes_test.go index 59f1d0e5..0c0bdf44 100644 --- a/e2e/handshakes_test.go +++ b/e2e/handshakes_test.go @@ -4,28 +4,33 @@ package e2e import ( - "fmt" - "net" + "net/netip" + "slices" "testing" "time" - "github.com/sirupsen/logrus" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" "github.com/slackhq/nebula" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" "github.com/slackhq/nebula/e2e/router" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/udp" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" ) func BenchmarkHotPath(b *testing.B) { - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, _, _, _ := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 1}, nil) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil) + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) // Put their info in our lighthouse - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) // Start the servers myControl.Start() @@ -34,29 +39,81 @@ func BenchmarkHotPath(b *testing.B) { r := router.NewR(b, myControl, theirControl) r.CancelFlowLogs() + assertTunnel(b, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + // Pre-build the IP packet bytes once so the bench measures the data plane, + // not gopacket SerializeLayers overhead. + prebuilt := BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")) + + // EnableFanIn switches the router to a 0-alloc routing path. Required + // for hot-path benchmarks; would conflict with GetFromUDP-using tests. + r.EnableFanIn() + + b.ResetTimer() + + for n := 0; n < b.N; n++ { + myControl.InjectTunPacket(prebuilt) + // Release the TUN-side bytes back to the harness freelist; the bench + // just confirms a packet arrived, the contents aren't inspected. + overlay.ReleaseTunBuf(r.RouteForAllUntilTxTun(theirControl)) + } + + myControl.Stop() + theirControl.Stop() +} + +func BenchmarkHotPathRelay(b *testing.B) { + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) + + // Teach my how to get to the relay and that their can be reached via the relay + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + // Build a router so we don't have to reason who gets which packet + r := router.NewR(b, myControl, relayControl, theirControl) + r.CancelFlowLogs() + + // Start the servers + myControl.Start() + relayControl.Start() + theirControl.Start() + + assertTunnel(b, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) + + prebuilt := BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")) + r.EnableFanIn() + + b.ResetTimer() + for n := 0; n < b.N; n++ { - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) - _ = r.RouteForAllUntilTxTun(theirControl) + myControl.InjectTunPacket(prebuilt) + overlay.ReleaseTunBuf(r.RouteForAllUntilTxTun(theirControl)) } myControl.Stop() theirControl.Stop() + relayControl.Stop() } func TestGoodHandshake(t *testing.T) { - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 1}, nil) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil) + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) // Put their info in our lighthouse - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) // Start the servers myControl.Start() theirControl.Start() t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) t.Log("Have them consume my stage 0 packet. They have a tunnel now") theirControl.InjectUDPPacket(myControl.GetFromUDP(true)) @@ -77,38 +134,150 @@ func TestGoodHandshake(t *testing.T) { myControl.WaitForType(1, 0, theirControl) t.Log("Make sure our host infos are correct") - assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl) + assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl) t.Log("Get that cached packet and make sure it looks right") myCachedPacket := theirControl.GetFromTun(true) - assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet.IP, theirVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) t.Log("Do a bidirectional tunnel test") r := router.NewR(t, myControl, theirControl) defer r.RenderFlow() - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) r.RenderHostmaps("Final hostmaps", myControl, theirControl) myControl.Stop() theirControl.Stop() - //TODO: assert hostmaps } -func TestWrongResponderHandshake(t *testing.T) { - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) +func TestGoodHandshakeNoOverlap(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "2001::69/24", nil) //look ma, cross-stack! + + // Put their info in our lighthouse + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + // Start the servers + myControl.Start() + theirControl.Start() + + empty := []byte{} + t.Log("do something to cause a handshake") + myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, empty, empty) + + t.Log("Have them consume my stage 0 packet. They have a tunnel now") + theirControl.InjectUDPPacket(myControl.GetFromUDP(true)) + + t.Log("Get their stage 1 packet") + stage1Packet := theirControl.GetFromUDP(true) + + t.Log("Have me consume their stage 1 packet. I have a tunnel now") + myControl.InjectUDPPacket(stage1Packet) + + t.Log("Wait until we see a test packet come through to make sure we give the tunnel time to complete") + myControl.WaitForType(header.Test, 0, theirControl) - // The IPs here are chosen on purpose: - // The current remote handling will sort by preference, public, and then lexically. - // So we need them to have a higher address than evil (we could apply a preference though) - myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 100}, nil) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 99}, nil) - evilControl, evilVpnIp, evilUdpAddr, _ := newSimpleServer(ca, caKey, "evil", net.IP{10, 0, 0, 2}, nil) + t.Log("Make sure our host infos are correct") + assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl) + + myControl.Stop() + theirControl.Stop() +} - // Add their real udp addr, which should be tried after evil. - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) +func TestWrongResponderHandshake(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.100/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.99/24", nil) + evilControl, evilVpnIp, evilUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "evil", "10.128.0.2/24", nil) // Put the evil udp addr in for their vpn Ip, this is a case of being lied to by the lighthouse. - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, evilUdpAddr) + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), evilUdpAddr) + + // Build a router so we don't have to reason who gets which packet + r := router.NewR(t, myControl, theirControl, evilControl) + defer r.RenderFlow() + + // Start the servers + myControl.Start() + theirControl.Start() + evilControl.Start() + + t.Log("Start the handshake process, we will route until we see the evil tunnel closed") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + + h := &header.H{} + r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType { + err := h.Parse(p.Data) + if err != nil { + panic(err) + } + + if h.Type == header.CloseTunnel && p.To == evilUdpAddr { + return router.RouteAndExit + } + + return router.KeepRouting + }) + + t.Log("Evil tunnel is closed, inject the correct udp addr for them") + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + pendingHi := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), true) + assert.NotContains(t, pendingHi.RemoteAddrs, evilUdpAddr) + + t.Log("Route until we see the cached packet") + r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType { + err := h.Parse(p.Data) + if err != nil { + panic(err) + } + + if p.To == theirUdpAddr && h.Type == 1 { + return router.RouteAndExit + } + + return router.KeepRouting + }) + + t.Log("My cached packet should be received by them") + myCachedPacket := theirControl.GetFromTun(true) + assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) + + t.Log("Test the tunnel with them") + assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + t.Log("Flush all packets from all controllers") + r.FlushAll() + + t.Log("Ensure ensure I don't have any hostinfo artifacts from evil") + assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIp[0].Addr(), true), "My pending hostmap should not contain evil") + assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIp[0].Addr(), false), "My main hostmap should not contain evil") + + r.RenderHostmaps("Final hostmaps", myControl, theirControl, evilControl) + t.Log("Success!") + myControl.Stop() + theirControl.Stop() +} + +func TestWrongResponderHandshakeStaticHostMap(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.99/24", nil) + evilControl, evilVpnIp, evilUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "evil", "10.128.0.2/24", nil) + o := m{ + "static_host_map": m{ + theirVpnIpNet[0].Addr().String(): []string{evilUdpAddr.String()}, + }, + } + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.100/24", o) + + // Put the evil udp addr in for their vpn addr, this is a case of a remote at a static entry changing its vpn addr. + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), evilUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, theirControl, evilControl) @@ -119,41 +288,58 @@ func TestWrongResponderHandshake(t *testing.T) { theirControl.Start() evilControl.Start() - t.Log("Start the handshake process, we will route until we see our cached packet get sent to them") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) + t.Log("Start the handshake process, we will route until we see the evil tunnel closed") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + + h := &header.H{} r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType { - h := &header.H{} err := h.Parse(p.Data) if err != nil { panic(err) } - if p.ToIp.Equal(theirUdpAddr.IP) && p.ToPort == uint16(theirUdpAddr.Port) && h.Type == 1 { + if h.Type == header.CloseTunnel && p.To == evilUdpAddr { return router.RouteAndExit } return router.KeepRouting }) - //TODO: Assert pending hostmap - I should have a correct hostinfo for them now + t.Log("Evil tunnel is closed, inject the correct udp addr for them") + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + pendingHi := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), true) + assert.NotContains(t, pendingHi.RemoteAddrs, evilUdpAddr) + + t.Log("Route until we see the cached packet") + r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType { + err := h.Parse(p.Data) + if err != nil { + panic(err) + } + + if p.To == theirUdpAddr && h.Type == 1 { + return router.RouteAndExit + } + + return router.KeepRouting + }) t.Log("My cached packet should be received by them") myCachedPacket := theirControl.GetFromTun(true) - assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet.IP, theirVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) t.Log("Test the tunnel with them") - assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl) - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) t.Log("Flush all packets from all controllers") r.FlushAll() t.Log("Ensure ensure I don't have any hostinfo artifacts from evil") - assert.Nil(t, myControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(evilVpnIp.IP), true), "My pending hostmap should not contain evil") - assert.Nil(t, myControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(evilVpnIp.IP), false), "My main hostmap should not contain evil") + assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIp[0].Addr(), true), "My pending hostmap should not contain evil") + assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIp[0].Addr(), false), "My main hostmap should not contain evil") //NOTE: if evil lost the handshake race it may still have a tunnel since me would reject the handshake since the tunnel is complete - //TODO: assert hostmaps for everyone r.RenderHostmaps("Final hostmaps", myControl, theirControl, evilControl) t.Log("Success!") myControl.Stop() @@ -161,16 +347,17 @@ func TestWrongResponderHandshake(t *testing.T) { } func TestStage1Race(t *testing.T) { + t.Parallel() // This tests ensures that two hosts handshaking with each other at the same time will allow traffic to flow // But will eventually collapse down to a single tunnel - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 1}, nil) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil) + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) // Put their info in our lighthouse and vice versa - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) - theirControl.InjectLightHouseAddr(myVpnIpNet.IP, myUdpAddr) + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, theirControl) @@ -181,8 +368,8 @@ func TestStage1Race(t *testing.T) { theirControl.Start() t.Log("Trigger a handshake to start on both me and them") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) - theirControl.InjectTunUDPPacket(myVpnIpNet.IP, 80, 80, []byte("Hi from them")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))) t.Log("Get both stage 1 handshake packets") myHsForThem := myControl.GetFromUDP(true) @@ -194,14 +381,14 @@ func TestStage1Race(t *testing.T) { r.Log("Route until they receive a message packet") myCachedPacket := r.RouteForAllUntilTxTun(theirControl) - assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet.IP, theirVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) r.Log("Their cached packet should be received by me") theirCachedPacket := r.RouteForAllUntilTxTun(myControl) - assertUdpPacket(t, []byte("Hi from them"), theirCachedPacket, theirVpnIpNet.IP, myVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from them"), theirCachedPacket, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80) r.Log("Do a bidirectional tunnel test") - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) myHostmapHosts := myControl.ListHostmapHosts(false) myHostmapIndexes := myControl.ListHostmapIndexes(false) @@ -218,8 +405,8 @@ func TestStage1Race(t *testing.T) { r.Log("Spin until connection manager tears down a tunnel") - for len(myControl.GetHostmap().Indexes)+len(theirControl.GetHostmap().Indexes) > 2 { - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + for myControl.GetHostmapIndexCount()+theirControl.GetHostmapIndexCount() > 2 { + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) t.Log("Connection manager hasn't ticked yet") time.Sleep(time.Second) } @@ -241,13 +428,14 @@ func TestStage1Race(t *testing.T) { } func TestUncleanShutdownRaceLoser(t *testing.T) { - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 1}, nil) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil) + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) // Teach my how to get to the relay and that their can be reached via the relay - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) - theirControl.InjectLightHouseAddr(myVpnIpNet.IP, myUdpAddr) + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, theirControl) @@ -258,29 +446,31 @@ func TestUncleanShutdownRaceLoser(t *testing.T) { theirControl.Start() r.Log("Trigger a handshake from me to them") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) p := r.RouteForAllUntilTxTun(theirControl) - assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet.IP, theirVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) r.Log("Nuke my hostmap") myHostmap := myControl.GetHostmap() - myHostmap.Hosts = map[iputil.VpnIp]*nebula.HostInfo{} + myHostmap.Lock() + myHostmap.Hosts = map[netip.Addr]*nebula.HostInfo{} myHostmap.Indexes = map[uint32]*nebula.HostInfo{} myHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{} + myHostmap.Unlock() - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me again")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me again"))) p = r.RouteForAllUntilTxTun(theirControl) - assertUdpPacket(t, []byte("Hi from me again"), p, myVpnIpNet.IP, theirVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from me again"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) r.Log("Assert the tunnel works") - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) r.Log("Wait for the dead index to go away") - start := len(theirControl.GetHostmap().Indexes) + start := theirControl.GetHostmapIndexCount() for { - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) - if len(theirControl.GetHostmap().Indexes) < start { + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + if theirControl.GetHostmapIndexCount() < start { break } time.Sleep(time.Second) @@ -290,13 +480,14 @@ func TestUncleanShutdownRaceLoser(t *testing.T) { } func TestUncleanShutdownRaceWinner(t *testing.T) { - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 1}, nil) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil) + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) // Teach my how to get to the relay and that their can be reached via the relay - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) - theirControl.InjectLightHouseAddr(myVpnIpNet.IP, myUdpAddr) + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, theirControl) @@ -307,31 +498,33 @@ func TestUncleanShutdownRaceWinner(t *testing.T) { theirControl.Start() r.Log("Trigger a handshake from me to them") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) p := r.RouteForAllUntilTxTun(theirControl) - assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet.IP, theirVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) r.RenderHostmaps("Final hostmaps", myControl, theirControl) r.Log("Nuke my hostmap") theirHostmap := theirControl.GetHostmap() - theirHostmap.Hosts = map[iputil.VpnIp]*nebula.HostInfo{} + theirHostmap.Lock() + theirHostmap.Hosts = map[netip.Addr]*nebula.HostInfo{} theirHostmap.Indexes = map[uint32]*nebula.HostInfo{} theirHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{} + theirHostmap.Unlock() - theirControl.InjectTunUDPPacket(myVpnIpNet.IP, 80, 80, []byte("Hi from them again")) + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them again"))) p = r.RouteForAllUntilTxTun(myControl) - assertUdpPacket(t, []byte("Hi from them again"), p, theirVpnIpNet.IP, myVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from them again"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80) r.RenderHostmaps("Derp hostmaps", myControl, theirControl) r.Log("Assert the tunnel works") - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) r.Log("Wait for the dead index to go away") - start := len(myControl.GetHostmap().Indexes) + start := myControl.GetHostmapIndexCount() for { - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) - if len(myControl.GetHostmap().Indexes) < start { + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + if myControl.GetHostmapIndexCount() < start { break } time.Sleep(time.Second) @@ -341,15 +534,46 @@ func TestUncleanShutdownRaceWinner(t *testing.T) { } func TestRelays(t *testing.T) { - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, _, _ := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 1}, m{"relay": m{"use_relays": true}}) - relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(ca, caKey, "relay ", net.IP{10, 0, 0, 128}, m{"relay": m{"am_relay": true}}) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them ", net.IP{10, 0, 0, 2}, m{"relay": m{"use_relays": true}}) + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) + + // Teach my how to get to the relay and that their can be reached via the relay + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + // Build a router so we don't have to reason who gets which packet + r := router.NewR(t, myControl, relayControl, theirControl) + defer r.RenderFlow() + + // Start the servers + myControl.Start() + relayControl.Start() + theirControl.Start() + + t.Log("Trigger a handshake from me to them via the relay") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + + p := r.RouteForAllUntilTxTun(theirControl) + r.Log("Assert the tunnel works") + assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) + r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl) +} + +func TestRelaysDontCareAboutIps(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay ", "2001::9999/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) // Teach my how to get to the relay and that their can be reached via the relay - myControl.InjectLightHouseAddr(relayVpnIpNet.IP, relayUdpAddr) - myControl.InjectRelays(theirVpnIpNet.IP, []net.IP{relayVpnIpNet.IP}) - relayControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, relayControl, theirControl) @@ -361,31 +585,227 @@ func TestRelays(t *testing.T) { theirControl.Start() t.Log("Trigger a handshake from me to them via the relay") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) p := r.RouteForAllUntilTxTun(theirControl) r.Log("Assert the tunnel works") - assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet.IP, theirVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) + r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl) +} + +func TestReestablishRelays(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) + + // Teach my how to get to the relay and that their can be reached via the relay + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + // Build a router so we don't have to reason who gets which packet + r := router.NewR(t, myControl, relayControl, theirControl) + defer r.RenderFlow() + + // Start the servers + myControl.Start() + relayControl.Start() + theirControl.Start() + + t.Log("Trigger a handshake from me to them via the relay") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + + p := r.RouteForAllUntilTxTun(theirControl) + r.Log("Assert the tunnel works") + assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) + + t.Log("Ensure packet traversal from them to me via the relay") + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))) + + p = r.RouteForAllUntilTxTun(myControl) + r.Log("Assert the tunnel works") + assertUdpPacket(t, []byte("Hi from them"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80) + + // If we break the relay's connection to 'them', 'me' needs to detect and recover the connection + r.Log("Close the tunnel") + relayControl.CloseTunnel(theirVpnIpNet[0].Addr(), true) + + start := myControl.GetHostmapIndexCount() + curIndexes := myControl.GetHostmapIndexCount() + for curIndexes >= start { + curIndexes = myControl.GetHostmapIndexCount() + r.Logf("Wait for the dead index to go away:start=%v indexes, current=%v indexes", start, curIndexes) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me should fail"))) + + r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType { + return router.RouteAndExit + }) + time.Sleep(2 * time.Second) + } + r.Log("Dead index went away. Woot!") + r.RenderHostmaps("Me removed hostinfo", myControl, relayControl, theirControl) + // Next packet should re-establish a relayed connection and work just great. + + t.Logf("Assert the tunnel...") + for { + t.Log("RouteForAllUntilTxTun") + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + + p = r.RouteForAllUntilTxTun(theirControl) + r.Log("Assert the tunnel works") + packet := gopacket.NewPacket(p, layers.LayerTypeIPv4, gopacket.Lazy) + v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4) + if slices.Compare(v4.SrcIP, myVpnIpNet[0].Addr().AsSlice()) != 0 { + t.Logf("SrcIP is unexpected...this is not the packet I'm looking for. Keep looking") + continue + } + if slices.Compare(v4.DstIP, theirVpnIpNet[0].Addr().AsSlice()) != 0 { + t.Logf("DstIP is unexpected...this is not the packet I'm looking for. Keep looking") + continue + } + + udp := packet.Layer(layers.LayerTypeUDP).(*layers.UDP) + if udp == nil { + t.Log("Not a UDP packet. This is not the packet I'm looking for. Keep looking") + continue + } + data := packet.ApplicationLayer() + if data == nil { + t.Log("No data found in packet. This is not the packet I'm looking for. Keep looking.") + continue + } + if string(data.Payload()) != "Hi from me" { + t.Logf("Unexpected payload: '%v', keep looking", string(data.Payload())) + continue + } + t.Log("I found my lost packet. I am so happy.") + break + } + t.Log("Assert the tunnel works the other way, too") + for { + t.Log("RouteForAllUntilTxTun") + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))) + + p = r.RouteForAllUntilTxTun(myControl) + r.Log("Assert the tunnel works") + packet := gopacket.NewPacket(p, layers.LayerTypeIPv4, gopacket.Lazy) + v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4) + if slices.Compare(v4.DstIP, myVpnIpNet[0].Addr().AsSlice()) != 0 { + t.Logf("Dst is unexpected...this is not the packet I'm looking for. Keep looking") + continue + } + if slices.Compare(v4.SrcIP, theirVpnIpNet[0].Addr().AsSlice()) != 0 { + t.Logf("SrcIP is unexpected...this is not the packet I'm looking for. Keep looking") + continue + } + + udp := packet.Layer(layers.LayerTypeUDP).(*layers.UDP) + if udp == nil { + t.Log("Not a UDP packet. This is not the packet I'm looking for. Keep looking") + continue + } + data := packet.ApplicationLayer() + if data == nil { + t.Log("No data found in packet. This is not the packet I'm looking for. Keep looking.") + continue + } + if string(data.Payload()) != "Hi from them" { + t.Logf("Unexpected payload: '%v', keep looking", string(data.Payload())) + continue + } + t.Log("I found my lost packet. I am so happy.") + break + } + r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl) + +} + +func TestRelayHandshakeOverDisestablishedEntry(t *testing.T) { + t.Parallel() + // If them tears down the tunnel while me keeps Established relay state, me's next + // handshake flows through the relay with no fresh CreateRelayRequest and lands on + // them's Disestablished terminal relay entry. them must re-establish that entry, or + // its first transmit deletes its only relay and the tunnel is born transmit-dead: + // them can receive but every send is silently dropped. + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) + + // Teach my how to get to the relay and that their can be reached via the relay + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + // Build a router so we don't have to reason who gets which packet + r := router.NewR(t, myControl, relayControl, theirControl) + defer r.RenderFlow() + + // Start the servers + myControl.Start() + relayControl.Start() + theirControl.Start() + + t.Log("Trigger a handshake from me to them via the relay") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + + p := r.RouteForAllUntilTxTun(theirControl) + assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) + oldIdx := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false).LocalIndex + + t.Log("Close the tunnel on them only, marking their relay entry Disestablished") + theirControl.CloseTunnel(myVpnIpNet[0].Addr(), true) + + t.Log("Re-handshake from me, riding the still-Established relay state") + myControl.ReHandshake(theirVpnIpNet[0].Addr()) + for { + h := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) + if h != nil && h.LocalIndex != oldIdx && h.RemoteIndex != 0 { + break + } + r.RouteForAllExitFunc(func(*udp.Packet, *nebula.Control) router.ExitType { + return router.RouteAndExit + }) + } + + hAtThem := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + require.NotNil(t, hAtThem, "them should have completed the relayed handshake") + require.Equal(t, []netip.Addr{relayVpnIpNet[0].Addr()}, hAtThem.CurrentRelaysToMe, "them should know a relay for the new tunnel") + + t.Log("Send from them to me; their only relay entry must survive the transmit") + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))) + require.Never(t, func() bool { + h := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + return h == nil || len(h.CurrentRelaysToMe) == 0 + }, time.Second, 10*time.Millisecond, "them deleted its only relay entry; the tunnel is permanently transmit-dead") + + p = r.RouteForAllUntilTxTun(myControl) + assertUdpPacket(t, []byte("Hi from them"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80) r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl) - //TODO: assert we actually used the relay even though it should be impossible for a tunnel to have occurred without it } func TestStage1RaceRelays(t *testing.T) { + t.Parallel() //NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 1}, m{"relay": m{"use_relays": true}}) - relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(ca, caKey, "relay ", net.IP{10, 0, 0, 128}, m{"relay": m{"am_relay": true}}) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them ", net.IP{10, 0, 0, 2}, m{"relay": m{"use_relays": true}}) + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) // Teach my how to get to the relay and that their can be reached via the relay - myControl.InjectLightHouseAddr(relayVpnIpNet.IP, relayUdpAddr) - theirControl.InjectLightHouseAddr(relayVpnIpNet.IP, relayUdpAddr) + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + theirControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) - myControl.InjectRelays(theirVpnIpNet.IP, []net.IP{relayVpnIpNet.IP}) - theirControl.InjectRelays(myVpnIpNet.IP, []net.IP{relayVpnIpNet.IP}) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + theirControl.InjectRelays(myVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) - relayControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) - relayControl.InjectLightHouseAddr(myVpnIpNet.IP, myUdpAddr) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + relayControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, relayControl, theirControl) @@ -397,14 +817,14 @@ func TestStage1RaceRelays(t *testing.T) { theirControl.Start() r.Log("Get a tunnel between me and relay") - assertTunnel(t, myVpnIpNet.IP, relayVpnIpNet.IP, myControl, relayControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), myControl, relayControl, r) r.Log("Get a tunnel between them and relay") - assertTunnel(t, theirVpnIpNet.IP, relayVpnIpNet.IP, theirControl, relayControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), theirControl, relayControl, r) r.Log("Trigger a handshake from both them and me via relay to them and me") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) - theirControl.InjectTunUDPPacket(myVpnIpNet.IP, 80, 80, []byte("Hi from them")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))) r.Log("Wait for a packet from them to me") p := r.RouteForAllUntilTxTun(myControl) @@ -415,27 +835,25 @@ func TestStage1RaceRelays(t *testing.T) { myControl.Stop() theirControl.Stop() relayControl.Stop() - // - ////TODO: assert hostmaps } func TestStage1RaceRelays2(t *testing.T) { + t.Parallel() //NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 1}, m{"relay": m{"use_relays": true}}) - relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(ca, caKey, "relay ", net.IP{10, 0, 0, 128}, m{"relay": m{"am_relay": true}}) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them ", net.IP{10, 0, 0, 2}, m{"relay": m{"use_relays": true}}) - l := NewTestLogger() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) // Teach my how to get to the relay and that their can be reached via the relay - myControl.InjectLightHouseAddr(relayVpnIpNet.IP, relayUdpAddr) - theirControl.InjectLightHouseAddr(relayVpnIpNet.IP, relayUdpAddr) + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + theirControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) - myControl.InjectRelays(theirVpnIpNet.IP, []net.IP{relayVpnIpNet.IP}) - theirControl.InjectRelays(myVpnIpNet.IP, []net.IP{relayVpnIpNet.IP}) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + theirControl.InjectRelays(myVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) - relayControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) - relayControl.InjectLightHouseAddr(myVpnIpNet.IP, myUdpAddr) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + relayControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, relayControl, theirControl) @@ -447,76 +865,66 @@ func TestStage1RaceRelays2(t *testing.T) { theirControl.Start() r.Log("Get a tunnel between me and relay") - l.Info("Get a tunnel between me and relay") - assertTunnel(t, myVpnIpNet.IP, relayVpnIpNet.IP, myControl, relayControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), myControl, relayControl, r) r.Log("Get a tunnel between them and relay") - l.Info("Get a tunnel between them and relay") - assertTunnel(t, theirVpnIpNet.IP, relayVpnIpNet.IP, theirControl, relayControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), theirControl, relayControl, r) r.Log("Trigger a handshake from both them and me via relay to them and me") - l.Info("Trigger a handshake from both them and me via relay to them and me") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) - theirControl.InjectTunUDPPacket(myVpnIpNet.IP, 80, 80, []byte("Hi from them")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))) //r.RouteUntilAfterMsgType(myControl, header.Control, header.MessageNone) //r.RouteUntilAfterMsgType(theirControl, header.Control, header.MessageNone) - r.Log("Wait for a packet from them to me") - l.Info("Wait for a packet from them to me; myControl") + r.Log("Wait for a packet from them to me; myControl") r.RouteForAllUntilTxTun(myControl) - l.Info("Wait for a packet from them to me; theirControl") + r.Log("Wait for a packet from them to me; theirControl") r.RouteForAllUntilTxTun(theirControl) r.Log("Assert the tunnel works") - l.Info("Assert the tunnel works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) t.Log("Wait until we remove extra tunnels") - l.Info("Wait until we remove extra tunnels") - l.WithFields( - logrus.Fields{ - "myControl": len(myControl.GetHostmap().Indexes), - "theirControl": len(theirControl.GetHostmap().Indexes), - "relayControl": len(relayControl.GetHostmap().Indexes), - }).Info("Waiting for hostinfos to be removed...") - hostInfos := len(myControl.GetHostmap().Indexes) + len(theirControl.GetHostmap().Indexes) + len(relayControl.GetHostmap().Indexes) + t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d", + myControl.GetHostmapIndexCount(), + theirControl.GetHostmapIndexCount(), + relayControl.GetHostmapIndexCount(), + ) + hostInfos := myControl.GetHostmapIndexCount() + theirControl.GetHostmapIndexCount() + relayControl.GetHostmapIndexCount() retries := 60 for hostInfos > 6 && retries > 0 { - hostInfos = len(myControl.GetHostmap().Indexes) + len(theirControl.GetHostmap().Indexes) + len(relayControl.GetHostmap().Indexes) - l.WithFields( - logrus.Fields{ - "myControl": len(myControl.GetHostmap().Indexes), - "theirControl": len(theirControl.GetHostmap().Indexes), - "relayControl": len(relayControl.GetHostmap().Indexes), - }).Info("Waiting for hostinfos to be removed...") - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + hostInfos = myControl.GetHostmapIndexCount() + theirControl.GetHostmapIndexCount() + relayControl.GetHostmapIndexCount() + t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d", + myControl.GetHostmapIndexCount(), + theirControl.GetHostmapIndexCount(), + relayControl.GetHostmapIndexCount(), + ) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) t.Log("Connection manager hasn't ticked yet") time.Sleep(time.Second) retries-- } r.Log("Assert the tunnel works") - l.Info("Assert the tunnel works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) myControl.Stop() theirControl.Stop() relayControl.Stop() - - // - ////TODO: assert hostmaps } + func TestRehandshakingRelays(t *testing.T) { - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, _, _ := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 1}, m{"relay": m{"use_relays": true}}) - relayControl, relayVpnIpNet, relayUdpAddr, relayConfig := newSimpleServer(ca, caKey, "relay ", net.IP{10, 0, 0, 128}, m{"relay": m{"am_relay": true}}) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them ", net.IP{10, 0, 0, 2}, m{"relay": m{"use_relays": true}}) + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, relayConfig := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) // Teach my how to get to the relay and that their can be reached via the relay - myControl.InjectLightHouseAddr(relayVpnIpNet.IP, relayUdpAddr) - myControl.InjectRelays(theirVpnIpNet.IP, []net.IP{relayVpnIpNet.IP}) - relayControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, relayControl, theirControl) @@ -528,19 +936,19 @@ func TestRehandshakingRelays(t *testing.T) { theirControl.Start() t.Log("Trigger a handshake from me to them via the relay") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) p := r.RouteForAllUntilTxTun(theirControl) r.Log("Assert the tunnel works") - assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet.IP, theirVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl) // When I update the certificate for the relay, both me and them will have 2 host infos for the relay, // and the main host infos will not have any relay state to handle the me<->relay<->them tunnel. r.Log("Renew relay certificate and spin until me and them sees it") - _, _, myNextPrivKey, myNextPEM := NewTestCert(ca, caKey, "relay", time.Now(), time.Now().Add(5*time.Minute), relayVpnIpNet, nil, []string{"new group"}) + _, _, myNextPrivKey, myNextPEM := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "relay", time.Now(), time.Now().Add(5*time.Minute), relayVpnIpNet, nil, []string{"new group"}) - caB, err := ca.MarshalToPEM() + caB, err := ca.MarshalPEM() if err != nil { panic(err) } @@ -551,14 +959,14 @@ func TestRehandshakingRelays(t *testing.T) { "key": string(myNextPrivKey), } rc, err := yaml.Marshal(relayConfig.Settings) - assert.NoError(t, err) + require.NoError(t, err) relayConfig.ReloadConfigString(string(rc)) for { r.Log("Assert the tunnel works between myVpnIpNet and relayVpnIpNet") - assertTunnel(t, myVpnIpNet.IP, relayVpnIpNet.IP, myControl, relayControl, r) - c := myControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(relayVpnIpNet.IP), false) - if len(c.Cert.Details.Groups) != 0 { + assertTunnel(t, myVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), myControl, relayControl, r) + c := myControl.GetHostInfoByVpnAddr(relayVpnIpNet[0].Addr(), false) + if len(c.Cert.Groups()) != 0 { // We have a new certificate now r.Log("Certificate between my and relay is updated!") break @@ -569,9 +977,9 @@ func TestRehandshakingRelays(t *testing.T) { for { r.Log("Assert the tunnel works between theirVpnIpNet and relayVpnIpNet") - assertTunnel(t, theirVpnIpNet.IP, relayVpnIpNet.IP, theirControl, relayControl, r) - c := theirControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(relayVpnIpNet.IP), false) - if len(c.Cert.Details.Groups) != 0 { + assertTunnel(t, theirVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), theirControl, relayControl, r) + c := theirControl.GetHostInfoByVpnAddr(relayVpnIpNet[0].Addr(), false) + if len(c.Cert.Groups()) != 0 { // We have a new certificate now r.Log("Certificate between their and relay is updated!") break @@ -581,29 +989,29 @@ func TestRehandshakingRelays(t *testing.T) { } r.Log("Assert the relay tunnel still works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl) // We should have two hostinfos on all sides - for len(myControl.GetHostmap().Indexes) != 2 { - t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(myControl.GetHostmap().Indexes)) + for myControl.GetHostmapIndexCount() != 2 { + t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", myControl.GetHostmapIndexCount()) r.Log("Assert the relay tunnel still works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) r.Log("yupitdoes") time.Sleep(time.Second) } t.Logf("myControl hostinfos got cleaned up!") - for len(theirControl.GetHostmap().Indexes) != 2 { - t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(theirControl.GetHostmap().Indexes)) + for theirControl.GetHostmapIndexCount() != 2 { + t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", theirControl.GetHostmapIndexCount()) r.Log("Assert the relay tunnel still works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) r.Log("yupitdoes") time.Sleep(time.Second) } t.Logf("theirControl hostinfos got cleaned up!") - for len(relayControl.GetHostmap().Indexes) != 2 { - t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(relayControl.GetHostmap().Indexes)) + for relayControl.GetHostmapIndexCount() != 2 { + t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", relayControl.GetHostmapIndexCount()) r.Log("Assert the relay tunnel still works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) r.Log("yupitdoes") time.Sleep(time.Second) } @@ -611,16 +1019,17 @@ func TestRehandshakingRelays(t *testing.T) { } func TestRehandshakingRelaysPrimary(t *testing.T) { + t.Parallel() // This test is the same as TestRehandshakingRelays but one of the terminal types is a primary swap winner - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, _, _ := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 128}, m{"relay": m{"use_relays": true}}) - relayControl, relayVpnIpNet, relayUdpAddr, relayConfig := newSimpleServer(ca, caKey, "relay ", net.IP{10, 0, 0, 1}, m{"relay": m{"am_relay": true}}) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them ", net.IP{10, 0, 0, 2}, m{"relay": m{"use_relays": true}}) + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.128/24", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, relayConfig := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.1/24", m{"relay": m{"am_relay": true}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}}) // Teach my how to get to the relay and that their can be reached via the relay - myControl.InjectLightHouseAddr(relayVpnIpNet.IP, relayUdpAddr) - myControl.InjectRelays(theirVpnIpNet.IP, []net.IP{relayVpnIpNet.IP}) - relayControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) + myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()}) + relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, relayControl, theirControl) @@ -632,19 +1041,19 @@ func TestRehandshakingRelaysPrimary(t *testing.T) { theirControl.Start() t.Log("Trigger a handshake from me to them via the relay") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) p := r.RouteForAllUntilTxTun(theirControl) r.Log("Assert the tunnel works") - assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet.IP, theirVpnIpNet.IP, 80, 80) + assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80) r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl) // When I update the certificate for the relay, both me and them will have 2 host infos for the relay, // and the main host infos will not have any relay state to handle the me<->relay<->them tunnel. r.Log("Renew relay certificate and spin until me and them sees it") - _, _, myNextPrivKey, myNextPEM := NewTestCert(ca, caKey, "relay", time.Now(), time.Now().Add(5*time.Minute), relayVpnIpNet, nil, []string{"new group"}) + _, _, myNextPrivKey, myNextPEM := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "relay", time.Now(), time.Now().Add(5*time.Minute), relayVpnIpNet, nil, []string{"new group"}) - caB, err := ca.MarshalToPEM() + caB, err := ca.MarshalPEM() if err != nil { panic(err) } @@ -655,14 +1064,14 @@ func TestRehandshakingRelaysPrimary(t *testing.T) { "key": string(myNextPrivKey), } rc, err := yaml.Marshal(relayConfig.Settings) - assert.NoError(t, err) + require.NoError(t, err) relayConfig.ReloadConfigString(string(rc)) for { r.Log("Assert the tunnel works between myVpnIpNet and relayVpnIpNet") - assertTunnel(t, myVpnIpNet.IP, relayVpnIpNet.IP, myControl, relayControl, r) - c := myControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(relayVpnIpNet.IP), false) - if len(c.Cert.Details.Groups) != 0 { + assertTunnel(t, myVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), myControl, relayControl, r) + c := myControl.GetHostInfoByVpnAddr(relayVpnIpNet[0].Addr(), false) + if len(c.Cert.Groups()) != 0 { // We have a new certificate now r.Log("Certificate between my and relay is updated!") break @@ -673,9 +1082,9 @@ func TestRehandshakingRelaysPrimary(t *testing.T) { for { r.Log("Assert the tunnel works between theirVpnIpNet and relayVpnIpNet") - assertTunnel(t, theirVpnIpNet.IP, relayVpnIpNet.IP, theirControl, relayControl, r) - c := theirControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(relayVpnIpNet.IP), false) - if len(c.Cert.Details.Groups) != 0 { + assertTunnel(t, theirVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), theirControl, relayControl, r) + c := theirControl.GetHostInfoByVpnAddr(relayVpnIpNet[0].Addr(), false) + if len(c.Cert.Groups()) != 0 { // We have a new certificate now r.Log("Certificate between their and relay is updated!") break @@ -685,29 +1094,29 @@ func TestRehandshakingRelaysPrimary(t *testing.T) { } r.Log("Assert the relay tunnel still works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl) // We should have two hostinfos on all sides - for len(myControl.GetHostmap().Indexes) != 2 { - t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(myControl.GetHostmap().Indexes)) + for myControl.GetHostmapIndexCount() != 2 { + t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", myControl.GetHostmapIndexCount()) r.Log("Assert the relay tunnel still works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) r.Log("yupitdoes") time.Sleep(time.Second) } t.Logf("myControl hostinfos got cleaned up!") - for len(theirControl.GetHostmap().Indexes) != 2 { - t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(theirControl.GetHostmap().Indexes)) + for theirControl.GetHostmapIndexCount() != 2 { + t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", theirControl.GetHostmapIndexCount()) r.Log("Assert the relay tunnel still works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) r.Log("yupitdoes") time.Sleep(time.Second) } t.Logf("theirControl hostinfos got cleaned up!") - for len(relayControl.GetHostmap().Indexes) != 2 { - t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(relayControl.GetHostmap().Indexes)) + for relayControl.GetHostmapIndexCount() != 2 { + t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", relayControl.GetHostmapIndexCount()) r.Log("Assert the relay tunnel still works") - assertTunnel(t, theirVpnIpNet.IP, myVpnIpNet.IP, theirControl, myControl, r) + assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) r.Log("yupitdoes") time.Sleep(time.Second) } @@ -715,13 +1124,14 @@ func TestRehandshakingRelaysPrimary(t *testing.T) { } func TestRehandshaking(t *testing.T) { - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, myUdpAddr, myConfig := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 2}, nil) - theirControl, theirVpnIpNet, theirUdpAddr, theirConfig := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 1}, nil) + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, myConfig := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.2/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, theirConfig := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.1/24", nil) // Put their info in our lighthouse and vice versa - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) - theirControl.InjectLightHouseAddr(myVpnIpNet.IP, myUdpAddr) + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, theirControl) @@ -732,14 +1142,14 @@ func TestRehandshaking(t *testing.T) { theirControl.Start() t.Log("Stand up a tunnel between me and them") - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) r.RenderHostmaps("Starting hostmaps", myControl, theirControl) r.Log("Renew my certificate and spin until their sees it") - _, _, myNextPrivKey, myNextPEM := NewTestCert(ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), myVpnIpNet, nil, []string{"new group"}) + _, _, myNextPrivKey, myNextPEM := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), myVpnIpNet, nil, []string{"new group"}) - caB, err := ca.MarshalToPEM() + caB, err := ca.MarshalPEM() if err != nil { panic(err) } @@ -750,13 +1160,13 @@ func TestRehandshaking(t *testing.T) { "key": string(myNextPrivKey), } rc, err := yaml.Marshal(myConfig.Settings) - assert.NoError(t, err) + require.NoError(t, err) myConfig.ReloadConfigString(string(rc)) for { - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) - c := theirControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(myVpnIpNet.IP), false) - if len(c.Cert.Details.Groups) != 0 { + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + if len(c.Cert.Groups()) != 0 { // We have a new certificate now break } @@ -764,37 +1174,38 @@ func TestRehandshaking(t *testing.T) { time.Sleep(time.Second) } + r.Log("Got the new cert") // Flip their firewall to only allowing the new group to catch the tunnels reverting incorrectly rc, err = yaml.Marshal(theirConfig.Settings) - assert.NoError(t, err) + require.NoError(t, err) var theirNewConfig m - assert.NoError(t, yaml.Unmarshal(rc, &theirNewConfig)) - theirFirewall := theirNewConfig["firewall"].(map[interface{}]interface{}) + require.NoError(t, yaml.Unmarshal(rc, &theirNewConfig)) + theirFirewall := theirNewConfig["firewall"].(map[string]any) theirFirewall["inbound"] = []m{{ "proto": "any", "port": "any", "group": "new group", }} rc, err = yaml.Marshal(theirNewConfig) - assert.NoError(t, err) + require.NoError(t, err) theirConfig.ReloadConfigString(string(rc)) r.Log("Spin until there is only 1 tunnel") - for len(myControl.GetHostmap().Indexes)+len(theirControl.GetHostmap().Indexes) > 2 { - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + for myControl.GetHostmapIndexCount()+theirControl.GetHostmapIndexCount() > 2 { + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) t.Log("Connection manager hasn't ticked yet") time.Sleep(time.Second) } - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) myFinalHostmapHosts := myControl.ListHostmapHosts(false) myFinalHostmapIndexes := myControl.ListHostmapIndexes(false) theirFinalHostmapHosts := theirControl.ListHostmapHosts(false) theirFinalHostmapIndexes := theirControl.ListHostmapIndexes(false) // Make sure the correct tunnel won - c := theirControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(myVpnIpNet.IP), false) - assert.Contains(t, c.Cert.Details.Groups, "new group") + c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + assert.Contains(t, c.Cert.Groups(), "new group") // We should only have a single tunnel now on both sides assert.Len(t, myFinalHostmapHosts, 1) @@ -809,15 +1220,16 @@ func TestRehandshaking(t *testing.T) { } func TestRehandshakingLoser(t *testing.T) { + t.Parallel() // The purpose of this test is that the race loser renews their certificate and rehandshakes. The final tunnel // Should be the one with the new certificate - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, myUdpAddr, myConfig := newSimpleServer(ca, caKey, "me ", net.IP{10, 0, 0, 2}, nil) - theirControl, theirVpnIpNet, theirUdpAddr, theirConfig := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 1}, nil) + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, myConfig := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.2/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, theirConfig := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.1/24", nil) // Put their info in our lighthouse and vice versa - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) - theirControl.InjectLightHouseAddr(myVpnIpNet.IP, myUdpAddr) + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) // Build a router so we don't have to reason who gets which packet r := router.NewR(t, myControl, theirControl) @@ -828,18 +1240,17 @@ func TestRehandshakingLoser(t *testing.T) { theirControl.Start() t.Log("Stand up a tunnel between me and them") - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) - tt1 := myControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(theirVpnIpNet.IP), false) - tt2 := theirControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(myVpnIpNet.IP), false) - fmt.Println(tt1.LocalIndex, tt2.LocalIndex) + myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) + theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) r.RenderHostmaps("Starting hostmaps", myControl, theirControl) r.Log("Renew their certificate and spin until mine sees it") - _, _, theirNextPrivKey, theirNextPEM := NewTestCert(ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), theirVpnIpNet, nil, []string{"their new group"}) + _, _, theirNextPrivKey, theirNextPEM := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), theirVpnIpNet, nil, []string{"their new group"}) - caB, err := ca.MarshalToPEM() + caB, err := ca.MarshalPEM() if err != nil { panic(err) } @@ -850,15 +1261,14 @@ func TestRehandshakingLoser(t *testing.T) { "key": string(theirNextPrivKey), } rc, err := yaml.Marshal(theirConfig.Settings) - assert.NoError(t, err) + require.NoError(t, err) theirConfig.ReloadConfigString(string(rc)) for { - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) - theirCertInMe := myControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(theirVpnIpNet.IP), false) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + theirCertInMe := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) - _, theirNewGroup := theirCertInMe.Cert.Details.InvertedGroups["their new group"] - if theirNewGroup { + if slices.Contains(theirCertInMe.Cert.Groups(), "their new group") { break } @@ -867,35 +1277,35 @@ func TestRehandshakingLoser(t *testing.T) { // Flip my firewall to only allowing the new group to catch the tunnels reverting incorrectly rc, err = yaml.Marshal(myConfig.Settings) - assert.NoError(t, err) + require.NoError(t, err) var myNewConfig m - assert.NoError(t, yaml.Unmarshal(rc, &myNewConfig)) - theirFirewall := myNewConfig["firewall"].(map[interface{}]interface{}) + require.NoError(t, yaml.Unmarshal(rc, &myNewConfig)) + theirFirewall := myNewConfig["firewall"].(map[string]any) theirFirewall["inbound"] = []m{{ "proto": "any", "port": "any", "group": "their new group", }} rc, err = yaml.Marshal(myNewConfig) - assert.NoError(t, err) + require.NoError(t, err) myConfig.ReloadConfigString(string(rc)) r.Log("Spin until there is only 1 tunnel") - for len(myControl.GetHostmap().Indexes)+len(theirControl.GetHostmap().Indexes) > 2 { - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + for myControl.GetHostmapIndexCount()+theirControl.GetHostmapIndexCount() > 2 { + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) t.Log("Connection manager hasn't ticked yet") time.Sleep(time.Second) } - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) myFinalHostmapHosts := myControl.ListHostmapHosts(false) myFinalHostmapIndexes := myControl.ListHostmapIndexes(false) theirFinalHostmapHosts := theirControl.ListHostmapHosts(false) theirFinalHostmapIndexes := theirControl.ListHostmapIndexes(false) // Make sure the correct tunnel won - theirCertInMe := myControl.GetHostInfoByVpnIp(iputil.Ip2VpnIp(theirVpnIpNet.IP), false) - assert.Contains(t, theirCertInMe.Cert.Details.Groups, "their new group") + theirCertInMe := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) + assert.Contains(t, theirCertInMe.Cert.Groups(), "their new group") // We should only have a single tunnel now on both sides assert.Len(t, myFinalHostmapHosts, 1) @@ -909,16 +1319,17 @@ func TestRehandshakingLoser(t *testing.T) { } func TestRaceRegression(t *testing.T) { + t.Parallel() // This test forces stage 1, stage 2, stage 1 to be received by me from them // We had a bug where we were not finding the duplicate handshake and responding to the final stage 1 which // caused a cross-linked hostinfo - ca, _, caKey, _ := NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(ca, caKey, "me", net.IP{10, 0, 0, 1}, nil) - theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(ca, caKey, "them", net.IP{10, 0, 0, 2}, nil) + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) // Put their info in our lighthouse - myControl.InjectLightHouseAddr(theirVpnIpNet.IP, theirUdpAddr) - theirControl.InjectLightHouseAddr(myVpnIpNet.IP, myUdpAddr) + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) // Start the servers myControl.Start() @@ -932,8 +1343,8 @@ func TestRaceRegression(t *testing.T) { //them rx stage:2 initiatorIndex=120607833 responderIndex=4209862089 t.Log("Start both handshakes") - myControl.InjectTunUDPPacket(theirVpnIpNet.IP, 80, 80, []byte("Hi from me")) - theirControl.InjectTunUDPPacket(myVpnIpNet.IP, 80, 80, []byte("Hi from them")) + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))) t.Log("Get both stage 1") myStage1ForThem := myControl.GetFromUDP(true) @@ -944,7 +1355,6 @@ func TestRaceRegression(t *testing.T) { myControl.InjectUDPPacket(theirStage1ForMe) theirControl.InjectUDPPacket(myStage1ForThem) - //TODO: ensure stage 2 t.Log("Get both stage 2") myStage2ForThem := myControl.GetFromUDP(true) theirStage2ForMe := theirControl.GetFromUDP(true) @@ -963,14 +1373,308 @@ func TestRaceRegression(t *testing.T) { r.RenderHostmaps("Starting hostmaps", myControl, theirControl) t.Log("Make sure the tunnel still works") - assertTunnel(t, myVpnIpNet.IP, theirVpnIpNet.IP, myControl, theirControl, r) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + myControl.Stop() + theirControl.Stop() +} + +func TestV2NonPrimaryWithLighthouse(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh ", "10.128.0.1/24, ff::1/64", m{"lighthouse": m{"am_lighthouse": true}}) + + o := m{ + "static_host_map": m{ + lhVpnIpNet[1].Addr().String(): []string{lhUdpAddr.String()}, + }, + "lighthouse": m{ + "hosts": []string{lhVpnIpNet[1].Addr().String()}, + "local_allow_list": m{ + // Try and block our lighthouse updates from using the actual addresses assigned to this computer + // If we start discovering addresses the test router doesn't know about then test traffic cant flow + "10.0.0.0/24": true, + "::/0": false, + }, + }, + } + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me ", "10.128.0.2/24, ff::2/64", o) + theirControl, theirVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24, ff::3/64", o) + + // Build a router so we don't have to reason who gets which packet + r := router.NewR(t, lhControl, myControl, theirControl) + defer r.RenderFlow() + + // Start the servers + lhControl.Start() + myControl.Start() + theirControl.Start() + + t.Log("Stand up an ipv6 tunnel between me and them") + assert.True(t, myVpnIpNet[1].Addr().Is6()) + assert.True(t, theirVpnIpNet[1].Addr().Is6()) + assertTunnel(t, myVpnIpNet[1].Addr(), theirVpnIpNet[1].Addr(), myControl, theirControl, r) + + lhControl.Stop() + myControl.Stop() + theirControl.Stop() +} + +func TestV2NonPrimaryWithOffNetLighthouse(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh ", "2001::1/64", m{"lighthouse": m{"am_lighthouse": true}}) + + o := m{ + "static_host_map": m{ + lhVpnIpNet[0].Addr().String(): []string{lhUdpAddr.String()}, + }, + "lighthouse": m{ + "hosts": []string{lhVpnIpNet[0].Addr().String()}, + "local_allow_list": m{ + // Try and block our lighthouse updates from using the actual addresses assigned to this computer + // If we start discovering addresses the test router doesn't know about then test traffic cant flow + "10.0.0.0/24": true, + "::/0": false, + }, + }, + } + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me ", "10.128.0.2/24, ff::2/64", o) + theirControl, theirVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24, ff::3/64", o) + + // Build a router so we don't have to reason who gets which packet + r := router.NewR(t, lhControl, myControl, theirControl) + defer r.RenderFlow() + + // Start the servers + lhControl.Start() + myControl.Start() + theirControl.Start() + + t.Log("Stand up an ipv6 tunnel between me and them") + assert.True(t, myVpnIpNet[1].Addr().Is6()) + assert.True(t, theirVpnIpNet[1].Addr().Is6()) + assertTunnel(t, myVpnIpNet[1].Addr(), theirVpnIpNet[1].Addr(), myControl, theirControl, r) + lhControl.Stop() myControl.Stop() theirControl.Stop() } -//TODO: test -// Race winner renews and handshakes -// Race loser renews and handshakes -// Does race winner repin the cert to old? -//TODO: add a test with many lies +func TestLighthouseUpdateOnReload(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + // Create the lighthouse + lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{"lighthouse": m{"am_lighthouse": true}}) + + // Create a client with NO lighthouse configured and a long update interval. + // The initial SendUpdate at startup will be a no-op since no lighthouses are known. + myControl, myVpnIpNet, _, myConfig := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{ + "lighthouse": m{ + "interval": 600, + "local_allow_list": m{ + "10.0.0.0/24": true, + "::/0": false, + }, + }, + }) + + r := router.NewR(t, lhControl, myControl) + defer r.RenderFlow() + + lhControl.Start() + myControl.Start() + + // Drain any startup packets (there should be none meaningful) + r.FlushAll() + + // Verify lighthouse has no knowledge of the client + assert.Nil(t, lhControl.QueryLighthouse(myVpnIpNet[0].Addr())) + + // Build a new config that adds the lighthouse + newSettings := make(m) + for k, v := range myConfig.Settings { + newSettings[k] = v + } + newSettings["static_host_map"] = m{ + lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()}, + } + newSettings["lighthouse"] = m{ + "hosts": []any{lhVpnIpNet[0].Addr().String()}, + "interval": 600, + "local_allow_list": m{ + "10.0.0.0/24": true, + "::/0": false, + }, + } + newCfg, err := yaml.Marshal(newSettings) + require.NoError(t, err) + + // Reload the config. The lighthouse.hosts change triggers TriggerUpdate, + // which wakes the update worker. It calls SendUpdate, initiating a + // handshake to the new lighthouse and caching the HostUpdateNotification. + require.NoError(t, myConfig.ReloadConfigString(string(newCfg))) + + // Route until the lighthouse receives the HostUpdateNotification. + // This covers: handshake stage 1, stage 2, then the cached update. + done := make(chan struct{}) + go func() { + r.RouteForAllUntilAfterMsgTypeTo(lhControl, header.LightHouse, 0) + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for lighthouse update after config reload") + } + + // Verify lighthouse now has the client's addresses + assert.NotNil(t, lhControl.QueryLighthouse(myVpnIpNet[0].Addr())) + + r.RenderHostmaps("Final hostmaps", lhControl, myControl) + lhControl.Stop() + myControl.Stop() +} + +func TestGoodHandshakeUnsafeDest(t *testing.T) { + t.Parallel() + unsafePrefix := "192.168.6.0/24" + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdpAndUnsafeNetworks(cert.Version2, ca, caKey, "spooky", "10.128.0.2/24", netip.MustParseAddrPort("10.64.0.2:4242"), unsafePrefix, nil) + route := m{"route": unsafePrefix, "via": theirVpnIpNet[0].Addr().String()} + myCfg := m{ + "tun": m{ + "unsafe_routes": []m{route}, + }, + } + myControl, myVpnIpNet, myUdpAddr, myConfig := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", myCfg) + t.Logf("my config %v", myConfig) + // Put their info in our lighthouse + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + + spookyDest := netip.MustParseAddr("192.168.6.4") + + // Start the servers + myControl.Start() + theirControl.Start() + + t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side") + myControl.InjectTunPacket(BuildTunUDPPacket(spookyDest, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + + t.Log("Have them consume my stage 0 packet. They have a tunnel now") + theirControl.InjectUDPPacket(myControl.GetFromUDP(true)) + + t.Log("Get their stage 1 packet so that we can play with it") + stage1Packet := theirControl.GetFromUDP(true) + + t.Log("I consume a garbage packet with a proper nebula header for our tunnel") + // this should log a statement and get ignored, allowing the real handshake packet to complete the tunnel + badPacket := stage1Packet.Copy() + badPacket.Data = badPacket.Data[:len(badPacket.Data)-header.Len] + myControl.InjectUDPPacket(badPacket) + + t.Log("Have me consume their real stage 1 packet. I have a tunnel now") + myControl.InjectUDPPacket(stage1Packet) + + t.Log("Wait until we see my cached packet come through") + myControl.WaitForType(1, 0, theirControl) + + t.Log("Make sure our host infos are correct") + assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl) + + t.Log("Get that cached packet and make sure it looks right") + myCachedPacket := theirControl.GetFromTun(true) + assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), spookyDest, 80, 80) + + //reply + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, spookyDest, 80, []byte("Hi from the spookyman"))) + //wait for reply + theirControl.WaitForType(1, 0, myControl) + theirCachedPacket := myControl.GetFromTun(true) + assertUdpPacket(t, []byte("Hi from the spookyman"), theirCachedPacket, spookyDest, myVpnIpNet[0].Addr(), 80, 80) + + t.Log("Do a bidirectional tunnel test") + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + r.RenderHostmaps("Final hostmaps", myControl, theirControl) + myControl.Stop() + theirControl.Stop() +} + +func TestMultiVpnAddrDeletePrimaryKeepsSecondAddr(t *testing.T) { + t.Parallel() + // Regression for the hostmap multi-vpnAddr delete bug. A dual-stack (v4+v6) V2-cert peer that + // handshakes twice at once ends up with two hostinfos linked in the shared next/prev chain, with the + // primary owning both addresses. Deleting that primary (e.g. connection manager dropping it, a + // CloseTunnel, a collision) must promote the surviving sibling for EVERY address. The pre-fix code + // unlinked the chain once per address, so it promoted the sibling for the first address and orphaned + // the second: the peer stayed reachable at its v4 addr but not its v6 addr despite a live tunnel. + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me ", "10.128.0.1/24,fd00::1/64", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.2/24,fd00::2/64", nil) + + // This bug only exists for peers carrying more than one vpn address + require.Len(t, theirVpnIpNet, 2) + theirV4 := theirVpnIpNet[0].Addr() + theirV6 := theirVpnIpNet[1].Addr() + + // Put their info in our lighthouse and vice versa + myControl.InjectLightHouseAddr(theirV4, theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + // Build a router so we don't have to reason who gets which packet + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + + myControl.Start() + theirControl.Start() + + // Race a handshake so both of us build a hostinfo for the other, leaving my hostmap with a single + // host (them) backed by two linked hostinfos, just like TestStage1Race. + myControl.InjectTunPacket(BuildTunUDPPacket(theirV4, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))) + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirV4, 80, []byte("Hi from them"))) + + myHsForThem := myControl.GetFromUDP(true) + theirHsForMe := theirControl.GetFromUDP(true) + + r.InjectUDPPacket(theirControl, myControl, theirHsForMe) + r.InjectUDPPacket(myControl, theirControl, myHsForThem) + + r.RouteForAllUntilTxTun(theirControl) + r.RouteForAllUntilTxTun(myControl) + + r.RenderHostmaps("Racing hostmaps", myControl, theirControl) + + // Two hostinfos for them means the shared next/prev chain has a sibling to promote. The Hosts map has + // one entry per vpn address (two, for dual stack), so the index count is what tells us there are two + // hostinfos. + require.Len(t, myControl.ListHostmapIndexes(false), 2) + + // The primary owns both of their addresses + primaryV4 := myControl.GetHostInfoByVpnAddr(theirV4, false) + primaryV6 := myControl.GetHostInfoByVpnAddr(theirV6, false) + require.NotNil(t, primaryV4) + require.NotNil(t, primaryV6) + require.Equal(t, primaryV4.LocalIndex, primaryV6.LocalIndex, "both addrs should point at the same primary") + + // Delete the primary tunnel. localOnly so we don't perturb their side, we only care about my hostmap. + require.True(t, myControl.CloseTunnel(theirV4, true)) + + // The surviving sibling must still serve BOTH addresses. + survivorV4 := myControl.GetHostInfoByVpnAddr(theirV4, false) + survivorV6 := myControl.GetHostInfoByVpnAddr(theirV6, false) + require.NotNil(t, survivorV4, "v4 addr should still resolve to the surviving tunnel") + // Pre-fix this is nil: the second address was orphaned when the primary was deleted. + require.NotNil(t, survivorV6, "v6 addr was orphaned after deleting the primary (multi-vpnAddr delete bug)") + assert.Equal(t, survivorV4.LocalIndex, survivorV6.LocalIndex, "both addrs should promote to the same survivor") + assert.NotEqual(t, primaryV4.LocalIndex, survivorV4.LocalIndex, "a different hostinfo should now be primary") + + r.RenderHostmaps("Final hostmaps", myControl, theirControl) + + myControl.Stop() + theirControl.Stop() +} diff --git a/e2e/helpers.go b/e2e/helpers.go deleted file mode 100644 index 13146ab7..00000000 --- a/e2e/helpers.go +++ /dev/null @@ -1,118 +0,0 @@ -package e2e - -import ( - "crypto/rand" - "io" - "net" - "time" - - "github.com/slackhq/nebula/cert" - "golang.org/x/crypto/curve25519" - "golang.org/x/crypto/ed25519" -) - -// NewTestCaCert will generate a CA cert -func NewTestCaCert(before, after time.Time, ips, subnets []*net.IPNet, groups []string) (*cert.NebulaCertificate, []byte, []byte, []byte) { - pub, priv, err := ed25519.GenerateKey(rand.Reader) - if before.IsZero() { - before = time.Now().Add(time.Second * -60).Round(time.Second) - } - if after.IsZero() { - after = time.Now().Add(time.Second * 60).Round(time.Second) - } - - nc := &cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "test ca", - NotBefore: time.Unix(before.Unix(), 0), - NotAfter: time.Unix(after.Unix(), 0), - PublicKey: pub, - IsCA: true, - InvertedGroups: make(map[string]struct{}), - }, - } - - if len(ips) > 0 { - nc.Details.Ips = ips - } - - if len(subnets) > 0 { - nc.Details.Subnets = subnets - } - - if len(groups) > 0 { - nc.Details.Groups = groups - } - - err = nc.Sign(cert.Curve_CURVE25519, priv) - if err != nil { - panic(err) - } - - pem, err := nc.MarshalToPEM() - if err != nil { - panic(err) - } - - return nc, pub, priv, pem -} - -// NewTestCert will generate a signed certificate with the provided details. -// Expiry times are defaulted if you do not pass them in -func NewTestCert(ca *cert.NebulaCertificate, key []byte, name string, before, after time.Time, ip *net.IPNet, subnets []*net.IPNet, groups []string) (*cert.NebulaCertificate, []byte, []byte, []byte) { - issuer, err := ca.Sha256Sum() - if err != nil { - panic(err) - } - - if before.IsZero() { - before = time.Now().Add(time.Second * -60).Round(time.Second) - } - - if after.IsZero() { - after = time.Now().Add(time.Second * 60).Round(time.Second) - } - - pub, rawPriv := x25519Keypair() - - nc := &cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: name, - Ips: []*net.IPNet{ip}, - Subnets: subnets, - Groups: groups, - NotBefore: time.Unix(before.Unix(), 0), - NotAfter: time.Unix(after.Unix(), 0), - PublicKey: pub, - IsCA: false, - Issuer: issuer, - InvertedGroups: make(map[string]struct{}), - }, - } - - err = nc.Sign(ca.Details.Curve, key) - if err != nil { - panic(err) - } - - pem, err := nc.MarshalToPEM() - if err != nil { - panic(err) - } - - return nc, pub, cert.MarshalX25519PrivateKey(rawPriv), pem -} - -func x25519Keypair() ([]byte, []byte) { - privkey := make([]byte, 32) - if _, err := io.ReadFull(rand.Reader, privkey); err != nil { - panic(err) - } - - pubkey, err := curve25519.X25519(privkey, curve25519.Basepoint) - if err != nil { - panic(err) - } - - return pubkey, privkey -} diff --git a/e2e/helpers_test.go b/e2e/helpers_test.go index b05c84a2..b555fbc4 100644 --- a/e2e/helpers_test.go +++ b/e2e/helpers_test.go @@ -4,42 +4,109 @@ package e2e import ( - "fmt" "io" - "net" + "net/netip" "os" + "strings" "testing" "time" + "log/slog" + "dario.cat/mergo" "github.com/google/gopacket" "github.com/google/gopacket/layers" - "github.com/sirupsen/logrus" + "github.com/slackhq/nebula" "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/e2e/router" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/logging" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" ) -type m map[string]interface{} +type m = map[string]any // newSimpleServer creates a nebula instance with many assumptions -func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, udpIp net.IP, overrides m) (*nebula.Control, *net.IPNet, *net.UDPAddr, *config.C) { +func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) { + var vpnNetworks []netip.Prefix + for _, sn := range strings.Split(sVpnNetworks, ",") { + vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn)) + if err != nil { + panic(err) + } + vpnNetworks = append(vpnNetworks, vpnIpNet) + } + + if len(vpnNetworks) == 0 { + panic("no vpn networks") + } + + var udpAddr netip.AddrPort + if vpnNetworks[0].Addr().Is4() { + budpIp := vpnNetworks[0].Addr().As4() + budpIp[1] -= 128 + udpAddr = netip.AddrPortFrom(netip.AddrFrom4(budpIp), 4242) + } else { + budpIp := vpnNetworks[0].Addr().As16() + // beef for funsies + budpIp[2] = 190 + budpIp[3] = 239 + udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242) + } + return newSimpleServerWithUdp(v, caCrt, caKey, name, sVpnNetworks, udpAddr, overrides) +} + +func newSimpleServerWithUdp(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) { + return newSimpleServerWithUdpAndUnsafeNetworks(v, caCrt, caKey, name, sVpnNetworks, udpAddr, "", overrides) +} + +func newSimpleServerWithUdpAndUnsafeNetworks(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, sUnsafeNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) { l := NewTestLogger() - vpnIpNet := &net.IPNet{IP: make([]byte, len(udpIp)), Mask: net.IPMask{255, 255, 255, 0}} - copy(vpnIpNet.IP, udpIp) - vpnIpNet.IP[1] += 128 - udpAddr := net.UDPAddr{ - IP: udpIp, - Port: 4242, + var vpnNetworks []netip.Prefix + for _, sn := range strings.Split(sVpnNetworks, ",") { + vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn)) + if err != nil { + panic(err) + } + vpnNetworks = append(vpnNetworks, vpnIpNet) } - _, _, myPrivKey, myPEM := NewTestCert(caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnIpNet, nil, []string{}) - caB, err := caCrt.MarshalToPEM() + if len(vpnNetworks) == 0 { + panic("no vpn networks") + } + + firewallInbound := []m{{ + "proto": "any", + "port": "any", + "host": "any", + }} + + var unsafeNetworks []netip.Prefix + if sUnsafeNetworks != "" { + firewallInbound = []m{{ + "proto": "any", + "port": "any", + "host": "any", + "local_cidr": "0.0.0.0/0", + }} + + for _, sn := range strings.Split(sUnsafeNetworks, ",") { + x, err := netip.ParsePrefix(strings.TrimSpace(sn)) + if err != nil { + panic(err) + } + unsafeNetworks = append(unsafeNetworks, x) + } + } + + _, _, myPrivKey, myPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnNetworks, unsafeNetworks, []string{}) + + caB, err := caCrt.MarshalPEM() if err != nil { panic(err) } @@ -51,6 +118,103 @@ func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, u "key": string(myPrivKey), }, //"tun": m{"disabled": true}, + "firewall": m{ + "outbound": []m{{ + "proto": "any", + "port": "any", + "host": "any", + }}, + "inbound": firewallInbound, + }, + //"handshakes": m{ + // "try_interval": "1s", + //}, + "listen": m{ + "host": udpAddr.Addr().String(), + "port": udpAddr.Port(), + }, + "logging": m{ + "level": testLogLevelName(), + }, + "timers": m{ + "pending_deletion_interval": 2, + "connection_alive_interval": 2, + }, + } + + if overrides != nil { + final := m{} + err = mergo.Merge(&final, overrides, mergo.WithAppendSlice) + if err != nil { + panic(err) + } + err = mergo.Merge(&final, mc, mergo.WithAppendSlice) + if err != nil { + panic(err) + } + mc = final + } + + cb, err := yaml.Marshal(mc) + if err != nil { + panic(err) + } + + c := config.NewC(l) + c.LoadString(string(cb)) + + control, err := nebula.Main(c, false, "e2e-test", l, nil) + + if err != nil { + panic(err) + } + + return control, vpnNetworks, udpAddr, c +} + +// newServer creates a nebula instance with fewer assumptions +func newServer(caCrt []cert.Certificate, certs []cert.Certificate, key []byte, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) { + l := NewTestLogger() + + vpnNetworks := certs[len(certs)-1].Networks() + + var udpAddr netip.AddrPort + if vpnNetworks[0].Addr().Is4() { + budpIp := vpnNetworks[0].Addr().As4() + budpIp[1] -= 128 + udpAddr = netip.AddrPortFrom(netip.AddrFrom4(budpIp), 4242) + } else { + budpIp := vpnNetworks[0].Addr().As16() + // beef for funsies + budpIp[2] = 190 + budpIp[3] = 239 + udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242) + } + + caStr := "" + for _, ca := range caCrt { + x, err := ca.MarshalPEM() + if err != nil { + panic(err) + } + caStr += string(x) + } + certStr := "" + for _, c := range certs { + x, err := c.MarshalPEM() + if err != nil { + panic(err) + } + certStr += string(x) + } + + mc := m{ + "pki": m{ + "ca": caStr, + "cert": certStr, + "key": string(key), + }, + //"tun": m{"disabled": true}, "firewall": m{ "outbound": []m{{ "proto": "any", @@ -67,12 +231,11 @@ func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, u // "try_interval": "1s", //}, "listen": m{ - "host": udpAddr.IP.String(), - "port": udpAddr.Port, + "host": udpAddr.Addr().String(), + "port": udpAddr.Port(), }, "logging": m{ - "timestamp_format": fmt.Sprintf("%v 15:04:05.000000", name), - "level": l.Level.String(), + "level": testLogLevelName(), }, "timers": m{ "pending_deletion_interval": 2, @@ -81,11 +244,16 @@ func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, u } if overrides != nil { - err = mergo.Merge(&overrides, mc, mergo.WithAppendSlice) + final := m{} + err := mergo.Merge(&final, overrides, mergo.WithAppendSlice) if err != nil { panic(err) } - mc = overrides + err = mergo.Merge(&final, mc, mergo.WithAppendSlice) + if err != nil { + panic(err) + } + mc = final } cb, err := yaml.Marshal(mc) @@ -94,7 +262,8 @@ func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, u } c := config.NewC(l) - c.LoadString(string(cb)) + cStr := string(cb) + c.LoadString(cStr) control, err := nebula.Main(c, false, "e2e-test", l, nil) @@ -102,7 +271,7 @@ func newSimpleServer(caCrt *cert.NebulaCertificate, caKey []byte, name string, u panic(err) } - return control, vpnIpNet, &udpAddr, c + return control, vpnNetworks, udpAddr, c } type doneCb func() @@ -123,64 +292,73 @@ func deadline(t *testing.T, seconds time.Duration) doneCb { } } -func assertTunnel(t *testing.T, vpnIpA, vpnIpB net.IP, controlA, controlB *nebula.Control, r *router.R) { +func assertTunnel(t testing.TB, vpnIpA, vpnIpB netip.Addr, controlA, controlB *nebula.Control, r *router.R) { // Send a packet from them to me - controlB.InjectTunUDPPacket(vpnIpA, 80, 90, []byte("Hi from B")) + controlB.InjectTunPacket(BuildTunUDPPacket(vpnIpA, 80, vpnIpB, 90, []byte("Hi from B"))) bPacket := r.RouteForAllUntilTxTun(controlA) assertUdpPacket(t, []byte("Hi from B"), bPacket, vpnIpB, vpnIpA, 90, 80) // And once more from me to them - controlA.InjectTunUDPPacket(vpnIpB, 80, 90, []byte("Hello from A")) + controlA.InjectTunPacket(BuildTunUDPPacket(vpnIpB, 80, vpnIpA, 90, []byte("Hello from A"))) aPacket := r.RouteForAllUntilTxTun(controlB) assertUdpPacket(t, []byte("Hello from A"), aPacket, vpnIpA, vpnIpB, 90, 80) } -func assertHostInfoPair(t *testing.T, addrA, addrB *net.UDPAddr, vpnIpA, vpnIpB net.IP, controlA, controlB *nebula.Control) { +func assertHostInfoPair(t testing.TB, addrA, addrB netip.AddrPort, vpnNetsA, vpnNetsB []netip.Prefix, controlA, controlB *nebula.Control) { // Get both host infos - hBinA := controlA.GetHostInfoByVpnIp(iputil.Ip2VpnIp(vpnIpB), false) - assert.NotNil(t, hBinA, "Host B was not found by vpnIp in controlA") + //TODO: CERT-V2 we may want to loop over each vpnAddr and assert all the things + hBinA := controlA.GetHostInfoByVpnAddr(vpnNetsB[0].Addr(), false) + require.NotNil(t, hBinA, "Host B was not found by vpnAddr in controlA") - hAinB := controlB.GetHostInfoByVpnIp(iputil.Ip2VpnIp(vpnIpA), false) - assert.NotNil(t, hAinB, "Host A was not found by vpnIp in controlB") + hAinB := controlB.GetHostInfoByVpnAddr(vpnNetsA[0].Addr(), false) + require.NotNil(t, hAinB, "Host A was not found by vpnAddr in controlB") // Check that both vpn and real addr are correct - assert.Equal(t, vpnIpB, hBinA.VpnIp, "Host B VpnIp is wrong in control A") - assert.Equal(t, vpnIpA, hAinB.VpnIp, "Host A VpnIp is wrong in control B") + assert.EqualValues(t, getAddrs(vpnNetsB), hBinA.VpnAddrs, "Host B VpnIp is wrong in control A") + assert.EqualValues(t, getAddrs(vpnNetsA), hAinB.VpnAddrs, "Host A VpnIp is wrong in control B") - assert.Equal(t, addrB.IP.To16(), hBinA.CurrentRemote.IP.To16(), "Host B remote ip is wrong in control A") - assert.Equal(t, addrA.IP.To16(), hAinB.CurrentRemote.IP.To16(), "Host A remote ip is wrong in control B") - - assert.Equal(t, addrB.Port, int(hBinA.CurrentRemote.Port), "Host B remote port is wrong in control A") - assert.Equal(t, addrA.Port, int(hAinB.CurrentRemote.Port), "Host A remote port is wrong in control B") + assert.Equal(t, addrB, hBinA.CurrentRemote, "Host B remote is wrong in control A") + assert.Equal(t, addrA, hAinB.CurrentRemote, "Host A remote is wrong in control B") // Check that our indexes match assert.Equal(t, hBinA.LocalIndex, hAinB.RemoteIndex, "Host B local index does not match host A remote index") assert.Equal(t, hBinA.RemoteIndex, hAinB.LocalIndex, "Host B remote index does not match host A local index") +} + +func assertUdpPacket(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) { + if toIp.Is6() { + assertUdpPacket6(t, expected, b, fromIp, toIp, fromPort, toPort) + } else { + assertUdpPacket4(t, expected, b, fromIp, toIp, fromPort, toPort) + } +} + +func assertUdpPacket6(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) { + packet := gopacket.NewPacket(b, layers.LayerTypeIPv6, gopacket.Lazy) + v6 := packet.Layer(layers.LayerTypeIPv6).(*layers.IPv6) + assert.NotNil(t, v6, "No ipv6 data found") + + assert.Equal(t, fromIp.AsSlice(), []byte(v6.SrcIP), "Source ip was incorrect") + assert.Equal(t, toIp.AsSlice(), []byte(v6.DstIP), "Dest ip was incorrect") + + udp := packet.Layer(layers.LayerTypeUDP).(*layers.UDP) + assert.NotNil(t, udp, "No udp data found") + + assert.Equal(t, fromPort, uint16(udp.SrcPort), "Source port was incorrect") + assert.Equal(t, toPort, uint16(udp.DstPort), "Dest port was incorrect") - //TODO: Would be nice to assert this memory - //checkIndexes := func(name string, hm *HostMap, hi *HostInfo) { - // hBbyIndex := hmA.Indexes[hBinA.localIndexId] - // assert.NotNil(t, hBbyIndex, "Could not host info by local index in %s", name) - // assert.Equal(t, &hBbyIndex, &hBinA, "%s Indexes map did not point to the right host info", name) - // - // //TODO: remote indexes are susceptible to collision - // hBbyRemoteIndex := hmA.RemoteIndexes[hBinA.remoteIndexId] - // assert.NotNil(t, hBbyIndex, "Could not host info by remote index in %s", name) - // assert.Equal(t, &hBbyRemoteIndex, &hBinA, "%s RemoteIndexes did not point to the right host info", name) - //} - // - //// Check hostmap indexes too - //checkIndexes("hmA", hmA, hBinA) - //checkIndexes("hmB", hmB, hAinB) + data := packet.ApplicationLayer() + assert.NotNil(t, data) + assert.Equal(t, expected, data.Payload(), "Data was incorrect") } -func assertUdpPacket(t *testing.T, expected, b []byte, fromIp, toIp net.IP, fromPort, toPort uint16) { +func assertUdpPacket4(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) { packet := gopacket.NewPacket(b, layers.LayerTypeIPv4, gopacket.Lazy) v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4) assert.NotNil(t, v4, "No ipv4 data found") - assert.Equal(t, fromIp, v4.SrcIP, "Source ip was incorrect") - assert.Equal(t, toIp, v4.DstIP, "Dest ip was incorrect") + assert.Equal(t, fromIp.AsSlice(), []byte(v4.SrcIP), "Source ip was incorrect") + assert.Equal(t, toIp.AsSlice(), []byte(v4.DstIP), "Dest ip was incorrect") udp := packet.Layer(layers.LayerTypeUDP).(*layers.UDP) assert.NotNil(t, udp, "No udp data found") @@ -193,24 +371,95 @@ func assertUdpPacket(t *testing.T, expected, b []byte, fromIp, toIp net.IP, from assert.Equal(t, expected, data.Payload(), "Data was incorrect") } -func NewTestLogger() *logrus.Logger { - l := logrus.New() +func getAddrs(ns []netip.Prefix) []netip.Addr { + var a []netip.Addr + for _, n := range ns { + a = append(a, n.Addr()) + } + return a +} +func NewTestLogger() *slog.Logger { v := os.Getenv("TEST_LOGS") if v == "" { - l.SetOutput(io.Discard) - l.SetLevel(logrus.PanicLevel) - return l + return slog.New(slog.NewTextHandler(io.Discard, nil)) } + level := slog.LevelInfo switch v { case "2": - l.SetLevel(logrus.DebugLevel) + level = slog.LevelDebug + case "3": + level = logging.LevelTrace + } + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})) +} + +// testLogLevelName returns the level name string accepted by logging.ApplyConfig +// for the current TEST_LOGS setting. Kept in sync with NewTestLogger. +func testLogLevelName() string { + switch os.Getenv("TEST_LOGS") { + case "2": + return "debug" case "3": - l.SetLevel(logrus.TraceLevel) - default: - l.SetLevel(logrus.InfoLevel) + return "trace" + case "": + return "info" + } + return "info" +} + +// BuildTunUDPPacket assembles an IP+UDP packet suitable for Control.InjectTunPacket. +// Using UDP here because it's a simpler protocol. +func BuildTunUDPPacket(toAddr netip.Addr, toPort uint16, fromAddr netip.Addr, fromPort uint16, data []byte) []byte { + serialize := make([]gopacket.SerializableLayer, 0) + var netLayer gopacket.NetworkLayer + if toAddr.Is6() { + if !fromAddr.Is6() { + panic("Cant send ipv6 to ipv4") + } + ip := &layers.IPv6{ + Version: 6, + NextHeader: layers.IPProtocolUDP, + SrcIP: fromAddr.Unmap().AsSlice(), + DstIP: toAddr.Unmap().AsSlice(), + } + serialize = append(serialize, ip) + netLayer = ip + } else { + if !fromAddr.Is4() { + panic("Cant send ipv4 to ipv6") + } + + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + Protocol: layers.IPProtocolUDP, + SrcIP: fromAddr.Unmap().AsSlice(), + DstIP: toAddr.Unmap().AsSlice(), + } + serialize = append(serialize, ip) + netLayer = ip + } + + udp := layers.UDP{ + SrcPort: layers.UDPPort(fromPort), + DstPort: layers.UDPPort(toPort), + } + if err := udp.SetNetworkLayerForChecksum(netLayer); err != nil { + panic(err) + } + + buffer := gopacket.NewSerializeBuffer() + opt := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + serialize = append(serialize, &udp, gopacket.Payload(data)) + if err := gopacket.SerializeLayers(buffer, opt, serialize...); err != nil { + panic(err) } - return l + return buffer.Bytes() } diff --git a/e2e/leak_test.go b/e2e/leak_test.go new file mode 100644 index 00000000..576d67a8 --- /dev/null +++ b/e2e/leak_test.go @@ -0,0 +1,47 @@ +//go:build e2e_testing +// +build e2e_testing + +package e2e + +import ( + "testing" + "time" + + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/e2e/router" + "go.uber.org/goleak" +) + +// TestNoGoroutineLeaks brings up two nebula instances, completes a tunnel, +// stops both, and asserts no goroutines leak past the shutdown. goleak's +// retry mechanism gives the wg.Wait()-driven goroutines a moment to drain +// before failing the assertion. +// +// Intentionally NOT t.Parallel()'d: concurrent tests would have their own +// goroutines running and trip the assertion. +func TestNoGoroutineLeaks(t *testing.T) { + defer goleak.VerifyNone(t) + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + myControl.Stop() + theirControl.Stop() + r.RenderFlow() + + // Settle period: Stop() is non-blocking; the wg-driven goroutines need + // a moment to drain. goleak retries internally too, but a short explicit + // settle reduces flakes when the suite is busy. + time.Sleep(50 * time.Millisecond) +} diff --git a/e2e/rebind_test.go b/e2e/rebind_test.go new file mode 100644 index 00000000..2547f739 --- /dev/null +++ b/e2e/rebind_test.go @@ -0,0 +1,225 @@ +//go:build e2e_testing +// +build e2e_testing + +package e2e + +import ( + "net/netip" + "testing" + "time" + + "github.com/slackhq/nebula" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/e2e/router" + "github.com/slackhq/nebula/header" + "github.com/slackhq/nebula/udp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// reportedAddrs is what the lighthouse would hand a peer asking where vpnAddr is. +func reportedAddrs(t *testing.T, lh *nebula.Control, vpnAddr netip.Addr) []netip.AddrPort { + t.Helper() + cm := lh.QueryLighthouse(vpnAddr) + if cm == nil { + return nil + } + var out []netip.AddrPort + for _, c := range *cm { + out = append(out, c.Reported...) + out = append(out, c.Learned...) + } + return out +} + +// waitForLighthouseMsg routes until a lighthouse message lands on lh, or gives up. Reports whether one arrived. +func waitForLighthouseMsg(t *testing.T, r *router.R, lh *nebula.Control, wait time.Duration) bool { + t.Helper() + h := &header.H{} + return r.RouteForAllExitFuncOrTimeout(wait, func(p *udp.Packet, c *nebula.Control) router.ExitType { + if c != lh { + return router.KeepRouting + } + // Punches are a single byte and never parse, they are just not what we are after + if err := h.Parse(p.Data); err != nil { + return router.KeepRouting + } + if h.Type == header.LightHouse { + return router.RouteAndExit + } + return router.KeepRouting + }) +} + +// A laptop that changes networks has to tell the lighthouse promptly, otherwise the lighthouse keeps handing peers +// the old address and their punches land nowhere. On a long lighthouse interval the only thing that closes that +// window is the rebind, which on darwin the network change monitor drives. The e2e build compiles the monitor out, +// so we call RebindUDPServer directly, which is the same thing the monitor does. +func TestRebindSendsLighthouseUpdate(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{ + "lighthouse": m{"am_lighthouse": true}, + }) + + // 600s interval, so nothing scheduled can send an update during this test. A rebind is the only thing that can. + myControl, _, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{ + "lighthouse": m{ + "hosts": []any{lhVpnIpNet[0].Addr().String()}, + "interval": 600, + }, + "static_host_map": m{ + lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()}, + }, + }) + + r := router.NewR(t, lhControl, myControl) + defer r.RenderFlow() + + lhControl.Start() + myControl.Start() + + // Let the startup registration finish, then clear everything it left behind + require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration") + r.RouteFor(time.Millisecond * 400) + + // Nothing should be talking to the lighthouse on its own now + require.False(t, waitForLighthouseMsg(t, r, lhControl, time.Millisecond*200), + "nothing should reach the lighthouse before the rebind") + + myControl.RebindUDPServer() + + assert.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), + "a rebind should push an update to the lighthouse rather than waiting out the interval") + + lhControl.Stop() + myControl.Stop() +} + +// The other half of a rebind: every live tunnel requeries the lighthouse on its next send. That query is what makes +// the lighthouse tell the peer to punch toward our new address, which is the part that actually revives a tunnel +// whose remote NAT state died while we were on a different network. +func TestRebindRequeriesPeersOnNextSend(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{ + "lighthouse": m{"am_lighthouse": true}, + }) + + lhCfg := m{ + "lighthouse": m{ + "hosts": []any{lhVpnIpNet[0].Addr().String()}, + "interval": 600, + // Without this the peers advertise this machine's real addresses and then try to punch at them, + // which the router has no route for. + "local_allow_list": m{ + "10.0.0.0/24": true, + "::/0": false, + }, + }, + "static_host_map": m{ + lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()}, + }, + } + + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", lhCfg) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", lhCfg) + + r := router.NewR(t, lhControl, myControl, theirControl) + defer r.RenderFlow() + + lhControl.Start() + myControl.Start() + theirControl.Start() + r.RouteFor(time.Millisecond * 500) + + // Point the peers at each other directly, this test is about the rebind and not about lighthouse discovery + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("initial"))) + r.RouteFor(time.Second) + require.NotNil(t, myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false), "expected a tunnel to them") + r.RouteFor(time.Millisecond * 300) + + // Assert on what the peer sees rather than on lighthouse traffic. A query for them makes the lighthouse send + // them a punch notification, which is the whole point. Our own update to the lighthouse sends them nothing, + // so this cannot be satisfied by the update the rebind itself pushes. + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("quiet"))) + require.False(t, waitForLighthouseMsg(t, r, theirControl, time.Millisecond*300), + "an ordinary send should not requery the lighthouse") + + myControl.RebindUDPServer() + r.RouteFor(time.Millisecond * 300) // let the update the rebind itself sends pass by + + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("after rebind"))) + assert.True(t, waitForLighthouseMsg(t, r, theirControl, time.Second*5), + "the first send after a rebind should requery the lighthouse, which then tells the peer to punch at us") + + lhControl.Stop() + myControl.Stop() + theirControl.Stop() +} + +// The scenario this whole thing exists for: a laptop sleeps at the office and wakes up at home on a new address. +// Until it tells the lighthouse, the lighthouse keeps handing peers the office address, so their punches land +// nowhere and the tunnel stays dead. On a long interval the rebind is the only thing that closes that window. +func TestRebindAdvertisesNewAddressAfterMove(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{ + "lighthouse": m{"am_lighthouse": true}, + }) + + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{ + "lighthouse": m{ + "hosts": []any{lhVpnIpNet[0].Addr().String()}, + "interval": 600, + }, + "static_host_map": m{ + lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()}, + }, + }) + + // Advertise wherever we currently are rather than this machine's real NICs, read fresh each time so a move + // is picked up. + myControl.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr { + return []netip.Addr{myControl.GetUDPAddr().Addr()} + }) + + r := router.NewR(t, lhControl, myControl) + defer r.RenderFlow() + + lhControl.Start() + myControl.Start() + + require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration") + r.RouteFor(time.Millisecond * 400) + + require.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), myUdpAddr, + "the lighthouse should know the address we started on") + + // Wake up somewhere else + newAddr := netip.MustParseAddrPort("10.0.0.99:4242") + myControl.SetUDPAddr(newAddr) + r.AddRoute(newAddr.Addr(), newAddr.Port(), myControl) + + // Nothing has told the lighthouse, and with interval 600 nothing scheduled will + r.RouteFor(time.Millisecond * 400) + require.NotContains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr, + "the lighthouse should still be handing out the old address before the rebind") + + myControl.RebindUDPServer() + require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an update after the rebind") + r.RouteFor(time.Millisecond * 400) + + assert.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr, + "after the rebind the lighthouse should hand peers our new address") + + lhControl.Stop() + myControl.Stop() +} diff --git a/e2e/recovery_timing_test.go b/e2e/recovery_timing_test.go new file mode 100644 index 00000000..0fe356c8 --- /dev/null +++ b/e2e/recovery_timing_test.go @@ -0,0 +1,136 @@ +//go:build e2e_testing +// +build e2e_testing + +package e2e + +import ( + "testing" + "time" + + "github.com/slackhq/nebula" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/e2e/router" + "github.com/slackhq/nebula/udp" +) + +// TestRecoveryTiming measures how long a tunnel takes to come back after the peer stops accepting our traffic, +// which is what a laptop waking on a new network looks like from the peer's side: its NAT has no state for where +// we are now, so everything we send disappears. +// +// It is a measurement, not a pass/fail assertion. Recovery is timed to the moment the peer punches back at us, +// since that is when its NAT opens and the tunnel is usable again. +// +// go test -tags e2e_testing -v -run TestRecoveryTiming ./e2e/ +func TestRecoveryTiming(t *testing.T) { + for _, tc := range []struct { + name string + rebind bool + }{ + {"no trigger", false}, + {"rebind counter", true}, + } { + t.Run(tc.name, func(t *testing.T) { + d, lost := measureRecovery(t, tc.rebind) + t.Logf("RESULT %-16s recovered in %-9v (%d packets lost)", tc.name, d.Round(time.Millisecond), lost) + }) + } +} + +// measureRecovery returns how long until the peer punched back, and how many of our packets died meanwhile. When +// rebind is true we call RebindUDPServer once the tunnel goes dark, which is what the darwin network change +// monitor does and what iOS has always done. When false, nothing tells nebula anything is wrong. +func measureRecovery(t *testing.T, rebind bool) (time.Duration, int) { + t.Helper() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{ + "lighthouse": m{"am_lighthouse": true}, + }) + + peerCfg := m{ + "lighthouse": m{ + "hosts": []any{lhVpnIpNet[0].Addr().String()}, + "interval": 600, + "local_allow_list": m{ + "10.0.0.0/24": true, + "::/0": false, + }, + }, + "static_host_map": m{ + lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()}, + }, + } + + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", peerCfg) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", peerCfg) + + r := router.NewR(t, lhControl, myControl, theirControl) + defer r.RenderFlow() + defer func() { + lhControl.Stop() + myControl.Stop() + theirControl.Stop() + }() + + lhControl.Start() + myControl.Start() + theirControl.Start() + r.RouteFor(time.Millisecond * 500) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("establish"))) + r.RouteFor(time.Second) + if myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) == nil { + t.Fatal("failed to establish the tunnel we are measuring") + } + r.RouteFor(time.Millisecond * 500) + + // From here the peer's NAT has no state for us, everything we send it disappears + start := time.Now() + blackholed := 0 + var recovered time.Duration + + if rebind { + myControl.RebindUDPServer() + } + + // Keep the tun busy the way someone retrying a stalled connection would + stop := make(chan struct{}) + defer close(stop) + go func() { + tick := time.NewTicker(time.Millisecond * 200) + defer tick.Stop() + for { + select { + case <-stop: + return + case <-tick.C: + myControl.InjectTunPacket(BuildTunUDPPacket( + theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("retry"))) + } + } + }() + + r.RouteForAllExitFuncOrTimeout(time.Second*30, func(p *udp.Packet, c *nebula.Control) router.ExitType { + if c == theirControl && p.From == myControl.GetUDPAddr() { + blackholed++ + return router.Drop + } + + // The peer reaching us directly is the moment its NAT opened, whether that is a punch or a handshake + if c == myControl && p.From == theirUdpAddr { + recovered = time.Since(start) + return router.RouteAndExit + } + + return router.KeepRouting + }) + + if recovered == 0 { + t.Fatalf("no recovery within 30s (%d packets blackholed)", blackholed) + } + return recovered, blackholed +} diff --git a/e2e/router/hostmap.go b/e2e/router/hostmap.go index 120be696..f2805d0c 100644 --- a/e2e/router/hostmap.go +++ b/e2e/router/hostmap.go @@ -5,11 +5,11 @@ package router import ( "fmt" + "net/netip" "sort" "strings" "github.com/slackhq/nebula" - "github.com/slackhq/nebula/iputil" ) type edge struct { @@ -58,8 +58,9 @@ func renderHostmap(c *nebula.Control) (string, []*edge) { var lines []string var globalLines []*edge - clusterName := strings.Trim(c.GetCert().Details.Name, " ") - clusterVpnIp := c.GetCert().Details.Ips[0].IP + crt := c.GetCertState().GetDefaultCertificate() + clusterName := strings.Trim(crt.Name(), " ") + clusterVpnIp := crt.Networks()[0].Addr() r := fmt.Sprintf("\tsubgraph %s[\"%s (%s)\"]\n", clusterName, clusterName, clusterVpnIp) hm := c.GetHostmap() @@ -101,8 +102,8 @@ func renderHostmap(c *nebula.Control) (string, []*edge) { for _, idx := range indexes { hi, ok := hm.Indexes[idx] if ok { - r += fmt.Sprintf("\t\t\t%v.%v[\"%v (%v)\"]\n", clusterName, idx, idx, hi.GetVpnIp()) - remoteClusterName := strings.Trim(hi.GetCert().Details.Name, " ") + r += fmt.Sprintf("\t\t\t%v.%v[\"%v (%v)\"]\n", clusterName, idx, idx, hi.GetVpnAddrs()) + remoteClusterName := strings.Trim(hi.GetCert().Certificate.Name(), " ") globalLines = append(globalLines, &edge{from: fmt.Sprintf("%v.%v", clusterName, idx), to: fmt.Sprintf("%v.%v", remoteClusterName, hi.GetRemoteIndex())}) _ = hi } @@ -118,14 +119,14 @@ func renderHostmap(c *nebula.Control) (string, []*edge) { return r, globalLines } -func sortedHosts(hosts map[iputil.VpnIp]*nebula.HostInfo) []iputil.VpnIp { - keys := make([]iputil.VpnIp, 0, len(hosts)) +func sortedHosts(hosts map[netip.Addr]*nebula.HostInfo) []netip.Addr { + keys := make([]netip.Addr, 0, len(hosts)) for key := range hosts { keys = append(keys, key) } sort.SliceStable(keys, func(i, j int) bool { - return keys[i] > keys[j] + return keys[i].Compare(keys[j]) > 0 }) return keys diff --git a/e2e/router/router.go b/e2e/router/router.go index 730853a9..2556079c 100644 --- a/e2e/router/router.go +++ b/e2e/router/router.go @@ -6,14 +6,14 @@ package router import ( "context" "fmt" - "net" + "net/netip" "os" "path/filepath" "reflect" + "regexp" "sort" - "strconv" - "strings" "sync" + "sync/atomic" "testing" "time" @@ -21,26 +21,54 @@ import ( "github.com/google/gopacket/layers" "github.com/slackhq/nebula" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/udp" "golang.org/x/exp/maps" ) +// outNatKey is the (from, to) pair used by outNat. Comparable struct, so it works as a map key without the +// allocation cost of a string-concat key. +type outNatKey struct { + from, to netip.AddrPort +} + +// fannedPacket pairs a UDP TX packet with its source control so the router can route it after popping from +// the fan-in channel. +type fannedPacket struct { + from *nebula.Control + pkt *udp.Packet +} + type R struct { // Simple map of the ip:port registered on a control to the control // Basically a router, right? - controls map[string]*nebula.Control + controls map[netip.AddrPort]*nebula.Control // A map for inbound packets for a control that doesn't know about this address - inNat map[string]*nebula.Control + inNat map[netip.AddrPort]*nebula.Control // A last used map, if an inbound packet hit the inNat map then // all return packets should use the same last used inbound address for the outbound sender - // map[from address + ":" + to address] => ip:port to rewrite in the udp packet to receiver - outNat map[string]net.UDPAddr + outNat map[outNatKey]netip.AddrPort // A map of vpn ip to the nebula control it belongs to - vpnControls map[iputil.VpnIp]*nebula.Control + vpnControls map[netip.Addr]*nebula.Control + + // Cached select infrastructure for RouteForAllUntilTxTun. + // The controls map is immutable after NewR so the cases are good for the test lifetime. + // We only rebuild if a different receiver is asked. + selRecvCtl *nebula.Control + selCases []reflect.SelectCase + selCtls []*nebula.Control + + // Optional fan-in mode for hot-path benchmarks: one forwarder goroutine per control drains UDP TX into udpFanIn, + // so RouteForAllUntilTxTun can do a fixed 2-way native select instead of paying reflect.Select per call. + // Off by default (would otherwise interleave with tests that use GetFromUDP directly on the same control). + // Enabled by EnableFanIn. + udpFanIn chan fannedPacket + stopFanIn chan struct{} + fanInWG sync.WaitGroup + fanInMu sync.Mutex + fanInOn atomic.Bool ignoreFlows []ignoreFlow flow []flowEntry @@ -86,6 +114,28 @@ type packet struct { packet *udp.Packet tun bool // a packet pulled off a tun device rx bool // the packet was received by a udp device + + // h is the nebula header, parsed once when the packet is recorded. parseErr says why there isn't one, which + // the flow log reports rather than hiding. Punchy sends a single byte, so an unparseable packet is normal. + h header.H + parseErr error +} + +// fromAddr and toAddr are the addresses this packet actually travelled between. Reading them off the control +// instead would misreport the whole history once a test moves a node. Tun packets are synthesized without +// addresses, so they fall back to the control. +func (p *packet) fromAddr() netip.AddrPort { + if p.tun || !p.packet.From.IsValid() { + return p.from.GetUDPAddr() + } + return p.packet.From +} + +func (p *packet) toAddr() netip.AddrPort { + if p.tun || !p.packet.To.IsValid() { + return p.to.GetUDPAddr() + } + return p.packet.To } func (p *packet) WasReceived() { @@ -103,6 +153,9 @@ const ( ExitNow ExitType = 1 // RouteAndExit routes this packet and exits immediately afterwards RouteAndExit ExitType = 2 + // Drop discards this packet without delivering it and keeps routing. Use it to simulate a blackhole, such as + // a restrictive NAT refusing traffic from an address it has not seen. + Drop ExitType = 3 ) type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType @@ -113,18 +166,20 @@ type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType func NewR(t testing.TB, controls ...*nebula.Control) *R { ctx, cancel := context.WithCancel(context.Background()) - if err := os.MkdirAll("mermaid", 0755); err != nil { + // t.Name() contains a slash for subtests, so the flow log can land in a nested directory + fn := filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name())) + if err := os.MkdirAll(filepath.Dir(fn), 0755); err != nil { panic(err) } r := &R{ - controls: make(map[string]*nebula.Control), - vpnControls: make(map[iputil.VpnIp]*nebula.Control), - inNat: make(map[string]*nebula.Control), - outNat: make(map[string]net.UDPAddr), + controls: make(map[netip.AddrPort]*nebula.Control), + vpnControls: make(map[netip.Addr]*nebula.Control), + inNat: make(map[netip.AddrPort]*nebula.Control), + outNat: make(map[outNatKey]netip.AddrPort), flow: []flowEntry{}, ignoreFlows: []ignoreFlow{}, - fn: filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name())), + fn: fn, t: t, cancelRender: cancel, } @@ -135,10 +190,13 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R { for _, c := range controls { addr := c.GetUDPAddr() if _, ok := r.controls[addr]; ok { - panic("Duplicate listen address: " + addr) + panic("Duplicate listen address: " + addr.String()) + } + + for _, vpnAddr := range c.GetVpnAddrs() { + r.vpnControls[vpnAddr] = c } - r.vpnControls[c.GetVpnIp()] = c r.controls[addr] = c } @@ -152,8 +210,10 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R { case <-ctx.Done(): return case <-clockSource.C: + r.Lock() r.renderHostmaps("clock tick") r.renderFlow() + r.Unlock() } } }() @@ -165,13 +225,13 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R { // It does not look at the addr attached to the instance. // If a route is used, this will behave like a NAT for the return path. // Rewriting the source ip:port to what was last sent to from the origin -func (r *R) AddRoute(ip net.IP, port uint16, c *nebula.Control) { +func (r *R) AddRoute(ip netip.Addr, port uint16, c *nebula.Control) { r.Lock() defer r.Unlock() - inAddr := net.JoinHostPort(ip.String(), fmt.Sprintf("%v", port)) + inAddr := netip.AddrPortFrom(ip, port) if _, ok := r.inNat[inAddr]; ok { - panic("Duplicate listen address inNat: " + inAddr) + panic("Duplicate listen address inNat: " + inAddr.String()) } r.inNat[inAddr] = c } @@ -179,15 +239,21 @@ func (r *R) AddRoute(ip net.IP, port uint16, c *nebula.Control) { // RenderFlow renders the packet flow seen up until now and stops further automatic renders from happening. func (r *R) RenderFlow() { r.cancelRender() + r.Lock() + defer r.Unlock() r.renderFlow() } // CancelFlowLogs stops flow logs from being tracked and destroys any logs already collected func (r *R) CancelFlowLogs() { r.cancelRender() + r.Lock() r.flow = nil + r.Unlock() } +// renderFlow writes the flow log to disk. Caller must hold r.Lock. renderFlow reads r.flow / r.additionalGraphs and +// the *packet pointers stashed inside, all of which are mutated under the same lock by routing paths. func (r *R) renderFlow() { if r.flow == nil { return @@ -198,7 +264,7 @@ func (r *R) renderFlow() { panic(err) } - var participants = map[string]struct{}{} + var participants = map[netip.AddrPort]struct{}{} var participantsVals []string fmt.Fprintln(f, "```mermaid") @@ -210,16 +276,16 @@ func (r *R) renderFlow() { continue } - addr := e.packet.from.GetUDPAddr() + addr := e.packet.fromAddr() if _, ok := participants[addr]; ok { continue } participants[addr] = struct{}{} - sanAddr := strings.Replace(addr, ":", "-", 1) + sanAddr := normalizeName(addr.String()) participantsVals = append(participantsVals, sanAddr) fmt.Fprintf( f, " participant %s as Nebula: %s
UDP: %s\n", - sanAddr, e.packet.from.GetVpnIp(), sanAddr, + sanAddr, e.packet.from.GetVpnAddrs(), sanAddr, ) } @@ -229,7 +295,6 @@ func (r *R) renderFlow() { } // Print packets - h := &header.H{} for _, e := range r.flow { if e.packet == nil { //fmt.Fprintf(f, " note over %s: %s\n", strings.Join(participantsVals, ", "), e.note) @@ -241,21 +306,22 @@ func (r *R) renderFlow() { fmt.Fprintln(f, r.formatUdpPacket(p)) } else { - if err := h.Parse(p.packet.Data); err != nil { - panic(err) - } - line := "--x" if p.rx { line = "->>" } - fmt.Fprintf(f, - " %s%s%s: %s(%s), index %v, counter: %v\n", - strings.Replace(p.from.GetUDPAddr(), ":", "-", 1), + detail := fmt.Sprintf("%s(%s), index %v, counter: %v", + p.h.TypeName(), p.h.SubTypeName(), p.h.RemoteIndex, p.h.MessageCounter) + if p.parseErr != nil { + detail = fmt.Sprintf("unparsed, %v (%d bytes)", p.parseErr, len(p.packet.Data)) + } + + fmt.Fprintf(f, " %s%s%s: %s\n", + normalizeName(p.fromAddr().String()), line, - strings.Replace(p.to.GetUDPAddr(), ":", "-", 1), - h.TypeName(), h.SubTypeName(), h.RemoteIndex, h.MessageCounter, + normalizeName(p.toAddr().String()), + detail, ) } } @@ -269,6 +335,11 @@ func (r *R) renderFlow() { } } +func normalizeName(s string) string { + rx := regexp.MustCompile("[\\[\\]\\:]") + return rx.ReplaceAllLiteralString(s, "_") +} + // IgnoreFlow tells the router to stop recording future flows that matches the provided criteria. // messageType and subType will target nebula underlay packets while tun will target nebula overlay packets // NOTE: This is a very broad system, if you set tun to true then no more tun traffic will be rendered @@ -305,7 +376,7 @@ func (r *R) RenderHostmaps(title string, controls ...*nebula.Control) { func (r *R) renderHostmaps(title string) { c := maps.Values(r.controls) sort.SliceStable(c, func(i, j int) bool { - return c[i].GetVpnIp() > c[j].GetVpnIp() + return c[i].GetVpnAddrs()[0].Compare(c[j].GetVpnAddrs()[0]) > 0 }) s := renderHostmaps(c...) @@ -364,29 +435,34 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool r.renderHostmaps(fmt.Sprintf("Packet %v", len(r.flow))) - if len(r.ignoreFlows) > 0 { - var h header.H - err := h.Parse(p.Data) - if err != nil { - panic(err) - } + var h header.H + var parseErr error + if !tun { + parseErr = h.Parse(p.Data) + } - for _, i := range r.ignoreFlows { - if !tun { - if i.messageType == h.Type && i.subType == h.Subtype { - return nil - } - } else if i.tun.HasValue && i.tun.IsTrue { + // Decide before copying, the copy comes from a freelist and an ignored packet would never be released + for _, i := range r.ignoreFlows { + if tun { + if i.tun.HasValue && i.tun.IsTrue { return nil } + continue + } + + // A packet we could not parse has no type to match against, so no rule can ignore it + if parseErr == nil && i.messageType == h.Type && i.subType == h.Subtype { + return nil } } fp := &packet{ - from: from, - to: to, - packet: p.Copy(), - tun: tun, + from: from, + to: to, + packet: p.Copy(), + tun: tun, + h: h, + parseErr: parseErr, } r.flow = append(r.flow, flowEntry{packet: fp}) @@ -420,79 +496,165 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) [] // Nope, lets push the sender along case p := <-udpTx: - outAddr := sender.GetUDPAddr() r.Lock() - inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort)) - c := r.getControl(outAddr, inAddr, p) + a := sender.GetUDPAddr() + c := r.getControl(a, p.To, p) if c == nil { r.Unlock() - panic("No control for udp tx") + panic("No control for udp tx " + a.String()) } fp := r.unlockedInjectFlow(sender, c, p, false) - c.InjectUDPPacket(p) + c.InjectUDPPacket(p) // copies internally; original is ours to release fp.WasReceived() r.Unlock() + p.Release() } } } -// RouteForAllUntilTxTun will route for everyone and return when a packet is seen on receivers tun -// If the router doesn't have the nebula controller for that address, we panic +// RouteForAllUntilTxTun will route for everyone and return when a packet is seen on the receiver's tun. +// If a control's UDP TX address can't be matched to a registered control, we panic. +// +// For allocation-sensitive callers (hot-path benchmarks, in particular relay +// benches with 3+ controls), call EnableFanIn() first. func (r *R) RouteForAllUntilTxTun(receiver *nebula.Control) []byte { - sc := make([]reflect.SelectCase, len(r.controls)+1) - cm := make([]*nebula.Control, len(r.controls)+1) - - i := 0 - sc[i] = reflect.SelectCase{ - Dir: reflect.SelectRecv, - Chan: reflect.ValueOf(receiver.GetTunTxChan()), - Send: reflect.Value{}, + if r.fanInOn.Load() { + return r.routeFanIn(receiver) } - cm[i] = receiver + return r.routeReflect(receiver) +} - i++ - for _, c := range r.controls { - sc[i] = reflect.SelectCase{ - Dir: reflect.SelectRecv, - Chan: reflect.ValueOf(c.GetUDPTxChan()), - Send: reflect.Value{}, +// routeFanIn is the alloc-free path used when EnableFanIn is in effect. +func (r *R) routeFanIn(receiver *nebula.Control) []byte { + tunTx := receiver.GetTunTxChan() + for { + select { + case p := <-tunTx: + r.Lock() + if r.flow != nil { + np := udp.Packet{Data: make([]byte, len(p))} + copy(np.Data, p) + r.unlockedInjectFlow(receiver, receiver, &np, true) + } + r.Unlock() + return p + case fp := <-r.udpFanIn: + r.routeUDP(fp.from, fp.pkt) } - - cm[i] = c - i++ } +} +// routeReflect is the default reflect.Select-based path. Pays the boxing allocation per call but doesn't interfere +// with tests that pull packets directly from controls' UDP TX channels via GetFromUDP. +func (r *R) routeReflect(receiver *nebula.Control) []byte { + sc, cm := r.selectCasesFor(receiver) for { x, rx, _ := reflect.Select(sc) - r.Lock() - if x == 0 { - // we are the tun tx, we can exit p := rx.Interface().([]byte) - np := udp.Packet{Data: make([]byte, len(p))} - copy(np.Data, p) - - r.unlockedInjectFlow(cm[x], cm[x], &np, true) + r.Lock() + if r.flow != nil { + np := udp.Packet{Data: make([]byte, len(p))} + copy(np.Data, p) + r.unlockedInjectFlow(cm[x], cm[x], &np, true) + } r.Unlock() return p + } + r.routeUDP(cm[x], rx.Interface().(*udp.Packet)) + } +} - } else { - // we are a udp tx, route and continue - p := rx.Interface().(*udp.Packet) - outAddr := cm[x].GetUDPAddr() +// EnableFanIn switches RouteForAllUntilTxTun to the alloc-free fan-in path. +// One forwarder goroutine per registered control drains UDP TX into a shared channel that RouteForAllUntilTxTun selects +// on alongside the receiver's TUN TX channel. +func (r *R) EnableFanIn() { + r.fanInMu.Lock() + defer r.fanInMu.Unlock() + if r.fanInOn.Load() { + return + } + r.udpFanIn = make(chan fannedPacket, 32) + r.stopFanIn = make(chan struct{}) + for _, c := range r.controls { + r.startFanInWorker(c) + } + r.fanInOn.Store(true) + r.t.Cleanup(r.stopFanInWorkers) +} - inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort)) - c := r.getControl(outAddr, inAddr, p) - if c == nil { - r.Unlock() - panic("No control for udp tx") +// startFanInWorker spawns a goroutine that drains c's UDP TX into r.udpFanIn. +func (r *R) startFanInWorker(c *nebula.Control) { + r.fanInWG.Add(1) + udpTx := c.GetUDPTxChan() + go func() { + defer r.fanInWG.Done() + for { + select { + case <-r.stopFanIn: + return + case p := <-udpTx: + select { + case <-r.stopFanIn: + p.Release() + return + case r.udpFanIn <- fannedPacket{from: c, pkt: p}: + } } - fp := r.unlockedInjectFlow(cm[x], c, p, false) - c.InjectUDPPacket(p) - fp.WasReceived() } - r.Unlock() + }() +} + +// stopFanInWorkers signals the fan-in goroutines to exit and waits for them. +func (r *R) stopFanInWorkers() { + r.fanInMu.Lock() + wasOn := r.fanInOn.Swap(false) + r.fanInMu.Unlock() + if !wasOn { + return } + close(r.stopFanIn) + r.fanInWG.Wait() +} + +// routeUDP forwards a UDP TX packet from the named source control to the destination control derived from p.To, +// releasing the source packet after InjectUDPPacket has copied its bytes into a fresh pool slot. +func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) { + r.Lock() + defer r.Unlock() + a := from.GetUDPAddr() + c := r.getControl(a, p.To, p) + if c == nil { + panic(fmt.Sprintf("No control for udp tx %s", p.To)) + } + fp := r.unlockedInjectFlow(from, c, p, false) + c.InjectUDPPacket(p) // copies internally; original is ours to release + fp.WasReceived() + p.Release() +} + +// selectCasesFor returns the SelectCase array used by routeReflect: one slot for the receiver's TUN TX channel followed +// by one per control's UDP TX channel. Cached for the test lifetime, only rebuilt if the receiver changes. +func (r *R) selectCasesFor(receiver *nebula.Control) ([]reflect.SelectCase, []*nebula.Control) { + r.Lock() + defer r.Unlock() + if r.selRecvCtl == receiver && r.selCases != nil { + return r.selCases, r.selCtls + } + sc := make([]reflect.SelectCase, len(r.controls)+1) + cm := make([]*nebula.Control, len(r.controls)+1) + sc[0] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(receiver.GetTunTxChan())} + cm[0] = receiver + i := 1 + for _, c := range r.controls { + sc[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(c.GetUDPTxChan())} + cm[i] = c + i++ + } + r.selRecvCtl = receiver + r.selCases = sc + r.selCtls = cm + return sc, cm } // RouteExitFunc will call the whatDo func with each udp packet from sender. @@ -509,18 +671,17 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) { panic(err) } - outAddr := sender.GetUDPAddr() - inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort)) - receiver := r.getControl(outAddr, inAddr, p) + receiver := r.getControl(sender.GetUDPAddr(), p.To, p) if receiver == nil { r.Unlock() - panic("Can't route for host: " + inAddr) + panic("Can't RouteExitFunc for host: " + p.To.String()) } e := whatDo(p, receiver) switch e { case ExitNow: r.Unlock() + p.Release() return case RouteAndExit: @@ -528,8 +689,13 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) { receiver.InjectUDPPacket(p) fp.WasReceived() r.Unlock() + p.Release() return + case Drop: + // Record it so the flow log shows the attempt, but never hand it to the receiver + r.unlockedInjectFlow(sender, receiver, p, false) + case KeepRouting: fp := r.unlockedInjectFlow(sender, receiver, p, false) receiver.InjectUDPPacket(p) @@ -540,6 +706,7 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) { } r.Unlock() + p.Release() } } @@ -559,6 +726,85 @@ func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType header.Messag }) } +// RouteFor routes everything that shows up for the given duration and then returns. Use it to let a test settle +// deterministically rather than sleeping and hoping: a single FlushAll races a completing handshake, which queues +// more packets right behind it. +func (r *R) RouteFor(d time.Duration) { + r.RouteForAllExitFuncOrTimeout(d, func(*udp.Packet, *nebula.Control) ExitType { + return KeepRouting + }) +} + +// RouteForAllExitFuncOrTimeout is RouteForAllExitFunc with a deadline, reporting whether whatDo asked to exit +// before time ran out. The unbounded version blocks forever on a quiet network, so this is what a test needs to +// assert that something does NOT happen, or to route for a fixed settling period. +func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc) bool { + sc := make([]reflect.SelectCase, 0, len(r.controls)+1) + cm := make([]*nebula.Control, 0, len(r.controls)) + + for _, c := range r.controls { + sc = append(sc, reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(c.GetUDPTxChan()), + Send: reflect.Value{}, + }) + cm = append(cm, c) + } + + timer := time.NewTimer(timeout) + defer timer.Stop() + sc = append(sc, reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(timer.C), + Send: reflect.Value{}, + }) + + for { + x, rx, _ := reflect.Select(sc) + if x == len(cm) { + return false + } + + r.Lock() + p := rx.Interface().(*udp.Packet) + receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p) + if receiver == nil { + r.Unlock() + panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String()) + } + + e := whatDo(p, receiver) + switch e { + case ExitNow: + r.Unlock() + p.Release() + return true + + case RouteAndExit: + fp := r.unlockedInjectFlow(cm[x], receiver, p, false) + receiver.InjectUDPPacket(p) + fp.WasReceived() + r.Unlock() + p.Release() + return true + + case Drop: + // Record it so the flow log shows the attempt, but never hand it to the receiver + r.unlockedInjectFlow(cm[x], receiver, p, false) + + case KeepRouting: + fp := r.unlockedInjectFlow(cm[x], receiver, p, false) + receiver.InjectUDPPacket(p) + fp.WasReceived() + + default: + panic(fmt.Sprintf("Unknown exitFunc return: %v", e)) + } + r.Unlock() + p.Release() + } +} + func (r *R) RouteForAllUntilAfterMsgTypeTo(receiver *nebula.Control, msgType header.MessageType, subType header.MessageSubType) { h := &header.H{} r.RouteForAllExitFunc(func(p *udp.Packet, r *nebula.Control) ExitType { @@ -590,13 +836,13 @@ func (r *R) InjectUDPPacket(sender, receiver *nebula.Control, packet *udp.Packet // RouteForUntilAfterToAddr will route for sender and return only after it sees and sends a packet destined for toAddr // finish can be any of the exitType values except `keepRouting`, the default value is `routeAndExit` // If the router doesn't have the nebula controller for that address, we panic -func (r *R) RouteForUntilAfterToAddr(sender *nebula.Control, toAddr *net.UDPAddr, finish ExitType) { +func (r *R) RouteForUntilAfterToAddr(sender *nebula.Control, toAddr netip.AddrPort, finish ExitType) { if finish == KeepRouting { finish = RouteAndExit } r.RouteExitFunc(sender, func(p *udp.Packet, r *nebula.Control) ExitType { - if p.ToIp.Equal(toAddr.IP) && p.ToPort == uint16(toAddr.Port) { + if p.To == toAddr { return finish } @@ -630,19 +876,17 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) { r.Lock() p := rx.Interface().(*udp.Packet) - - outAddr := cm[x].GetUDPAddr() - inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort)) - receiver := r.getControl(outAddr, inAddr, p) + receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p) if receiver == nil { r.Unlock() - panic("Can't route for host: " + inAddr) + panic("Can't RouteForAllExitFunc for host: " + p.To.String()) } e := whatDo(p, receiver) switch e { case ExitNow: r.Unlock() + p.Release() return case RouteAndExit: @@ -650,8 +894,13 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) { receiver.InjectUDPPacket(p) fp.WasReceived() r.Unlock() + p.Release() return + case Drop: + // Record it so the flow log shows the attempt, but never hand it to the receiver + r.unlockedInjectFlow(cm[x], receiver, p, false) + case KeepRouting: fp := r.unlockedInjectFlow(cm[x], receiver, p, false) receiver.InjectUDPPacket(p) @@ -661,6 +910,7 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) { panic(fmt.Sprintf("Unknown exitFunc return: %v", e)) } r.Unlock() + p.Release() } } @@ -697,41 +947,27 @@ func (r *R) FlushAll() { p := rx.Interface().(*udp.Packet) - outAddr := cm[x].GetUDPAddr() - inAddr := net.JoinHostPort(p.ToIp.String(), fmt.Sprintf("%v", p.ToPort)) - receiver := r.getControl(outAddr, inAddr, p) + receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p) if receiver == nil { r.Unlock() - panic("Can't route for host: " + inAddr) + panic("Can't FlushAll for host: " + p.To.String()) } + receiver.InjectUDPPacket(p) r.Unlock() + p.Release() } } // getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change // This is an internal router function, the caller must hold the lock -func (r *R) getControl(fromAddr, toAddr string, p *udp.Packet) *nebula.Control { - if newAddr, ok := r.outNat[fromAddr+":"+toAddr]; ok { - p.FromIp = newAddr.IP - p.FromPort = uint16(newAddr.Port) +func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control { + if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok { + p.From = newAddr } c, ok := r.inNat[toAddr] if ok { - sHost, sPort, err := net.SplitHostPort(toAddr) - if err != nil { - panic(err) - } - - port, err := strconv.Atoi(sPort) - if err != nil { - panic(err) - } - - r.outNat[c.GetUDPAddr()+":"+fromAddr] = net.UDPAddr{ - IP: net.ParseIP(sHost), - Port: port, - } + r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr return c } @@ -739,29 +975,42 @@ func (r *R) getControl(fromAddr, toAddr string, p *udp.Packet) *nebula.Control { } func (r *R) formatUdpPacket(p *packet) string { - packet := gopacket.NewPacket(p.packet.Data, layers.LayerTypeIPv4, gopacket.Lazy) - v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4) - if v4 == nil { - panic("not an ipv4 packet") + var packet gopacket.Packet + var srcAddr netip.Addr + + packet = gopacket.NewPacket(p.packet.Data, layers.LayerTypeIPv6, gopacket.Lazy) + if packet.ErrorLayer() == nil { + v6 := packet.Layer(layers.LayerTypeIPv6).(*layers.IPv6) + if v6 == nil { + panic("not an ipv6 packet") + } + srcAddr, _ = netip.AddrFromSlice(v6.SrcIP) + } else { + packet = gopacket.NewPacket(p.packet.Data, layers.LayerTypeIPv4, gopacket.Lazy) + v6 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4) + if v6 == nil { + panic("not an ipv6 packet") + } + srcAddr, _ = netip.AddrFromSlice(v6.SrcIP) } from := "unknown" - if c, ok := r.vpnControls[iputil.Ip2VpnIp(v4.SrcIP)]; ok { - from = c.GetUDPAddr() + if c, ok := r.vpnControls[srcAddr]; ok { + from = c.GetUDPAddr().String() } - udp := packet.Layer(layers.LayerTypeUDP).(*layers.UDP) - if udp == nil { + udpLayer := packet.Layer(layers.LayerTypeUDP).(*layers.UDP) + if udpLayer == nil { panic("not a udp packet") } data := packet.ApplicationLayer() return fmt.Sprintf( " %s-->>%s: src port: %v
dest port: %v
data: \"%v\"\n", - strings.Replace(from, ":", "-", 1), - strings.Replace(p.to.GetUDPAddr(), ":", "-", 1), - udp.SrcPort, - udp.DstPort, + normalizeName(from), + normalizeName(p.to.GetUDPAddr().String()), + udpLayer.SrcPort, + udpLayer.DstPort, string(data.Payload()), ) } diff --git a/e2e/sshd_test.go b/e2e/sshd_test.go new file mode 100644 index 00000000..e91f1bd0 --- /dev/null +++ b/e2e/sshd_test.go @@ -0,0 +1,125 @@ +//go:build e2e_testing +// +build e2e_testing + +package e2e + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/pem" + "net" + "strings" + "testing" + "time" + + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/ssh" +) + +func TestSSHDLifecycle(t *testing.T) { + // TestSSHDLifecycle exercises the in-process sshd through several config reloads and a Control.Stop. + ca, _, caKey, _ := cert_test.NewTestCaCert( + cert.Version1, cert.Curve_CURVE25519, + time.Now(), time.Now().Add(10*time.Minute), + nil, nil, []string{}, + ) + + hostKeyPEM := generateSSHHostKey(t) + clientSigner, clientAuthKey := generateSSHClientKey(t) + sshdAddr := allocLoopbackPort(t) + + overrides := m{ + "sshd": m{ + "enabled": true, + "listen": sshdAddr, + "host_key": hostKeyPEM, + "authorized_users": []m{{ + "user": "tester", + "keys": []string{clientAuthKey}, + }}, + }, + } + control, _, _, _ := newSimpleServer(cert.Version1, ca, caKey, "sshd-test", "10.222.0.1/24", overrides) + control.Start() + t.Cleanup(func() { control.Stop() }) + + // sshd binds in a goroutine after Start returns; wait for it. + require.Eventually(t, func() bool { return canDial(sshdAddr) }, 2*time.Second, 25*time.Millisecond, + "sshd never started listening") + + for i := 1; i <= 3; i++ { + out := sshExecReload(t, sshdAddr, clientSigner) + assert.Contains(t, out, "Reloading config", "reload cycle %d", i) + require.Eventually(t, func() bool { return canDial(sshdAddr) }, 2*time.Second, 25*time.Millisecond, + "sshd not listening after reload cycle %d", i) + } + + control.Stop() + require.Eventually(t, func() bool { return !canDial(sshdAddr) }, 2*time.Second, 25*time.Millisecond, + "sshd still listening after Control.Stop") +} + +func canDial(addr string) bool { + c, err := net.DialTimeout("tcp", addr, 100*time.Millisecond) + if err != nil { + return false + } + _ = c.Close() + return true +} + +// allocLoopbackPort grabs an unused TCP port on 127.0.0.1, closes it, and returns the address. There +// is a small race between releasing the port and the sshd reclaiming it; in practice the OS keeps the +// port available long enough for the test to bind it. +func allocLoopbackPort(t *testing.T) string { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := l.Addr().String() + require.NoError(t, l.Close()) + return addr +} + +func generateSSHHostKey(t *testing.T) string { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + block, err := ssh.MarshalPrivateKey(priv, "nebula-e2e-host") + require.NoError(t, err) + return string(pem.EncodeToMemory(block)) +} + +func generateSSHClientKey(t *testing.T) (ssh.Signer, string) { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + signer, err := ssh.NewSignerFromKey(priv) + require.NoError(t, err) + auth := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(signer.PublicKey()))) + return signer, auth +} + +func sshExecReload(t *testing.T, addr string, signer ssh.Signer) string { + t.Helper() + cfg := &ssh.ClientConfig{ + User: "tester", + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 2 * time.Second, + } + client, err := ssh.Dial("tcp", addr, cfg) + require.NoError(t, err) + defer client.Close() + + sess, err := client.NewSession() + require.NoError(t, err) + defer sess.Close() + + // reload tears the channel down before sending exit-status, so Output returns an error on the + // channel close. The output buffer still has whatever the reload callback wrote before that. + out, _ := sess.Output("reload") + return string(out) +} diff --git a/e2e/tunnels_test.go b/e2e/tunnels_test.go new file mode 100644 index 00000000..7874cc79 --- /dev/null +++ b/e2e/tunnels_test.go @@ -0,0 +1,573 @@ +//go:build e2e_testing +// +build e2e_testing + +package e2e + +import ( + "fmt" + "net/netip" + "testing" + "time" + + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/e2e/router" + "github.com/slackhq/nebula/header" + "github.com/slackhq/nebula/udp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestDropInactiveTunnels(t *testing.T) { + t.Parallel() + // The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides + // under ideal conditions + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "5s"}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "10m"}}) + + // Share our underlay information + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + // Start the servers + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + + r.Log("Assert the tunnel between me and them works") + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + r.Log("Go inactive and wait for the tunnels to get dropped") + waitStart := time.Now() + for { + myIndexes := myControl.GetHostmapIndexCount() + theirIndexes := theirControl.GetHostmapIndexCount() + if myIndexes == 0 && theirIndexes == 0 { + break + } + + since := time.Since(waitStart) + r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since) + if since > time.Second*30 { + t.Fatal("Tunnel should have been declared inactive after 5 seconds and before 30 seconds") + } + + time.Sleep(1 * time.Second) + r.FlushAll() + } + + r.Logf("Inactive tunnels were dropped within %v", time.Since(waitStart)) + myControl.Stop() + theirControl.Stop() +} + +func TestCertUpgrade(t *testing.T) { + t.Parallel() + // The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides + // under ideal conditions + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + caB, err := ca.MarshalPEM() + if err != nil { + panic(err) + } + ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + ca2B, err := ca2.MarshalPEM() + if err != nil { + panic(err) + } + caStr := fmt.Sprintf("%s\n%s", caB, ca2B) + + myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{}) + _, myCert2Pem := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2) + + theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{}) + theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2) + + myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert}, myPrivKey, m{}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{}) + + // Share our underlay information + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + // Start the servers + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + + r.Log("Assert the tunnel between me and them works") + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + r.Log("yay") + //todo ??? + time.Sleep(1 * time.Second) + r.FlushAll() + + mc := m{ + "pki": m{ + "ca": caStr, + "cert": string(myCert2Pem), + "key": string(myPrivKey), + }, + //"tun": m{"disabled": true}, + "firewall": myC.Settings["firewall"], + //"handshakes": m{ + // "try_interval": "1s", + //}, + "listen": myC.Settings["listen"], + "logging": myC.Settings["logging"], + "timers": myC.Settings["timers"], + } + + cb, err := yaml.Marshal(mc) + if err != nil { + panic(err) + } + + r.Logf("reload new v2-only config") + err = myC.ReloadConfigString(string(cb)) + assert.NoError(t, err) + r.Log("yay, spin until their sees it") + waitStart := time.Now() + for { + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + if c == nil { + r.Log("nil") + } else { + version := c.Cert.Version() + r.Logf("version %d", version) + if version == cert.Version2 { + break + } + } + since := time.Since(waitStart) + if since > time.Second*10 { + t.Fatal("Cert should be new by now") + } + time.Sleep(time.Second) + } + + r.RenderHostmaps("Final hostmaps", myControl, theirControl) + + myControl.Stop() + theirControl.Stop() +} + +func TestCertDowngrade(t *testing.T) { + t.Parallel() + // The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides + // under ideal conditions + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + caB, err := ca.MarshalPEM() + if err != nil { + panic(err) + } + ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + ca2B, err := ca2.MarshalPEM() + if err != nil { + panic(err) + } + caStr := fmt.Sprintf("%s\n%s", caB, ca2B) + + myCert, _, myPrivKey, myCertPem := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{}) + myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2) + + theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{}) + theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2) + + myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{}) + + // Share our underlay information + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + // Start the servers + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + + r.Log("Assert the tunnel between me and them works") + //assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) + //r.Log("yay") + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + r.Log("yay") + //todo ??? + time.Sleep(1 * time.Second) + r.FlushAll() + + mc := m{ + "pki": m{ + "ca": caStr, + "cert": string(myCertPem), + "key": string(myPrivKey), + }, + "firewall": myC.Settings["firewall"], + "listen": myC.Settings["listen"], + "logging": myC.Settings["logging"], + "timers": myC.Settings["timers"], + } + + cb, err := yaml.Marshal(mc) + if err != nil { + panic(err) + } + + r.Logf("reload new v1-only config") + err = myC.ReloadConfigString(string(cb)) + assert.NoError(t, err) + r.Log("yay, spin until their sees it") + waitStart := time.Now() + for { + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) + if c == nil || c2 == nil { + r.Log("nil") + } else { + version := c.Cert.Version() + theirVersion := c2.Cert.Version() + r.Logf("version %d,%d", version, theirVersion) + if version == cert.Version1 { + break + } + } + since := time.Since(waitStart) + if since > time.Second*5 { + r.Log("it is unusual that the cert is not new yet, but not a failure yet") + } + if since > time.Second*10 { + r.Log("wtf") + t.Fatal("Cert should be new by now") + } + time.Sleep(time.Second) + } + + r.RenderHostmaps("Final hostmaps", myControl, theirControl) + + myControl.Stop() + theirControl.Stop() +} + +func TestCertMismatchCorrection(t *testing.T) { + t.Parallel() + // The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides + // under ideal conditions + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + + myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{}) + myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2) + + theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{}) + theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2) + + myControl, myVpnIpNet, myUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{}) + + // Share our underlay information + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + // Start the servers + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + defer r.RenderFlow() + + r.Log("Assert the tunnel between me and them works") + //assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) + //r.Log("yay") + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + r.Log("yay") + //todo ??? + time.Sleep(1 * time.Second) + r.FlushAll() + + waitStart := time.Now() + for { + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) + if c == nil || c2 == nil { + r.Log("nil") + } else { + version := c.Cert.Version() + theirVersion := c2.Cert.Version() + r.Logf("version %d,%d", version, theirVersion) + if version == theirVersion { + break + } + } + since := time.Since(waitStart) + if since > time.Second*5 { + r.Log("wtf") + } + if since > time.Second*10 { + r.Log("wtf") + t.Fatal("Cert should be new by now") + } + time.Sleep(time.Second) + } + + r.RenderHostmaps("Final hostmaps", myControl, theirControl) + + myControl.Stop() + theirControl.Stop() +} + +func TestCrossStackRelaysWork(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay ", "10.128.0.128/24,fc00::128/64", m{"relay": m{"am_relay": true}}) + theirUdp := netip.MustParseAddrPort("10.0.0.2:4242") + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, "them ", "fc00::2/64", theirUdp, m{"relay": m{"use_relays": true}}) + + //myVpnV4 := myVpnIpNet[0] + myVpnV6 := myVpnIpNet[1] + relayVpnV4 := relayVpnIpNet[0] + relayVpnV6 := relayVpnIpNet[1] + theirVpnV6 := theirVpnIpNet[0] + + // Teach my how to get to the relay and that their can be reached via the relay + myControl.InjectLightHouseAddr(relayVpnV4.Addr(), relayUdpAddr) + myControl.InjectLightHouseAddr(relayVpnV6.Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnV6.Addr(), []netip.Addr{relayVpnV6.Addr()}) + relayControl.InjectLightHouseAddr(theirVpnV6.Addr(), theirUdpAddr) + + // Build a router so we don't have to reason who gets which packet + r := router.NewR(t, myControl, relayControl, theirControl) + defer r.RenderFlow() + + // Start the servers + myControl.Start() + relayControl.Start() + theirControl.Start() + + t.Log("Trigger a handshake from me to them via the relay") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me"))) + + p := r.RouteForAllUntilTxTun(theirControl) + r.Log("Assert the tunnel works") + assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80) + + t.Log("reply?") + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them"))) + p = r.RouteForAllUntilTxTun(myControl) + assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80) + + r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl) + //t.Log("finish up") + //myControl.Stop() + //theirControl.Stop() + //relayControl.Stop() +} + +// TestRelayReplayProtection asserts that a relay (forwarding-type) node rejects +// replayed relay frames. A captured relay frame, re-injected with the same +// message counter, must be dropped by the replay window rather than re-forwarded +// to the relay target. Before the fix, handleOutsideRelayPacket authenticated the +// frame but never advanced the replay window, so every replay was re-forwarded. +func TestRelayReplayProtection(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}}) + relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay ", "10.128.0.128/24,fc00::128/64", m{"relay": m{"am_relay": true}}) + theirUdp := netip.MustParseAddrPort("10.0.0.2:4242") + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, "them ", "fc00::2/64", theirUdp, m{"relay": m{"use_relays": true}}) + + myVpnV6 := myVpnIpNet[1] + relayVpnV4 := relayVpnIpNet[0] + relayVpnV6 := relayVpnIpNet[1] + theirVpnV6 := theirVpnIpNet[0] + + // Teach me how to reach the relay and that them is reachable via the relay + myControl.InjectLightHouseAddr(relayVpnV4.Addr(), relayUdpAddr) + myControl.InjectLightHouseAddr(relayVpnV6.Addr(), relayUdpAddr) + myControl.InjectRelays(theirVpnV6.Addr(), []netip.Addr{relayVpnV6.Addr()}) + relayControl.InjectLightHouseAddr(theirVpnV6.Addr(), theirUdpAddr) + + r := router.NewR(t, myControl, relayControl, theirControl) + defer r.RenderFlow() + + myControl.Start() + relayControl.Start() + theirControl.Start() + + // Establish the relayed tunnel in both directions so all handshakes complete. + t.Log("Establish the relayed tunnel") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me"))) + p := r.RouteForAllUntilTxTun(theirControl) + assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80) + theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them"))) + p = r.RouteForAllUntilTxTun(myControl) + assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80) + + // Drain anything still queued on me's UDP tx so the next packet we pull is the + // relay frame we are about to generate. + for myControl.GetFromUDP(false) != nil { + } + + // Capture a single legitimate relay frame that me transmits toward the relay. + t.Log("Capture a relay frame from me -> relay") + myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("replay me"))) + relayFrame := myControl.GetFromUDP(true) + require.Equal(t, relayUdpAddr, relayFrame.To, "captured frame should be addressed to the relay") + var fh header.H + require.NoError(t, fh.Parse(relayFrame.Data)) + require.Equal(t, header.Message, fh.Type) + require.Equal(t, header.MessageRelay, fh.Subtype) + + // drainForwards counts relay frames the relay forwards toward them within the + // settle window. We match on destination + (Message, MessageRelay) so the + // relay's own direct traffic to them can't be miscounted. + drainForwards := func(settle time.Duration) int { + ch := relayControl.GetUDPTxChan() + count := 0 + for { + select { + case pkt := <-ch: + var ph header.H + if pkt.To == theirUdpAddr && ph.Parse(pkt.Data) == nil && + ph.Type == header.Message && ph.Subtype == header.MessageRelay { + count++ + } + pkt.Release() + case <-time.After(settle): + return count + } + } + } + + // First delivery of the captured frame: the relay should forward it once. + t.Log("Deliver the captured frame once; relay forwards it to them") + relayControl.InjectUDPPacket(relayFrame) + require.Equal(t, 1, drainForwards(200*time.Millisecond), "relay should forward the first, legitimate copy") + + // Replay the exact same frame several times. A correct replay window rejects + // these duplicates so the relay forwards none of them. + t.Log("Replay the captured frame; relay must drop the duplicates") + const replays = 3 + for i := 0; i < replays; i++ { + relayControl.InjectUDPPacket(relayFrame) + } + forwarded := drainForwards(200 * time.Millisecond) + assert.Equal(t, 0, forwarded, "relay re-forwarded %d/%d replayed relay frames; replay protection is ineffective on relay tunnels", forwarded, replays) + + r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl) +} + +func TestCloseTunnelAuthenticated(t *testing.T) { + t.Parallel() + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "5s"}}) + theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "10m"}}) + + // Share our underlay information + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + + // Start the servers + myControl.Start() + theirControl.Start() + + r := router.NewR(t, myControl, theirControl) + + r.Log("Assert the tunnel between me and them works") + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + + r.Log("Close the tunnel") + myControl.CloseTunnel(theirVpnIpNet[0].Addr(), false) + r.FlushAll() + + waitStart := time.Now() + for { + myIndexes := myControl.GetHostmapIndexCount() + theirIndexes := theirControl.GetHostmapIndexCount() + if myIndexes == 0 && theirIndexes == 0 { + break + } + + since := time.Since(waitStart) + r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since) + if since > time.Second*6 { + t.Fatal("Tunnel should have been declared inactive after 2 seconds and before 6 seconds") + } + + time.Sleep(1 * time.Second) + //r.FlushAll() + } + + r.Logf("Happy path success, tunnels were dropped within %v", time.Since(waitStart)) + + myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr) + theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr) + r.Log("Assert another tunnel between me and them works") + assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) + hi := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) + if hi == nil { + t.Fatal("There is no hostinfo for this tunnel") + } + myHi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false) + if myHi == nil { + t.Fatal("There is no hostinfo for my tunnel") + } + r.Log("It does") + + buf := make([]byte, 1024) + hdr := header.H{ + Version: 1, + Type: header.CloseTunnel, + Subtype: 0, + Reserved: 0, + RemoteIndex: hi.RemoteIndex, + MessageCounter: 5, + } + out, err := hdr.Encode(buf) + if err != nil { + t.Fatal(err) + } + + pkt := &udp.Packet{ + To: hi.CurrentRemote, + From: myHi.CurrentRemote, + Data: out, + } + r.InjectUDPPacket(myControl, theirControl, pkt) + r.Log("Injected bogus close tunnel. Let's see!") + waitStart = time.Now() + for { + myIndexes := myControl.GetHostmapIndexCount() + theirIndexes := theirControl.GetHostmapIndexCount() + if myIndexes == 0 { + t.Fatal("myIndexes should not be 0") + } + if theirIndexes == 0 { + t.Fatal("theirIndexes should not be 0, they should have rejected this bogus packet") + } + + since := time.Since(waitStart) + r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since) + if since > time.Second*4 { + t.Log("The tunnel would have been gone by now") + break + } + + time.Sleep(1 * time.Second) + r.FlushAll() + } + + myControl.Stop() + theirControl.Stop() +} diff --git a/examples/config.yml b/examples/config.yml index ff5b4031..d5409bce 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -13,6 +13,12 @@ pki: # disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid. #disconnect_invalid: true + # initiating_version controls which certificate version is used when initiating handshakes. + # This setting only applies if both a v1 and a v2 certificate are configured, in which case it will default to `1`. + # Once all hosts in the mesh are configured with both a v1 and v2 certificate then this should be changed to `2`. + # After all hosts in the mesh are using a v2 certificate then v1 certificates are no longer needed. + # initiating_version: 1 + # The static host map defines a set of hosts with fixed IP addresses on the internet (or any network). # A host can have multiple fixed IP addresses defined here, and nebula will try each when establishing a tunnel. # The syntax is: @@ -120,8 +126,8 @@ lighthouse: # Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined, # however using port 0 will dynamically assign a port and is recommended for roaming nodes. listen: - # To listen on both any ipv4 and ipv6 use "::" - host: 0.0.0.0 + # To listen on only ipv4, use "0.0.0.0" + host: "::" port: 4242 # Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg) # default is 64, does not support reload @@ -132,12 +138,37 @@ listen: # max, net.core.rmem_max and net.core.wmem_max #read_buffer: 10485760 #write_buffer: 10485760 + + # On Windows only + # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port. + # WFP sits below Windows Defender Firewall, so this lets peer handshakes reach Nebula's outside socket regardless + # of WDF's inbound rules. + # Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable. + #windows_bypass_wdf: true + + # On macOS only + # macOS scopes the udp socket to the interface it was created on, so moving between networks (wifi to wired, + # office to home) leaves Nebula sending out an interface that no longer has a route. When true, Nebula watches + # the routing socket and rebinds the listener once the change settles. + # iOS does not use this, the host app drives the same rebind itself. + # Default true. Not reloadable. + #rebind_on_network_change: true + # By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection # in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running # on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes. # valid values: always, never, private # This setting is reloadable. #send_recv_error: always + # Similar to send_recv_error, this option lets you configure if you want to accept "recv_error" packets from remote hosts. + # valid values: always, never, private + # This setting is reloadable. + #accept_recv_error: always + # The so_sock option is a Linux-specific feature that allows all outgoing Nebula packets to be tagged with a specific identifier. + # This tagging enables IP rule-based filtering. For example, it supports 0.0.0.0/0 unsafe_routes, + # allowing for more precise routing decisions based on the packet tags. Default is 0 meaning no mark is set. + # This setting is reloadable. + #so_mark: 0 # Routines is the number of thread pairs to run that consume from the tun and UDP queues. # Currently, this defaults to 1 which means we have 1 tun queue reader and 1 @@ -148,17 +179,21 @@ listen: punchy: # Continues to punch inbound/outbound at a regular interval to avoid expiration of firewall nat mappings + # This setting is reloadable. punch: true # respond means that a node you are trying to reach will connect back out to you if your hole punching fails # this is extremely useful if one node is behind a difficult nat, such as a symmetric NAT # Default is false + # This setting is reloadable. #respond: true # delays a punch response for misbehaving NATs, default is 1 second. + # This setting is reloadable. #delay: 1s # set the delay before attempting punchy.respond. Default is 5 seconds. respond must be true to take effect. + # This setting is reloadable. #respond_delay: 5s # Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes @@ -167,8 +202,7 @@ punchy: # Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest # path to a network adjacent nebula node. -# NOTE: the previous option "local_range" only allowed definition of a single range -# and has been deprecated for "preferred_ranges" +# This setting is reloadable. #preferred_ranges: ["172.16.0.0/24"] # sshd can expose informational and administrative functions via ssh. This can expose informational and administrative @@ -181,12 +215,21 @@ punchy: # A file containing the ssh host private key to use # A decent way to generate one: ssh-keygen -t ed25519 -f ssh_host_ed25519_key -N "" < /dev/null #host_key: ./ssh_host_ed25519_key - # A file containing a list of authorized public keys + # Authorized users and their public keys #authorized_users: #- user: steeeeve # keys can be an array of strings or single string #keys: #- "ssh public key string" + # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access. + #trusted_cas: + #- "ssh public key string" + # sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile, + # save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory, + # and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug. + # The directory is NOT automatically created. + # Overriding this to "" is the same as "/" and will allow overwriting any path on the host. + #sandbox_dir: /var/tmp/nebula-debug # EXPERIMENTAL: relay support for networks that can't establish direct connections. relay: @@ -226,10 +269,32 @@ tun: # Unsafe routes allows you to route traffic over nebula to non-nebula nodes # Unsafe routes should be avoided unless you have hosts/services that cannot run nebula - # NOTE: The nebula certificate of the "via" node *MUST* have the "route" defined as a subnet in its certificate + # Supports weighted ECMP if you define a list of gateways, this can be used for load balancing or redundancy to hosts outside of nebula + # NOTES: + # * You will only see a single gateway in the routing table if you are not on linux + # * If a gateway is not reachable through the overlay another gateway will be selected to send the traffic through, ignoring weights + # + # unsafe_routes: + # # Multiple gateways without defining a weight defaults to a weight of 1, this will balance traffic equally between the three gateways + # - route: 192.168.87.0/24 + # via: + # - gateway: 10.0.0.1 + # - gateway: 10.0.0.2 + # - gateway: 10.0.0.3 + # # Multiple gateways with a weight, this will balance traffic accordingly + # - route: 192.168.87.0/24 + # via: + # - gateway: 10.0.0.1 + # weight: 10 + # - gateway: 10.0.0.2 + # weight: 5 + # + # NOTE: The nebula certificate of the "via" node(s) *MUST* have the "route" defined as a subnet in its certificate + # `via`: single node or list of gateways to use for this route # `mtu`: will default to tun mtu if this option is not specified # `metric`: will default to 0 if this option is not specified # `install`: will default to true, controls whether this route is installed in the systems routing table. + # This setting is reloadable. unsafe_routes: #- route: 172.16.1.0/24 # via: 192.168.100.99 @@ -237,28 +302,49 @@ tun: # metric: 100 # install: true + # On Windows only, sets the network category of the nebula interface. Without this, Windows often + # leaves the network as "Unidentified" and treats it as Public, which makes the host firewall more + # restrictive than you usually want for an overlay between trusted peers. Valid values: + # private - treat the nebula network as a private/trusted network (default) + # public - treat it as a public/untrusted network + # domain - treat it as a domain-authenticated network + # unset - leave whatever Windows decided alone + # Not reloadable. + #network_category: private + + # On Windows only + # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to the nebula adapter LUID. + # WFP sits below Windows Defender Firewall, so this lets inbound traffic through regardless of WDF rules. + # Filters are auto-removed when the adapter goes away. + # See listen.windows_bypass_wdf for the matching control over inbound to nebula's outside UDP listener. + # Default true; set to false to leave WDF in charge of inbound decisions on the nebula interface. Not reloadable. + #windows_bypass_wdf: true + # On linux only, set to true to manage unsafe routes directly on the system route table with gateway routes instead of # in nebula configuration files. Default false, not reloadable. #use_system_route_table: false + # Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default). + # If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss. + # SO_RCVBUFFORCE is used to avoid having to raise the system wide max + #use_system_route_table_buffer_size: 0 -# TODO # Configure logging level logging: - # panic, fatal, error, warning, info, or debug. Default is info + # trace, debug, info, warn, or error. Default is info and is reloadable. + # fatal and panic are accepted for backwards compatibility and map to error. + #NOTE: Debug and trace modes can log remotely controlled/untrusted data which can quickly fill a disk in some + # scenarios. Debug and trace logging are also CPU intensive and will decrease performance overall. + # Only enable debug or trace logging while actively investigating an issue. level: info - # json or text formats currently available. Default is text + # json or text formats currently available. Default is text. format: text - # Disable timestamp logging. useful when output is redirected to logging system that already adds timestamps. Default is false + # Disable timestamp logging. Useful when output is redirected to a logging system that already adds timestamps. Default is false. #disable_timestamp: true - # timestamp format is specified in Go time format, see: - # https://golang.org/pkg/time/#pkg-constants - # default when `format: json`: "2006-01-02T15:04:05Z07:00" (RFC3339) - # default when `format: text`: - # when TTY attached: seconds since beginning of execution - # otherwise: "2006-01-02T15:04:05Z07:00" (RFC3339) - # As an example, to log as RFC3339 with millisecond precision, set to: - #timestamp_format: "2006-01-02T15:04:05.000Z07:00" + # Timestamps use RFC3339Nano ("2006-01-02T15:04:05.999999999Z07:00") and are not configurable. +# The stats section is reloadable. A HUP may change the backend, toggle stats +# on or off, switch the listen/host address, or pick up new DNS for the +# configured graphite host. #stats: #type: graphite #prefix: nebula @@ -276,10 +362,12 @@ logging: # enables counter metrics for meta packets # e.g.: `messages.tx.handshake` # NOTE: `message.{tx,rx}.recv_error` is always emitted + # Not reloadable. #message_metrics: false # enables detailed counter metrics for lighthouse packets # e.g.: `lighthouse.rx.HostQuery` + # Not reloadable. #lighthouse_metrics: false # Handshake Manager Settings @@ -297,6 +385,18 @@ logging: # after receiving the response for lighthouse queries #trigger_buffer: 64 +# Tunnel manager settings +#tunnels: + # drop_inactive controls whether inactive tunnels are maintained or dropped after the inactive_timeout period has + # elapsed. + # In general, it is a good idea to enable this setting. It will be enabled by default in a future release. + # This setting is reloadable + #drop_inactive: false + + # inactivity_timeout controls how long a tunnel MUST NOT see any inbound or outbound traffic before being considered + # inactive and eligible to be dropped. + # This setting is reloadable + #inactivity_timeout: 10m # Nebula security group configuration firewall: @@ -305,15 +405,15 @@ firewall: # `drop` (default): silently drop the packet. # `reject`: send a reject reply. # - For TCP, this will be a RST "Connection Reset" packet. - # - For other protocols, this will be an ICMP port unreachable packet. + # - For other protocols, this will be an ICMP "Destination unreachable: Communication administratively prohibited" packet. outbound_action: drop inbound_action: drop - # Controls the default value for local_cidr. Default is true, will be deprecated after v1.9 and defaulted to false. - # This setting only affects nebula hosts with subnets encoded in their certificate. A nebula host acting as an - # unsafe router with `default_local_cidr_any: true` will expose their unsafe routes to every inbound rule regardless - # of the actual destination for the packet. Setting this to false requires each inbound rule to contain a `local_cidr` - # if the intention is to allow traffic to flow to an unsafe route. + # THIS FLAG IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE. (Defaults to false.) + # This setting only affects nebula hosts exposing unsafe_routes. When set to false, each inbound rule must contain a + # `local_cidr` if the intention is to allow traffic to flow to an unsafe route. When set to true, every firewall rule + # will apply to all configured unsafe_routes regardless of the actual destination of the packet, unless `local_cidr` + # is explicitly defined. This is usually not the desired behavior and should be avoided! #default_local_cidr_any: false conntrack: @@ -325,15 +425,16 @@ firewall: # Rules are comprised of a protocol, port, and one or more of host, group, or CIDR # Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND (local cidr) # - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available). - # code: same as port but makes more sense when talking about ICMP, TODO: this is not currently implemented in a way that works, use `any` # proto: `any`, `tcp`, `udp`, or `icmp` + # a port specification is ignored if proto is `icmp` # host: `any` or a literal hostname, ie `test-host` # group: `any` or a literal group name, ie `default-group` # groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass - # cidr: a remote CIDR, `0.0.0.0/0` is any. - # local_cidr: a local CIDR, `0.0.0.0/0` is any. This could be used to filter destinations when using unsafe_routes. - # Default is `any` unless the certificate contains subnets and then the default is the ip issued in the certificate - # if `default_local_cidr_any` is false, otherwise its `any`. + # cidr: a remote CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. `any` means any ip family and address. + # local_cidr: a local CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. `any` means any ip family and address. + # This can be used to filter destinations when using unsafe_routes. + # By default, this is set to only the VPN (overlay) networks assigned via the certificate networks field unless `default_local_cidr_any` is set to true. + # If there are unsafe_routes present in this config file, `local_cidr` should be set appropriately for the intended us case. # ca_name: An issuing CA name # ca_sha: An issuing CA shasum diff --git a/examples/go_service/main.go b/examples/go_service/main.go index f46273ac..3f98fe3d 100644 --- a/examples/go_service/main.go +++ b/examples/go_service/main.go @@ -4,8 +4,13 @@ import ( "bufio" "fmt" "log" + "net" + "os" + "github.com/slackhq/nebula" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/logging" + "github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/service" ) @@ -54,16 +59,24 @@ pki: cert: /home/rice/Developer/nebula-config/app.crt key: /home/rice/Developer/nebula-config/app.key ` - var config config.C - if err := config.LoadString(configStr); err != nil { + var cfg config.C + if err := cfg.LoadString(configStr); err != nil { return err } - service, err := service.New(&config) + + logger := logging.NewLogger(os.Stdout) + + ctrl, err := nebula.Main(&cfg, false, "custom-app", logger, overlay.NewUserDeviceFromConfig) + if err != nil { + return err + } + + svc, err := service.New(ctrl) if err != nil { return err } - ln, err := service.Listen("tcp", ":1234") + ln, err := svc.Listen("tcp", ":1234") if err != nil { return err } @@ -73,16 +86,24 @@ pki: log.Printf("accept error: %s", err) break } - defer conn.Close() + defer func(conn net.Conn) { + _ = conn.Close() + }(conn) log.Printf("got connection") - conn.Write([]byte("hello world\n")) + _, err = conn.Write([]byte("hello world\n")) + if err != nil { + log.Printf("write error: %s", err) + } scanner := bufio.NewScanner(conn) for scanner.Scan() { message := scanner.Text() - fmt.Fprintf(conn, "echo: %q\n", message) + _, err = fmt.Fprintf(conn, "echo: %q\n", message) + if err != nil { + log.Printf("write error: %s", err) + } log.Printf("got message %q", message) } @@ -92,8 +113,8 @@ pki: } } - service.Close() - if err := service.Wait(); err != nil { + _ = svc.Close() + if err := svc.Wait(); err != nil { return err } return nil diff --git a/examples/quickstart-vagrant/README.md b/examples/quickstart-vagrant/README.md deleted file mode 100644 index 108de9e6..00000000 --- a/examples/quickstart-vagrant/README.md +++ /dev/null @@ -1,138 +0,0 @@ -# Quickstart Guide - -This guide is intended to bring up a vagrant environment with 1 lighthouse and 2 generic hosts running nebula. - -## Creating the virtualenv for ansible - -Within the `quickstart/` directory, do the following - -``` -# make a virtual environment -virtualenv venv - -# get into the virtualenv -source venv/bin/activate - -# install ansible -pip install -r requirements.yml -``` - -## Bringing up the vagrant environment - -A plugin that is used for the Vagrant environment is `vagrant-hostmanager` - -To install, run - -``` -vagrant plugin install vagrant-hostmanager -``` - -All hosts within the Vagrantfile are brought up with - -`vagrant up` - -Once the boxes are up, go into the `ansible/` directory and deploy the playbook by running - -`ansible-playbook playbook.yml -i inventory -u vagrant` - -## Testing within the vagrant env - -Once the ansible run is done, hop onto a vagrant box - -`vagrant ssh generic1.vagrant` - -or specifically - -`ssh vagrant@` (password for the vagrant user on the boxes is `vagrant`) - -See `/etc/nebula/config.yml` on a box for firewall rules. - -To see full handshakes and hostmaps, change the logging config of `/etc/nebula/config.yml` on the vagrant boxes from -info to debug. - -You can watch nebula logs by running - -``` -sudo journalctl -fu nebula -``` - -Refer to the nebula src code directory's README for further instructions on configuring nebula. - -## Troubleshooting - -### Is nebula up and running? - -Run and verify that - -``` -ifconfig -``` - -shows you an interface with the name `nebula1` being up. - -``` -vagrant@generic1:~$ ifconfig nebula1 -nebula1: flags=4305 mtu 1300 - inet 10.168.91.210 netmask 255.128.0.0 destination 10.168.91.210 - inet6 fe80::aeaf:b105:e6dc:936c prefixlen 64 scopeid 0x20 - unspec 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 txqueuelen 500 (UNSPEC) - RX packets 2 bytes 168 (168.0 B) - RX errors 0 dropped 0 overruns 0 frame 0 - TX packets 11 bytes 600 (600.0 B) - TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 -``` - -### Connectivity - -Are you able to ping other boxes on the private nebula network? - -The following are the private nebula ip addresses of the vagrant env - -``` -generic1.vagrant [nebula_ip] 10.168.91.210 -generic2.vagrant [nebula_ip] 10.168.91.220 -lighthouse1.vagrant [nebula_ip] 10.168.91.230 -``` - -Try pinging generic1.vagrant to and from any other box using its nebula ip above. - -Double check the nebula firewall rules under /etc/nebula/config.yml to make sure that connectivity is allowed for your use-case if on a specific port. - -``` -vagrant@lighthouse1:~$ grep -A21 firewall /etc/nebula/config.yml -firewall: - conntrack: - tcp_timeout: 12m - udp_timeout: 3m - default_timeout: 10m - - inbound: - - proto: icmp - port: any - host: any - - proto: any - port: 22 - host: any - - proto: any - port: 53 - host: any - - outbound: - - proto: any - port: any - host: any -``` diff --git a/examples/quickstart-vagrant/Vagrantfile b/examples/quickstart-vagrant/Vagrantfile deleted file mode 100644 index ab9408fa..00000000 --- a/examples/quickstart-vagrant/Vagrantfile +++ /dev/null @@ -1,40 +0,0 @@ -Vagrant.require_version ">= 2.2.6" - -nodes = [ - { :hostname => 'generic1.vagrant', :ip => '172.11.91.210', :box => 'bento/ubuntu-18.04', :ram => '512', :cpus => 1}, - { :hostname => 'generic2.vagrant', :ip => '172.11.91.220', :box => 'bento/ubuntu-18.04', :ram => '512', :cpus => 1}, - { :hostname => 'lighthouse1.vagrant', :ip => '172.11.91.230', :box => 'bento/ubuntu-18.04', :ram => '512', :cpus => 1}, -] - -Vagrant.configure("2") do |config| - - config.ssh.insert_key = false - - if Vagrant.has_plugin?('vagrant-cachier') - config.cache.enable :apt - else - printf("** Install vagrant-cachier plugin to speedup deploy: `vagrant plugin install vagrant-cachier`.**\n") - end - - if Vagrant.has_plugin?('vagrant-hostmanager') - config.hostmanager.enabled = true - config.hostmanager.manage_host = true - config.hostmanager.include_offline = true - else - config.vagrant.plugins = "vagrant-hostmanager" - end - - nodes.each do |node| - config.vm.define node[:hostname] do |node_config| - node_config.vm.box = node[:box] - node_config.vm.hostname = node[:hostname] - node_config.vm.network :private_network, ip: node[:ip] - node_config.vm.provider :virtualbox do |vb| - vb.memory = node[:ram] - vb.cpus = node[:cpus] - vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] - vb.customize ['guestproperty', 'set', :id, '/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold', 10000] - end - end - end -end diff --git a/examples/quickstart-vagrant/ansible/ansible.cfg b/examples/quickstart-vagrant/ansible/ansible.cfg deleted file mode 100644 index 518a4f1f..00000000 --- a/examples/quickstart-vagrant/ansible/ansible.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[defaults] -host_key_checking = False -private_key_file = ~/.vagrant.d/insecure_private_key -become = yes diff --git a/examples/quickstart-vagrant/ansible/filter_plugins/to_nebula_ip.py b/examples/quickstart-vagrant/ansible/filter_plugins/to_nebula_ip.py deleted file mode 100644 index a21e82d4..00000000 --- a/examples/quickstart-vagrant/ansible/filter_plugins/to_nebula_ip.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/python - - -class FilterModule(object): - def filters(self): - return { - 'to_nebula_ip': self.to_nebula_ip, - 'map_to_nebula_ips': self.map_to_nebula_ips, - } - - def to_nebula_ip(self, ip_str): - ip_list = list(map(int, ip_str.split("."))) - ip_list[0] = 10 - ip_list[1] = 168 - ip = '.'.join(map(str, ip_list)) - return ip - - def map_to_nebula_ips(self, ip_strs): - ip_list = [ self.to_nebula_ip(ip_str) for ip_str in ip_strs ] - ips = ', '.join(ip_list) - return ips diff --git a/examples/quickstart-vagrant/ansible/inventory b/examples/quickstart-vagrant/ansible/inventory deleted file mode 100644 index 0bae4072..00000000 --- a/examples/quickstart-vagrant/ansible/inventory +++ /dev/null @@ -1,11 +0,0 @@ -[all] -generic1.vagrant -generic2.vagrant -lighthouse1.vagrant - -[generic] -generic1.vagrant -generic2.vagrant - -[lighthouse] -lighthouse1.vagrant diff --git a/examples/quickstart-vagrant/ansible/playbook.yml b/examples/quickstart-vagrant/ansible/playbook.yml deleted file mode 100644 index c3c7d9ff..00000000 --- a/examples/quickstart-vagrant/ansible/playbook.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: test connection to vagrant boxes - hosts: all - tasks: - - debug: msg=ok - -- name: build nebula binaries locally - connection: local - hosts: localhost - tasks: - - command: chdir=../../../ make build/linux-amd64/"{{ item }}" - with_items: - - nebula - - nebula-cert - tags: - - build-nebula - -- name: install nebula on all vagrant hosts - hosts: all - become: yes - gather_facts: yes - roles: - - nebula diff --git a/examples/quickstart-vagrant/ansible/roles/nebula/defaults/main.yml b/examples/quickstart-vagrant/ansible/roles/nebula/defaults/main.yml deleted file mode 100644 index f8e7a991..00000000 --- a/examples/quickstart-vagrant/ansible/roles/nebula/defaults/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -# defaults file for nebula -nebula_config_directory: "/etc/nebula/" diff --git a/examples/quickstart-vagrant/ansible/roles/nebula/files/systemd.nebula.service b/examples/quickstart-vagrant/ansible/roles/nebula/files/systemd.nebula.service deleted file mode 100644 index fd7a0671..00000000 --- a/examples/quickstart-vagrant/ansible/roles/nebula/files/systemd.nebula.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Nebula overlay networking tool -Wants=basic.target network-online.target nss-lookup.target time-sync.target -After=basic.target network.target network-online.target -Before=sshd.service - -[Service] -SyslogIdentifier=nebula -ExecReload=/bin/kill -HUP $MAINPID -ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml -Restart=always - -[Install] -WantedBy=multi-user.target diff --git a/examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.crt b/examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.crt deleted file mode 100644 index 6148687e..00000000 --- a/examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.crt +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN NEBULA CERTIFICATE----- -CkAKDm5lYnVsYSB0ZXN0IENBKNXC1NYFMNXIhO0GOiCmVYeZ9tkB4WEnawmkrca+ -hsAg9otUFhpAowZeJ33KVEABEkAORybHQUUyVFbKYzw0JHfVzAQOHA4kwB1yP9IV -KpiTw9+ADz+wA+R5tn9B+L8+7+Apc+9dem4BQULjA5mRaoYN ------END NEBULA CERTIFICATE----- diff --git a/examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.key b/examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.key deleted file mode 100644 index 394043c6..00000000 --- a/examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.key +++ /dev/null @@ -1,4 +0,0 @@ ------BEGIN NEBULA ED25519 PRIVATE KEY----- -FEXZKMSmg8CgIODR0ymUeNT3nbnVpMi7nD79UgkCRHWmVYeZ9tkB4WEnawmkrca+ -hsAg9otUFhpAowZeJ33KVA== ------END NEBULA ED25519 PRIVATE KEY----- diff --git a/examples/quickstart-vagrant/ansible/roles/nebula/handlers/main.yml b/examples/quickstart-vagrant/ansible/roles/nebula/handlers/main.yml deleted file mode 100644 index 0e09599a..00000000 --- a/examples/quickstart-vagrant/ansible/roles/nebula/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# handlers file for nebula - -- name: restart nebula - service: name=nebula state=restarted diff --git a/examples/quickstart-vagrant/ansible/roles/nebula/tasks/main.yml b/examples/quickstart-vagrant/ansible/roles/nebula/tasks/main.yml deleted file mode 100644 index ffc89d55..00000000 --- a/examples/quickstart-vagrant/ansible/roles/nebula/tasks/main.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- -# tasks file for nebula - -- name: get the vagrant network interface and set fact - set_fact: - vagrant_ifce: "ansible_{{ ansible_interfaces | difference(['lo',ansible_default_ipv4.alias]) | sort | first }}" - tags: - - nebula-conf - -- name: install built nebula binary - copy: src="../../../../../build/linux-amd64/{{ item }}" dest="/usr/local/bin" mode=0755 - with_items: - - nebula - - nebula-cert - -- name: create nebula config directory - file: path="{{ nebula_config_directory }}" state=directory mode=0755 - -- name: temporarily copy over root.crt and root.key to sign - copy: src={{ item }} dest=/opt/{{ item }} - with_items: - - vagrant-test-ca.key - - vagrant-test-ca.crt - -- name: remove previously signed host certificate - file: dest=/etc/nebula/{{ item }} state=absent - with_items: - - host.crt - - host.key - -- name: sign using the root key - command: nebula-cert sign -ca-crt /opt/vagrant-test-ca.crt -ca-key /opt/vagrant-test-ca.key -duration 4320h -groups vagrant -ip {{ hostvars[inventory_hostname][vagrant_ifce]['ipv4']['address'] | to_nebula_ip }}/9 -name {{ ansible_hostname }}.nebula -out-crt /etc/nebula/host.crt -out-key /etc/nebula/host.key - -- name: remove root.key used to sign - file: dest=/opt/{{ item }} state=absent - with_items: - - vagrant-test-ca.key - -- name: write the content of the trusted ca certificate - copy: src="vagrant-test-ca.crt" dest="/etc/nebula/vagrant-test-ca.crt" - notify: restart nebula - -- name: Create config directory - file: path="{{ nebula_config_directory }}" owner=root group=root mode=0755 state=directory - -- name: nebula config - template: src=config.yml.j2 dest="/etc/nebula/config.yml" mode=0644 owner=root group=root - notify: restart nebula - tags: - - nebula-conf - -- name: nebula systemd - copy: src=systemd.nebula.service dest="/etc/systemd/system/nebula.service" mode=0644 owner=root group=root - register: addconf - notify: restart nebula - -- name: maybe reload systemd - shell: systemctl daemon-reload - when: addconf.changed - -- name: nebula running - service: name="nebula" state=started enabled=yes diff --git a/examples/quickstart-vagrant/ansible/roles/nebula/templates/config.yml.j2 b/examples/quickstart-vagrant/ansible/roles/nebula/templates/config.yml.j2 deleted file mode 100644 index a05b1e3c..00000000 --- a/examples/quickstart-vagrant/ansible/roles/nebula/templates/config.yml.j2 +++ /dev/null @@ -1,85 +0,0 @@ -pki: - ca: /etc/nebula/vagrant-test-ca.crt - cert: /etc/nebula/host.crt - key: /etc/nebula/host.key - -# Port Nebula will be listening on -listen: - host: 0.0.0.0 - port: 4242 - -# sshd can expose informational and administrative functions via ssh -sshd: - # Toggles the feature - enabled: true - # Host and port to listen on - listen: 127.0.0.1:2222 - # A file containing the ssh host private key to use - host_key: /etc/ssh/ssh_host_ed25519_key - # A file containing a list of authorized public keys - authorized_users: -{% for user in nebula_users %} - - user: {{ user.name }} - keys: -{% for key in user.ssh_auth_keys %} - - "{{ key }}" -{% endfor %} -{% endfor %} - -local_range: 10.168.0.0/16 - -static_host_map: -# lighthouse - {{ hostvars[groups['lighthouse'][0]][vagrant_ifce]['ipv4']['address'] | to_nebula_ip }}: ["{{ hostvars[groups['lighthouse'][0]][vagrant_ifce]['ipv4']['address']}}:4242"] - -default_route: "0.0.0.0" - -lighthouse: -{% if 'lighthouse' in group_names %} - am_lighthouse: true - serve_dns: true -{% else %} - am_lighthouse: false -{% endif %} - interval: 60 -{% if 'generic' in group_names %} - hosts: - - {{ hostvars[groups['lighthouse'][0]][vagrant_ifce]['ipv4']['address'] | to_nebula_ip }} -{% endif %} - -# Configure the private interface -tun: - dev: nebula1 - # Sets MTU of the tun dev. - # MTU of the tun must be smaller than the MTU of the eth0 interface - mtu: 1300 - -# TODO -# Configure logging level -logging: - level: info - format: json - -firewall: - conntrack: - tcp_timeout: 12m - udp_timeout: 3m - default_timeout: 10m - - inbound: - - proto: icmp - port: any - host: any - - proto: any - port: 22 - host: any -{% if "lighthouse" in groups %} - - proto: any - port: 53 - host: any -{% endif %} - - outbound: - - proto: any - port: any - host: any diff --git a/examples/quickstart-vagrant/ansible/roles/nebula/vars/main.yml b/examples/quickstart-vagrant/ansible/roles/nebula/vars/main.yml deleted file mode 100644 index 7a3ae5d7..00000000 --- a/examples/quickstart-vagrant/ansible/roles/nebula/vars/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# vars file for nebula - -nebula_users: - - name: user1 - ssh_auth_keys: - - "ed25519 place-your-ssh-public-key-here" diff --git a/examples/quickstart-vagrant/requirements.yml b/examples/quickstart-vagrant/requirements.yml deleted file mode 100644 index 90d40550..00000000 --- a/examples/quickstart-vagrant/requirements.yml +++ /dev/null @@ -1 +0,0 @@ -ansible diff --git a/examples/service_scripts/nebula.open-rc b/examples/service_scripts/nebula.open-rc new file mode 100644 index 00000000..2beca66c --- /dev/null +++ b/examples/service_scripts/nebula.open-rc @@ -0,0 +1,35 @@ +#!/sbin/openrc-run +# +# nebula service for open-rc systems + +extra_commands="checkconfig" + +: ${NEBULA_CONFDIR:=${RC_PREFIX%/}/etc/nebula} +: ${NEBULA_CONFIG:=${NEBULA_CONFDIR}/config.yml} +: ${NEBULA_BINARY:=${NEBULA_BINARY}${RC_PREFIX%/}/usr/local/sbin/nebula} + +command="${NEBULA_BINARY}" +command_args="${NEBULA_OPTS} -config ${NEBULA_CONFIG}" + +supervisor="supervise-daemon" + +description="A scalable overlay networking tool with a focus on performance, simplicity and security" + +required_dirs="${NEBULA_CONFDIR}" +required_files="${NEBULA_CONFIG}" + +checkconfig() { + "${command}" -test ${command_args} || return 1 +} + +start_pre() { + if [ "${RC_CMD}" != "restart" ] ; then + checkconfig || return $? + fi +} + +stop_pre() { + if [ "${RC_CMD}" = "restart" ] ; then + checkconfig || return $? + fi +} diff --git a/examples/service_scripts/nebula.service b/examples/service_scripts/nebula.service index ab5218f8..295a6fcd 100644 --- a/examples/service_scripts/nebula.service +++ b/examples/service_scripts/nebula.service @@ -8,6 +8,15 @@ Before=sshd.service Type=notify NotifyAccess=main SyslogIdentifier=nebula + +# Uncomment to run as an unprivileged user with only CAP_NET_ADMIN. Requires a +# nebula user that owns the config directory. Add CAP_NET_BIND_SERVICE to both +# lines if any listener (lighthouse DNS, listen.port, stats, sshd) binds <1024. +#User=nebula +#Group=nebula +#CapabilityBoundingSet=CAP_NET_ADMIN +#AmbientCapabilities=CAP_NET_ADMIN + ExecReload=/bin/kill -HUP $MAINPID ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml Restart=always diff --git a/firewall.go b/firewall.go index b5d79d65..f0fc79c9 100644 --- a/firewall.go +++ b/firewall.go @@ -1,38 +1,34 @@ package nebula import ( + "context" "crypto/sha256" - "encoding/binary" "encoding/hex" "errors" "fmt" "hash/fnv" - "net" + "log/slog" + "net/netip" "reflect" + "slices" "strconv" "strings" "sync" "time" + "github.com/gaissmai/bart" "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cert" - "github.com/slackhq/nebula/cidr" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/firewall" ) -const tcpACK = 0x10 -const tcpFIN = 0x01 - type FirewallInterface interface { - AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip *net.IPNet, localIp *net.IPNet, caName string, caSha string) error + AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error } type conn struct { Expires time.Time // Time when this conntrack entry will expire - Sent time.Time // If tcp rtt tracking is enabled this will be when Seq was last set - Seq uint32 // If tcp rtt tracking is enabled this will be the seq we are looking for an ack // record why the original connection passed the firewall, so we can re-validate // after ruleset changes. Note, rulesVersion is a uint16 so that these two @@ -48,8 +44,8 @@ type Firewall struct { InRules *FirewallTable OutRules *FirewallTable - InSendReject bool - OutSendReject bool + InboundSendReject bool + OutboundSendReject bool //TODO: we should have many more options for TCP, an option for ICMP, and mimic the kernel a bit better // https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt @@ -57,27 +53,29 @@ type Firewall struct { UDPTimeout time.Duration //linux: 180s max DefaultTimeout time.Duration //linux: 600s - // Used to ensure we don't emit local packets for ips we don't own - localIps *cidr.Tree4[struct{}] - assignedCIDR *net.IPNet - hasSubnets bool + // routableNetworks describes the vpn addresses as well as any unsafe networks issued to us in the certificate. + // The vpn addresses are a full bit match while the unsafe networks only match the prefix + routableNetworks *bart.Lite + + // assignedNetworks is a list of vpn networks assigned to us in the certificate. + assignedNetworks []netip.Prefix + // unsafeNetworks is the list of unsafe networks issued to us in the certificate + unsafeNetworks []netip.Prefix rules string rulesVersion uint16 defaultLocalCIDRAny bool - trackTCPRTT bool - metricTCPRTT metrics.Histogram incomingMetrics firewallMetrics outgoingMetrics firewallMetrics - l *logrus.Logger + l *slog.Logger } type firewallMetrics struct { - droppedLocalIP metrics.Counter - droppedRemoteIP metrics.Counter - droppedNoRule metrics.Counter + droppedLocalAddr metrics.Counter + droppedRemoteAddr metrics.Counter + droppedNoRule metrics.Counter } type FirewallConntrack struct { @@ -116,7 +114,7 @@ type FirewallRule struct { Any *firewallLocalCIDR Hosts map[string]*firewallLocalCIDR Groups []*firewallGroups - CIDR *cidr.Tree4[*firewallLocalCIDR] + CIDR *bart.Table[*firewallLocalCIDR] } type firewallGroups struct { @@ -130,106 +128,111 @@ type firewallPort map[int32]*FirewallCA type firewallLocalCIDR struct { Any bool - LocalCIDR *cidr.Tree4[struct{}] + LocalCIDR *bart.Lite } // NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts. -func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c *cert.NebulaCertificate) *Firewall { +// The certificate provided should be the highest version loaded in memory. +func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c cert.Certificate) *Firewall { //TODO: error on 0 duration - var min, max time.Duration + var tmin, tmax time.Duration if tcpTimeout < UDPTimeout { - min = tcpTimeout - max = UDPTimeout + tmin = tcpTimeout + tmax = UDPTimeout } else { - min = UDPTimeout - max = tcpTimeout + tmin = UDPTimeout + tmax = tcpTimeout } - if defaultTimeout < min { - min = defaultTimeout - } else if defaultTimeout > max { - max = defaultTimeout + if defaultTimeout < tmin { + tmin = defaultTimeout + } else if defaultTimeout > tmax { + tmax = defaultTimeout } - localIps := cidr.NewTree4[struct{}]() - var assignedCIDR *net.IPNet - for _, ip := range c.Details.Ips { - ipNet := &net.IPNet{IP: ip.IP, Mask: net.IPMask{255, 255, 255, 255}} - localIps.AddCIDR(ipNet, struct{}{}) - - if assignedCIDR == nil { - // Only grabbing the first one in the cert since any more than that currently has undefined behavior - assignedCIDR = ipNet - } + routableNetworks := new(bart.Lite) + var assignedNetworks []netip.Prefix + for _, network := range c.Networks() { + nprefix := netip.PrefixFrom(network.Addr(), network.Addr().BitLen()) + routableNetworks.Insert(nprefix) + assignedNetworks = append(assignedNetworks, network) } - for _, n := range c.Details.Subnets { - localIps.AddCIDR(n, struct{}{}) + unsafeNetworks := c.UnsafeNetworks() + for _, n := range unsafeNetworks { + routableNetworks.Insert(n) } return &Firewall{ Conntrack: &FirewallConntrack{ Conns: make(map[firewall.Packet]*conn), - TimerWheel: NewTimerWheel[firewall.Packet](min, max), + TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax), }, - InRules: newFirewallTable(), - OutRules: newFirewallTable(), - TCPTimeout: tcpTimeout, - UDPTimeout: UDPTimeout, - DefaultTimeout: defaultTimeout, - localIps: localIps, - assignedCIDR: assignedCIDR, - hasSubnets: len(c.Details.Subnets) > 0, - l: l, - - metricTCPRTT: metrics.GetOrRegisterHistogram("network.tcp.rtt", nil, metrics.NewExpDecaySample(1028, 0.015)), + InRules: newFirewallTable(), + OutRules: newFirewallTable(), + TCPTimeout: tcpTimeout, + UDPTimeout: UDPTimeout, + DefaultTimeout: defaultTimeout, + routableNetworks: routableNetworks, + assignedNetworks: assignedNetworks, + unsafeNetworks: unsafeNetworks, + l: l, + incomingMetrics: firewallMetrics{ - droppedLocalIP: metrics.GetOrRegisterCounter("firewall.incoming.dropped.local_ip", nil), - droppedRemoteIP: metrics.GetOrRegisterCounter("firewall.incoming.dropped.remote_ip", nil), - droppedNoRule: metrics.GetOrRegisterCounter("firewall.incoming.dropped.no_rule", nil), + droppedLocalAddr: metrics.GetOrRegisterCounter("firewall.incoming.dropped.local_addr", nil), + droppedRemoteAddr: metrics.GetOrRegisterCounter("firewall.incoming.dropped.remote_addr", nil), + droppedNoRule: metrics.GetOrRegisterCounter("firewall.incoming.dropped.no_rule", nil), }, outgoingMetrics: firewallMetrics{ - droppedLocalIP: metrics.GetOrRegisterCounter("firewall.outgoing.dropped.local_ip", nil), - droppedRemoteIP: metrics.GetOrRegisterCounter("firewall.outgoing.dropped.remote_ip", nil), - droppedNoRule: metrics.GetOrRegisterCounter("firewall.outgoing.dropped.no_rule", nil), + droppedLocalAddr: metrics.GetOrRegisterCounter("firewall.outgoing.dropped.local_addr", nil), + droppedRemoteAddr: metrics.GetOrRegisterCounter("firewall.outgoing.dropped.remote_addr", nil), + droppedNoRule: metrics.GetOrRegisterCounter("firewall.outgoing.dropped.no_rule", nil), }, } } -func NewFirewallFromConfig(l *logrus.Logger, nc *cert.NebulaCertificate, c *config.C) (*Firewall, error) { +func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewall, error) { + certificate := cs.getCertificate(cert.Version2) + if certificate == nil { + certificate = cs.getCertificate(cert.Version1) + } + + if certificate == nil { + panic("No certificate available to reconfigure the firewall") + } + fw := NewFirewall( l, c.GetDuration("firewall.conntrack.tcp_timeout", time.Minute*12), c.GetDuration("firewall.conntrack.udp_timeout", time.Minute*3), c.GetDuration("firewall.conntrack.default_timeout", time.Minute*10), - nc, + certificate, //TODO: max_connections ) - //TODO: Flip to false after v1.9 release - fw.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", true) + fw.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", false) inboundAction := c.GetString("firewall.inbound_action", "drop") switch inboundAction { case "reject": - fw.InSendReject = true + fw.InboundSendReject = true case "drop": - fw.InSendReject = false + fw.InboundSendReject = false default: - l.WithField("action", inboundAction).Warn("invalid firewall.inbound_action, defaulting to `drop`") - fw.InSendReject = false + l.Warn("invalid firewall.inbound_action, defaulting to `drop`", "action", inboundAction) + fw.InboundSendReject = false } outboundAction := c.GetString("firewall.outbound_action", "drop") switch outboundAction { case "reject": - fw.OutSendReject = true + fw.OutboundSendReject = true case "drop": - fw.OutSendReject = false + fw.OutboundSendReject = false default: - l.WithField("action", inboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`") - fw.OutSendReject = false + l.Warn("invalid firewall.outbound_action, defaulting to `drop`", "action", outboundAction) + fw.OutboundSendReject = false } err := AddFirewallRulesFromConfig(l, false, c, fw) @@ -246,32 +249,7 @@ func NewFirewallFromConfig(l *logrus.Logger, nc *cert.NebulaCertificate, c *conf } // AddRule properly creates the in memory rule structure for a firewall table. -func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip *net.IPNet, localIp *net.IPNet, caName string, caSha string) error { - // Under gomobile, stringing a nil pointer with fmt causes an abort in debug mode for iOS - // https://github.com/golang/go/issues/14131 - sIp := "" - if ip != nil { - sIp = ip.String() - } - lIp := "" - if localIp != nil { - lIp = localIp.String() - } - - // We need this rule string because we generate a hash. Removing this will break firewall reload. - ruleString := fmt.Sprintf( - "incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s", - incoming, proto, startPort, endPort, groups, host, sIp, lIp, caName, caSha, - ) - f.rules += ruleString + "\n" - - direction := "incoming" - if !incoming { - direction = "outgoing" - } - f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "ip": sIp, "localIp": lIp, "caName": caName, "caSha": caSha}). - Info("Firewall rule added") - +func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error { var ( ft *FirewallTable fp firewallPort @@ -288,7 +266,13 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort fp = ft.TCP case firewall.ProtoUDP: fp = ft.UDP - case firewall.ProtoICMP: + case firewall.ProtoICMP, firewall.ProtoICMPv6: + //ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided + if startPort != firewall.PortAny { + f.l.Warn("ignoring port specification for ICMP firewall rule", "startPort", startPort) + } + startPort = firewall.PortAny + endPort = firewall.PortAny fp = ft.ICMP case firewall.ProtoAny: fp = ft.AnyProto @@ -296,7 +280,22 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort return fmt.Errorf("unknown protocol %v", proto) } - return fp.addRule(f, startPort, endPort, groups, host, ip, localIp, caName, caSha) + // We need this rule string because we generate a hash. Removing this will break firewall reload. + ruleString := fmt.Sprintf( + "incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s", + incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha, + ) + f.rules += ruleString + "\n" + + direction := "incoming" + if !incoming { + direction = "outgoing" + } + f.l.Info("Firewall rule added", + "firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha}, + ) + + return fp.addRule(f, startPort, endPort, groups, host, cidr, localCidr, caName, caSha) } // GetRuleHash returns a hash representation of all inbound and outbound rules @@ -317,7 +316,7 @@ func (f *Firewall) GetRuleHashes() string { return "SHA:" + f.GetRuleHash() + ",FNV:" + strconv.FormatUint(uint64(f.GetRuleHashFNV()), 10) } -func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw FirewallInterface) error { +func AddFirewallRulesFromConfig(l *slog.Logger, inbound bool, c *config.C, fw FirewallInterface) error { var table string if inbound { table = "firewall.inbound" @@ -330,13 +329,12 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw return nil } - rs, ok := r.([]interface{}) + rs, ok := r.([]any) if !ok { return fmt.Errorf("%s failed to parse, should be an array of rules", table) } for i, t := range rs { - var groups []string r, err := convertRule(l, t, table, i) if err != nil { return fmt.Errorf("%s rule #%v; %s", table, i, err) @@ -346,23 +344,10 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i) } - if r.Host == "" && len(r.Groups) == 0 && r.Group == "" && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" { + if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" { return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i) } - if len(r.Groups) > 0 { - groups = r.Groups - } - - if r.Group != "" { - // Check if we have both groups and group provided in the rule config - if len(groups) > 0 { - return fmt.Errorf("%s rule #%v; only one of group or groups should be defined, both provided", table, i) - } - - groups = []string{r.Group} - } - var sPort, errPort string if r.Code != "" { errPort = "code" @@ -372,42 +357,55 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw sPort = r.Port } - startPort, endPort, err := parsePort(sPort) - if err != nil { - return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err) - } - var proto uint8 + var startPort, endPort int32 switch r.Proto { case "any": proto = firewall.ProtoAny + startPort, endPort, err = parsePort(sPort) case "tcp": proto = firewall.ProtoTCP + startPort, endPort, err = parsePort(sPort) case "udp": proto = firewall.ProtoUDP + startPort, endPort, err = parsePort(sPort) case "icmp": proto = firewall.ProtoICMP + startPort = firewall.PortAny + endPort = firewall.PortAny + if sPort != "" { + l.Warn("ignoring port specification for ICMP firewall rule", "port", sPort) + } default: return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto) } + if err != nil { + return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err) + } - var cidr *net.IPNet - if r.Cidr != "" { - _, cidr, err = net.ParseCIDR(r.Cidr) + if r.Cidr != "" && r.Cidr != "any" { + _, err = netip.ParsePrefix(r.Cidr) if err != nil { return fmt.Errorf("%s rule #%v; cidr did not parse; %s", table, i, err) } } - var localCidr *net.IPNet - if r.LocalCidr != "" { - _, localCidr, err = net.ParseCIDR(r.LocalCidr) + if r.LocalCidr != "" && r.LocalCidr != "any" { + _, err = netip.ParsePrefix(r.LocalCidr) if err != nil { return fmt.Errorf("%s rule #%v; local_cidr did not parse; %s", table, i, err) } } - err = fw.AddRule(inbound, proto, startPort, endPort, groups, r.Host, cidr, localCidr, r.CAName, r.CASha) + if warning := r.sanity(); warning != nil { + l.Warn("firewall rule sanity check", + "table", table, + "rule", i, + "warning", warning, + ) + } + + err = fw.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha) if err != nil { return fmt.Errorf("%s rule #%v; `%s`", table, i, err) } @@ -416,40 +414,53 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw return nil } -var ErrInvalidRemoteIP = errors.New("remote IP is not in remote certificate subnets") -var ErrInvalidLocalIP = errors.New("local IP is not in list of handled local IPs") +var ErrUnknownNetworkType = errors.New("unknown network type") +var ErrPeerRejected = errors.New("remote address is not within a network that we handle") +var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate networks") +var ErrInvalidLocalIP = errors.New("local address is not in list of handled local addresses") var ErrNoMatchingRule = errors.New("no matching rule in firewall table") // Drop returns an error if the packet should be dropped, explaining why. It // returns nil if the packet should not be dropped. -func (f *Firewall) Drop(packet []byte, fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) error { - // Check if we spoke to this tuple, if we did then allow this packet - if f.inConns(packet, fp, incoming, h, caPool, localCache) { - return nil - } - - // Make sure remote address matches nebula certificate - if remoteCidr := h.remoteCidr; remoteCidr != nil { - ok, _ := remoteCidr.Contains(fp.RemoteIP) - if !ok { - f.metrics(incoming).droppedRemoteIP.Inc(1) +func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error { + // Make sure remote address matches nebula certificate, and determine how to treat it + if h.networks == nil { + // Simple case: Certificate has one address and no unsafe networks + if h.vpnAddrs[0] != fp.RemoteAddr { + f.metrics(incoming).droppedRemoteAddr.Inc(1) return ErrInvalidRemoteIP } } else { - // Simple case: Certificate has one IP and no subnets - if fp.RemoteIP != h.vpnIp { - f.metrics(incoming).droppedRemoteIP.Inc(1) + nwType, ok := h.networks.Lookup(fp.RemoteAddr) + if !ok { + f.metrics(incoming).droppedRemoteAddr.Inc(1) return ErrInvalidRemoteIP } + switch nwType { + case NetworkTypeVPN: + break // nothing special + case NetworkTypeVPNPeer: + f.metrics(incoming).droppedRemoteAddr.Inc(1) + return ErrPeerRejected // reject for now, one day this may have different FW rules + case NetworkTypeUnsafe: + break // nothing special, one day this may have different FW rules + default: + f.metrics(incoming).droppedRemoteAddr.Inc(1) + return ErrUnknownNetworkType //should never happen + } } // Make sure we are supposed to be handling this local ip address - ok, _ := f.localIps.Contains(fp.LocalIP) - if !ok { - f.metrics(incoming).droppedLocalIP.Inc(1) + if !f.routableNetworks.Contains(fp.LocalAddr) { + f.metrics(incoming).droppedLocalAddr.Inc(1) return ErrInvalidLocalIP } + // Check if we spoke to this tuple, if we did then allow this packet + if f.inConns(fp, h, caPool, localCache) { + return nil + } + table := f.OutRules if incoming { table = f.InRules @@ -462,7 +473,7 @@ func (f *Firewall) Drop(packet []byte, fp firewall.Packet, incoming bool, h *Hos } // We always want to conntrack since it is a faster operation - f.addConn(packet, fp, incoming) + f.addConn(fp, incoming) return nil } @@ -475,7 +486,7 @@ func (f *Firewall) metrics(incoming bool) firewallMetrics { } } -// Destroy cleans up any known cyclical references so the object can be free'd my GC. This should be called if a new +// Destroy cleans up any known cyclical references so the object can be freed by GC. This should be called if a new // firewall object is created func (f *Firewall) Destroy() { //TODO: clean references if/when needed @@ -491,7 +502,7 @@ func (f *Firewall) EmitStats() { metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV())) } -func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) bool { +func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool { if localCache != nil { if _, ok := localCache[fp]; ok { return true @@ -523,26 +534,26 @@ func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h * // We now know which firewall table to check against if !table.match(fp, c.incoming, h.ConnectionState.peerCert, caPool) { - if f.l.Level >= logrus.DebugLevel { - h.logger(f.l). - WithField("fwPacket", fp). - WithField("incoming", c.incoming). - WithField("rulesVersion", f.rulesVersion). - WithField("oldRulesVersion", c.rulesVersion). - Debugln("dropping old conntrack entry, does not match new ruleset") + if f.l.Enabled(context.Background(), slog.LevelDebug) { + h.logger(f.l).Debug("dropping old conntrack entry, does not match new ruleset", + "fwPacket", fp, + "incoming", c.incoming, + "rulesVersion", f.rulesVersion, + "oldRulesVersion", c.rulesVersion, + ) } delete(conntrack.Conns, fp) conntrack.Unlock() return false } - if f.l.Level >= logrus.DebugLevel { - h.logger(f.l). - WithField("fwPacket", fp). - WithField("incoming", c.incoming). - WithField("rulesVersion", f.rulesVersion). - WithField("oldRulesVersion", c.rulesVersion). - Debugln("keeping old conntrack entry, does match new ruleset") + if f.l.Enabled(context.Background(), slog.LevelDebug) { + h.logger(f.l).Debug("keeping old conntrack entry, does match new ruleset", + "fwPacket", fp, + "incoming", c.incoming, + "rulesVersion", f.rulesVersion, + "oldRulesVersion", c.rulesVersion, + ) } c.rulesVersion = f.rulesVersion @@ -551,11 +562,6 @@ func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h * switch fp.Protocol { case firewall.ProtoTCP: c.Expires = time.Now().Add(f.TCPTimeout) - if incoming { - f.checkTCPRTT(c, packet) - } else { - setTCPRTTTracking(c, packet) - } case firewall.ProtoUDP: c.Expires = time.Now().Add(f.UDPTimeout) default: @@ -571,16 +577,13 @@ func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h * return true } -func (f *Firewall) addConn(packet []byte, fp firewall.Packet, incoming bool) { +func (f *Firewall) addConn(fp firewall.Packet, incoming bool) { var timeout time.Duration c := &conn{} switch fp.Protocol { case firewall.ProtoTCP: timeout = f.TCPTimeout - if !incoming { - setTCPRTTTracking(c, packet) - } case firewall.ProtoUDP: timeout = f.UDPTimeout default: @@ -606,7 +609,6 @@ func (f *Firewall) addConn(packet []byte, fp firewall.Packet, incoming bool) { // Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel // Caller must own the connMutex lock! func (f *Firewall) evict(p firewall.Packet) { - //TODO: report a stat if the tcp rtt tracking was never resolved? // Are we still tracking this conn? conntrack := f.Conntrack t, ok := conntrack.Conns[p] @@ -627,7 +629,7 @@ func (f *Firewall) evict(p firewall.Packet) { delete(conntrack.Conns, p) } -func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool { +func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool { if ft.AnyProto.match(p, incoming, c, caPool) { return true } @@ -641,7 +643,7 @@ func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.NebulaC if ft.UDP.match(p, incoming, c, caPool) { return true } - case firewall.ProtoICMP: + case firewall.ProtoICMP, firewall.ProtoICMPv6: if ft.ICMP.match(p, incoming, c, caPool) { return true } @@ -650,7 +652,7 @@ func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.NebulaC return false } -func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, ip *net.IPNet, localIp *net.IPNet, caName string, caSha string) error { +func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error { if startPort > endPort { return fmt.Errorf("start port was lower than end port") } @@ -663,7 +665,7 @@ func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, grou } } - if err := fp[i].addRule(f, groups, host, ip, localIp, caName, caSha); err != nil { + if err := fp[i].addRule(f, groups, host, cidr, localCidr, caName, caSha); err != nil { return err } } @@ -671,12 +673,19 @@ func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, grou return nil } -func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool { +func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool { // We don't have any allowed ports, bail if fp == nil { return false } + // this branch is here to catch traffic from FirewallTable.Any.match and FirewallTable.ICMP.match + if p.Protocol == firewall.ProtoICMP || p.Protocol == firewall.ProtoICMPv6 { + // port numbers are re-used for connection tracking of ICMP, + // but we don't want to actually filter on them. + return fp[firewall.PortAny].match(p, c, caPool) + } + var port int32 if p.Fragment { @@ -694,12 +703,12 @@ func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.NebulaCer return fp[firewall.PortAny].match(p, c, caPool) } -func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, ip, localIp *net.IPNet, caName, caSha string) error { +func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, localCidr, caName, caSha string) error { fr := func() *FirewallRule { return &FirewallRule{ Hosts: make(map[string]*firewallLocalCIDR), Groups: make([]*firewallGroups, 0), - CIDR: cidr.NewTree4[*firewallLocalCIDR](), + CIDR: new(bart.Table[*firewallLocalCIDR]), } } @@ -708,14 +717,14 @@ func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, ip, loc fc.Any = fr() } - return fc.Any.addRule(f, groups, host, ip, localIp) + return fc.Any.addRule(f, groups, host, cidr, localCidr) } if caSha != "" { if _, ok := fc.CAShas[caSha]; !ok { fc.CAShas[caSha] = fr() } - err := fc.CAShas[caSha].addRule(f, groups, host, ip, localIp) + err := fc.CAShas[caSha].addRule(f, groups, host, cidr, localCidr) if err != nil { return err } @@ -725,7 +734,7 @@ func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, ip, loc if _, ok := fc.CANames[caName]; !ok { fc.CANames[caName] = fr() } - err := fc.CANames[caName].addRule(f, groups, host, ip, localIp) + err := fc.CANames[caName].addRule(f, groups, host, cidr, localCidr) if err != nil { return err } @@ -734,7 +743,7 @@ func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, ip, loc return nil } -func (fc *FirewallCA) match(p firewall.Packet, c *cert.NebulaCertificate, caPool *cert.NebulaCAPool) bool { +func (fc *FirewallCA) match(p firewall.Packet, c *cert.CachedCertificate, caPool *cert.CAPool) bool { if fc == nil { return false } @@ -743,38 +752,38 @@ func (fc *FirewallCA) match(p firewall.Packet, c *cert.NebulaCertificate, caPool return true } - if t, ok := fc.CAShas[c.Details.Issuer]; ok { + if t, ok := fc.CAShas[c.Certificate.Issuer()]; ok { if t.match(p, c) { return true } } - s, err := caPool.GetCAForCert(c) + s, err := caPool.GetCAForCert(c.Certificate) if err != nil { return false } - return fc.CANames[s.Details.Name].match(p, c) + return fc.CANames[s.Certificate.Name()].match(p, c) } -func (fr *FirewallRule) addRule(f *Firewall, groups []string, host string, ip *net.IPNet, localCIDR *net.IPNet) error { +func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localCidr string) error { flc := func() *firewallLocalCIDR { return &firewallLocalCIDR{ - LocalCIDR: cidr.NewTree4[struct{}](), + LocalCIDR: new(bart.Lite), } } - if fr.isAny(groups, host, ip) { + if fr.isAny(groups, host, cidr) { if fr.Any == nil { fr.Any = flc() } - return fr.Any.addRule(f, localCIDR) + return fr.Any.addRule(f, localCidr) } if len(groups) > 0 { nlc := flc() - err := nlc.addRule(f, localCIDR) + err := nlc.addRule(f, localCidr) if err != nil { return err } @@ -790,51 +799,53 @@ func (fr *FirewallRule) addRule(f *Firewall, groups []string, host string, ip *n if nlc == nil { nlc = flc() } - err := nlc.addRule(f, localCIDR) + err := nlc.addRule(f, localCidr) if err != nil { return err } fr.Hosts[host] = nlc } - if ip != nil { - _, nlc := fr.CIDR.GetCIDR(ip) + if cidr != "" { + c, err := netip.ParsePrefix(cidr) + if err != nil { + return err + } + nlc, _ := fr.CIDR.Get(c) if nlc == nil { nlc = flc() } - err := nlc.addRule(f, localCIDR) + err = nlc.addRule(f, localCidr) if err != nil { return err } - fr.CIDR.AddCIDR(ip, nlc) + fr.CIDR.Insert(c, nlc) } return nil } -func (fr *FirewallRule) isAny(groups []string, host string, ip *net.IPNet) bool { - if len(groups) == 0 && host == "" && ip == nil { +func (fr *FirewallRule) isAny(groups []string, host string, cidr string) bool { + if len(groups) == 0 && host == "" && cidr == "" { return true } - for _, group := range groups { - if group == "any" { - return true - } + if slices.Contains(groups, "any") { + return true } if host == "any" { return true } - if ip != nil && ip.Contains(net.IPv4(0, 0, 0, 0)) { + if cidr == "any" { return true } return false } -func (fr *FirewallRule) match(p firewall.Packet, c *cert.NebulaCertificate) bool { +func (fr *FirewallRule) match(p firewall.Packet, c *cert.CachedCertificate) bool { if fr == nil { return false } @@ -849,7 +860,7 @@ func (fr *FirewallRule) match(p firewall.Packet, c *cert.NebulaCertificate) bool found := false for _, g := range sg.Groups { - if _, ok := c.Details.InvertedGroups[g]; !ok { + if _, ok := c.InvertedGroups[g]; !ok { found = false break } @@ -863,35 +874,50 @@ func (fr *FirewallRule) match(p firewall.Packet, c *cert.NebulaCertificate) bool } if fr.Hosts != nil { - if flc, ok := fr.Hosts[c.Details.Name]; ok { + if flc, ok := fr.Hosts[c.Certificate.Name()]; ok { if flc.match(p, c) { return true } } } - return fr.CIDR.EachContains(p.RemoteIP, func(flc *firewallLocalCIDR) bool { - return flc.match(p, c) - }) + for _, v := range fr.CIDR.Supernets(netip.PrefixFrom(p.RemoteAddr, p.RemoteAddr.BitLen())) { + if v.match(p, c) { + return true + } + } + + return false } -func (flc *firewallLocalCIDR) addRule(f *Firewall, localIp *net.IPNet) error { - if localIp == nil { - if !f.hasSubnets || f.defaultLocalCIDRAny { +func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error { + if localCidr == "any" { + flc.Any = true + return nil + } + + if localCidr == "" { + if len(f.unsafeNetworks) == 0 || f.defaultLocalCIDRAny { flc.Any = true return nil } - localIp = f.assignedCIDR - } else if localIp.Contains(net.IPv4(0, 0, 0, 0)) { - flc.Any = true + for _, network := range f.assignedNetworks { + flc.LocalCIDR.Insert(network) + } + return nil + } - flc.LocalCIDR.AddCIDR(localIp, struct{}{}) + c, err := netip.ParsePrefix(localCidr) + if err != nil { + return err + } + flc.LocalCIDR.Insert(c) return nil } -func (flc *firewallLocalCIDR) match(p firewall.Packet, c *cert.NebulaCertificate) bool { +func (flc *firewallLocalCIDR) match(p firewall.Packet, c *cert.CachedCertificate) bool { if flc == nil { return false } @@ -900,8 +926,7 @@ func (flc *firewallLocalCIDR) match(p firewall.Packet, c *cert.NebulaCertificate return true } - ok, _ := flc.LocalCIDR.Contains(p.LocalIP) - return ok + return flc.LocalCIDR.Contains(p.LocalAddr) } type rule struct { @@ -909,7 +934,6 @@ type rule struct { Code string Proto string Host string - Group string Groups []string Cidr string LocalCidr string @@ -917,15 +941,15 @@ type rule struct { CASha string } -func convertRule(l *logrus.Logger, p interface{}, table string, i int) (rule, error) { +func convertRule(l *slog.Logger, p any, table string, i int) (rule, error) { r := rule{} - m, ok := p.(map[interface{}]interface{}) + m, ok := p.(map[string]any) if !ok { return r, errors.New("could not parse rule") } - toString := func(k string, m map[interface{}]interface{}) string { + toString := func(k string, m map[string]any) string { v, ok := m[k] if !ok { return "" @@ -943,15 +967,19 @@ func convertRule(l *logrus.Logger, p interface{}, table string, i int) (rule, er r.CASha = toString("ca_sha", m) // Make sure group isn't an array - if v, ok := m["group"].([]interface{}); ok { + if v, ok := m["group"].([]any); ok { if len(v) > 1 { return r, errors.New("group should contain a single value, an array with more than one entry was provided") } - l.Warnf("%s rule #%v; group was an array with a single value, converting to simple value", table, i) + l.Warn("group was an array with a single value, converting to simple value", + "table", table, + "rule", i, + ) m["group"] = v[0] } - r.Group = toString("group", m) + + singleGroup := toString("group", m) if rg, ok := m["groups"]; ok { switch reflect.TypeOf(rg).Kind() { @@ -968,91 +996,122 @@ func convertRule(l *logrus.Logger, p interface{}, table string, i int) (rule, er } } + //flatten group vs groups + if singleGroup != "" { + // Check if we have both groups and group provided in the rule config + if len(r.Groups) > 0 { + return r, fmt.Errorf("only one of group or groups should be defined, both provided") + } + r.Groups = []string{singleGroup} + } + return r, nil } -func parsePort(s string) (startPort, endPort int32, err error) { - if s == "any" { - startPort = firewall.PortAny - endPort = firewall.PortAny +// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value +// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr" +func (r *rule) sanity() error { + //port, proto, local_cidr are AND, no need to check here + //ca_sha and ca_name don't have a wildcard value, no need to check here + groupsEmpty := len(r.Groups) == 0 + hostEmpty := r.Host == "" + cidrEmpty := r.Cidr == "" - } else if s == "fragment" { - startPort = firewall.PortFragment - endPort = firewall.PortFragment + if (groupsEmpty && hostEmpty && cidrEmpty) == true { + return nil //no content! + } - } else if strings.Contains(s, `-`) { - sPorts := strings.SplitN(s, `-`, 2) - sPorts[0] = strings.Trim(sPorts[0], " ") - sPorts[1] = strings.Trim(sPorts[1], " ") + groupsHasAny := slices.Contains(r.Groups, "any") + if groupsHasAny && len(r.Groups) > 1 { + return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups) + } - if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" { - return 0, 0, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s) + if r.Host == "any" { + if !groupsEmpty { + return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups) } - rStartPort, err := strconv.Atoi(sPorts[0]) - if err != nil { - return 0, 0, fmt.Errorf("beginning range was not a number; `%s`", sPorts[0]) + if !cidrEmpty { + return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr) } + } - rEndPort, err := strconv.Atoi(sPorts[1]) - if err != nil { - return 0, 0, fmt.Errorf("ending range was not a number; `%s`", sPorts[1]) + if groupsHasAny { + if !hostEmpty && r.Host != "any" { + return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host) } - - startPort = int32(rStartPort) - endPort = int32(rEndPort) - - if startPort == firewall.PortAny { - endPort = firewall.PortAny + if !cidrEmpty { + return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr) } + } - } else { - rPort, err := strconv.Atoi(s) - if err != nil { - return 0, 0, fmt.Errorf("was not a number; `%s`", s) - } - startPort = int32(rPort) - endPort = startPort + if r.Code != "" { + return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code) } - return + //todo alert on cidr-any + + return nil } -// TODO: write tests for these -func setTCPRTTTracking(c *conn, p []byte) { - if c.Seq != 0 { - return +func parsePort(s string) (int32, int32, error) { + const notAPort int32 = -2 + if s == "any" { + return firewall.PortAny, firewall.PortAny, nil } - - ihl := int(p[0]&0x0f) << 2 - - // Don't track FIN packets - if p[ihl+13]&tcpFIN != 0 { - return + if s == "fragment" { + return firewall.PortFragment, firewall.PortFragment, nil + } + if !strings.Contains(s, `-`) { + rPort, err := parsePortValue("", s) + if err != nil { + return notAPort, notAPort, err + } + return rPort, rPort, nil } - c.Seq = binary.BigEndian.Uint32(p[ihl+4 : ihl+8]) - c.Sent = time.Now() -} + sPorts := strings.SplitN(s, `-`, 2) + for i := range sPorts { + sPorts[i] = strings.Trim(sPorts[i], " ") + } + if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" { + return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s) + } -func (f *Firewall) checkTCPRTT(c *conn, p []byte) bool { - if c.Seq == 0 { - return false + startPort, err := parsePortValue("beginning range ", sPorts[0]) + if err != nil { + return notAPort, notAPort, err } - ihl := int(p[0]&0x0f) << 2 - if p[ihl+13]&tcpACK == 0 { - return false + endPort, err := parsePortValue("ending range ", sPorts[1]) + if err != nil { + return notAPort, notAPort, err } - // Deal with wrap around, signed int cuts the ack window in half - // 0 is a bad ack, no data acknowledged - // positive number is a bad ack, ack is over half the window away - if int32(c.Seq-binary.BigEndian.Uint32(p[ihl+8:ihl+12])) >= 0 { - return false + if startPort == firewall.PortAny { + endPort = firewall.PortAny } - f.metricTCPRTT.Update(time.Since(c.Sent).Nanoseconds()) - c.Seq = 0 - return true + return startPort, endPort, nil +} + +// parsePortValue accepts a base-10 decimal in [0, 65535] and returns it +// widened to int32. Using strconv.ParseUint with bitSize 16 rejects +// negative input, out-of-range input (>65535), and any non-decimal byte +// by construction, so the int32 widening that follows is provably safe +// and cannot collide with firewall.PortAny (0) or firewall.PortFragment +// (-1) via integer truncation. +// +// prefix is prepended to both error messages so callers can disambiguate +// the single-port path (prefix="") from the range bounds (prefix="beginning +// range " / "ending range "), preserving the historical error strings. +func parsePortValue(prefix, s string) (int32, error) { + n, err := strconv.ParseUint(s, 10, 16) + if err == nil { + return int32(n), nil + } + if errors.Is(err, strconv.ErrRange) { + return 0, fmt.Errorf("%sout of range [0,65535]; `%s`", prefix, s) + } + return 0, fmt.Errorf("%swas not a number; `%s`", prefix, s) } diff --git a/firewall/cache.go b/firewall/cache.go index 71b83f43..ba4b9732 100644 --- a/firewall/cache.go +++ b/firewall/cache.go @@ -1,10 +1,10 @@ package firewall import ( + "context" + "log/slog" "sync/atomic" "time" - - "github.com/sirupsen/logrus" ) // ConntrackCache is used as a local routine cache to know if a given flow @@ -15,41 +15,49 @@ type ConntrackCacheTicker struct { cacheV uint64 cacheTick atomic.Uint64 + l *slog.Logger cache ConntrackCache } -func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker { +func NewConntrackCacheTicker(ctx context.Context, l *slog.Logger, d time.Duration) *ConntrackCacheTicker { if d == 0 { return nil } c := &ConntrackCacheTicker{ + l: l, cache: ConntrackCache{}, } - go c.tick(d) + go c.tick(ctx, d) return c } -func (c *ConntrackCacheTicker) tick(d time.Duration) { +func (c *ConntrackCacheTicker) tick(ctx context.Context, d time.Duration) { + t := time.NewTicker(d) + defer t.Stop() for { - time.Sleep(d) - c.cacheTick.Add(1) + select { + case <-ctx.Done(): + return + case <-t.C: + c.cacheTick.Add(1) + } } } // Get checks if the cache ticker has moved to the next version before returning // the map. If it has moved, we reset the map. -func (c *ConntrackCacheTicker) Get(l *logrus.Logger) ConntrackCache { +func (c *ConntrackCacheTicker) Get() ConntrackCache { if c == nil { return nil } if tick := c.cacheTick.Load(); tick != c.cacheV { c.cacheV = tick if ll := len(c.cache); ll > 0 { - if l.Level == logrus.DebugLevel { - l.WithField("len", ll).Debug("resetting conntrack cache") + if c.l.Enabled(context.Background(), slog.LevelDebug) { + c.l.Debug("resetting conntrack cache", "len", ll) } c.cache = make(ConntrackCache, ll) } diff --git a/firewall/cache_test.go b/firewall/cache_test.go new file mode 100644 index 00000000..ab807984 --- /dev/null +++ b/firewall/cache_test.go @@ -0,0 +1,69 @@ +package firewall + +import ( + "bytes" + "log/slog" + "strings" + "testing" + + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" +) + +// The tests below pin the log format produced by ConntrackCacheTicker.Get +// so changes cannot silently break what operators are grepping for. The +// ticker's internal state (cache + cacheTick) is poked directly to avoid +// racing a goroutine-driven tick in tests. + +func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheTicker { + t.Helper() + c := &ConntrackCacheTicker{ + l: l, + cache: make(ConntrackCache, cacheLen), + } + for i := 0; i < cacheLen; i++ { + c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{} + } + c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path + return c +} + +func TestConntrackCacheTicker_Get_TextFormat(t *testing.T) { + buf := &bytes.Buffer{} + l := test.NewLoggerWithOutputAndLevel(buf, slog.LevelDebug) + + c := newFixedTicker(t, l, 3) + c.Get() + + assert.Equal(t, "level=DEBUG msg=\"resetting conntrack cache\" len=3\n", buf.String()) +} + +func TestConntrackCacheTicker_Get_JSONFormat(t *testing.T) { + buf := &bytes.Buffer{} + l := test.NewJSONLoggerWithOutput(buf, slog.LevelDebug) + + c := newFixedTicker(t, l, 2) + c.Get() + + assert.JSONEq(t, `{"level":"DEBUG","msg":"resetting conntrack cache","len":2}`, strings.TrimSpace(buf.String())) +} + +func TestConntrackCacheTicker_Get_QuietBelowDebug(t *testing.T) { + buf := &bytes.Buffer{} + l := test.NewLoggerWithOutputAndLevel(buf, slog.LevelInfo) + + c := newFixedTicker(t, l, 5) + c.Get() + + assert.Empty(t, buf.String()) +} + +func TestConntrackCacheTicker_Get_QuietWhenCacheEmpty(t *testing.T) { + buf := &bytes.Buffer{} + l := test.NewLoggerWithOutputAndLevel(buf, slog.LevelDebug) + + c := newFixedTicker(t, l, 0) + c.Get() + + assert.Empty(t, buf.String()) +} diff --git a/firewall/packet.go b/firewall/packet.go index 1c4affda..2cbfb5ea 100644 --- a/firewall/packet.go +++ b/firewall/packet.go @@ -3,26 +3,29 @@ package firewall import ( "encoding/json" "fmt" - - "github.com/slackhq/nebula/iputil" + "net/netip" ) -type m map[string]interface{} +type m = map[string]any const ( - ProtoAny = 0 // When we want to handle HOPOPT (0) we can change this, if ever - ProtoTCP = 6 - ProtoUDP = 17 - ProtoICMP = 1 + ProtoAny = 0 // When we want to handle HOPOPT (0) we can change this, if ever + ProtoTCP = 6 + ProtoUDP = 17 + ProtoICMP = 1 + ProtoICMPv6 = 58 PortAny = 0 // Special value for matching `port: any` PortFragment = -1 // Special value for matching `port: fragment` ) type Packet struct { - LocalIP iputil.VpnIp - RemoteIP iputil.VpnIp - LocalPort uint16 + LocalAddr netip.Addr + RemoteAddr netip.Addr + // LocalPort is the destination port for incoming traffic, or the source port for outgoing. Zero for ICMP. + LocalPort uint16 + // RemotePort is the source port for incoming traffic, or the destination port for outgoing. + // For ICMP, it's the "identifier". This is only used for connection tracking, actual firewall rules will not filter on ICMP identifier RemotePort uint16 Protocol uint8 Fragment bool @@ -30,8 +33,8 @@ type Packet struct { func (fp *Packet) Copy() *Packet { return &Packet{ - LocalIP: fp.LocalIP, - RemoteIP: fp.RemoteIP, + LocalAddr: fp.LocalAddr, + RemoteAddr: fp.RemoteAddr, LocalPort: fp.LocalPort, RemotePort: fp.RemotePort, Protocol: fp.Protocol, @@ -46,14 +49,16 @@ func (fp Packet) MarshalJSON() ([]byte, error) { proto = "tcp" case ProtoICMP: proto = "icmp" + case ProtoICMPv6: + proto = "icmpv6" case ProtoUDP: proto = "udp" default: proto = fmt.Sprintf("unknown %v", fp.Protocol) } return json.Marshal(m{ - "LocalIP": fp.LocalIP.String(), - "RemoteIP": fp.RemoteIP.String(), + "LocalAddr": fp.LocalAddr.String(), + "RemoteAddr": fp.RemoteAddr.String(), "LocalPort": fp.LocalPort, "RemotePort": fp.RemotePort, "Protocol": proto, diff --git a/firewall_test.go b/firewall_test.go index 7d65cb50..499f3cc7 100644 --- a/firewall_test.go +++ b/firewall_test.go @@ -2,25 +2,25 @@ package nebula import ( "bytes" - "encoding/binary" "errors" + "log/slog" "math" - "net" + "net/netip" "testing" "time" - "github.com/rcrowley/go-metrics" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/firewall" - "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNewFirewall(t *testing.T) { l := test.NewLogger() - c := &cert.NebulaCertificate{} + c := &dummyCert{} fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c) conntrack := fw.Conntrack assert.NotNil(t, conntrack) @@ -58,149 +58,275 @@ func TestNewFirewall(t *testing.T) { } func TestFirewall_AddRule(t *testing.T) { - l := test.NewLogger() ob := &bytes.Buffer{} - l.SetOutput(ob) + l := test.NewLoggerWithOutput(ob) - c := &cert.NebulaCertificate{} + c := &dummyCert{} fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c) assert.NotNil(t, fw.InRules) assert.NotNil(t, fw.OutRules) - _, ti, _ := net.ParseCIDR("1.2.3.4/32") + ti, err := netip.ParsePrefix("1.2.3.4/32") + require.NoError(t, err) + + ti6, err := netip.ParsePrefix("fd12::34/128") + require.NoError(t, err) - assert.Nil(t, fw.AddRule(true, firewall.ProtoTCP, 1, 1, []string{}, "", nil, nil, "", "")) + require.NoError(t, fw.AddRule(true, firewall.ProtoTCP, 1, 1, []string{}, "", "", "", "", "")) // An empty rule is any assert.True(t, fw.InRules.TCP[1].Any.Any.Any) assert.Empty(t, fw.InRules.TCP[1].Any.Groups) assert.Empty(t, fw.InRules.TCP[1].Any.Hosts) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", nil, nil, "", "")) + require.NoError(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", "", "", "", "")) assert.Nil(t, fw.InRules.UDP[1].Any.Any) assert.Contains(t, fw.InRules.UDP[1].Any.Groups[0].Groups, "g1") assert.Empty(t, fw.InRules.UDP[1].Any.Hosts) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoICMP, 1, 1, []string{}, "h1", nil, nil, "", "")) - assert.Nil(t, fw.InRules.ICMP[1].Any.Any) - assert.Empty(t, fw.InRules.ICMP[1].Any.Groups) - assert.Contains(t, fw.InRules.ICMP[1].Any.Hosts, "h1") + require.NoError(t, fw.AddRule(true, firewall.ProtoICMP, 1, 1, []string{}, "h1", "", "", "", "")) + //no matter what port is given for icmp, it should end up as "any" + assert.Nil(t, fw.InRules.ICMP[firewall.PortAny].Any.Any) + assert.Empty(t, fw.InRules.ICMP[firewall.PortAny].Any.Groups) + assert.Contains(t, fw.InRules.ICMP[firewall.PortAny].Any.Hosts, "h1") + + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti.String(), "", "", "")) + assert.Nil(t, fw.OutRules.AnyProto[1].Any.Any) + _, ok := fw.OutRules.AnyProto[1].Any.CIDR.Get(ti) + assert.True(t, ok) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) - assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti, nil, "", "")) + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti6.String(), "", "", "")) assert.Nil(t, fw.OutRules.AnyProto[1].Any.Any) - ok, _ := fw.OutRules.AnyProto[1].Any.CIDR.GetCIDR(ti) + _, ok = fw.OutRules.AnyProto[1].Any.CIDR.Get(ti6) + assert.True(t, ok) + + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", "", ti.String(), "", "")) + assert.NotNil(t, fw.OutRules.AnyProto[1].Any.Any) + ok = fw.OutRules.AnyProto[1].Any.Any.LocalCIDR.Get(ti) assert.True(t, ok) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) - assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", nil, ti, "", "")) + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", "", ti6.String(), "", "")) assert.NotNil(t, fw.OutRules.AnyProto[1].Any.Any) - ok, _ = fw.OutRules.AnyProto[1].Any.Any.LocalCIDR.GetCIDR(ti) + ok = fw.OutRules.AnyProto[1].Any.Any.LocalCIDR.Get(ti6) assert.True(t, ok) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", nil, nil, "ca-name", "")) + require.NoError(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", "", "", "ca-name", "")) assert.Contains(t, fw.InRules.UDP[1].CANames, "ca-name") fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", nil, nil, "", "ca-sha")) + require.NoError(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", "", "", "", "ca-sha")) assert.Contains(t, fw.InRules.UDP[1].CAShas, "ca-sha") fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) - assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "any", nil, nil, "", "")) + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "any", "", "", "", "")) + assert.True(t, fw.OutRules.AnyProto[0].Any.Any.Any) + + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) + anyIp, err := netip.ParsePrefix("0.0.0.0/0") + require.NoError(t, err) + + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", anyIp.String(), "", "", "")) + assert.Nil(t, fw.OutRules.AnyProto[0].Any.Any) + table, ok := fw.OutRules.AnyProto[0].Any.CIDR.Lookup(netip.MustParseAddr("1.1.1.1")) + assert.True(t, table.Any) + table, ok = fw.OutRules.AnyProto[0].Any.CIDR.Lookup(netip.MustParseAddr("9::9")) + assert.False(t, ok) + + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) + anyIp6, err := netip.ParsePrefix("::/0") + require.NoError(t, err) + + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", anyIp6.String(), "", "", "")) + assert.Nil(t, fw.OutRules.AnyProto[0].Any.Any) + table, ok = fw.OutRules.AnyProto[0].Any.CIDR.Lookup(netip.MustParseAddr("9::9")) + assert.True(t, table.Any) + table, ok = fw.OutRules.AnyProto[0].Any.CIDR.Lookup(netip.MustParseAddr("1.1.1.1")) + assert.False(t, ok) + + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", "any", "", "", "")) assert.True(t, fw.OutRules.AnyProto[0].Any.Any.Any) fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) - _, anyIp, _ := net.ParseCIDR("0.0.0.0/0") - assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", anyIp, nil, "", "")) + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", "", anyIp.String(), "", "")) + assert.False(t, fw.OutRules.AnyProto[0].Any.Any.Any) + assert.True(t, fw.OutRules.AnyProto[0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("1.1.1.1"))) + assert.False(t, fw.OutRules.AnyProto[0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("9::9"))) + + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", "", anyIp6.String(), "", "")) + assert.False(t, fw.OutRules.AnyProto[0].Any.Any.Any) + assert.True(t, fw.OutRules.AnyProto[0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("9::9"))) + assert.False(t, fw.OutRules.AnyProto[0].Any.Any.LocalCIDR.Lookup(netip.MustParseAddr("1.1.1.1"))) + + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) + require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", "", "any", "", "")) assert.True(t, fw.OutRules.AnyProto[0].Any.Any.Any) // Test error conditions fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c) - assert.Error(t, fw.AddRule(true, math.MaxUint8, 0, 0, []string{}, "", nil, nil, "", "")) - assert.Error(t, fw.AddRule(true, firewall.ProtoAny, 10, 0, []string{}, "", nil, nil, "", "")) + require.Error(t, fw.AddRule(true, math.MaxUint8, 0, 0, []string{}, "", "", "", "", "")) + require.Error(t, fw.AddRule(true, firewall.ProtoAny, 10, 0, []string{}, "", "", "", "", "")) } func TestFirewall_Drop(t *testing.T) { - l := test.NewLogger() ob := &bytes.Buffer{} - l.SetOutput(ob) - + l := test.NewLoggerWithOutput(ob) + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8")) p := firewall.Packet{ - LocalIP: iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)), - RemoteIP: iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)), + LocalAddr: netip.MustParseAddr("1.2.3.4"), + RemoteAddr: netip.MustParseAddr("1.2.3.4"), LocalPort: 10, RemotePort: 90, Protocol: firewall.ProtoUDP, Fragment: false, } - ipNet := net.IPNet{ - IP: net.IPv4(1, 2, 3, 4), - Mask: net.IPMask{255, 255, 255, 0}, + c := dummyCert{ + name: "host1", + networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")}, + groups: []string{"default-group"}, + issuer: "signer-shasum", } - - c := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "host1", - Ips: []*net.IPNet{&ipNet}, - Groups: []string{"default-group"}, - InvertedGroups: map[string]struct{}{"default-group": {}}, - Issuer: "signer-shasum", + h := HostInfo{ + ConnectionState: &ConnectionState{ + peerCert: &cert.CachedCertificate{ + Certificate: &c, + InvertedGroups: map[string]struct{}{"default-group": {}}, + }, }, + vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")}, + } + h.buildNetworks(myVpnNetworksTable, &c) + + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", "")) + cp := cert.NewCAPool() + + // Drop outbound + assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil)) + // Allow inbound + resetConntrack(fw) + require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + // Allow outbound because conntrack + require.NoError(t, fw.Drop(p, false, &h, cp, nil)) + + // test remote mismatch + oldRemote := p.RemoteAddr + p.RemoteAddr = netip.MustParseAddr("1.2.3.10") + assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP) + p.RemoteAddr = oldRemote + + // ensure signer doesn't get in the way of group checks + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad")) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) + + // test caSha doesn't drop on match + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum")) + require.NoError(t, fw.Drop(p, true, &h, cp, nil)) + + // ensure ca name doesn't get in the way of group checks + cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", "")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", "")) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) + + // test caName doesn't drop on match + cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", "")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", "")) + require.NoError(t, fw.Drop(p, true, &h, cp, nil)) +} + +func TestFirewall_DropV6(t *testing.T) { + ob := &bytes.Buffer{} + l := test.NewLoggerWithOutput(ob) + + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("fd00::/7")) + + p := firewall.Packet{ + LocalAddr: netip.MustParseAddr("fd12::34"), + RemoteAddr: netip.MustParseAddr("fd12::34"), + LocalPort: 10, + RemotePort: 90, + Protocol: firewall.ProtoUDP, + Fragment: false, + } + + c := dummyCert{ + name: "host1", + networks: []netip.Prefix{netip.MustParsePrefix("fd12::34/120")}, + groups: []string{"default-group"}, + issuer: "signer-shasum", } h := HostInfo{ ConnectionState: &ConnectionState{ - peerCert: &c, + peerCert: &cert.CachedCertificate{ + Certificate: &c, + InvertedGroups: map[string]struct{}{"default-group": {}}, + }, }, - vpnIp: iputil.Ip2VpnIp(ipNet.IP), + vpnAddrs: []netip.Addr{netip.MustParseAddr("fd12::34")}, } - h.CreateRemoteCIDR(&c) + h.buildNetworks(myVpnNetworksTable, &c) fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", nil, nil, "", "")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", "")) cp := cert.NewCAPool() // Drop outbound - assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil)) // Allow inbound resetConntrack(fw) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil)) // Allow outbound because conntrack - assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil)) + require.NoError(t, fw.Drop(p, false, &h, cp, nil)) // test remote mismatch - oldRemote := p.RemoteIP - p.RemoteIP = iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 10)) - assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrInvalidRemoteIP) - p.RemoteIP = oldRemote + oldRemote := p.RemoteAddr + p.RemoteAddr = netip.MustParseAddr("fd12::56") + assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP) + p.RemoteAddr = oldRemote // ensure signer doesn't get in the way of group checks fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "", "signer-shasum")) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "", "signer-shasum-bad")) - assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad")) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) // test caSha doesn't drop on match fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "", "signer-shasum-bad")) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "", "signer-shasum")) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum")) + require.NoError(t, fw.Drop(p, true, &h, cp, nil)) // ensure ca name doesn't get in the way of group checks - cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}} + cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "ca-good", "")) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "ca-good-bad", "")) - assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", "")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", "")) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) // test caName doesn't drop on match - cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}} + cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}} fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "ca-good-bad", "")) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "ca-good", "")) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", "")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", "")) + require.NoError(t, fw.Drop(p, true, &h, cp, nil)) } func BenchmarkFirewallTable_match(b *testing.B) { @@ -209,15 +335,20 @@ func BenchmarkFirewallTable_match(b *testing.B) { TCP: firewallPort{}, } - _, n, _ := net.ParseCIDR("172.1.1.1/32") - goodLocalCIDRIP := iputil.Ip2VpnIp(n.IP) - _ = ft.TCP.addRule(f, 10, 10, []string{"good-group"}, "good-host", n, nil, "", "") - _ = ft.TCP.addRule(f, 100, 100, []string{"good-group"}, "good-host", nil, n, "", "") + pfix := netip.MustParsePrefix("172.1.1.1/32") + _ = ft.TCP.addRule(f, 10, 10, []string{"good-group"}, "good-host", pfix.String(), "", "", "") + _ = ft.TCP.addRule(f, 100, 100, []string{"good-group"}, "good-host", "", pfix.String(), "", "") + + pfix6 := netip.MustParsePrefix("fd11::11/128") + _ = ft.TCP.addRule(f, 10, 10, []string{"good-group"}, "good-host", pfix6.String(), "", "", "") + _ = ft.TCP.addRule(f, 100, 100, []string{"good-group"}, "good-host", "", pfix6.String(), "", "") cp := cert.NewCAPool() b.Run("fail on proto", func(b *testing.B) { // This benchmark is showing us the cost of failing to match the protocol - c := &cert.NebulaCertificate{} + c := &cert.CachedCertificate{ + Certificate: &dummyCert{}, + } for n := 0; n < b.N; n++ { assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoUDP}, true, c, cp)) } @@ -225,29 +356,52 @@ func BenchmarkFirewallTable_match(b *testing.B) { b.Run("pass proto, fail on port", func(b *testing.B) { // This benchmark is showing us the cost of matching a specific protocol but failing to match the port - c := &cert.NebulaCertificate{} + c := &cert.CachedCertificate{ + Certificate: &dummyCert{}, + } for n := 0; n < b.N; n++ { assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 1}, true, c, cp)) } }) b.Run("pass proto, port, fail on local CIDR", func(b *testing.B) { - c := &cert.NebulaCertificate{} - ip, _, _ := net.ParseCIDR("9.254.254.254/32") - lip := iputil.Ip2VpnIp(ip) + c := &cert.CachedCertificate{ + Certificate: &dummyCert{}, + } + ip := netip.MustParsePrefix("9.254.254.254/32") for n := 0; n < b.N; n++ { - assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalIP: lip}, true, c, cp)) + assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalAddr: ip.Addr()}, true, c, cp)) + } + }) + b.Run("pass proto, port, fail on local CIDRv6", func(b *testing.B) { + c := &cert.CachedCertificate{ + Certificate: &dummyCert{}, + } + ip := netip.MustParsePrefix("fd99::99/128") + for n := 0; n < b.N; n++ { + assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalAddr: ip.Addr()}, true, c, cp)) } }) b.Run("pass proto, port, any local CIDR, fail all group, name, and cidr", func(b *testing.B) { - _, ip, _ := net.ParseCIDR("9.254.254.254/32") - c := &cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - InvertedGroups: map[string]struct{}{"nope": {}}, - Name: "nope", - Ips: []*net.IPNet{ip}, + c := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "nope", + networks: []netip.Prefix{netip.MustParsePrefix("9.254.254.245/32")}, }, + InvertedGroups: map[string]struct{}{"nope": {}}, + } + for n := 0; n < b.N; n++ { + assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp)) + } + }) + b.Run("pass proto, port, any local CIDRv6, fail all group, name, and cidr", func(b *testing.B) { + c := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "nope", + networks: []netip.Prefix{netip.MustParsePrefix("fd99::99/128")}, + }, + InvertedGroups: map[string]struct{}{"nope": {}}, } for n := 0; n < b.N; n++ { assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp)) @@ -255,25 +409,36 @@ func BenchmarkFirewallTable_match(b *testing.B) { }) b.Run("pass proto, port, specific local CIDR, fail all group, name, and cidr", func(b *testing.B) { - _, ip, _ := net.ParseCIDR("9.254.254.254/32") - c := &cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - InvertedGroups: map[string]struct{}{"nope": {}}, - Name: "nope", - Ips: []*net.IPNet{ip}, + c := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "nope", + networks: []netip.Prefix{netip.MustParsePrefix("9.254.254.245/32")}, + }, + InvertedGroups: map[string]struct{}{"nope": {}}, + } + for n := 0; n < b.N; n++ { + assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalAddr: pfix.Addr()}, true, c, cp)) + } + }) + b.Run("pass proto, port, specific local CIDRv6, fail all group, name, and cidr", func(b *testing.B) { + c := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "nope", + networks: []netip.Prefix{netip.MustParsePrefix("fd99::99/128")}, }, + InvertedGroups: map[string]struct{}{"nope": {}}, } for n := 0; n < b.N; n++ { - assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalIP: goodLocalCIDRIP}, true, c, cp)) + assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalAddr: pfix6.Addr()}, true, c, cp)) } }) b.Run("pass on group on any local cidr", func(b *testing.B) { - c := &cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - InvertedGroups: map[string]struct{}{"good-group": {}}, - Name: "nope", + c := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "nope", }, + InvertedGroups: map[string]struct{}{"good-group": {}}, } for n := 0; n < b.N; n++ { assert.True(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp)) @@ -281,294 +446,627 @@ func BenchmarkFirewallTable_match(b *testing.B) { }) b.Run("pass on group on specific local cidr", func(b *testing.B) { - c := &cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - InvertedGroups: map[string]struct{}{"good-group": {}}, - Name: "nope", + c := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "nope", }, + InvertedGroups: map[string]struct{}{"good-group": {}}, } for n := 0; n < b.N; n++ { - assert.True(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalIP: goodLocalCIDRIP}, true, c, cp)) + assert.True(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalAddr: pfix.Addr()}, true, c, cp)) + } + }) + b.Run("pass on group on specific local cidr6", func(b *testing.B) { + c := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "nope", + }, + InvertedGroups: map[string]struct{}{"good-group": {}}, + } + for n := 0; n < b.N; n++ { + assert.True(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalAddr: pfix6.Addr()}, true, c, cp)) } }) b.Run("pass on name", func(b *testing.B) { - c := &cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - InvertedGroups: map[string]struct{}{"nope": {}}, - Name: "good-host", + c := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "good-host", }, + InvertedGroups: map[string]struct{}{"nope": {}}, } for n := 0; n < b.N; n++ { ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp) } }) - // - //b.Run("pass on ip", func(b *testing.B) { - // ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1)) - // c := &cert.NebulaCertificate{ - // Details: cert.NebulaCertificateDetails{ - // InvertedGroups: map[string]struct{}{"nope": {}}, - // Name: "good-host", - // }, - // } - // for n := 0; n < b.N; n++ { - // ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10, RemoteIP: ip}, true, c, cp) - // } - //}) - // - //b.Run("pass on local ip", func(b *testing.B) { - // ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1)) - // c := &cert.NebulaCertificate{ - // Details: cert.NebulaCertificateDetails{ - // InvertedGroups: map[string]struct{}{"nope": {}}, - // Name: "good-host", - // }, - // } - // for n := 0; n < b.N; n++ { - // ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10, LocalIP: ip}, true, c, cp) - // } - //}) - // - //_ = ft.TCP.addRule(0, 0, []string{"good-group"}, "good-host", n, n, "", "") - // - //b.Run("pass on ip with any port", func(b *testing.B) { - // ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1)) - // c := &cert.NebulaCertificate{ - // Details: cert.NebulaCertificateDetails{ - // InvertedGroups: map[string]struct{}{"nope": {}}, - // Name: "good-host", - // }, - // } - // for n := 0; n < b.N; n++ { - // ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, RemoteIP: ip}, true, c, cp) - // } - //}) - // - //b.Run("pass on local ip with any port", func(b *testing.B) { - // ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1)) - // c := &cert.NebulaCertificate{ - // Details: cert.NebulaCertificateDetails{ - // InvertedGroups: map[string]struct{}{"nope": {}}, - // Name: "good-host", - // }, - // } - // for n := 0; n < b.N; n++ { - // ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalIP: ip}, true, c, cp) - // } - //}) } func TestFirewall_Drop2(t *testing.T) { - l := test.NewLogger() ob := &bytes.Buffer{} - l.SetOutput(ob) + l := test.NewLoggerWithOutput(ob) + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8")) p := firewall.Packet{ - LocalIP: iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)), - RemoteIP: iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)), + LocalAddr: netip.MustParseAddr("1.2.3.4"), + RemoteAddr: netip.MustParseAddr("1.2.3.4"), LocalPort: 10, RemotePort: 90, Protocol: firewall.ProtoUDP, Fragment: false, } - ipNet := net.IPNet{ - IP: net.IPv4(1, 2, 3, 4), - Mask: net.IPMask{255, 255, 255, 0}, - } + network := netip.MustParsePrefix("1.2.3.4/24") - c := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "host1", - Ips: []*net.IPNet{&ipNet}, - InvertedGroups: map[string]struct{}{"default-group": {}, "test-group": {}}, + c := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host1", + networks: []netip.Prefix{network}, }, + InvertedGroups: map[string]struct{}{"default-group": {}, "test-group": {}}, } h := HostInfo{ ConnectionState: &ConnectionState{ peerCert: &c, }, - vpnIp: iputil.Ip2VpnIp(ipNet.IP), + vpnAddrs: []netip.Addr{network.Addr()}, } - h.CreateRemoteCIDR(&c) + h.buildNetworks(myVpnNetworksTable, c.Certificate) - c1 := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "host1", - Ips: []*net.IPNet{&ipNet}, - InvertedGroups: map[string]struct{}{"default-group": {}, "test-group-not": {}}, + c1 := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host1", + networks: []netip.Prefix{network}, }, + InvertedGroups: map[string]struct{}{"default-group": {}, "test-group-not": {}}, } h1 := HostInfo{ + vpnAddrs: []netip.Addr{network.Addr()}, ConnectionState: &ConnectionState{ peerCert: &c1, }, } - h1.CreateRemoteCIDR(&c1) + h1.buildNetworks(myVpnNetworksTable, c1.Certificate) - fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group", "test-group"}, "", nil, nil, "", "")) + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group", "test-group"}, "", "", "", "", "")) cp := cert.NewCAPool() // h1/c1 lacks the proper groups - assert.Error(t, fw.Drop([]byte{}, p, true, &h1, cp, nil), ErrNoMatchingRule) + require.ErrorIs(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule) // c has the proper groups resetConntrack(fw) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil)) } func TestFirewall_Drop3(t *testing.T) { - l := test.NewLogger() ob := &bytes.Buffer{} - l.SetOutput(ob) + l := test.NewLoggerWithOutput(ob) + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8")) p := firewall.Packet{ - LocalIP: iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)), - RemoteIP: iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)), + LocalAddr: netip.MustParseAddr("1.2.3.4"), + RemoteAddr: netip.MustParseAddr("1.2.3.4"), LocalPort: 1, RemotePort: 1, Protocol: firewall.ProtoUDP, Fragment: false, } - ipNet := net.IPNet{ - IP: net.IPv4(1, 2, 3, 4), - Mask: net.IPMask{255, 255, 255, 0}, - } - - c := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "host-owner", - Ips: []*net.IPNet{&ipNet}, + network := netip.MustParsePrefix("1.2.3.4/24") + c := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host-owner", + networks: []netip.Prefix{network}, }, } - c1 := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "host1", - Ips: []*net.IPNet{&ipNet}, - Issuer: "signer-sha-bad", + c1 := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host1", + networks: []netip.Prefix{network}, + issuer: "signer-sha-bad", }, } h1 := HostInfo{ ConnectionState: &ConnectionState{ peerCert: &c1, }, - vpnIp: iputil.Ip2VpnIp(ipNet.IP), + vpnAddrs: []netip.Addr{network.Addr()}, } - h1.CreateRemoteCIDR(&c1) + h1.buildNetworks(myVpnNetworksTable, c1.Certificate) - c2 := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "host2", - Ips: []*net.IPNet{&ipNet}, - Issuer: "signer-sha", + c2 := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host2", + networks: []netip.Prefix{network}, + issuer: "signer-sha", }, } h2 := HostInfo{ ConnectionState: &ConnectionState{ peerCert: &c2, }, - vpnIp: iputil.Ip2VpnIp(ipNet.IP), + vpnAddrs: []netip.Addr{network.Addr()}, } - h2.CreateRemoteCIDR(&c2) + h2.buildNetworks(myVpnNetworksTable, c2.Certificate) - c3 := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "host3", - Ips: []*net.IPNet{&ipNet}, - Issuer: "signer-sha-bad", + c3 := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host3", + networks: []netip.Prefix{network}, + issuer: "signer-sha-bad", }, } h3 := HostInfo{ ConnectionState: &ConnectionState{ peerCert: &c3, }, - vpnIp: iputil.Ip2VpnIp(ipNet.IP), + vpnAddrs: []netip.Addr{network.Addr()}, } - h3.CreateRemoteCIDR(&c3) + h3.buildNetworks(myVpnNetworksTable, c3.Certificate) - fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "host1", nil, nil, "", "")) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", nil, nil, "", "signer-sha")) + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "host1", "", "", "", "")) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "", "", "", "signer-sha")) cp := cert.NewCAPool() // c1 should pass because host match - assert.NoError(t, fw.Drop([]byte{}, p, true, &h1, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h1, cp, nil)) // c2 should pass because ca sha match resetConntrack(fw) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h2, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h2, cp, nil)) // c3 should fail because no match resetConntrack(fw) - assert.Equal(t, fw.Drop([]byte{}, p, true, &h3, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule) + + // Test a remote address match + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", "")) + require.NoError(t, fw.Drop(p, true, &h1, cp, nil)) } -func TestFirewall_DropConntrackReload(t *testing.T) { - l := test.NewLogger() +func TestFirewall_Drop3V6(t *testing.T) { ob := &bytes.Buffer{} - l.SetOutput(ob) + l := test.NewLoggerWithOutput(ob) + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("fd00::/7")) p := firewall.Packet{ - LocalIP: iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)), - RemoteIP: iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 4)), - LocalPort: 10, - RemotePort: 90, + LocalAddr: netip.MustParseAddr("fd12::34"), + RemoteAddr: netip.MustParseAddr("fd12::34"), + LocalPort: 1, + RemotePort: 1, Protocol: firewall.ProtoUDP, Fragment: false, } - ipNet := net.IPNet{ - IP: net.IPv4(1, 2, 3, 4), - Mask: net.IPMask{255, 255, 255, 0}, + network := netip.MustParsePrefix("fd12::34/120") + c := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host-owner", + networks: []netip.Prefix{network}, + }, } + h := HostInfo{ + ConnectionState: &ConnectionState{ + peerCert: &c, + }, + vpnAddrs: []netip.Addr{network.Addr()}, + } + h.buildNetworks(myVpnNetworksTable, c.Certificate) - c := cert.NebulaCertificate{ - Details: cert.NebulaCertificateDetails{ - Name: "host1", - Ips: []*net.IPNet{&ipNet}, - Groups: []string{"default-group"}, - InvertedGroups: map[string]struct{}{"default-group": {}}, - Issuer: "signer-shasum", + // Test a remote address match + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + cp := cert.NewCAPool() + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", "")) + require.NoError(t, fw.Drop(p, true, &h, cp, nil)) +} + +func TestFirewall_DropConntrackReload(t *testing.T) { + ob := &bytes.Buffer{} + l := test.NewLoggerWithOutput(ob) + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8")) + + p := firewall.Packet{ + LocalAddr: netip.MustParseAddr("1.2.3.4"), + RemoteAddr: netip.MustParseAddr("1.2.3.4"), + LocalPort: 10, + RemotePort: 90, + Protocol: firewall.ProtoUDP, + Fragment: false, + } + network := netip.MustParsePrefix("1.2.3.4/24") + + c := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host1", + networks: []netip.Prefix{network}, + groups: []string{"default-group"}, + issuer: "signer-shasum", }, + InvertedGroups: map[string]struct{}{"default-group": {}}, } h := HostInfo{ ConnectionState: &ConnectionState{ peerCert: &c, }, - vpnIp: iputil.Ip2VpnIp(ipNet.IP), + vpnAddrs: []netip.Addr{network.Addr()}, } - h.CreateRemoteCIDR(&c) + h.buildNetworks(myVpnNetworksTable, c.Certificate) - fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", nil, nil, "", "")) + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", "")) cp := cert.NewCAPool() // Drop outbound - assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule) // Allow inbound resetConntrack(fw) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + require.NoError(t, fw.Drop(p, true, &h, cp, nil)) // Allow outbound because conntrack - assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil)) + require.NoError(t, fw.Drop(p, false, &h, cp, nil)) oldFw := fw - fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 10, 10, []string{"any"}, "", nil, nil, "", "")) + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 10, 10, []string{"any"}, "", "", "", "", "")) fw.Conntrack = oldFw.Conntrack fw.rulesVersion = oldFw.rulesVersion + 1 // Allow outbound because conntrack and new rules allow port 10 - assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil)) + require.NoError(t, fw.Drop(p, false, &h, cp, nil)) oldFw = fw - fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) - assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 11, 11, []string{"any"}, "", nil, nil, "", "")) + fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 11, 11, []string{"any"}, "", "", "", "", "")) fw.Conntrack = oldFw.Conntrack fw.rulesVersion = oldFw.rulesVersion + 1 // Drop outbound because conntrack doesn't match new ruleset - assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule) +} + +func TestFirewall_ICMPPortBehavior(t *testing.T) { + ob := &bytes.Buffer{} + l := test.NewLoggerWithOutput(ob) + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8")) + + network := netip.MustParsePrefix("1.2.3.4/24") + + c := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host1", + networks: []netip.Prefix{network}, + groups: []string{"default-group"}, + issuer: "signer-shasum", + }, + InvertedGroups: map[string]struct{}{"default-group": {}}, + } + h := HostInfo{ + ConnectionState: &ConnectionState{ + peerCert: &c, + }, + vpnAddrs: []netip.Addr{network.Addr()}, + } + h.buildNetworks(myVpnNetworksTable, c.Certificate) + + cp := cert.NewCAPool() + + templ := firewall.Packet{ + LocalAddr: netip.MustParseAddr("1.2.3.4"), + RemoteAddr: netip.MustParseAddr("1.2.3.4"), + Protocol: firewall.ProtoICMP, + Fragment: false, + } + + t.Run("ICMP allowed", func(t *testing.T) { + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + require.NoError(t, fw.AddRule(true, firewall.ProtoICMP, 0, 0, []string{"any"}, "", "", "", "", "")) + t.Run("zero ports", func(t *testing.T) { + p := templ.Copy() + p.LocalPort = 0 + p.RemotePort = 0 + // Drop outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + // Allow inbound + resetConntrack(fw) + require.NoError(t, fw.Drop(*p, true, &h, cp, nil)) + //now also allow outbound + require.NoError(t, fw.Drop(*p, false, &h, cp, nil)) + }) + + t.Run("nonzero ports", func(t *testing.T) { + p := templ.Copy() + p.LocalPort = 0xabcd + p.RemotePort = 0x1234 + // Drop outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + // Allow inbound + resetConntrack(fw) + require.NoError(t, fw.Drop(*p, true, &h, cp, nil)) + //now also allow outbound + require.NoError(t, fw.Drop(*p, false, &h, cp, nil)) + }) + }) + + t.Run("Any proto, some ports allowed", func(t *testing.T) { + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 80, 444, []string{"any"}, "", "", "", "", "")) + t.Run("zero ports, still blocked", func(t *testing.T) { + p := templ.Copy() + p.LocalPort = 0 + p.RemotePort = 0 + // Drop outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + // Allow inbound + resetConntrack(fw) + assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule) + //now also allow outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + }) + + t.Run("nonzero ports, still blocked", func(t *testing.T) { + p := templ.Copy() + p.LocalPort = 0xabcd + p.RemotePort = 0x1234 + // Drop outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + // Allow inbound + resetConntrack(fw) + assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule) + //now also allow outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + }) + + t.Run("nonzero, matching ports, still blocked", func(t *testing.T) { + p := templ.Copy() + p.LocalPort = 80 + p.RemotePort = 80 + // Drop outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + // Allow inbound + resetConntrack(fw) + assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule) + //now also allow outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + }) + }) + t.Run("Any proto, any port", func(t *testing.T) { + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", "")) + t.Run("zero ports, allowed", func(t *testing.T) { + resetConntrack(fw) + p := templ.Copy() + p.LocalPort = 0 + p.RemotePort = 0 + // Drop outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + // Allow inbound + resetConntrack(fw) + require.NoError(t, fw.Drop(*p, true, &h, cp, nil)) + //now also allow outbound + require.NoError(t, fw.Drop(*p, false, &h, cp, nil)) + }) + + t.Run("nonzero ports, allowed", func(t *testing.T) { + resetConntrack(fw) + p := templ.Copy() + p.LocalPort = 0xabcd + p.RemotePort = 0x1234 + // Drop outbound + assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + // Allow inbound + resetConntrack(fw) + require.NoError(t, fw.Drop(*p, true, &h, cp, nil)) + //now also allow outbound + require.NoError(t, fw.Drop(*p, false, &h, cp, nil)) + //different ID is blocked + p.RemotePort++ + require.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule) + }) + }) + +} + +func TestFirewall_DropIPSpoofing(t *testing.T) { + ob := &bytes.Buffer{} + l := test.NewLoggerWithOutput(ob) + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24")) + + c := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host-owner", + networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")}, + }, + } + + c1 := cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "host", + networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")}, + unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")}, + }, + } + h1 := HostInfo{ + ConnectionState: &ConnectionState{ + peerCert: &c1, + }, + vpnAddrs: []netip.Addr{c1.Certificate.Networks()[0].Addr()}, + } + h1.buildNetworks(myVpnNetworksTable, c1.Certificate) + + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate) + + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "", "", "", "")) + cp := cert.NewCAPool() + + // Packet spoofed by `c1`. Note that the remote addr is not a valid one. + p := firewall.Packet{ + LocalAddr: netip.MustParseAddr("192.0.2.1"), + RemoteAddr: netip.MustParseAddr("192.0.2.3"), + LocalPort: 1, + RemotePort: 1, + Protocol: firewall.ProtoUDP, + Fragment: false, + } + assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP) +} + +func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) { + l := test.NewLoggerWithOutput(&bytes.Buffer{}) + + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24")) + + owner := &dummyCert{ + name: "owner", + networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")}, + } + + victim := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "victim", + networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")}, + }, + } + victimHI := HostInfo{ + ConnectionState: &ConnectionState{peerCert: victim}, + vpnAddrs: []netip.Addr{netip.MustParseAddr("192.0.2.2")}, + } + victimHI.buildNetworks(myVpnNetworksTable, victim.Certificate) + + attacker := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "attacker", + networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.3/24")}, + }, + } + attackerHI := HostInfo{ + ConnectionState: &ConnectionState{peerCert: attacker}, + vpnAddrs: []netip.Addr{netip.MustParseAddr("192.0.2.3")}, + } + attackerHI.buildNetworks(myVpnNetworksTable, attacker.Certificate) + + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner) + // Allow any inbound traffic that passes the cert / source-IP checks. + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", "")) + cp := cert.NewCAPool() + + flow := firewall.Packet{ + LocalAddr: netip.MustParseAddr("192.0.2.1"), + RemoteAddr: netip.MustParseAddr("192.0.2.2"), + LocalPort: 443, + RemotePort: 55000, + Protocol: firewall.ProtoUDP, + } + + require.NoError(t, fw.Drop(flow, true, &victimHI, cp, nil), + "victim's own traffic from its own overlay IP must be allowed") + + unseen := flow + unseen.RemotePort = 55001 + assert.Equal(t, ErrInvalidRemoteIP, fw.Drop(unseen, true, &attackerHI, cp, nil), + "sanity: attacker forging victim's source IP must be rejected when no conntrack entry exists") + + got := fw.Drop(flow, true, &attackerHI, cp, nil) + t.Logf("attacker replaying victim's 4-tuple: Drop returned %v (nil == packet ALLOWED == spoof succeeded)", got) + assert.Equal(t, ErrInvalidRemoteIP, got, + "SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got) +} + +// BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow +// (a conntrack hit). This is the fast path that the source-IP<->cert binding +// reordering adds work to, so it quantifies the cost of moving the address checks +// ahead of the conntrack lookup. Cases: +// - simple: peer cert has one address, no unsafe networks (h.networks == nil), +// so the remote-address check is a single netip.Addr compare. +// - complex: peer cert has unsafe networks (h.networks populated), so the +// remote-address check is a BART lookup. +// - noCache/localCache: whether a per-batch ConntrackCache is supplied, which in +// the original code let the fast path skip straight past the address checks. +func BenchmarkFirewallDropConntrackHit(b *testing.B) { + l := test.NewLoggerWithOutput(&bytes.Buffer{}) + + myVpnNetworksTable := new(bart.Lite) + myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24")) + + owner := &dummyCert{ + name: "owner", + networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")}, + } + + simpleCert := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "simple", + networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")}, + }, + } + simpleHost := &HostInfo{ + ConnectionState: &ConnectionState{peerCert: simpleCert}, + vpnAddrs: []netip.Addr{netip.MustParseAddr("192.0.2.2")}, + } + simpleHost.buildNetworks(myVpnNetworksTable, simpleCert.Certificate) + + complexCert := &cert.CachedCertificate{ + Certificate: &dummyCert{ + name: "complex", + networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")}, + unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")}, + }, + } + complexHost := &HostInfo{ + ConnectionState: &ConnectionState{peerCert: complexCert}, + vpnAddrs: []netip.Addr{netip.MustParseAddr("192.0.2.2")}, + } + complexHost.buildNetworks(myVpnNetworksTable, complexCert.Certificate) + + cp := cert.NewCAPool() + + flow := firewall.Packet{ + LocalAddr: netip.MustParseAddr("192.0.2.1"), + RemoteAddr: netip.MustParseAddr("192.0.2.2"), + LocalPort: 443, + RemotePort: 55000, + Protocol: firewall.ProtoUDP, + } + + cases := []struct { + name string + host *HostInfo + useCache bool + }{ + {"simple/noCache", simpleHost, false}, + {"simple/localCache", simpleHost, true}, + {"complex/noCache", complexHost, false}, + {"complex/localCache", complexHost, true}, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner) + require.NoError(b, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", "")) + + // Establish the conntrack entry so every benchmarked Drop is a hit. + require.NoError(b, fw.Drop(flow, true, tc.host, cp, nil)) + + var cache firewall.ConntrackCache + if tc.useCache { + cache = firewall.ConntrackCache{} + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := fw.Drop(flow, true, tc.host, cp, cache); err != nil { + b.Fatal(err) + } + } + }) + } } func BenchmarkLookup(b *testing.B) { @@ -642,319 +1140,566 @@ func BenchmarkLookup(b *testing.B) { ml(m, a) } }) - - //TODO: only way array lookup in array will help is if both are sorted, then maybe it's faster } func Test_parsePort(t *testing.T) { _, _, err := parsePort("") - assert.EqualError(t, err, "was not a number; ``") + require.EqualError(t, err, "was not a number; ``") _, _, err = parsePort(" ") - assert.EqualError(t, err, "was not a number; ` `") + require.EqualError(t, err, "was not a number; ` `") _, _, err = parsePort("-") - assert.EqualError(t, err, "appears to be a range but could not be parsed; `-`") + require.EqualError(t, err, "appears to be a range but could not be parsed; `-`") _, _, err = parsePort(" - ") - assert.EqualError(t, err, "appears to be a range but could not be parsed; ` - `") + require.EqualError(t, err, "appears to be a range but could not be parsed; ` - `") _, _, err = parsePort("a-b") - assert.EqualError(t, err, "beginning range was not a number; `a`") + require.EqualError(t, err, "beginning range was not a number; `a`") _, _, err = parsePort("1-b") - assert.EqualError(t, err, "ending range was not a number; `b`") + require.EqualError(t, err, "ending range was not a number; `b`") s, e, err := parsePort(" 1 - 2 ") assert.Equal(t, int32(1), s) assert.Equal(t, int32(2), e) - assert.Nil(t, err) + require.NoError(t, err) s, e, err = parsePort("0-1") assert.Equal(t, int32(0), s) assert.Equal(t, int32(0), e) - assert.Nil(t, err) + require.NoError(t, err) s, e, err = parsePort("9919") assert.Equal(t, int32(9919), s) assert.Equal(t, int32(9919), e) - assert.Nil(t, err) + require.NoError(t, err) s, e, err = parsePort("any") assert.Equal(t, int32(0), s) assert.Equal(t, int32(0), e) - assert.Nil(t, err) + require.NoError(t, err) +} + +// Test_parsePort_invalid covers inputs that must error. The named bug is +// that int32(strconv.Atoi("4294967296")) truncates to 0 == firewall.PortAny, +// silently turning a typo into a match-all-ports rule; the rest are +// representative syntax/range probes. +func Test_parsePort_invalid(t *testing.T) { + tests := []struct { + name string + input string + wantErrContains string + }{ + // Numeric overflow (the named bug + boundary). + {"named bug: 2^32 truncates to PortAny", "4294967296", "out of range"}, + {"just above max real port", "65536", "out of range"}, + + // Negatives route through the range branch and hit the empty-half + // guard; included as defense in depth so a future refactor cannot + // accidentally reach the int32 cast. + {"negative", "-1", "could not be parsed"}, + + // Syntax probes. + {"NUL between digits", "4\x002", "was not a number"}, + {"hex notation", "0x10", "was not a number"}, + {"scientific notation", "1e3", "was not a number"}, + {"leading whitespace", " 42", "was not a number"}, + {"fullwidth digits", "42", "was not a number"}, + + // Range branch. + {"range upper out of range", "1-65536", "ending range out of range"}, + {"range lower out of range", "65536-65537", "beginning range out of range"}, + {"range with negative upper", "1--1", "ending range was not a number"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, _, err := parsePort(tc.input) + require.Error(t, err, "input %q must error", tc.input) + require.ErrorContains(t, err, tc.wantErrContains) + }) + } +} + +// Test_parsePort_valid_boundaries locks in success cases at 0, 1, and 65535 +// so a future refactor cannot regress the boundaries. +func Test_parsePort_valid_boundaries(t *testing.T) { + tests := []struct { + name string + input string + wantStart int32 + wantEnd int32 + }{ + {"zero is PortAny", "0", 0, 0}, + {"min real port", "1", 1, 1}, + {"max real port", "65535", 65535, 65535}, + {"range zero to max forces end to zero", "0-65535", 0, 0}, + {"range max to max", "65535-65535", 65535, 65535}, + {"range one to max", "1-65535", 1, 65535}, + {"range with whitespace inside", " 1 - 2 ", 1, 2}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, e, err := parsePort(tc.input) + require.NoError(t, err) + assert.Equal(t, tc.wantStart, s, "start port") + assert.Equal(t, tc.wantEnd, e, "end port") + }) + } } func TestNewFirewallFromConfig(t *testing.T) { l := test.NewLogger() // Test a bad rule definition - c := &cert.NebulaCertificate{} - conf := config.NewC(l) - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": "asdf"} - _, err := NewFirewallFromConfig(l, c, conf) - assert.EqualError(t, err, "firewall.outbound failed to parse, should be an array of rules") + c := &dummyCert{} + cs, err := newCertState(cert.Version2, nil, c, false, cert.Curve_CURVE25519, nil, "aes") + require.NoError(t, err) + + conf := config.NewC(test.NewLogger()) + conf.Settings["firewall"] = map[string]any{"outbound": "asdf"} + _, err = NewFirewallFromConfig(l, cs, conf) + require.EqualError(t, err, "firewall.outbound failed to parse, should be an array of rules") // Test both port and code - conf = config.NewC(l) - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "1", "code": "2"}}} - _, err = NewFirewallFromConfig(l, c, conf) - assert.EqualError(t, err, "firewall.outbound rule #0; only one of port or code should be provided") + conf = config.NewC(test.NewLogger()) + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "code": "2"}}} + _, err = NewFirewallFromConfig(l, cs, conf) + require.EqualError(t, err, "firewall.outbound rule #0; only one of port or code should be provided") // Test missing host, group, cidr, ca_name and ca_sha - conf = config.NewC(l) - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{}}} - _, err = NewFirewallFromConfig(l, c, conf) - assert.EqualError(t, err, "firewall.outbound rule #0; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided") + conf = config.NewC(test.NewLogger()) + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{}}} + _, err = NewFirewallFromConfig(l, cs, conf) + require.EqualError(t, err, "firewall.outbound rule #0; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided") // Test code/port error - conf = config.NewC(l) - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"code": "a", "host": "testh"}}} - _, err = NewFirewallFromConfig(l, c, conf) - assert.EqualError(t, err, "firewall.outbound rule #0; code was not a number; `a`") + conf = config.NewC(test.NewLogger()) + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "a", "host": "testh", "proto": "any"}}} + _, err = NewFirewallFromConfig(l, cs, conf) + require.EqualError(t, err, "firewall.outbound rule #0; code was not a number; `a`") - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "a", "host": "testh"}}} - _, err = NewFirewallFromConfig(l, c, conf) - assert.EqualError(t, err, "firewall.outbound rule #0; port was not a number; `a`") + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "a", "host": "testh", "proto": "any"}}} + _, err = NewFirewallFromConfig(l, cs, conf) + require.EqualError(t, err, "firewall.outbound rule #0; port was not a number; `a`") // Test proto error - conf = config.NewC(l) - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"code": "1", "host": "testh"}}} - _, err = NewFirewallFromConfig(l, c, conf) - assert.EqualError(t, err, "firewall.outbound rule #0; proto was not understood; ``") + conf = config.NewC(test.NewLogger()) + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "1", "host": "testh"}}} + _, err = NewFirewallFromConfig(l, cs, conf) + require.EqualError(t, err, "firewall.outbound rule #0; proto was not understood; ``") // Test cidr parse error - conf = config.NewC(l) - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"code": "1", "cidr": "testh", "proto": "any"}}} - _, err = NewFirewallFromConfig(l, c, conf) - assert.EqualError(t, err, "firewall.outbound rule #0; cidr did not parse; invalid CIDR address: testh") + conf = config.NewC(test.NewLogger()) + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "1", "cidr": "testh", "proto": "any"}}} + _, err = NewFirewallFromConfig(l, cs, conf) + require.EqualError(t, err, "firewall.outbound rule #0; cidr did not parse; netip.ParsePrefix(\"testh\"): no '/'") // Test local_cidr parse error - conf = config.NewC(l) - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"code": "1", "local_cidr": "testh", "proto": "any"}}} - _, err = NewFirewallFromConfig(l, c, conf) - assert.EqualError(t, err, "firewall.outbound rule #0; local_cidr did not parse; invalid CIDR address: testh") + conf = config.NewC(test.NewLogger()) + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "1", "local_cidr": "testh", "proto": "any"}}} + _, err = NewFirewallFromConfig(l, cs, conf) + require.EqualError(t, err, "firewall.outbound rule #0; local_cidr did not parse; netip.ParsePrefix(\"testh\"): no '/'") // Test both group and groups - conf = config.NewC(l) - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "group": "a", "groups": []string{"b", "c"}}}} - _, err = NewFirewallFromConfig(l, c, conf) - assert.EqualError(t, err, "firewall.inbound rule #0; only one of group or groups should be defined, both provided") + conf = config.NewC(test.NewLogger()) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": "a", "groups": []string{"b", "c"}}}} + _, err = NewFirewallFromConfig(l, cs, conf) + require.EqualError(t, err, "firewall.inbound rule #0; only one of group or groups should be defined, both provided") } func TestAddFirewallRulesFromConfig(t *testing.T) { l := test.NewLogger() // Test adding tcp rule - conf := config.NewC(l) + conf := config.NewC(test.NewLogger()) mf := &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "tcp", "host": "a"}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, false, conf, mf)) - assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil, localIp: nil}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "tcp", "host": "a"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf)) + assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoTCP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall) // Test adding udp rule - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "udp", "host": "a"}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, false, conf, mf)) - assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil, localIp: nil}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "udp", "host": "a"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf)) + assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoUDP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall) // Test adding icmp rule - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"outbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "icmp", "host": "a"}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, false, conf, mf)) - assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil, localIp: nil}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "icmp", "host": "a"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf)) + assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: firewall.PortAny, endPort: firewall.PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall) + + // Test adding icmp rule no port + conf = config.NewC(test.NewLogger()) + mf = &mockFirewall{} + conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"proto": "icmp", "host": "a"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf)) + assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: firewall.PortAny, endPort: firewall.PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall) // Test adding any rule - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "host": "a"}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf)) - assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, host: "a", ip: nil, localIp: nil}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall) // Test adding rule with cidr - cidr := &net.IPNet{IP: net.ParseIP("10.0.0.0").To4(), Mask: net.IPv4Mask(255, 0, 0, 0)} - conf = config.NewC(l) + cidr := netip.MustParsePrefix("10.0.0.0/8") + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "cidr": cidr.String()}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf)) - assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr, localIp: nil}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr.String()}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr.String(), localIp: ""}, mf.lastCall) // Test adding rule with local_cidr - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) + mf = &mockFirewall{} + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr.String()}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr.String()}, mf.lastCall) + + // Test adding rule with cidr ipv6 + cidr6 := netip.MustParsePrefix("fd00::/8") + conf = config.NewC(test.NewLogger()) + mf = &mockFirewall{} + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": cidr6.String()}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: cidr6.String(), localIp: ""}, mf.lastCall) + + // Test adding rule with any cidr + conf = config.NewC(test.NewLogger()) + mf = &mockFirewall{} + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "any"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "any", localIp: ""}, mf.lastCall) + + // Test adding rule with junk cidr + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "local_cidr": cidr.String()}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf)) - assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: nil, localIp: cidr}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "cidr": "junk/junk"}}} + require.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP") + + // Test adding rule with local_cidr ipv6 + conf = config.NewC(test.NewLogger()) + mf = &mockFirewall{} + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": cidr6.String()}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: cidr6.String()}, mf.lastCall) + + // Test adding rule with any local_cidr + conf = config.NewC(test.NewLogger()) + mf = &mockFirewall{} + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "any"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, localIp: "any"}, mf.lastCall) + + // Test adding rule with junk local_cidr + conf = config.NewC(test.NewLogger()) + mf = &mockFirewall{} + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "local_cidr": "junk/junk"}}} + require.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; local_cidr did not parse; netip.ParsePrefix(\"junk/junk\"): ParseAddr(\"junk\"): unable to parse IP") // Test adding rule with ca_sha - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "ca_sha": "12312313123"}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf)) - assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: nil, localIp: nil, caSha: "12312313123"}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_sha": "12312313123"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caSha: "12312313123"}, mf.lastCall) // Test adding rule with ca_name - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "ca_name": "root01"}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf)) - assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: nil, localIp: nil, caName: "root01"}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "ca_name": "root01"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: nil, ip: "", localIp: "", caName: "root01"}, mf.lastCall) // Test single group - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "group": "a"}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf)) - assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: nil, localIp: nil}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "group": "a"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall) // Test single groups - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "groups": "a"}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf)) - assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: nil, localIp: nil}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": "a"}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a"}, ip: "", localIp: ""}, mf.lastCall) // Test multiple AND groups - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "groups": []string{"a", "b"}}}} - assert.Nil(t, AddFirewallRulesFromConfig(l, true, conf, mf)) - assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: nil, localIp: nil}, mf.lastCall) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "groups": []string{"a", "b"}}}} + require.NoError(t, AddFirewallRulesFromConfig(l, true, conf, mf)) + assert.Equal(t, addRuleCall{incoming: true, proto: firewall.ProtoAny, startPort: 1, endPort: 1, groups: []string{"a", "b"}, ip: "", localIp: ""}, mf.lastCall) // Test Add error - conf = config.NewC(l) + conf = config.NewC(test.NewLogger()) mf = &mockFirewall{} mf.nextCallReturn = errors.New("test error") - conf.Settings["firewall"] = map[interface{}]interface{}{"inbound": []interface{}{map[interface{}]interface{}{"port": "1", "proto": "any", "host": "a"}}} - assert.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; `test error`") -} - -func TestTCPRTTTracking(t *testing.T) { - b := make([]byte, 200) - - // Max ip IHL (60 bytes) and tcp IHL (60 bytes) - b[0] = 15 - b[60+12] = 15 << 4 - f := Firewall{ - metricTCPRTT: metrics.GetOrRegisterHistogram("nope", nil, metrics.NewExpDecaySample(1028, 0.015)), - } - - // Set SEQ to 1 - binary.BigEndian.PutUint32(b[60+4:60+8], 1) - - c := &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, uint32(1), c.Seq) - - // Bad ack - no ack flag - binary.BigEndian.PutUint32(b[60+8:60+12], 80) - assert.False(t, f.checkTCPRTT(c, b)) - - // Bad ack, number is too low - binary.BigEndian.PutUint32(b[60+8:60+12], 0) - b[60+13] = uint8(0x10) - assert.False(t, f.checkTCPRTT(c, b)) - - // Good ack - binary.BigEndian.PutUint32(b[60+8:60+12], 80) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) - - // Set SEQ to 1 - binary.BigEndian.PutUint32(b[60+4:60+8], 1) - c = &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, uint32(1), c.Seq) - - // Good acks - binary.BigEndian.PutUint32(b[60+8:60+12], 81) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) - - // Set SEQ to max uint32 - 20 - binary.BigEndian.PutUint32(b[60+4:60+8], ^uint32(0)-20) - c = &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, ^uint32(0)-20, c.Seq) - - // Good acks - binary.BigEndian.PutUint32(b[60+8:60+12], 81) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) - - // Set SEQ to max uint32 / 2 - binary.BigEndian.PutUint32(b[60+4:60+8], ^uint32(0)/2) - c = &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, ^uint32(0)/2, c.Seq) - - // Below - binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)/2-1) - assert.False(t, f.checkTCPRTT(c, b)) - assert.Equal(t, ^uint32(0)/2, c.Seq) - - // Halfway below - binary.BigEndian.PutUint32(b[60+8:60+12], uint32(0)) - assert.False(t, f.checkTCPRTT(c, b)) - assert.Equal(t, ^uint32(0)/2, c.Seq) - - // Halfway above is ok - binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) - - // Set SEQ to max uint32 - binary.BigEndian.PutUint32(b[60+4:60+8], ^uint32(0)) - c = &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, ^uint32(0), c.Seq) - - // Halfway + 1 above - binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)/2+1) - assert.False(t, f.checkTCPRTT(c, b)) - assert.Equal(t, ^uint32(0), c.Seq) - - // Halfway above - binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)/2) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) + conf.Settings["firewall"] = map[string]any{"inbound": []any{map[string]any{"port": "1", "proto": "any", "host": "a"}}} + require.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; `test error`") } func TestFirewall_convertRule(t *testing.T) { - l := test.NewLogger() ob := &bytes.Buffer{} - l.SetOutput(ob) + l := test.NewLoggerWithOutput(ob) // Ensure group array of 1 is converted and a warning is printed - c := map[interface{}]interface{}{ - "group": []interface{}{"group1"}, + c := map[string]any{ + "group": []any{"group1"}, } r, err := convertRule(l, c, "test", 1) - assert.Contains(t, ob.String(), "test rule #1; group was an array with a single value, converting to simple value") - assert.Nil(t, err) - assert.Equal(t, "group1", r.Group) + assert.Contains(t, ob.String(), "group was an array with a single value, converting to simple value") + assert.Contains(t, ob.String(), "table=test") + assert.Contains(t, ob.String(), "rule=1") + require.NoError(t, err) + assert.Equal(t, []string{"group1"}, r.Groups) // Ensure group array of > 1 is errord ob.Reset() - c = map[interface{}]interface{}{ - "group": []interface{}{"group1", "group2"}, + c = map[string]any{ + "group": []any{"group1", "group2"}, } r, err = convertRule(l, c, "test", 1) - assert.Equal(t, "", ob.String()) - assert.Error(t, err, "group should contain a single value, an array with more than one entry was provided") + assert.Empty(t, ob.String()) + require.Error(t, err, "group should contain a single value, an array with more than one entry was provided") // Make sure a well formed group is alright ob.Reset() - c = map[interface{}]interface{}{ + c = map[string]any{ "group": "group1", } r, err = convertRule(l, c, "test", 1) - assert.Nil(t, err) - assert.Equal(t, "group1", r.Group) + require.NoError(t, err) + assert.Equal(t, []string{"group1"}, r.Groups) +} + +func TestFirewall_convertRuleSanity(t *testing.T) { + ob := &bytes.Buffer{} + l := test.NewLoggerWithOutput(ob) + + noWarningPlease := []map[string]any{ + {"group": "group1"}, + {"groups": []any{"group2"}}, + {"host": "bob"}, + {"cidr": "1.1.1.1/1"}, + {"groups": []any{"group2"}, "host": "bob"}, + {"cidr": "1.1.1.1/1", "host": "bob"}, + {"groups": []any{"group2"}, "cidr": "1.1.1.1/1"}, + {"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"}, + } + for _, c := range noWarningPlease { + r, err := convertRule(l, c, "test", 1) + require.NoError(t, err) + require.NoError(t, r.sanity(), "should not generate a sanity warning, %+v", c) + } + + yesWarningPlease := []map[string]any{ + {"group": "group1"}, + {"groups": []any{"group2"}}, + {"cidr": "1.1.1.1/1"}, + {"groups": []any{"group2"}, "host": "bob"}, + {"cidr": "1.1.1.1/1", "host": "bob"}, + {"groups": []any{"group2"}, "cidr": "1.1.1.1/1"}, + {"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"}, + } + for _, c := range yesWarningPlease { + c["host"] = "any" + r, err := convertRule(l, c, "test", 1) + require.NoError(t, err) + err = r.sanity() + require.Error(t, err, "I wanted a warning: %+v", c) + } + //reset the list + yesWarningPlease = []map[string]any{ + {"group": "group1"}, + {"groups": []any{"group2"}}, + {"cidr": "1.1.1.1/1"}, + {"groups": []any{"group2"}, "host": "bob"}, + {"cidr": "1.1.1.1/1", "host": "bob"}, + {"groups": []any{"group2"}, "cidr": "1.1.1.1/1"}, + {"groups": []any{"group2"}, "cidr": "1.1.1.1/1", "host": "bob"}, + } + for _, c := range yesWarningPlease { + r, err := convertRule(l, c, "test", 1) + require.NoError(t, err) + r.Groups = append(r.Groups, "any") + err = r.sanity() + require.Error(t, err, "I wanted a warning: %+v", c) + } +} + +type testcase struct { + h *HostInfo + p firewall.Packet + c cert.Certificate + err error +} + +func (c *testcase) Test(t *testing.T, fw *Firewall) { + t.Helper() + cp := cert.NewCAPool() + resetConntrack(fw) + err := fw.Drop(c.p, true, c.h, cp, nil) + if c.err == nil { + require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr) + } else { + require.ErrorIs(t, c.err, err, "failed to drop remote address %s", c.p.RemoteAddr) + } +} + +func buildTestCase(setup testsetup, err error, theirPrefixes ...netip.Prefix) testcase { + c1 := dummyCert{ + name: "host1", + networks: theirPrefixes, + groups: []string{"default-group"}, + issuer: "signer-shasum", + } + h := HostInfo{ + ConnectionState: &ConnectionState{ + peerCert: &cert.CachedCertificate{ + Certificate: &c1, + InvertedGroups: map[string]struct{}{"default-group": {}}, + }, + }, + vpnAddrs: make([]netip.Addr, len(theirPrefixes)), + } + for i := range theirPrefixes { + h.vpnAddrs[i] = theirPrefixes[i].Addr() + } + h.buildNetworks(setup.myVpnNetworksTable, &c1) + p := firewall.Packet{ + LocalAddr: setup.c.Networks()[0].Addr(), //todo? + RemoteAddr: theirPrefixes[0].Addr(), + LocalPort: 10, + RemotePort: 90, + Protocol: firewall.ProtoUDP, + Fragment: false, + } + return testcase{ + h: &h, + p: p, + c: &c1, + err: err, + } +} + +type testsetup struct { + c dummyCert + myVpnNetworksTable *bart.Lite + fw *Firewall +} + +func newSetup(t *testing.T, l *slog.Logger, myPrefixes ...netip.Prefix) testsetup { + c := dummyCert{ + name: "me", + networks: myPrefixes, + groups: []string{"default-group"}, + issuer: "signer-shasum", + } + + return newSetupFromCert(t, l, c) +} + +func newSetupFromCert(t *testing.T, l *slog.Logger, c dummyCert) testsetup { + myVpnNetworksTable := new(bart.Lite) + for _, prefix := range c.Networks() { + myVpnNetworksTable.Insert(prefix) + } + fw := NewFirewall(l, time.Second, time.Minute, time.Hour, &c) + require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", "")) + + return testsetup{ + c: c, + fw: fw, + myVpnNetworksTable: myVpnNetworksTable, + } +} + +func TestFirewall_Drop_EnforceIPMatch(t *testing.T) { + t.Parallel() + ob := &bytes.Buffer{} + l := test.NewLoggerWithOutput(ob) + + myPrefix := netip.MustParsePrefix("1.1.1.1/8") + // for now, it's okay that these are all "incoming", the logic this test tries to check doesn't care about in/out + t.Run("allow inbound all matching", func(t *testing.T) { + t.Parallel() + setup := newSetup(t, l, myPrefix) + tc := buildTestCase(setup, nil, netip.MustParsePrefix("1.2.3.4/24")) + tc.Test(t, setup.fw) + }) + t.Run("allow inbound local matching", func(t *testing.T) { + t.Parallel() + setup := newSetup(t, l, myPrefix) + tc := buildTestCase(setup, ErrInvalidLocalIP, netip.MustParsePrefix("1.2.3.4/24")) + tc.p.LocalAddr = netip.MustParseAddr("1.2.3.8") + tc.Test(t, setup.fw) + }) + t.Run("block inbound remote mismatched", func(t *testing.T) { + t.Parallel() + setup := newSetup(t, l, myPrefix) + tc := buildTestCase(setup, ErrInvalidRemoteIP, netip.MustParsePrefix("1.2.3.4/24")) + tc.p.RemoteAddr = netip.MustParseAddr("9.9.9.9") + tc.Test(t, setup.fw) + }) + t.Run("Block a vpn peer packet", func(t *testing.T) { + t.Parallel() + setup := newSetup(t, l, myPrefix) + tc := buildTestCase(setup, ErrPeerRejected, netip.MustParsePrefix("2.2.2.2/24")) + tc.Test(t, setup.fw) + }) + twoPrefixes := []netip.Prefix{ + netip.MustParsePrefix("1.2.3.4/24"), netip.MustParsePrefix("2.2.2.2/24"), + } + t.Run("allow inbound one matching", func(t *testing.T) { + t.Parallel() + setup := newSetup(t, l, myPrefix) + tc := buildTestCase(setup, nil, twoPrefixes...) + tc.Test(t, setup.fw) + }) + t.Run("block inbound multimismatch", func(t *testing.T) { + t.Parallel() + setup := newSetup(t, l, myPrefix) + tc := buildTestCase(setup, ErrInvalidRemoteIP, twoPrefixes...) + tc.p.RemoteAddr = netip.MustParseAddr("9.9.9.9") + tc.Test(t, setup.fw) + }) + t.Run("allow inbound 2nd one matching", func(t *testing.T) { + t.Parallel() + setup2 := newSetup(t, l, netip.MustParsePrefix("2.2.2.1/24")) + tc := buildTestCase(setup2, nil, twoPrefixes...) + tc.p.RemoteAddr = twoPrefixes[1].Addr() + tc.Test(t, setup2.fw) + }) + t.Run("allow inbound unsafe route", func(t *testing.T) { + t.Parallel() + unsafePrefix := netip.MustParsePrefix("192.168.0.0/24") + c := dummyCert{ + name: "me", + networks: []netip.Prefix{myPrefix}, + unsafeNetworks: []netip.Prefix{unsafePrefix}, + groups: []string{"default-group"}, + issuer: "signer-shasum", + } + unsafeSetup := newSetupFromCert(t, l, c) + tc := buildTestCase(unsafeSetup, nil, twoPrefixes...) + tc.p.LocalAddr = netip.MustParseAddr("192.168.0.3") + tc.err = ErrNoMatchingRule + tc.Test(t, unsafeSetup.fw) //should hit firewall and bounce off + require.NoError(t, unsafeSetup.fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", unsafePrefix.String(), "", "")) + tc.err = nil + tc.Test(t, unsafeSetup.fw) //should pass + }) } type addRuleCall struct { @@ -964,8 +1709,8 @@ type addRuleCall struct { endPort int32 groups []string host string - ip *net.IPNet - localIp *net.IPNet + ip string + localIp string caName string caSha string } @@ -975,7 +1720,7 @@ type mockFirewall struct { nextCallReturn error } -func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip *net.IPNet, localIp *net.IPNet, caName string, caSha string) error { +func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip, localIp, caName string, caSha string) error { mf.lastCall = addRuleCall{ incoming: incoming, proto: proto, diff --git a/go.mod b/go.mod index abc1134e..9748492d 100644 --- a/go.mod +++ b/go.mod @@ -1,52 +1,56 @@ module github.com/slackhq/nebula -go 1.20 +go 1.26.0 require ( - dario.cat/mergo v1.0.0 + dario.cat/mergo v1.0.2 + filippo.io/bigmod v0.1.0 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be github.com/armon/go-radix v1.0.0 github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 - github.com/flynn/noise v1.0.1 + github.com/flynn/noise v1.1.0 + github.com/gaissmai/bart v0.28.0 github.com/gogo/protobuf v1.3.2 github.com/google/gopacket v1.1.19 - github.com/kardianos/service v1.2.2 - github.com/miekg/dns v1.1.58 + github.com/kardianos/service v1.3.0 + github.com/miekg/dns v1.1.72 + github.com/miekg/pkcs11 v1.1.2 github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f - github.com/prometheus/client_golang v1.18.0 + github.com/prometheus/client_golang v1.23.2 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 - github.com/sirupsen/logrus v1.9.3 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e - github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 - github.com/stretchr/testify v1.8.4 - github.com/vishvananda/netlink v1.2.1-beta.2 - golang.org/x/crypto v0.18.0 - golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 - golang.org/x/net v0.20.0 - golang.org/x/sync v0.6.0 - golang.org/x/sys v0.16.0 - golang.org/x/term v0.16.0 + github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 + github.com/stretchr/testify v1.11.1 + github.com/vishvananda/netlink v1.3.1 + go.uber.org/goleak v1.3.0 + go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/crypto v0.54.0 + golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 + golang.org/x/net v0.57.0 + golang.org/x/sync v0.22.0 + golang.org/x/sys v0.47.0 + golang.org/x/term v0.45.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b - golang.zx2c4.com/wireguard/windows v0.5.3 - google.golang.org/protobuf v1.32.0 - gopkg.in/yaml.v2 v2.4.0 - gvisor.dev/gvisor v0.0.0-20230504175454-7b0a1988a28f + golang.zx2c4.com/wireguard/windows v1.0.1 + google.golang.org/protobuf v1.36.11 + gopkg.in/yaml.v3 v3.0.1 + gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe ) require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/google/btree v1.0.1 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect - github.com/vishvananda/netns v0.0.4 // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - golang.org/x/tools v0.17.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/vishvananda/netns v0.0.5 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.45.0 // indirect ) diff --git a/go.sum b/go.sum index 6226e150..29d68429 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8= +filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -15,15 +17,17 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 h1:M5QgkYacWj0Xs8MhpIK/5uwU02icXpEoSo9sM2aRCps= github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.mod h1:xwIwAxMvYnVrGJPe2FKx5prTrnAjGOD8zvDOnxnrrkM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/flynn/noise v1.0.1 h1:vPp/jdQLXC6ppsXSj/pM3W1BIJ5FEHE2TulSJBpb43Y= -github.com/flynn/noise v1.0.1/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= +github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/gaissmai/bart v0.28.0 h1:89yZLo8NmyqD0RYgJ3QO9HhqqGGw+oWhf90cZm69Lko= +github.com/gaissmai/bart v0.28.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -44,14 +48,15 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= @@ -61,28 +66,35 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kardianos/service v1.2.2 h1:ZvePhAHfvo0A7Mftk/tEzqEZ7Q4lgnR8sGz4xu1YX60= -github.com/kardianos/service v1.2.2/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM= +github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFdTI= +github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= -github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= -github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/miekg/pkcs11 v1.1.2 h1:/VxmeAX5qU6Q3EwafypogwWbYryHFmF2RpkJmw3m4MQ= +github.com/miekg/pkcs11 v1.1.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f h1:8dM0ilqKL0Uzl42GABzzC4Oqlc3kGRILz0vgoff7nwg= @@ -96,66 +108,70 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= -github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= +github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw= +github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= -github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= -github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= +github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= +github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= +github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -166,8 +182,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -175,42 +191,40 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -219,8 +233,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo= golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4= -golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE= -golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI= +golang.zx2c4.com/wireguard/windows v1.0.1 h1:eOxiDVbywPC+ZQqvdCK7x+ZwWXKbYv50TtH8ysFIbw8= +golang.zx2c4.com/wireguard/windows v1.0.1/go.mod h1:+fbT3FFdX4zzYDLwJh5+HPEcNN/3HyNdzhNSVsQM+zs= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -229,8 +243,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -241,10 +255,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gvisor.dev/gvisor v0.0.0-20230504175454-7b0a1988a28f h1:8GE2MRjGiFmfpon8dekPI08jEuNMQzSffVHgdupcO4E= -gvisor.dev/gvisor v0.0.0-20230504175454-7b0a1988a28f/go.mod h1:pzr6sy8gDLfVmDAg8OYrlKvGEHw5C3PGTiBXBTCx76Q= +gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g= +gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU= diff --git a/handshake/credential.go b/handshake/credential.go new file mode 100644 index 00000000..f6cd5f41 --- /dev/null +++ b/handshake/credential.go @@ -0,0 +1,57 @@ +package handshake + +import ( + "crypto/rand" + + "github.com/flynn/noise" + "github.com/slackhq/nebula/cert" +) + +// Credential holds everything needed to participate in a handshake +// at a given cert version. Version and Curve are read from Cert; the public +// half of the static keypair likewise comes from Cert.PublicKey(). +type Credential struct { + Cert cert.Certificate // the certificate + Bytes []byte // pre-marshaled certificate bytes + privateKey []byte // static private key (public half lives in Cert) + cipherSuite noise.CipherSuite // pre-built cipher suite (DH + cipher + hash) +} + +// NewCredential creates a Credential with all material needed for handshake +// participation. The cipherSuite should be pre-built by the caller with the +// appropriate DH function, cipher, and hash. +func NewCredential( + c cert.Certificate, + hsBytes []byte, + privateKey []byte, + cipherSuite noise.CipherSuite, +) *Credential { + return &Credential{ + Cert: c, + Bytes: hsBytes, + privateKey: privateKey, + cipherSuite: cipherSuite, + } +} + +// buildHandshakeState creates a noise.HandshakeState from this credential. +func (hc *Credential) buildHandshakeState(initiator bool, pattern noise.HandshakePattern) (*noise.HandshakeState, error) { + return noise.NewHandshakeState(noise.Config{ + CipherSuite: hc.cipherSuite, + Random: rand.Reader, + Pattern: pattern, + Initiator: initiator, + StaticKeypair: noise.DHKey{Private: hc.privateKey, Public: hc.Cert.PublicKey()}, + PresharedKey: []byte{}, + PresharedKeyPlacement: 0, + }) +} + +// GetCredentialFunc returns the handshake credential for the given version, +// or nil if that version is not available. +// +// Implementations must return credentials drawn from a snapshot stable for +// the lifetime of any single Machine. The Machine may call this multiple +// times during a handshake (e.g. when negotiating to the peer's version) +// and assumes the underlying static keypair is consistent across calls. +type GetCredentialFunc func(v cert.Version) *Credential diff --git a/handshake/errors.go b/handshake/errors.go new file mode 100644 index 00000000..3bdcc947 --- /dev/null +++ b/handshake/errors.go @@ -0,0 +1,22 @@ +package handshake + +import "errors" + +var ( + ErrInitiateOnResponder = errors.New("initiate called on responder") + ErrInitiateAlreadyCalled = errors.New("initiate already called") + ErrInitiateNotCalled = errors.New("initiate must be called before ProcessPacket for initiators") + ErrPacketTooShort = errors.New("packet too short") + ErrPublicKeyMismatch = errors.New("public key mismatch between certificate and handshake") + ErrIncompleteHandshake = errors.New("handshake completed without receiving required content") + ErrMachineFailed = errors.New("handshake machine has failed") + ErrUnknownSubtype = errors.New("unknown handshake subtype") + ErrMissingContent = errors.New("expected handshake content but message was empty") + ErrUnexpectedContent = errors.New("received unexpected handshake content") + ErrInvalidRemoteIndex = errors.New("peer sent an invalid index in handshake payload") + ErrIndexAllocation = errors.New("failed to allocate local index") + ErrNoCredential = errors.New("no handshake credential available for cert version") + ErrAsymmetricCipherKeys = errors.New("noise produced only one cipher key") + ErrMultiMessageUnsupported = errors.New("multi-message handshake patterns are not yet supported by the manager") + ErrSubtypeMismatch = errors.New("packet subtype does not match handshake machine subtype") +) diff --git a/handshake/handshake.proto b/handshake/handshake.proto new file mode 100644 index 00000000..8eb32aa6 --- /dev/null +++ b/handshake/handshake.proto @@ -0,0 +1,29 @@ +// This file documents the wire format the nebula handshake speaks. It is +// not run through protoc; the encoder/decoder in payload.go is hand-written +// against this shape directly to keep the parser narrow and panic-free. +// +// Any change to the wire format must be reflected here, and adding a new +// field requires updating MarshalPayload / unmarshalPayloadDetails together +// with the field-uniqueness and wire-type checks in those functions. + +syntax = "proto3"; +package nebula.handshake; + +message NebulaHandshake { + NebulaHandshakeDetails Details = 1; + bytes Hmac = 2; +} + +message NebulaHandshakeDetails { + bytes Cert = 1; + uint32 InitiatorIndex = 2; + uint32 ResponderIndex = 3; + // Cookie was reserved for an anti-DoS mechanism that was never + // implemented. No released version of nebula has ever populated it; the + // hand-written parser silently skips it on read. + uint64 Cookie = 4 [deprecated = true]; + uint64 Time = 5; + uint32 CertVersion = 8; + // reserved for WIP multiport + reserved 6, 7; +} diff --git a/handshake/helpers_test.go b/handshake/helpers_test.go new file mode 100644 index 00000000..c72346cb --- /dev/null +++ b/handshake/helpers_test.go @@ -0,0 +1,116 @@ +package handshake + +import ( + "net/netip" + "testing" + "time" + + "github.com/flynn/noise" + "github.com/slackhq/nebula/cert" + ct "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/header" + "github.com/stretchr/testify/require" +) + +// testCertState holds cert material for a test peer. +type testCertState struct { + version cert.Version + creds map[cert.Version]*Credential +} + +func (s *testCertState) getCredential(v cert.Version) *Credential { + return s.creds[v] +} + +func newTestCertState( + t *testing.T, ca cert.Certificate, caKey []byte, name string, networks []netip.Prefix, +) *testCertState { + return newTestCertStateWithCipher(t, ca, caKey, name, networks, noise.CipherChaChaPoly) +} + +func newTestCertStateWithCipher( + t *testing.T, ca cert.Certificate, caKey []byte, name string, networks []netip.Prefix, + cipher noise.CipherFunc, +) *testCertState { + t.Helper() + c, _, rawPrivKey, _ := ct.NewTestCert( + cert.Version2, cert.Curve_CURVE25519, ca, caKey, + name, ca.NotBefore(), ca.NotAfter(), networks, nil, nil, + ) + + priv, _, _, err := cert.UnmarshalPrivateKeyFromPEM(rawPrivKey) + require.NoError(t, err) + + hsBytes, err := c.MarshalForHandshakes() + require.NoError(t, err) + + ncs := noise.NewCipherSuite(noise.DH25519, cipher, noise.HashSHA256) + return &testCertState{ + version: cert.Version2, + creds: map[cert.Version]*Credential{ + cert.Version2: NewCredential(c, hsBytes, priv, ncs), + }, + } +} + +func testVerifier(pool *cert.CAPool) CertVerifier { + return func(c cert.Certificate) (*cert.CachedCertificate, error) { + return pool.VerifyCertificate(time.Now(), c) + } +} + +func newTestMachine( + t *testing.T, + cs *testCertState, + verifier CertVerifier, + initiator bool, + localIndex uint32, +) *Machine { + t.Helper() + m, err := NewMachine( + cs.version, cs.getCredential, + verifier, func() (uint32, error) { return localIndex, nil }, + initiator, header.HandshakeIXPSK0, + ) + require.NoError(t, err) + return m +} + +func initiateHandshake( + t *testing.T, + initCS *testCertState, initVerifier CertVerifier, + respCS *testCertState, respVerifier CertVerifier, +) (initM, respM *Machine, respResult *Result, resp []byte, err error) { + t.Helper() + initM = newTestMachine(t, initCS, initVerifier, true, 100) + msg1, merr := initM.Initiate(nil) + require.NoError(t, merr) + + respM = newTestMachine(t, respCS, respVerifier, false, 200) + resp, respResult, err = respM.ProcessPacket(nil, msg1) + return +} + +func doFullHandshake( + t *testing.T, initCS, respCS *testCertState, caPool *cert.CAPool, +) (initResult, respResult *Result) { + t.Helper() + v := testVerifier(caPool) + + initM := newTestMachine(t, initCS, v, true, 1000) + respM := newTestMachine(t, respCS, v, false, 2000) + + msg1, err := initM.Initiate(nil) + require.NoError(t, err) + + resp, respResult, err := respM.ProcessPacket(nil, msg1) + require.NoError(t, err) + require.NotNil(t, respResult) + require.NotEmpty(t, resp) + + _, initResult, err = initM.ProcessPacket(nil, resp) + require.NoError(t, err) + require.NotNil(t, initResult) + + return initResult, respResult +} diff --git a/handshake/machine.go b/handshake/machine.go new file mode 100644 index 00000000..baf61589 --- /dev/null +++ b/handshake/machine.go @@ -0,0 +1,454 @@ +package handshake + +import ( + "bytes" + "fmt" + "slices" + "time" + + "github.com/flynn/noise" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/header" +) + +// IndexAllocator is called by the Machine to allocate a local index for the +// handshake. It is called at most once, when the first outgoing message that +// carries a payload is built. +// +// Implementations MUST NOT return 0. Zero is reserved as a sentinel meaning +// "no index assigned" on the wire and in the payload-presence checks. If an +// allocator ever returned 0, a legitimate handshake's payload could be +// indistinguishable from an empty one and would be rejected. +type IndexAllocator func() (uint32, error) + +// CertVerifier is called by the Machine after reconstructing the peer's +// certificate from the handshake. The verifier performs all validation +// (CA trust, expiry, policy checks, allow lists). +type CertVerifier func(cert.Certificate) (*cert.CachedCertificate, error) + +// Result contains the results of a successful handshake. +// Returned by ProcessPacket when the handshake is complete. +type Result struct { + EKey *noise.CipherState + DKey *noise.CipherState + Cipher noise.CipherFunc // identifies which post-handshake CipherState the data plane should wrap EKey/DKey in + MyCert cert.Certificate + RemoteCert *cert.CachedCertificate + RemoteIndex uint32 + LocalIndex uint32 + HandshakeTime uint64 + MessageIndex uint64 // number of messages exchanged during the handshake + Initiator bool +} + +// Machine drives a Noise handshake through N messages. It handles Noise +// protocol operations, certificate reconstruction, and payload encoding. +// Certificate validation is delegated to the caller via CertVerifier. +// +// A Machine is not safe for concurrent use. The caller must ensure that +// Initiate and ProcessPacket are not called concurrently. +// +// Error contract: when ProcessPacket or Initiate returns an error, callers +// must check Failed() to decide what to do next. If Failed() is false the +// underlying noise state was not advanced (the packet was rejected before +// ReadMessage took effect, or the rejection is non-fatal like a stale +// retransmit) and the Machine can accept another packet. If Failed() is +// true the Machine is unrecoverable and the caller must abandon it. +type Machine struct { + hs *noise.HandshakeState + getCred GetCredentialFunc + allocIndex IndexAllocator + verifier CertVerifier + result *Result + msgs []msgFlags + myVersion cert.Version + subtype header.MessageSubType + indexAllocated bool + remoteCertSet bool + payloadSet bool + failed bool +} + +// NewMachine creates a handshake state machine. The subtype determines both +// the noise pattern and the per-message content layout. The credential for +// `version` is fetched via getCred and used to seed the noise.HandshakeState. +// IndexAllocator is called lazily when the first outgoing payload is built. +func NewMachine( + version cert.Version, + getCred GetCredentialFunc, + verifier CertVerifier, + allocIndex IndexAllocator, + initiator bool, + subtype header.MessageSubType, +) (*Machine, error) { + info, err := subtypeInfoFor(subtype) + if err != nil { + return nil, err + } + + cred := getCred(version) + if cred == nil { + return nil, fmt.Errorf("%w: %v", ErrNoCredential, version) + } + + hs, err := cred.buildHandshakeState(initiator, info.pattern) + if err != nil { + return nil, fmt.Errorf("build noise state: %w", err) + } + + return &Machine{ + hs: hs, + subtype: subtype, + msgs: info.msgs, + getCred: getCred, + allocIndex: allocIndex, + verifier: verifier, + myVersion: version, + result: &Result{ + Initiator: initiator, + Cipher: cred.cipherSuite, + }, + }, nil +} + +// Failed returns true if the Machine is in an unrecoverable state. +func (m *Machine) Failed() bool { + return m.failed +} + +// Subtype returns the handshake subtype this Machine was built for. +func (m *Machine) Subtype() header.MessageSubType { + return m.subtype +} + +// MessageIndex returns the noise handshake message index, which equals the +// wire counter of the most recently sent or received message. +func (m *Machine) MessageIndex() int { + return m.hs.MessageIndex() +} + +// requireComplete checks that both a peer cert and payload have been received. +// Marks the machine as failed if not. +func (m *Machine) requireComplete() error { + if !m.payloadSet || !m.remoteCertSet { + m.failed = true + return ErrIncompleteHandshake + } + return nil +} + +// myMsgFlags returns the flags for the current outgoing message. +func (m *Machine) myMsgFlags() msgFlags { + idx := m.hs.MessageIndex() + if idx < len(m.msgs) { + return m.msgs[idx] + } + return msgFlags{} +} + +// peerMsgFlags returns the flags for the message we just read. +func (m *Machine) peerMsgFlags() msgFlags { + idx := m.hs.MessageIndex() - 1 + if idx >= 0 && idx < len(m.msgs) { + return m.msgs[idx] + } + return msgFlags{} +} + +// Initiate produces the first handshake message. Only valid for initiators, +// and must be called exactly once before ProcessPacket. +// +// out is a destination buffer the message is appended to and returned. Pass +// nil to allocate fresh, or pass a re-used buffer sliced to length 0 (e.g. +// buf[:0]) with sufficient capacity to avoid allocation. +// +// An error return may not indicate a fatal condition, check Failed() to +// determine if the Machine can still be used. +func (m *Machine) Initiate(out []byte) ([]byte, error) { + if m.failed { + return nil, ErrMachineFailed + } + if !m.result.Initiator { + m.failed = true + return nil, ErrInitiateOnResponder + } + if m.hs.MessageIndex() != 0 { + m.failed = true + return nil, ErrInitiateAlreadyCalled + } + + // At MessageIndex=0 with RemoteIndex still zero, buildResponse produces + // header counter 1 and remote index 0, which is what the initial message needs. + out, _, _, err := m.buildResponse(out) + if err != nil { + m.failed = true + return nil, err + } + return out, nil +} + +// ProcessPacket handles an incoming handshake message. It advances the Noise +// state, validates the peer certificate via the verifier, and optionally +// produces a response. +// +// out is a destination buffer the response is appended to and returned. Pass +// nil to allocate fresh, or pass a re-used buffer sliced to length 0 (e.g. +// buf[:0]) with sufficient capacity to avoid allocation. The returned slice +// is nil when no outgoing message is produced (handshake complete on this +// side, or final message of a multi-message pattern). +// +// Returns a non-nil Result when the handshake is complete. +// An error return may not indicate a fatal condition, check Failed() to +// determine if the Machine can still be used. +func (m *Machine) ProcessPacket(out, packet []byte) ([]byte, *Result, error) { + if m.failed { + return nil, nil, ErrMachineFailed + } + if len(packet) < header.Len { + return nil, nil, ErrPacketTooShort + } + // Reject packets whose subtype doesn't match the one this Machine was + // built for. A pending handshake that suddenly receives a different + // subtype on its index is either a stray packet that matched by chance + // or a peer protocol violation; drop it without failing the Machine so + // the legitimate retransmit can still complete. + if header.MessageSubType(packet[1]) != m.subtype { + return nil, nil, ErrSubtypeMismatch + } + if m.result.Initiator && m.hs.MessageIndex() == 0 { + m.failed = true + return nil, nil, ErrInitiateNotCalled + } + + // The (eKey, dKey) ordering here is correct for IX, where the initiator + // completes the handshake by reading the responder's stage-2 message. + // noise returns (cs1, cs2) where cs1 is the initiator->responder cipher. + // For 3-message patterns where a responder finishes by reading the final + // message, this ordering would be wrong; revisit when XX/pqIX lands. + msg, eKey, dKey, err := m.hs.ReadMessage(nil, packet[header.Len:]) + if err != nil { + // Noise ReadMessage failed. The noise library checkpoints and rolls back + // on failure, so the Machine is still alive. The caller can retry with + // a different packet. + return nil, nil, fmt.Errorf("noise ReadMessage: %w", err) + } + + // From here on, noise state has advanced. Any error is fatal. + flags := m.peerMsgFlags() + + if err := m.processPayload(msg, flags); err != nil { + return nil, nil, err + } + + // If ReadMessage derived keys, the handshake is complete. Noise should + // always produce both keys together; asymmetry is a protocol invariant + // violation. + if eKey != nil || dKey != nil { + if eKey == nil || dKey == nil { + m.failed = true + return nil, nil, ErrAsymmetricCipherKeys + } + if err := m.requireComplete(); err != nil { + return nil, nil, err + } + return nil, m.completed(eKey, dKey), nil + } + + // ReadMessage didn't complete, produce the next outgoing message + out, dk, ek, err := m.buildResponse(out) + if err != nil { + m.failed = true + return nil, nil, err + } + + if ek != nil || dk != nil { + if ek == nil || dk == nil { + m.failed = true + return nil, nil, ErrAsymmetricCipherKeys + } + if err := m.requireComplete(); err != nil { + return nil, nil, err + } + return out, m.completed(ek, dk), nil + } + + return out, nil, nil +} + +func (m *Machine) completed(eKey, dKey *noise.CipherState) *Result { + m.result.EKey = eKey + m.result.DKey = dKey + m.result.MessageIndex = uint64(m.hs.MessageIndex()) + return m.result +} + +func (m *Machine) processPayload(msg []byte, flags msgFlags) error { + if len(msg) == 0 { + if flags.expectsPayload || flags.expectsCert { + m.failed = true + return ErrMissingContent + } + return nil + } + + payload, err := UnmarshalPayload(msg) + if err != nil { + m.failed = true + return fmt.Errorf("unmarshal handshake: %w", err) + } + + // Assert the payload contains exactly what we expect + hasPayloadData := payload.InitiatorIndex != 0 || payload.ResponderIndex != 0 || payload.Time != 0 + if hasPayloadData != flags.expectsPayload { + m.failed = true + return ErrUnexpectedContent + } + + hasCertData := len(payload.Cert) > 0 + if hasCertData != flags.expectsCert { + m.failed = true + return ErrUnexpectedContent + } + + // Process payload + if flags.expectsPayload { + var remoteIndex uint32 + if m.result.Initiator { + remoteIndex = payload.ResponderIndex + } else { + remoteIndex = payload.InitiatorIndex + } + // The payload presence check above can be satisfied by Time alone, so a payload + // could still carry a zero index here. We need to reject it. + if remoteIndex == 0 { + m.failed = true + return ErrInvalidRemoteIndex + } + m.result.RemoteIndex = remoteIndex + m.result.HandshakeTime = payload.Time + m.payloadSet = true + } + + // Process certificate + if flags.expectsCert { + if err := m.validateCert(payload); err != nil { + return err + } + } + + return nil +} + +func (m *Machine) validateCert(payload Payload) error { + cred := m.getCred(m.myVersion) + if cred == nil { + m.failed = true + return fmt.Errorf("%w: %v", ErrNoCredential, m.myVersion) + } + rc, err := cert.Recombine( + cert.Version(payload.CertVersion), + payload.Cert, + m.hs.PeerStatic(), + cred.Cert.Curve(), + ) + if err != nil { + m.failed = true + return fmt.Errorf("recombine cert: %w", err) + } + + if !bytes.Equal(rc.PublicKey(), m.hs.PeerStatic()) { + m.failed = true + return ErrPublicKeyMismatch + } + + // Version negotiation, if the peer sent a different version and we have it, switch + if rc.Version() != m.myVersion { + if m.getCred(rc.Version()) != nil { + m.myVersion = rc.Version() + } + } + + verified, err := m.verifier(rc) + if err != nil { + m.failed = true + return fmt.Errorf("verify cert: %w", err) + } + + m.result.RemoteCert = verified + m.remoteCertSet = true + return nil +} + +func (m *Machine) marshalOutgoing(flags msgFlags) ([]byte, error) { + if !flags.expectsPayload && !flags.expectsCert { + return nil, nil + } + + var p Payload + if flags.expectsPayload { + if !m.indexAllocated { + index, err := m.allocIndex() + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrIndexAllocation, err) + } + m.result.LocalIndex = index + m.indexAllocated = true + } + + if m.result.Initiator { + p.InitiatorIndex = m.result.LocalIndex + } else { + p.ResponderIndex = m.result.LocalIndex + p.InitiatorIndex = m.result.RemoteIndex + } + p.Time = uint64(time.Now().UnixNano()) + } + if flags.expectsCert { + cred := m.getCred(m.myVersion) + if cred == nil { + return nil, fmt.Errorf("%w: %v", ErrNoCredential, m.myVersion) + } + p.Cert = cred.Bytes + p.CertVersion = uint32(cred.Cert.Version()) + m.result.MyCert = cred.Cert + } + + return MarshalPayload(nil, p), nil +} + +func (m *Machine) buildResponse(out []byte) ([]byte, *noise.CipherState, *noise.CipherState, error) { + flags := m.myMsgFlags() + hsBytes, err := m.marshalOutgoing(flags) + if err != nil { + return nil, nil, nil, err + } + + // Extend out by header.Len to make room for the header. slices.Grow is a + // no-op when the cap is already sufficient (the zero-copy case where the + // caller passed a pre-sized buffer). header.Encode overwrites the new + // bytes, so they don't need to be zeroed. + start := len(out) + out = slices.Grow(out, header.Len)[:start+header.Len] + header.Encode( + out[start:], + header.Version, header.Handshake, m.subtype, + m.result.RemoteIndex, + uint64(m.hs.MessageIndex()+1), + ) + + // noise.WriteMessage appends the encrypted handshake message to out, + // reusing capacity when present. + // + // The (dKey, eKey) ordering here is correct for IX, where the responder + // completes the handshake by writing the stage-2 message. noise returns + // (cs1, cs2) where cs1 is the initiator->responder cipher (which is the + // responder's decrypt key). For 3-message patterns where an initiator + // finishes by writing the final message, this ordering would be wrong; + // revisit when XX/pqIX lands. + out, dKey, eKey, err := m.hs.WriteMessage(out, hsBytes) + if err != nil { + return nil, nil, nil, fmt.Errorf("noise WriteMessage: %w", err) + } + + return out, dKey, eKey, nil +} diff --git a/handshake/machine_test.go b/handshake/machine_test.go new file mode 100644 index 00000000..01c968ed --- /dev/null +++ b/handshake/machine_test.go @@ -0,0 +1,680 @@ +package handshake + +import ( + "net/netip" + "testing" + "time" + + "github.com/flynn/noise" + "github.com/slackhq/nebula/cert" + ct "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/header" + "github.com/slackhq/nebula/noiseutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMachineIXHappyPath(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + + initCS := newTestCertState(t, ca, caKey, "initiator", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + respCS := newTestCertState(t, ca, caKey, "responder", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}) + + initR, respR := doFullHandshake(t, initCS, respCS, caPool) + + assert.Equal(t, "responder", initR.RemoteCert.Certificate.Name()) + assert.Equal(t, "initiator", respR.RemoteCert.Certificate.Name()) + + assert.Equal(t, uint32(1000), initR.LocalIndex) + assert.Equal(t, uint32(2000), initR.RemoteIndex) + assert.Equal(t, uint32(2000), respR.LocalIndex) + assert.Equal(t, uint32(1000), respR.RemoteIndex) + + assert.Equal(t, uint64(2), initR.MessageIndex, "IX has 2 messages") + assert.Equal(t, uint64(2), respR.MessageIndex, "IX has 2 messages") + + ct1, err := initR.EKey.Encrypt(nil, nil, []byte("hello")) + require.NoError(t, err) + pt1, err := respR.DKey.Decrypt(nil, nil, ct1) + require.NoError(t, err) + assert.Equal(t, []byte("hello"), pt1) + + ct2, err := respR.EKey.Encrypt(nil, nil, []byte("world")) + require.NoError(t, err) + pt2, err := initR.DKey.Decrypt(nil, nil, ct2) + require.NoError(t, err) + assert.Equal(t, []byte("world"), pt2) +} + +func TestMachineInitiateErrors(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + cs := newTestCertState(t, ca, caKey, "test", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + v := testVerifier(caPool) + + t.Run("initiate on responder", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + _, err := m.Initiate(nil) + require.ErrorIs(t, err, ErrInitiateOnResponder) + assert.True(t, m.Failed()) + }) + + t.Run("initiate called twice", func(t *testing.T) { + m := newTestMachine(t, cs, v, true, 100) + _, err := m.Initiate(nil) + require.NoError(t, err) + _, err = m.Initiate(nil) + require.ErrorIs(t, err, ErrInitiateAlreadyCalled) + assert.True(t, m.Failed()) + }) + + t.Run("process packet before initiate on initiator", func(t *testing.T) { + m := newTestMachine(t, cs, v, true, 100) + _, _, err := m.ProcessPacket(nil, make([]byte, 100)) + require.ErrorIs(t, err, ErrInitiateNotCalled) + assert.True(t, m.Failed()) + }) + + t.Run("calling failed machine", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + _, err := m.Initiate(nil) // fails: responder + require.Error(t, err) + _, err = m.Initiate(nil) // fails: already failed + require.ErrorIs(t, err, ErrMachineFailed) + }) +} + +func TestMachineProcessPacketErrors(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + cs := newTestCertState(t, ca, caKey, "test", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + v := testVerifier(caPool) + + t.Run("packet too short", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + _, _, err := m.ProcessPacket(nil, []byte{1, 2, 3}) + require.ErrorIs(t, err, ErrPacketTooShort) + assert.False(t, m.Failed(), "short packet should not kill machine") + }) + + t.Run("noise decryption failure is recoverable", func(t *testing.T) { + initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + initM := newTestMachine(t, initCS, v, true, 100) + msg1, err := initM.Initiate(nil) + require.NoError(t, err) + + respM := newTestMachine(t, cs, v, false, 200) + resp, _, err := respM.ProcessPacket(nil, msg1) + require.NoError(t, err) + + corrupted := make([]byte, len(resp)) + copy(corrupted, resp) + for i := header.Len; i < len(corrupted); i++ { + corrupted[i] ^= 0xff + } + _, _, err = initM.ProcessPacket(nil, corrupted) + require.Error(t, err) + assert.False(t, initM.Failed(), "noise failure should be recoverable") + + // And the machine should still complete a real handshake afterward. + _, result, err := initM.ProcessPacket(nil, resp) + require.NoError(t, err) + require.NotNil(t, result, "initiator should complete on the legitimate response") + }) + + t.Run("invalid cert is fatal", func(t *testing.T) { + otherCA, _, otherCAKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + otherCS := newTestCertState(t, otherCA, otherCAKey, "other", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}) + + initM := newTestMachine(t, otherCS, testVerifier(ct.NewTestCAPool(otherCA)), true, 100) + msg1, err := initM.Initiate(nil) + require.NoError(t, err) + + respM := newTestMachine(t, cs, v, false, 200) + _, _, err = respM.ProcessPacket(nil, msg1) + require.Error(t, err) + assert.True(t, respM.Failed(), "cert validation failure should kill machine") + }) + + t.Run("subtype mismatch is recoverable", func(t *testing.T) { + initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + initM := newTestMachine(t, initCS, v, true, 100) + msg1, err := initM.Initiate(nil) + require.NoError(t, err) + + // Mutate the subtype byte (offset 1 in the header) to a value the + // responder Machine wasn't built for. + bad := make([]byte, len(msg1)) + copy(bad, msg1) + bad[1] = 0xff + + respM := newTestMachine(t, cs, v, false, 200) + _, _, err = respM.ProcessPacket(nil, bad) + require.ErrorIs(t, err, ErrSubtypeMismatch) + assert.False(t, respM.Failed(), "subtype mismatch should not kill the machine") + + // And the machine should still complete a real handshake afterward. + resp, result, err := respM.ProcessPacket(nil, msg1) + require.NoError(t, err) + require.NotNil(t, result, "responder should complete on the legitimate stage-1 packet") + assert.NotEmpty(t, resp, "responder should produce a stage-2 reply") + }) +} + +// TestMachineProcessPayload exercises processPayload's internal validation +// directly. Most of these failure modes can't be reached black-box once the +// subtype check at the top of ProcessPacket gates external callers, so we +// drive them by hand here for coverage. +func TestMachineProcessPayload(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + cs := newTestCertState(t, ca, caKey, "test", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + v := testVerifier(caPool) + + t.Run("empty message with expects fails", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + err := m.processPayload(nil, msgFlags{expectsPayload: true, expectsCert: true}) + require.ErrorIs(t, err, ErrMissingContent) + assert.True(t, m.Failed()) + }) + + t.Run("empty message with no expects passes", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + err := m.processPayload(nil, msgFlags{}) + require.NoError(t, err) + assert.False(t, m.Failed()) + }) + + t.Run("malformed protobuf is fatal", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + err := m.processPayload([]byte{0xff, 0xff, 0xff}, msgFlags{expectsPayload: true, expectsCert: true}) + require.Error(t, err) + assert.True(t, m.Failed()) + }) + + t.Run("unexpected payload data is fatal", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + // A payload with index data when none was expected. + bytes := MarshalPayload(nil, Payload{InitiatorIndex: 42, Time: 1}) + err := m.processPayload(bytes, msgFlags{expectsPayload: false, expectsCert: false}) + require.ErrorIs(t, err, ErrUnexpectedContent) + assert.True(t, m.Failed()) + }) + + t.Run("unexpected cert data is fatal", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + // A payload with cert when none was expected. + bytes := MarshalPayload(nil, Payload{Cert: []byte{1, 2, 3}, CertVersion: 2}) + err := m.processPayload(bytes, msgFlags{expectsPayload: false, expectsCert: false}) + require.ErrorIs(t, err, ErrUnexpectedContent) + assert.True(t, m.Failed()) + }) + + t.Run("missing payload data when expected is fatal", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + // Cert present, but no index/time fields. + bytes := MarshalPayload(nil, Payload{Cert: []byte{1, 2, 3}, CertVersion: 2}) + err := m.processPayload(bytes, msgFlags{expectsPayload: true, expectsCert: true}) + require.ErrorIs(t, err, ErrUnexpectedContent) + assert.True(t, m.Failed()) + }) + + t.Run("zero initiator index on responder is fatal", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + bytes := MarshalPayload(nil, Payload{InitiatorIndex: 0, Time: 1}) + err := m.processPayload(bytes, msgFlags{expectsPayload: true}) + require.ErrorIs(t, err, ErrInvalidRemoteIndex) + assert.True(t, m.Failed()) + assert.Zero(t, m.result.RemoteIndex) + }) + + t.Run("zero responder index on initiator is fatal", func(t *testing.T) { + m := newTestMachine(t, cs, v, true, 100) + bytes := MarshalPayload(nil, Payload{InitiatorIndex: 100, ResponderIndex: 0, Time: 1}) + err := m.processPayload(bytes, msgFlags{expectsPayload: true}) + require.ErrorIs(t, err, ErrInvalidRemoteIndex) + assert.True(t, m.Failed()) + assert.Zero(t, m.result.RemoteIndex) + }) +} + +// TestMachineRequireComplete checks the fail-on-incomplete-handshake path +// directly. Like processPayload above this isn't reachable from a normal IX +// flow, so we drive it by hand. +func TestMachineRequireComplete(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + cs := newTestCertState(t, ca, caKey, "test", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + v := testVerifier(caPool) + + t.Run("missing both fails", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + err := m.requireComplete() + require.ErrorIs(t, err, ErrIncompleteHandshake) + assert.True(t, m.Failed()) + }) + + t.Run("payload only fails", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + m.payloadSet = true + err := m.requireComplete() + require.ErrorIs(t, err, ErrIncompleteHandshake) + assert.True(t, m.Failed()) + }) + + t.Run("cert only fails", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + m.remoteCertSet = true + err := m.requireComplete() + require.ErrorIs(t, err, ErrIncompleteHandshake) + assert.True(t, m.Failed()) + }) + + t.Run("both set passes", func(t *testing.T) { + m := newTestMachine(t, cs, v, false, 100) + m.payloadSet = true + m.remoteCertSet = true + err := m.requireComplete() + require.NoError(t, err) + assert.False(t, m.Failed()) + }) +} + +func TestMachineAESCipher(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + + initCS := newTestCertStateWithCipher( + t, ca, caKey, "init", + []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}, + noiseutil.CipherAESGCM, + ) + respCS := newTestCertStateWithCipher( + t, ca, caKey, "resp", + []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, + noiseutil.CipherAESGCM, + ) + + initR, respR := doFullHandshake(t, initCS, respCS, caPool) + + ct1, err := initR.EKey.Encrypt(nil, nil, []byte("works")) + require.NoError(t, err) + pt1, err := respR.DKey.Decrypt(nil, nil, ct1) + require.NoError(t, err) + assert.Equal(t, []byte("works"), pt1) + + ct2, err := respR.EKey.Encrypt(nil, nil, []byte("back")) + require.NoError(t, err) + pt2, err := initR.DKey.Decrypt(nil, nil, ct2) + require.NoError(t, err) + assert.Equal(t, []byte("back"), pt2) +} + +func TestResultFields(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + respCS := newTestCertState(t, ca, caKey, "resp", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}) + + initR, respR := doFullHandshake(t, initCS, respCS, caPool) + + assert.True(t, initR.Initiator) + assert.False(t, respR.Initiator) + assert.NotZero(t, initR.HandshakeTime) + assert.NotZero(t, respR.HandshakeTime) + assert.NotNil(t, initR.RemoteCert) + assert.NotNil(t, respR.RemoteCert) +} + +func TestMachineBufferReuse(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + respCS := newTestCertState(t, ca, caKey, "resp", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}) + v := testVerifier(caPool) + + initM := newTestMachine(t, initCS, v, true, 1000) + respM := newTestMachine(t, respCS, v, false, 2000) + + msg1, err := initM.Initiate(nil) + require.NoError(t, err) + + t.Run("response writes into provided buffer", func(t *testing.T) { + buf := make([]byte, 0, 4096) + resp, result, err := respM.ProcessPacket(buf, msg1) + require.NoError(t, err) + require.NotNil(t, result) + + assert.NotEmpty(t, resp, "response should have content") + assert.Equal(t, &buf[:1][0], &resp[:1][0], + "response should reuse the provided buffer's backing array") + }) + + t.Run("initiate writes into provided buffer", func(t *testing.T) { + initM2 := newTestMachine(t, initCS, v, true, 3000) + buf := make([]byte, 0, 4096) + msg, err := initM2.Initiate(buf) + require.NoError(t, err) + + assert.NotEmpty(t, msg, "initiate should have content") + assert.Equal(t, &buf[:1][0], &msg[:1][0], + "initiate should reuse the provided buffer's backing array") + }) + + t.Run("nil out still works", func(t *testing.T) { + initM2 := newTestMachine(t, initCS, v, true, 4000) + respM2 := newTestMachine(t, respCS, v, false, 5000) + + msg1, err := initM2.Initiate(nil) + require.NoError(t, err) + + resp, _, err := respM2.ProcessPacket(nil, msg1) + require.NoError(t, err) + + out, result, err := initM2.ProcessPacket(nil, resp) + require.NoError(t, err) + assert.NotNil(t, result) + assert.Nil(t, out, "initiator should have no response for IX msg2") + }) +} + +func TestMachineMsgIndexTracking(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + respCS := newTestCertState(t, ca, caKey, "resp", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}) + v := testVerifier(caPool) + + initM := newTestMachine(t, initCS, v, true, 100) + respM := newTestMachine(t, respCS, v, false, 200) + + msg1, err := initM.Initiate(nil) + require.NoError(t, err) + + resp1, result1, err := respM.ProcessPacket(nil, msg1) + require.NoError(t, err) + assert.NotNil(t, result1) + + _, result2, err := initM.ProcessPacket(nil, resp1) + require.NoError(t, err) + assert.NotNil(t, result2) +} + +func TestMachineThreeMessagePattern(t *testing.T) { + registerTestXXInfo(t) + + // Use HandshakeXX (3 messages) to verify the Machine handles multi-message + // patterns correctly. XX flow: + // msg1 (I->R): [E] - payload only, no cert + // msg2 (R->I): [E, ee, S, es] - payload + cert + // msg3 (I->R): [S, se] - cert only (no payload, not first two) + + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + v := testVerifier(caPool) + + initCS := newTestCertState(t, ca, caKey, "init", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}) + respCS := newTestCertState(t, ca, caKey, "resp", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}) + + initM, err := NewMachine( + cert.Version2, + initCS.getCredential, v, + func() (uint32, error) { return 1000, nil }, + true, header.HandshakeXXPSK0, + ) + require.NoError(t, err) + + respM, err := NewMachine( + cert.Version2, + respCS.getCredential, v, + func() (uint32, error) { return 2000, nil }, + false, header.HandshakeXXPSK0, + ) + require.NoError(t, err) + + // msg1: initiator -> responder (E only, no cert) + msg1, err := initM.Initiate(nil) + require.NoError(t, err) + assert.NotEmpty(t, msg1) + + // Responder processes msg1, should not complete yet, should produce msg2 + msg2, result, err := respM.ProcessPacket(nil, msg1) + require.NoError(t, err) + assert.Nil(t, result, "XX should not complete on msg1") + assert.NotEmpty(t, msg2, "responder should produce msg2") + + // Initiator processes msg2: gets responder's cert, produces msg3, and + // completes (WriteMessage for msg3 derives keys) + msg3, initResult, err := initM.ProcessPacket(nil, msg2) + require.NoError(t, err) + require.NotNil(t, initResult, "XX initiator should complete after reading msg2 and writing msg3") + assert.NotEmpty(t, msg3, "initiator should produce msg3") + assert.Equal(t, "resp", initResult.RemoteCert.Certificate.Name()) + + // Responder processes msg3: gets initiator's cert and completes + _, respResult, err := respM.ProcessPacket(nil, msg3) + require.NoError(t, err) + require.NotNil(t, respResult, "XX responder should complete on msg3") + assert.Equal(t, "init", respResult.RemoteCert.Certificate.Name()) + + assert.Equal(t, uint64(3), initResult.MessageIndex, "XX has 3 messages") + assert.Equal(t, uint64(3), respResult.MessageIndex, "XX has 3 messages") + + // Verify keys work + ct1, err := initResult.EKey.Encrypt(nil, nil, []byte("three messages")) + require.NoError(t, err) + pt1, err := respResult.DKey.Decrypt(nil, nil, ct1) + require.NoError(t, err) + assert.Equal(t, []byte("three messages"), pt1) +} + +// NOTE: ErrIncompleteHandshake is tested implicitly. It can't be triggered with +// IX since the cert is always in the payload. A 3-message pattern test (HybridIX) +// should exercise the case where cert arrives in msg3 and verify that completing +// without it fails. + +func TestMachineExpiredCert(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, + time.Now().Add(-24*time.Hour), time.Now().Add(24*time.Hour), + nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + + expCert, _, expKeyPEM, _ := ct.NewTestCert( + cert.Version2, cert.Curve_CURVE25519, ca, caKey, + "expired", time.Now().Add(-2*time.Hour), time.Now().Add(-1*time.Hour), + []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}, nil, nil, + ) + expKey, _, _, err := cert.UnmarshalPrivateKeyFromPEM(expKeyPEM) + require.NoError(t, err) + expHsBytes, err := expCert.MarshalForHandshakes() + require.NoError(t, err) + ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256) + + expiredCS := &testCertState{ + version: cert.Version2, + creds: map[cert.Version]*Credential{ + cert.Version2: NewCredential(expCert, expHsBytes, expKey, ncs), + }, + } + + respCS := newTestCertState( + t, ca, caKey, "responder", + []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, + ) + + _, respM, _, _, err := initiateHandshake( + t, expiredCS, testVerifier(caPool), + respCS, testVerifier(caPool), + ) + require.ErrorContains(t, err, "verify cert") + assert.True(t, respM.Failed()) +} + +func TestMachineNoCertNetworks(t *testing.T) { + ca, _, caKey, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca) + + caHsBytes, err := ca.MarshalForHandshakes() + require.NoError(t, err) + ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256) + + noNetCS := &testCertState{ + version: cert.Version2, + creds: map[cert.Version]*Credential{ + cert.Version2: NewCredential(ca, caHsBytes, caKey, ncs), + }, + } + + respCS := newTestCertState( + t, ca, caKey, "responder", + []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, + ) + + _, respM, _, _, err := initiateHandshake( + t, noNetCS, testVerifier(caPool), + respCS, testVerifier(caPool), + ) + require.Error(t, err) + assert.True(t, respM.Failed()) +} + +func TestMachineDifferentCAs(t *testing.T) { + ca1, _, caKey1, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + ca2, _, caKey2, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + + initCS := newTestCertState( + t, ca1, caKey1, "init", + []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}, + ) + respCS := newTestCertState( + t, ca2, caKey2, "resp", + []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, + ) + + _, respM, _, _, err := initiateHandshake( + t, initCS, testVerifier(ct.NewTestCAPool(ca1)), + respCS, testVerifier(ct.NewTestCAPool(ca2)), + ) + require.ErrorContains(t, err, "verify cert") + assert.True(t, respM.Failed()) +} + +func TestMachineVersionNegotiation(t *testing.T) { + ca1, _, caKey1, _ := ct.NewTestCaCert( + cert.Version1, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + ca2, _, caKey2, _ := ct.NewTestCaCert( + cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil, + ) + caPool := ct.NewTestCAPool(ca1, ca2) + + makeMultiVersionResp := func(t *testing.T) *testCertState { + t.Helper() + respCertV1, _, respKeyPEM, _ := ct.NewTestCert( + cert.Version1, cert.Curve_CURVE25519, ca1, caKey1, "resp", + ca1.NotBefore(), ca1.NotAfter(), + []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil, nil, + ) + respKey, _, _, _ := cert.UnmarshalPrivateKeyFromPEM(respKeyPEM) + respCertV2, _ := ct.NewTestCertDifferentVersion(respCertV1, cert.Version2, ca2, caKey2) + respHsV1, _ := respCertV1.MarshalForHandshakes() + respHsV2, _ := respCertV2.MarshalForHandshakes() + ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256) + return &testCertState{ + version: cert.Version1, + creds: map[cert.Version]*Credential{ + cert.Version1: NewCredential(respCertV1, respHsV1, respKey, ncs), + cert.Version2: NewCredential(respCertV2, respHsV2, respKey, ncs), + }, + } + } + + t.Run("responder matches initiator version", func(t *testing.T) { + initCS := newTestCertState( + t, ca2, caKey2, "init", + []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}, + ) + respCS := makeMultiVersionResp(t) + v := testVerifier(caPool) + + initM, _, respResult, resp, err := initiateHandshake( + t, initCS, v, + respCS, v, + ) + require.NoError(t, err) + require.NotNil(t, respResult) + + assert.Equal(t, cert.Version2, respResult.MyCert.Version(), + "responder should negotiate to initiator's version") + + _, initResult, err := initM.ProcessPacket(nil, resp) + require.NoError(t, err) + require.NotNil(t, initResult) + assert.Equal(t, cert.Version2, initResult.RemoteCert.Certificate.Version(), + "initiator should see V2 cert from responder") + }) + + t.Run("responder keeps version when no match available", func(t *testing.T) { + initCS := newTestCertState( + t, ca2, caKey2, "init", + []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}, + ) + + respCert, _, respKeyPEM, _ := ct.NewTestCert( + cert.Version1, cert.Curve_CURVE25519, ca1, caKey1, "resp", + ca1.NotBefore(), ca1.NotAfter(), + []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, nil, nil, + ) + respKey, _, _, _ := cert.UnmarshalPrivateKeyFromPEM(respKeyPEM) + respHs, _ := respCert.MarshalForHandshakes() + ncs := noise.NewCipherSuite(noise.DH25519, noise.CipherChaChaPoly, noise.HashSHA256) + respCS := &testCertState{ + version: cert.Version1, + creds: map[cert.Version]*Credential{ + cert.Version1: NewCredential(respCert, respHs, respKey, ncs), + }, + } + + v := testVerifier(caPool) + _, _, respResult, _, err := initiateHandshake( + t, initCS, v, + respCS, v, + ) + require.NoError(t, err) + require.NotNil(t, respResult) + + assert.Equal(t, cert.Version1, respResult.MyCert.Version(), + "responder should keep V1 when V2 not available") + }) +} diff --git a/handshake/patterns.go b/handshake/patterns.go new file mode 100644 index 00000000..a0cc1a70 --- /dev/null +++ b/handshake/patterns.go @@ -0,0 +1,54 @@ +package handshake + +import ( + "fmt" + + "github.com/flynn/noise" + "github.com/slackhq/nebula/header" +) + +// msgFlags tracks what application data a handshake message carries. +type msgFlags struct { + expectsPayload bool // message carries indexes and time + expectsCert bool // message carries the certificate +} + +// subtypeInfo bundles the noise pattern with the per-message flags for a +// given handshake subtype. +type subtypeInfo struct { + pattern noise.HandshakePattern + msgs []msgFlags +} + +// subtypeInfos defines the noise pattern and message content layout for each +// handshake subtype. +var subtypeInfos = map[header.MessageSubType]subtypeInfo{ + // IX: 2 messages, both carry payload and cert + header.HandshakeIXPSK0: { + pattern: noise.HandshakeIX, + msgs: []msgFlags{ + {expectsPayload: true, expectsCert: true}, + {expectsPayload: true, expectsCert: true}, + }, + }, + + // XX: 3 messages + // msg1 (I->R): payload only + // msg2 (R->I): payload + cert + // msg3 (I->R): cert only + //header.HandshakeXXPSK0: { + // pattern: noise.HandshakeXX, + // msgs: []msgFlags{ + // {expectsPayload: true, expectsCert: false}, + // {expectsPayload: true, expectsCert: true}, + // {expectsPayload: false, expectsCert: true}, + // }, + //}, +} + +func subtypeInfoFor(subtype header.MessageSubType) (subtypeInfo, error) { + if info, ok := subtypeInfos[subtype]; ok { + return info, nil + } + return subtypeInfo{}, fmt.Errorf("%w: %d", ErrUnknownSubtype, subtype) +} diff --git a/handshake/patterns_test.go b/handshake/patterns_test.go new file mode 100644 index 00000000..d6207e00 --- /dev/null +++ b/handshake/patterns_test.go @@ -0,0 +1,63 @@ +package handshake + +import ( + "testing" + + "github.com/flynn/noise" + "github.com/slackhq/nebula/header" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSubtypeInfo(t *testing.T) { + t.Run("IX", func(t *testing.T) { + info, err := subtypeInfoFor(header.HandshakeIXPSK0) + require.NoError(t, err) + assert.Equal(t, noise.HandshakeIX.Name, info.pattern.Name) + require.Len(t, info.msgs, 2) + // msg1: payload + cert + assert.True(t, info.msgs[0].expectsPayload) + assert.True(t, info.msgs[0].expectsCert) + // msg2: payload + cert + assert.True(t, info.msgs[1].expectsPayload) + assert.True(t, info.msgs[1].expectsCert) + }) + + t.Run("XX", func(t *testing.T) { + registerTestXXInfo(t) + info, err := subtypeInfoFor(header.HandshakeXXPSK0) + require.NoError(t, err) + assert.Equal(t, noise.HandshakeXX.Name, info.pattern.Name) + require.Len(t, info.msgs, 3) + // msg1: payload only + assert.True(t, info.msgs[0].expectsPayload) + assert.False(t, info.msgs[0].expectsCert) + // msg2: payload + cert + assert.True(t, info.msgs[1].expectsPayload) + assert.True(t, info.msgs[1].expectsCert) + // msg3: cert only + assert.False(t, info.msgs[2].expectsPayload) + assert.True(t, info.msgs[2].expectsCert) + }) + + t.Run("unknown subtype returns error", func(t *testing.T) { + _, err := subtypeInfoFor(99) + require.ErrorIs(t, err, ErrUnknownSubtype) + }) +} + +// registerTestXXInfo temporarily registers XX subtype info for testing. +func registerTestXXInfo(t *testing.T) { + t.Helper() + subtypeInfos[header.HandshakeXXPSK0] = subtypeInfo{ + pattern: noise.HandshakeXX, + msgs: []msgFlags{ + {expectsPayload: true, expectsCert: false}, + {expectsPayload: true, expectsCert: true}, + {expectsPayload: false, expectsCert: true}, + }, + } + t.Cleanup(func() { + delete(subtypeInfos, header.HandshakeXXPSK0) + }) +} diff --git a/handshake/payload.go b/handshake/payload.go new file mode 100644 index 00000000..4567fc0d --- /dev/null +++ b/handshake/payload.go @@ -0,0 +1,173 @@ +package handshake + +import ( + "errors" + "math" + + "google.golang.org/protobuf/encoding/protowire" +) + +var ( + errInvalidHandshakeMessage = errors.New("invalid handshake message") + errInvalidHandshakeDetails = errors.New("invalid handshake details") +) + +// Payload represents the decoded fields of a handshake message. +// Wire format is protobuf-compatible with NebulaHandshake{Details: NebulaHandshakeDetails{...}}. +type Payload struct { + Cert []byte + InitiatorIndex uint32 + ResponderIndex uint32 + Time uint64 + CertVersion uint32 +} + +// Proto field numbers for NebulaHandshakeDetails +const ( + fieldCert = 1 // bytes + fieldInitiatorIndex = 2 // uint32 + fieldResponderIndex = 3 // uint32 + fieldTime = 5 // uint64 + fieldCertVersion = 8 // uint32 +) + +// MarshalPayload encodes a handshake payload in protobuf wire format compatible +// with NebulaHandshake{Details: NebulaHandshakeDetails{...}}. +// Returns out (which may be nil), with the marshalled Payload appended to it. +func MarshalPayload(out []byte, p Payload) []byte { + var details []byte + + if len(p.Cert) > 0 { + details = protowire.AppendTag(details, fieldCert, protowire.BytesType) + details = protowire.AppendBytes(details, p.Cert) + } + if p.InitiatorIndex != 0 { + details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType) + details = protowire.AppendVarint(details, uint64(p.InitiatorIndex)) + } + if p.ResponderIndex != 0 { + details = protowire.AppendTag(details, fieldResponderIndex, protowire.VarintType) + details = protowire.AppendVarint(details, uint64(p.ResponderIndex)) + } + if p.Time != 0 { + details = protowire.AppendTag(details, fieldTime, protowire.VarintType) + details = protowire.AppendVarint(details, p.Time) + } + if p.CertVersion != 0 { + details = protowire.AppendTag(details, fieldCertVersion, protowire.VarintType) + details = protowire.AppendVarint(details, uint64(p.CertVersion)) + } + + out = protowire.AppendTag(out, 1, protowire.BytesType) + out = protowire.AppendBytes(out, details) + + return out +} + +// UnmarshalPayload decodes a protobuf-encoded NebulaHandshake message. +func UnmarshalPayload(b []byte) (Payload, error) { + var p Payload + + for len(b) > 0 { + num, typ, n := protowire.ConsumeTag(b) + if n < 0 { + return p, errInvalidHandshakeMessage + } + b = b[n:] + + switch { + case num == 1 && typ == protowire.BytesType: + details, n := protowire.ConsumeBytes(b) + if n < 0 { + return p, errInvalidHandshakeMessage + } + b = b[n:] + if err := unmarshalPayloadDetails(&p, details); err != nil { + return p, err + } + default: + n := protowire.ConsumeFieldValue(num, typ, b) + if n < 0 { + return p, errInvalidHandshakeMessage + } + b = b[n:] + } + } + + return p, nil +} + +func unmarshalPayloadDetails(p *Payload, b []byte) error { + for len(b) > 0 { + num, typ, n := protowire.ConsumeTag(b) + if n < 0 { + return errInvalidHandshakeDetails + } + b = b[n:] + + // For known field numbers, reject any non-matching wire type as a + // hard error rather than silently skipping. The caller will catch + // missing-field cases downstream, but a wire-type mismatch on a tag + // we know is a peer protocol violation worth flagging here. + // Repeated occurrences of a singular field follow proto3 last-wins. + switch num { + case fieldCert: + if typ != protowire.BytesType { + return errInvalidHandshakeDetails + } + v, n := protowire.ConsumeBytes(b) + if n < 0 { + return errInvalidHandshakeDetails + } + p.Cert = append([]byte(nil), v...) + b = b[n:] + case fieldInitiatorIndex: + if typ != protowire.VarintType { + return errInvalidHandshakeDetails + } + v, n := protowire.ConsumeVarint(b) + if n < 0 || v > math.MaxUint32 { + return errInvalidHandshakeDetails + } + p.InitiatorIndex = uint32(v) + b = b[n:] + case fieldResponderIndex: + if typ != protowire.VarintType { + return errInvalidHandshakeDetails + } + v, n := protowire.ConsumeVarint(b) + if n < 0 || v > math.MaxUint32 { + return errInvalidHandshakeDetails + } + p.ResponderIndex = uint32(v) + b = b[n:] + case fieldTime: + if typ != protowire.VarintType { + return errInvalidHandshakeDetails + } + v, n := protowire.ConsumeVarint(b) + if n < 0 { + return errInvalidHandshakeDetails + } + p.Time = v + b = b[n:] + case fieldCertVersion: + if typ != protowire.VarintType { + return errInvalidHandshakeDetails + } + v, n := protowire.ConsumeVarint(b) + if n < 0 || v > math.MaxUint32 { + return errInvalidHandshakeDetails + } + p.CertVersion = uint32(v) + b = b[n:] + default: + n := protowire.ConsumeFieldValue(num, typ, b) + if n < 0 { + return errInvalidHandshakeDetails + } + b = b[n:] + } + } + return nil +} diff --git a/handshake/payload_test.go b/handshake/payload_test.go new file mode 100644 index 00000000..2ff3231c --- /dev/null +++ b/handshake/payload_test.go @@ -0,0 +1,361 @@ +package handshake + +import ( + "bytes" + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" +) + +func TestPayloadRoundTrip(t *testing.T) { + t.Run("all fields set", func(t *testing.T) { + data := MarshalPayload(nil, Payload{ + Cert: []byte("test-cert-bytes"), + CertVersion: 2, + InitiatorIndex: 12345, + ResponderIndex: 67890, + Time: 1234567890, + }) + + got, err := UnmarshalPayload(data) + require.NoError(t, err) + + assert.Equal(t, []byte("test-cert-bytes"), got.Cert) + assert.Equal(t, uint32(12345), got.InitiatorIndex) + assert.Equal(t, uint32(67890), got.ResponderIndex) + assert.Equal(t, uint64(1234567890), got.Time) + assert.Equal(t, uint32(2), got.CertVersion) + }) + + t.Run("minimal fields", func(t *testing.T) { + data := MarshalPayload(nil, Payload{InitiatorIndex: 1}) + + got, err := UnmarshalPayload(data) + require.NoError(t, err) + + assert.Equal(t, uint32(1), got.InitiatorIndex) + assert.Equal(t, uint32(0), got.ResponderIndex) + assert.Equal(t, uint64(0), got.Time) + assert.Nil(t, got.Cert) + }) + + t.Run("empty payload", func(t *testing.T) { + data := MarshalPayload(nil, Payload{}) + + got, err := UnmarshalPayload(data) + require.NoError(t, err) + + assert.Equal(t, uint32(0), got.InitiatorIndex) + }) + + t.Run("large cert bytes", func(t *testing.T) { + bigCert := make([]byte, 4096) + for i := range bigCert { + bigCert[i] = byte(i % 256) + } + + data := MarshalPayload(nil, Payload{ + Cert: bigCert, + CertVersion: 2, + InitiatorIndex: 999, + }) + + got, err := UnmarshalPayload(data) + require.NoError(t, err) + + assert.Equal(t, bigCert, got.Cert) + assert.Equal(t, uint32(999), got.InitiatorIndex) + }) + + t.Run("append to existing buffer", func(t *testing.T) { + prefix := []byte("prefix") + data := MarshalPayload(prefix, Payload{InitiatorIndex: 42}) + + assert.Equal(t, []byte("prefix"), data[:6]) + + got, err := UnmarshalPayload(data[6:]) + require.NoError(t, err) + assert.Equal(t, uint32(42), got.InitiatorIndex) + }) +} + +func TestPayloadUnknownFields(t *testing.T) { + t.Run("unknown field in outer message is skipped", func(t *testing.T) { + // Marshal a normal payload then append an unknown field (field 99, varint) + data := MarshalPayload(nil, Payload{InitiatorIndex: 42}) + data = protowire.AppendTag(data, 99, protowire.VarintType) + data = protowire.AppendVarint(data, 12345) + + got, err := UnmarshalPayload(data) + require.NoError(t, err) + assert.Equal(t, uint32(42), got.InitiatorIndex) + }) + + t.Run("unknown field in details is skipped", func(t *testing.T) { + // Build details with a known field + unknown field + var details []byte + details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType) + details = protowire.AppendVarint(details, 77) + // Unknown field 50, varint + details = protowire.AppendTag(details, 50, protowire.VarintType) + details = protowire.AppendVarint(details, 9999) + // Another known field after the unknown one + details = protowire.AppendTag(details, fieldResponderIndex, protowire.VarintType) + details = protowire.AppendVarint(details, 88) + + // Wrap in outer message + var data []byte + data = protowire.AppendTag(data, 1, protowire.BytesType) + data = protowire.AppendBytes(data, details) + + got, err := UnmarshalPayload(data) + require.NoError(t, err) + assert.Equal(t, uint32(77), got.InitiatorIndex) + assert.Equal(t, uint32(88), got.ResponderIndex) + }) + + t.Run("reserved fields 6 and 7 are skipped", func(t *testing.T) { + // Fields 6 and 7 are reserved in the proto definition + var details []byte + details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType) + details = protowire.AppendVarint(details, 100) + details = protowire.AppendTag(details, 6, protowire.VarintType) + details = protowire.AppendVarint(details, 1) + details = protowire.AppendTag(details, 7, protowire.VarintType) + details = protowire.AppendVarint(details, 2) + + var data []byte + data = protowire.AppendTag(data, 1, protowire.BytesType) + data = protowire.AppendBytes(data, details) + + got, err := UnmarshalPayload(data) + require.NoError(t, err) + assert.Equal(t, uint32(100), got.InitiatorIndex) + }) +} + +func TestPayloadBytesConsumed(t *testing.T) { + t.Run("all bytes consumed on valid input", func(t *testing.T) { + original := Payload{ + Cert: []byte("cert"), + CertVersion: 2, + InitiatorIndex: 100, + ResponderIndex: 200, + Time: 999, + } + data := MarshalPayload(nil, original) + + got, err := UnmarshalPayload(data) + require.NoError(t, err) + + // Re-marshal and compare — proves we consumed and reproduced all fields + remarshaled := MarshalPayload(nil, got) + assert.Equal(t, data, remarshaled) + }) +} + +// wrapDetails wraps raw detail bytes in the outer NebulaHandshake envelope +// so UnmarshalPayload can reach unmarshalPayloadDetails. +func wrapDetails(details []byte) []byte { + var out []byte + out = protowire.AppendTag(out, 1, protowire.BytesType) + out = protowire.AppendBytes(out, details) + return out +} + +func TestPayloadUnmarshalErrors(t *testing.T) { + t.Run("nil input", func(t *testing.T) { + got, err := UnmarshalPayload(nil) + require.NoError(t, err) + assert.Equal(t, uint32(0), got.InitiatorIndex) + }) + + t.Run("truncated outer tag", func(t *testing.T) { + _, err := UnmarshalPayload([]byte{0x80}) + assert.Error(t, err) + }) + + t.Run("truncated outer details field", func(t *testing.T) { + _, err := UnmarshalPayload([]byte{0x0a, 0x64, 0x01, 0x02, 0x03, 0x04, 0x05}) + assert.Error(t, err) + }) + + t.Run("truncated outer unknown field", func(t *testing.T) { + // Valid tag for unknown field 99 varint, but no value follows + var data []byte + data = protowire.AppendTag(data, 99, protowire.VarintType) + _, err := UnmarshalPayload(data) + assert.Error(t, err) + }) + + t.Run("truncated details tag", func(t *testing.T) { + _, err := UnmarshalPayload(wrapDetails([]byte{0x80})) + assert.Error(t, err) + }) + + t.Run("truncated cert bytes", func(t *testing.T) { + // Field 1 (cert), bytes type, length 10 but only 2 bytes + var details []byte + details = protowire.AppendTag(details, fieldCert, protowire.BytesType) + details = append(details, 0x0a, 0x01, 0x02) // length 10, only 2 bytes + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("truncated initiator index varint", func(t *testing.T) { + var details []byte + details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType) + details = append(details, 0x80) // incomplete varint + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("truncated responder index varint", func(t *testing.T) { + var details []byte + details = protowire.AppendTag(details, fieldResponderIndex, protowire.VarintType) + details = append(details, 0x80) + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("truncated time varint", func(t *testing.T) { + var details []byte + details = protowire.AppendTag(details, fieldTime, protowire.VarintType) + details = append(details, 0x80) + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("truncated cert version varint", func(t *testing.T) { + var details []byte + details = protowire.AppendTag(details, fieldCertVersion, protowire.VarintType) + details = append(details, 0x80) + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("truncated unknown field in details", func(t *testing.T) { + var details []byte + details = protowire.AppendTag(details, 50, protowire.VarintType) + details = append(details, 0x80) // incomplete varint + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("cert with wrong wire type rejected", func(t *testing.T) { + // fieldCert as Varint instead of Bytes. + var details []byte + details = protowire.AppendTag(details, fieldCert, protowire.VarintType) + details = protowire.AppendVarint(details, 42) + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("initiator index with wrong wire type rejected", func(t *testing.T) { + // fieldInitiatorIndex as Bytes instead of Varint. + var details []byte + details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.BytesType) + details = protowire.AppendBytes(details, []byte{1, 2, 3}) + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("time with wrong wire type rejected", func(t *testing.T) { + var details []byte + details = protowire.AppendTag(details, fieldTime, protowire.BytesType) + details = protowire.AppendBytes(details, []byte{1, 2, 3}) + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("cert version with wrong wire type rejected", func(t *testing.T) { + var details []byte + details = protowire.AppendTag(details, fieldCertVersion, protowire.BytesType) + details = protowire.AppendBytes(details, []byte{1, 2, 3}) + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("repeated singular field follows proto3 last-wins", func(t *testing.T) { + // Per proto3, multiple instances of a singular field are accepted and + // the last value wins. We keep this behavior so that peers using + // alternative encoders aren't rejected. + var details []byte + details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType) + details = protowire.AppendVarint(details, 1) + details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType) + details = protowire.AppendVarint(details, 42) + got, err := UnmarshalPayload(wrapDetails(details)) + require.NoError(t, err) + assert.Equal(t, uint32(42), got.InitiatorIndex) + }) + + t.Run("initiator index varint overflow rejected", func(t *testing.T) { + var details []byte + details = protowire.AppendTag(details, fieldInitiatorIndex, protowire.VarintType) + details = protowire.AppendVarint(details, math.MaxUint32+1) + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + + t.Run("cert version varint overflow rejected", func(t *testing.T) { + var details []byte + details = protowire.AppendTag(details, fieldCertVersion, protowire.VarintType) + details = protowire.AppendVarint(details, math.MaxUint32+1) + _, err := UnmarshalPayload(wrapDetails(details)) + assert.Error(t, err) + }) + +} + +// FuzzPayload feeds arbitrary bytes through UnmarshalPayload to confirm it +// never panics, and for any input that parses cleanly, that re-marshal + +// re-parse is a fix-point. Inputs come from an authenticated peer (post- +// noise-decrypt), so the threat model is "valid peer behaving arbitrarily," +// not "unauthenticated injection." +func FuzzPayload(f *testing.F) { + // Seed corpus with a handful of known-good shapes. + f.Add(MarshalPayload(nil, Payload{})) + f.Add(MarshalPayload(nil, Payload{Cert: []byte{1, 2, 3}, CertVersion: 2})) + f.Add(MarshalPayload(nil, Payload{InitiatorIndex: 42, Time: 1})) + f.Add(MarshalPayload(nil, Payload{ + Cert: []byte("seed-cert"), + InitiatorIndex: 1, + ResponderIndex: 2, + Time: 3, + CertVersion: 2, + })) + f.Add([]byte{}) + f.Add([]byte{0xff}) + + f.Fuzz(func(t *testing.T, data []byte) { + p1, err := UnmarshalPayload(data) + if err != nil { + return + } + + // For any input that parses, re-marshaling and re-parsing must + // yield an equivalent Payload. This catches dispatch bugs (e.g. + // emitting a field on marshal that we don't accept on parse) and + // any non-idempotent parsing behavior. + b2 := MarshalPayload(nil, p1) + p2, err := UnmarshalPayload(b2) + if err != nil { + t.Fatalf("re-parse of self-marshaled payload failed: %v\nintermediate: %x\n", err, b2) + } + if !payloadsEqual(p1, p2) { + t.Fatalf("re-marshal not idempotent\nfirst: %+v\nsecond: %+v", p1, p2) + } + }) +} + +func payloadsEqual(a, b Payload) bool { + return bytes.Equal(a.Cert, b.Cert) && + a.InitiatorIndex == b.InitiatorIndex && + a.ResponderIndex == b.ResponderIndex && + a.Time == b.Time && + a.CertVersion == b.CertVersion +} diff --git a/handshake_ix.go b/handshake_ix.go deleted file mode 100644 index 1905c004..00000000 --- a/handshake_ix.go +++ /dev/null @@ -1,479 +0,0 @@ -package nebula - -import ( - "time" - - "github.com/flynn/noise" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" - "github.com/slackhq/nebula/udp" -) - -// NOISE IX Handshakes - -// This function constructs a handshake packet, but does not actually send it -// Sending is done by the handshake manager -func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool { - err := f.handshakeManager.allocateIndex(hh) - if err != nil { - f.l.WithError(err).WithField("vpnIp", hh.hostinfo.vpnIp). - WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to generate index") - return false - } - - certState := f.pki.GetCertState() - ci := NewConnectionState(f.l, f.cipher, certState, true, noise.HandshakeIX, []byte{}, 0) - hh.hostinfo.ConnectionState = ci - - hsProto := &NebulaHandshakeDetails{ - InitiatorIndex: hh.hostinfo.localIndexId, - Time: uint64(time.Now().UnixNano()), - Cert: certState.RawCertificateNoKey, - } - - hsBytes := []byte{} - - hs := &NebulaHandshake{ - Details: hsProto, - } - hsBytes, err = hs.Marshal() - - if err != nil { - f.l.WithError(err).WithField("vpnIp", hh.hostinfo.vpnIp). - WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to marshal handshake message") - return false - } - - h := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, 0, 1) - ci.messageCounter.Add(1) - - msg, _, _, err := ci.H.WriteMessage(h, hsBytes) - if err != nil { - f.l.WithError(err).WithField("vpnIp", hh.hostinfo.vpnIp). - WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage") - return false - } - - // We are sending handshake packet 1, so we don't expect to receive - // handshake packet 1 from the responder - ci.window.Update(f.l, 1) - - hh.hostinfo.HandshakePacket[0] = msg - hh.ready = true - return true -} - -func ixHandshakeStage1(f *Interface, addr *udp.Addr, via *ViaSender, packet []byte, h *header.H) { - certState := f.pki.GetCertState() - ci := NewConnectionState(f.l, f.cipher, certState, false, noise.HandshakeIX, []byte{}, 0) - // Mark packet 1 as seen so it doesn't show up as missed - ci.window.Update(f.l, 1) - - msg, _, _, err := ci.H.ReadMessage(nil, packet[header.Len:]) - if err != nil { - f.l.WithError(err).WithField("udpAddr", addr). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.ReadMessage") - return - } - - hs := &NebulaHandshake{} - err = hs.Unmarshal(msg) - /* - l.Debugln("GOT INDEX: ", hs.Details.InitiatorIndex) - */ - if err != nil || hs.Details == nil { - f.l.WithError(err).WithField("udpAddr", addr). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed unmarshal handshake message") - return - } - - remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.pki.GetCAPool()) - if err != nil { - f.l.WithError(err).WithField("udpAddr", addr). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert). - Info("Invalid certificate from host") - return - } - vpnIp := iputil.Ip2VpnIp(remoteCert.Details.Ips[0].IP) - certName := remoteCert.Details.Name - fingerprint, _ := remoteCert.Sha256Sum() - issuer := remoteCert.Details.Issuer - - if vpnIp == f.myVpnIp { - f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Refusing to handshake with myself") - return - } - - if addr != nil { - if !f.lightHouse.GetRemoteAllowList().Allow(vpnIp, addr.IP) { - f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake") - return - } - } - - myIndex, err := generateIndex(f.l) - if err != nil { - f.l.WithError(err).WithField("vpnIp", vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to generate index") - return - } - - hostinfo := &HostInfo{ - ConnectionState: ci, - localIndexId: myIndex, - remoteIndexId: hs.Details.InitiatorIndex, - vpnIp: vpnIp, - HandshakePacket: make(map[uint8][]byte, 0), - lastHandshakeTime: hs.Details.Time, - relayState: RelayState{ - relays: map[iputil.VpnIp]struct{}{}, - relayForByIp: map[iputil.VpnIp]*Relay{}, - relayForByIdx: map[uint32]*Relay{}, - }, - } - - f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex). - WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). - Info("Handshake message received") - - hs.Details.ResponderIndex = myIndex - hs.Details.Cert = certState.RawCertificateNoKey - // Update the time in case their clock is way off from ours - hs.Details.Time = uint64(time.Now().UnixNano()) - - hsBytes, err := hs.Marshal() - if err != nil { - f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to marshal handshake message") - return - } - - nh := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, hs.Details.InitiatorIndex, 2) - msg, dKey, eKey, err := ci.H.WriteMessage(nh, hsBytes) - if err != nil { - f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage") - return - } else if dKey == nil || eKey == nil { - f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Noise did not arrive at a key") - return - } - - hostinfo.HandshakePacket[0] = make([]byte, len(packet[header.Len:])) - copy(hostinfo.HandshakePacket[0], packet[header.Len:]) - - // Regardless of whether you are the sender or receiver, you should arrive here - // and complete standing up the connection. - hostinfo.HandshakePacket[2] = make([]byte, len(msg)) - copy(hostinfo.HandshakePacket[2], msg) - - // We are sending handshake packet 2, so we don't expect to receive - // handshake packet 2 from the initiator. - ci.window.Update(f.l, 2) - - ci.peerCert = remoteCert - ci.dKey = NewNebulaCipherState(dKey) - ci.eKey = NewNebulaCipherState(eKey) - - hostinfo.remotes = f.lightHouse.QueryCache(vpnIp) - hostinfo.SetRemote(addr) - hostinfo.CreateRemoteCIDR(remoteCert) - - existing, err := f.handshakeManager.CheckAndComplete(hostinfo, 0, f) - if err != nil { - switch err { - case ErrAlreadySeen: - // Update remote if preferred - if existing.SetRemoteIfPreferred(f.hostMap, addr) { - // Send a test packet to ensure the other side has also switched to - // the preferred remote - f.SendMessageToVpnIp(header.Test, header.TestRequest, vpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu)) - } - - msg = existing.HandshakePacket[2] - f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1) - if addr != nil { - err := f.outside.WriteTo(msg, addr) - if err != nil { - f.l.WithField("vpnIp", existing.vpnIp).WithField("udpAddr", addr). - WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true). - WithError(err).Error("Failed to send handshake message") - } else { - f.l.WithField("vpnIp", existing.vpnIp).WithField("udpAddr", addr). - WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true). - Info("Handshake message sent") - } - return - } else { - if via == nil { - f.l.Error("Handshake send failed: both addr and via are nil.") - return - } - hostinfo.relayState.InsertRelayTo(via.relayHI.vpnIp) - f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false) - f.l.WithField("vpnIp", existing.vpnIp).WithField("relay", via.relayHI.vpnIp). - WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true). - Info("Handshake message sent") - return - } - case ErrExistingHostInfo: - // This means there was an existing tunnel and this handshake was older than the one we are currently based on - f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("oldHandshakeTime", existing.lastHandshakeTime). - WithField("newHandshakeTime", hostinfo.lastHandshakeTime). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex). - WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). - Info("Handshake too old") - - // Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues - f.SendMessageToVpnIp(header.Test, header.TestRequest, vpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu)) - return - case ErrLocalIndexCollision: - // This means we failed to insert because of collision on localIndexId. Just let the next handshake packet retry - f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex). - WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). - WithField("localIndex", hostinfo.localIndexId).WithField("collision", existing.vpnIp). - Error("Failed to add HostInfo due to localIndex collision") - return - default: - // Shouldn't happen, but just in case someone adds a new error type to CheckAndComplete - // And we forget to update it here - f.l.WithError(err).WithField("vpnIp", vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex). - WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). - Error("Failed to add HostInfo to HostMap") - return - } - } - - // Do the send - f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1) - if addr != nil { - err = f.outside.WriteTo(msg, addr) - if err != nil { - f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex). - WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}). - WithError(err).Error("Failed to send handshake") - } else { - f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex). - WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}). - Info("Handshake message sent") - } - } else { - if via == nil { - f.l.Error("Handshake send failed: both addr and via are nil.") - return - } - hostinfo.relayState.InsertRelayTo(via.relayHI.vpnIp) - f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false) - f.l.WithField("vpnIp", vpnIp).WithField("relay", via.relayHI.vpnIp). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex). - WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}). - Info("Handshake message sent") - } - - f.connectionManager.AddTrafficWatch(hostinfo.localIndexId) - hostinfo.ConnectionState.messageCounter.Store(2) - hostinfo.remotes.ResetBlockedRemotes() - - return -} - -func ixHandshakeStage2(f *Interface, addr *udp.Addr, via *ViaSender, hh *HandshakeHostInfo, packet []byte, h *header.H) bool { - if hh == nil { - // Nothing here to tear down, got a bogus stage 2 packet - return true - } - - hh.Lock() - defer hh.Unlock() - - hostinfo := hh.hostinfo - if addr != nil { - if !f.lightHouse.GetRemoteAllowList().Allow(hostinfo.vpnIp, addr.IP) { - f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake") - return false - } - } - - ci := hostinfo.ConnectionState - msg, eKey, dKey, err := ci.H.ReadMessage(nil, packet[header.Len:]) - if err != nil { - f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). - WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h). - Error("Failed to call noise.ReadMessage") - - // We don't want to tear down the connection on a bad ReadMessage because it could be an attacker trying - // to DOS us. Every other error condition after should to allow a possible good handshake to complete in the - // near future - return false - } else if dKey == nil || eKey == nil { - f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). - WithField("handshake", m{"stage": 2, "style": "ix_psk0"}). - Error("Noise did not arrive at a key") - - // This should be impossible in IX but just in case, if we get here then there is no chance to recover - // the handshake state machine. Tear it down - return true - } - - hs := &NebulaHandshake{} - err = hs.Unmarshal(msg) - if err != nil || hs.Details == nil { - f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). - WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("Failed unmarshal handshake message") - - // The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again - return true - } - - remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.pki.GetCAPool()) - if err != nil { - f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). - WithField("cert", remoteCert).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}). - Error("Invalid certificate from host") - - // The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again - return true - } - - vpnIp := iputil.Ip2VpnIp(remoteCert.Details.Ips[0].IP) - certName := remoteCert.Details.Name - fingerprint, _ := remoteCert.Sha256Sum() - issuer := remoteCert.Details.Issuer - - // Ensure the right host responded - if vpnIp != hostinfo.vpnIp { - f.l.WithField("intendedVpnIp", hostinfo.vpnIp).WithField("haveVpnIp", vpnIp). - WithField("udpAddr", addr).WithField("certName", certName). - WithField("handshake", m{"stage": 2, "style": "ix_psk0"}). - Info("Incorrect host responded to handshake") - - // Release our old handshake from pending, it should not continue - f.handshakeManager.DeleteHostInfo(hostinfo) - - // Create a new hostinfo/handshake for the intended vpn ip - f.handshakeManager.StartHandshake(hostinfo.vpnIp, func(newHH *HandshakeHostInfo) { - //TODO: this doesnt know if its being added or is being used for caching a packet - // Block the current used address - newHH.hostinfo.remotes = hostinfo.remotes - newHH.hostinfo.remotes.BlockRemote(addr) - - // Get the correct remote list for the host we did handshake with - hostinfo.remotes = f.lightHouse.QueryCache(vpnIp) - - f.l.WithField("blockedUdpAddrs", newHH.hostinfo.remotes.CopyBlockedRemotes()).WithField("vpnIp", vpnIp). - WithField("remotes", newHH.hostinfo.remotes.CopyAddrs(f.hostMap.preferredRanges)). - Info("Blocked addresses for handshakes") - - // Swap the packet store to benefit the original intended recipient - newHH.packetStore = hh.packetStore - hh.packetStore = []*cachedPacket{} - - // Finally, put the correct vpn ip in the host info, tell them to close the tunnel, and return true to tear down - hostinfo.vpnIp = vpnIp - f.sendCloseTunnel(hostinfo) - }) - - return true - } - - // Mark packet 2 as seen so it doesn't show up as missed - ci.window.Update(f.l, 2) - - duration := time.Since(hh.startTime).Nanoseconds() - f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr). - WithField("certName", certName). - WithField("fingerprint", fingerprint). - WithField("issuer", issuer). - WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex). - WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}). - WithField("durationNs", duration). - WithField("sentCachedPackets", len(hh.packetStore)). - Info("Handshake message received") - - hostinfo.remoteIndexId = hs.Details.ResponderIndex - hostinfo.lastHandshakeTime = hs.Details.Time - - // Store their cert and our symmetric keys - ci.peerCert = remoteCert - ci.dKey = NewNebulaCipherState(dKey) - ci.eKey = NewNebulaCipherState(eKey) - - // Make sure the current udpAddr being used is set for responding - if addr != nil { - hostinfo.SetRemote(addr) - } else { - hostinfo.relayState.InsertRelayTo(via.relayHI.vpnIp) - } - - // Build up the radix for the firewall if we have subnets in the cert - hostinfo.CreateRemoteCIDR(remoteCert) - - // Complete our handshake and update metrics, this will replace any existing tunnels for this vpnIp - f.handshakeManager.Complete(hostinfo, f) - f.connectionManager.AddTrafficWatch(hostinfo.localIndexId) - - hostinfo.ConnectionState.messageCounter.Store(2) - - if f.l.Level >= logrus.DebugLevel { - hostinfo.logger(f.l).Debugf("Sending %d stored packets", len(hh.packetStore)) - } - - if len(hh.packetStore) > 0 { - nb := make([]byte, 12, 12) - out := make([]byte, mtu) - for _, cp := range hh.packetStore { - cp.callback(cp.messageType, cp.messageSubType, hostinfo, cp.packet, nb, out) - } - f.cachedPacketMetrics.sent.Inc(int64(len(hh.packetStore))) - } - - hostinfo.remotes.ResetBlockedRemotes() - f.metricHandshakes.Update(duration) - - return false -} diff --git a/handshake_manager.go b/handshake_manager.go index b568cc88..b4bebdc7 100644 --- a/handshake_manager.go +++ b/handshake_manager.go @@ -6,14 +6,16 @@ import ( "crypto/rand" "encoding/binary" "errors" - "net" + "log/slog" + "net/netip" + "slices" "sync" "time" "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/handshake" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/udp" ) @@ -21,7 +23,18 @@ const ( DefaultHandshakeTryInterval = time.Millisecond * 100 DefaultHandshakeRetries = 10 DefaultHandshakeTriggerBuffer = 64 - DefaultUseRelays = true + + // maxCachedPackets is how many unsent packets we'll buffer per pending + // handshake before dropping further ones. + maxCachedPackets = 100 + + // HandshakePacket map keys mirror the IX protocol stage convention: + // stage 0 = the initiator's first message (and what the responder + // receives, stripped of header) + // stage 2 = the responder's reply + // Other handshake patterns will need new keys when added. + handshakePacketStage0 uint8 = 0 + handshakePacketStage2 uint8 = 2 ) var ( @@ -29,15 +42,13 @@ var ( tryInterval: DefaultHandshakeTryInterval, retries: DefaultHandshakeRetries, triggerBuffer: DefaultHandshakeTriggerBuffer, - useRelays: DefaultUseRelays, } ) type HandshakeConfig struct { tryInterval time.Duration - retries int + retries int64 triggerBuffer int - useRelays bool messageMetrics *MessageMetrics } @@ -46,71 +57,74 @@ type HandshakeManager struct { // Mutex for interacting with the vpnIps and indexes maps sync.RWMutex - vpnIps map[iputil.VpnIp]*HandshakeHostInfo + vpnIps map[netip.Addr]*HandshakeHostInfo indexes map[uint32]*HandshakeHostInfo mainHostMap *HostMap lightHouse *LightHouse outside udp.Conn config HandshakeConfig - OutboundHandshakeTimer *LockingTimerWheel[iputil.VpnIp] + OutboundHandshakeTimer *LockingTimerWheel[netip.Addr] messageMetrics *MessageMetrics metricInitiated metrics.Counter metricTimedOut metrics.Counter f *Interface - l *logrus.Logger + l *slog.Logger // can be used to trigger outbound handshake for the given vpnIp - trigger chan iputil.VpnIp + trigger chan netip.Addr } type HandshakeHostInfo struct { sync.Mutex - startTime time.Time // Time that we first started trying with this handshake - ready bool // Is the handshake ready - counter int // How many attempts have we made so far - lastRemotes []*udp.Addr // Remotes that we sent to during the previous attempt - packetStore []*cachedPacket // A set of packets to be transmitted once the handshake completes + startTime time.Time // Time that we first started trying with this handshake + ready bool // Is the handshake ready + initiatingVersionOverride cert.Version // Should we use a non-default cert version for this handshake? + counter int64 // How many attempts have we made so far + lastRemotes []netip.AddrPort // Remotes that we sent to during the previous attempt + lastRelays []netip.Addr // Relays we attempted to use during the previous attempt + packetStore []*cachedPacket // A set of packets to be transmitted once the handshake completes hostinfo *HostInfo + machine *handshake.Machine // The handshake state machine, set during stage 0 (initiator) or beginHandshake (responder multi-message) } -func (hh *HandshakeHostInfo) cachePacket(l *logrus.Logger, t header.MessageType, st header.MessageSubType, packet []byte, f packetCallback, m *cachedPacketMetrics) { - if len(hh.packetStore) < 100 { +func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, st header.MessageSubType, packet []byte, f packetCallback, m *cachedPacketMetrics) { + if len(hh.packetStore) < maxCachedPackets { tempPacket := make([]byte, len(packet)) copy(tempPacket, packet) hh.packetStore = append(hh.packetStore, &cachedPacket{t, st, f, tempPacket}) - if l.Level >= logrus.DebugLevel { - hh.hostinfo.logger(l). - WithField("length", len(hh.packetStore)). - WithField("stored", true). - Debugf("Packet store") + if l.Enabled(context.Background(), slog.LevelDebug) { + hh.hostinfo.logger(l).Debug("Packet store", + "length", len(hh.packetStore), + "stored", true, + ) } } else { m.dropped.Inc(1) - if l.Level >= logrus.DebugLevel { - hh.hostinfo.logger(l). - WithField("length", len(hh.packetStore)). - WithField("stored", false). - Debugf("Packet store") + if l.Enabled(context.Background(), slog.LevelDebug) { + hh.hostinfo.logger(l).Debug("Packet store", + "length", len(hh.packetStore), + "stored", false, + ) } } } -func NewHandshakeManager(l *logrus.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager { +func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager { return &HandshakeManager{ - vpnIps: map[iputil.VpnIp]*HandshakeHostInfo{}, + vpnIps: map[netip.Addr]*HandshakeHostInfo{}, indexes: map[uint32]*HandshakeHostInfo{}, mainHostMap: mainHostMap, lightHouse: lightHouse, outside: outside, config: config, - trigger: make(chan iputil.VpnIp, config.triggerBuffer), - OutboundHandshakeTimer: NewLockingTimerWheel[iputil.VpnIp](config.tryInterval, hsTimeout(config.retries, config.tryInterval)), + trigger: make(chan netip.Addr, config.triggerBuffer), + OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)), messageMetrics: config.messageMetrics, metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil), metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil), @@ -118,59 +132,79 @@ func NewHandshakeManager(l *logrus.Logger, mainHostMap *HostMap, lightHouse *Lig } } -func (c *HandshakeManager) Run(ctx context.Context) { - clockSource := time.NewTicker(c.config.tryInterval) +func (hm *HandshakeManager) Run(ctx context.Context) { + clockSource := time.NewTicker(hm.config.tryInterval) defer clockSource.Stop() for { select { case <-ctx.Done(): return - case vpnIP := <-c.trigger: - c.handleOutbound(vpnIP, true) + case vpnIP := <-hm.trigger: + hm.handleOutbound(vpnIP, true) case now := <-clockSource.C: - c.NextOutboundHandshakeTimerTick(now) + hm.NextOutboundHandshakeTimerTick(now) } } } -func (hm *HandshakeManager) HandleIncoming(addr *udp.Addr, via *ViaSender, packet []byte, h *header.H) { +func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *header.H) { + // Gate on known handshake subtypes. Unknown subtypes (or future ones we + // don't yet support) are dropped here rather than silently routed through + // the IX path. Add a case when introducing a new pattern. + switch h.Subtype { + case header.HandshakeIXPSK0: + // supported + default: + hm.l.Debug("dropping handshake with unsupported subtype", + "from", via, "subtype", h.Subtype) + return + } + // First remote allow list check before we know the vpnIp - if addr != nil { - if !hm.lightHouse.GetRemoteAllowList().AllowUnknownVpnIp(addr.IP) { - hm.l.WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake") + if !via.IsRelayed { + if !hm.lightHouse.GetRemoteAllowList().AllowUnknownVpnAddr(via.UdpAddr.Addr()) { + hm.l.Debug("lighthouse.remote_allow_list denied incoming handshake", "from", via) return } } - switch h.Subtype { - case header.HandshakeIXPSK0: - switch h.MessageCounter { - case 1: - ixHandshakeStage1(hm.f, addr, via, packet, h) - - case 2: - newHostinfo := hm.queryIndex(h.RemoteIndex) - tearDown := ixHandshakeStage2(hm.f, addr, via, newHostinfo, packet, h) - if tearDown && newHostinfo != nil { - hm.DeleteHostInfo(newHostinfo.hostinfo) - } + // First message of a new handshake. The wire format requires RemoteIndex + // to be zero here (the initiator has no responder index to fill in yet), + // and generateIndex never allocates 0, so any non-zero RemoteIndex on a + // stage-1 packet is malformed or someone probing for an index collision. + // Drop without paying the cost of running noise on a pending Machine. + if h.MessageCounter == 1 { + if h.RemoteIndex != 0 { + hm.l.Debug("dropping stage-1 handshake with non-zero RemoteIndex", + "from", via, "remoteIndex", h.RemoteIndex) + return } + hm.beginHandshake(via, packet, h) + return + } + + // Continuation message must match a pending handshake by index. + // Anything else is an orphaned packet (e.g., late retransmit after + // timeout) and is dropped. + if hh := hm.queryIndex(h.RemoteIndex); hh != nil { + hm.continueHandshake(via, hh, packet) + return } } -func (c *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time) { - c.OutboundHandshakeTimer.Advance(now) +func (hm *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time) { + hm.OutboundHandshakeTimer.Advance(now) for { - vpnIp, has := c.OutboundHandshakeTimer.Purge() + vpnIp, has := hm.OutboundHandshakeTimer.Purge() if !has { break } - c.handleOutbound(vpnIp, false) + hm.handleOutbound(vpnIp, false) } } -func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTriggered bool) { +func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered bool) { hh := hm.queryVpnIp(vpnIp) if hh == nil { return @@ -181,12 +215,21 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger hostinfo := hh.hostinfo // If we are out of time, clean up if hh.counter >= hm.config.retries { - hh.hostinfo.logger(hm.l).WithField("udpAddrs", hh.hostinfo.remotes.CopyAddrs(hm.mainHostMap.preferredRanges)). - WithField("initiatorIndex", hh.hostinfo.localIndexId). - WithField("remoteIndex", hh.hostinfo.remoteIndexId). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). - WithField("durationNs", time.Since(hh.startTime).Nanoseconds()). - Info("Handshake timed out") + fields := []any{ + "udpAddrs", hh.hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges()), + "initiatorIndex", hh.hostinfo.localIndexId, + "durationNs", time.Since(hh.startTime).Nanoseconds(), + } + // hh.machine can be nil here if buildStage0Packet never succeeded + // (e.g., no certificate available). In that case there's no useful + // handshake metadata to log. + if hh.machine != nil { + fields = append(fields, "handshake", m{ + "stage": uint64(hh.machine.MessageIndex()), + "style": header.SubTypeName(header.Handshake, hh.machine.Subtype()), + }) + } + hh.hostinfo.logger(hm.l).Info("Handshake timed out", fields...) hm.metricTimedOut.Inc(1) hm.DeleteHostInfo(hostinfo) return @@ -197,22 +240,35 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger // Check if we have a handshake packet to transmit yet if !hh.ready { - if !ixHandshakeStage0(hm.f, hh) { + if !hm.buildStage0Packet(hh) { hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter)) return } } + // TODO: this hardcodes "always retransmit stage 0", which is correct for + // IX (the initiator only ever sends one packet, msg1) but wrong the + // moment a 3+ message pattern lands. The retry loop should resend the + // most recent outgoing message, not always stage 0. That implies + // HandshakeHostInfo tracking a single "currentOutbound" packet (bytes + + // header metadata) that gets replaced as the handshake progresses, + // instead of indexing into HandshakePacket. + stage0 := hostinfo.HandshakePacket[handshakePacketStage0] + hsFields := m{ + "stage": uint64(hh.machine.MessageIndex()), + "style": header.SubTypeName(header.Handshake, hh.machine.Subtype()), + } + // Get a remotes object if we don't already have one. // This is mainly to protect us as this should never be the case // NB ^ This comment doesn't jive. It's how the thing gets initialized. // It's the common path. Should it update every time, in case a future LH query/queries give us more info? if hostinfo.remotes == nil { - hostinfo.remotes = hm.lightHouse.QueryCache(vpnIp) + hostinfo.remotes = hm.lightHouse.QueryCache([]netip.Addr{vpnIp}) } - remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.preferredRanges) - remotesHaveChanged := !udp.AddrSlice(remotes).Equal(hh.lastRemotes) + remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges()) + remotesHaveChanged := !slices.Equal(remotes, hh.lastRemotes) // We only care about a lighthouse trigger if we have new remotes to send to. // This is a very specific optimization for a fast lighthouse reply. @@ -223,7 +279,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger hh.lastRemotes = remotes - // TODO: this will generate a load of queries for hosts with only 1 ip + // This will generate a load of queries for hosts with only 1 ip // (such as ones registered to the lighthouse with only a private IP) // So we only do it one time after attempting 5 handshakes already. if len(remotes) <= 1 && hh.counter == 5 { @@ -234,15 +290,23 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger } // Send the handshake to all known ips, stage 2 takes care of assigning the hostinfo.remote based on the first to reply - var sentTo []*udp.Addr - hostinfo.remotes.ForEach(hm.mainHostMap.preferredRanges, func(addr *udp.Addr, _ bool) { - hm.messageMetrics.Tx(header.Handshake, header.MessageSubType(hostinfo.HandshakePacket[0][1]), 1) - err := hm.outside.WriteTo(hostinfo.HandshakePacket[0], addr) + var sentTo []netip.AddrPort + hostinfo.remotes.ForEach(hm.mainHostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) { + hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1) + err := hm.outside.WriteTo(stage0, addr) if err != nil { - hostinfo.logger(hm.l).WithField("udpAddr", addr). - WithField("initiatorIndex", hostinfo.localIndexId). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). - WithError(err).Error("Failed to send handshake message") + // These repeat every attempt, so match the success log below and only shout when the remotes changed + level := slog.LevelDebug + if remotesHaveChanged { + level = slog.LevelError + } + + hostinfo.logger(hm.l).Log(context.Background(), level, "Failed to send handshake message", + "udpAddr", addr, + "initiatorIndex", hostinfo.localIndexId, + "handshake", hsFields, + "error", err, + ) } else { sentTo = append(sentTo, addr) @@ -252,101 +316,21 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger // Don't be too noisy or confusing if we fail to send a handshake - if we don't get through we'll eventually log a timeout, // so only log when the list of remotes has changed if remotesHaveChanged { - hostinfo.logger(hm.l).WithField("udpAddrs", sentTo). - WithField("initiatorIndex", hostinfo.localIndexId). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). - Info("Handshake message sent") - } else if hm.l.IsLevelEnabled(logrus.DebugLevel) { - hostinfo.logger(hm.l).WithField("udpAddrs", sentTo). - WithField("initiatorIndex", hostinfo.localIndexId). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). - Debug("Handshake message sent") - } - - if hm.config.useRelays && len(hostinfo.remotes.relays) > 0 { - hostinfo.logger(hm.l).WithField("relays", hostinfo.remotes.relays).Info("Attempt to relay through hosts") - // Send a RelayRequest to all known Relay IP's - for _, relay := range hostinfo.remotes.relays { - // Don't relay to myself, and don't relay through the host I'm trying to connect to - if *relay == vpnIp || *relay == hm.lightHouse.myVpnIp { - continue - } - relayHostInfo := hm.mainHostMap.QueryVpnIp(*relay) - if relayHostInfo == nil || relayHostInfo.remote == nil { - hostinfo.logger(hm.l).WithField("relay", relay.String()).Info("Establish tunnel to relay target") - hm.f.Handshake(*relay) - continue - } - // Check the relay HostInfo to see if we already established a relay through it - if existingRelay, ok := relayHostInfo.relayState.QueryRelayForByIp(vpnIp); ok { - switch existingRelay.State { - case Established: - hostinfo.logger(hm.l).WithField("relay", relay.String()).Info("Send handshake via relay") - hm.f.SendVia(relayHostInfo, existingRelay, hostinfo.HandshakePacket[0], make([]byte, 12), make([]byte, mtu), false) - case Requested: - hostinfo.logger(hm.l).WithField("relay", relay.String()).Info("Re-send CreateRelay request") - // Re-send the CreateRelay request, in case the previous one was lost. - m := NebulaControl{ - Type: NebulaControl_CreateRelayRequest, - InitiatorRelayIndex: existingRelay.LocalIndex, - RelayFromIp: uint32(hm.lightHouse.myVpnIp), - RelayToIp: uint32(vpnIp), - } - msg, err := m.Marshal() - if err != nil { - hostinfo.logger(hm.l). - WithError(err). - Error("Failed to marshal Control message to create relay") - } else { - // This must send over the hostinfo, not over hm.Hosts[ip] - hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu)) - hm.l.WithFields(logrus.Fields{ - "relayFrom": hm.lightHouse.myVpnIp, - "relayTo": vpnIp, - "initiatorRelayIndex": existingRelay.LocalIndex, - "relay": *relay}). - Info("send CreateRelayRequest") - } - default: - hostinfo.logger(hm.l). - WithField("vpnIp", vpnIp). - WithField("state", existingRelay.State). - WithField("relay", relayHostInfo.vpnIp). - Errorf("Relay unexpected state") - } - } else { - // No relays exist or requested yet. - if relayHostInfo.remote != nil { - idx, err := AddRelay(hm.l, relayHostInfo, hm.mainHostMap, vpnIp, nil, TerminalType, Requested) - if err != nil { - hostinfo.logger(hm.l).WithField("relay", relay.String()).WithError(err).Info("Failed to add relay to hostmap") - } - - m := NebulaControl{ - Type: NebulaControl_CreateRelayRequest, - InitiatorRelayIndex: idx, - RelayFromIp: uint32(hm.lightHouse.myVpnIp), - RelayToIp: uint32(vpnIp), - } - msg, err := m.Marshal() - if err != nil { - hostinfo.logger(hm.l). - WithError(err). - Error("Failed to marshal Control message to create relay") - } else { - hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu)) - hm.l.WithFields(logrus.Fields{ - "relayFrom": hm.lightHouse.myVpnIp, - "relayTo": vpnIp, - "initiatorRelayIndex": idx, - "relay": *relay}). - Info("send CreateRelayRequest") - } - } - } - } + hostinfo.logger(hm.l).Info("Handshake message sent", + "udpAddrs", sentTo, + "initiatorIndex", hostinfo.localIndexId, + "handshake", hsFields, + ) + } else if hm.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(hm.l).Debug("Handshake message sent", + "udpAddrs", sentTo, + "initiatorIndex", hostinfo.localIndexId, + "handshake", hsFields, + ) } + hm.f.relayManager.StartRelays(hm.f, vpnIp, hh, stage0) + // If a lighthouse triggered this attempt then we are still in the timer wheel and do not need to re-add if !lighthouseTriggered { hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter)) @@ -355,42 +339,42 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger // GetOrHandshake will try to find a hostinfo with a fully formed tunnel or start a new handshake if one is not present // The 2nd argument will be true if the hostinfo is ready to transmit traffic -func (hm *HandshakeManager) GetOrHandshake(vpnIp iputil.VpnIp, cacheCb func(*HandshakeHostInfo)) (*HostInfo, bool) { - // Check the main hostmap and maintain a read lock if our host is not there +func (hm *HandshakeManager) GetOrHandshake(vpnIp netip.Addr, cacheCb func(*HandshakeHostInfo)) (*HostInfo, bool) { hm.mainHostMap.RLock() - if h, ok := hm.mainHostMap.Hosts[vpnIp]; ok { - hm.mainHostMap.RUnlock() + h, ok := hm.mainHostMap.Hosts[vpnIp] + hm.mainHostMap.RUnlock() + + if ok { // Do not attempt promotion if you are a lighthouse if !hm.lightHouse.amLighthouse { - h.TryPromoteBest(hm.mainHostMap.preferredRanges, hm.f) + h.TryPromoteBest(hm.mainHostMap.GetPreferredRanges(), hm.f) } return h, true } - defer hm.mainHostMap.RUnlock() return hm.StartHandshake(vpnIp, cacheCb), false } // StartHandshake will ensure a handshake is currently being attempted for the provided vpn ip -func (hm *HandshakeManager) StartHandshake(vpnIp iputil.VpnIp, cacheCb func(*HandshakeHostInfo)) *HostInfo { +func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*HandshakeHostInfo)) *HostInfo { hm.Lock() - defer hm.Unlock() - if hh, ok := hm.vpnIps[vpnIp]; ok { + if hh, ok := hm.vpnIps[vpnAddr]; ok { // We are already trying to handshake with this vpn ip if cacheCb != nil { cacheCb(hh) } + hm.Unlock() return hh.hostinfo } hostinfo := &HostInfo{ - vpnIp: vpnIp, + vpnAddrs: []netip.Addr{vpnAddr}, HandshakePacket: make(map[uint8][]byte, 0), relayState: RelayState{ - relays: map[iputil.VpnIp]struct{}{}, - relayForByIp: map[iputil.VpnIp]*Relay{}, - relayForByIdx: map[uint32]*Relay{}, + relays: nil, + relayForByAddr: map[netip.Addr]*Relay{}, + relayForByIdx: map[uint32]*Relay{}, }, } @@ -398,9 +382,9 @@ func (hm *HandshakeManager) StartHandshake(vpnIp iputil.VpnIp, cacheCb func(*Han hostinfo: hostinfo, startTime: time.Now(), } - hm.vpnIps[vpnIp] = hh + hm.vpnIps[vpnAddr] = hh hm.metricInitiated.Inc(1) - hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval) + hm.OutboundHandshakeTimer.Add(vpnAddr, hm.config.tryInterval) if cacheCb != nil { cacheCb(hh) @@ -408,20 +392,21 @@ func (hm *HandshakeManager) StartHandshake(vpnIp iputil.VpnIp, cacheCb func(*Han // If this is a static host, we don't need to wait for the HostQueryReply // We can trigger the handshake right now - _, doTrigger := hm.lightHouse.GetStaticHostList()[vpnIp] + _, doTrigger := hm.lightHouse.GetStaticHostList()[vpnAddr] if !doTrigger { // Add any calculated remotes, and trigger early handshake if one found - doTrigger = hm.lightHouse.addCalculatedRemotes(vpnIp) + doTrigger = hm.lightHouse.addCalculatedRemotes(vpnAddr) } if doTrigger { select { - case hm.trigger <- vpnIp: + case hm.trigger <- vpnAddr: default: } } - hm.lightHouse.QueryServer(vpnIp) + hm.Unlock() + hm.lightHouse.QueryServer(vpnAddr) return hostinfo } @@ -442,23 +427,20 @@ var ( // // ErrLocalIndexCollision if we already have an entry in the main or pending // hostmap for the hostinfo.localIndexId. -func (c *HandshakeManager) CheckAndComplete(hostinfo *HostInfo, handshakePacket uint8, f *Interface) (*HostInfo, error) { - c.mainHostMap.Lock() - defer c.mainHostMap.Unlock() - c.Lock() - defer c.Unlock() +func (hm *HandshakeManager) CheckAndComplete(hostinfo *HostInfo, handshakePacket uint8, f *Interface) (*HostInfo, error) { + hm.mainHostMap.Lock() + defer hm.mainHostMap.Unlock() + hm.Lock() + defer hm.Unlock() // Check if we already have a tunnel with this vpn ip - existingHostInfo, found := c.mainHostMap.Hosts[hostinfo.vpnIp] + existingHostInfo, found := hm.mainHostMap.Hosts[hostinfo.vpnAddrs[0]] if found && existingHostInfo != nil { - testHostInfo := existingHostInfo - for testHostInfo != nil { - // Is it just a delayed handshake packet? + // Is it just a delayed handshake packet? Check every hostinfo we hold for this address. + for _, testHostInfo := range hm.mainHostMap.unlockedGetHostList(hostinfo.vpnAddrs[0]) { if bytes.Equal(hostinfo.HandshakePacket[handshakePacket], testHostInfo.HandshakePacket[handshakePacket]) { return testHostInfo, ErrAlreadySeen } - - testHostInfo = testHostInfo.next } // Is this a newer handshake? @@ -466,31 +448,31 @@ func (c *HandshakeManager) CheckAndComplete(hostinfo *HostInfo, handshakePacket return existingHostInfo, ErrExistingHostInfo } - existingHostInfo.logger(c.l).Info("Taking new handshake") + existingHostInfo.logger(hm.l).Info("Taking new handshake") } - existingIndex, found := c.mainHostMap.Indexes[hostinfo.localIndexId] + existingIndex, found := hm.mainHostMap.Indexes[hostinfo.localIndexId] if found { // We have a collision, but for a different hostinfo return existingIndex, ErrLocalIndexCollision } - existingPendingIndex, found := c.indexes[hostinfo.localIndexId] + existingPendingIndex, found := hm.indexes[hostinfo.localIndexId] if found && existingPendingIndex.hostinfo != hostinfo { // We have a collision, but for a different hostinfo - return existingIndex, ErrLocalIndexCollision + return existingPendingIndex.hostinfo, ErrLocalIndexCollision } - existingRemoteIndex, found := c.mainHostMap.RemoteIndexes[hostinfo.remoteIndexId] - if found && existingRemoteIndex != nil && existingRemoteIndex.vpnIp != hostinfo.vpnIp { + existingRemoteIndex, found := hm.mainHostMap.RemoteIndexes[hostinfo.remoteIndexId] + if found && existingRemoteIndex != nil && existingRemoteIndex.vpnAddrs[0] != hostinfo.vpnAddrs[0] { // We have a collision, but this can happen since we can't control // the remote ID. Just log about the situation as a note. - hostinfo.logger(c.l). - WithField("remoteIndex", hostinfo.remoteIndexId).WithField("collision", existingRemoteIndex.vpnIp). - Info("New host shadows existing host remoteIndex") + hostinfo.logger(hm.l).Info("New host shadows existing host remoteIndex", + "collision", existingRemoteIndex.vpnAddrs, + ) } - c.mainHostMap.unlockedAddHostInfo(hostinfo, f) + hm.mainHostMap.unlockedAddHostInfo(hostinfo, f) return existingHostInfo, nil } @@ -507,9 +489,9 @@ func (hm *HandshakeManager) Complete(hostinfo *HostInfo, f *Interface) { if found && existingRemoteIndex != nil { // We have a collision, but this can happen since we can't control // the remote ID. Just log about the situation as a note. - hostinfo.logger(hm.l). - WithField("remoteIndex", hostinfo.remoteIndexId).WithField("collision", existingRemoteIndex.vpnIp). - Info("New host shadows existing host remoteIndex") + hostinfo.logger(hm.l).Info("New host shadows existing host remoteIndex", + "collision", existingRemoteIndex.vpnAddrs, + ) } // We need to remove from the pending hostmap first to avoid undoing work when after to the main hostmap. @@ -520,16 +502,16 @@ func (hm *HandshakeManager) Complete(hostinfo *HostInfo, f *Interface) { // allocateIndex generates a unique localIndexId for this HostInfo // and adds it to the pendingHostMap. Will error if we are unable to generate // a unique localIndexId -func (hm *HandshakeManager) allocateIndex(hh *HandshakeHostInfo) error { +func (hm *HandshakeManager) allocateIndex(hh *HandshakeHostInfo) (uint32, error) { hm.mainHostMap.RLock() defer hm.mainHostMap.RUnlock() hm.Lock() defer hm.Unlock() - for i := 0; i < 32; i++ { + for range 32 { index, err := generateIndex(hm.l) if err != nil { - return err + return 0, err } _, inPending := hm.indexes[index] @@ -538,38 +520,44 @@ func (hm *HandshakeManager) allocateIndex(hh *HandshakeHostInfo) error { if !inMain && !inPending { hh.hostinfo.localIndexId = index hm.indexes[index] = hh - return nil + return index, nil } } - return errors.New("failed to generate unique localIndexId") + return 0, errors.New("failed to generate unique localIndexId") } -func (c *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) { - c.Lock() - defer c.Unlock() - c.unlockedDeleteHostInfo(hostinfo) +func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) { + hm.Lock() + defer hm.Unlock() + hm.unlockedDeleteHostInfo(hostinfo) } -func (c *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) { - delete(c.vpnIps, hostinfo.vpnIp) - if len(c.vpnIps) == 0 { - c.vpnIps = map[iputil.VpnIp]*HandshakeHostInfo{} +func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) { + for _, addr := range hostinfo.vpnAddrs { + if cur, ok := hm.vpnIps[addr]; ok && cur.hostinfo == hostinfo { + delete(hm.vpnIps, addr) + } } - delete(c.indexes, hostinfo.localIndexId) - if len(c.vpnIps) == 0 { - c.indexes = map[uint32]*HandshakeHostInfo{} + if len(hm.vpnIps) == 0 { + hm.vpnIps = map[netip.Addr]*HandshakeHostInfo{} } - if c.l.Level >= logrus.DebugLevel { - c.l.WithField("hostMap", m{"mapTotalSize": len(c.vpnIps), - "vpnIp": hostinfo.vpnIp, "indexNumber": hostinfo.localIndexId, "remoteIndexNumber": hostinfo.remoteIndexId}). - Debug("Pending hostmap hostInfo deleted") + delete(hm.indexes, hostinfo.localIndexId) + if len(hm.indexes) == 0 { + hm.indexes = map[uint32]*HandshakeHostInfo{} + } + + if hm.l.Enabled(context.Background(), slog.LevelDebug) { + hm.l.Debug("Pending hostmap hostInfo deleted", + "hostMap", m{"mapTotalSize": len(hm.vpnIps), + "vpnAddrs": hostinfo.vpnAddrs, "indexNumber": hostinfo.localIndexId, "remoteIndexNumber": hostinfo.remoteIndexId}, + ) } } -func (hm *HandshakeManager) QueryVpnIp(vpnIp iputil.VpnIp) *HostInfo { +func (hm *HandshakeManager) QueryVpnAddr(vpnIp netip.Addr) *HostInfo { hh := hm.queryVpnIp(vpnIp) if hh != nil { return hh.hostinfo @@ -578,7 +566,7 @@ func (hm *HandshakeManager) QueryVpnIp(vpnIp iputil.VpnIp) *HostInfo { } -func (hm *HandshakeManager) queryVpnIp(vpnIp iputil.VpnIp) *HandshakeHostInfo { +func (hm *HandshakeManager) queryVpnIp(vpnIp netip.Addr) *HandshakeHostInfo { hm.RLock() defer hm.RUnlock() return hm.vpnIps[vpnIp] @@ -598,42 +586,42 @@ func (hm *HandshakeManager) queryIndex(index uint32) *HandshakeHostInfo { return hm.indexes[index] } -func (c *HandshakeManager) GetPreferredRanges() []*net.IPNet { - return c.mainHostMap.preferredRanges +func (hm *HandshakeManager) GetPreferredRanges() []netip.Prefix { + return hm.mainHostMap.GetPreferredRanges() } -func (c *HandshakeManager) ForEachVpnIp(f controlEach) { - c.RLock() - defer c.RUnlock() +func (hm *HandshakeManager) ForEachVpnAddr(f controlEach) { + hm.RLock() + defer hm.RUnlock() - for _, v := range c.vpnIps { + for _, v := range hm.vpnIps { f(v.hostinfo) } } -func (c *HandshakeManager) ForEachIndex(f controlEach) { - c.RLock() - defer c.RUnlock() +func (hm *HandshakeManager) ForEachIndex(f controlEach) { + hm.RLock() + defer hm.RUnlock() - for _, v := range c.indexes { + for _, v := range hm.indexes { f(v.hostinfo) } } -func (c *HandshakeManager) EmitStats() { - c.RLock() - hostLen := len(c.vpnIps) - indexLen := len(c.indexes) - c.RUnlock() +func (hm *HandshakeManager) EmitStats() { + hm.RLock() + hostLen := len(hm.vpnIps) + indexLen := len(hm.indexes) + hm.RUnlock() metrics.GetOrRegisterGauge("hostmap.pending.hosts", nil).Update(int64(hostLen)) metrics.GetOrRegisterGauge("hostmap.pending.indexes", nil).Update(int64(indexLen)) - c.mainHostMap.EmitStats() + hm.mainHostMap.EmitStats() } // Utility functions below -func generateIndex(l *logrus.Logger) (uint32, error) { +func generateIndex(l *slog.Logger) (uint32, error) { b := make([]byte, 4) // Let zero mean we don't know the ID, so don't generate zero @@ -641,20 +629,538 @@ func generateIndex(l *logrus.Logger) (uint32, error) { for index == 0 { _, err := rand.Read(b) if err != nil { - l.Errorln(err) + l.Error("Failed to generate index", "error", err) return 0, err } index = binary.BigEndian.Uint32(b) } - if l.Level >= logrus.DebugLevel { - l.WithField("index", index). - Debug("Generated index") + if l.Enabled(context.Background(), slog.LevelDebug) { + l.Debug("Generated index", "index", index) } return index, nil } -func hsTimeout(tries int, interval time.Duration) time.Duration { - return time.Duration(tries / 2 * ((2 * int(interval)) + (tries-1)*int(interval))) +func hsTimeout(tries int64, interval time.Duration) time.Duration { + return time.Duration(tries / 2 * ((2 * int64(interval)) + (tries-1)*int64(interval))) +} + +// buildStage0Packet creates the initial handshake packet for the initiator. +func (hm *HandshakeManager) buildStage0Packet(hh *HandshakeHostInfo) bool { + cs := hm.f.pki.getCertState() + v := cs.DefaultVersion() + if hh.initiatingVersionOverride != cert.VersionPre1 { + v = hh.initiatingVersionOverride + } else if v < cert.Version2 { + for _, a := range hh.hostinfo.vpnAddrs { + if a.Is6() { + v = cert.Version2 + break + } + } + } + + cred := cs.GetCredential(v) + if cred == nil { + hm.f.l.Error("Unable to handshake with host because no certificate is available", + "vpnAddrs", hh.hostinfo.vpnAddrs, "certVersion", v) + return false + } + + machine, err := handshake.NewMachine( + v, cs.GetCredential, + hm.certVerifier(), func() (uint32, error) { return hm.allocateIndex(hh) }, + true, header.HandshakeIXPSK0, + ) + if err != nil { + hm.f.l.Error("Failed to create handshake machine", + "vpnAddrs", hh.hostinfo.vpnAddrs, "error", err) + return false + } + + msg, err := machine.Initiate(nil) + if err != nil { + hm.f.l.Error("Failed to initiate handshake", + "vpnAddrs", hh.hostinfo.vpnAddrs, "error", err) + return false + } + + // hostinfo.ConnectionState stays nil until the handshake completes in + // continueHandshake. Pre-completion control surfaces guard with nil + // checks; the data plane never observes a pending hostinfo. + hh.hostinfo.HandshakePacket[handshakePacketStage0] = msg + hh.machine = machine + hh.ready = true + return true +} + +// beginHandshake handles an incoming handshake packet that doesn't match any +// existing pending handshake. It creates a new responder Machine and processes +// the first message. +func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *header.H) { + f := hm.f + cs := f.pki.getCertState() + + v := cs.DefaultVersion() + if cs.GetCredential(v) == nil { + f.l.Error("Unable to handshake with host because no certificate is available", + "from", via, "certVersion", v) + return + } + + machine, err := handshake.NewMachine( + v, cs.GetCredential, + hm.certVerifier(), func() (uint32, error) { return generateIndex(f.l) }, + false, header.HandshakeIXPSK0, + ) + if err != nil { + f.l.Error("Failed to create handshake machine", "from", via, "error", err) + return + } + + response, result, err := machine.ProcessPacket(nil, packet) + if err != nil { + f.l.Error("Failed to process handshake packet", "from", via, "error", err) + return + } + + if result == nil { + // Multi-message pattern: the responder Machine would need to be + // registered in hm.indexes so a future inbound packet finds it via + // continueHandshake. The current manager doesn't do that yet, so + // fail loudly rather than silently dropping the in-flight handshake. + // TODO: support multi-message responder flows (XX, pqIX, etc.). + // See also the IX-shaped cipher key assignment in handshake.Machine. + f.l.Error("multi-message handshake responder is not supported", + "from", via, "error", handshake.ErrMultiMessageUnsupported) + return + } + + remoteCert := result.RemoteCert + if remoteCert == nil { + f.l.Error("Handshake did not produce a peer certificate", "from", via) + return + } + + // Validate peer identity + vpnAddrs, anyVpnAddrsInCommon, ok := hm.validatePeerCert(via, remoteCert) + if !ok { + return + } + + hostinfo := &HostInfo{ + ConnectionState: newConnectionStateFromResult(result), + localIndexId: result.LocalIndex, + remoteIndexId: result.RemoteIndex, + vpnAddrs: vpnAddrs, + HandshakePacket: make(map[uint8][]byte, 0), + lastHandshakeTime: result.HandshakeTime, + relayState: RelayState{ + relays: nil, + relayForByAddr: map[netip.Addr]*Relay{}, + relayForByIdx: map[uint32]*Relay{}, + }, + } + + msg := "Handshake message received" + if !anyVpnAddrsInCommon { + msg = "Handshake message received, but no vpnNetworks in common." + } + f.l.Info(msg, + "vpnAddrs", vpnAddrs, + "from", via, + "certName", remoteCert.Certificate.Name(), + "certVersion", remoteCert.Certificate.Version(), + "fingerprint", remoteCert.Fingerprint, + "issuer", remoteCert.Certificate.Issuer(), + "initiatorIndex", result.RemoteIndex, + "responderIndex", result.LocalIndex, + "handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())}, + ) + + // packet aliases the listener's incoming buffer, so this copy must stay. + hostinfo.HandshakePacket[handshakePacketStage0] = make([]byte, len(packet[header.Len:])) + copy(hostinfo.HandshakePacket[handshakePacketStage0], packet[header.Len:]) + + // response was freshly allocated by ProcessPacket; safe to retain directly. + if response != nil { + hostinfo.HandshakePacket[handshakePacketStage2] = response + } + + hostinfo.remotes = f.lightHouse.QueryCache(vpnAddrs) + if !via.IsRelayed { + hostinfo.SetRemote(via.UdpAddr) + } + hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate) + + existing, err := hm.CheckAndComplete(hostinfo, handshakePacketStage0, f) + if err != nil { + hm.handleCheckAndCompleteError(err, existing, hostinfo, via) + return + } + + hm.sendHandshakeResponse(via, response, hostinfo, false) + hostinfo.remotes.RefreshFromHandshake(vpnAddrs) + + // Don't wait for UpdateWorker + if f.lightHouse.IsAnyLighthouseAddr(vpnAddrs) { + f.lightHouse.TriggerUpdate() + } +} + +// continueHandshake feeds an incoming packet to an existing pending handshake Machine. +func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostInfo, packet []byte) { + f := hm.f + + hh.Lock() + defer hh.Unlock() + + // Re-verify hh is still tracked. Between queryIndex returning and us taking + // hh.Lock, handleOutbound may have timed out and deleted it. Once we hold + // hh.Lock no other deleter can race our index: handleOutbound also takes + // hh.Lock first, and handleRecvError targets a main-hostmap entry with a + // different localIndexId. + hm.RLock() + cur, ok := hm.indexes[hh.hostinfo.localIndexId] + hm.RUnlock() + if !ok || cur != hh { + return + } + + hostinfo := hh.hostinfo + if !via.IsRelayed { + if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) { + f.l.Debug("lighthouse.remote_allow_list denied incoming handshake", + "vpnAddrs", hostinfo.vpnAddrs, "from", via) + return + } + } + + machine := hh.machine + if machine == nil { + f.l.Error("No handshake machine available for continuation", + "vpnAddrs", hostinfo.vpnAddrs, "from", via) + hm.DeleteHostInfo(hostinfo) + return + } + + response, result, err := machine.ProcessPacket(nil, packet) + if err != nil { + // Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn. + if machine.Failed() { + f.l.Warn("Failed to process handshake packet, abandoning", + "vpnAddrs", hostinfo.vpnAddrs, "from", via, "error", err) + hm.DeleteHostInfo(hostinfo) + } else { + f.l.Debug("Failed to process handshake packet", + "vpnAddrs", hostinfo.vpnAddrs, "from", via, "error", err) + } + return + } + + if response != nil { + hm.sendHandshakeResponse(via, response, hostinfo, false) + } + + if result == nil { + return + } + + // Handshake complete; build the ConnectionState now that we have keys and a verified peer cert. + hostinfo.ConnectionState = newConnectionStateFromResult(result) + + remoteCert := result.RemoteCert + if remoteCert == nil { + f.l.Error("Handshake completed without peer certificate", + "vpnAddrs", hostinfo.vpnAddrs, "from", via) + hm.DeleteHostInfo(hostinfo) + return + } + + vpnNetworks := remoteCert.Certificate.Networks() + hostinfo.remoteIndexId = result.RemoteIndex + hostinfo.lastHandshakeTime = result.HandshakeTime + + if !via.IsRelayed { + hostinfo.SetRemote(via.UdpAddr) + } else { + hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0]) + } + + // Verify correct host responded (initiator check) + vpnAddrs := make([]netip.Addr, len(vpnNetworks)) + correctHostResponded := false + anyVpnAddrsInCommon := false + for i, network := range vpnNetworks { + // inside.go drops self-routed packets at the firewall stage, but we'd + // rather not let a self-handshake complete in the first place: it + // wastes a hostmap slot, suppresses no log, and obscures routing + // misconfig. Explicit refusal here mirrors the responder-side check + // in validatePeerCert. + if f.myVpnAddrsTable.Contains(network.Addr()) { + f.l.Error("Refusing to handshake with myself", + "vpnNetworks", vpnNetworks, + "from", via, + "certName", remoteCert.Certificate.Name(), + "certVersion", remoteCert.Certificate.Version(), + "fingerprint", remoteCert.Fingerprint, + "issuer", remoteCert.Certificate.Issuer(), + "handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())}, + ) + hm.DeleteHostInfo(hostinfo) + return + } + vpnAddrs[i] = network.Addr() + if hostinfo.vpnAddrs[0] == network.Addr() { + correctHostResponded = true + } + if f.myVpnNetworksTable.Contains(network.Addr()) { + anyVpnAddrsInCommon = true + } + } + + if !correctHostResponded { + f.l.Info("Incorrect host responded to handshake", + "intendedVpnAddrs", hostinfo.vpnAddrs, + "haveVpnNetworks", vpnNetworks, + "from", via, + "certName", remoteCert.Certificate.Name(), + "certVersion", remoteCert.Certificate.Version(), + "fingerprint", remoteCert.Fingerprint, + "issuer", remoteCert.Certificate.Issuer(), + "handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())}, + ) + + hm.DeleteHostInfo(hostinfo) + hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) { + newHH.hostinfo.remotes = hostinfo.remotes + newHH.hostinfo.remotes.BlockRemote(via) + newHH.packetStore = hh.packetStore + hh.packetStore = []*cachedPacket{} + hostinfo.vpnAddrs = vpnAddrs + f.sendCloseTunnel(hostinfo) + }) + return + } + + duration := time.Since(hh.startTime).Nanoseconds() + msg := "Handshake message received" + if !anyVpnAddrsInCommon { + msg = "Handshake message received, but no vpnNetworks in common." + } + f.l.Info(msg, + "vpnAddrs", vpnAddrs, + "from", via, + "certName", remoteCert.Certificate.Name(), + "certVersion", remoteCert.Certificate.Version(), + "fingerprint", remoteCert.Fingerprint, + "issuer", remoteCert.Certificate.Issuer(), + "initiatorIndex", result.LocalIndex, + "responderIndex", result.RemoteIndex, + "handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())}, + "durationNs", duration, + "sentCachedPackets", len(hh.packetStore), + ) + + hostinfo.vpnAddrs = vpnAddrs + hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate) + + hm.Complete(hostinfo, f) + + if len(hh.packetStore) > 0 { + if f.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(f.l).Debug("Sending stored packets", "count", len(hh.packetStore)) + } + nb := make([]byte, 12, 12) + out := make([]byte, mtu) + for _, cp := range hh.packetStore { + cp.callback(cp.messageType, cp.messageSubType, hostinfo, cp.packet, nb, out) + } + f.cachedPacketMetrics.sent.Inc(int64(len(hh.packetStore))) + } + + hostinfo.remotes.RefreshFromHandshake(vpnAddrs) + f.metricHandshakes.Update(duration) + + // Don't wait for UpdateWorker + if f.lightHouse.IsAnyLighthouseAddr(vpnAddrs) { + f.lightHouse.TriggerUpdate() + } +} + +// validatePeerCert checks the peer certificate for self-connection and remote allow list. +// Returns the VPN addrs, whether any of them fall within one of our own VPN +// networks, and true if valid; false if rejected. +func (hm *HandshakeManager) validatePeerCert(via ViaSender, remoteCert *cert.CachedCertificate) ([]netip.Addr, bool, bool) { + f := hm.f + vpnNetworks := remoteCert.Certificate.Networks() + + // The cert package rejects host certs with no networks at parse time, so + // reaching this state would mean an invariant was bypassed elsewhere. + // Refuse explicitly so downstream code (which indexes vpnAddrs[0]) can't + // panic if that invariant ever changes. + if len(vpnNetworks) == 0 { + f.l.Info("No networks in certificate", + "from", via, "cert", remoteCert) + return nil, false, false + } + + vpnAddrs := make([]netip.Addr, len(vpnNetworks)) + anyVpnAddrsInCommon := false + + for i, network := range vpnNetworks { + if f.myVpnAddrsTable.Contains(network.Addr()) { + f.l.Error("Refusing to handshake with myself", + "vpnNetworks", vpnNetworks, + "from", via, + "certName", remoteCert.Certificate.Name(), + "certVersion", remoteCert.Certificate.Version(), + "fingerprint", remoteCert.Fingerprint, + "issuer", remoteCert.Certificate.Issuer(), + ) + return nil, false, false + } + vpnAddrs[i] = network.Addr() + if f.myVpnNetworksTable.Contains(network.Addr()) { + anyVpnAddrsInCommon = true + } + } + + if !via.IsRelayed { + if !f.lightHouse.GetRemoteAllowList().AllowAll(vpnAddrs, via.UdpAddr.Addr()) { + f.l.Debug("lighthouse.remote_allow_list denied incoming handshake", + "vpnAddrs", vpnAddrs, "from", via) + return nil, false, false + } + } + + return vpnAddrs, anyVpnAddrsInCommon, true +} + +// sendHandshakeResponse sends a handshake response via the appropriate transport. +// cached is true when msg is a stored response being retransmitted because +// the peer's stage-1 retransmit landed (the ErrAlreadySeen path); false on a +// fresh response. +func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hostinfo *HostInfo, cached bool) { + if msg == nil { + return + } + + f := hm.f + f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1) + + // Common log fields. peerCert may be nil during intermediate + // multi-message flows (handshake hasn't completed yet); skip the cert + // block if so. + logFields := []any{ + "vpnAddrs", hostinfo.vpnAddrs, + "handshake", m{"stage": uint64(2), "style": header.SubTypeName(header.Handshake, header.HandshakeIXPSK0)}, + "cached", cached, + "initiatorIndex", hostinfo.remoteIndexId, + "responderIndex", hostinfo.localIndexId, + } + if peerCert := hostinfo.ConnectionState.peerCert; peerCert != nil { + logFields = append(logFields, + "certName", peerCert.Certificate.Name(), + "certVersion", peerCert.Certificate.Version(), + "fingerprint", peerCert.Fingerprint, + "issuer", peerCert.Certificate.Issuer(), + ) + } + + if !via.IsRelayed { + fields := append(logFields, "from", via) + err := f.outside.WriteTo(msg, via.UdpAddr) + if err != nil { + f.l.Error("Failed to send handshake message", append(fields, "error", err)...) + } else { + f.l.Info("Handshake message sent", fields...) + } + } else { + if via.relay == nil { + f.l.Error("Handshake send failed: both addr and via.relay are nil.") + return + } + hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0]) + // We received a valid handshake on this relay, so make sure the relay + // state reflects that, in case it had been marked Disestablished. + via.relayHI.relayState.UpdateRelayForByIdxState(via.relay.LocalIndex, Established) + f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false) + f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...) + } +} + +// handleCheckAndCompleteError handles errors from CheckAndComplete. +// This only fires from the responder-side beginHandshake path, after the +// peer cert has been validated and ConnectionState populated, so peerCert +// is always non-nil for the cases that log it. +func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hostinfo *HostInfo, via ViaSender) { + f := hm.f + peerCert := hostinfo.ConnectionState.peerCert + hsFields := m{"stage": uint64(1), "style": header.SubTypeName(header.Handshake, header.HandshakeIXPSK0)} + + switch err { + case ErrAlreadySeen: + if existing.SetRemoteIfPreferred(f.hostMap, via) { + f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu)) + } + // Resend the original response. The peer is committed to that response's + // ephemeral keys; a freshly-built one would have different keys and break + // the tunnel even though both sides "completed" the handshake. + if msg := existing.HandshakePacket[handshakePacketStage2]; msg != nil { + hm.sendHandshakeResponse(via, msg, existing, true) + } + + case ErrExistingHostInfo: + f.l.Info("Handshake too old", + "vpnAddrs", hostinfo.vpnAddrs, + "from", via, + "certName", peerCert.Certificate.Name(), + "certVersion", peerCert.Certificate.Version(), + "fingerprint", peerCert.Fingerprint, + "issuer", peerCert.Certificate.Issuer(), + "oldHandshakeTime", existing.lastHandshakeTime, + "newHandshakeTime", hostinfo.lastHandshakeTime, + "initiatorIndex", hostinfo.remoteIndexId, + "responderIndex", hostinfo.localIndexId, + "handshake", hsFields, + ) + f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu)) + + case ErrLocalIndexCollision: + f.l.Error("Failed to add HostInfo due to localIndex collision", + "vpnAddrs", hostinfo.vpnAddrs, + "from", via, + "certName", peerCert.Certificate.Name(), + "certVersion", peerCert.Certificate.Version(), + "fingerprint", peerCert.Fingerprint, + "issuer", peerCert.Certificate.Issuer(), + "localIndex", hostinfo.localIndexId, + "initiatorIndex", hostinfo.remoteIndexId, + "responderIndex", hostinfo.localIndexId, + "handshake", hsFields, + ) + + default: + f.l.Error("Failed to add HostInfo to HostMap", + "vpnAddrs", hostinfo.vpnAddrs, + "from", via, + "error", err, + "certName", peerCert.Certificate.Name(), + "certVersion", peerCert.Certificate.Version(), + "fingerprint", peerCert.Fingerprint, + "issuer", peerCert.Certificate.Issuer(), + "initiatorIndex", hostinfo.remoteIndexId, + "responderIndex", hostinfo.localIndexId, + "handshake", hsFields, + ) + } +} + +// certVerifier returns a CertVerifier that validates certs against the current CA pool. +func (hm *HandshakeManager) certVerifier() handshake.CertVerifier { + return func(c cert.Certificate) (*cert.CachedCertificate, error) { + return hm.f.pki.GetCAPool().VerifyCertificate(time.Now(), c) + } } diff --git a/handshake_manager_test.go b/handshake_manager_test.go index 303aa505..5f8383e4 100644 --- a/handshake_manager_test.go +++ b/handshake_manager_test.go @@ -1,13 +1,13 @@ package nebula import ( - "net" + "net/netip" "testing" "time" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/test" "github.com/slackhq/nebula/udp" "github.com/stretchr/testify/assert" @@ -15,18 +15,20 @@ import ( func Test_NewHandshakeManagerVpnIp(t *testing.T) { l := test.NewLogger() - _, vpncidr, _ := net.ParseCIDR("172.1.1.1/24") - _, localrange, _ := net.ParseCIDR("10.1.1.1/24") - ip := iputil.Ip2VpnIp(net.ParseIP("172.1.1.2")) - preferredRanges := []*net.IPNet{localrange} - mainHM := NewHostMap(l, vpncidr, preferredRanges) + localrange := netip.MustParsePrefix("10.1.1.1/24") + ip := netip.MustParseAddr("172.1.1.2") + + preferredRanges := []netip.Prefix{localrange} + mainHM := newHostMap(l) + mainHM.preferredRanges.Store(&preferredRanges) + lh := newTestLighthouse() cs := &CertState{ - RawCertificate: []byte{}, - PrivateKey: []byte{}, - Certificate: &cert.NebulaCertificate{}, - RawCertificateNoKey: []byte{}, + initiatingVersion: cert.Version1, + privateKey: []byte{}, + v1Cert: &dummyCert{version: cert.Version1}, + v1Credential: nil, } blah := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig) @@ -40,10 +42,10 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) { i2 := blah.StartHandshake(ip, nil) assert.Same(t, i, i2) - i.remotes = NewRemoteList(nil) + i.remotes = NewRemoteList([]netip.Addr{}, nil) // Adding something to pending should not affect the main hostmap - assert.Len(t, mainHM.Hosts, 0) + assert.Empty(t, mainHM.Hosts) // Confirm they are in the pending index list assert.Contains(t, blah.vpnIps, ip) @@ -64,7 +66,7 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) { assert.NotContains(t, blah.vpnIps, ip) } -func testCountTimerWheelEntries(tw *LockingTimerWheel[iputil.VpnIp]) (c int) { +func testCountTimerWheelEntries(tw *LockingTimerWheel[netip.Addr]) (c int) { for _, i := range tw.t.wheel { n := i.Head for n != nil { @@ -78,16 +80,158 @@ func testCountTimerWheelEntries(tw *LockingTimerWheel[iputil.VpnIp]) (c int) { type mockEncWriter struct { } -func (mw *mockEncWriter) SendMessageToVpnIp(t header.MessageType, st header.MessageSubType, vpnIp iputil.VpnIp, p, nb, out []byte) { +func (mw *mockEncWriter) SendMessageToVpnAddr(_ header.MessageType, _ header.MessageSubType, _ netip.Addr, _, _, _ []byte) { return } -func (mw *mockEncWriter) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, nocopy bool) { +func (mw *mockEncWriter) SendVia(_ *HostInfo, _ *Relay, _, _, _ []byte, _ bool) { return } -func (mw *mockEncWriter) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) { +func (mw *mockEncWriter) SendMessageToHostInfo(_ header.MessageType, _ header.MessageSubType, _ *HostInfo, _, _, _ []byte) { return } -func (mw *mockEncWriter) Handshake(vpnIP iputil.VpnIp) {} +func (mw *mockEncWriter) Handshake(_ netip.Addr) {} + +func (mw *mockEncWriter) GetHostInfo(_ netip.Addr) *HostInfo { + return nil +} + +func (mw *mockEncWriter) GetCertState() *CertState { + return &CertState{initiatingVersion: cert.Version2} +} + +func TestValidatePeerCert(t *testing.T) { + l := test.NewLogger() + + myNetwork := netip.MustParsePrefix("10.0.0.1/24") + myAddrTable := new(bart.Lite) + myAddrTable.Insert(netip.PrefixFrom(myNetwork.Addr(), myNetwork.Addr().BitLen())) + myNetTable := new(bart.Lite) + myNetTable.Insert(myNetwork.Masked()) + + newHM := func() *HandshakeManager { + hm := NewHandshakeManager(l, newHostMap(l), newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig) + hm.f = &Interface{ + handshakeManager: hm, + pki: &PKI{}, + l: l, + myVpnAddrsTable: myAddrTable, + myVpnNetworksTable: myNetTable, + lightHouse: hm.lightHouse, + } + return hm + } + + cached := func(networks ...netip.Prefix) *cert.CachedCertificate { + return &cert.CachedCertificate{ + Certificate: &dummyCert{name: "peer", networks: networks}, + } + } + + via := ViaSender{ + UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242"), + IsRelayed: true, // skip the remote allow list (covered separately) + } + + t.Run("addr inside our networks sets anyVpnAddrsInCommon", func(t *testing.T) { + hm := newHM() + // 10.0.0.2 falls inside our 10.0.0.0/24 + addrs, common, ok := hm.validatePeerCert(via, cached(netip.MustParsePrefix("10.0.0.2/24"))) + assert.True(t, ok) + assert.True(t, common) + assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.0.0.2")}, addrs) + }) + + t.Run("addr outside our networks leaves anyVpnAddrsInCommon false", func(t *testing.T) { + hm := newHM() + addrs, common, ok := hm.validatePeerCert(via, cached(netip.MustParsePrefix("192.168.1.5/24"))) + assert.True(t, ok) + assert.False(t, common) + assert.Equal(t, []netip.Addr{netip.MustParseAddr("192.168.1.5")}, addrs) + }) + + t.Run("any matching network is enough", func(t *testing.T) { + hm := newHM() + addrs, common, ok := hm.validatePeerCert(via, cached( + netip.MustParsePrefix("192.168.1.5/24"), + netip.MustParsePrefix("10.0.0.42/24"), + )) + assert.True(t, ok) + assert.True(t, common) + assert.Len(t, addrs, 2) + }) + + t.Run("self-handshake is rejected", func(t *testing.T) { + hm := newHM() + // 10.0.0.1 is in myVpnAddrsTable + addrs, common, ok := hm.validatePeerCert(via, cached(netip.MustParsePrefix("10.0.0.1/24"))) + assert.False(t, ok) + assert.False(t, common) + assert.Nil(t, addrs) + }) + + t.Run("cert with no networks is rejected", func(t *testing.T) { + hm := newHM() + addrs, common, ok := hm.validatePeerCert(via, cached()) + assert.False(t, ok) + assert.False(t, common) + assert.Nil(t, addrs) + }) +} + +func TestHandleIncomingDispatch(t *testing.T) { + l := test.NewLogger() + + newHM := func() *HandshakeManager { + hm := NewHandshakeManager(l, newHostMap(l), newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig) + hm.f = &Interface{ + handshakeManager: hm, + pki: &PKI{}, + l: l, + } + return hm + } + + via := ViaSender{ + UdpAddr: netip.MustParseAddrPort("198.51.100.7:4242"), + IsRelayed: true, // bypass remote allow list + } + + // A packet body of zero length is fine for these tests: dispatch is + // gated on header fields, and we assert that we never reach noise/cert + // processing for any of the malformed shapes here. + pkt := make([]byte, header.Len) + + t.Run("unsupported subtype dropped", func(t *testing.T) { + hm := newHM() + h := &header.H{Type: header.Handshake, Subtype: header.MessageSubType(99), MessageCounter: 1} + hm.HandleIncoming(via, pkt, h) + assert.Empty(t, hm.indexes, "no pending handshake should be created") + }) + + t.Run("stage-1 with non-zero RemoteIndex dropped", func(t *testing.T) { + hm := newHM() + h := &header.H{ + Type: header.Handshake, + Subtype: header.HandshakeIXPSK0, + RemoteIndex: 0xdeadbeef, + MessageCounter: 1, + } + hm.HandleIncoming(via, pkt, h) + assert.Empty(t, hm.indexes, "spoofed stage-1 must not create a pending machine") + }) + + t.Run("continuation with no matching pending index dropped", func(t *testing.T) { + hm := newHM() + h := &header.H{ + Type: header.Handshake, + Subtype: header.HandshakeIXPSK0, + RemoteIndex: 0xcafef00d, + MessageCounter: 2, + } + hm.HandleIncoming(via, pkt, h) + assert.Empty(t, hm.indexes, "orphan stage-2 must not create state") + }) +} diff --git a/header/header.go b/header/header.go index 50b7d62b..b973141f 100644 --- a/header/header.go +++ b/header/header.go @@ -19,7 +19,7 @@ import ( // |-----------------------------------------------------------------------| // | payload... | -type m map[string]interface{} +type m = map[string]any const ( Version uint8 = 1 @@ -174,6 +174,10 @@ func (h *H) SubTypeName() string { return SubTypeName(h.Type, h.Subtype) } +func (h *H) IsValidSubType() bool { + return IsValidSubType(h.Type, h.Subtype) +} + // SubTypeName will transform a nebula message sub type into a human string func SubTypeName(t MessageType, s MessageSubType) string { if n, ok := subTypeMap[t]; ok { @@ -185,6 +189,16 @@ func SubTypeName(t MessageType, s MessageSubType) string { return "unknown" } +func IsValidSubType(t MessageType, s MessageSubType) bool { + if n, ok := subTypeMap[t]; ok { + if _, ok := (*n)[s]; ok { + return true + } + } + + return false +} + // NewHeader turns bytes into a header func NewHeader(b []byte) (*H, error) { h := new(H) diff --git a/header/header_test.go b/header/header_test.go index 765a0066..a7e53742 100644 --- a/header/header_test.go +++ b/header/header_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type headerTest struct { @@ -111,7 +112,7 @@ func TestHeader_String(t *testing.T) { func TestHeader_MarshalJSON(t *testing.T) { b, err := (&H{100, Test, TestRequest, 99, 98, 97}).MarshalJSON() - assert.Nil(t, err) + require.NoError(t, err) assert.Equal( t, "{\"messageCounter\":97,\"remoteIndex\":98,\"reserved\":99,\"subType\":\"testRequest\",\"type\":\"test\",\"version\":100}", diff --git a/hostmap.go b/hostmap.go index a5adeb99..45515fc3 100644 --- a/hostmap.go +++ b/hostmap.go @@ -1,27 +1,30 @@ package nebula import ( + "context" + "encoding/json" "errors" + "fmt" + "log/slog" "net" + "net/netip" + "slices" "sync" "sync/atomic" "time" + "github.com/gaissmai/bart" "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cert" - "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" - "github.com/slackhq/nebula/udp" + "github.com/slackhq/nebula/logging" ) -// const ProbeLen = 100 const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address const defaultReQueryEvery = 5000 // Count of packets sent before re-querying a hostinfo to the lighthouse const defaultReQueryWait = time.Minute // Minimum amount of seconds to wait before re-querying a hostinfo the lighthouse. Evaluated every ReQueryEvery const MaxRemotes = 10 -const maxRecvError = 4 // MaxHostInfosPerVpnIp is the max number of hostinfos we will track for a given vpn ip // 5 allows for an initial handshake and each host pair re-handshaking twice @@ -35,6 +38,7 @@ const ( Requested = iota PeerRequested Established + Disestablished ) const ( @@ -48,19 +52,26 @@ type Relay struct { State int LocalIndex uint32 RemoteIndex uint32 - PeerIp iputil.VpnIp + PeerAddr netip.Addr } type HostMap struct { - sync.RWMutex //Because we concurrently read and write to our maps - Indexes map[uint32]*HostInfo - Relays map[uint32]*HostInfo // Maps a Relay IDX to a Relay HostInfo object - RemoteIndexes map[uint32]*HostInfo - Hosts map[iputil.VpnIp]*HostInfo - preferredRanges []*net.IPNet - vpnCIDR *net.IPNet - metricsEnabled bool - l *logrus.Logger + sync.RWMutex //Because we concurrently read and write to our maps + Indexes map[uint32]*HostInfo + Relays map[uint32]*HostInfo // Maps a Relay IDX to a Relay HostInfo object + RemoteIndexes map[uint32]*HostInfo + // Hosts maps a vpn address to its primary hostinfo, one entry per address we hold a tunnel + // for. moreHosts only has an entry while an address is held by 2 or more hostinfos and stores + // the full most-recent-first list; moreHosts[a][0] is always the same hostinfo as Hosts[a]. + // Each address gets its own independent list, so a hostinfo owning multiple addresses can + // never corrupt another address's ordering the way the old shared next/prev chain could. + // Entries in moreHosts are only ever written by unlockedSetHostsForAddr; Hosts is written + // directly only in the single-hostinfo fast paths where moreHosts is known to have no entry, + // and unlockedDeleteHostInfo swaps either map for a fresh one when it fully drains. + Hosts map[netip.Addr]*HostInfo + moreHosts map[netip.Addr][]*HostInfo + preferredRanges atomic.Pointer[[]netip.Prefix] + l *slog.Logger } // For synchronization, treat the pointed-to Relay struct as immutable. To edit the Relay @@ -69,15 +80,45 @@ type HostMap struct { type RelayState struct { sync.RWMutex - relays map[iputil.VpnIp]struct{} // Set of VpnIp's of Hosts to use as relays to access this peer - relayForByIp map[iputil.VpnIp]*Relay // Maps VpnIps of peers for which this HostInfo is a relay to some Relay info - relayForByIdx map[uint32]*Relay // Maps a local index to some Relay info + relays []netip.Addr // Ordered set of VpnAddrs of Hosts to use as relays to access this peer + // For data race avoidance, the contents of a *Relay are treated immutably. To update a *Relay, copy the existing data, + // modify what needs to be updated, and store the new modified copy in the relayForByIp and relayForByIdx maps (with + // the RelayState Lock held) + relayForByAddr map[netip.Addr]*Relay // Maps vpnAddr of peers for which this HostInfo is a relay to some Relay info + relayForByIdx map[uint32]*Relay // Maps a local index to some Relay info } -func (rs *RelayState) DeleteRelay(ip iputil.VpnIp) { +func (rs *RelayState) DeleteRelay(ip netip.Addr) { rs.Lock() defer rs.Unlock() - delete(rs.relays, ip) + for idx, val := range rs.relays { + if val == ip { + rs.relays = append(rs.relays[:idx], rs.relays[idx+1:]...) + return + } + } +} + +func (rs *RelayState) UpdateRelayForByIpState(vpnIp netip.Addr, state int) { + rs.Lock() + defer rs.Unlock() + if r, ok := rs.relayForByAddr[vpnIp]; ok { + newRelay := *r + newRelay.State = state + rs.relayForByAddr[newRelay.PeerAddr] = &newRelay + rs.relayForByIdx[newRelay.LocalIndex] = &newRelay + } +} + +func (rs *RelayState) UpdateRelayForByIdxState(idx uint32, state int) { + rs.Lock() + defer rs.Unlock() + if r, ok := rs.relayForByIdx[idx]; ok { + newRelay := *r + newRelay.State = state + rs.relayForByAddr[newRelay.PeerAddr] = &newRelay + rs.relayForByIdx[newRelay.LocalIndex] = &newRelay + } } func (rs *RelayState) CopyAllRelayFor() []*Relay { @@ -90,34 +131,34 @@ func (rs *RelayState) CopyAllRelayFor() []*Relay { return ret } -func (rs *RelayState) GetRelayForByIp(ip iputil.VpnIp) (*Relay, bool) { +func (rs *RelayState) GetRelayForByAddr(addr netip.Addr) (*Relay, bool) { rs.RLock() defer rs.RUnlock() - r, ok := rs.relayForByIp[ip] + r, ok := rs.relayForByAddr[addr] return r, ok } -func (rs *RelayState) InsertRelayTo(ip iputil.VpnIp) { +func (rs *RelayState) InsertRelayTo(ip netip.Addr) { rs.Lock() defer rs.Unlock() - rs.relays[ip] = struct{}{} + if !slices.Contains(rs.relays, ip) { + rs.relays = append(rs.relays, ip) + } } -func (rs *RelayState) CopyRelayIps() []iputil.VpnIp { +func (rs *RelayState) CopyRelayIps() []netip.Addr { rs.RLock() defer rs.RUnlock() - ret := make([]iputil.VpnIp, 0, len(rs.relays)) - for ip := range rs.relays { - ret = append(ret, ip) - } + ret := make([]netip.Addr, len(rs.relays)) + copy(ret, rs.relays) return ret } -func (rs *RelayState) CopyRelayForIps() []iputil.VpnIp { +func (rs *RelayState) CopyRelayForIps() []netip.Addr { rs.RLock() defer rs.RUnlock() - currentRelays := make([]iputil.VpnIp, 0, len(rs.relayForByIp)) - for relayIp := range rs.relayForByIp { + currentRelays := make([]netip.Addr, 0, len(rs.relayForByAddr)) + for relayIp := range rs.relayForByAddr { currentRelays = append(currentRelays, relayIp) } return currentRelays @@ -133,22 +174,10 @@ func (rs *RelayState) CopyRelayForIdxs() []uint32 { return ret } -func (rs *RelayState) RemoveRelay(localIdx uint32) (iputil.VpnIp, bool) { +func (rs *RelayState) CompleteRelayByIP(vpnIp netip.Addr, remoteIdx uint32) bool { rs.Lock() defer rs.Unlock() - r, ok := rs.relayForByIdx[localIdx] - if !ok { - return iputil.VpnIp(0), false - } - delete(rs.relayForByIdx, localIdx) - delete(rs.relayForByIp, r.PeerIp) - return r.PeerIp, true -} - -func (rs *RelayState) CompleteRelayByIP(vpnIp iputil.VpnIp, remoteIdx uint32) bool { - rs.Lock() - defer rs.Unlock() - r, ok := rs.relayForByIp[vpnIp] + r, ok := rs.relayForByAddr[vpnIp] if !ok { return false } @@ -156,7 +185,7 @@ func (rs *RelayState) CompleteRelayByIP(vpnIp iputil.VpnIp, remoteIdx uint32) bo newRelay.State = Established newRelay.RemoteIndex = remoteIdx rs.relayForByIdx[r.LocalIndex] = &newRelay - rs.relayForByIp[r.PeerIp] = &newRelay + rs.relayForByAddr[r.PeerAddr] = &newRelay return true } @@ -171,14 +200,14 @@ func (rs *RelayState) CompleteRelayByIdx(localIdx uint32, remoteIdx uint32) (*Re newRelay.State = Established newRelay.RemoteIndex = remoteIdx rs.relayForByIdx[r.LocalIndex] = &newRelay - rs.relayForByIp[r.PeerIp] = &newRelay + rs.relayForByAddr[r.PeerAddr] = &newRelay return &newRelay, true } -func (rs *RelayState) QueryRelayForByIp(vpnIp iputil.VpnIp) (*Relay, bool) { +func (rs *RelayState) QueryRelayForByIp(vpnIp netip.Addr) (*Relay, bool) { rs.RLock() defer rs.RUnlock() - r, ok := rs.relayForByIp[vpnIp] + r, ok := rs.relayForByAddr[vpnIp] return r, ok } @@ -189,24 +218,41 @@ func (rs *RelayState) QueryRelayForByIdx(idx uint32) (*Relay, bool) { return r, ok } -func (rs *RelayState) InsertRelay(ip iputil.VpnIp, idx uint32, r *Relay) { +func (rs *RelayState) InsertRelay(ip netip.Addr, idx uint32, r *Relay) { rs.Lock() defer rs.Unlock() - rs.relayForByIp[ip] = r + rs.relayForByAddr[ip] = r rs.relayForByIdx[idx] = r } +type NetworkType uint8 + +const ( + NetworkTypeUnknown NetworkType = iota + // NetworkTypeVPN is a network that overlaps one or more of the vpnNetworks in our certificate + NetworkTypeVPN + // NetworkTypeVPNPeer is a network that does not overlap one of our networks + NetworkTypeVPNPeer + // NetworkTypeUnsafe is a network from Certificate.UnsafeNetworks() + NetworkTypeUnsafe +) + type HostInfo struct { - remote *udp.Addr + remote atomic.Pointer[netip.AddrPort] remotes *RemoteList promoteCounter atomic.Uint32 ConnectionState *ConnectionState remoteIndexId uint32 localIndexId uint32 - vpnIp iputil.VpnIp - recvError atomic.Uint32 - remoteCidr *cidr.Tree4[struct{}] - relayState RelayState + + // vpnAddrs is a list of vpn addresses assigned to this host that are within our own vpn networks + // The host may have other vpn addresses that are outside our + // vpn networks but were removed because they are not usable + vpnAddrs []netip.Addr + + // networks is a combination of specific vpn addresses (not prefixes!) and full unsafe networks assigned to this host. + networks *bart.Table[NetworkType] + relayState RelayState // HandshakePacket records the packets used to create this hostinfo // We need these to avoid replayed handshake packets creating new hostinfos which causes churn @@ -227,17 +273,36 @@ type HostInfo struct { lastHandshakeTime uint64 lastRoam time.Time - lastRoamRemote *udp.Addr + lastRoamRemote netip.AddrPort + + //TODO: in, out, and others might benefit from being an atomic.Int32. We could collapse connectionManager pendingDeletion, relayUsed, and in/out into this 1 thing + in, out, pendingDeletion atomic.Bool - // Used to track other hostinfos for this vpn ip since only 1 can be primary - // Synchronised via hostmap lock and not the hostinfo lock. - next, prev *HostInfo + // lastUsed tracks the last time ConnectionManager checked the tunnel and it was in use. + // This value will be behind against actual tunnel utilization in the hot path. + // This should only be used by the ConnectionManagers ticker routine. + lastUsed time.Time } type ViaSender struct { + UdpAddr netip.AddrPort relayHI *HostInfo // relayHI is the host info object of the relay - remoteIdx uint32 // remoteIdx is the index included in the header of the received packet relay *Relay // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us. + IsRelayed bool // IsRelayed is true if the packet was sent through a relay +} + +func (v ViaSender) String() string { + if v.IsRelayed { + return fmt.Sprintf("%s (relayed)", v.UdpAddr) + } + return v.UdpAddr.String() +} + +func (v ViaSender) MarshalJSON() ([]byte, error) { + if v.IsRelayed { + return json.Marshal(m{"relay": v.UdpAddr}) + } + return json.Marshal(m{"direct": v.UdpAddr}) } type cachedPacket struct { @@ -254,21 +319,57 @@ type cachedPacketMetrics struct { dropped metrics.Counter } -func NewHostMap(l *logrus.Logger, vpnCIDR *net.IPNet, preferredRanges []*net.IPNet) *HostMap { - h := map[iputil.VpnIp]*HostInfo{} - i := map[uint32]*HostInfo{} - r := map[uint32]*HostInfo{} - relays := map[uint32]*HostInfo{} - m := HostMap{ - Indexes: i, - Relays: relays, - RemoteIndexes: r, - Hosts: h, - preferredRanges: preferredRanges, - vpnCIDR: vpnCIDR, - l: l, +func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap { + hm := newHostMap(l) + + hm.reload(c, true) + c.RegisterReloadCallback(func(c *config.C) { + hm.reload(c, false) + }) + + l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges()) + + return hm +} + +func newHostMap(l *slog.Logger) *HostMap { + return &HostMap{ + Indexes: map[uint32]*HostInfo{}, + Relays: map[uint32]*HostInfo{}, + RemoteIndexes: map[uint32]*HostInfo{}, + Hosts: map[netip.Addr]*HostInfo{}, + moreHosts: map[netip.Addr][]*HostInfo{}, + l: l, + } +} + +func (hm *HostMap) reload(c *config.C, initial bool) { + if initial || c.HasChanged("preferred_ranges") { + var preferredRanges []netip.Prefix + rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{}) + + for _, rawPreferredRange := range rawPreferredRanges { + preferredRange, err := netip.ParsePrefix(rawPreferredRange) + + if err != nil { + hm.l.Warn("Failed to parse preferred ranges, ignoring", + "error", err, + "range", rawPreferredRanges, + ) + continue + } + + preferredRanges = append(preferredRanges, preferredRange) + } + + oldRanges := hm.preferredRanges.Swap(&preferredRanges) + if !initial { + hm.l.Info("preferred_ranges changed", + "oldPreferredRanges", *oldRanges, + "newPreferredRanges", preferredRanges, + ) + } } - return &m } // EmitStats reports host, index, and relay counts to the stats collection system @@ -286,24 +387,55 @@ func (hm *HostMap) EmitStats() { metrics.GetOrRegisterGauge("hostmap.main.relayIndexes", nil).Update(int64(relaysLen)) } -func (hm *HostMap) RemoveRelay(localIdx uint32) { - hm.Lock() - _, ok := hm.Relays[localIdx] - if !ok { - hm.Unlock() +// unlockedSetHostsForAddr stores the per-address hostinfo list (list[0] is the primary). An empty +// list removes the address. This is the one place Hosts and moreHosts are written together, keep +// it that way. Callers must hold the write lock. +func (hm *HostMap) unlockedSetHostsForAddr(addr netip.Addr, list []*HostInfo) { + if len(list) == 0 { + delete(hm.Hosts, addr) + delete(hm.moreHosts, addr) return } - delete(hm.Relays, localIdx) - hm.Unlock() + hm.Hosts[addr] = list[0] + if len(list) > 1 { + hm.moreHosts[addr] = list + } else { + delete(hm.moreHosts, addr) + } +} + +// unlockedGetHostList returns every hostinfo holding addr, primary first, or nil if we have no +// tunnel for addr. The common single-hostinfo case builds a fresh one element list, so keep this +// off the packet hot path; the primary is a direct Hosts read. Callers must hold the lock (read +// or write). +func (hm *HostMap) unlockedGetHostList(addr netip.Addr) []*HostInfo { + if list, ok := hm.moreHosts[addr]; ok { + return list + } + if h, ok := hm.Hosts[addr]; ok { + return []*HostInfo{h} + } + return nil +} + +// removeHostInfo returns list with hi removed (order preserved), or list unchanged if hi is +// absent. It deletes in place: every mutator holds the hostmap write lock and no reader ever +// retains a slice across a mutation (readers iterate under RLock), so there is no snapshot to +// invalidate. +func removeHostInfo(list []*HostInfo, hi *HostInfo) []*HostInfo { + idx := slices.Index(list, hi) + if idx < 0 { + return list + } + return slices.Delete(list, idx, idx+1) } -// DeleteHostInfo will fully unlink the hostinfo and return true if it was the final hostinfo for this vpn ip +// DeleteHostInfo will fully unlink the hostinfo and return true if no other hostinfo still holds +// any of its vpn addrs, meaning we no longer have a tunnel to the peer func (hm *HostMap) DeleteHostInfo(hostinfo *HostInfo) bool { // Delete the host itself, ensuring it's not modified anymore hm.Lock() - // If we have a previous or next hostinfo then we are not the last one for this vpn ip - final := (hostinfo.next == nil && hostinfo.prev == nil) - hm.unlockedDeleteHostInfo(hostinfo) + final := hm.unlockedDeleteHostInfo(hostinfo) hm.Unlock() return final @@ -315,60 +447,66 @@ func (hm *HostMap) MakePrimary(hostinfo *HostInfo) { hm.unlockedMakePrimary(hostinfo) } -func (hm *HostMap) unlockedMakePrimary(hostinfo *HostInfo) { - oldHostinfo := hm.Hosts[hostinfo.vpnIp] - if oldHostinfo == hostinfo { - return - } - - if hostinfo.prev != nil { - hostinfo.prev.next = hostinfo.next - } - - if hostinfo.next != nil { - hostinfo.next.prev = hostinfo.prev +// unlockedMakePrimary reports whether hostinfo is (now) the primary for each of its addresses, +// false only when it is no longer in the hostmap at all. +func (hm *HostMap) unlockedMakePrimary(hostinfo *HostInfo) bool { + // A hostinfo that is no longer in the hostmap must not be re-inserted here. Callers can race + // tunnel teardown, deciding to promote under the read lock and only taking the write lock + // after a delete fully unlinked the hostinfo (connection manager swapPrimary, AddRelay). Every + // live hostinfo is registered in Indexes by unlockedAddHostInfo, so this is a membership test. + if hm.Indexes[hostinfo.localIndexId] != hostinfo { + return false } - hm.Hosts[hostinfo.vpnIp] = hostinfo - - if oldHostinfo == nil { - return + // Move hostinfo to the front (primary) of each of its address lists. The lists are + // independent per address, so this can never leave a dangling entry the way promoting + // against a single shared chain could. + for _, addr := range hostinfo.vpnAddrs { + if hm.Hosts[addr] == hostinfo { + // Already primary for this address, the list is already in the right order + continue + } + list := removeHostInfo(hm.unlockedGetHostList(addr), hostinfo) + list = append([]*HostInfo{hostinfo}, list...) + hm.unlockedSetHostsForAddr(addr, list) } - - hostinfo.next = oldHostinfo - oldHostinfo.prev = hostinfo - hostinfo.prev = nil + return true } -func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) { - primary, ok := hm.Hosts[hostinfo.vpnIp] - if ok && primary == hostinfo { - // The vpnIp pointer points to the same hostinfo as the local index id, we can remove it - delete(hm.Hosts, hostinfo.vpnIp) - if len(hm.Hosts) == 0 { - hm.Hosts = map[iputil.VpnIp]*HostInfo{} - } - - if hostinfo.next != nil { - // We had more than 1 hostinfo at this vpnip, promote the next in the list to primary - hm.Hosts[hostinfo.vpnIp] = hostinfo.next - // It is primary, there is no previous hostinfo now - hostinfo.next.prev = nil - } - - } else { - // Relink if we were in the middle of multiple hostinfos for this vpn ip - if hostinfo.prev != nil { - hostinfo.prev.next = hostinfo.next - } - - if hostinfo.next != nil { - hostinfo.next.prev = hostinfo.prev +// unlockedDeleteHostInfo removes hostinfo from every one of its address lists and from the index +// maps. It returns true if this was the last hostinfo for all of its addresses (we no longer have +// any tunnel to the peer), which the caller uses to decide whether to clear learned lighthouse +// state and disestablish relays. +func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) bool { + // Remove this hostinfo from each of its address lists. The lists are independent, so a + // sibling is never promoted to an address it does not own and no other list is touched. + final := true + for _, addr := range hostinfo.vpnAddrs { + if list, ok := hm.moreHosts[addr]; ok { + list = removeHostInfo(list, hostinfo) + hm.unlockedSetHostsForAddr(addr, list) + if len(list) > 0 { + final = false + } + } else if existing, ok := hm.Hosts[addr]; ok { + if existing == hostinfo { + // Common case, the only hostinfo for this address. moreHosts has no entry to clean up. + delete(hm.Hosts, addr) + } else { + // We don't hold this address but another hostinfo does, we still have a tunnel to the peer + final = false + } } } - hostinfo.next = nil - hostinfo.prev = nil + // Go maps never shrink their buckets, replace fully drained maps so a node that churned + // through a large peer count gives the memory back. Same idiom as the index maps below. + if len(hm.Hosts) == 0 { + hm.Hosts = map[netip.Addr]*HostInfo{} + } + if len(hm.moreHosts) == 0 { + hm.moreHosts = map[netip.Addr][]*HostInfo{} + } // The remote index uses index ids outside our control so lets make sure we are only removing // the remote index pointer here if it points to the hostinfo we are deleting @@ -385,15 +523,24 @@ func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) { hm.Indexes = map[uint32]*HostInfo{} } - if hm.l.Level >= logrus.DebugLevel { - hm.l.WithField("hostMap", m{"mapTotalSize": len(hm.Hosts), - "vpnIp": hostinfo.vpnIp, "indexNumber": hostinfo.localIndexId, "remoteIndexNumber": hostinfo.remoteIndexId}). - Debug("Hostmap hostInfo deleted") + if hm.l.Enabled(context.Background(), slog.LevelDebug) { + hm.l.Debug("Hostmap hostInfo deleted", + "hostMap", m{"mapTotalSize": len(hm.Hosts), + "vpnAddrs": hostinfo.vpnAddrs, "indexNumber": hostinfo.localIndexId, "remoteIndexNumber": hostinfo.remoteIndexId}, + ) } + if final { + // I have lost connectivity to my peers. My relay tunnel is likely broken. Mark the next + // hops as 'Requested' so that new relay tunnels are created in the future. + hm.unlockedDisestablishVpnAddrRelayFor(hostinfo) + } + // Clean up any local relay indexes for which I am acting as a relay hop for _, localRelayIdx := range hostinfo.relayState.CopyRelayForIdxs() { delete(hm.Relays, localRelayIdx) } + + return final } func (hm *HostMap) QueryIndex(index uint32) *HostInfo { @@ -429,35 +576,65 @@ func (hm *HostMap) QueryReverseIndex(index uint32) *HostInfo { } } -func (hm *HostMap) QueryVpnIp(vpnIp iputil.VpnIp) *HostInfo { - return hm.queryVpnIp(vpnIp, nil) +func (hm *HostMap) QueryVpnAddr(vpnIp netip.Addr) *HostInfo { + return hm.queryVpnAddr(vpnIp, nil) } -func (hm *HostMap) QueryVpnIpRelayFor(targetIp, relayHostIp iputil.VpnIp) (*HostInfo, *Relay, error) { +func (hm *HostMap) QueryVpnAddrsRelayFor(targetIps []netip.Addr, relayHostIp netip.Addr) (*HostInfo, *Relay, error) { hm.RLock() defer hm.RUnlock() + // This runs per relayed packet, so check the primary with a single map probe and only consult + // moreHosts when the primary can't relay for us. h, ok := hm.Hosts[relayHostIp] if !ok { return nil, nil, errors.New("unable to find host") } - for h != nil { + + for _, targetIp := range targetIps { r, ok := h.relayState.QueryRelayForByIp(targetIp) if ok && r.State == Established { return h, r, nil } - h = h.next } + + if list, ok := hm.moreHosts[relayHostIp]; ok { + // list[0] is the primary we already checked + for _, h := range list[1:] { + for _, targetIp := range targetIps { + r, ok := h.relayState.QueryRelayForByIp(targetIp) + if ok && r.State == Established { + return h, r, nil + } + } + } + } + return nil, nil, errors.New("unable to find host with relay") } -func (hm *HostMap) queryVpnIp(vpnIp iputil.VpnIp, promoteIfce *Interface) *HostInfo { +func (hm *HostMap) unlockedDisestablishVpnAddrRelayFor(hi *HostInfo) { + for _, relayHostIp := range hi.relayState.CopyRelayIps() { + for _, h := range hm.unlockedGetHostList(relayHostIp) { + h.relayState.UpdateRelayForByIpState(hi.vpnAddrs[0], Disestablished) + } + } + for _, rs := range hi.relayState.CopyAllRelayFor() { + if rs.Type == ForwardingType { + for _, h := range hm.unlockedGetHostList(rs.PeerAddr) { + h.relayState.UpdateRelayForByIpState(hi.vpnAddrs[0], Disestablished) + } + } + } +} + +func (hm *HostMap) queryVpnAddr(vpnIp netip.Addr, promoteIfce *Interface) *HostInfo { hm.RLock() if h, ok := hm.Hosts[vpnIp]; ok { hm.RUnlock() // Do not attempt promotion if you are a lighthouse if promoteIfce != nil && !promoteIfce.lightHouse.amLighthouse { - h.TryPromoteBest(hm.preferredRanges, promoteIfce) + h.TryPromoteBest(hm.GetPreferredRanges(), promoteIfce) } return h @@ -470,44 +647,61 @@ func (hm *HostMap) queryVpnIp(vpnIp iputil.VpnIp, promoteIfce *Interface) *HostI // unlockedAddHostInfo assumes you have a write-lock and will add a hostinfo object to the hostmap Indexes and RemoteIndexes maps. // If an entry exists for the Hosts table (vpnIp -> hostinfo) then the provided hostinfo will be made primary func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) { - if f.serveDns { + if f.dnsServer != nil { remoteCert := hostinfo.ConnectionState.peerCert - dnsR.Add(remoteCert.Details.Name+".", remoteCert.Details.Ips[0].IP.String()) + f.dnsServer.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs) } - - existing := hm.Hosts[hostinfo.vpnIp] - hm.Hosts[hostinfo.vpnIp] = hostinfo - - if existing != nil { - hostinfo.next = existing - existing.prev = hostinfo + for _, addr := range hostinfo.vpnAddrs { + hm.unlockedInnerAddHostInfo(addr, hostinfo, f) } hm.Indexes[hostinfo.localIndexId] = hostinfo hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo - if hm.l.Level >= logrus.DebugLevel { - hm.l.WithField("hostMap", m{"vpnIp": hostinfo.vpnIp, "mapTotalSize": len(hm.Hosts), - "hostinfo": m{"existing": true, "localIndexId": hostinfo.localIndexId, "hostId": hostinfo.vpnIp}}). - Debug("Hostmap vpnIp added") + hostinfo.out.Store(true) + if f.connectionManager != nil { // f.connectionManager is only nil in some unit tests + f.connectionManager.trafficTimer.Add(hostinfo.localIndexId, f.connectionManager.checkInterval) } - i := 1 - check := hostinfo - for check != nil { - if i > MaxHostInfosPerVpnIp { - hm.unlockedDeleteHostInfo(check) - } - check = check.next - i++ + if hm.l.Enabled(context.Background(), slog.LevelDebug) { + hm.l.Debug("Hostmap vpnIp added", + "hostMap", m{"vpnAddrs": hostinfo.vpnAddrs, "mapTotalSize": len(hm.Hosts), + "hostinfo": m{"existing": true, "localIndexId": hostinfo.localIndexId, "vpnAddrs": hostinfo.vpnAddrs}}, + ) + } +} + +func (hm *HostMap) unlockedInnerAddHostInfo(vpnAddr netip.Addr, hostinfo *HostInfo, f *Interface) { + existing, ok := hm.Hosts[vpnAddr] + if !ok { + // Common case, the first hostinfo for this address. moreHosts stays empty. + hm.Hosts[vpnAddr] = hostinfo + return + } + + // The new hostinfo becomes the primary for this address. Remove any stale copy of it first so + // we never hold a duplicate, then prepend. + list, ok := hm.moreHosts[vpnAddr] + if !ok { + list = []*HostInfo{existing} + } + list = removeHostInfo(list, hostinfo) + list = append([]*HostInfo{hostinfo}, list...) + hm.unlockedSetHostsForAddr(vpnAddr, list) + + // Enforce the per-address cap by fully retiring the oldest hostinfo once we exceed it. + // Deleting it removes it from all of its addresses and the index maps, matching prior behavior. + if len(list) > MaxHostInfosPerVpnIp { + hm.unlockedDeleteHostInfo(list[len(list)-1]) } } -func (hm *HostMap) GetPreferredRanges() []*net.IPNet { - return hm.preferredRanges +func (hm *HostMap) GetPreferredRanges() []netip.Prefix { + //NOTE: if preferredRanges is ever not stored before a load this will fail to dereference a nil pointer + return *hm.preferredRanges.Load() } -func (hm *HostMap) ForEachVpnIp(f controlEach) { +func (hm *HostMap) ForEachVpnAddr(f controlEach) { hm.RLock() defer hm.RUnlock() @@ -527,14 +721,14 @@ func (hm *HostMap) ForEachIndex(f controlEach) { // TryPromoteBest handles re-querying lighthouses and probing for better paths // NOTE: It is an error to call this if you are a lighthouse since they should not roam clients! -func (i *HostInfo) TryPromoteBest(preferredRanges []*net.IPNet, ifce *Interface) { +func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interface) { c := i.promoteCounter.Add(1) if c%ifce.tryPromoteEvery.Load() == 0 { - remote := i.remote + remote := i.GetRemote() // return early if we are already on a preferred remote - if remote != nil { - rIP := remote.IP + if remote.IsValid() { + rIP := remote.Addr() for _, l := range preferredRanges { if l.Contains(rIP) { return @@ -542,8 +736,8 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []*net.IPNet, ifce *Interface) } } - i.remotes.ForEach(preferredRanges, func(addr *udp.Addr, preferred bool) { - if remote != nil && (addr == nil || !preferred) { + i.remotes.ForEach(preferredRanges, func(addr netip.AddrPort, preferred bool) { + if remote.IsValid() && (!addr.IsValid() || !preferred) { return } @@ -561,48 +755,56 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []*net.IPNet, ifce *Interface) } i.nextLHQuery.Store(now + ifce.reQueryWait.Load()) - ifce.lightHouse.QueryServer(i.vpnIp) + ifce.lightHouse.QueryServer(i.vpnAddrs[0]) } } -func (i *HostInfo) GetCert() *cert.NebulaCertificate { +func (i *HostInfo) GetCert() *cert.CachedCertificate { if i.ConnectionState != nil { return i.ConnectionState.peerCert } return nil } -func (i *HostInfo) SetRemote(remote *udp.Addr) { +func (i *HostInfo) GetRemote() netip.AddrPort { + if p := i.remote.Load(); p != nil { + return *p + } + return netip.AddrPort{} +} + +// TODO: Maybe use ViaSender here? +func (i *HostInfo) SetRemote(remote netip.AddrPort) { // We copy here because we likely got this remote from a source that reuses the object - if !i.remote.Equals(remote) { - i.remote = remote.Copy() - i.remotes.LearnRemote(i.vpnIp, remote.Copy()) + if i.GetRemote() != remote { + i.remote.Store(&remote) + i.remotes.LearnRemote(i.vpnAddrs[0], remote) } } // SetRemoteIfPreferred returns true if the remote was changed. The lastRoam // time on the HostInfo will also be updated. -func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote *udp.Addr) bool { - if newRemote == nil { - // relays have nil udp Addrs +func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool { + if via.IsRelayed { return false } - currentRemote := i.remote - if currentRemote == nil { - i.SetRemote(newRemote) + + currentRemote := i.GetRemote() + if !currentRemote.IsValid() { + i.SetRemote(via.UdpAddr) return true } // NOTE: We do this loop here instead of calling `isPreferred` in // remote_list.go so that we only have to loop over preferredRanges once. newIsPreferred := false - for _, l := range hm.preferredRanges { + for _, l := range hm.GetPreferredRanges() { // return early if we are already on a preferred remote - if l.Contains(currentRemote.IP) { + if l.Contains(currentRemote.Addr()) { return false } - if l.Contains(newRemote.IP) { + if l.Contains(via.UdpAddr.Addr()) { newIsPreferred = true } } @@ -610,9 +812,9 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote *udp.Addr) bool { if newIsPreferred { // Consider this a roaming event i.lastRoam = time.Now() - i.lastRoamRemote = currentRemote.Copy() + i.lastRoamRemote = currentRemote - i.SetRemote(newRemote) + i.SetRemote(via.UdpAddr) return true } @@ -620,42 +822,44 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote *udp.Addr) bool { return false } -func (i *HostInfo) RecvErrorExceeded() bool { - if i.recvError.Add(1) >= maxRecvError { - return true - } - return true -} - -func (i *HostInfo) CreateRemoteCIDR(c *cert.NebulaCertificate) { - if len(c.Details.Ips) == 1 && len(c.Details.Subnets) == 0 { - // Simple case, no CIDRTree needed - return +// buildNetworks fills in the networks field of HostInfo. It accepts a cert.Certificate so you never ever mix the network types up. +func (i *HostInfo) buildNetworks(myVpnNetworksTable *bart.Lite, c cert.Certificate) { + if len(c.Networks()) == 1 && len(c.UnsafeNetworks()) == 0 { + if myVpnNetworksTable.Contains(c.Networks()[0].Addr()) { + return // Simple case, no BART needed + } } - remoteCidr := cidr.NewTree4[struct{}]() - for _, ip := range c.Details.Ips { - remoteCidr.AddCIDR(&net.IPNet{IP: ip.IP, Mask: net.IPMask{255, 255, 255, 255}}, struct{}{}) + i.networks = new(bart.Table[NetworkType]) + for _, network := range c.Networks() { + nprefix := netip.PrefixFrom(network.Addr(), network.Addr().BitLen()) + if myVpnNetworksTable.Contains(network.Addr()) { + i.networks.Insert(nprefix, NetworkTypeVPN) + } else { + i.networks.Insert(nprefix, NetworkTypeVPNPeer) + } } - for _, n := range c.Details.Subnets { - remoteCidr.AddCIDR(n, struct{}{}) + for _, network := range c.UnsafeNetworks() { + i.networks.Insert(network, NetworkTypeUnsafe) } - i.remoteCidr = remoteCidr } -func (i *HostInfo) logger(l *logrus.Logger) *logrus.Entry { +// logger returns a derived slog.Logger with per-hostinfo fields pre-bound. +func (i *HostInfo) logger(l *slog.Logger) *slog.Logger { if i == nil { - return logrus.NewEntry(l) + return l } - li := l.WithField("vpnIp", i.vpnIp). - WithField("localIndex", i.localIndexId). - WithField("remoteIndex", i.remoteIndexId) + li := l.With( + "vpnAddrs", i.vpnAddrs, + "localIndex", i.localIndexId, + "remoteIndex", i.remoteIndexId, + ) if connState := i.ConnectionState; connState != nil { if peerCert := connState.peerCert; peerCert != nil { - li = li.WithField("certName", peerCert.Details.Name) + li = li.With("certName", peerCert.Certificate.Name()) } } @@ -664,44 +868,56 @@ func (i *HostInfo) logger(l *logrus.Logger) *logrus.Entry { // Utility functions -func localIps(l *logrus.Logger, allowList *LocalAllowList) *[]net.IP { +func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr { //FIXME: This function is pretty garbage - var ips []net.IP + var finalAddrs []netip.Addr ifaces, _ := net.Interfaces() for _, i := range ifaces { allow := allowList.AllowName(i.Name) - if l.Level >= logrus.TraceLevel { - l.WithField("interfaceName", i.Name).WithField("allow", allow).Trace("localAllowList.AllowName") + if l.Enabled(context.Background(), logging.LevelTrace) { + l.Log(context.Background(), logging.LevelTrace, "localAllowList.AllowName", + "interfaceName", i.Name, + "allow", allow, + ) } if !allow { continue } addrs, _ := i.Addrs() - for _, addr := range addrs { - var ip net.IP - switch v := addr.(type) { + for _, rawAddr := range addrs { + var addr netip.Addr + switch v := rawAddr.(type) { case *net.IPNet: //continue - ip = v.IP + addr, _ = netip.AddrFromSlice(v.IP) case *net.IPAddr: - ip = v.IP + addr, _ = netip.AddrFromSlice(v.IP) } - //TODO: Filtering out link local for now, this is probably the most correct thing - //TODO: Would be nice to filter out SLAAC MAC based ips as well - if ip.IsLoopback() == false && !ip.IsLinkLocalUnicast() { - allow := allowList.Allow(ip) - if l.Level >= logrus.TraceLevel { - l.WithField("localIp", ip).WithField("allow", allow).Trace("localAllowList.Allow") + if !addr.IsValid() { + if l.Enabled(context.Background(), slog.LevelDebug) { + l.Debug("addr was invalid", "localAddr", rawAddr) + } + continue + } + addr = addr.Unmap() + + if addr.IsLoopback() == false && addr.IsLinkLocalUnicast() == false { + isAllowed := allowList.Allow(addr) + if l.Enabled(context.Background(), logging.LevelTrace) { + l.Log(context.Background(), logging.LevelTrace, "localAllowList.Allow", + "localAddr", addr, + "allowed", isAllowed, + ) } - if !allow { + if !isAllowed { continue } - ips = append(ips, ip) + finalAddrs = append(finalAddrs, addr) } } } - return &ips + return finalAddrs } diff --git a/hostmap_test.go b/hostmap_test.go index c1c0dcea..9cfebe17 100644 --- a/hostmap_test.go +++ b/hostmap_test.go @@ -1,113 +1,109 @@ package nebula import ( - "net" + "net/netip" + "slices" "testing" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// chainIds returns the localIndexIds of the hostinfos holding addr, primary (index 0) first. It +// also validates the Hosts/moreHosts sync contract on every call so a mutation that broke it +// fails fast. +func chainIds(t *testing.T, hm *HostMap, addr netip.Addr) []uint32 { + t.Helper() + assertHostMapInvariants(t, hm) + list := hm.unlockedGetHostList(addr) + ids := make([]uint32, len(list)) + for i, h := range list { + ids[i] = h.localIndexId + } + return ids +} + +// assertHostMapInvariants checks the Hosts/moreHosts contract: moreHosts only holds addresses +// with 2 or more hostinfos, its first entry is always the primary in Hosts, lists never hold +// duplicates, every hostinfo in a list owns the address and is registered in Indexes, and every +// indexed hostinfo is reachable through each of its addresses. +func assertHostMapInvariants(t *testing.T, hm *HostMap) { + t.Helper() + for addr, list := range hm.moreHosts { + require.GreaterOrEqualf(t, len(list), 2, "moreHosts[%s] must hold at least 2 hostinfos", addr) + require.Samef(t, hm.Hosts[addr], list[0], "moreHosts[%s][0] must match the primary in Hosts", addr) + seen := map[*HostInfo]bool{} + for _, h := range list { + require.NotNilf(t, h, "moreHosts[%s] must never hold a nil hostinfo", addr) + require.Falsef(t, seen[h], "moreHosts[%s] holds hostinfo %d twice", addr, h.localIndexId) + seen[h] = true + require.Samef(t, hm.Indexes[h.localIndexId], h, "moreHosts[%s] member %d is not registered in Indexes", addr, h.localIndexId) + require.Truef(t, slices.Contains(h.vpnAddrs, addr), "moreHosts[%s] member %d does not own the address", addr, h.localIndexId) + } + } + for addr, h := range hm.Hosts { + require.NotNilf(t, h, "Hosts[%s] must never be nil", addr) + require.Samef(t, hm.Indexes[h.localIndexId], h, "Hosts[%s] primary %d is not registered in Indexes", addr, h.localIndexId) + require.Truef(t, slices.Contains(h.vpnAddrs, addr), "Hosts[%s] primary (index %d) does not own the address", addr, h.localIndexId) + } + for idx, h := range hm.Indexes { + require.Equalf(t, idx, h.localIndexId, "Indexes[%d] holds hostinfo with localIndexId %d", idx, h.localIndexId) + for _, va := range h.vpnAddrs { + require.Truef(t, slices.Contains(hm.unlockedGetHostList(va), h), "indexed hostinfo %d is missing from the list for %s", idx, va) + } + } +} + func TestHostMap_MakePrimary(t *testing.T) { l := test.NewLogger() - hm := NewHostMap( - l, - &net.IPNet{ - IP: net.IP{10, 0, 0, 1}, - Mask: net.IPMask{255, 255, 255, 0}, - }, - []*net.IPNet{}, - ) + hm := newHostMap(l) f := &Interface{} + a := netip.MustParseAddr("0.0.0.1") - h1 := &HostInfo{vpnIp: 1, localIndexId: 1} - h2 := &HostInfo{vpnIp: 1, localIndexId: 2} - h3 := &HostInfo{vpnIp: 1, localIndexId: 3} - h4 := &HostInfo{vpnIp: 1, localIndexId: 4} + h1 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1} + h2 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 2} + h3 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 3} + h4 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 4} hm.unlockedAddHostInfo(h4, f) hm.unlockedAddHostInfo(h3, f) hm.unlockedAddHostInfo(h2, f) hm.unlockedAddHostInfo(h1, f) - // Make sure we go h1 -> h2 -> h3 -> h4 - prim := hm.QueryVpnIp(1) - assert.Equal(t, h1.localIndexId, prim.localIndexId) - assert.Equal(t, h2.localIndexId, prim.next.localIndexId) - assert.Nil(t, prim.prev) - assert.Equal(t, h1.localIndexId, h2.prev.localIndexId) - assert.Equal(t, h3.localIndexId, h2.next.localIndexId) - assert.Equal(t, h2.localIndexId, h3.prev.localIndexId) - assert.Equal(t, h4.localIndexId, h3.next.localIndexId) - assert.Equal(t, h3.localIndexId, h4.prev.localIndexId) - assert.Nil(t, h4.next) - - // Swap h3/middle to primary + // Most-recently-added is primary: h1, h2, h3, h4 + assert.Equal(t, []uint32{1, 2, 3, 4}, chainIds(t, hm, a)) + assert.Equal(t, h1, hm.QueryVpnAddr(a)) + + // Swap the middle to primary: h3, h1, h2, h4 hm.MakePrimary(h3) + assert.Equal(t, []uint32{3, 1, 2, 4}, chainIds(t, hm, a)) + assert.Equal(t, h3, hm.QueryVpnAddr(a)) - // Make sure we go h3 -> h1 -> h2 -> h4 - prim = hm.QueryVpnIp(1) - assert.Equal(t, h3.localIndexId, prim.localIndexId) - assert.Equal(t, h1.localIndexId, prim.next.localIndexId) - assert.Nil(t, prim.prev) - assert.Equal(t, h2.localIndexId, h1.next.localIndexId) - assert.Equal(t, h3.localIndexId, h1.prev.localIndexId) - assert.Equal(t, h4.localIndexId, h2.next.localIndexId) - assert.Equal(t, h1.localIndexId, h2.prev.localIndexId) - assert.Equal(t, h2.localIndexId, h4.prev.localIndexId) - assert.Nil(t, h4.next) - - // Swap h4/tail to primary + // Swap the tail to primary: h4, h3, h1, h2 hm.MakePrimary(h4) + assert.Equal(t, []uint32{4, 3, 1, 2}, chainIds(t, hm, a)) - // Make sure we go h4 -> h3 -> h1 -> h2 - prim = hm.QueryVpnIp(1) - assert.Equal(t, h4.localIndexId, prim.localIndexId) - assert.Equal(t, h3.localIndexId, prim.next.localIndexId) - assert.Nil(t, prim.prev) - assert.Equal(t, h1.localIndexId, h3.next.localIndexId) - assert.Equal(t, h4.localIndexId, h3.prev.localIndexId) - assert.Equal(t, h2.localIndexId, h1.next.localIndexId) - assert.Equal(t, h3.localIndexId, h1.prev.localIndexId) - assert.Equal(t, h1.localIndexId, h2.prev.localIndexId) - assert.Nil(t, h2.next) - - // Swap h4 again should be no-op + // Swapping the current primary again is a no-op hm.MakePrimary(h4) - - // Make sure we go h4 -> h3 -> h1 -> h2 - prim = hm.QueryVpnIp(1) - assert.Equal(t, h4.localIndexId, prim.localIndexId) - assert.Equal(t, h3.localIndexId, prim.next.localIndexId) - assert.Nil(t, prim.prev) - assert.Equal(t, h1.localIndexId, h3.next.localIndexId) - assert.Equal(t, h4.localIndexId, h3.prev.localIndexId) - assert.Equal(t, h2.localIndexId, h1.next.localIndexId) - assert.Equal(t, h3.localIndexId, h1.prev.localIndexId) - assert.Equal(t, h1.localIndexId, h2.prev.localIndexId) - assert.Nil(t, h2.next) + assert.Equal(t, []uint32{4, 3, 1, 2}, chainIds(t, hm, a)) } func TestHostMap_DeleteHostInfo(t *testing.T) { l := test.NewLogger() - hm := NewHostMap( - l, - &net.IPNet{ - IP: net.IP{10, 0, 0, 1}, - Mask: net.IPMask{255, 255, 255, 0}, - }, - []*net.IPNet{}, - ) + hm := newHostMap(l) f := &Interface{} + a := netip.MustParseAddr("0.0.0.1") - h1 := &HostInfo{vpnIp: 1, localIndexId: 1} - h2 := &HostInfo{vpnIp: 1, localIndexId: 2} - h3 := &HostInfo{vpnIp: 1, localIndexId: 3} - h4 := &HostInfo{vpnIp: 1, localIndexId: 4} - h5 := &HostInfo{vpnIp: 1, localIndexId: 5} - h6 := &HostInfo{vpnIp: 1, localIndexId: 6} + h1 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1} + h2 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 2} + h3 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 3} + h4 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 4} + h5 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 5} + h6 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 6} hm.unlockedAddHostInfo(h6, f) hm.unlockedAddHostInfo(h5, f) @@ -116,92 +112,292 @@ func TestHostMap_DeleteHostInfo(t *testing.T) { hm.unlockedAddHostInfo(h2, f) hm.unlockedAddHostInfo(h1, f) - // h6 should be deleted - assert.Nil(t, h6.next) - assert.Nil(t, h6.prev) - h := hm.QueryIndex(h6.localIndexId) - assert.Nil(t, h) - - // Make sure we go h1 -> h2 -> h3 -> h4 -> h5 - prim := hm.QueryVpnIp(1) - assert.Equal(t, h1.localIndexId, prim.localIndexId) - assert.Equal(t, h2.localIndexId, prim.next.localIndexId) - assert.Nil(t, prim.prev) - assert.Equal(t, h1.localIndexId, h2.prev.localIndexId) - assert.Equal(t, h3.localIndexId, h2.next.localIndexId) - assert.Equal(t, h2.localIndexId, h3.prev.localIndexId) - assert.Equal(t, h4.localIndexId, h3.next.localIndexId) - assert.Equal(t, h3.localIndexId, h4.prev.localIndexId) - assert.Equal(t, h5.localIndexId, h4.next.localIndexId) - assert.Equal(t, h4.localIndexId, h5.prev.localIndexId) - assert.Nil(t, h5.next) - - // Delete primary - hm.DeleteHostInfo(h1) - assert.Nil(t, h1.prev) - assert.Nil(t, h1.next) - - // Make sure we go h2 -> h3 -> h4 -> h5 - prim = hm.QueryVpnIp(1) - assert.Equal(t, h2.localIndexId, prim.localIndexId) - assert.Equal(t, h3.localIndexId, prim.next.localIndexId) - assert.Nil(t, prim.prev) - assert.Equal(t, h3.localIndexId, h2.next.localIndexId) - assert.Equal(t, h2.localIndexId, h3.prev.localIndexId) - assert.Equal(t, h4.localIndexId, h3.next.localIndexId) - assert.Equal(t, h3.localIndexId, h4.prev.localIndexId) - assert.Equal(t, h5.localIndexId, h4.next.localIndexId) - assert.Equal(t, h4.localIndexId, h5.prev.localIndexId) - assert.Nil(t, h5.next) - - // Delete in the middle - hm.DeleteHostInfo(h3) - assert.Nil(t, h3.prev) - assert.Nil(t, h3.next) - - // Make sure we go h2 -> h4 -> h5 - prim = hm.QueryVpnIp(1) - assert.Equal(t, h2.localIndexId, prim.localIndexId) - assert.Equal(t, h4.localIndexId, prim.next.localIndexId) - assert.Nil(t, prim.prev) - assert.Equal(t, h4.localIndexId, h2.next.localIndexId) - assert.Equal(t, h2.localIndexId, h4.prev.localIndexId) - assert.Equal(t, h5.localIndexId, h4.next.localIndexId) - assert.Equal(t, h4.localIndexId, h5.prev.localIndexId) - assert.Nil(t, h5.next) - - // Delete the tail - hm.DeleteHostInfo(h5) - assert.Nil(t, h5.prev) - assert.Nil(t, h5.next) - - // Make sure we go h2 -> h4 - prim = hm.QueryVpnIp(1) - assert.Equal(t, h2.localIndexId, prim.localIndexId) - assert.Equal(t, h4.localIndexId, prim.next.localIndexId) - assert.Nil(t, prim.prev) - assert.Equal(t, h4.localIndexId, h2.next.localIndexId) - assert.Equal(t, h2.localIndexId, h4.prev.localIndexId) - assert.Nil(t, h4.next) - - // Delete the head - hm.DeleteHostInfo(h2) - assert.Nil(t, h2.prev) - assert.Nil(t, h2.next) - - // Make sure we only have h4 - prim = hm.QueryVpnIp(1) - assert.Equal(t, h4.localIndexId, prim.localIndexId) - assert.Nil(t, prim.prev) - assert.Nil(t, prim.next) - assert.Nil(t, h4.next) - - // Delete the only item - hm.DeleteHostInfo(h4) - assert.Nil(t, h4.prev) - assert.Nil(t, h4.next) - - // Make sure we have nil - prim = hm.QueryVpnIp(1) - assert.Nil(t, prim) + // h6 is evicted by the MaxHostInfosPerVpnIp cap; the rest are newest-first. + assert.Nil(t, hm.QueryIndex(h6.localIndexId)) + assert.Equal(t, []uint32{1, 2, 3, 4, 5}, chainIds(t, hm, a)) + + // Delete primary; not final since siblings remain. + assert.False(t, hm.DeleteHostInfo(h1)) + assert.Equal(t, []uint32{2, 3, 4, 5}, chainIds(t, hm, a)) + + // Deleting the same hostinfo again must not report final while siblings remain and must not + // disturb the list. The old chain code got this wrong: the first delete nil'd next/prev, so a + // second delete looked final and wiped lighthouse state out from under the live sibling. + assert.False(t, hm.DeleteHostInfo(h1)) + assert.Equal(t, []uint32{2, 3, 4, 5}, chainIds(t, hm, a)) + + // Delete a middle node. + assert.False(t, hm.DeleteHostInfo(h3)) + assert.Equal(t, []uint32{2, 4, 5}, chainIds(t, hm, a)) + + // Delete the tail. + assert.False(t, hm.DeleteHostInfo(h5)) + assert.Equal(t, []uint32{2, 4}, chainIds(t, hm, a)) + + // Delete the head; h4 remains and becomes primary. + assert.False(t, hm.DeleteHostInfo(h2)) + assert.Equal(t, []uint32{4}, chainIds(t, hm, a)) + assert.Equal(t, h4, hm.QueryVpnAddr(a)) + + // Delete the only remaining item; final is true and the address is gone. + assert.True(t, hm.DeleteHostInfo(h4)) + assert.Empty(t, chainIds(t, hm, a)) + assert.Nil(t, hm.QueryVpnAddr(a)) + + // Deleting an already-gone hostinfo is still final; nothing holds the address anymore. + assert.True(t, hm.DeleteHostInfo(h4)) + assert.Empty(t, chainIds(t, hm, a)) +} + +// TestHostMap_MakePrimary_DeletedHostInfo covers promoting a hostinfo that lost a race with +// tunnel teardown: swapPrimary and AddRelay decide to promote while holding a stale pointer and +// only take the write lock after a delete fully unlinked the hostinfo. MakePrimary must be a +// no-op, not a resurrection that installs an unmanaged primary. +func TestHostMap_MakePrimary_DeletedHostInfo(t *testing.T) { + l := test.NewLogger() + hm := newHostMap(l) + f := &Interface{} + a := netip.MustParseAddr("0.0.0.1") + + h1 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1} + h2 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 2} + hm.unlockedAddHostInfo(h1, f) + hm.unlockedAddHostInfo(h2, f) + + // h1 is fully deleted while another goroutine still holds a pointer to it. + assert.False(t, hm.DeleteHostInfo(h1)) + assert.Equal(t, []uint32{2}, chainIds(t, hm, a)) + + // The stale promote must not bring it back. + hm.MakePrimary(h1) + assert.Equal(t, []uint32{2}, chainIds(t, hm, a)) + assert.Equal(t, h2, hm.QueryVpnAddr(a)) + assert.Nil(t, hm.QueryIndex(h1.localIndexId)) +} + +// TestHostMap_QueryVpnAddrsRelayFor_NonPrimary makes sure a relay established on an older +// hostinfo is still found after a newer tunnel without relay state takes primary for the same +// address. The lookup checks the primary first and falls back to the rest of the list. +func TestHostMap_QueryVpnAddrsRelayFor_NonPrimary(t *testing.T) { + l := test.NewLogger() + hm := newHostMap(l) + f := &Interface{} + relayAddr := netip.MustParseAddr("0.0.0.9") + target := netip.MustParseAddr("0.0.0.1") + + older := &HostInfo{ + vpnAddrs: []netip.Addr{relayAddr}, + localIndexId: 1, + relayState: RelayState{ + relayForByAddr: map[netip.Addr]*Relay{}, + relayForByIdx: map[uint32]*Relay{}, + }, + } + older.relayState.InsertRelay(target, 100, &Relay{Type: ForwardingType, State: Established, LocalIndex: 100, PeerAddr: target}) + hm.unlockedAddHostInfo(older, f) + + // The relay is found on the primary. + h, r, err := hm.QueryVpnAddrsRelayFor([]netip.Addr{target}, relayAddr) + require.NoError(t, err) + assert.Equal(t, older, h) + assert.Equal(t, uint32(100), r.LocalIndex) + + // A re-handshake with no relay state takes primary; the established relay on the older + // hostinfo must still be found through the fallback. + newer := &HostInfo{vpnAddrs: []netip.Addr{relayAddr}, localIndexId: 2} + hm.unlockedAddHostInfo(newer, f) + assert.Equal(t, []uint32{2, 1}, chainIds(t, hm, relayAddr)) + + h, r, err = hm.QueryVpnAddrsRelayFor([]netip.Addr{target}, relayAddr) + require.NoError(t, err) + assert.Equal(t, older, h) + assert.Equal(t, uint32(100), r.LocalIndex) + + // No hostinfo at all is a plain miss. + _, _, err = hm.QueryVpnAddrsRelayFor([]netip.Addr{target}, netip.MustParseAddr("0.0.0.42")) + require.Error(t, err) +} + +// TestHostMap_DeleteHostInfo_MultipleVpnAddrs exercises the case where a hostinfo carries more than one +// vpnAddr and shares its next/prev chain with a live sibling. Deleting the head must not corrupt the +// sibling: every address the sibling owns has to keep pointing at it. The pre-fix code unlinked the shared +// chain once per vpnAddr, so on the first address it nil'd next/prev, and on the second address the node +// looked already-detached: it dropped the map entry instead of promoting the sibling (and tripped the +// isLastHostinfo relay teardown). See unlockedDeleteHostInfo. +func TestHostMap_DeleteHostInfo_MultipleVpnAddrs(t *testing.T) { + l := test.NewLogger() + hm := newHostMap(l) + + f := &Interface{} + + a := netip.MustParseAddr("0.0.0.1") + b := netip.MustParseAddr("0.0.0.2") + + // Two tunnels for the same peer, each reachable at both a and b. + other := &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: 1} + head := &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: 2} + + hm.unlockedAddHostInfo(other, f) + hm.unlockedAddHostInfo(head, f) + + // head is primary for both addresses, other is next in each address's list. + assert.Equal(t, head, hm.QueryVpnAddr(a)) + assert.Equal(t, head, hm.QueryVpnAddr(b)) + assert.Equal(t, []uint32{2, 1}, chainIds(t, hm, a)) + assert.Equal(t, []uint32{2, 1}, chainIds(t, hm, b)) + + // Delete the head. other is still live, so it must become primary for BOTH addresses. + assert.False(t, hm.DeleteHostInfo(head)) + assert.Equal(t, other, hm.QueryVpnAddr(a)) + assert.Equal(t, other, hm.QueryVpnAddr(b)) + assert.Equal(t, []uint32{1}, chainIds(t, hm, a)) + assert.Equal(t, []uint32{1}, chainIds(t, hm, b)) + + // head is fully removed from the index map. + assert.Nil(t, hm.QueryIndex(head.localIndexId)) +} + +// TestHostMap_DeleteHostInfo_DivergentVpnAddrs covers chained hostinfos for the same peer whose +// vpnAddrs sets differ (a re-handshake cert added a second address). Deleting the superset node +// must not promote a sibling to an address it does not own. +func TestHostMap_DeleteHostInfo_DivergentVpnAddrs(t *testing.T) { + l := test.NewLogger() + hm := newHostMap(l) + f := &Interface{} + a := netip.MustParseAddr("0.0.0.1") + b := netip.MustParseAddr("0.0.0.2") + + // sub owns only a; super (a newer handshake) owns a and b. + sub := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1} + super := &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: 2} + hm.unlockedAddHostInfo(sub, f) + hm.unlockedAddHostInfo(super, f) + + assert.Equal(t, []uint32{2, 1}, chainIds(t, hm, a)) + assert.Equal(t, []uint32{2}, chainIds(t, hm, b)) + + // Delete super: a promotes to sub (which owns it); b has no remaining owner and must be + // removed, not dangled at sub (which does not own b). + assert.False(t, hm.DeleteHostInfo(super)) + assert.Equal(t, []uint32{1}, chainIds(t, hm, a)) + assert.Empty(t, chainIds(t, hm, b)) + assert.Equal(t, sub, hm.QueryVpnAddr(a)) + assert.Nil(t, hm.QueryVpnAddr(b)) + assert.Nil(t, hm.QueryIndex(super.localIndexId)) + + // Deleting sub cleans up fully. + assert.True(t, hm.DeleteHostInfo(sub)) + assert.Nil(t, hm.QueryVpnAddr(a)) + assertHostMapInvariants(t, hm) +} + +// TestHostMap_AddDivergentOverlap covers a new hostinfo claiming addresses currently owned by two +// DIFFERENT hostinfos. The old single shared next/prev chain overwrote a pointer and orphaned one +// of them (in Indexes but unreachable via its address); independent per-address lists cannot. +func TestHostMap_AddDivergentOverlap(t *testing.T) { + l := test.NewLogger() + hm := newHostMap(l) + f := &Interface{} + a := netip.MustParseAddr("0.0.0.1") + b := netip.MustParseAddr("0.0.0.2") + + hiA := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1} + hiP := &HostInfo{vpnAddrs: []netip.Addr{b}, localIndexId: 2} + hm.unlockedAddHostInfo(hiA, f) + hm.unlockedAddHostInfo(hiP, f) + + hiB := &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: 3} + hm.unlockedAddHostInfo(hiB, f) + + assert.Equal(t, []uint32{3, 1}, chainIds(t, hm, a)) + assert.Equal(t, []uint32{3, 2}, chainIds(t, hm, b)) + // hiA is still reachable via its address (not orphaned) and still indexed. + assert.Contains(t, chainIds(t, hm, a), hiA.localIndexId) + assert.NotNil(t, hm.QueryIndex(hiA.localIndexId)) +} + +// TestHostMap_MaxHostInfosPerVpnIp_MultipleVpnAddrs verifies the MaxHostInfosPerVpnIp overflow prune +// (unlockedInnerAddHostInfo calls unlockedDeleteHostInfo on the oldest node once the chain is too long) +// still behaves when hostinfos carry more than one vpnAddr. The pruned node is always the tail, so it is +// primary for none of the addresses, and both address chains must stay consistent afterwards. +func TestHostMap_MaxHostInfosPerVpnIp_MultipleVpnAddrs(t *testing.T) { + l := test.NewLogger() + hm := newHostMap(l) + + f := &Interface{} + + a := netip.MustParseAddr("0.0.0.1") + b := netip.MustParseAddr("0.0.0.2") + + // Add one more than the cap, newest last so it becomes head. Every hostinfo owns both a and b. + hostinfos := make([]*HostInfo, 0, MaxHostInfosPerVpnIp+1) + for i := 0; i <= MaxHostInfosPerVpnIp; i++ { + hostinfos = append(hostinfos, &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: uint32(i + 1)}) + } + // Add oldest first (highest index in our slice) so the very first one added is the overflow victim. + for i := len(hostinfos) - 1; i >= 0; i-- { + hm.unlockedAddHostInfo(hostinfos[i], f) + } + + oldest := hostinfos[len(hostinfos)-1] + + // The oldest hostinfo was pruned from both lists and the index map. + assert.Nil(t, hm.QueryIndex(oldest.localIndexId)) + + // Both addresses hold exactly MaxHostInfosPerVpnIp survivors in the same order; oldest is absent. + require.Len(t, chainIds(t, hm, a), MaxHostInfosPerVpnIp) + assert.Equal(t, chainIds(t, hm, a), chainIds(t, hm, b), "both addresses must list the same survivors in the same order") + assert.NotContains(t, chainIds(t, hm, a), oldest.localIndexId) + assert.Equal(t, hm.QueryVpnAddr(a), hm.QueryVpnAddr(b)) +} + +func TestHostMap_reload(t *testing.T) { + l := test.NewLogger() + c := config.NewC(test.NewLogger()) + + hm := NewHostMapFromConfig(l, c) + + toS := func(ipn []netip.Prefix) []string { + var s []string + for _, n := range ipn { + s = append(s, n.String()) + } + return s + } + + assert.Empty(t, hm.GetPreferredRanges()) + + c.ReloadConfigString("preferred_ranges: [1.1.1.0/24, 10.1.1.0/24]") + assert.Equal(t, []string{"1.1.1.0/24", "10.1.1.0/24"}, toS(hm.GetPreferredRanges())) + + c.ReloadConfigString("preferred_ranges: [1.1.1.1/32]") + assert.Equal(t, []string{"1.1.1.1/32"}, toS(hm.GetPreferredRanges())) +} + +func TestHostMap_RelayState(t *testing.T) { + h1 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 1} + a1 := netip.MustParseAddr("::1") + a2 := netip.MustParseAddr("2001::1") + + h1.relayState.InsertRelayTo(a1) + assert.Equal(t, []netip.Addr{a1}, h1.relayState.relays) + h1.relayState.InsertRelayTo(a2) + assert.Equal(t, []netip.Addr{a1, a2}, h1.relayState.relays) + // Ensure that the first relay added is the first one returned in the copy + currentRelays := h1.relayState.CopyRelayIps() + require.Len(t, currentRelays, 2) + assert.Equal(t, a1, currentRelays[0]) + + // Deleting the last one in the list works ok + h1.relayState.DeleteRelay(a2) + assert.Equal(t, []netip.Addr{a1}, h1.relayState.relays) + + // Deleting an element not in the list works ok + h1.relayState.DeleteRelay(a2) + assert.Equal(t, []netip.Addr{a1}, h1.relayState.relays) + + // Deleting the only element in the list works ok + h1.relayState.DeleteRelay(a1) + assert.Equal(t, []netip.Addr{}, h1.relayState.relays) + } diff --git a/hostmap_tester.go b/hostmap_tester.go index 0d5d41bf..a6ac6d44 100644 --- a/hostmap_tester.go +++ b/hostmap_tester.go @@ -1,14 +1,15 @@ //go:build e2e_testing -// +build e2e_testing package nebula // This file contains functions used to export information to the e2e testing framework -import "github.com/slackhq/nebula/iputil" +import ( + "net/netip" +) -func (i *HostInfo) GetVpnIp() iputil.VpnIp { - return i.vpnIp +func (i *HostInfo) GetVpnAddrs() []netip.Addr { + return i.vpnAddrs } func (i *HostInfo) GetLocalIndex() uint32 { diff --git a/inside.go b/inside.go index 62309628..a80b2e96 100644 --- a/inside.go +++ b/inside.go @@ -1,59 +1,68 @@ package nebula import ( - "github.com/sirupsen/logrus" + "context" + "log/slog" + "net/netip" + "github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/header" "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/noiseutil" - "github.com/slackhq/nebula/udp" + "github.com/slackhq/nebula/routing" ) func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) { err := newPacket(packet, false, fwPacket) if err != nil { - if f.l.Level >= logrus.DebugLevel { - f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err) + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("Error while validating outbound packet", + "packet", packet, + "error", err, + ) } return } // Ignore local broadcast packets - if f.dropLocalBroadcast && fwPacket.RemoteIP == f.localBroadcast { - return + if f.dropLocalBroadcast { + if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) { + return + } } - if fwPacket.RemoteIP == f.myVpnIp { + if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) { // Immediately forward packets from self to self. // This should only happen on Darwin-based and FreeBSD hosts, which - // routes packets from the Nebula IP to the Nebula IP through the Nebula + // routes packets from the Nebula addr to the Nebula addr through the Nebula // TUN device. if immediatelyForwardToSelf { _, err := f.readers[q].Write(packet) if err != nil { - f.l.WithError(err).Error("Failed to forward to tun") + f.l.Error("Failed to forward to tun", "error", err) } } // Otherwise, drop. On linux, we should never see these packets - Linux - // routes packets from the nebula IP to the nebula IP through the loopback device. + // routes packets from the nebula addr to the nebula addr through the loopback device. return } - // Ignore broadcast packets - if f.dropMulticast && isMulticast(fwPacket.RemoteIP) { + // Ignore multicast packets + if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() { return } - hostinfo, ready := f.getOrHandshake(fwPacket.RemoteIP, func(hh *HandshakeHostInfo) { + hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) { hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics) }) if hostinfo == nil { f.rejectInside(packet, out, q) - if f.l.Level >= logrus.DebugLevel { - f.l.WithField("vpnIp", fwPacket.RemoteIP). - WithField("fwPacket", fwPacket). - Debugln("dropping outbound packet, vpnIp not in our CIDR or in unsafe routes") + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks", + "vpnAddr", fwPacket.RemoteAddr, + "fwPacket", fwPacket, + ) } return } @@ -62,23 +71,23 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet return } - dropReason := f.firewall.Drop(packet, *fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache) + dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache) if dropReason == nil { - f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, nil, packet, nb, out, q) + f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q) } else { f.rejectInside(packet, out, q) - if f.l.Level >= logrus.DebugLevel { - hostinfo.logger(f.l). - WithField("fwPacket", fwPacket). - WithField("reason", dropReason). - Debugln("dropping outbound packet") + if f.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(f.l).Debug("dropping outbound packet", + "fwPacket", fwPacket, + "reason", dropReason, + ) } } } func (f *Interface) rejectInside(packet []byte, out []byte, q int) { - if !f.firewall.InSendReject { + if !f.firewall.OutboundSendReject { return } @@ -89,12 +98,12 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) { _, err := f.readers[q].Write(out) if err != nil { - f.l.WithError(err).Error("Failed to write to tun") + f.l.Error("Failed to write to tun", "error", err) } } func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) { - if !f.firewall.OutSendReject { + if !f.firewall.InboundSendReject { return } @@ -104,67 +113,143 @@ func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo * } if len(out) > iputil.MaxRejectPacketSize { - if f.l.GetLevel() >= logrus.InfoLevel { - f.l. - WithField("packet", packet). - WithField("outPacket", out). - Info("rejectOutside: packet too big, not sending") + if f.l.Enabled(context.Background(), slog.LevelInfo) { + f.l.Info("rejectOutside: packet too big, not sending", + "packet", packet, + "outPacket", out, + ) } return } - f.sendNoMetrics(header.Message, 0, ci, hostinfo, nil, out, nb, packet, q) + f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q) } -func (f *Interface) Handshake(vpnIp iputil.VpnIp) { - f.getOrHandshake(vpnIp, nil) +// Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established +// it does not check if it is within our vpn networks! +func (f *Interface) Handshake(vpnAddr netip.Addr) { + f.handshakeManager.GetOrHandshake(vpnAddr, nil) } -// getOrHandshake returns nil if the vpnIp is not routable. +// getOrHandshakeNoRouting returns nil if the vpnAddr is not routable. // If the 2nd return var is false then the hostinfo is not ready to be used in a tunnel -func (f *Interface) getOrHandshake(vpnIp iputil.VpnIp, cacheCallback func(*HandshakeHostInfo)) (*HostInfo, bool) { - if !ipMaskContains(f.lightHouse.myVpnIp, f.lightHouse.myVpnZeros, vpnIp) { - vpnIp = f.inside.RouteFor(vpnIp) - if vpnIp == 0 { - return nil, false +func (f *Interface) getOrHandshakeNoRouting(vpnAddr netip.Addr, cacheCallback func(*HandshakeHostInfo)) (*HostInfo, bool) { + if f.myVpnNetworksTable.Contains(vpnAddr) { + return f.handshakeManager.GetOrHandshake(vpnAddr, cacheCallback) + } + + return nil, false +} + +// getOrHandshakeConsiderRouting will try to find the HostInfo to handle this packet, starting a handshake if necessary. +// If the 2nd return var is false then the hostinfo is not ready to be used in a tunnel. +func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cacheCallback func(*HandshakeHostInfo)) (*HostInfo, bool) { + destinationAddr := fwPacket.RemoteAddr + + hostinfo, ready := f.getOrHandshakeNoRouting(destinationAddr, cacheCallback) + + // Host is inside the mesh, no routing required + if hostinfo != nil { + return hostinfo, ready + } + + gateways := f.inside.RoutesFor(destinationAddr) + + switch len(gateways) { + case 0: + return nil, false + case 1: + // Single gateway route + return f.handshakeManager.GetOrHandshake(gateways[0].Addr(), cacheCallback) + default: + // Multi gateway route, perform ECMP categorization + gatewayAddr, balancingOk := routing.BalancePacket(fwPacket, gateways) + + if !balancingOk { + // This happens if the gateway buckets were not calculated, this _should_ never happen + f.l.Error("Gateway buckets not calculated, fallback from ECMP to random routing. Please report this bug.") + } + + var handshakeInfoForChosenGateway *HandshakeHostInfo + var hhReceiver = func(hh *HandshakeHostInfo) { + handshakeInfoForChosenGateway = hh } + + // Store the handshakeHostInfo for later. + // If this node is not reachable we will attempt other nodes, if none are reachable we will + // cache the packet for this gateway. + if hostinfo, ready = f.handshakeManager.GetOrHandshake(gatewayAddr, hhReceiver); ready { + return hostinfo, true + } + + // It appears the selected gateway cannot be reached, find another gateway to fallback on. + // The current implementation breaks ECMP but that seems better than no connectivity. + // If ECMP is also required when a gateway is down then connectivity status + // for each gateway needs to be kept and the weights recalculated when they go up or down. + // This would also need to interact with unsafe_route updates through reloading the config or + // use of the use_system_route_table option + + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("Calculated gateway for ECMP not available, attempting other gateways", + "destination", destinationAddr, + "originalGateway", gatewayAddr, + ) + } + + for i := range gateways { + // Skip the gateway that failed previously + if gateways[i].Addr() == gatewayAddr { + continue + } + + // We do not need the HandshakeHostInfo since we cache the packet in the originally chosen gateway + if hostinfo, ready = f.handshakeManager.GetOrHandshake(gateways[i].Addr(), nil); ready { + return hostinfo, true + } + } + + // No gateways reachable, cache the packet in the originally chosen gateway + cacheCallback(handshakeInfoForChosenGateway) + return hostinfo, false } - return f.handshakeManager.GetOrHandshake(vpnIp, cacheCallback) } func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) { fp := &firewall.Packet{} err := newPacket(p, false, fp) if err != nil { - f.l.Warnf("error while parsing outgoing packet for firewall check; %v", err) + f.l.Warn("error while parsing outgoing packet for firewall check", "error", err) return } // check if packet is in outbound fw rules - dropReason := f.firewall.Drop(p, *fp, false, hostinfo, f.pki.GetCAPool(), nil) + dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil) if dropReason != nil { - if f.l.Level >= logrus.DebugLevel { - f.l.WithField("fwPacket", fp). - WithField("reason", dropReason). - Debugln("dropping cached packet") + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("dropping cached packet", + "fwPacket", fp, + "reason", dropReason, + ) } return } - f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, nil, p, nb, out, 0) + f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0) } -// SendMessageToVpnIp handles real ip:port lookup and sends to the current best known address for vpnIp -func (f *Interface) SendMessageToVpnIp(t header.MessageType, st header.MessageSubType, vpnIp iputil.VpnIp, p, nb, out []byte) { - hostInfo, ready := f.getOrHandshake(vpnIp, func(hh *HandshakeHostInfo) { +// SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr. +// This function ignores myVpnNetworksTable, and will always attempt to treat the address as a vpnAddr +func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte) { + hostInfo, ready := f.handshakeManager.GetOrHandshake(vpnAddr, func(hh *HandshakeHostInfo) { hh.cachePacket(f.l, t, st, p, f.SendMessageToHostInfo, f.cachedPacketMetrics) }) if hostInfo == nil { - if f.l.Level >= logrus.DebugLevel { - f.l.WithField("vpnIp", vpnIp). - Debugln("dropping SendMessageToVpnIp, vpnIp not in our CIDR or in unsafe routes") + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("dropping SendMessageToVpnAddr, vpnAddr not in our vpn networks or in unsafe routes", + "vpnAddr", vpnAddr, + ) } return } @@ -182,10 +267,10 @@ func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.Messag func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p, nb, out []byte) { f.messageMetrics.Tx(t, st, 1) - f.sendNoMetrics(t, st, ci, hostinfo, nil, p, nb, out, 0) + f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, nb, out, 0) } -func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udp.Addr, p, nb, out []byte) { +func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte) { f.messageMetrics.Tx(t, st, 1) f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0) } @@ -212,7 +297,7 @@ func (f *Interface) SendVia(via *HostInfo, c := via.ConnectionState.messageCounter.Add(1) out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c) - f.connectionManager.Out(via.localIndexId) + f.connectionManager.Out(via) // Authenticate the header and payload, but do not encrypt for this message type. // The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload. @@ -220,12 +305,12 @@ func (f *Interface) SendVia(via *HostInfo, if noiseutil.EncryptLockNeeded { via.ConnectionState.writeLock.Unlock() } - via.logger(f.l). - WithField("outCap", cap(out)). - WithField("payloadLen", len(ad)). - WithField("headerLen", len(out)). - WithField("cipherOverhead", via.ConnectionState.eKey.Overhead()). - Error("SendVia out buffer not large enough for relay") + via.logger(f.l).Error("SendVia out buffer not large enough for relay", + "outCap", cap(out), + "payloadLen", len(ad), + "headerLen", len(out), + "cipherOverhead", via.ConnectionState.eKey.Overhead(), + ) return } @@ -245,22 +330,21 @@ func (f *Interface) SendVia(via *HostInfo, via.ConnectionState.writeLock.Unlock() } if err != nil { - via.logger(f.l).WithError(err).Info("Failed to EncryptDanger in sendVia") + via.logger(f.l).Info("Failed to EncryptDanger in sendVia", "error", err) return } - err = f.writers[0].WriteTo(out, via.remote) + err = f.writers[0].WriteTo(out, via.GetRemote()) if err != nil { - via.logger(f.l).WithError(err).Info("Failed to WriteTo in sendVia") + via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err) } f.connectionManager.RelayUsed(relay.LocalIndex) } -func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote *udp.Addr, p, nb, out []byte, q int) { +func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int) { if ci.eKey == nil { - //TODO: log warning return } - useRelay := remote == nil && hostinfo.remote == nil + useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid() fullOut := out if useRelay { @@ -281,17 +365,19 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType //l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p) out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c) - f.connectionManager.Out(hostinfo.localIndexId) + f.connectionManager.Out(hostinfo) // Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against - // all our IPs and enable a faster roaming. + // all our addrs and enable a faster roaming. if t != header.CloseTunnel && hostinfo.lastRebindCount != f.rebindCount { //NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is // finally used again. This tunnel would eventually be torn down and recreated if this action didn't help. - f.lightHouse.QueryServer(hostinfo.vpnIp) + f.lightHouse.QueryServer(hostinfo.vpnAddrs[0]) hostinfo.lastRebindCount = f.rebindCount - if f.l.Level >= logrus.DebugLevel { - f.l.WithField("vpnIp", hostinfo.vpnIp).Debug("Lighthouse update triggered for punch due to rebind counter") + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("Lighthouse update triggered for punch due to rebind counter", + "vpnAddrs", hostinfo.vpnAddrs, + ) } } @@ -301,32 +387,40 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType ci.writeLock.Unlock() } if err != nil { - hostinfo.logger(f.l).WithError(err). - WithField("udpAddr", remote).WithField("counter", c). - WithField("attemptedCounter", c). - Error("Failed to encrypt outgoing packet") + hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet", + "error", err, + "udpAddr", remote, + "counter", c, + ) return } - if remote != nil { + if remote.IsValid() { err = f.writers[q].WriteTo(out, remote) if err != nil { - hostinfo.logger(f.l).WithError(err). - WithField("udpAddr", remote).Error("Failed to write outgoing packet") + hostinfo.logger(f.l).Error("Failed to write outgoing packet", + "error", err, + "udpAddr", remote, + ) } - } else if hostinfo.remote != nil { - err = f.writers[q].WriteTo(out, hostinfo.remote) + } else if hr := hostinfo.GetRemote(); hr.IsValid() { + err = f.writers[q].WriteTo(out, hr) if err != nil { - hostinfo.logger(f.l).WithError(err). - WithField("udpAddr", remote).Error("Failed to write outgoing packet") + hostinfo.logger(f.l).Error("Failed to write outgoing packet", + "error", err, + "udpAddr", hr, + ) } } else { // Try to send via a relay for _, relayIP := range hostinfo.relayState.CopyRelayIps() { - relayHostInfo, relay, err := f.hostMap.QueryVpnIpRelayFor(hostinfo.vpnIp, relayIP) + relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP) if err != nil { hostinfo.relayState.DeleteRelay(relayIP) - hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo") + hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo", + "relay", relayIP, + "error", err, + ) continue } f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true) @@ -334,8 +428,3 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType } } } - -func isMulticast(ip iputil.VpnIp) bool { - // Class D multicast - return (((ip >> 24) & 0xff) & 0xf0) == 0xe0 -} diff --git a/inside_bsd.go b/inside_bsd.go index c9c7730d..dc847878 100644 --- a/inside_bsd.go +++ b/inside_bsd.go @@ -1,5 +1,4 @@ //go:build darwin || dragonfly || freebsd || netbsd || openbsd -// +build darwin dragonfly freebsd netbsd openbsd package nebula diff --git a/inside_generic.go b/inside_generic.go index 0bb2345a..bdcc1a6a 100644 --- a/inside_generic.go +++ b/inside_generic.go @@ -1,5 +1,4 @@ //go:build !darwin && !dragonfly && !freebsd && !netbsd && !openbsd -// +build !darwin,!dragonfly,!freebsd,!netbsd,!openbsd package nebula diff --git a/interface.go b/interface.go index d16348aa..c44f38b3 100644 --- a/interface.go +++ b/interface.go @@ -5,18 +5,20 @@ import ( "errors" "fmt" "io" - "net" - "os" - "runtime" + "log/slog" + "net/netip" + "slices" + "sync" "sync/atomic" "time" + "github.com/gaissmai/bart" "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" + + "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/udp" ) @@ -24,59 +26,61 @@ import ( const mtu = 9001 type InterfaceConfig struct { - HostMap *HostMap - Outside udp.Conn - Inside overlay.Device - pki *PKI - Cipher string - Firewall *Firewall - ServeDns bool - HandshakeManager *HandshakeManager - lightHouse *LightHouse - checkInterval time.Duration - pendingDeletionInterval time.Duration - DropLocalBroadcast bool - DropMulticast bool - routines int - MessageMetrics *MessageMetrics - version string - relayManager *relayManager - punchy *Punchy + HostMap *HostMap + Outside udp.Conn + Inside overlay.Device + pki *PKI + Cipher string + Firewall *Firewall + DnsServer *dnsServer + HandshakeManager *HandshakeManager + lightHouse *LightHouse + connectionManager *connectionManager + DropLocalBroadcast bool + DropMulticast bool + routines int + MessageMetrics *MessageMetrics + version string + relayManager *relayManager + punchy *Punchy tryPromoteEvery uint32 reQueryEvery uint32 reQueryWait time.Duration ConntrackCacheTimeout time.Duration - l *logrus.Logger + l *slog.Logger } type Interface struct { - hostMap *HostMap - outside udp.Conn - inside overlay.Device - pki *PKI - cipher string - firewall *Firewall - connectionManager *connectionManager - handshakeManager *HandshakeManager - serveDns bool - createTime time.Time - lightHouse *LightHouse - localBroadcast iputil.VpnIp - myVpnIp iputil.VpnIp - dropLocalBroadcast bool - dropMulticast bool - routines int - disconnectInvalid atomic.Bool - closed atomic.Bool - relayManager *relayManager + hostMap *HostMap + outside udp.Conn + inside overlay.Device + pki *PKI + firewall *Firewall + connectionManager *connectionManager + handshakeManager *HandshakeManager + dnsServer *dnsServer + createTime time.Time + lightHouse *LightHouse + myBroadcastAddrsTable *bart.Lite + myVpnAddrs []netip.Addr // A list of addresses assigned to us via our certificate + myVpnAddrsTable *bart.Lite + myVpnNetworks []netip.Prefix // A list of networks assigned to us via our certificate + myVpnNetworksTable *bart.Lite + dropLocalBroadcast bool + dropMulticast bool + routines int + disconnectInvalid atomic.Bool + closed atomic.Bool + relayManager *relayManager tryPromoteEvery atomic.Uint32 reQueryEvery atomic.Uint32 reQueryWait atomic.Int64 - sendRecvErrorConfig sendRecvErrorConfig + sendRecvErrorConfig recvErrorConfig + acceptRecvErrorConfig recvErrorConfig // rebindCount is used to decide if an active tunnel should trigger a punch notification through a lighthouse rebindCount int8 @@ -84,14 +88,22 @@ type Interface struct { conntrackCacheTimeout time.Duration + ctx context.Context writers []udp.Conn readers []io.ReadWriteCloser + wg sync.WaitGroup + + // fatalErr holds the first unexpected reader error that caused shutdown. + // nil means "no fatal error" (yet) + fatalErr atomic.Pointer[error] + // triggerShutdown is a function that will be run exactly once, when onFatal swaps something non-nil into fatalErr + triggerShutdown func() metricHandshakes metrics.Histogram messageMetrics *MessageMetrics cachedPacketMetrics *cachedPacketMetrics - l *logrus.Logger + l *slog.Logger } type EncWriter interface { @@ -102,39 +114,41 @@ type EncWriter interface { out []byte, nocopy bool, ) - SendMessageToVpnIp(t header.MessageType, st header.MessageSubType, vpnIp iputil.VpnIp, p, nb, out []byte) + SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) - Handshake(vpnIp iputil.VpnIp) + Handshake(vpnAddr netip.Addr) + GetHostInfo(vpnAddr netip.Addr) *HostInfo + GetCertState() *CertState } -type sendRecvErrorConfig uint8 +type recvErrorConfig uint8 const ( - sendRecvErrorAlways sendRecvErrorConfig = iota - sendRecvErrorNever - sendRecvErrorPrivate + recvErrorAlways recvErrorConfig = iota + recvErrorNever + recvErrorPrivate ) -func (s sendRecvErrorConfig) ShouldSendRecvError(ip net.IP) bool { +func (s recvErrorConfig) ShouldRecvError(endpoint netip.AddrPort) bool { switch s { - case sendRecvErrorPrivate: - return ip.IsPrivate() - case sendRecvErrorAlways: + case recvErrorPrivate: + return endpoint.Addr().IsPrivate() + case recvErrorAlways: return true - case sendRecvErrorNever: + case recvErrorNever: return false default: - panic(fmt.Errorf("invalid sendRecvErrorConfig value: %d", s)) + panic(fmt.Errorf("invalid recvErrorConfig value: %d", s)) } } -func (s sendRecvErrorConfig) String() string { +func (s recvErrorConfig) String() string { switch s { - case sendRecvErrorAlways: + case recvErrorAlways: return "always" - case sendRecvErrorNever: + case recvErrorNever: return "never" - case sendRecvErrorPrivate: + case recvErrorPrivate: return "private" default: return fmt.Sprintf("invalid(%d)", s) @@ -154,30 +168,35 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) { if c.Firewall == nil { return nil, errors.New("no firewall rules") } + if c.connectionManager == nil { + return nil, errors.New("no connection manager") + } - certificate := c.pki.GetCertState().Certificate - myVpnIp := iputil.Ip2VpnIp(certificate.Details.Ips[0].IP) + cs := c.pki.getCertState() ifce := &Interface{ - pki: c.pki, - hostMap: c.HostMap, - outside: c.Outside, - inside: c.Inside, - cipher: c.Cipher, - firewall: c.Firewall, - serveDns: c.ServeDns, - handshakeManager: c.HandshakeManager, - createTime: time.Now(), - lightHouse: c.lightHouse, - localBroadcast: myVpnIp | ^iputil.Ip2VpnIp(certificate.Details.Ips[0].Mask), - dropLocalBroadcast: c.DropLocalBroadcast, - dropMulticast: c.DropMulticast, - routines: c.routines, - version: c.version, - writers: make([]udp.Conn, c.routines), - readers: make([]io.ReadWriteCloser, c.routines), - myVpnIp: myVpnIp, - relayManager: c.relayManager, - + ctx: ctx, + pki: c.pki, + hostMap: c.HostMap, + outside: c.Outside, + inside: c.Inside, + firewall: c.Firewall, + dnsServer: c.DnsServer, + handshakeManager: c.HandshakeManager, + createTime: time.Now(), + lightHouse: c.lightHouse, + dropLocalBroadcast: c.DropLocalBroadcast, + dropMulticast: c.DropMulticast, + routines: c.routines, + version: c.version, + writers: make([]udp.Conn, c.routines), + readers: make([]io.ReadWriteCloser, c.routines), + myVpnNetworks: cs.myVpnNetworks, + myVpnNetworksTable: cs.myVpnNetworksTable, + myVpnAddrs: cs.myVpnAddrs, + myVpnAddrsTable: cs.myVpnAddrsTable, + myBroadcastAddrsTable: cs.myVpnBroadcastAddrsTable, + relayManager: c.relayManager, + connectionManager: c.connectionManager, conntrackCacheTimeout: c.ConntrackCacheTimeout, metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)), @@ -194,7 +213,10 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) { ifce.reQueryEvery.Store(c.reQueryEvery) ifce.reQueryWait.Store(int64(c.reQueryWait)) - ifce.connectionManager = newConnectionManager(ctx, c.l, ifce, c.checkInterval, c.pendingDeletionInterval, c.punchy) + ifce.connectionManager.intf = ifce + + // Held until Close so waiting on the interface blocks until the resources are actually released + ifce.wg.Add(1) return ifce, nil } @@ -202,18 +224,28 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) { // activate creates the interface on the host. After the interface is created, any // other services that want to bind listeners to its IP may do so successfully. However, // the interface isn't going to process anything until run() is called. -func (f *Interface) activate() { +func (f *Interface) activate() error { // actually turn on tun dev addr, err := f.outside.LocalAddr() if err != nil { - f.l.WithError(err).Error("Failed to get udp listen address") + f.l.Error("Failed to get udp listen address", "error", err) } - f.l.WithField("interface", f.inside.Name()).WithField("network", f.inside.Cidr().String()). - WithField("build", f.version).WithField("udpAddr", addr). - WithField("boringcrypto", boringEnabled()). - Info("Nebula interface is active") + f.l.Info("Nebula interface is active", + "interface", f.inside.Name(), + "networks", f.myVpnNetworks, + "build", f.version, + "udpAddr", addr, + "boringcrypto", boringEnabled(), + ) + + if f.routines > 1 { + if !f.inside.SupportsMultiqueue() || !f.outside.SupportsMultipleReaders() { + f.routines = 1 + f.l.Warn("routines is not supported on this platform, falling back to a single routine") + } + } metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines)) @@ -223,75 +255,116 @@ func (f *Interface) activate() { if i > 0 { reader, err = f.inside.NewMultiQueueReader() if err != nil { - f.l.Fatal(err) + return err } } f.readers[i] = reader } - if err := f.inside.Activate(); err != nil { - f.inside.Close() - f.l.Fatal(err) + // On error the caller owns the cleanup, Control.Start cancels the service context + // before releasing our resources so a waiter never observes a live context + if err = f.inside.Activate(); err != nil { + return err } + + return nil } func (f *Interface) run() { // Launch n queues to read packets from udp for i := 0; i < f.routines; i++ { - go f.listenOut(i) + f.wg.Go(func() { + f.listenOut(i) + }) } // Launch n queues to read packets from tun dev for i := 0; i < f.routines; i++ { - go f.listenIn(f.readers[i], i) + f.wg.Go(func() { + f.listenIn(f.readers[i], i) + }) } + } -func (f *Interface) listenOut(i int) { - runtime.LockOSThread() +func (f *Interface) wait() error { + f.wg.Wait() + if e := f.fatalErr.Load(); e != nil { + return *e + } + return nil +} +// onFatal stores the first fatal reader error, and calls triggerShutdown if it was the first one +func (f *Interface) onFatal(err error) { + swapped := f.fatalErr.CompareAndSwap(nil, &err) + if !swapped { + return + } + if f.triggerShutdown != nil { + f.triggerShutdown() + } +} + +func (f *Interface) listenOut(i int) { var li udp.Conn - // TODO clean this up with a coherent interface for each outside connection if i > 0 { li = f.writers[i] } else { li = f.outside } + ctCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout) lhh := f.lightHouse.NewRequestHandler() - conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout) - li.ListenOut(readOutsidePackets(f), lhHandleRequest(lhh, f), conntrackCache, i) + plaintext := make([]byte, udp.MTU) + h := &header.H{} + fwPacket := &firewall.Packet{} + nb := make([]byte, 12, 12) + + err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) { + f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get()) + }) + + // An error after teardown began is shutdown noise, the closed flag covers resources + // Close releases itself and the cancelled ctx covers ones torn down by their owners + // reacting to it, like the user device pipes + if err != nil && !f.closed.Load() && f.ctx.Err() == nil { + f.l.Error("Error while reading inbound packet, closing", "error", err) + f.onFatal(err) + } + + f.l.Debug("underlay reader is done", "reader", i) } func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) { - runtime.LockOSThread() - packet := make([]byte, mtu) out := make([]byte, mtu) fwPacket := &firewall.Packet{} nb := make([]byte, 12, 12) - conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout) + conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout) for { n, err := reader.Read(packet) if err != nil { - if errors.Is(err, os.ErrClosed) && f.closed.Load() { - return + // Same shutdown noise handling as listenOut + if !f.closed.Load() && f.ctx.Err() == nil { + f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i) + f.onFatal(err) } - - f.l.WithError(err).Error("Error while reading outbound packet") - // This only seems to happen when something fatal happens to the fd, so exit. - os.Exit(2) + break } - f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l)) + f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get()) } + + f.l.Debug("overlay reader is done", "reader", i) } func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) { c.RegisterReloadCallback(f.reloadFirewall) c.RegisterReloadCallback(f.reloadSendRecvError) + c.RegisterReloadCallback(f.reloadAcceptRecvError) c.RegisterReloadCallback(f.reloadDisconnectInvalid) c.RegisterReloadCallback(f.reloadMisc) @@ -305,21 +378,30 @@ func (f *Interface) reloadDisconnectInvalid(c *config.C) { if initial || c.HasChanged("pki.disconnect_invalid") { f.disconnectInvalid.Store(c.GetBool("pki.disconnect_invalid", true)) if !initial { - f.l.Infof("pki.disconnect_invalid changed to %v", f.disconnectInvalid.Load()) + f.l.Info("pki.disconnect_invalid changed", "value", f.disconnectInvalid.Load()) } } } func (f *Interface) reloadFirewall(c *config.C) { - //TODO: need to trigger/detect if the certificate changed too - if c.HasChanged("firewall") == false { + cs := f.pki.getCertState() + curCert := cs.getCertificate(cert.Version2) + if curCert == nil { + curCert = cs.getCertificate(cert.Version1) + } + + // The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction. + // Check to see if that set has changed, and if so, rebuild the firewall. + certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks) + + if !c.HasChanged("firewall") && !certUnsafeChanged { f.l.Debug("No firewall config change detected") return } - fw, err := NewFirewallFromConfig(f.l, f.pki.GetCertState().Certificate, c) + fw, err := NewFirewallFromConfig(f.l, cs, c) if err != nil { - f.l.WithError(err).Error("Error while creating firewall during reload") + f.l.Error("Error while creating firewall during reload", "error", err) return } @@ -332,10 +414,11 @@ func (f *Interface) reloadFirewall(c *config.C) { // If rulesVersion is back to zero, we have wrapped all the way around. Be // safe and just reset conntrack in this case. if fw.rulesVersion == 0 { - f.l.WithField("firewallHashes", fw.GetRuleHashes()). - WithField("oldFirewallHashes", oldFw.GetRuleHashes()). - WithField("rulesVersion", fw.rulesVersion). - Warn("firewall rulesVersion has overflowed, resetting conntrack") + f.l.Warn("firewall rulesVersion has overflowed, resetting conntrack", + "firewallHashes", fw.GetRuleHashes(), + "oldFirewallHashes", oldFw.GetRuleHashes(), + "rulesVersion", fw.rulesVersion, + ) } else { fw.Conntrack = conntrack } @@ -343,10 +426,11 @@ func (f *Interface) reloadFirewall(c *config.C) { f.firewall = fw oldFw.Destroy() - f.l.WithField("firewallHashes", fw.GetRuleHashes()). - WithField("oldFirewallHashes", oldFw.GetRuleHashes()). - WithField("rulesVersion", fw.rulesVersion). - Info("New firewall has been installed") + f.l.Info("New firewall has been installed", + "firewallHashes", fw.GetRuleHashes(), + "oldFirewallHashes", oldFw.GetRuleHashes(), + "rulesVersion", fw.rulesVersion, + ) } func (f *Interface) reloadSendRecvError(c *config.C) { @@ -355,21 +439,43 @@ func (f *Interface) reloadSendRecvError(c *config.C) { switch stringValue { case "always": - f.sendRecvErrorConfig = sendRecvErrorAlways + f.sendRecvErrorConfig = recvErrorAlways case "never": - f.sendRecvErrorConfig = sendRecvErrorNever + f.sendRecvErrorConfig = recvErrorNever case "private": - f.sendRecvErrorConfig = sendRecvErrorPrivate + f.sendRecvErrorConfig = recvErrorPrivate default: if c.GetBool("listen.send_recv_error", true) { - f.sendRecvErrorConfig = sendRecvErrorAlways + f.sendRecvErrorConfig = recvErrorAlways } else { - f.sendRecvErrorConfig = sendRecvErrorNever + f.sendRecvErrorConfig = recvErrorNever } } - f.l.WithField("sendRecvError", f.sendRecvErrorConfig.String()). - Info("Loaded send_recv_error config") + f.l.Info("Loaded send_recv_error config", "sendRecvError", f.sendRecvErrorConfig.String()) + } +} + +func (f *Interface) reloadAcceptRecvError(c *config.C) { + if c.InitialLoad() || c.HasChanged("listen.accept_recv_error") { + stringValue := c.GetString("listen.accept_recv_error", "always") + + switch stringValue { + case "always": + f.acceptRecvErrorConfig = recvErrorAlways + case "never": + f.acceptRecvErrorConfig = recvErrorNever + case "private": + f.acceptRecvErrorConfig = recvErrorPrivate + default: + if c.GetBool("listen.accept_recv_error", true) { + f.acceptRecvErrorConfig = recvErrorAlways + } else { + f.acceptRecvErrorConfig = recvErrorNever + } + } + + f.l.Info("Loaded accept_recv_error config", "acceptRecvError", f.acceptRecvErrorConfig.String()) } } @@ -400,30 +506,75 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) { udpStats := udp.NewUDPStatsEmitter(f.writers) certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) + certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil) + certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil) + + emit := func() { + f.firewall.EmitStats() + f.handshakeManager.EmitStats() + udpStats() + + certState := f.pki.getCertState() + defaultCrt := certState.GetDefaultCertificate() + certExpirationGauge.Update(int64(defaultCrt.NotAfter().Sub(time.Now()) / time.Second)) + certInitiatingVersion.Update(int64(defaultCrt.Version())) + + // Report the max certificate version we are capable of using + if certState.v2Cert != nil { + certMaxVersion.Update(int64(certState.v2Cert.Version())) + } else { + certMaxVersion.Update(int64(certState.v1Cert.Version())) + } + } + + // Prime gauges so a Prometheus scrape that lands before the first tick + // sees real values instead of the zero defaults (issue #907). + emit() for { select { case <-ctx.Done(): return case <-ticker.C: - f.firewall.EmitStats() - f.handshakeManager.EmitStats() - udpStats() - certExpirationGauge.Update(int64(f.pki.GetCertState().Certificate.Details.NotAfter.Sub(time.Now()) / time.Second)) + emit() } } } +func (f *Interface) GetHostInfo(vpnIp netip.Addr) *HostInfo { + return f.hostMap.QueryVpnAddr(vpnIp) +} + +func (f *Interface) GetCertState() *CertState { + return f.pki.getCertState() +} + +// Close releases the interface's resources: the udp sockets and the tun device. +// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated, +// calls after the first return nil without doing anything. func (f *Interface) Close() error { - f.closed.Store(true) + if !f.closed.CompareAndSwap(false, true) { + return nil + } + + var errs []error - for _, u := range f.writers { + // Release the udp readers + for i, u := range f.writers { err := u.Close() if err != nil { - f.l.WithError(err).Error("Error while closing udp socket") + f.l.Error("Error while closing udp socket", "error", err, "writer", i) + errs = append(errs, err) } } - // Release the tun device - return f.inside.Close() + // Release the tun device (closing the tun also closes all readers) + closeErr := f.inside.Close() + if closeErr != nil { + errs = append(errs, closeErr) + } + + // Release the construction token so waiters know the resources are gone + f.wg.Done() + return errors.Join(errs...) } diff --git a/interface_emit_test.go b/interface_emit_test.go new file mode 100644 index 00000000..b0a9d025 --- /dev/null +++ b/interface_emit_test.go @@ -0,0 +1,73 @@ +//go:build linux || darwin + +package nebula + +import ( + "context" + "net/netip" + "testing" + "time" + + "github.com/rcrowley/go-metrics" + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/firewall" + "github.com/slackhq/nebula/overlay/overlaytest" + "github.com/slackhq/nebula/test" + "github.com/slackhq/nebula/udp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Test_emitStats_primesGauges covers issue #907: a Prometheus scrape that +// landed before the first ticker fire used to read 0 for the cert gauges. +// emitStats now primes the gauges before entering the ticker loop. We assert +// the gauge is zero before the first call and non-zero after. +func Test_emitStats_primesGauges(t *testing.T) { + defer metrics.DefaultRegistry.UnregisterAll() + + l := test.NewLogger() + hostMap := newHostMap(l) + preferredRanges := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")} + hostMap.preferredRanges.Store(&preferredRanges) + + notAfter := time.Now().Add(time.Hour) + cs := &CertState{ + initiatingVersion: cert.Version1, + privateKey: []byte{}, + v1Cert: &dummyCert{version: cert.Version1, notAfter: notAfter}, + v1Credential: nil, + } + + lh := newTestLighthouse() + ifce := &Interface{ + hostMap: hostMap, + inside: &overlaytest.NoopTun{}, + outside: &udp.NoopConn{}, + firewall: &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.Packet]*conn{}}}, + lightHouse: lh, + pki: &PKI{}, + handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig), + l: l, + // On linux, udp.NewUDPStatsEmitter indexes writers[0] and asserts to + // *udp.StdConn. A zero value works: getMemInfo sees a nil rawConn, + // returns an error, and the emitter falls through to a no-op. + writers: []udp.Conn{&udp.StdConn{}}, + } + ifce.pki.cs.Store(cs) + + ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) + require.Zero(t, ttlGauge.Value(), "gauge should be zero before emitStats runs") + + // Pre-cancel the context so emitStats returns after priming the gauges + // without ever reading from ticker.C. The one hour interval is just a + // belt-and-suspenders, the test does not expect the ticker to fire. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + ifce.emitStats(ctx, time.Hour) + + ttl := ttlGauge.Value() + assert.Positive(t, ttl, "ttl gauge should be primed by emitStats before its first tick") + assert.LessOrEqual(t, ttl, int64(3600)) + assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value()) + assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value()) +} diff --git a/interface_test.go b/interface_test.go new file mode 100644 index 00000000..1b912bbb --- /dev/null +++ b/interface_test.go @@ -0,0 +1,120 @@ +package nebula + +import ( + "net/netip" + "testing" + + "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestReloadFirewall_CertUnsafeNetworksChanged verifies that reloadFirewall +// rebuilds the firewall when only the certificate's UnsafeNetworks have changed, +// even if the firewall section of the YAML has not. +func TestReloadFirewall_CertUnsafeNetworksChanged(t *testing.T) { + l := test.NewLogger() + + vpnNet := netip.MustParsePrefix("10.0.0.1/24") + initialUnsafe := []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")} + + // dummyCert avoids dragging the real signing pipeline into a unit test. + c1 := &dummyCert{ + version: cert.Version2, + networks: []netip.Prefix{vpnNet}, + unsafeNetworks: initialUnsafe, + } + pki := &PKI{} + pki.cs.Store(&CertState{v2Cert: c1, initiatingVersion: cert.Version2}) + + rawYAML := `firewall: + outbound: + - port: any + proto: any + host: any + inbound: + - port: any + proto: any + host: any +` + cfg := config.NewC(l) + require.NoError(t, cfg.LoadString(rawYAML)) + + fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg) + require.NoError(t, err) + require.Equal(t, initialUnsafe, fw.unsafeNetworks) + + f := &Interface{ + pki: pki, + firewall: fw, + l: l, + } + + // Swap the cert with a different UnsafeNetworks set. + newUnsafe := []netip.Prefix{ + netip.MustParsePrefix("198.51.100.0/24"), + netip.MustParsePrefix("203.0.113.0/24"), + } + c2 := &dummyCert{ + version: cert.Version2, + networks: []netip.Prefix{vpnNet}, + unsafeNetworks: newUnsafe, + } + pki.cs.Store(&CertState{v2Cert: c2, initiatingVersion: cert.Version2}) + + // Reload with the same YAML so HasChanged("firewall") reports false. + require.NoError(t, cfg.ReloadConfigString(rawYAML)) + require.False(t, cfg.HasChanged("firewall")) + + f.reloadFirewall(cfg) + + assert.NotSame(t, fw, f.firewall, "firewall pointer should have been replaced") + assert.Equal(t, newUnsafe, f.firewall.unsafeNetworks) + assert.True(t, f.firewall.routableNetworks.Contains(netip.MustParseAddr("203.0.113.5"))) +} + +// TestReloadFirewall_NoChange verifies that reloadFirewall is a no-op when +// neither the firewall config nor the cert's UnsafeNetworks have changed. +func TestReloadFirewall_NoChange(t *testing.T) { + l := test.NewLogger() + + vpnNet := netip.MustParsePrefix("10.0.0.1/24") + unsafe := []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")} + + c1 := &dummyCert{ + version: cert.Version2, + networks: []netip.Prefix{vpnNet}, + unsafeNetworks: unsafe, + } + pki := &PKI{} + pki.cs.Store(&CertState{v2Cert: c1, initiatingVersion: cert.Version2}) + + rawYAML := `firewall: + outbound: + - port: any + proto: any + host: any + inbound: + - port: any + proto: any + host: any +` + cfg := config.NewC(l) + require.NoError(t, cfg.LoadString(rawYAML)) + + fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg) + require.NoError(t, err) + + f := &Interface{ + pki: pki, + firewall: fw, + l: l, + } + + require.NoError(t, cfg.ReloadConfigString(rawYAML)) + f.reloadFirewall(cfg) + + assert.Same(t, fw, f.firewall, "firewall should not have been replaced") +} diff --git a/iputil/packet.go b/iputil/packet.go index b18e5244..c0c1921e 100644 --- a/iputil/packet.go +++ b/iputil/packet.go @@ -4,26 +4,54 @@ import ( "encoding/binary" "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" ) const ( - // Need 96 bytes for the largest reject packet: + // MaxIPv4RejectPacketSize is the largest IPv4 reject packet: // - 20 byte ipv4 header // - 8 byte icmpv4 header // - 68 byte body (60 byte max orig ipv4 header + 8 byte orig icmpv4 header) - MaxRejectPacketSize = ipv4.HeaderLen + 8 + 60 + 8 + maxIPv4RejectPacketSize = ipv4.HeaderLen + 8 + 60 + 8 + + // MaxRejectPacketSize is sized for the largest possible reject packet (IPv6): + // - 40 byte ipv6 header + // - 8 byte icmpv6 header + // - up to 1000 byte body (original packet, possibly truncated. We want to stay + // under the MTU with Nebula overhead included) + maxIPv6RejectPacketSize = ipv6.HeaderLen + 8 + 1000 + + MaxRejectPacketSize = maxIPv6RejectPacketSize ) func CreateRejectPacket(packet []byte, out []byte) []byte { - if len(packet) < ipv4.HeaderLen || int(packet[0]>>4) != ipv4.Version { + if len(packet) < 1 { return nil } - switch packet[9] { - case 6: // tcp - return ipv4CreateRejectTCPPacket(packet, out) + version := int(packet[0] >> 4) + switch version { + case ipv4.Version: + if len(packet) < ipv4.HeaderLen { + return nil + } + // Do not send reject packets for non-first fragments + if packet[6]&0x1f != 0 || packet[7] != 0 { + return nil + } + switch packet[9] { + case 6: // tcp + return ipv4CreateRejectTCPPacket(packet, out) + default: + return ipv4CreateRejectICMPPacket(packet, out) + } + case ipv6.Version: + if len(packet) < ipv6.HeaderLen { + return nil + } + return ipv6CreateRejectPacket(packet, out) default: - return ipv4CreateRejectICMPPacket(packet, out) + return nil } } @@ -35,12 +63,17 @@ func ipv4CreateRejectICMPPacket(packet []byte, out []byte) []byte { return nil } - // ICMP reply includes original header and first 8 bytes of the packet - packetLen := len(packet) - if packetLen > ihl+8 { - packetLen = ihl + 8 + // Do not generate ICMP errors in response to ICMP error packets + if packet[9] == 1 && len(packet) > ihl { + icmpType := packet[ihl] + if icmpType == 3 || icmpType == 4 || icmpType == 5 || icmpType == 11 || icmpType == 12 { + return nil + } } + // ICMP reply includes original header and first 8 bytes of the packet + packetLen := min(len(packet), ihl+8) + outLen := ipv4.HeaderLen + 8 + packetLen if outLen > cap(out) { return nil @@ -71,14 +104,14 @@ func ipv4CreateRejectICMPPacket(packet []byte, out []byte) []byte { // ICMP Destination Unreachable icmpOut := out[ipv4.HeaderLen:] - icmpOut[0] = 3 // type (Destination unreachable) - icmpOut[1] = 3 // code (Port unreachable error) - icmpOut[2] = 0 // checksum - icmpOut[3] = 0 // . - icmpOut[4] = 0 // unused - icmpOut[5] = 0 // . - icmpOut[6] = 0 // . - icmpOut[7] = 0 // . + icmpOut[0] = 3 // type (Destination unreachable) + icmpOut[1] = 13 // code (Communication administratively prohibited) + icmpOut[2] = 0 // checksum + icmpOut[3] = 0 // . + icmpOut[4] = 0 // unused + icmpOut[5] = 0 // . + icmpOut[6] = 0 // . + icmpOut[7] = 0 // . // Copy original IP header and first 8 bytes as body copy(icmpOut[8:], packet[:packetLen]) @@ -165,7 +198,193 @@ func ipv4CreateRejectTCPPacket(packet []byte, out []byte) []byte { return out } +func ipv6CreateRejectPacket(packet []byte, out []byte) []byte { + proto, offset, isFragment := ipv6FindUpperProtocol(packet) + if isFragment { + return nil + } + switch proto { + case 6: // tcp + return ipv6CreateRejectTCPPacket(packet, out, offset) + default: + return ipv6CreateRejectICMPPacket(packet, out, proto, offset) + } +} + +func ipv6CreateRejectICMPPacket(packet []byte, out []byte, proto uint8, offset int) []byte { + // Do not generate ICMPv6 errors in response to ICMPv6 error packets + if proto == 58 && len(packet) > offset { + icmpType := packet[offset] + if icmpType >= 1 && icmpType <= 4 { + return nil + } + } + + // Include as much of the original packet as possible, up to 1000 bytes, + // so the response fits comfortably within any tunnel MTU. + packetLen := min(len(packet), 1000) + + outLen := ipv6.HeaderLen + 8 + packetLen + if outLen > cap(out) { + return nil + } + + out = out[:outLen] + + // IPv6 header + ipHdr := out[0:ipv6.HeaderLen] + ipHdr[0] = ipv6.Version << 4 // version, traffic class (high bits) + ipHdr[1] = 0 // traffic class (low bits), flow label (high bits) + ipHdr[2] = 0 // flow label + ipHdr[3] = 0 // flow label + + payloadLen := uint16(outLen - ipv6.HeaderLen) + binary.BigEndian.PutUint16(ipHdr[4:], payloadLen) // payload length + ipHdr[6] = 58 // next header (ICMPv6) + ipHdr[7] = 64 // hop limit + + // Swap dest / src IPs (each 16 bytes, src at 8, dst at 24) + copy(ipHdr[8:24], packet[24:40]) + copy(ipHdr[24:40], packet[8:24]) + + // ICMPv6 Destination Unreachable + icmpOut := out[ipv6.HeaderLen:] + icmpOut[0] = 1 // type (Destination Unreachable) + icmpOut[1] = 1 // code (Communication with destination administratively prohibited) + icmpOut[2] = 0 // checksum + icmpOut[3] = 0 // . + icmpOut[4] = 0 // unused + icmpOut[5] = 0 // . + icmpOut[6] = 0 // . + icmpOut[7] = 0 // . + + copy(icmpOut[8:], packet[:packetLen]) + + // ICMPv6 checksum uses a pseudo-header + csum := ipv6PseudoheaderChecksum(ipHdr[8:24], ipHdr[24:40], 58, uint32(payloadLen)) + binary.BigEndian.PutUint16(icmpOut[2:], tcpipChecksum(icmpOut, csum)) + + return out +} + +func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte { + const tcpLen = 20 + + if len(packet) < offset+tcpLen { + return nil + } + + outLen := ipv6.HeaderLen + tcpLen + if outLen > cap(out) { + return nil + } + + out = out[:outLen] + + // IPv6 header + ipHdr := out[0:ipv6.HeaderLen] + ipHdr[0] = ipv6.Version << 4 // version, traffic class (high bits) + ipHdr[1] = 0 // traffic class (low bits), flow label (high bits) + ipHdr[2] = 0 // flow label + ipHdr[3] = 0 // flow label + + binary.BigEndian.PutUint16(ipHdr[4:], tcpLen) // payload length + ipHdr[6] = 6 // next header (TCP) + ipHdr[7] = 64 // hop limit + + // Swap dest / src IPs + copy(ipHdr[8:24], packet[24:40]) + copy(ipHdr[24:40], packet[8:24]) + + // TCP RST + tcpIn := packet[offset:] + var ackSeq, seq uint32 + outFlags := byte(0b00000100) // RST + + inAck := tcpIn[13]&0b00010000 != 0 + if inAck { + seq = binary.BigEndian.Uint32(tcpIn[8:]) + } else { + inSyn := uint32((tcpIn[13] & 0b00000010) >> 1) + inFin := uint32(tcpIn[13] & 0b00000001) + ackSeq = binary.BigEndian.Uint32(tcpIn[4:]) + inSyn + inFin + uint32(len(tcpIn)) - uint32(tcpIn[12]>>4)<<2 + outFlags |= 0b00010000 // ACK + } + + tcpOut := out[ipv6.HeaderLen:] + // Swap dest / src ports + copy(tcpOut[0:2], tcpIn[2:4]) + copy(tcpOut[2:4], tcpIn[0:2]) + binary.BigEndian.PutUint32(tcpOut[4:], seq) + binary.BigEndian.PutUint32(tcpOut[8:], ackSeq) + tcpOut[12] = (tcpLen >> 2) << 4 // data offset, reserved, NS + tcpOut[13] = outFlags // CWR, ECE, URG, ACK, PSH, RST, SYN, FIN + tcpOut[14] = 0 // window size + tcpOut[15] = 0 // . + tcpOut[16] = 0 // checksum + tcpOut[17] = 0 // . + tcpOut[18] = 0 // URG Pointer + tcpOut[19] = 0 // . + + // Calculate checksum with IPv6 pseudo-header + csum := ipv6PseudoheaderChecksum(ipHdr[8:24], ipHdr[24:40], 6, tcpLen) + binary.BigEndian.PutUint16(tcpOut[16:], tcpipChecksum(tcpOut, csum)) + + return out +} + +func ipv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool) { + nextHeader = packet[6] + offset = ipv6.HeaderLen + + for { + switch nextHeader { + case 0, 43, 60: // Hop-by-Hop, Routing, Destination + if len(packet) < offset+2 { + return nextHeader, offset, isFragment + } + nextHeader = packet[offset] + offset += (int(packet[offset+1]) + 1) << 3 + + case 44: // Fragment + if len(packet) < offset+8 { + return nextHeader, offset, isFragment + } + if packet[offset+2] != 0 || packet[offset+3]&0xf8 != 0 { + isFragment = true + } + nextHeader = packet[offset] + offset += 8 + + case 51: // AH + if len(packet) < offset+2 { + return nextHeader, offset, isFragment + } + nextHeader = packet[offset] + offset += (int(packet[offset+1]) + 2) << 2 + + default: + return nextHeader, offset, isFragment + } + } +} + func CreateICMPEchoResponse(packet, out []byte) []byte { + if len(packet) < 1 { + return nil + } + + switch packet[0] >> 4 { + case 4: + return createICMPv4EchoResponse(packet, out) + case 6: + return createICMPv6EchoResponse(packet, out) + default: + return nil + } +} + +func createICMPv4EchoResponse(packet, out []byte) []byte { // Return early if this is not a simple ICMP Echo Request //TODO: make constants out of these if !(len(packet) >= 28 && len(packet) <= 9001 && packet[0] == 0x45 && packet[9] == 0x01 && packet[20] == 0x08) { @@ -199,6 +418,43 @@ func CreateICMPEchoResponse(packet, out []byte) []byte { return out } +func createICMPv6EchoResponse(packet, out []byte) []byte { + // IPv6 header (40 bytes) + ICMPv6 header (8 bytes minimum) + if len(packet) < ipv6.HeaderLen+8 || len(packet) > 9001 { + return nil + } + + // Next Header must be ICMPv6 (58) + if packet[6] != 58 { + return nil + } + + // ICMPv6 type must be Echo Request (128) + if packet[ipv6.HeaderLen] != 128 { + return nil + } + + out = out[:len(packet)] + copy(out, packet) + + // Swap src/dst addresses (bytes 8-23 and 24-39) + copy(out[8:24], packet[24:40]) + copy(out[24:40], packet[8:24]) + + // Change ICMPv6 type to Echo Reply (129) + icmp := out[ipv6.HeaderLen:] + icmp[0] = 129 + icmp[2] = 0 + icmp[3] = 0 + + // ICMPv6 checksum uses a pseudo-header with src, dst, length, and next header + payloadLen := uint32(len(icmp)) + csum := ipv6PseudoheaderChecksum(out[8:24], out[24:40], 58, payloadLen) + binary.BigEndian.PutUint16(icmp[2:], tcpipChecksum(icmp, csum)) + + return out +} + // calculates the TCP/IP checksum defined in rfc1071. The passed-in // csum is any initial checksum data that's already been computed. // @@ -236,3 +492,18 @@ func ipv4PseudoheaderChecksum(src, dst []byte, proto, length uint32) (csum uint3 csum += length >> 16 return csum } + +// based on: +// - https://github.com/google/gopacket/blob/v1.1.19/layers/tcpip.go#L37-L48 +func ipv6PseudoheaderChecksum(src, dst []byte, proto, length uint32) (csum uint32) { + for i := 0; i < 16; i += 2 { + csum += uint32(src[i]) << 8 + csum += uint32(src[i+1]) + csum += uint32(dst[i]) << 8 + csum += uint32(dst[i+1]) + } + csum += proto + csum += length & 0xffff + csum += length >> 16 + return csum +} diff --git a/iputil/packet_test.go b/iputil/packet_test.go index e1d0d95d..6d567d51 100644 --- a/iputil/packet_test.go +++ b/iputil/packet_test.go @@ -1,11 +1,13 @@ package iputil import ( + "encoding/binary" "net" "testing" "github.com/stretchr/testify/assert" "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" ) func Test_CreateRejectPacket(t *testing.T) { @@ -43,7 +45,7 @@ func Test_CreateRejectPacket(t *testing.T) { } b = append(b, []byte{0, 3, 0, 4, 0, 0, 0, 0}...) - expectedLen = MaxRejectPacketSize + expectedLen = maxIPv4RejectPacketSize out = make([]byte, MaxRejectPacketSize) rejectPacket = CreateRejectPacket(b, out) assert.NotNil(t, rejectPacket) @@ -71,3 +73,404 @@ func Test_CreateRejectPacket(t *testing.T) { assert.NotNil(t, rejectPacket) assert.Len(t, rejectPacket, expectedLen) } + +func Test_CreateRejectPacket_NoFragment(t *testing.T) { + out := make([]byte, MaxRejectPacketSize) + + // IPv4: non-zero fragment offset should not generate reject packet + h := ipv4.Header{ + Len: 20, + Src: net.IPv4(10, 0, 0, 1), + Dst: net.IPv4(10, 0, 0, 2), + Protocol: 17, // UDP + } + b, err := h.Marshal() + if err != nil { + t.Fatalf("h.Marshal: %v", err) + } + b = append(b, make([]byte, 8)...) + // Set fragment offset to non-zero (byte 6-7, offset in 8-byte units) + b[6] = 0x00 + b[7] = 0x01 + assert.Nil(t, CreateRejectPacket(b, out)) + + // MF flag with zero offset (first fragment) should still generate reject + b[6] = 0x20 // MF flag set + b[7] = 0x00 + assert.NotNil(t, CreateRejectPacket(b, out)) + + // Non-fragment should still generate reject packet + b[6] = 0x00 + b[7] = 0x00 + assert.NotNil(t, CreateRejectPacket(b, out)) + + // DF flag only (not a fragment) should still generate reject packet + b[6] = 0x40 + b[7] = 0x00 + assert.NotNil(t, CreateRejectPacket(b, out)) +} + +func Test_CreateRejectPacketIPv6_NoFragment(t *testing.T) { + src := net.ParseIP("fd00::1") + dst := net.ParseIP("fd00::2") + out := make([]byte, MaxRejectPacketSize) + + // IPv6 with Fragment header and non-zero offset should not generate reject + fragHeader := []byte{ + 17, // next header: UDP + 0, // reserved + 0, 9, // fragment offset=1 (shifted left 3), M=1 + 0, 0, 0, 1, // identification + } + udpPayload := make([]byte, 8) + payload := append(fragHeader, udpPayload...) + packet := makeIPv6Packet(src, dst, 44, payload) // next header 44 = Fragment + assert.Nil(t, CreateRejectPacket(packet, out)) + + // Fragment header with zero offset (first fragment) should still generate reject + fragHeader[2] = 0 + fragHeader[3] = 1 // offset=0, M=1 + payload = append(fragHeader, udpPayload...) + packet = makeIPv6Packet(src, dst, 44, payload) + assert.NotNil(t, CreateRejectPacket(packet, out)) +} + +func Test_CreateRejectPacket_NoICMPError(t *testing.T) { + out := make([]byte, MaxRejectPacketSize) + + // ICMP error types should not generate reject packets + icmpErrorTypes := []byte{3, 4, 5, 11, 12} + for _, icmpType := range icmpErrorTypes { + h := ipv4.Header{ + Len: 20, + Src: net.IPv4(10, 0, 0, 1), + Dst: net.IPv4(10, 0, 0, 2), + Protocol: 1, // ICMP + } + + b, err := h.Marshal() + if err != nil { + t.Fatalf("h.Marshal: %v", err) + } + b = append(b, icmpType, 0, 0, 0, 0, 0, 0, 0) + + rejectPacket := CreateRejectPacket(b, out) + assert.Nil(t, rejectPacket, "ICMP type %d should not generate a reject packet", icmpType) + } + + // ICMP non-error types should still generate reject packets + icmpNonErrorTypes := []byte{0, 8, 13, 14} + for _, icmpType := range icmpNonErrorTypes { + h := ipv4.Header{ + Len: 20, + Src: net.IPv4(10, 0, 0, 1), + Dst: net.IPv4(10, 0, 0, 2), + Protocol: 1, // ICMP + } + + b, err := h.Marshal() + if err != nil { + t.Fatalf("h.Marshal: %v", err) + } + b = append(b, icmpType, 0, 0, 0, 0, 0, 0, 0) + + rejectPacket := CreateRejectPacket(b, out) + assert.NotNil(t, rejectPacket, "ICMP type %d should generate a reject packet", icmpType) + } +} + +func makeIPv6Packet(src, dst net.IP, nextHeader uint8, payload []byte) []byte { + b := make([]byte, ipv6.HeaderLen+len(payload)) + b[0] = ipv6.Version << 4 + binary.BigEndian.PutUint16(b[4:], uint16(len(payload))) + b[6] = nextHeader + b[7] = 64 + copy(b[8:24], src.To16()) + copy(b[24:40], dst.To16()) + copy(b[ipv6.HeaderLen:], payload) + return b +} + +func Test_CreateRejectPacketIPv6_ICMP(t *testing.T) { + src := net.ParseIP("fd00::1") + dst := net.ParseIP("fd00::2") + + // Small UDP packet: entire original included in body + udpPayload := make([]byte, 20) + udpPayload[0] = 0x00 // src port high + udpPayload[1] = 0x50 // src port low (80) + udpPayload[2] = 0x01 // dst port high + udpPayload[3] = 0xBB // dst port low (443) + packet := makeIPv6Packet(src, dst, 17, udpPayload) + + out := make([]byte, MaxRejectPacketSize) + rejectPacket := CreateRejectPacket(packet, out) + assert.NotNil(t, rejectPacket) + + // Small packet fits entirely: 40 (ipv6 hdr) + 8 (icmpv6 hdr) + 60 (original) + expectedLen := ipv6.HeaderLen + 8 + len(packet) + assert.Len(t, rejectPacket, expectedLen) + + // Verify version + assert.Equal(t, byte(ipv6.Version<<4), rejectPacket[0]&0xf0) + // Verify next header is ICMPv6 (58) + assert.Equal(t, byte(58), rejectPacket[6]) + // Verify src/dst are swapped + assert.Equal(t, dst.To16(), net.IP(rejectPacket[8:24])) + assert.Equal(t, src.To16(), net.IP(rejectPacket[24:40])) + // Verify ICMPv6 type=1 (Dest Unreachable), code=1 (Administratively prohibited) + assert.Equal(t, byte(1), rejectPacket[ipv6.HeaderLen]) + assert.Equal(t, byte(1), rejectPacket[ipv6.HeaderLen+1]) + // Verify entire original packet is included in body + assert.Equal(t, packet, rejectPacket[ipv6.HeaderLen+8:]) + + // Large packet: body is truncated to 1000 bytes + largePkt := makeIPv6Packet(src, dst, 17, make([]byte, 1200)) + rejectPacket = CreateRejectPacket(largePkt, out) + assert.NotNil(t, rejectPacket) + assert.Len(t, rejectPacket, ipv6.HeaderLen+8+1000) + assert.Equal(t, largePkt[:1000], rejectPacket[ipv6.HeaderLen+8:]) +} + +func Test_CreateRejectPacketIPv6_TCP(t *testing.T) { + src := net.ParseIP("fd00::1") + dst := net.ParseIP("fd00::2") + + // TCP SYN packet (next header 6) + tcpPayload := make([]byte, 20) + tcpPayload[0] = 0x00 // src port high + tcpPayload[1] = 0x50 // src port low (80) + tcpPayload[2] = 0x01 // dst port high + tcpPayload[3] = 0xBB // dst port low (443) + binary.BigEndian.PutUint32(tcpPayload[4:], 1000) // seq + binary.BigEndian.PutUint32(tcpPayload[8:], 0) // ack seq + tcpPayload[12] = (20 >> 2) << 4 // data offset + tcpPayload[13] = 0b00000010 // SYN flag + + packet := makeIPv6Packet(src, dst, 6, tcpPayload) + + out := make([]byte, MaxRejectPacketSize) + rejectPacket := CreateRejectPacket(packet, out) + assert.NotNil(t, rejectPacket) + + // Expected: 40 (ipv6 hdr) + 20 (tcp RST) + expectedLen := ipv6.HeaderLen + 20 + assert.Len(t, rejectPacket, expectedLen) + + // Verify version + assert.Equal(t, byte(ipv6.Version<<4), rejectPacket[0]&0xf0) + // Verify next header is TCP (6) + assert.Equal(t, byte(6), rejectPacket[6]) + // Verify src/dst are swapped + assert.Equal(t, dst.To16(), net.IP(rejectPacket[8:24])) + assert.Equal(t, src.To16(), net.IP(rejectPacket[24:40])) + // Verify ports are swapped + tcpOut := rejectPacket[ipv6.HeaderLen:] + assert.Equal(t, uint16(443), binary.BigEndian.Uint16(tcpOut[0:2])) + assert.Equal(t, uint16(80), binary.BigEndian.Uint16(tcpOut[2:4])) + // RST+ACK flags (since input was SYN without ACK) + assert.Equal(t, byte(0b00010100), tcpOut[13]) + // ack_seq = original seq (1000) + SYN (1) + FIN (0) + segment data (0) + assert.Equal(t, uint32(1001), binary.BigEndian.Uint32(tcpOut[8:])) +} + +func Test_CreateRejectPacketIPv6_TCPWithACK(t *testing.T) { + src := net.ParseIP("fd00::1") + dst := net.ParseIP("fd00::2") + + // TCP packet with ACK set + tcpPayload := make([]byte, 20) + tcpPayload[0] = 0x00 + tcpPayload[1] = 0x50 + tcpPayload[2] = 0x01 + tcpPayload[3] = 0xBB + binary.BigEndian.PutUint32(tcpPayload[4:], 1000) // seq + binary.BigEndian.PutUint32(tcpPayload[8:], 2000) // ack seq + tcpPayload[12] = (20 >> 2) << 4 // data offset + tcpPayload[13] = 0b00010000 // ACK flag + + packet := makeIPv6Packet(src, dst, 6, tcpPayload) + + out := make([]byte, MaxRejectPacketSize) + rejectPacket := CreateRejectPacket(packet, out) + assert.NotNil(t, rejectPacket) + + tcpOut := rejectPacket[ipv6.HeaderLen:] + // RST only (no ACK) since input had ACK + assert.Equal(t, byte(0b00000100), tcpOut[13]) + // seq = original ack_seq + assert.Equal(t, uint32(2000), binary.BigEndian.Uint32(tcpOut[4:])) +} + +func Test_CreateRejectPacketIPv6_NoICMPError(t *testing.T) { + src := net.ParseIP("fd00::1") + dst := net.ParseIP("fd00::2") + out := make([]byte, MaxRejectPacketSize) + + // ICMPv6 error types (1-4) should not generate reject packets + for icmpType := byte(1); icmpType <= 4; icmpType++ { + payload := make([]byte, 8) + payload[0] = icmpType + packet := makeIPv6Packet(src, dst, 58, payload) + + rejectPacket := CreateRejectPacket(packet, out) + assert.Nil(t, rejectPacket, "ICMPv6 type %d should not generate a reject packet", icmpType) + } + + // ICMPv6 non-error types should still generate reject packets + nonErrorTypes := []byte{128, 129, 133, 134} + for _, icmpType := range nonErrorTypes { + payload := make([]byte, 8) + payload[0] = icmpType + packet := makeIPv6Packet(src, dst, 58, payload) + + rejectPacket := CreateRejectPacket(packet, out) + assert.NotNil(t, rejectPacket, "ICMPv6 type %d should generate a reject packet", icmpType) + } +} + +func Test_CreateRejectPacketIPv6_TooShort(t *testing.T) { + // Packet too short to be valid IPv6 + out := make([]byte, MaxRejectPacketSize) + assert.Nil(t, CreateRejectPacket([]byte{0x60}, out)) + assert.Nil(t, CreateRejectPacket(make([]byte, 39), out)) +} + +func Test_CreateRejectPacketIPv6_ExtensionHeaders(t *testing.T) { + src := net.ParseIP("fd00::1") + dst := net.ParseIP("fd00::2") + + // IPv6 + Hop-by-Hop extension header + TCP + hopByHop := []byte{ + 6, // next header: TCP + 0, // length (8 bytes total) + 0, 0, // padding + 0, 0, 0, 0, + } + tcpPayload := make([]byte, 20) + tcpPayload[0] = 0x00 + tcpPayload[1] = 0x50 + tcpPayload[2] = 0x01 + tcpPayload[3] = 0xBB + binary.BigEndian.PutUint32(tcpPayload[4:], 1000) + binary.BigEndian.PutUint32(tcpPayload[8:], 2000) + tcpPayload[12] = (20 >> 2) << 4 + tcpPayload[13] = 0b00010000 // ACK + + payload := append(hopByHop, tcpPayload...) + packet := makeIPv6Packet(src, dst, 0, payload) // next header 0 = Hop-by-Hop + + out := make([]byte, MaxRejectPacketSize) + rejectPacket := CreateRejectPacket(packet, out) + assert.NotNil(t, rejectPacket) + + // Should produce TCP RST + expectedLen := ipv6.HeaderLen + 20 + assert.Len(t, rejectPacket, expectedLen) + assert.Equal(t, byte(6), rejectPacket[6]) // next header is TCP + tcpOut := rejectPacket[ipv6.HeaderLen:] + assert.Equal(t, byte(0b00000100), tcpOut[13]) // RST only +} + +func TestCreateICMPEchoResponse_IPv4(t *testing.T) { + // Build a simple IPv4 ICMP Echo Request + packet := make([]byte, 28) + packet[0] = 0x45 // version 4, IHL 5 + binary.BigEndian.PutUint16(packet[2:], uint16(28)) // total length + packet[8] = 64 // TTL + packet[9] = 1 // protocol ICMP + copy(packet[12:16], net.IPv4(10, 0, 0, 1).To4()) // src + copy(packet[16:20], net.IPv4(10, 0, 0, 2).To4()) // dst + packet[20] = 8 // ICMP Echo Request + + out := make([]byte, len(packet)) + result := CreateICMPEchoResponse(packet, out) + assert.NotNil(t, result) + assert.Equal(t, byte(0x45), result[0]) + // src/dst swapped + assert.Equal(t, net.IPv4(10, 0, 0, 2).To4(), net.IP(result[12:16])) + assert.Equal(t, net.IPv4(10, 0, 0, 1).To4(), net.IP(result[16:20])) + // ICMP Echo Reply + assert.Equal(t, byte(0), result[20]) +} + +func TestCreateICMPEchoResponse_IPv6(t *testing.T) { + src := net.ParseIP("fd00::1").To16() + dst := net.ParseIP("fd00::2").To16() + + // Build an IPv6 ICMPv6 Echo Request packet + // IPv6 header (40 bytes) + ICMPv6 (8 bytes) + packet := make([]byte, 48) + packet[0] = 0x60 // version 6 + payloadLen := uint16(8) // ICMPv6 header only + binary.BigEndian.PutUint16(packet[4:], payloadLen) + packet[6] = 58 // Next Header: ICMPv6 + packet[7] = 64 // Hop Limit + copy(packet[8:24], src) // src address + copy(packet[24:40], dst) // dst address + + // ICMPv6 Echo Request + icmp := packet[40:] + icmp[0] = 128 // type: Echo Request + icmp[1] = 0 // code + binary.BigEndian.PutUint16(icmp[4:], 1) // identifier + binary.BigEndian.PutUint16(icmp[6:], 1) // sequence number + + // Compute correct checksum for the request + csum := ipv6PseudoheaderChecksum(src, dst, 58, uint32(payloadLen)) + binary.BigEndian.PutUint16(icmp[2:], tcpipChecksum(icmp, csum)) + + out := make([]byte, len(packet)) + result := CreateICMPEchoResponse(packet, out) + assert.NotNil(t, result) + + // Version should still be 6 + assert.Equal(t, byte(6), result[0]>>4) + // src/dst swapped + assert.Equal(t, dst, net.IP(result[8:24])) + assert.Equal(t, src, net.IP(result[24:40])) + // ICMPv6 Echo Reply type + assert.Equal(t, byte(129), result[40]) + + // Verify checksum is valid (tcpipChecksum returns 0 when data+checksum is correct) + respIcmp := result[40:] + verifyCsum := ipv6PseudoheaderChecksum(result[8:24], result[24:40], 58, uint32(payloadLen)) + assert.Equal(t, uint16(0), tcpipChecksum(respIcmp, verifyCsum)) +} + +func TestCreateICMPEchoResponse_IPv6_NotEchoRequest(t *testing.T) { + src := net.ParseIP("fd00::1").To16() + dst := net.ParseIP("fd00::2").To16() + + packet := make([]byte, 48) + packet[0] = 0x60 + binary.BigEndian.PutUint16(packet[4:], 8) + packet[6] = 58 + packet[7] = 64 + copy(packet[8:24], src) + copy(packet[24:40], dst) + + // ICMPv6 type 1 (Destination Unreachable) - not Echo Request + packet[40] = 1 + + out := make([]byte, len(packet)) + result := CreateICMPEchoResponse(packet, out) + assert.Nil(t, result) +} + +func TestCreateICMPEchoResponse_IPv6_NotICMPv6(t *testing.T) { + src := net.ParseIP("fd00::1").To16() + dst := net.ParseIP("fd00::2").To16() + + packet := make([]byte, 48) + packet[0] = 0x60 + binary.BigEndian.PutUint16(packet[4:], 8) + packet[6] = 6 // TCP, not ICMPv6 + packet[7] = 64 + copy(packet[8:24], src) + copy(packet[24:40], dst) + + out := make([]byte, len(packet)) + result := CreateICMPEchoResponse(packet, out) + assert.Nil(t, result) +} diff --git a/iputil/util.go b/iputil/util.go deleted file mode 100644 index 65f7677a..00000000 --- a/iputil/util.go +++ /dev/null @@ -1,93 +0,0 @@ -package iputil - -import ( - "encoding/binary" - "fmt" - "net" - "net/netip" -) - -type VpnIp uint32 - -const maxIPv4StringLen = len("255.255.255.255") - -func (ip VpnIp) String() string { - b := make([]byte, maxIPv4StringLen) - - n := ubtoa(b, 0, byte(ip>>24)) - b[n] = '.' - n++ - - n += ubtoa(b, n, byte(ip>>16&255)) - b[n] = '.' - n++ - - n += ubtoa(b, n, byte(ip>>8&255)) - b[n] = '.' - n++ - - n += ubtoa(b, n, byte(ip&255)) - return string(b[:n]) -} - -func (ip VpnIp) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf("\"%s\"", ip.String())), nil -} - -func (ip VpnIp) ToIP() net.IP { - nip := make(net.IP, 4) - binary.BigEndian.PutUint32(nip, uint32(ip)) - return nip -} - -func (ip VpnIp) ToNetIpAddr() netip.Addr { - var nip [4]byte - binary.BigEndian.PutUint32(nip[:], uint32(ip)) - return netip.AddrFrom4(nip) -} - -func Ip2VpnIp(ip []byte) VpnIp { - if len(ip) == 16 { - return VpnIp(binary.BigEndian.Uint32(ip[12:16])) - } - return VpnIp(binary.BigEndian.Uint32(ip)) -} - -func ToNetIpAddr(ip net.IP) (netip.Addr, error) { - addr, ok := netip.AddrFromSlice(ip) - if !ok { - return netip.Addr{}, fmt.Errorf("invalid net.IP: %v", ip) - } - return addr, nil -} - -func ToNetIpPrefix(ipNet net.IPNet) (netip.Prefix, error) { - addr, err := ToNetIpAddr(ipNet.IP) - if err != nil { - return netip.Prefix{}, err - } - ones, bits := ipNet.Mask.Size() - if ones == 0 && bits == 0 { - return netip.Prefix{}, fmt.Errorf("invalid net.IP: %v", ipNet) - } - return netip.PrefixFrom(addr, ones), nil -} - -// ubtoa encodes the string form of the integer v to dst[start:] and -// returns the number of bytes written to dst. The caller must ensure -// that dst has sufficient length. -func ubtoa(dst []byte, start int, v byte) int { - if v < 10 { - dst[start] = v + '0' - return 1 - } else if v < 100 { - dst[start+1] = v%10 + '0' - dst[start] = v/10 + '0' - return 2 - } - - dst[start+2] = v%10 + '0' - dst[start+1] = (v/10)%10 + '0' - dst[start] = v/100 + '0' - return 3 -} diff --git a/iputil/util_test.go b/iputil/util_test.go deleted file mode 100644 index 712d4264..00000000 --- a/iputil/util_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package iputil - -import ( - "net" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestVpnIp_String(t *testing.T) { - assert.Equal(t, "255.255.255.255", Ip2VpnIp(net.ParseIP("255.255.255.255")).String()) - assert.Equal(t, "1.255.255.255", Ip2VpnIp(net.ParseIP("1.255.255.255")).String()) - assert.Equal(t, "1.1.255.255", Ip2VpnIp(net.ParseIP("1.1.255.255")).String()) - assert.Equal(t, "1.1.1.255", Ip2VpnIp(net.ParseIP("1.1.1.255")).String()) - assert.Equal(t, "1.1.1.1", Ip2VpnIp(net.ParseIP("1.1.1.1")).String()) - assert.Equal(t, "0.0.0.0", Ip2VpnIp(net.ParseIP("0.0.0.0")).String()) -} diff --git a/lighthouse.go b/lighthouse.go index aa54c4bc..9cece233 100644 --- a/lighthouse.go +++ b/lighthouse.go @@ -5,46 +5,44 @@ import ( "encoding/binary" "errors" "fmt" + "log/slog" "net" "net/netip" + "slices" + "strconv" "sync" "sync/atomic" "time" - "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" + "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/util" ) -//TODO: if a lighthouse doesn't have an answer, clients AGGRESSIVELY REQUERY.. why? handshake manager and/or getOrHandshake? -//TODO: nodes are roaming lighthouses, this is bad. How are they learning? - var ErrHostNotKnown = errors.New("host not known") - -type netIpAndPort struct { - ip net.IP - port uint16 -} +var ErrBadDetailsVpnAddr = errors.New("invalid packet, malformed detailsVpnAddr") type LightHouse struct { - //TODO: We need a timer wheel to kick out vpnIps that haven't reported in a long time + //TODO: We need a timer wheel to kick out vpnAddrs that haven't reported in a long time sync.RWMutex //Because we concurrently read and write to our maps ctx context.Context amLighthouse bool - myVpnIp iputil.VpnIp - myVpnZeros iputil.VpnIp - myVpnNet *net.IPNet - punchConn udp.Conn - punchy *Punchy + + myVpnNetworks []netip.Prefix + myVpnNetworksTable *bart.Lite + punchy *Punchy + + // localAddrsFn enumerates the underlay addresses we advertise. It is a field so tests can supply simulated + // addresses rather than whatever this machine's NICs happen to be. Set it before Start. + localAddrsFn func(*LocalAllowList) []netip.Addr // Local cache of answers from light houses - // map of vpn Ip to answers - addrMap map[iputil.VpnIp]*RemoteList + // map of vpn addr to answers + addrMap map[netip.Addr]*RemoteList // filters remote addresses allowed for each host // - When we are a lighthouse, this filters what addresses we store and @@ -57,35 +55,35 @@ type LightHouse struct { localAllowList atomic.Pointer[LocalAllowList] // used to trigger the HandshakeManager when we receive HostQueryReply - handshakeTrigger chan<- iputil.VpnIp + handshakeTrigger chan<- netip.Addr // staticList exists to avoid having a bool in each addrMap entry // since static should be rare - staticList atomic.Pointer[map[iputil.VpnIp]struct{}] - lighthouses atomic.Pointer[map[iputil.VpnIp]struct{}] + staticList atomic.Pointer[map[netip.Addr]struct{}] + lighthouses atomic.Pointer[[]netip.Addr] interval atomic.Int64 updateCancel context.CancelFunc ifce EncWriter nebulaPort uint32 // 32 bits because protobuf does not have a uint16 - advertiseAddrs atomic.Pointer[[]netIpAndPort] + advertiseAddrs atomic.Pointer[[]netip.AddrPort] - // IP's of relays that can be used by peers to access me - relaysForMe atomic.Pointer[[]iputil.VpnIp] + // Addr's of relays that can be used by peers to access me + relaysForMe atomic.Pointer[[]netip.Addr] - queryChan chan iputil.VpnIp + updateTrigger chan struct{} + queryChan chan netip.Addr - calculatedRemotes atomic.Pointer[cidr.Tree4[[]*calculatedRemote]] // Maps VpnIp to []*calculatedRemote + calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote - metrics *MessageMetrics - metricHolepunchTx metrics.Counter - l *logrus.Logger + metrics *MessageMetrics + l *slog.Logger } // NewLightHouseFromConfig will build a Lighthouse struct from the values provided in the config object // addrMap should be nil unless this is during a config reload -func NewLightHouseFromConfig(ctx context.Context, l *logrus.Logger, c *config.C, myVpnNet *net.IPNet, pc udp.Conn, p *Punchy) (*LightHouse, error) { +func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, cs *CertState, pc udp.Conn, p *Punchy) (*LightHouse, error) { amLighthouse := c.GetBool("lighthouse.am_lighthouse", false) nebulaPort := uint32(c.GetInt("listen.port", 0)) if amLighthouse && nebulaPort == 0 { @@ -98,33 +96,32 @@ func NewLightHouseFromConfig(ctx context.Context, l *logrus.Logger, c *config.C, if err != nil { return nil, util.NewContextualError("Failed to get listening port", nil, err) } - nebulaPort = uint32(uPort.Port) + nebulaPort = uint32(uPort.Port()) } - ones, _ := myVpnNet.Mask.Size() h := LightHouse{ - ctx: ctx, - amLighthouse: amLighthouse, - myVpnIp: iputil.Ip2VpnIp(myVpnNet.IP), - myVpnZeros: iputil.VpnIp(32 - ones), - myVpnNet: myVpnNet, - addrMap: make(map[iputil.VpnIp]*RemoteList), - nebulaPort: nebulaPort, - punchConn: pc, - punchy: p, - queryChan: make(chan iputil.VpnIp, c.GetUint32("handshakes.query_buffer", 64)), - l: l, - } - lighthouses := make(map[iputil.VpnIp]struct{}) + ctx: ctx, + amLighthouse: amLighthouse, + myVpnNetworks: cs.myVpnNetworks, + myVpnNetworksTable: cs.myVpnNetworksTable, + addrMap: make(map[netip.Addr]*RemoteList), + nebulaPort: nebulaPort, + punchy: p, + updateTrigger: make(chan struct{}, 1), + queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)), + l: l, + } + h.localAddrsFn = func(al *LocalAllowList) []netip.Addr { + return localAddrs(h.l, al) + } + + lighthouses := make([]netip.Addr, 0) h.lighthouses.Store(&lighthouses) - staticList := make(map[iputil.VpnIp]struct{}) + staticList := make(map[netip.Addr]struct{}) h.staticList.Store(&staticList) if c.GetBool("stats.lighthouse_metrics", false) { h.metrics = newLighthouseMetrics() - h.metricHolepunchTx = metrics.GetOrRegisterCounter("messages.tx.holepunch", nil) - } else { - h.metricHolepunchTx = metrics.NilCounter{} } err := h.reload(c, true) @@ -138,7 +135,7 @@ func NewLightHouseFromConfig(ctx context.Context, l *logrus.Logger, c *config.C, case *util.ContextualError: v.Log(l) case error: - l.WithError(err).Error("failed to reload lighthouse") + l.Error("failed to reload lighthouse", "error", err) } }) @@ -147,11 +144,11 @@ func NewLightHouseFromConfig(ctx context.Context, l *logrus.Logger, c *config.C, return &h, nil } -func (lh *LightHouse) GetStaticHostList() map[iputil.VpnIp]struct{} { +func (lh *LightHouse) GetStaticHostList() map[netip.Addr]struct{} { return *lh.staticList.Load() } -func (lh *LightHouse) GetLighthouses() map[iputil.VpnIp]struct{} { +func (lh *LightHouse) GetLighthouses() []netip.Addr { return *lh.lighthouses.Load() } @@ -163,15 +160,15 @@ func (lh *LightHouse) GetLocalAllowList() *LocalAllowList { return lh.localAllowList.Load() } -func (lh *LightHouse) GetAdvertiseAddrs() []netIpAndPort { +func (lh *LightHouse) GetAdvertiseAddrs() []netip.AddrPort { return *lh.advertiseAddrs.Load() } -func (lh *LightHouse) GetRelaysForMe() []iputil.VpnIp { +func (lh *LightHouse) GetRelaysForMe() []netip.Addr { return *lh.relaysForMe.Load() } -func (lh *LightHouse) getCalculatedRemotes() *cidr.Tree4[[]*calculatedRemote] { +func (lh *LightHouse) getCalculatedRemotes() *bart.Table[[]*calculatedRemote] { return lh.calculatedRemotes.Load() } @@ -182,25 +179,42 @@ func (lh *LightHouse) GetUpdateInterval() int64 { func (lh *LightHouse) reload(c *config.C, initial bool) error { if initial || c.HasChanged("lighthouse.advertise_addrs") { rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{}) - advAddrs := make([]netIpAndPort, 0) + advAddrs := make([]netip.AddrPort, 0) for i, rawAddr := range rawAdvAddrs { - fIp, fPort, err := udp.ParseIPAndPort(rawAddr) + host, sport, err := net.SplitHostPort(rawAddr) if err != nil { return util.NewContextualError("Unable to parse lighthouse.advertise_addrs entry", m{"addr": rawAddr, "entry": i + 1}, err) } - if fPort == 0 { - fPort = uint16(lh.nebulaPort) + addrs, err := net.DefaultResolver.LookupNetIP(context.Background(), "ip", host) + if err != nil { + return util.NewContextualError("Unable to lookup lighthouse.advertise_addrs entry", m{"addr": rawAddr, "entry": i + 1}, err) + } + if len(addrs) == 0 { + return util.NewContextualError("Unable to lookup lighthouse.advertise_addrs entry", m{"addr": rawAddr, "entry": i + 1}, nil) } - if ip4 := fIp.To4(); ip4 != nil && lh.myVpnNet.Contains(fIp) { - lh.l.WithField("addr", rawAddr).WithField("entry", i+1). - Warn("Ignoring lighthouse.advertise_addrs report because it is within the nebula network range") + port, err := strconv.Atoi(sport) + if err != nil { + return util.NewContextualError("Unable to parse port in lighthouse.advertise_addrs entry", m{"addr": rawAddr, "entry": i + 1}, err) + } + + if port == 0 { + port = int(lh.nebulaPort) + } + + //TODO: we could technically insert all returned addrs instead of just the first one if a dns lookup was used + addr := addrs[0].Unmap() + if lh.myVpnNetworksTable.Contains(addr) { + lh.l.Warn("Ignoring lighthouse.advertise_addrs report because it is within the nebula network range", + "addr", rawAddr, + "entry", i+1, + ) continue } - advAddrs = append(advAddrs, netIpAndPort{ip: fIp, port: fPort}) + advAddrs = append(advAddrs, netip.AddrPortFrom(addr, uint16(port))) } lh.advertiseAddrs.Store(&advAddrs) @@ -214,7 +228,9 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { lh.interval.Store(int64(c.GetInt("lighthouse.interval", 10))) if !initial { - lh.l.Infof("lighthouse.interval changed to %v", lh.interval.Load()) + lh.l.Info("lighthouse.interval changed", + "interval", lh.interval.Load(), + ) if lh.updateCancel != nil { // May not always have a running routine @@ -233,7 +249,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { lh.remoteAllowList.Store(ral) if !initial { - //TODO: a diff will be annoyingly difficult lh.l.Info("lighthouse.remote_allow_list and/or lighthouse.remote_allow_ranges has changed") } } @@ -246,7 +261,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { lh.localAllowList.Store(lal) if !initial { - //TODO: a diff will be annoyingly difficult lh.l.Info("lighthouse.local_allow_list has changed") } } @@ -259,7 +273,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { lh.calculatedRemotes.Store(cr) if !initial { - //TODO: a diff will be annoyingly difficult lh.l.Info("lighthouse.calculated_remotes has changed") } } @@ -267,26 +280,42 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { //NOTE: many things will get much simpler when we combine static_host_map and lighthouse.hosts in config if initial || c.HasChanged("static_host_map") || c.HasChanged("static_map.cadence") || c.HasChanged("static_map.network") || c.HasChanged("static_map.lookup_timeout") { // Clean up. Entries still in the static_host_map will be re-built. - // Entries no longer present must have their (possible) background DNS goroutines stopped. - if existingStaticList := lh.staticList.Load(); existingStaticList != nil { + ourselves := lh.myVpnNetworks[0].Addr() + oldStaticList := lh.staticList.Load() + if oldStaticList != nil { lh.RLock() - for staticVpnIp := range *existingStaticList { - if am, ok := lh.addrMap[staticVpnIp]; ok && am != nil { - am.hr.Cancel() + for staticVpnAddr := range *oldStaticList { + if am, ok := lh.addrMap[staticVpnAddr]; ok && am != nil { + am.ResetForOwner(ourselves) } } lh.RUnlock() } + // Build a new list based on current config. - staticList := make(map[iputil.VpnIp]struct{}) - err := lh.loadStaticMap(c, lh.myVpnNet, staticList) + staticList := make(map[netip.Addr]struct{}) + err := lh.loadStaticMap(c, staticList) if err != nil { return err } + // For entries removed from static_host_map, stop the DNS goroutine and drop the cached addrs. + // All addrs must come from the lighthouses now that it's no longer a static host. + if oldStaticList != nil { + lh.RLock() + for staticVpnAddr := range *oldStaticList { + if _, stillStatic := staticList[staticVpnAddr]; stillStatic { + continue + } + if am, ok := lh.addrMap[staticVpnAddr]; ok && am != nil { + am.ClearHostnameResults() + } + } + lh.RUnlock() + } + lh.staticList.Store(&staticList) if !initial { - //TODO: we should remove any remote list entries for static hosts that were removed/modified? if c.HasChanged("static_host_map") { lh.l.Info("static_host_map has changed") } @@ -303,16 +332,16 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { } if initial || c.HasChanged("lighthouse.hosts") { - lhMap := make(map[iputil.VpnIp]struct{}) - err := lh.parseLighthouses(c, lh.myVpnNet, lhMap) + lhList, err := lh.parseLighthouses(c) if err != nil { return err } - lh.lighthouses.Store(&lhMap) + lh.lighthouses.Store(&lhList) if !initial { //NOTE: we are not tearing down existing lighthouse connections because they might be used for non lighthouse traffic lh.l.Info("lighthouse.hosts has changed") + lh.TriggerUpdate() } } @@ -323,16 +352,20 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { if len(c.GetStringSlice("relay.relays", nil)) > 0 { lh.l.Info("Ignoring relays from config because am_relay is true") } - relaysForMe := []iputil.VpnIp{} + relaysForMe := []netip.Addr{} lh.relaysForMe.Store(&relaysForMe) case false: - relaysForMe := []iputil.VpnIp{} + relaysForMe := []netip.Addr{} for _, v := range c.GetStringSlice("relay.relays", nil) { - lh.l.WithField("relay", v).Info("Read relay from config") - - configRIP := net.ParseIP(v) - if configRIP != nil { - relaysForMe = append(relaysForMe, iputil.Ip2VpnIp(configRIP)) + configRIP, err := netip.ParseAddr(v) + if err != nil { + lh.l.Warn("Parse relay from config failed", + "relay", v, + "error", err, + ) + } else { + lh.l.Info("Read relay from config", "relay", v) + relaysForMe = append(relaysForMe, configRIP) } } lh.relaysForMe.Store(&relaysForMe) @@ -342,35 +375,40 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { return nil } -func (lh *LightHouse) parseLighthouses(c *config.C, tunCidr *net.IPNet, lhMap map[iputil.VpnIp]struct{}) error { +func (lh *LightHouse) parseLighthouses(c *config.C) ([]netip.Addr, error) { lhs := c.GetStringSlice("lighthouse.hosts", []string{}) if lh.amLighthouse && len(lhs) != 0 { lh.l.Warn("lighthouse.am_lighthouse enabled on node but upstream lighthouses exist in config") } + out := make([]netip.Addr, len(lhs)) for i, host := range lhs { - ip := net.ParseIP(host) - if ip == nil { - return util.NewContextualError("Unable to parse lighthouse host entry", m{"host": host, "entry": i + 1}, nil) + addr, err := netip.ParseAddr(host) + if err != nil { + return nil, util.NewContextualError("Unable to parse lighthouse host entry", m{"host": host, "entry": i + 1}, err) } - if !tunCidr.Contains(ip) { - return util.NewContextualError("lighthouse host is not in our subnet, invalid", m{"vpnIp": ip, "network": tunCidr.String()}, nil) + + if !lh.myVpnNetworksTable.Contains(addr) { + lh.l.Warn("lighthouse host is not within our networks, lighthouse functionality will work but layer 3 network traffic to the lighthouse will not", + "vpnAddr", addr, + "networks", lh.myVpnNetworks, + ) } - lhMap[iputil.Ip2VpnIp(ip)] = struct{}{} + out[i] = addr } - if !lh.amLighthouse && len(lhMap) == 0 { + if !lh.amLighthouse && len(out) == 0 { lh.l.Warn("No lighthouse.hosts configured, this host will only be able to initiate tunnels with static_host_map entries") } staticList := lh.GetStaticHostList() - for lhIP, _ := range lhMap { - if _, ok := staticList[lhIP]; !ok { - return fmt.Errorf("lighthouse %s does not have a static_host_map entry", lhIP) + for i := range out { + if _, ok := staticList[out[i]]; !ok { + return nil, fmt.Errorf("lighthouse %s does not have a static_host_map entry", out[i]) } } - return nil + return out, nil } func getStaticMapCadence(c *config.C) (time.Duration, error) { @@ -399,7 +437,7 @@ func getStaticMapNetwork(c *config.C) (string, error) { return network, nil } -func (lh *LightHouse) loadStaticMap(c *config.C, tunCidr *net.IPNet, staticList map[iputil.VpnIp]struct{}) error { +func (lh *LightHouse) loadStaticMap(c *config.C, staticList map[netip.Addr]struct{}) error { d, err := getStaticMapCadence(c) if err != nil { return err @@ -410,35 +448,38 @@ func (lh *LightHouse) loadStaticMap(c *config.C, tunCidr *net.IPNet, staticList return err } - lookup_timeout, err := getStaticMapLookupTimeout(c) + lookupTimeout, err := getStaticMapLookupTimeout(c) if err != nil { return err } - shm := c.GetMap("static_host_map", map[interface{}]interface{}{}) + shm := c.GetMap("static_host_map", map[string]any{}) i := 0 for k, v := range shm { - rip := net.ParseIP(fmt.Sprintf("%v", k)) - if rip == nil { - return util.NewContextualError("Unable to parse static_host_map entry", m{"host": k, "entry": i + 1}, nil) + vpnAddr, err := netip.ParseAddr(fmt.Sprintf("%v", k)) + if err != nil { + return util.NewContextualError("Unable to parse static_host_map entry", m{"host": k, "entry": i + 1}, err) } - if !tunCidr.Contains(rip) { - return util.NewContextualError("static_host_map key is not in our subnet, invalid", m{"vpnIp": rip, "network": tunCidr.String(), "entry": i + 1}, nil) + if !lh.myVpnNetworksTable.Contains(vpnAddr) { + lh.l.Warn("static_host_map key is not within our networks, layer 3 network traffic to this host will not work", + "vpnAddr", vpnAddr, + "networks", lh.myVpnNetworks, + "entry", i+1, + ) } - vpnIp := iputil.Ip2VpnIp(rip) - vals, ok := v.([]interface{}) + vals, ok := v.([]any) if !ok { - vals = []interface{}{v} + vals = []any{v} } remoteAddrs := []string{} for _, v := range vals { remoteAddrs = append(remoteAddrs, fmt.Sprintf("%v", v)) } - err := lh.addStaticRemotes(i, d, network, lookup_timeout, vpnIp, remoteAddrs, staticList) + err = lh.addStaticRemotes(i, d, network, lookupTimeout, vpnAddr, remoteAddrs, staticList) if err != nil { return err } @@ -448,12 +489,12 @@ func (lh *LightHouse) loadStaticMap(c *config.C, tunCidr *net.IPNet, staticList return nil } -func (lh *LightHouse) Query(ip iputil.VpnIp) *RemoteList { - if !lh.IsLighthouseIP(ip) { - lh.QueryServer(ip) +func (lh *LightHouse) Query(vpnAddr netip.Addr) *RemoteList { + if !lh.IsLighthouseAddr(vpnAddr) { + lh.QueryServer(vpnAddr) } lh.RLock() - if v, ok := lh.addrMap[ip]; ok { + if v, ok := lh.addrMap[vpnAddr]; ok { lh.RUnlock() return v } @@ -462,18 +503,18 @@ func (lh *LightHouse) Query(ip iputil.VpnIp) *RemoteList { } // QueryServer is asynchronous so no reply should be expected -func (lh *LightHouse) QueryServer(ip iputil.VpnIp) { - // Don't put lighthouse ips in the query channel because we can't query lighthouses about lighthouses - if lh.amLighthouse || lh.IsLighthouseIP(ip) { +func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) { + // Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses + if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) { return } - lh.queryChan <- ip + lh.queryChan <- vpnAddr } -func (lh *LightHouse) QueryCache(ip iputil.VpnIp) *RemoteList { +func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList { lh.RLock() - if v, ok := lh.addrMap[ip]; ok { + if v, ok := lh.addrMap[vpnAddrs[0]]; ok { lh.RUnlock() return v } @@ -482,24 +523,27 @@ func (lh *LightHouse) QueryCache(ip iputil.VpnIp) *RemoteList { lh.Lock() defer lh.Unlock() // Add an entry if we don't already have one - return lh.unlockedGetRemoteList(ip) + return lh.unlockedGetRemoteList(vpnAddrs) //todo CERT-V2 this contains addrmap lookups we could potentially skip } // queryAndPrepMessage is a lock helper on RemoteList, assisting the caller to build a lighthouse message containing -// details from the remote list. It looks for a hit in the addrMap and a hit in the RemoteList under the owner vpnIp +// details from the remote list. It looks for a hit in the addrMap and a hit in the RemoteList under the owner vpnAddr // If one is found then f() is called with proper locking, f() must return result of n.MarshalTo() -func (lh *LightHouse) queryAndPrepMessage(vpnIp iputil.VpnIp, f func(*cache) (int, error)) (bool, int, error) { +func (lh *LightHouse) queryAndPrepMessage(vpnAddr netip.Addr, f func(*cache) (int, error)) (bool, int, error) { lh.RLock() // Do we have an entry in the main cache? - if v, ok := lh.addrMap[vpnIp]; ok { + if v, ok := lh.addrMap[vpnAddr]; ok { // Swap lh lock for remote list lock v.RLock() defer v.RUnlock() lh.RUnlock() - // vpnIp should also be the owner here since we are a lighthouse. - c := v.cache[vpnIp] + // We may be asking about a non primary address so lets get the primary address + if slices.Contains(v.vpnAddrs, vpnAddr) { + vpnAddr = v.vpnAddrs[0] + } + c := v.cache[vpnAddr] // Make sure we have if c != nil { n, err := f(c) @@ -511,234 +555,219 @@ func (lh *LightHouse) queryAndPrepMessage(vpnIp iputil.VpnIp, f func(*cache) (in return false, 0, nil } -func (lh *LightHouse) DeleteVpnIp(vpnIp iputil.VpnIp) { - // First we check the static mapping - // and do nothing if it is there - if _, ok := lh.GetStaticHostList()[vpnIp]; ok { - return +func (lh *LightHouse) DeleteVpnAddrs(allVpnAddrs []netip.Addr) { + // First we check the static host map. If any of the VpnAddrs to be deleted are present, do nothing. + staticList := lh.GetStaticHostList() + for _, addr := range allVpnAddrs { + if _, ok := staticList[addr]; ok { + return + } } - lh.Lock() - //l.Debugln(lh.addrMap) - delete(lh.addrMap, vpnIp) - if lh.l.Level >= logrus.DebugLevel { - lh.l.Debugf("deleting %s from lighthouse.", vpnIp) + // None of the VpnAddrs were present. Now we can do the deletes. + lh.Lock() + rm, ok := lh.addrMap[allVpnAddrs[0]] + if ok { + debugEnabled := lh.l.Enabled(context.Background(), slog.LevelDebug) + for _, addr := range allVpnAddrs { + srm := lh.addrMap[addr] + if srm == rm { + delete(lh.addrMap, addr) + if debugEnabled { + lh.l.Debug("deleting from lighthouse", "vpnAddr", addr) + } + } + } } - lh.Unlock() } -// AddStaticRemote adds a static host entry for vpnIp as ourselves as the owner +// AddStaticRemote adds a static host entry for vpnAddr as ourselves as the owner // We are the owner because we don't want a lighthouse server to advertise for static hosts it was configured with // And we don't want a lighthouse query reply to interfere with our learned cache if we are a client // NOTE: this function should not interact with any hot path objects, like lh.staticList, the caller should handle it -func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, timeout time.Duration, vpnIp iputil.VpnIp, toAddrs []string, staticList map[iputil.VpnIp]struct{}) error { +func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, timeout time.Duration, vpnAddr netip.Addr, toAddrs []string, staticList map[netip.Addr]struct{}) error { lh.Lock() - am := lh.unlockedGetRemoteList(vpnIp) + am := lh.unlockedGetRemoteList([]netip.Addr{vpnAddr}) am.Lock() defer am.Unlock() ctx := lh.ctx lh.Unlock() hr, err := NewHostnameResults(ctx, lh.l, d, network, timeout, toAddrs, func() { - // This callback runs whenever the DNS hostname resolver finds a different set of IP's + // This callback runs whenever the DNS hostname resolver finds a different set of addr's // in its resolution for hostnames. am.Lock() defer am.Unlock() am.shouldRebuild = true }) if err != nil { - return util.NewContextualError("Static host address could not be parsed", m{"vpnIp": vpnIp, "entry": i + 1}, err) + return util.NewContextualError("Static host address could not be parsed", m{"vpnAddr": vpnAddr, "entry": i + 1}, err) } am.unlockedSetHostnamesResults(hr) - for _, addrPort := range hr.GetIPs() { - + for _, addrPort := range hr.GetAddrs() { + if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) { + continue + } switch { case addrPort.Addr().Is4(): - to := NewIp4AndPortFromNetIP(addrPort.Addr(), addrPort.Port()) - if !lh.unlockedShouldAddV4(vpnIp, to) { - continue - } - am.unlockedPrependV4(lh.myVpnIp, to) + am.unlockedPrependV4(lh.myVpnNetworks[0].Addr(), netAddrToProtoV4AddrPort(addrPort.Addr(), addrPort.Port())) case addrPort.Addr().Is6(): - to := NewIp6AndPortFromNetIP(addrPort.Addr(), addrPort.Port()) - if !lh.unlockedShouldAddV6(vpnIp, to) { - continue - } - am.unlockedPrependV6(lh.myVpnIp, to) + am.unlockedPrependV6(lh.myVpnNetworks[0].Addr(), netAddrToProtoV6AddrPort(addrPort.Addr(), addrPort.Port())) } } // Mark it as static in the caller provided map - staticList[vpnIp] = struct{}{} + staticList[vpnAddr] = struct{}{} return nil } // addCalculatedRemotes adds any calculated remotes based on the // lighthouse.calculated_remotes configuration. It returns true if any // calculated remotes were added -func (lh *LightHouse) addCalculatedRemotes(vpnIp iputil.VpnIp) bool { +func (lh *LightHouse) addCalculatedRemotes(vpnAddr netip.Addr) bool { tree := lh.getCalculatedRemotes() if tree == nil { return false } - ok, calculatedRemotes := tree.MostSpecificContains(vpnIp) + calculatedRemotes, ok := tree.Lookup(vpnAddr) if !ok { return false } - var calculated []*Ip4AndPort + var calculatedV4 []*V4AddrPort + var calculatedV6 []*V6AddrPort for _, cr := range calculatedRemotes { - c := cr.Apply(vpnIp) - if c != nil { - calculated = append(calculated, c) + if vpnAddr.Is4() { + c := cr.ApplyV4(vpnAddr) + if c != nil { + calculatedV4 = append(calculatedV4, c) + } + } else if vpnAddr.Is6() { + c := cr.ApplyV6(vpnAddr) + if c != nil { + calculatedV6 = append(calculatedV6, c) + } } } lh.Lock() - am := lh.unlockedGetRemoteList(vpnIp) + am := lh.unlockedGetRemoteList([]netip.Addr{vpnAddr}) am.Lock() defer am.Unlock() lh.Unlock() - am.unlockedSetV4(lh.myVpnIp, vpnIp, calculated, lh.unlockedShouldAddV4) - - return len(calculated) > 0 -} + if len(calculatedV4) > 0 { + am.unlockedSetV4(lh.myVpnNetworks[0].Addr(), vpnAddr, calculatedV4, lh.unlockedShouldAddV4) + } -// unlockedGetRemoteList assumes you have the lh lock -func (lh *LightHouse) unlockedGetRemoteList(vpnIp iputil.VpnIp) *RemoteList { - am, ok := lh.addrMap[vpnIp] - if !ok { - am = NewRemoteList(func(a netip.Addr) bool { return lh.shouldAdd(vpnIp, a) }) - lh.addrMap[vpnIp] = am + if len(calculatedV6) > 0 { + am.unlockedSetV6(lh.myVpnNetworks[0].Addr(), vpnAddr, calculatedV6, lh.unlockedShouldAddV6) } - return am -} -func (lh *LightHouse) shouldAdd(vpnIp iputil.VpnIp, to netip.Addr) bool { - switch { - case to.Is4(): - ipBytes := to.As4() - ip := iputil.Ip2VpnIp(ipBytes[:]) - allow := lh.GetRemoteAllowList().AllowIpV4(vpnIp, ip) - if lh.l.Level >= logrus.TraceLevel { - lh.l.WithField("remoteIp", vpnIp).WithField("allow", allow).Trace("remoteAllowList.Allow") - } - if !allow || ipMaskContains(lh.myVpnIp, lh.myVpnZeros, ip) { - return false - } - case to.Is6(): - ipBytes := to.As16() + return len(calculatedV4) > 0 || len(calculatedV6) > 0 +} - hi := binary.BigEndian.Uint64(ipBytes[:8]) - lo := binary.BigEndian.Uint64(ipBytes[8:]) - allow := lh.GetRemoteAllowList().AllowIpV6(vpnIp, hi, lo) - if lh.l.Level >= logrus.TraceLevel { - lh.l.WithField("remoteIp", to).WithField("allow", allow).Trace("remoteAllowList.Allow") +// unlockedGetRemoteList assumes you have the lh lock +func (lh *LightHouse) unlockedGetRemoteList(allAddrs []netip.Addr) *RemoteList { + // before we go and make a new remotelist, we need to make sure we don't have one for any of this set of vpnaddrs yet + for i, addr := range allAddrs { + am, ok := lh.addrMap[addr] + if ok { + if i != 0 { + lh.addrMap[allAddrs[0]] = am + } + return am } + } - // We don't check our vpn network here because nebula does not support ipv6 on the inside - if !allow { - return false - } + am := NewRemoteList(allAddrs, lh.shouldAdd) + for _, addr := range allAddrs { + lh.addrMap[addr] = am } - return true + return am } -// unlockedShouldAddV4 checks if to is allowed by our allow list -func (lh *LightHouse) unlockedShouldAddV4(vpnIp iputil.VpnIp, to *Ip4AndPort) bool { - allow := lh.GetRemoteAllowList().AllowIpV4(vpnIp, iputil.VpnIp(to.Ip)) - if lh.l.Level >= logrus.TraceLevel { - lh.l.WithField("remoteIp", vpnIp).WithField("allow", allow).Trace("remoteAllowList.Allow") +func (lh *LightHouse) shouldAdd(vpnAddrs []netip.Addr, to netip.Addr) bool { + allow := lh.GetRemoteAllowList().AllowAll(vpnAddrs, to) + if lh.l.Enabled(context.Background(), logging.LevelTrace) { + lh.l.Log(context.Background(), logging.LevelTrace, "remoteAllowList.Allow", + "vpnAddrs", vpnAddrs, + "udpAddr", to, + "allow", allow, + ) + } + if !allow { + return false } - if !allow || ipMaskContains(lh.myVpnIp, lh.myVpnZeros, iputil.VpnIp(to.Ip)) { + if lh.myVpnNetworksTable.Contains(to) { return false } return true } -// unlockedShouldAddV6 checks if to is allowed by our allow list -func (lh *LightHouse) unlockedShouldAddV6(vpnIp iputil.VpnIp, to *Ip6AndPort) bool { - allow := lh.GetRemoteAllowList().AllowIpV6(vpnIp, to.Hi, to.Lo) - if lh.l.Level >= logrus.TraceLevel { - lh.l.WithField("remoteIp", lhIp6ToIp(to)).WithField("allow", allow).Trace("remoteAllowList.Allow") +// unlockedShouldAddV4 checks if to is allowed by our allow list +func (lh *LightHouse) unlockedShouldAddV4(vpnAddr netip.Addr, to *V4AddrPort) bool { + udpAddr := protoV4AddrPortToNetAddrPort(to) + allow := lh.GetRemoteAllowList().Allow(vpnAddr, udpAddr.Addr()) + if lh.l.Enabled(context.Background(), logging.LevelTrace) { + lh.l.Log(context.Background(), logging.LevelTrace, "remoteAllowList.Allow", + "vpnAddr", vpnAddr, + "udpAddr", udpAddr, + "allow", allow, + ) } - // We don't check our vpn network here because nebula does not support ipv6 on the inside if !allow { return false } - return true -} + if lh.myVpnNetworksTable.Contains(udpAddr.Addr()) { + return false + } -func lhIp6ToIp(v *Ip6AndPort) net.IP { - ip := make(net.IP, 16) - binary.BigEndian.PutUint64(ip[:8], v.Hi) - binary.BigEndian.PutUint64(ip[8:], v.Lo) - return ip + return true } -func (lh *LightHouse) IsLighthouseIP(vpnIp iputil.VpnIp) bool { - if _, ok := lh.GetLighthouses()[vpnIp]; ok { - return true +// unlockedShouldAddV6 checks if to is allowed by our allow list +func (lh *LightHouse) unlockedShouldAddV6(vpnAddr netip.Addr, to *V6AddrPort) bool { + udpAddr := protoV6AddrPortToNetAddrPort(to) + allow := lh.GetRemoteAllowList().Allow(vpnAddr, udpAddr.Addr()) + if lh.l.Enabled(context.Background(), logging.LevelTrace) { + lh.l.Log(context.Background(), logging.LevelTrace, "remoteAllowList.Allow", + "vpnAddr", vpnAddr, + "udpAddr", udpAddr, + "allow", allow, + ) } - return false -} -func NewLhQueryByInt(VpnIp iputil.VpnIp) *NebulaMeta { - return &NebulaMeta{ - Type: NebulaMeta_HostQuery, - Details: &NebulaMetaDetails{ - VpnIp: uint32(VpnIp), - }, + if !allow { + return false } -} -func NewIp4AndPort(ip net.IP, port uint32) *Ip4AndPort { - ipp := Ip4AndPort{Port: port} - ipp.Ip = uint32(iputil.Ip2VpnIp(ip)) - return &ipp -} - -func NewIp4AndPortFromNetIP(ip netip.Addr, port uint16) *Ip4AndPort { - v4Addr := ip.As4() - return &Ip4AndPort{ - Ip: binary.BigEndian.Uint32(v4Addr[:]), - Port: uint32(port), + if lh.myVpnNetworksTable.Contains(udpAddr.Addr()) { + return false } -} -func NewIp6AndPort(ip net.IP, port uint32) *Ip6AndPort { - return &Ip6AndPort{ - Hi: binary.BigEndian.Uint64(ip[:8]), - Lo: binary.BigEndian.Uint64(ip[8:]), - Port: port, - } + return true } -func NewIp6AndPortFromNetIP(ip netip.Addr, port uint16) *Ip6AndPort { - ip6Addr := ip.As16() - return &Ip6AndPort{ - Hi: binary.BigEndian.Uint64(ip6Addr[:8]), - Lo: binary.BigEndian.Uint64(ip6Addr[8:]), - Port: uint32(port), - } -} -func NewUDPAddrFromLH4(ipp *Ip4AndPort) *udp.Addr { - ip := ipp.Ip - return udp.NewAddr( - net.IPv4(byte(ip&0xff000000>>24), byte(ip&0x00ff0000>>16), byte(ip&0x0000ff00>>8), byte(ip&0x000000ff)), - uint16(ipp.Port), - ) +func (lh *LightHouse) IsLighthouseAddr(vpnAddr netip.Addr) bool { + l := lh.GetLighthouses() + return slices.Contains(l, vpnAddr) } -func NewUDPAddrFromLH6(ipp *Ip6AndPort) *udp.Addr { - return udp.NewAddr(lhIp6ToIp(ipp), uint16(ipp.Port)) +func (lh *LightHouse) IsAnyLighthouseAddr(vpnAddrs []netip.Addr) bool { + l := lh.GetLighthouses() + for i := range vpnAddrs { + if slices.Contains(l, vpnAddrs[i]) { + return true + } + } + return false } func (lh *LightHouse) startQueryWorker() { @@ -754,31 +783,95 @@ func (lh *LightHouse) startQueryWorker() { select { case <-lh.ctx.Done(): return - case ip := <-lh.queryChan: - lh.innerQueryServer(ip, nb, out) + case addr := <-lh.queryChan: + lh.innerQueryServer(addr, nb, out) } } }() } -func (lh *LightHouse) innerQueryServer(ip iputil.VpnIp, nb, out []byte) { - if lh.IsLighthouseIP(ip) { +func (lh *LightHouse) innerQueryServer(addr netip.Addr, nb, out []byte) { + if lh.IsLighthouseAddr(addr) { return } - // Send a query to the lighthouses and hope for the best next time - query, err := NewLhQueryByInt(ip).Marshal() - if err != nil { - lh.l.WithError(err).WithField("vpnIp", ip).Error("Failed to marshal lighthouse query payload") - return + msg := &NebulaMeta{ + Type: NebulaMeta_HostQuery, + Details: &NebulaMetaDetails{}, } + var v1Query, v2Query []byte + var err error + var v cert.Version + queried := 0 lighthouses := lh.GetLighthouses() - lh.metricTx(NebulaMeta_HostQuery, int64(len(lighthouses))) - for n := range lighthouses { - lh.ifce.SendMessageToVpnIp(header.LightHouse, 0, n, query, nb, out) + for _, lhVpnAddr := range lighthouses { + hi := lh.ifce.GetHostInfo(lhVpnAddr) + if hi != nil { + v = hi.ConnectionState.myCert.Version() + } else { + v = lh.ifce.GetCertState().initiatingVersion + } + + if v == cert.Version1 { + if !addr.Is4() { + lh.l.Error("Can't query lighthouse for v6 address using a v1 protocol", + "queryVpnAddr", addr, + "lighthouseAddr", lhVpnAddr, + ) + continue + } + + if v1Query == nil { + b := addr.As4() + msg.Details.VpnAddr = nil + msg.Details.OldVpnAddr = binary.BigEndian.Uint32(b[:]) + + v1Query, err = msg.Marshal() + if err != nil { + lh.l.Error("Failed to marshal lighthouse v1 query payload", + "error", err, + "queryVpnAddr", addr, + "lighthouseAddr", lhVpnAddr, + ) + continue + } + } + + lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Query, nb, out) + queried++ + + } else if v == cert.Version2 { + if v2Query == nil { + msg.Details.OldVpnAddr = 0 + msg.Details.VpnAddr = netAddrToProtoAddr(addr) + + v2Query, err = msg.Marshal() + if err != nil { + lh.l.Error("Failed to marshal lighthouse v2 query payload", + "error", err, + "queryVpnAddr", addr, + "lighthouseAddr", lhVpnAddr, + ) + continue + } + } + + lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Query, nb, out) + queried++ + + } else { + lh.l.Debug("unsupported protocol version", + "op", "query", + "queryVpnAddr", addr, + "version", v, + ) + continue + } } + + lh.metricTx(NebulaMeta_HostQuery, int64(queried)) } func (lh *LightHouse) StartUpdateWorker() { @@ -802,66 +895,145 @@ func (lh *LightHouse) StartUpdateWorker() { return case <-clockSource.C: continue + case <-lh.updateTrigger: + continue } } }() } +// TriggerUpdate requests an immediate lighthouse update. This is a non-blocking +// operation intended to be called after a handshake completes with a lighthouse, +// so the lighthouse has our current addresses without waiting for the next +// periodic update. +func (lh *LightHouse) TriggerUpdate() { + select { + case lh.updateTrigger <- struct{}{}: + default: + } +} + func (lh *LightHouse) SendUpdate() { - var v4 []*Ip4AndPort - var v6 []*Ip6AndPort + var v4 []*V4AddrPort + var v6 []*V6AddrPort for _, e := range lh.GetAdvertiseAddrs() { - if ip := e.ip.To4(); ip != nil { - v4 = append(v4, NewIp4AndPort(e.ip, uint32(e.port))) + if e.Addr().Is4() { + v4 = append(v4, netAddrToProtoV4AddrPort(e.Addr(), e.Port())) } else { - v6 = append(v6, NewIp6AndPort(e.ip, uint32(e.port))) + v6 = append(v6, netAddrToProtoV6AddrPort(e.Addr(), e.Port())) } } lal := lh.GetLocalAllowList() - for _, e := range *localIps(lh.l, lal) { - if ip4 := e.To4(); ip4 != nil && ipMaskContains(lh.myVpnIp, lh.myVpnZeros, iputil.Ip2VpnIp(ip4)) { + for _, e := range lh.localAddrsFn(lal) { + if lh.myVpnNetworksTable.Contains(e) { continue } - // Only add IPs that aren't my VPN/tun IP - if ip := e.To4(); ip != nil { - v4 = append(v4, NewIp4AndPort(e, lh.nebulaPort)) + // Only add addrs that aren't my VPN/tun networks + if e.Is4() { + v4 = append(v4, netAddrToProtoV4AddrPort(e, uint16(lh.nebulaPort))) } else { - v6 = append(v6, NewIp6AndPort(e, lh.nebulaPort)) + v6 = append(v6, netAddrToProtoV6AddrPort(e, uint16(lh.nebulaPort))) } } - var relays []uint32 - for _, r := range lh.GetRelaysForMe() { - relays = append(relays, (uint32)(r)) - } - - m := &NebulaMeta{ - Type: NebulaMeta_HostUpdateNotification, - Details: &NebulaMetaDetails{ - VpnIp: uint32(lh.myVpnIp), - Ip4AndPorts: v4, - Ip6AndPorts: v6, - RelayVpnIp: relays, - }, - } - - lighthouses := lh.GetLighthouses() - lh.metricTx(NebulaMeta_HostUpdateNotification, int64(len(lighthouses))) nb := make([]byte, 12, 12) out := make([]byte, mtu) - mm, err := m.Marshal() - if err != nil { - lh.l.WithError(err).Error("Error while marshaling for lighthouse update") - return - } + var v1Update, v2Update []byte + var err error + updated := 0 + lighthouses := lh.GetLighthouses() + + for _, lhVpnAddr := range lighthouses { + var v cert.Version + hi := lh.ifce.GetHostInfo(lhVpnAddr) + if hi != nil { + v = hi.ConnectionState.myCert.Version() + } else { + v = lh.ifce.GetCertState().initiatingVersion + } + if v == cert.Version1 { + if v1Update == nil { + if !lh.myVpnNetworks[0].Addr().Is4() { + lh.l.Warn("cannot update lighthouse using v1 protocol without an IPv4 address", + "lighthouseAddr", lhVpnAddr, + ) + continue + } + var relays []uint32 + for _, r := range lh.GetRelaysForMe() { + if !r.Is4() { + continue + } + b := r.As4() + relays = append(relays, binary.BigEndian.Uint32(b[:])) + } + b := lh.myVpnNetworks[0].Addr().As4() + msg := NebulaMeta{ + Type: NebulaMeta_HostUpdateNotification, + Details: &NebulaMetaDetails{ + V4AddrPorts: v4, + V6AddrPorts: v6, + OldRelayVpnAddrs: relays, + OldVpnAddr: binary.BigEndian.Uint32(b[:]), + }, + } + + v1Update, err = msg.Marshal() + if err != nil { + lh.l.Error("Error while marshaling for lighthouse v1 update", + "error", err, + "lighthouseAddr", lhVpnAddr, + ) + continue + } + } + + lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Update, nb, out) + updated++ + + } else if v == cert.Version2 { + if v2Update == nil { + var relays []*Addr + for _, r := range lh.GetRelaysForMe() { + relays = append(relays, netAddrToProtoAddr(r)) + } - for vpnIp := range lighthouses { - lh.ifce.SendMessageToVpnIp(header.LightHouse, 0, vpnIp, mm, nb, out) + msg := NebulaMeta{ + Type: NebulaMeta_HostUpdateNotification, + Details: &NebulaMetaDetails{ + V4AddrPorts: v4, + V6AddrPorts: v6, + RelayVpnAddrs: relays, + }, + } + + v2Update, err = msg.Marshal() + if err != nil { + lh.l.Error("Error while marshaling for lighthouse v2 update", + "error", err, + "lighthouseAddr", lhVpnAddr, + ) + continue + } + } + + lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Update, nb, out) + updated++ + + } else { + lh.l.Debug("unsupported protocol version", + "op", "update", + "version", v, + ) + continue + } } + + lh.metricTx(NebulaMeta_HostUpdateNotification, int64(updated)) } type LightHouseHandler struct { @@ -870,7 +1042,7 @@ type LightHouseHandler struct { out []byte pb []byte meta *NebulaMeta - l *logrus.Logger + l *slog.Logger } func (lh *LightHouse) NewRequestHandler() *LightHouseHandler { @@ -904,34 +1076,34 @@ func (lhh *LightHouseHandler) resetMeta() *NebulaMeta { lhh.meta.Reset() // Keep the array memory around - details.Ip4AndPorts = details.Ip4AndPorts[:0] - details.Ip6AndPorts = details.Ip6AndPorts[:0] - details.RelayVpnIp = details.RelayVpnIp[:0] + details.V4AddrPorts = details.V4AddrPorts[:0] + details.V6AddrPorts = details.V6AddrPorts[:0] + details.RelayVpnAddrs = details.RelayVpnAddrs[:0] + details.OldRelayVpnAddrs = details.OldRelayVpnAddrs[:0] + details.OldVpnAddr = 0 + details.VpnAddr = nil lhh.meta.Details = details return lhh.meta } -func lhHandleRequest(lhh *LightHouseHandler, f *Interface) udp.LightHouseHandlerFunc { - return func(rAddr *udp.Addr, vpnIp iputil.VpnIp, p []byte) { - lhh.HandleRequest(rAddr, vpnIp, p, f) - } -} - -func (lhh *LightHouseHandler) HandleRequest(rAddr *udp.Addr, vpnIp iputil.VpnIp, p []byte, w EncWriter) { +func (lhh *LightHouseHandler) HandleRequest(rAddr netip.AddrPort, fromVpnAddrs []netip.Addr, p []byte, w EncWriter) { n := lhh.resetMeta() err := n.Unmarshal(p) if err != nil { - lhh.l.WithError(err).WithField("vpnIp", vpnIp).WithField("udpAddr", rAddr). - Error("Failed to unmarshal lighthouse packet") - //TODO: send recv_error? + lhh.l.Error("Failed to unmarshal lighthouse packet", + "error", err, + "vpnAddrs", fromVpnAddrs, + "udpAddr", rAddr, + ) return } if n.Details == nil { - lhh.l.WithField("vpnIp", vpnIp).WithField("udpAddr", rAddr). - Error("Invalid lighthouse update") - //TODO: send recv_error? + lhh.l.Error("Invalid lighthouse update", + "vpnAddrs", fromVpnAddrs, + "udpAddr", rAddr, + ) return } @@ -939,41 +1111,64 @@ func (lhh *LightHouseHandler) HandleRequest(rAddr *udp.Addr, vpnIp iputil.VpnIp, switch n.Type { case NebulaMeta_HostQuery: - lhh.handleHostQuery(n, vpnIp, rAddr, w) + lhh.handleHostQuery(n, fromVpnAddrs, rAddr, w) case NebulaMeta_HostQueryReply: - lhh.handleHostQueryReply(n, vpnIp) + lhh.handleHostQueryReply(n, fromVpnAddrs) case NebulaMeta_HostUpdateNotification: - lhh.handleHostUpdateNotification(n, vpnIp, w) + lhh.handleHostUpdateNotification(n, fromVpnAddrs, w) case NebulaMeta_HostMovedNotification: case NebulaMeta_HostPunchNotification: - lhh.handleHostPunchNotification(n, vpnIp, w) + lhh.handleHostPunchNotification(n, fromVpnAddrs, w) case NebulaMeta_HostUpdateNotificationAck: // noop } } -func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, vpnIp iputil.VpnIp, addr *udp.Addr, w EncWriter) { +func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []netip.Addr, addr netip.AddrPort, w EncWriter) { // Exit if we don't answer queries if !lhh.lh.amLighthouse { - if lhh.l.Level >= logrus.DebugLevel { - lhh.l.Debugln("I don't answer queries, but received from: ", addr) + if lhh.l.Enabled(context.Background(), slog.LevelDebug) { + lhh.l.Debug("I don't answer queries, but received one", "from", addr) } return } - //TODO: we can DRY this further - reqVpnIp := n.Details.VpnIp - //TODO: Maybe instead of marshalling into n we marshal into a new `r` to not nuke our current request data - found, ln, err := lhh.lh.queryAndPrepMessage(iputil.VpnIp(n.Details.VpnIp), func(c *cache) (int, error) { + queryVpnAddr, useVersion, err := n.Details.GetVpnAddrAndVersion() + if err != nil { + if lhh.l.Enabled(context.Background(), slog.LevelDebug) { + lhh.l.Debug("Dropping malformed HostQuery", + "from", fromVpnAddrs, + "details", n.Details, + ) + } + return + } + if useVersion == cert.Version1 && queryVpnAddr.Is6() { + // this case really shouldn't be possible to represent, but reject it anyway. + if lhh.l.Enabled(context.Background(), slog.LevelDebug) { + lhh.l.Debug("invalid vpn addr for v1 handleHostQuery", + "vpnAddrs", fromVpnAddrs, + "queryVpnAddr", queryVpnAddr, + ) + } + return + } + + found, ln, err := lhh.lh.queryAndPrepMessage(queryVpnAddr, func(c *cache) (int, error) { n = lhh.resetMeta() n.Type = NebulaMeta_HostQueryReply - n.Details.VpnIp = reqVpnIp + if useVersion == cert.Version1 { + b := queryVpnAddr.As4() + n.Details.OldVpnAddr = binary.BigEndian.Uint32(b[:]) + } else { + n.Details.VpnAddr = netAddrToProtoAddr(queryVpnAddr) + } - lhh.coalesceAnswers(c, n) + lhh.coalesceAnswers(useVersion, c, n) return n.MarshalTo(lhh.pb) }) @@ -983,20 +1178,57 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, vpnIp iputil.VpnIp, } if err != nil { - lhh.l.WithError(err).WithField("vpnIp", vpnIp).Error("Failed to marshal lighthouse host query reply") + lhh.l.Error("Failed to marshal lighthouse host query reply", + "error", err, + "vpnAddrs", fromVpnAddrs, + ) return } lhh.lh.metricTx(NebulaMeta_HostQueryReply, 1) - w.SendMessageToVpnIp(header.LightHouse, 0, vpnIp, lhh.pb[:ln], lhh.nb, lhh.out[:0]) + w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0]) + + lhh.sendHostPunchNotification(n, fromVpnAddrs, queryVpnAddr, w) +} - // This signals the other side to punch some zero byte udp packets - found, ln, err = lhh.lh.queryAndPrepMessage(vpnIp, func(c *cache) (int, error) { +// sendHostPunchNotification signals the other side to punch some zero byte udp packets +func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, punchNotifDest netip.Addr, w EncWriter) { + whereToPunch := fromVpnAddrs[0] + found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) { n = lhh.resetMeta() n.Type = NebulaMeta_HostPunchNotification - n.Details.VpnIp = uint32(vpnIp) + targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest) + var useVersion cert.Version + if targetHI == nil { + useVersion = lhh.lh.ifce.GetCertState().initiatingVersion + } else { + crt := targetHI.GetCert().Certificate + useVersion = crt.Version() + // we can only retarget if we have a hostinfo + newDest, ok := findNetworkUnion(crt.Networks(), fromVpnAddrs) + if ok { + whereToPunch = newDest + } else { + if lhh.l.Enabled(context.Background(), slog.LevelDebug) { + lhh.l.Debug("unable to punch to host, no addresses in common", + "to", crt.Networks(), + ) + } + } + } - lhh.coalesceAnswers(c, n) + if useVersion == cert.Version1 { + if !whereToPunch.Is4() { + return 0, fmt.Errorf("invalid vpn addr for v1 handleHostQuery") + } + b := whereToPunch.As4() + n.Details.OldVpnAddr = binary.BigEndian.Uint32(b[:]) + } else if useVersion == cert.Version2 { + n.Details.VpnAddr = netAddrToProtoAddr(whereToPunch) + } else { + return 0, errors.New("unsupported version") + } + lhh.coalesceAnswers(useVersion, c, n) return n.MarshalTo(lhh.pb) }) @@ -1006,153 +1238,306 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, vpnIp iputil.VpnIp, } if err != nil { - lhh.l.WithError(err).WithField("vpnIp", vpnIp).Error("Failed to marshal lighthouse host was queried for") + lhh.l.Error("Failed to marshal lighthouse host was queried for", + "error", err, + "vpnAddrs", fromVpnAddrs, + ) return } lhh.lh.metricTx(NebulaMeta_HostPunchNotification, 1) - w.SendMessageToVpnIp(header.LightHouse, 0, iputil.VpnIp(reqVpnIp), lhh.pb[:ln], lhh.nb, lhh.out[:0]) + w.SendMessageToVpnAddr(header.LightHouse, 0, punchNotifDest, lhh.pb[:ln], lhh.nb, lhh.out[:0]) } -func (lhh *LightHouseHandler) coalesceAnswers(c *cache, n *NebulaMeta) { +func (lhh *LightHouseHandler) coalesceAnswers(v cert.Version, c *cache, n *NebulaMeta) { if c.v4 != nil { if c.v4.learned != nil { - n.Details.Ip4AndPorts = append(n.Details.Ip4AndPorts, c.v4.learned) + n.Details.V4AddrPorts = append(n.Details.V4AddrPorts, c.v4.learned) } if c.v4.reported != nil && len(c.v4.reported) > 0 { - n.Details.Ip4AndPorts = append(n.Details.Ip4AndPorts, c.v4.reported...) + n.Details.V4AddrPorts = append(n.Details.V4AddrPorts, c.v4.reported...) } } if c.v6 != nil { if c.v6.learned != nil { - n.Details.Ip6AndPorts = append(n.Details.Ip6AndPorts, c.v6.learned) + n.Details.V6AddrPorts = append(n.Details.V6AddrPorts, c.v6.learned) } if c.v6.reported != nil && len(c.v6.reported) > 0 { - n.Details.Ip6AndPorts = append(n.Details.Ip6AndPorts, c.v6.reported...) + n.Details.V6AddrPorts = append(n.Details.V6AddrPorts, c.v6.reported...) } } if c.relay != nil { - n.Details.RelayVpnIp = append(n.Details.RelayVpnIp, c.relay.relay...) + if v == cert.Version1 { + b := [4]byte{} + for _, r := range c.relay.relay { + if !r.Is4() { + continue + } + b = r.As4() + n.Details.OldRelayVpnAddrs = append(n.Details.OldRelayVpnAddrs, binary.BigEndian.Uint32(b[:])) + } + } else if v == cert.Version2 { + for _, r := range c.relay.relay { + n.Details.RelayVpnAddrs = append(n.Details.RelayVpnAddrs, netAddrToProtoAddr(r)) + } + } else { + if lhh.l.Enabled(context.Background(), slog.LevelDebug) { + lhh.l.Debug("unsupported protocol version", + "op", "coalesceAnswers", + "version", v, + ) + } + } } } -func (lhh *LightHouseHandler) handleHostQueryReply(n *NebulaMeta, vpnIp iputil.VpnIp) { - if !lhh.lh.IsLighthouseIP(vpnIp) { +func (lhh *LightHouseHandler) handleHostQueryReply(n *NebulaMeta, fromVpnAddrs []netip.Addr) { + if !lhh.lh.IsAnyLighthouseAddr(fromVpnAddrs) { + return + } + + certVpnAddr, _, err := n.Details.GetVpnAddrAndVersion() + if err != nil { + if lhh.l.Enabled(context.Background(), slog.LevelDebug) { + lhh.l.Error("dropping malformed HostQueryReply", + "error", err, + "vpnAddrs", fromVpnAddrs, + ) + } return } + relays := n.Details.GetRelays() lhh.lh.Lock() - am := lhh.lh.unlockedGetRemoteList(iputil.VpnIp(n.Details.VpnIp)) + am := lhh.lh.unlockedGetRemoteList([]netip.Addr{certVpnAddr}) am.Lock() lhh.lh.Unlock() - certVpnIp := iputil.VpnIp(n.Details.VpnIp) - am.unlockedSetV4(vpnIp, certVpnIp, n.Details.Ip4AndPorts, lhh.lh.unlockedShouldAddV4) - am.unlockedSetV6(vpnIp, certVpnIp, n.Details.Ip6AndPorts, lhh.lh.unlockedShouldAddV6) - am.unlockedSetRelay(vpnIp, certVpnIp, n.Details.RelayVpnIp) + am.unlockedSetV4(fromVpnAddrs[0], certVpnAddr, n.Details.V4AddrPorts, lhh.lh.unlockedShouldAddV4) + am.unlockedSetV6(fromVpnAddrs[0], certVpnAddr, n.Details.V6AddrPorts, lhh.lh.unlockedShouldAddV6) + am.unlockedSetRelay(fromVpnAddrs[0], relays) am.Unlock() // Non-blocking attempt to trigger, skip if it would block select { - case lhh.lh.handshakeTrigger <- iputil.VpnIp(n.Details.VpnIp): + case lhh.lh.handshakeTrigger <- certVpnAddr: default: } } -func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, vpnIp iputil.VpnIp, w EncWriter) { +func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, w EncWriter) { if !lhh.lh.amLighthouse { - if lhh.l.Level >= logrus.DebugLevel { - lhh.l.Debugln("I am not a lighthouse, do not take host updates: ", vpnIp) + if lhh.l.Enabled(context.Background(), slog.LevelDebug) { + lhh.l.Debug("I am not a lighthouse, do not take host updates", "from", fromVpnAddrs) } return } - //Simple check that the host sent this not someone else - if n.Details.VpnIp != uint32(vpnIp) { - if lhh.l.Level >= logrus.DebugLevel { - lhh.l.WithField("vpnIp", vpnIp).WithField("answer", iputil.VpnIp(n.Details.VpnIp)).Debugln("Host sent invalid update") + // not using GetVpnAddrAndVersion because we don't want to error on a blank detailsVpnAddr + var detailsVpnAddr netip.Addr + var useVersion cert.Version + if n.Details.OldVpnAddr != 0 { //v1 always sets this field + b := [4]byte{} + binary.BigEndian.PutUint32(b[:], n.Details.OldVpnAddr) + detailsVpnAddr = netip.AddrFrom4(b) + useVersion = cert.Version1 + } else if n.Details.VpnAddr != nil { //this field is "optional" in v2, but if it's set, we should enforce it + detailsVpnAddr = protoAddrToNetAddr(n.Details.VpnAddr) + useVersion = cert.Version2 + } else { + detailsVpnAddr = netip.Addr{} + useVersion = cert.Version2 + } + + //Simple check that the host sent this not someone else, if detailsVpnAddr is filled + if detailsVpnAddr.IsValid() && !slices.Contains(fromVpnAddrs, detailsVpnAddr) { + if lhh.l.Enabled(context.Background(), slog.LevelDebug) { + lhh.l.Debug("Host sent invalid update", + "vpnAddrs", fromVpnAddrs, + "answer", detailsVpnAddr, + ) } return } + relays := n.Details.GetRelays() + lhh.lh.Lock() - am := lhh.lh.unlockedGetRemoteList(vpnIp) + am := lhh.lh.unlockedGetRemoteList(fromVpnAddrs) am.Lock() lhh.lh.Unlock() - certVpnIp := iputil.VpnIp(n.Details.VpnIp) - am.unlockedSetV4(vpnIp, certVpnIp, n.Details.Ip4AndPorts, lhh.lh.unlockedShouldAddV4) - am.unlockedSetV6(vpnIp, certVpnIp, n.Details.Ip6AndPorts, lhh.lh.unlockedShouldAddV6) - am.unlockedSetRelay(vpnIp, certVpnIp, n.Details.RelayVpnIp) + am.unlockedSetV4(fromVpnAddrs[0], fromVpnAddrs[0], n.Details.V4AddrPorts, lhh.lh.unlockedShouldAddV4) + am.unlockedSetV6(fromVpnAddrs[0], fromVpnAddrs[0], n.Details.V6AddrPorts, lhh.lh.unlockedShouldAddV6) + am.unlockedSetRelay(fromVpnAddrs[0], relays) am.Unlock() n = lhh.resetMeta() n.Type = NebulaMeta_HostUpdateNotificationAck - n.Details.VpnIp = uint32(vpnIp) - ln, err := n.MarshalTo(lhh.pb) + switch useVersion { + case cert.Version1: + if !fromVpnAddrs[0].Is4() { + lhh.l.Error("Can not send HostUpdateNotificationAck for a ipv6 vpn ip in a v1 message", + "vpnAddrs", fromVpnAddrs, + ) + return + } + vpnAddrB := fromVpnAddrs[0].As4() + n.Details.OldVpnAddr = binary.BigEndian.Uint32(vpnAddrB[:]) + case cert.Version2: + // do nothing, we want to send a blank message + default: + lhh.l.Error("invalid protocol version", "useVersion", useVersion) + return + } + ln, err := n.MarshalTo(lhh.pb) if err != nil { - lhh.l.WithError(err).WithField("vpnIp", vpnIp).Error("Failed to marshal lighthouse host update ack") + lhh.l.Error("Failed to marshal lighthouse host update ack", + "error", err, + "vpnAddrs", fromVpnAddrs, + ) return } lhh.lh.metricTx(NebulaMeta_HostUpdateNotificationAck, 1) - w.SendMessageToVpnIp(header.LightHouse, 0, vpnIp, lhh.pb[:ln], lhh.nb, lhh.out[:0]) + w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0]) } -func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, vpnIp iputil.VpnIp, w EncWriter) { - if !lhh.lh.IsLighthouseIP(vpnIp) { +func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, w EncWriter) { + //It's possible the lighthouse is communicating with us using a non primary vpn addr, + //which means we need to compare all fromVpnAddrs against all configured lighthouse vpn addrs. + if !lhh.lh.IsAnyLighthouseAddr(fromVpnAddrs) { return } - empty := []byte{0} - punch := func(vpnPeer *udp.Addr) { - if vpnPeer == nil { - return + detailsVpnAddr, _, err := n.Details.GetVpnAddrAndVersion() + if err != nil { + if lhh.l.Enabled(context.Background(), slog.LevelDebug) { + lhh.l.Debug("dropping invalid HostPunchNotification", + "details", n.Details, + "error", err, + ) } + return + } - go func() { - time.Sleep(lhh.lh.punchy.GetDelay()) - lhh.lh.metricHolepunchTx.Inc(1) - lhh.lh.punchConn.WriteTo(empty, vpnPeer) - }() + remoteAllowList := lhh.lh.GetRemoteAllowList() + for _, a := range n.Details.V4AddrPorts { + if a == nil { + continue + } + b := protoV4AddrPortToNetAddrPort(a) + if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) { + lhh.lh.punchy.Schedule(b, detailsVpnAddr) + } + } - if lhh.l.Level >= logrus.DebugLevel { - //TODO: lacking the ip we are actually punching on, old: l.Debugf("Punching %s on %d for %s", IntIp(a.Ip), a.Port, IntIp(n.Details.VpnIp)) - lhh.l.Debugf("Punching on %d for %s", vpnPeer.Port, iputil.VpnIp(n.Details.VpnIp)) + for _, a := range n.Details.V6AddrPorts { + if a == nil { + continue + } + b := protoV6AddrPortToNetAddrPort(a) + if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) { + lhh.lh.punchy.Schedule(b, detailsVpnAddr) } } - for _, a := range n.Details.Ip4AndPorts { - punch(NewUDPAddrFromLH4(a)) + // This sends a nebula test packet to the host trying to contact us. In the case + // of a double nat or other difficult scenario, this may help establish + // a tunnel. ScheduleRespond is a no-op when punchy.respond is disabled. + lhh.lh.punchy.ScheduleRespond(detailsVpnAddr) +} + +func protoAddrToNetAddr(addr *Addr) netip.Addr { + b := [16]byte{} + binary.BigEndian.PutUint64(b[:8], addr.Hi) + binary.BigEndian.PutUint64(b[8:], addr.Lo) + return netip.AddrFrom16(b).Unmap() +} + +func protoV4AddrPortToNetAddrPort(ap *V4AddrPort) netip.AddrPort { + b := [4]byte{} + binary.BigEndian.PutUint32(b[:], ap.Addr) + return netip.AddrPortFrom(netip.AddrFrom4(b), uint16(ap.Port)) +} + +func protoV6AddrPortToNetAddrPort(ap *V6AddrPort) netip.AddrPort { + b := [16]byte{} + binary.BigEndian.PutUint64(b[:8], ap.Hi) + binary.BigEndian.PutUint64(b[8:], ap.Lo) + return netip.AddrPortFrom(netip.AddrFrom16(b).Unmap(), uint16(ap.Port)) +} + +func netAddrToProtoAddr(addr netip.Addr) *Addr { + b := addr.As16() + return &Addr{ + Hi: binary.BigEndian.Uint64(b[:8]), + Lo: binary.BigEndian.Uint64(b[8:]), + } +} + +func netAddrToProtoV4AddrPort(addr netip.Addr, port uint16) *V4AddrPort { + v4Addr := addr.As4() + return &V4AddrPort{ + Addr: binary.BigEndian.Uint32(v4Addr[:]), + Port: uint32(port), } +} - for _, a := range n.Details.Ip6AndPorts { - punch(NewUDPAddrFromLH6(a)) +func netAddrToProtoV6AddrPort(addr netip.Addr, port uint16) *V6AddrPort { + v6Addr := addr.As16() + return &V6AddrPort{ + Hi: binary.BigEndian.Uint64(v6Addr[:8]), + Lo: binary.BigEndian.Uint64(v6Addr[8:]), + Port: uint32(port), } +} - // This sends a nebula test packet to the host trying to contact us. In the case - // of a double nat or other difficult scenario, this may help establish - // a tunnel. - if lhh.lh.punchy.GetRespond() { - queryVpnIp := iputil.VpnIp(n.Details.VpnIp) - go func() { - time.Sleep(lhh.lh.punchy.GetRespondDelay()) - if lhh.l.Level >= logrus.DebugLevel { - lhh.l.Debugf("Sending a nebula test packet to vpn ip %s", queryVpnIp) +func (d *NebulaMetaDetails) GetRelays() []netip.Addr { + var relays []netip.Addr + if len(d.OldRelayVpnAddrs) > 0 { + b := [4]byte{} + for _, r := range d.OldRelayVpnAddrs { + binary.BigEndian.PutUint32(b[:], r) + relays = append(relays, netip.AddrFrom4(b)) + } + } + + if len(d.RelayVpnAddrs) > 0 { + for _, r := range d.RelayVpnAddrs { + if r != nil { + relays = append(relays, protoAddrToNetAddr(r)) + } + } + } + return relays +} + +// FindNetworkUnion returns the first netip.Addr contained in the list of provided netip.Prefix, if able +func findNetworkUnion(prefixes []netip.Prefix, addrs []netip.Addr) (netip.Addr, bool) { + for i := range prefixes { + for j := range addrs { + if prefixes[i].Contains(addrs[j]) { + return addrs[j], true } - //NOTE: we have to allocate a new output buffer here since we are spawning a new goroutine - // for each punchBack packet. We should move this into a timerwheel or a single goroutine - // managed by a channel. - w.SendMessageToVpnIp(header.Test, header.TestRequest, queryVpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu)) - }() + } } + return netip.Addr{}, false } -// ipMaskContains checks if testIp is contained by ip after applying a cidr -// zeros is 32 - bits from net.IPMask.Size() -func ipMaskContains(ip iputil.VpnIp, zeros iputil.VpnIp, testIp iputil.VpnIp) bool { - return (testIp^ip)>>zeros == 0 +func (d *NebulaMetaDetails) GetVpnAddrAndVersion() (netip.Addr, cert.Version, error) { + if d.OldVpnAddr != 0 { + b := [4]byte{} + binary.BigEndian.PutUint32(b[:], d.OldVpnAddr) + detailsVpnAddr := netip.AddrFrom4(b) + return detailsVpnAddr, cert.Version1, nil + } else if d.VpnAddr != nil { + detailsVpnAddr := protoAddrToNetAddr(d.VpnAddr) + return detailsVpnAddr, cert.Version2, nil + } else { + return netip.Addr{}, cert.Version1, ErrBadDetailsVpnAddr + } } diff --git a/lighthouse_test.go b/lighthouse_test.go index 66427e33..81c883ff 100644 --- a/lighthouse_test.go +++ b/lighthouse_test.go @@ -1,151 +1,151 @@ package nebula import ( - "context" + "encoding/binary" "fmt" - "net" + "net/netip" "testing" + "github.com/gaissmai/bart" + "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/test" - "github.com/slackhq/nebula/udp" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" ) -//TODO: Add a test to ensure udpAddr is copied and not reused - func TestOldIPv4Only(t *testing.T) { // This test ensures our new ipv6 enabled LH protobuf IpAndPorts works with the old style to enable backwards compatibility b := []byte{8, 129, 130, 132, 80, 16, 10} - var m Ip4AndPort + var m V4AddrPort err := m.Unmarshal(b) - assert.NoError(t, err) - assert.Equal(t, "10.1.1.1", iputil.VpnIp(m.GetIp()).String()) -} - -func TestNewLhQuery(t *testing.T) { - myIp := net.ParseIP("192.1.1.1") - myIpint := iputil.Ip2VpnIp(myIp) - - // Generating a new lh query should work - a := NewLhQueryByInt(myIpint) - - // The result should be a nebulameta protobuf - assert.IsType(t, &NebulaMeta{}, a) - - // It should also Marshal fine - b, err := a.Marshal() - assert.Nil(t, err) - - // and then Unmarshal fine - n := &NebulaMeta{} - err = n.Unmarshal(b) - assert.Nil(t, err) - + require.NoError(t, err) + ip := netip.MustParseAddr("10.1.1.1") + bp := ip.As4() + assert.Equal(t, binary.BigEndian.Uint32(bp[:]), m.GetAddr()) } func Test_lhStaticMapping(t *testing.T) { l := test.NewLogger() - _, myVpnNet, _ := net.ParseCIDR("10.128.0.1/16") + myVpnNet := netip.MustParsePrefix("10.128.0.1/16") + nt := new(bart.Lite) + nt.Insert(myVpnNet) + cs := &CertState{ + myVpnNetworks: []netip.Prefix{myVpnNet}, + myVpnNetworksTable: nt, + } lh1 := "10.128.0.2" c := config.NewC(l) - c.Settings["lighthouse"] = map[interface{}]interface{}{"hosts": []interface{}{lh1}} - c.Settings["static_host_map"] = map[interface{}]interface{}{lh1: []interface{}{"1.1.1.1:4242"}} - _, err := NewLightHouseFromConfig(context.Background(), l, c, myVpnNet, nil, nil) - assert.Nil(t, err) + c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}} + c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}} + _, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil) + require.NoError(t, err) lh2 := "10.128.0.3" c = config.NewC(l) - c.Settings["lighthouse"] = map[interface{}]interface{}{"hosts": []interface{}{lh1, lh2}} - c.Settings["static_host_map"] = map[interface{}]interface{}{lh1: []interface{}{"100.1.1.1:4242"}} - _, err = NewLightHouseFromConfig(context.Background(), l, c, myVpnNet, nil, nil) - assert.EqualError(t, err, "lighthouse 10.128.0.3 does not have a static_host_map entry") + c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1, lh2}} + c.Settings["static_host_map"] = map[string]any{lh1: []any{"100.1.1.1:4242"}} + _, err = NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil) + require.EqualError(t, err, "lighthouse 10.128.0.3 does not have a static_host_map entry") } func TestReloadLighthouseInterval(t *testing.T) { l := test.NewLogger() - _, myVpnNet, _ := net.ParseCIDR("10.128.0.1/16") + myVpnNet := netip.MustParsePrefix("10.128.0.1/16") + nt := new(bart.Lite) + nt.Insert(myVpnNet) + cs := &CertState{ + myVpnNetworks: []netip.Prefix{myVpnNet}, + myVpnNetworksTable: nt, + } lh1 := "10.128.0.2" c := config.NewC(l) - c.Settings["lighthouse"] = map[interface{}]interface{}{ - "hosts": []interface{}{lh1}, + c.Settings["lighthouse"] = map[string]any{ + "hosts": []any{lh1}, "interval": "1s", } - c.Settings["static_host_map"] = map[interface{}]interface{}{lh1: []interface{}{"1.1.1.1:4242"}} - lh, err := NewLightHouseFromConfig(context.Background(), l, c, myVpnNet, nil, nil) - assert.NoError(t, err) + c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}} + lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil) + require.NoError(t, err) lh.ifce = &mockEncWriter{} // The first one routine is kicked off by main.go currently, lets make sure that one dies - c.ReloadConfigString("lighthouse:\n interval: 5") + require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 5")) assert.Equal(t, int64(5), lh.interval.Load()) // Subsequent calls are killed off by the LightHouse.Reload function - c.ReloadConfigString("lighthouse:\n interval: 10") + require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 10")) assert.Equal(t, int64(10), lh.interval.Load()) // If this completes then nothing is stealing our reload routine - c.ReloadConfigString("lighthouse:\n interval: 11") + require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 11")) assert.Equal(t, int64(11), lh.interval.Load()) } func BenchmarkLighthouseHandleRequest(b *testing.B) { l := test.NewLogger() - _, myVpnNet, _ := net.ParseCIDR("10.128.0.1/0") + myVpnNet := netip.MustParsePrefix("10.128.0.1/0") + nt := new(bart.Lite) + nt.Insert(myVpnNet) + cs := &CertState{ + myVpnNetworks: []netip.Prefix{myVpnNet}, + myVpnNetworksTable: nt, + } c := config.NewC(l) - lh, err := NewLightHouseFromConfig(context.Background(), l, c, myVpnNet, nil, nil) - if !assert.NoError(b, err) { - b.Fatal() - } - - hAddr := udp.NewAddrFromString("4.5.6.7:12345") - hAddr2 := udp.NewAddrFromString("4.5.6.7:12346") - lh.addrMap[3] = NewRemoteList(nil) - lh.addrMap[3].unlockedSetV4( - 3, - 3, - []*Ip4AndPort{ - NewIp4AndPort(hAddr.IP, uint32(hAddr.Port)), - NewIp4AndPort(hAddr2.IP, uint32(hAddr2.Port)), + lh, err := NewLightHouseFromConfig(b.Context(), l, c, cs, nil, nil) + require.NoError(b, err) + + hAddr := netip.MustParseAddrPort("4.5.6.7:12345") + hAddr2 := netip.MustParseAddrPort("4.5.6.7:12346") + + vpnIp3 := netip.MustParseAddr("0.0.0.3") + lh.addrMap[vpnIp3] = NewRemoteList([]netip.Addr{vpnIp3}, nil) + lh.addrMap[vpnIp3].unlockedSetV4( + vpnIp3, + vpnIp3, + []*V4AddrPort{ + netAddrToProtoV4AddrPort(hAddr.Addr(), hAddr.Port()), + netAddrToProtoV4AddrPort(hAddr2.Addr(), hAddr2.Port()), }, - func(iputil.VpnIp, *Ip4AndPort) bool { return true }, + func(netip.Addr, *V4AddrPort) bool { return true }, ) - rAddr := udp.NewAddrFromString("1.2.2.3:12345") - rAddr2 := udp.NewAddrFromString("1.2.2.3:12346") - lh.addrMap[2] = NewRemoteList(nil) - lh.addrMap[2].unlockedSetV4( - 3, - 3, - []*Ip4AndPort{ - NewIp4AndPort(rAddr.IP, uint32(rAddr.Port)), - NewIp4AndPort(rAddr2.IP, uint32(rAddr2.Port)), + rAddr := netip.MustParseAddrPort("1.2.2.3:12345") + rAddr2 := netip.MustParseAddrPort("1.2.2.3:12346") + vpnIp2 := netip.MustParseAddr("0.0.0.3") + lh.addrMap[vpnIp2] = NewRemoteList([]netip.Addr{vpnIp2}, nil) + lh.addrMap[vpnIp2].unlockedSetV4( + vpnIp3, + vpnIp3, + []*V4AddrPort{ + netAddrToProtoV4AddrPort(rAddr.Addr(), rAddr.Port()), + netAddrToProtoV4AddrPort(rAddr2.Addr(), rAddr2.Port()), }, - func(iputil.VpnIp, *Ip4AndPort) bool { return true }, + func(netip.Addr, *V4AddrPort) bool { return true }, ) mw := &mockEncWriter{} + hi := []netip.Addr{vpnIp2} b.Run("notfound", func(b *testing.B) { lhh := lh.NewRequestHandler() req := &NebulaMeta{ Type: NebulaMeta_HostQuery, Details: &NebulaMetaDetails{ - VpnIp: 4, - Ip4AndPorts: nil, + OldVpnAddr: 4, + V4AddrPorts: nil, }, } p, err := req.Marshal() - assert.NoError(b, err) + require.NoError(b, err) for n := 0; n < b.N; n++ { - lhh.HandleRequest(rAddr, 2, p, mw) + lhh.HandleRequest(rAddr, hi, p, mw) } }) b.Run("found", func(b *testing.B) { @@ -153,15 +153,15 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) { req := &NebulaMeta{ Type: NebulaMeta_HostQuery, Details: &NebulaMetaDetails{ - VpnIp: 3, - Ip4AndPorts: nil, + OldVpnAddr: 3, + V4AddrPorts: nil, }, } p, err := req.Marshal() - assert.NoError(b, err) + require.NoError(b, err) for n := 0; n < b.N; n++ { - lhh.HandleRequest(rAddr, 2, p, mw) + lhh.HandleRequest(rAddr, hi, p, mw) } }) } @@ -169,71 +169,80 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) { func TestLighthouse_Memory(t *testing.T) { l := test.NewLogger() - myUdpAddr0 := &udp.Addr{IP: net.ParseIP("10.0.0.2"), Port: 4242} - myUdpAddr1 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4242} - myUdpAddr2 := &udp.Addr{IP: net.ParseIP("172.16.0.2"), Port: 4242} - myUdpAddr3 := &udp.Addr{IP: net.ParseIP("100.152.0.2"), Port: 4242} - myUdpAddr4 := &udp.Addr{IP: net.ParseIP("24.15.0.2"), Port: 4242} - myUdpAddr5 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4243} - myUdpAddr6 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4244} - myUdpAddr7 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4245} - myUdpAddr8 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4246} - myUdpAddr9 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4247} - myUdpAddr10 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4248} - myUdpAddr11 := &udp.Addr{IP: net.ParseIP("192.168.0.2"), Port: 4249} - myVpnIp := iputil.Ip2VpnIp(net.ParseIP("10.128.0.2")) - - theirUdpAddr0 := &udp.Addr{IP: net.ParseIP("10.0.0.3"), Port: 4242} - theirUdpAddr1 := &udp.Addr{IP: net.ParseIP("192.168.0.3"), Port: 4242} - theirUdpAddr2 := &udp.Addr{IP: net.ParseIP("172.16.0.3"), Port: 4242} - theirUdpAddr3 := &udp.Addr{IP: net.ParseIP("100.152.0.3"), Port: 4242} - theirUdpAddr4 := &udp.Addr{IP: net.ParseIP("24.15.0.3"), Port: 4242} - theirVpnIp := iputil.Ip2VpnIp(net.ParseIP("10.128.0.3")) + myUdpAddr0 := netip.MustParseAddrPort("10.0.0.2:4242") + myUdpAddr1 := netip.MustParseAddrPort("192.168.0.2:4242") + myUdpAddr2 := netip.MustParseAddrPort("172.16.0.2:4242") + myUdpAddr3 := netip.MustParseAddrPort("100.152.0.2:4242") + myUdpAddr4 := netip.MustParseAddrPort("24.15.0.2:4242") + myUdpAddr5 := netip.MustParseAddrPort("192.168.0.2:4243") + myUdpAddr6 := netip.MustParseAddrPort("192.168.0.2:4244") + myUdpAddr7 := netip.MustParseAddrPort("192.168.0.2:4245") + myUdpAddr8 := netip.MustParseAddrPort("192.168.0.2:4246") + myUdpAddr9 := netip.MustParseAddrPort("192.168.0.2:4247") + myUdpAddr10 := netip.MustParseAddrPort("192.168.0.2:4248") + myUdpAddr11 := netip.MustParseAddrPort("192.168.0.2:4249") + myVpnIp := netip.MustParseAddr("10.128.0.2") + + theirUdpAddr0 := netip.MustParseAddrPort("10.0.0.3:4242") + theirUdpAddr1 := netip.MustParseAddrPort("192.168.0.3:4242") + theirUdpAddr2 := netip.MustParseAddrPort("172.16.0.3:4242") + theirUdpAddr3 := netip.MustParseAddrPort("100.152.0.3:4242") + theirUdpAddr4 := netip.MustParseAddrPort("24.15.0.3:4242") + theirVpnIp := netip.MustParseAddr("10.128.0.3") c := config.NewC(l) - c.Settings["lighthouse"] = map[interface{}]interface{}{"am_lighthouse": true} - c.Settings["listen"] = map[interface{}]interface{}{"port": 4242} - lh, err := NewLightHouseFromConfig(context.Background(), l, c, &net.IPNet{IP: net.IP{10, 128, 0, 1}, Mask: net.IPMask{255, 255, 255, 0}}, nil, nil) - assert.NoError(t, err) + c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true} + c.Settings["listen"] = map[string]any{"port": 4242} + + myVpnNet := netip.MustParsePrefix("10.128.0.1/24") + nt := new(bart.Lite) + nt.Insert(myVpnNet) + cs := &CertState{ + myVpnNetworks: []netip.Prefix{myVpnNet}, + myVpnNetworksTable: nt, + } + lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil) + lh.ifce = &mockEncWriter{} + require.NoError(t, err) lhh := lh.NewRequestHandler() // Test that my first update responds with just that - newLHHostUpdate(myUdpAddr0, myVpnIp, []*udp.Addr{myUdpAddr1, myUdpAddr2}, lhh) + newLHHostUpdate(myUdpAddr0, myVpnIp, []netip.AddrPort{myUdpAddr1, myUdpAddr2}, lhh) r := newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh) - assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr2) + assertIp4InArray(t, r.msg.Details.V4AddrPorts, myUdpAddr1, myUdpAddr2) // Ensure we don't accumulate addresses - newLHHostUpdate(myUdpAddr0, myVpnIp, []*udp.Addr{myUdpAddr3}, lhh) + newLHHostUpdate(myUdpAddr0, myVpnIp, []netip.AddrPort{myUdpAddr3}, lhh) r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh) - assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr3) + assertIp4InArray(t, r.msg.Details.V4AddrPorts, myUdpAddr3) // Grow it back to 2 - newLHHostUpdate(myUdpAddr0, myVpnIp, []*udp.Addr{myUdpAddr1, myUdpAddr4}, lhh) + newLHHostUpdate(myUdpAddr0, myVpnIp, []netip.AddrPort{myUdpAddr1, myUdpAddr4}, lhh) r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh) - assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr4) + assertIp4InArray(t, r.msg.Details.V4AddrPorts, myUdpAddr1, myUdpAddr4) // Update a different host and ask about it - newLHHostUpdate(theirUdpAddr0, theirVpnIp, []*udp.Addr{theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4}, lhh) + newLHHostUpdate(theirUdpAddr0, theirVpnIp, []netip.AddrPort{theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4}, lhh) r = newLHHostRequest(theirUdpAddr0, theirVpnIp, theirVpnIp, lhh) - assertIp4InArray(t, r.msg.Details.Ip4AndPorts, theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4) + assertIp4InArray(t, r.msg.Details.V4AddrPorts, theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4) // Have both hosts ask about the other r = newLHHostRequest(theirUdpAddr0, theirVpnIp, myVpnIp, lhh) - assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr4) + assertIp4InArray(t, r.msg.Details.V4AddrPorts, myUdpAddr1, myUdpAddr4) r = newLHHostRequest(myUdpAddr0, myVpnIp, theirVpnIp, lhh) - assertIp4InArray(t, r.msg.Details.Ip4AndPorts, theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4) + assertIp4InArray(t, r.msg.Details.V4AddrPorts, theirUdpAddr1, theirUdpAddr2, theirUdpAddr3, theirUdpAddr4) // Make sure we didn't get changed r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh) - assertIp4InArray(t, r.msg.Details.Ip4AndPorts, myUdpAddr1, myUdpAddr4) + assertIp4InArray(t, r.msg.Details.V4AddrPorts, myUdpAddr1, myUdpAddr4) // Ensure proper ordering and limiting // Send 12 addrs, get 10 back, the last 2 removed, allowing the duplicate to remain (clients dedupe) newLHHostUpdate( myUdpAddr0, myVpnIp, - []*udp.Addr{ + []netip.AddrPort{ myUdpAddr1, myUdpAddr2, myUdpAddr3, @@ -251,47 +260,187 @@ func TestLighthouse_Memory(t *testing.T) { r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh) assertIp4InArray( t, - r.msg.Details.Ip4AndPorts, + r.msg.Details.V4AddrPorts, myUdpAddr1, myUdpAddr2, myUdpAddr3, myUdpAddr4, myUdpAddr5, myUdpAddr5, myUdpAddr6, myUdpAddr7, myUdpAddr8, myUdpAddr9, ) // Make sure we won't add ips in our vpn network - bad1 := &udp.Addr{IP: net.ParseIP("10.128.0.99"), Port: 4242} - bad2 := &udp.Addr{IP: net.ParseIP("10.128.0.100"), Port: 4242} - good := &udp.Addr{IP: net.ParseIP("1.128.0.99"), Port: 4242} - newLHHostUpdate(myUdpAddr0, myVpnIp, []*udp.Addr{bad1, bad2, good}, lhh) + bad1 := netip.MustParseAddrPort("10.128.0.99:4242") + bad2 := netip.MustParseAddrPort("10.128.0.100:4242") + good := netip.MustParseAddrPort("1.128.0.99:4242") + newLHHostUpdate(myUdpAddr0, myVpnIp, []netip.AddrPort{bad1, bad2, good}, lhh) r = newLHHostRequest(myUdpAddr0, myVpnIp, myVpnIp, lhh) - assertIp4InArray(t, r.msg.Details.Ip4AndPorts, good) + assertIp4InArray(t, r.msg.Details.V4AddrPorts, good) } func TestLighthouse_reload(t *testing.T) { l := test.NewLogger() c := config.NewC(l) - c.Settings["lighthouse"] = map[interface{}]interface{}{"am_lighthouse": true} - c.Settings["listen"] = map[interface{}]interface{}{"port": 4242} - lh, err := NewLightHouseFromConfig(context.Background(), l, c, &net.IPNet{IP: net.IP{10, 128, 0, 1}, Mask: net.IPMask{255, 255, 255, 0}}, nil, nil) - assert.NoError(t, err) - - nc := map[interface{}]interface{}{ - "static_host_map": map[interface{}]interface{}{ - "10.128.0.2": []interface{}{"1.1.1.1:4242"}, + c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true} + c.Settings["listen"] = map[string]any{"port": 4242} + + myVpnNet := netip.MustParsePrefix("10.128.0.1/24") + nt := new(bart.Lite) + nt.Insert(myVpnNet) + cs := &CertState{ + myVpnNetworks: []netip.Prefix{myVpnNet}, + myVpnNetworksTable: nt, + } + + lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil) + require.NoError(t, err) + + nc := map[string]any{ + "static_host_map": map[string]any{ + "10.128.0.2": []any{"1.1.1.1:4242"}, }, } rc, err := yaml.Marshal(nc) - assert.NoError(t, err) + require.NoError(t, err) c.ReloadConfigString(string(rc)) err = lh.reload(c, false) - assert.NoError(t, err) + require.NoError(t, err) } -func newLHHostRequest(fromAddr *udp.Addr, myVpnIp, queryVpnIp iputil.VpnIp, lhh *LightHouseHandler) testLhReply { - req := &NebulaMeta{ - Type: NebulaMeta_HostQuery, - Details: &NebulaMetaDetails{ - VpnIp: uint32(queryVpnIp), +// TestLighthouse_reloadStaticHostMap verifies that reloading static_host_map applies the new +// config rather than appending to it. See issue #718. +func TestLighthouse_reloadStaticHostMap(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true} + c.Settings["listen"] = map[string]any{"port": 4242} + c.Settings["static_host_map"] = map[string]any{ + "10.128.0.2": []any{"1.1.1.1:4242"}, + } + + myVpnNet := netip.MustParsePrefix("10.128.0.1/24") + nt := new(bart.Lite) + nt.Insert(myVpnNet) + cs := &CertState{ + myVpnNetworks: []netip.Prefix{myVpnNet}, + myVpnNetworksTable: nt, + } + + lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil) + require.NoError(t, err) + + staticHost := netip.MustParseAddr("10.128.0.2") + otherHost := netip.MustParseAddr("10.128.0.3") + + // Capture the RemoteList pointer up front; an in-flight handshake would hold the same one + // on hostinfo.remotes, so it must reflect every reload below. + pinned := lh.Query(staticHost) + require.NotNil(t, pinned) + assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, pinned.CopyAddrs([]netip.Prefix{})) + + // Replace the remote address. The new address should be the only entry. + nc := map[string]any{ + "static_host_map": map[string]any{ + "10.128.0.2": []any{"2.2.2.2:4242"}, }, } + rc, err := yaml.Marshal(nc) + require.NoError(t, err) + require.NoError(t, c.ReloadConfigString(string(rc))) + + rl := lh.Query(staticHost) + require.NotNil(t, rl) + assert.Same(t, pinned, rl, "RemoteList pointer must stay stable so in-flight handshakes pick up the change") + assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, rl.CopyAddrs([]netip.Prefix{})) + + // Reload back to the original IP. Mirrors the round-trip in issue #718 step 6-8 where + // the buggy reload produced [1.1.1.1, 2.2.2.2, 1.1.1.1] instead of [1.1.1.1]. + nc = map[string]any{ + "static_host_map": map[string]any{ + "10.128.0.2": []any{"1.1.1.1:4242"}, + }, + } + rc, err = yaml.Marshal(nc) + require.NoError(t, err) + require.NoError(t, c.ReloadConfigString(string(rc))) + + rl = lh.Query(staticHost) + require.NotNil(t, rl) + assert.Same(t, pinned, rl) + assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{})) + + // Reload with the same config. An unchanged entry must not duplicate. + require.NoError(t, c.ReloadConfigString(string(rc))) + + rl = lh.Query(staticHost) + require.NotNil(t, rl) + assert.Same(t, pinned, rl) + assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{})) + + // Switch back to 2.2.2.2 so the rest of the test continues against a known address. + nc = map[string]any{ + "static_host_map": map[string]any{ + "10.128.0.2": []any{"2.2.2.2:4242"}, + }, + } + rc, err = yaml.Marshal(nc) + require.NoError(t, err) + require.NoError(t, c.ReloadConfigString(string(rc))) + + // Add a second host alongside the first. Both should be present, neither duplicated. + nc = map[string]any{ + "static_host_map": map[string]any{ + "10.128.0.2": []any{"2.2.2.2:4242"}, + "10.128.0.3": []any{"3.3.3.3:4242"}, + }, + } + rc, err = yaml.Marshal(nc) + require.NoError(t, err) + require.NoError(t, c.ReloadConfigString(string(rc))) + + rl = lh.Query(staticHost) + require.NotNil(t, rl) + assert.Same(t, pinned, rl, "adding a sibling entry must not displace the existing RemoteList") + assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, rl.CopyAddrs([]netip.Prefix{})) + + rl = lh.Query(otherHost) + require.NotNil(t, rl) + assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("3.3.3.3:4242")}, rl.CopyAddrs([]netip.Prefix{})) + + // Drop the first host entirely. The vpnAddr is no longer marked static, our owner + // contribution is cleared, but the addrMap entry stays in place so non-static cache + // data (from lighthouse queries) on the same RemoteList isn't lost. In-flight handshakes + // that already had the pointer see an empty address list rather than retrying stale ones. + nc = map[string]any{ + "static_host_map": map[string]any{ + "10.128.0.3": []any{"3.3.3.3:4242"}, + }, + } + rc, err = yaml.Marshal(nc) + require.NoError(t, err) + require.NoError(t, c.ReloadConfigString(string(rc))) + + _, isStatic := lh.GetStaticHostList()[staticHost] + assert.False(t, isStatic) + + rl = lh.Query(staticHost) + require.NotNil(t, rl) + assert.Same(t, pinned, rl) + assert.Empty(t, rl.CopyAddrs([]netip.Prefix{})) + + rl = lh.Query(otherHost) + require.NotNil(t, rl) + assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("3.3.3.3:4242")}, rl.CopyAddrs([]netip.Prefix{})) +} + +func newLHHostRequest(fromAddr netip.AddrPort, myVpnIp, queryVpnIp netip.Addr, lhh *LightHouseHandler) testLhReply { + req := &NebulaMeta{ + Type: NebulaMeta_HostQuery, + Details: &NebulaMetaDetails{}, + } + + if queryVpnIp.Is4() { + bip := queryVpnIp.As4() + req.Details.OldVpnAddr = binary.BigEndian.Uint32(bip[:]) + } else { + req.Details.VpnAddr = netAddrToProtoAddr(queryVpnIp) + } b, err := req.Marshal() if err != nil { @@ -302,21 +451,29 @@ func newLHHostRequest(fromAddr *udp.Addr, myVpnIp, queryVpnIp iputil.VpnIp, lhh w := &testEncWriter{ metaFilter: &filter, } - lhh.HandleRequest(fromAddr, myVpnIp, b, w) + lhh.HandleRequest(fromAddr, []netip.Addr{myVpnIp}, b, w) return w.lastReply } -func newLHHostUpdate(fromAddr *udp.Addr, vpnIp iputil.VpnIp, addrs []*udp.Addr, lhh *LightHouseHandler) { +func newLHHostUpdate(fromAddr netip.AddrPort, vpnIp netip.Addr, addrs []netip.AddrPort, lhh *LightHouseHandler) { req := &NebulaMeta{ - Type: NebulaMeta_HostUpdateNotification, - Details: &NebulaMetaDetails{ - VpnIp: uint32(vpnIp), - Ip4AndPorts: make([]*Ip4AndPort, len(addrs)), - }, + Type: NebulaMeta_HostUpdateNotification, + Details: &NebulaMetaDetails{}, } - for k, v := range addrs { - req.Details.Ip4AndPorts[k] = &Ip4AndPort{Ip: uint32(iputil.Ip2VpnIp(v.IP)), Port: uint32(v.Port)} + if vpnIp.Is4() { + bip := vpnIp.As4() + req.Details.OldVpnAddr = binary.BigEndian.Uint32(bip[:]) + } else { + req.Details.VpnAddr = netAddrToProtoAddr(vpnIp) + } + + for _, v := range addrs { + if v.Addr().Is4() { + req.Details.V4AddrPorts = append(req.Details.V4AddrPorts, netAddrToProtoV4AddrPort(v.Addr(), v.Port())) + } else { + req.Details.V6AddrPorts = append(req.Details.V6AddrPorts, netAddrToProtoV6AddrPort(v.Addr(), v.Port())) + } } b, err := req.Marshal() @@ -325,96 +482,25 @@ func newLHHostUpdate(fromAddr *udp.Addr, vpnIp iputil.VpnIp, addrs []*udp.Addr, } w := &testEncWriter{} - lhh.HandleRequest(fromAddr, vpnIp, b, w) -} - -//TODO: this is a RemoteList test -//func Test_lhRemoteAllowList(t *testing.T) { -// l := NewLogger() -// c := NewConfig(l) -// c.Settings["remoteallowlist"] = map[interface{}]interface{}{ -// "10.20.0.0/12": false, -// } -// allowList, err := c.GetAllowList("remoteallowlist", false) -// assert.Nil(t, err) -// -// lh1 := "10.128.0.2" -// lh1IP := net.ParseIP(lh1) -// -// udpServer, _ := NewListener(l, "0.0.0.0", 0, true) -// -// lh := NewLightHouse(l, true, &net.IPNet{IP: net.IP{0, 0, 0, 1}, Mask: net.IPMask{255, 255, 255, 0}}, []uint32{ip2int(lh1IP)}, 10, 10003, udpServer, false, 1, false) -// lh.SetRemoteAllowList(allowList) -// -// // A disallowed ip should not enter the cache but we should end up with an empty entry in the addrMap -// remote1IP := net.ParseIP("10.20.0.3") -// remotes := lh.unlockedGetRemoteList(ip2int(remote1IP)) -// remotes.unlockedPrependV4(ip2int(remote1IP), NewIp4AndPort(remote1IP, 4242)) -// assert.NotNil(t, lh.addrMap[ip2int(remote1IP)]) -// assert.Empty(t, lh.addrMap[ip2int(remote1IP)].CopyAddrs([]*net.IPNet{})) -// -// // Make sure a good ip enters the cache and addrMap -// remote2IP := net.ParseIP("10.128.0.3") -// remote2UDPAddr := NewUDPAddr(remote2IP, uint16(4242)) -// lh.addRemoteV4(ip2int(remote2IP), ip2int(remote2IP), NewIp4AndPort(remote2UDPAddr.IP, uint32(remote2UDPAddr.Port)), false, false) -// assertUdpAddrInArray(t, lh.addrMap[ip2int(remote2IP)].CopyAddrs([]*net.IPNet{}), remote2UDPAddr) -// -// // Another good ip gets into the cache, ordering is inverted -// remote3IP := net.ParseIP("10.128.0.4") -// remote3UDPAddr := NewUDPAddr(remote3IP, uint16(4243)) -// lh.addRemoteV4(ip2int(remote2IP), ip2int(remote2IP), NewIp4AndPort(remote3UDPAddr.IP, uint32(remote3UDPAddr.Port)), false, false) -// assertUdpAddrInArray(t, lh.addrMap[ip2int(remote2IP)].CopyAddrs([]*net.IPNet{}), remote2UDPAddr, remote3UDPAddr) -// -// // If we exceed the length limit we should only have the most recent addresses -// addedAddrs := []*udpAddr{} -// for i := 0; i < 11; i++ { -// remoteUDPAddr := NewUDPAddr(net.IP{10, 128, 0, 4}, uint16(4243+i)) -// lh.addRemoteV4(ip2int(remote2IP), ip2int(remote2IP), NewIp4AndPort(remoteUDPAddr.IP, uint32(remoteUDPAddr.Port)), false, false) -// // The first entry here is a duplicate, don't add it to the assert list -// if i != 0 { -// addedAddrs = append(addedAddrs, remoteUDPAddr) -// } -// } -// -// // We should only have the last 10 of what we tried to add -// assert.True(t, len(addedAddrs) >= 10, "We should have tried to add at least 10 addresses") -// assertUdpAddrInArray( -// t, -// lh.addrMap[ip2int(remote2IP)].CopyAddrs([]*net.IPNet{}), -// addedAddrs[0], -// addedAddrs[1], -// addedAddrs[2], -// addedAddrs[3], -// addedAddrs[4], -// addedAddrs[5], -// addedAddrs[6], -// addedAddrs[7], -// addedAddrs[8], -// addedAddrs[9], -// ) -//} - -func Test_ipMaskContains(t *testing.T) { - assert.True(t, ipMaskContains(iputil.Ip2VpnIp(net.ParseIP("10.0.0.1")), 32-24, iputil.Ip2VpnIp(net.ParseIP("10.0.0.255")))) - assert.False(t, ipMaskContains(iputil.Ip2VpnIp(net.ParseIP("10.0.0.1")), 32-24, iputil.Ip2VpnIp(net.ParseIP("10.0.1.1")))) - assert.True(t, ipMaskContains(iputil.Ip2VpnIp(net.ParseIP("10.0.0.1")), 32, iputil.Ip2VpnIp(net.ParseIP("10.0.1.1")))) + lhh.HandleRequest(fromAddr, []netip.Addr{vpnIp}, b, w) } type testLhReply struct { nebType header.MessageType nebSubType header.MessageSubType - vpnIp iputil.VpnIp + vpnIp netip.Addr msg *NebulaMeta } type testEncWriter struct { - lastReply testLhReply - metaFilter *NebulaMeta_MessageType + lastReply testLhReply + metaFilter *NebulaMeta_MessageType + protocolVersion cert.Version } func (tw *testEncWriter) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, nocopy bool) { } -func (tw *testEncWriter) Handshake(vpnIp iputil.VpnIp) { +func (tw *testEncWriter) Handshake(vpnIp netip.Addr) { } func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, _, _ []byte) { @@ -424,7 +510,7 @@ func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.M tw.lastReply = testLhReply{ nebType: t, nebSubType: st, - vpnIp: hostinfo.vpnIp, + vpnIp: hostinfo.vpnAddrs[0], msg: msg, } } @@ -434,7 +520,7 @@ func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.M } } -func (tw *testEncWriter) SendMessageToVpnIp(t header.MessageType, st header.MessageSubType, vpnIp iputil.VpnIp, p, _, _ []byte) { +func (tw *testEncWriter) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnIp netip.Addr, p, _, _ []byte) { msg := &NebulaMeta{} err := msg.Unmarshal(p) if tw.metaFilter == nil || msg.Type == *tw.metaFilter { @@ -451,36 +537,204 @@ func (tw *testEncWriter) SendMessageToVpnIp(t header.MessageType, st header.Mess } } +func (tw *testEncWriter) GetHostInfo(vpnIp netip.Addr) *HostInfo { + return nil +} + +func (tw *testEncWriter) GetCertState() *CertState { + return &CertState{initiatingVersion: tw.protocolVersion} +} + // assertIp4InArray asserts every address in want is at the same position in have and that the lengths match -func assertIp4InArray(t *testing.T, have []*Ip4AndPort, want ...*udp.Addr) { +func assertIp4InArray(t *testing.T, have []*V4AddrPort, want ...netip.AddrPort) { if !assert.Len(t, have, len(want)) { return } for k, w := range want { - if !(have[k].Ip == uint32(iputil.Ip2VpnIp(w.IP)) && have[k].Port == uint32(w.Port)) { - assert.Fail(t, fmt.Sprintf("Response did not contain: %v:%v at %v; %v", w.IP, w.Port, k, translateV4toUdpAddr(have))) + h := protoV4AddrPortToNetAddrPort(have[k]) + if !(h == w) { + assert.Fail(t, fmt.Sprintf("Response did not contain: %v at %v, found %v", w, k, h)) } } } -// assertUdpAddrInArray asserts every address in want is at the same position in have and that the lengths match -func assertUdpAddrInArray(t *testing.T, have []*udp.Addr, want ...*udp.Addr) { - if !assert.Len(t, have, len(want)) { - return +func Test_findNetworkUnion(t *testing.T) { + var out netip.Addr + var ok bool + + tenDot := netip.MustParsePrefix("10.0.0.0/8") + oneSevenTwo := netip.MustParsePrefix("172.16.0.0/16") + fe80 := netip.MustParsePrefix("fe80::/8") + fc00 := netip.MustParsePrefix("fc00::/7") + + a1 := netip.MustParseAddr("10.0.0.1") + afe81 := netip.MustParseAddr("fe80::1") + + //simple + out, ok = findNetworkUnion([]netip.Prefix{tenDot}, []netip.Addr{a1}) + assert.True(t, ok) + assert.Equal(t, out, a1) + + //mixed lengths + out, ok = findNetworkUnion([]netip.Prefix{tenDot}, []netip.Addr{a1, afe81}) + assert.True(t, ok) + assert.Equal(t, out, a1) + out, ok = findNetworkUnion([]netip.Prefix{tenDot, oneSevenTwo}, []netip.Addr{a1}) + assert.True(t, ok) + assert.Equal(t, out, a1) + + //mixed family + out, ok = findNetworkUnion([]netip.Prefix{tenDot, oneSevenTwo, fe80}, []netip.Addr{a1}) + assert.True(t, ok) + assert.Equal(t, out, a1) + out, ok = findNetworkUnion([]netip.Prefix{tenDot, oneSevenTwo, fe80}, []netip.Addr{a1, afe81}) + assert.True(t, ok) + assert.Equal(t, out, a1) + + //ordering + out, ok = findNetworkUnion([]netip.Prefix{tenDot, oneSevenTwo, fe80}, []netip.Addr{afe81, a1}) + assert.True(t, ok) + assert.Equal(t, out, a1) + out, ok = findNetworkUnion([]netip.Prefix{fe80, tenDot, oneSevenTwo}, []netip.Addr{afe81, a1}) + assert.True(t, ok) + assert.Equal(t, out, afe81) + + //some mismatches + out, ok = findNetworkUnion([]netip.Prefix{tenDot, oneSevenTwo, fe80}, []netip.Addr{afe81}) + assert.True(t, ok) + assert.Equal(t, out, afe81) + out, ok = findNetworkUnion([]netip.Prefix{oneSevenTwo, fe80}, []netip.Addr{a1, afe81}) + assert.True(t, ok) + assert.Equal(t, out, afe81) + + //falsey cases + out, ok = findNetworkUnion([]netip.Prefix{oneSevenTwo, fe80}, []netip.Addr{a1}) + assert.False(t, ok) + out, ok = findNetworkUnion([]netip.Prefix{fc00, fe80}, []netip.Addr{a1}) + assert.False(t, ok) + out, ok = findNetworkUnion([]netip.Prefix{oneSevenTwo, fc00}, []netip.Addr{a1, afe81}) + assert.False(t, ok) + out, ok = findNetworkUnion([]netip.Prefix{fc00}, []netip.Addr{a1, afe81}) + assert.False(t, ok) +} + +func TestLighthouse_Dont_Delete_Static_Hosts(t *testing.T) { + l := test.NewLogger() + + myUdpAddr2 := netip.MustParseAddrPort("1.2.3.4:4242") + + testSameHostNotStatic := netip.MustParseAddr("10.128.0.41") + testStaticHost := netip.MustParseAddr("10.128.0.42") + //myVpnIp := netip.MustParseAddr("10.128.0.2") + + c := config.NewC(l) + lh1 := "10.128.0.2" + c.Settings["lighthouse"] = map[string]any{ + "hosts": []any{lh1}, + "interval": "1s", } - for k, w := range want { - if !(have[k].IP.Equal(w.IP) && have[k].Port == w.Port) { - assert.Fail(t, fmt.Sprintf("Response did not contain: %v at %v; %v", w, k, have)) - } + c.Settings["listen"] = map[string]any{"port": 4242} + c.Settings["static_host_map"] = map[string]any{ + lh1: []any{"1.1.1.1:4242"}, + "10.128.0.42": []any{"1.2.3.4:4242"}, + } + + myVpnNet := netip.MustParsePrefix("10.128.0.1/24") + nt := new(bart.Lite) + nt.Insert(myVpnNet) + cs := &CertState{ + myVpnNetworks: []netip.Prefix{myVpnNet}, + myVpnNetworksTable: nt, } + lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil) + require.NoError(t, err) + lh.ifce = &mockEncWriter{} + + //test that we actually have the static entry: + out := lh.Query(testStaticHost) + assert.NotNil(t, out) + assert.Equal(t, out.vpnAddrs[0], testStaticHost) + out.Rebuild([]netip.Prefix{}) //why tho + assert.Equal(t, out.addrs[0], myUdpAddr2) + + //bolt on a lower numbered primary IP + am := lh.unlockedGetRemoteList([]netip.Addr{testStaticHost}) + am.vpnAddrs = []netip.Addr{testSameHostNotStatic, testStaticHost} + lh.addrMap[testSameHostNotStatic] = am + out.Rebuild([]netip.Prefix{}) //??? + + //test that we actually have the static entry: + out = lh.Query(testStaticHost) + assert.NotNil(t, out) + assert.Equal(t, out.vpnAddrs[0], testSameHostNotStatic) + assert.Equal(t, out.vpnAddrs[1], testStaticHost) + assert.Equal(t, out.addrs[0], myUdpAddr2) + + //test that we actually have the static entry for BOTH: + out2 := lh.Query(testSameHostNotStatic) + assert.Same(t, out2, out) + + //now do the delete + lh.DeleteVpnAddrs([]netip.Addr{testSameHostNotStatic, testStaticHost}) + //verify + out = lh.Query(testSameHostNotStatic) + assert.NotNil(t, out) + if out == nil { + t.Fatal("expected non-nil query for the static host") + } + assert.Equal(t, out.vpnAddrs[0], testSameHostNotStatic) + assert.Equal(t, out.vpnAddrs[1], testStaticHost) + assert.Equal(t, out.addrs[0], myUdpAddr2) } -func translateV4toUdpAddr(ips []*Ip4AndPort) []*udp.Addr { - addrs := make([]*udp.Addr, len(ips)) - for k, v := range ips { - addrs[k] = NewUDPAddrFromLH4(v) +func TestLighthouse_DeletesWork(t *testing.T) { + l := test.NewLogger() + + myUdpAddr2 := netip.MustParseAddrPort("1.2.3.4:4242") + testHost := netip.MustParseAddr("10.128.0.42") + + c := config.NewC(l) + lh1 := "10.128.0.2" + c.Settings["lighthouse"] = map[string]any{ + "hosts": []any{lh1}, + "interval": "1s", + } + + c.Settings["listen"] = map[string]any{"port": 4242} + c.Settings["static_host_map"] = map[string]any{ + lh1: []any{"1.1.1.1:4242"}, + } + + myVpnNet := netip.MustParsePrefix("10.128.0.1/24") + nt := new(bart.Lite) + nt.Insert(myVpnNet) + cs := &CertState{ + myVpnNetworks: []netip.Prefix{myVpnNet}, + myVpnNetworksTable: nt, } - return addrs + lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil) + require.NoError(t, err) + lh.ifce = &mockEncWriter{} + + //insert the host + am := lh.unlockedGetRemoteList([]netip.Addr{testHost}) + am.vpnAddrs = []netip.Addr{testHost} + am.addrs = []netip.AddrPort{myUdpAddr2} + lh.addrMap[testHost] = am + am.Rebuild([]netip.Prefix{}) //??? + + //test that we actually have the entry: + out := lh.Query(testHost) + assert.NotNil(t, out) + assert.Equal(t, out.vpnAddrs[0], testHost) + out.Rebuild([]netip.Prefix{}) //why tho + assert.Equal(t, out.addrs[0], myUdpAddr2) + + //now do the delete + lh.DeleteVpnAddrs([]netip.Addr{testHost}) + //verify + out = lh.Query(testHost) + assert.Nil(t, out) } diff --git a/logger.go b/logger.go deleted file mode 100644 index aaf6f29c..00000000 --- a/logger.go +++ /dev/null @@ -1,45 +0,0 @@ -package nebula - -import ( - "fmt" - "strings" - "time" - - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/config" -) - -func configLogger(l *logrus.Logger, c *config.C) error { - // set up our logging level - logLevel, err := logrus.ParseLevel(strings.ToLower(c.GetString("logging.level", "info"))) - if err != nil { - return fmt.Errorf("%s; possible levels: %s", err, logrus.AllLevels) - } - l.SetLevel(logLevel) - - disableTimestamp := c.GetBool("logging.disable_timestamp", false) - timestampFormat := c.GetString("logging.timestamp_format", "") - fullTimestamp := (timestampFormat != "") - if timestampFormat == "" { - timestampFormat = time.RFC3339 - } - - logFormat := strings.ToLower(c.GetString("logging.format", "text")) - switch logFormat { - case "text": - l.Formatter = &logrus.TextFormatter{ - TimestampFormat: timestampFormat, - FullTimestamp: fullTimestamp, - DisableTimestamp: disableTimestamp, - } - case "json": - l.Formatter = &logrus.JSONFormatter{ - TimestampFormat: timestampFormat, - DisableTimestamp: disableTimestamp, - } - default: - return fmt.Errorf("unknown log format `%s`. possible formats: %s", logFormat, []string{"text", "json"}) - } - - return nil -} diff --git a/logging/logger.go b/logging/logger.go new file mode 100644 index 00000000..bbc10bb3 --- /dev/null +++ b/logging/logger.go @@ -0,0 +1,233 @@ +// Package logging wires the nebula runtime-reconfigurable slog handler used +// by nebula.Main and the nebula CLI binaries. Callers build a logger with +// NewLogger, then call ApplyConfig at startup and from a config reload +// callback to push logging.level, logging.format, and +// logging.disable_timestamp changes onto the logger without rebuilding it. +package logging + +import ( + "context" + "fmt" + "io" + "log/slog" + "strings" + "sync/atomic" + "time" +) + +// Config is the subset of *config.C that ApplyConfig reads. Declaring it +// here keeps the logging package from depending on config directly, which +// would cycle through the shared test helpers (test.NewLogger imports +// logging, and config's tests import test). *config.C satisfies this +// interface structurally with no adapter. +type Config interface { + GetString(key, def string) string + GetBool(key string, def bool) bool +} + +// LevelTrace is a custom slog level below Debug, used when logging.level is +// "trace". slog has no builtin trace level; the value is one step below +// slog.LevelDebug in slog's 4-point spacing. +const LevelTrace = slog.Level(-8) + +// NewLogger returns a *slog.Logger whose level, format, and timestamp +// emission can be reconfigured at runtime via ApplyConfig and the SSH debug +// commands. The default configuration is info-level text output so log +// calls made before ApplyConfig runs still produce output. Timestamps +// follow slog's default RFC3339Nano format; set logging.disable_timestamp +// in config to suppress them. +// +// ApplyConfig and the SSH commands discover the reconfig surface via +// structural type-assertion on l.Handler(), so replacement implementations +// (tests, platform-specific sinks) need only implement the subset of +// {SetLevel(slog.Level), SetFormat(string) error, SetDisableTimestamp(bool)} +// they care about. Callers that pass a plain *slog.Logger without these +// methods get a silent no-op; reconfiguration is always opt-in. +func NewLogger(w io.Writer) *slog.Logger { + return slog.New(NewHandler(w)) +} + +// NewHandler builds the *Handler that NewLogger wraps. Exported for +// platform-specific sinks (notably cmd/nebula-service/logs_windows.go) +// that want to wrap the handler with extra behavior, such as tagging each +// record with its Event Log severity, while still benefiting from all the +// level / format / timestamp / WithAttrs machinery implemented here. +func NewHandler(w io.Writer) *Handler { + root := &handlerRoot{} + root.level.Set(slog.LevelInfo) + opts := &slog.HandlerOptions{Level: &root.level} + return &Handler{ + root: root, + text: slog.NewTextHandler(w, opts), + json: slog.NewJSONHandler(w, opts), + } +} + +// handlerRoot carries the reconfiguration state shared by every logger +// derived from a NewHandler call. All fields are consulted on the log +// path and updated lock-free. +type handlerRoot struct { + level slog.LevelVar + disableTimestamp atomic.Bool + // jsonMode picks which of the pre-derived inner handlers Handler.Handle + // dispatches to. Flipping it propagates instantly to every derived logger + // without rebuilding or chain-replaying anything. + jsonMode atomic.Bool +} + +// Handler is the slog.Handler returned by NewHandler. It holds two +// pre-derived slog handlers -- one text, one json -- both built from the +// same accumulated WithAttrs/WithGroup state. Handle picks which one to +// dispatch to based on handlerRoot.jsonMode, so a SetFormat call takes +// effect immediately across the whole process without having to rebuild +// any derived loggers. +type Handler struct { + root *handlerRoot + text slog.Handler + json slog.Handler +} + +func (h *Handler) Enabled(_ context.Context, l slog.Level) bool { + return h.root.level.Level() <= l +} + +func (h *Handler) Handle(ctx context.Context, r slog.Record) error { + if h.root.disableTimestamp.Load() { + r.Time = time.Time{} + } + if h.root.jsonMode.Load() { + return h.json.Handle(ctx, r) + } + return h.text.Handle(ctx, r) +} + +func (h *Handler) WithAttrs(attrs []slog.Attr) slog.Handler { + if len(attrs) == 0 { + return h + } + return &Handler{ + root: h.root, + text: h.text.WithAttrs(attrs), + json: h.json.WithAttrs(attrs), + } +} + +func (h *Handler) WithGroup(name string) slog.Handler { + if name == "" { + return h + } + return &Handler{ + root: h.root, + text: h.text.WithGroup(name), + json: h.json.WithGroup(name), + } +} + +// SetLevel updates the effective log level. Propagates to every derived +// logger via the shared LevelVar. +func (h *Handler) SetLevel(level slog.Level) { h.root.level.Set(level) } + +// GetLevel reports the current log level. +func (h *Handler) GetLevel() slog.Level { return h.root.level.Level() } + +// SetFormat flips the output format atomically. Valid formats are "text" +// and "json". Every derived logger sees the new format on its next Handle +// call; no rebuild or registration is required. +func (h *Handler) SetFormat(format string) error { + switch format { + case "text": + h.root.jsonMode.Store(false) + case "json": + h.root.jsonMode.Store(true) + default: + return fmt.Errorf("unknown log format `%s`. possible formats: %s", format, []string{"text", "json"}) + } + return nil +} + +// GetFormat reports the currently selected format name. +func (h *Handler) GetFormat() string { + if h.root.jsonMode.Load() { + return "json" + } + return "text" +} + +// SetDisableTimestamp toggles whether Handle zeroes r.Time before +// dispatching (slog's builtin text/json handlers skip emitting the time +// attribute on a zero time). +func (h *Handler) SetDisableTimestamp(v bool) { h.root.disableTimestamp.Store(v) } + +// ApplyConfig reads logging.level, logging.format, and (optionally) +// logging.disable_timestamp from c and applies them to l. The reconfig +// surface is discovered via structural type-assertion on l.Handler(), so +// foreign handlers silently opt out of whichever capabilities they do not +// implement. +// +// nebula.Main does NOT call this function on your behalf; callers that want +// config-driven log level / format / timestamp updates invoke it at +// startup and register it as a reload callback themselves. This keeps the +// library from mutating an embedder's logger without their say-so. +func ApplyConfig(l *slog.Logger, c Config) error { + h := l.Handler() + + lvl, err := ParseLevel(strings.ToLower(c.GetString("logging.level", "info"))) + if err != nil { + return err + } + if ls, ok := h.(interface{ SetLevel(slog.Level) }); ok { + ls.SetLevel(lvl) + } + + format := strings.ToLower(c.GetString("logging.format", "text")) + if fs, ok := h.(interface{ SetFormat(string) error }); ok { + if err := fs.SetFormat(format); err != nil { + return err + } + } + + if ts, ok := h.(interface{ SetDisableTimestamp(bool) }); ok { + ts.SetDisableTimestamp(c.GetBool("logging.disable_timestamp", false)) + } + return nil +} + +// ParseLevel converts a config-string level name ("trace", "debug", "info", +// "warn"/"warning", "error", "fatal"/"panic") to a slog.Level. "fatal" and +// "panic" are accepted for backwards compatibility with pre-slog configs +// and both map to slog.LevelError. +func ParseLevel(s string) (slog.Level, error) { + switch s { + case "trace": + return LevelTrace, nil + case "debug": + return slog.LevelDebug, nil + case "info": + return slog.LevelInfo, nil + case "warn", "warning": + return slog.LevelWarn, nil + case "error": + return slog.LevelError, nil + case "fatal", "panic": + return slog.LevelError, nil + default: + return 0, fmt.Errorf("not a valid logging level: %q", s) + } +} + +// LevelName returns a human-readable name for a slog.Level matching the +// strings accepted by ParseLevel. +func LevelName(l slog.Level) string { + switch { + case l <= LevelTrace: + return "trace" + case l <= slog.LevelDebug: + return "debug" + case l <= slog.LevelInfo: + return "info" + case l <= slog.LevelWarn: + return "warn" + default: + return "error" + } +} diff --git a/logging/logger_bench_test.go b/logging/logger_bench_test.go new file mode 100644 index 00000000..eb29c1c3 --- /dev/null +++ b/logging/logger_bench_test.go @@ -0,0 +1,90 @@ +package logging + +import ( + "context" + "io" + "log/slog" + "testing" +) + +// BenchmarkLogger_* compare the handler returned by NewLogger against a +// stock slog text handler. The key thing we care about is the per-log +// cost on a logger that has been derived via .With(), because that is the +// shape subsystems store on their structs (HostInfo.logger(), +// lh.l.With("subsystem", ...), etc.) and call from hot paths. + +func BenchmarkLogger_Stock_RootInfo(b *testing.B) { + l := slog.New(slog.DiscardHandler) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + l.Info("hello", "i", i) + } +} + +func BenchmarkLogger_Nebula_RootInfo(b *testing.B) { + l := NewLogger(io.Discard) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + l.Info("hello", "i", i) + } +} + +func BenchmarkLogger_Stock_DerivedInfo(b *testing.B) { + l := slog.New(slog.DiscardHandler).With( + "subsystem", "bench", + "localIndex", 1234, + ) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + l.Info("hello", "i", i) + } +} + +func BenchmarkLogger_Nebula_DerivedInfo(b *testing.B) { + l := NewLogger(io.Discard).With( + "subsystem", "bench", + "localIndex", 1234, + ) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + l.Info("hello", "i", i) + } +} + +// Gated-off-path benchmarks: mimic the typical hot-path shape +// `if l.Enabled(ctx, slog.LevelDebug) { ... }` where the log is gated below +// the active level. This is the dominant pattern in inside.go/outside.go and +// what we pay on every packet. +func BenchmarkLogger_Stock_DerivedEnabledGateMiss(b *testing.B) { + l := slog.New(slog.DiscardHandler).With( + "subsystem", "bench", + "localIndex", 1234, + ) + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if l.Enabled(ctx, slog.LevelDebug) { + l.Debug("hello", "i", i) + } + } +} + +func BenchmarkLogger_Nebula_DerivedEnabledGateMiss(b *testing.B) { + l := NewLogger(io.Discard).With( + "subsystem", "bench", + "localIndex", 1234, + ) + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if l.Enabled(ctx, slog.LevelDebug) { + l.Debug("hello", "i", i) + } + } +} diff --git a/main.go b/main.go index 8c94e80d..2ef2031b 100644 --- a/main.go +++ b/main.go @@ -2,23 +2,25 @@ package nebula import ( "context" - "encoding/binary" "fmt" + "log/slog" "net" + "net/netip" + "runtime/debug" + "strings" "time" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/sshd" "github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/util" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v3" ) -type m map[string]interface{} +type m = map[string]any -func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) { +func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) { ctx, cancel := context.WithCancel(context.Background()) // Automatically cancel the context if Main returns an error, to signal all created goroutines to quit. defer func() { @@ -27,9 +29,8 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg } }() - l := logger - l.Formatter = &logrus.TextFormatter{ - FullTimestamp: true, + if buildVersion == "" { + buildVersion = moduleVersion() } // Print the config if in test, the exit comes later @@ -40,37 +41,21 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg } // Print the final config - l.Println(string(b)) + l.Info(string(b)) } - err := configLogger(l, c) - if err != nil { - return nil, util.ContextualizeIfNeeded("Failed to configure the logger", err) - } - - c.RegisterReloadCallback(func(c *config.C) { - err := configLogger(l, c) - if err != nil { - l.WithError(err).Error("Failed to configure the logger") - } - }) - pki, err := NewPKIFromConfig(l, c) if err != nil { return nil, util.ContextualizeIfNeeded("Failed to load PKI from config", err) } - certificate := pki.GetCertState().Certificate - fw, err := NewFirewallFromConfig(l, certificate, c) + fw, err := NewFirewallFromConfig(l, pki.getCertState(), c) if err != nil { return nil, util.ContextualizeIfNeeded("Error while loading firewall rules", err) } - l.WithField("firewallHashes", fw.GetRuleHashes()).Info("Firewall started") - - // TODO: make sure mask is 4 bytes - tunCidr := certificate.Details.Ips[0] + l.Info("Firewall started", "firewallHashes", fw.GetRuleHashes()) - ssh, err := sshd.NewSSHServer(l.WithField("subsystem", "sshd")) + ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd")) if err != nil { return nil, util.ContextualizeIfNeeded("Error while creating SSH server", err) } @@ -79,7 +64,8 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg if c.GetBool("sshd.enabled", false) { sshStart, err = configSSH(l, ssh, c) if err != nil { - return nil, util.ContextualizeIfNeeded("Error while configuring the sshd", err) + l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err) + sshStart = nil } } @@ -96,19 +82,15 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg routines = 1 } if routines > 1 { - l.WithField("routines", routines).Info("Using multiple routines") + l.Info("Using multiple routines", "routines", routines) } } else { // deprecated and undocumented tunQueues := c.GetInt("tun.routines", 1) udpQueues := c.GetInt("listen.routines", 1) - if tunQueues > udpQueues { - routines = tunQueues - } else { - routines = udpQueues - } + routines = max(tunQueues, udpQueues) if routines != 1 { - l.WithField("routines", routines).Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead") + l.Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead", "routines", routines) } } @@ -121,7 +103,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg conntrackCacheTimeout = 1 * time.Second } if conntrackCacheTimeout > 0 { - l.WithField("duration", conntrackCacheTimeout).Info("Using routine-local conntrack cache") + l.Info("Using routine-local conntrack cache", "duration", conntrackCacheTimeout) } var tun overlay.Device @@ -132,7 +114,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg deviceFactory = overlay.NewDeviceFromConfig } - tun, err = deviceFactory(c, l, tunCidr, routines) + tun, err = deviceFactory(c, l, pki.getCertState().myVpnNetworks, routines) if err != nil { return nil, util.ContextualizeIfNeeded("Failed to get a tun/tap device", err) } @@ -148,23 +130,38 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg udpConns := make([]udp.Conn, routines) port := c.GetInt("listen.port", 0) + // Callers get no handle to these until the Control is returned, release them on any error. + defer func() { + if reterr != nil { + for _, u := range udpConns { + if u != nil { + _ = u.Close() + } + } + } + }() + if !configTest { rawListenHost := c.GetString("listen.host", "0.0.0.0") - var listenHost *net.IPAddr + var listenHost netip.Addr if rawListenHost == "[::]" { // Old guidance was to provide the literal `[::]` in `listen.host` but that won't resolve. - listenHost = &net.IPAddr{IP: net.IPv6zero} + listenHost = netip.IPv6Unspecified() } else { - listenHost, err = net.ResolveIPAddr("ip", rawListenHost) + ips, err := net.DefaultResolver.LookupNetIP(context.Background(), "ip", rawListenHost) if err != nil { return nil, util.ContextualizeIfNeeded("Failed to resolve listen.host", err) } + if len(ips) == 0 { + return nil, util.ContextualizeIfNeeded("Failed to resolve listen.host", err) + } + listenHost = ips[0].Unmap() } for i := 0; i < routines; i++ { - l.Infof("listening %q %d", listenHost.IP, port) - udpServer, err := udp.NewListener(l, listenHost.IP, port, routines > 1, c.GetInt("listen.batch", 64)) + l.Info("listening", "addr", netip.AddrPortFrom(listenHost, uint16(port))) + udpServer, err := udp.NewListener(l, listenHost, port, routines > 1, c.GetInt("listen.batch", 64)) if err != nil { return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err) } @@ -178,59 +175,15 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg if err != nil { return nil, util.NewContextualError("Failed to get listening port", nil, err) } - port = int(uPort.Port) + port = int(uPort.Port()) } } } - // Set up my internal host map - var preferredRanges []*net.IPNet - rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{}) - // First, check if 'preferred_ranges' is set and fallback to 'local_range' - if len(rawPreferredRanges) > 0 { - for _, rawPreferredRange := range rawPreferredRanges { - _, preferredRange, err := net.ParseCIDR(rawPreferredRange) - if err != nil { - return nil, util.ContextualizeIfNeeded("Failed to parse preferred ranges", err) - } - preferredRanges = append(preferredRanges, preferredRange) - } - } - - // local_range was superseded by preferred_ranges. If it is still present, - // merge the local_range setting into preferred_ranges. We will probably - // deprecate local_range and remove in the future. - rawLocalRange := c.GetString("local_range", "") - if rawLocalRange != "" { - _, localRange, err := net.ParseCIDR(rawLocalRange) - if err != nil { - return nil, util.ContextualizeIfNeeded("Failed to parse local_range", err) - } - - // Check if the entry for local_range was already specified in - // preferred_ranges. Don't put it into the slice twice if so. - var found bool - for _, r := range preferredRanges { - if r.String() == localRange.String() { - found = true - break - } - } - if !found { - preferredRanges = append(preferredRanges, localRange) - } - } - - hostMap := NewHostMap(l, tunCidr, preferredRanges) - hostMap.metricsEnabled = c.GetBool("stats.message_metrics", false) - - l. - WithField("network", hostMap.vpnCIDR.String()). - WithField("preferredRanges", hostMap.preferredRanges). - Info("Main HostMap created") - - punchy := NewPunchyFromConfig(l, c) - lightHouse, err := NewLightHouseFromConfig(ctx, l, c, tunCidr, udpConns[0], punchy) + hostMap := NewHostMapFromConfig(l, c) + punchy := NewPunchyFromConfig(l, c, udpConns[0]) + connManager := newConnectionManagerFromConfig(l, c, hostMap, punchy) + lightHouse, err := NewLightHouseFromConfig(ctx, l, c, pki.getCertState(), udpConns[0], punchy) if err != nil { return nil, util.ContextualizeIfNeeded("Failed to initialize lighthouse handler", err) } @@ -242,68 +195,45 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg messageMetrics = newMessageMetricsOnlyRecvError() } - useRelays := c.GetBool("relay.use_relays", DefaultUseRelays) && !c.GetBool("relay.am_relay", false) - handshakeConfig := HandshakeConfig{ - tryInterval: c.GetDuration("handshakes.try_interval", DefaultHandshakeTryInterval), - retries: c.GetInt("handshakes.retries", DefaultHandshakeRetries), - triggerBuffer: c.GetInt("handshakes.trigger_buffer", DefaultHandshakeTriggerBuffer), - useRelays: useRelays, - + tryInterval: c.GetDuration("handshakes.try_interval", DefaultHandshakeTryInterval), + retries: int64(c.GetInt("handshakes.retries", DefaultHandshakeRetries)), + triggerBuffer: c.GetInt("handshakes.trigger_buffer", DefaultHandshakeTriggerBuffer), messageMetrics: messageMetrics, } handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig) lightHouse.handshakeTrigger = handshakeManager.trigger - serveDns := false - if c.GetBool("lighthouse.serve_dns", false) { - if c.GetBool("lighthouse.am_lighthouse", false) { - serveDns = true - } else { - l.Warn("DNS server refusing to run because this host is not a lighthouse.") - } + ds, err := newDnsServerFromConfig(ctx, l, pki, hostMap, c) + if err != nil { + l.Warn("Failed to start DNS responder", "error", err) } - checkInterval := c.GetInt("timers.connection_alive_interval", 5) - pendingDeletionInterval := c.GetInt("timers.pending_deletion_interval", 10) - ifConfig := &InterfaceConfig{ - HostMap: hostMap, - Inside: tun, - Outside: udpConns[0], - pki: pki, - Cipher: c.GetString("cipher", "aes"), - Firewall: fw, - ServeDns: serveDns, - HandshakeManager: handshakeManager, - lightHouse: lightHouse, - checkInterval: time.Second * time.Duration(checkInterval), - pendingDeletionInterval: time.Second * time.Duration(pendingDeletionInterval), - tryPromoteEvery: c.GetUint32("counters.try_promote", defaultPromoteEvery), - reQueryEvery: c.GetUint32("counters.requery_every_packets", defaultReQueryEvery), - reQueryWait: c.GetDuration("timers.requery_wait_duration", defaultReQueryWait), - DropLocalBroadcast: c.GetBool("tun.drop_local_broadcast", false), - DropMulticast: c.GetBool("tun.drop_multicast", false), - routines: routines, - MessageMetrics: messageMetrics, - version: buildVersion, - relayManager: NewRelayManager(ctx, l, hostMap, c), - punchy: punchy, - + HostMap: hostMap, + Inside: tun, + Outside: udpConns[0], + pki: pki, + Firewall: fw, + DnsServer: ds, + HandshakeManager: handshakeManager, + connectionManager: connManager, + lightHouse: lightHouse, + tryPromoteEvery: c.GetUint32("counters.try_promote", defaultPromoteEvery), + reQueryEvery: c.GetUint32("counters.requery_every_packets", defaultReQueryEvery), + reQueryWait: c.GetDuration("timers.requery_wait_duration", defaultReQueryWait), + DropLocalBroadcast: c.GetBool("tun.drop_local_broadcast", false), + DropMulticast: c.GetBool("tun.drop_multicast", false), + routines: routines, + MessageMetrics: messageMetrics, + version: buildVersion, + relayManager: NewRelayManager(ctx, l, hostMap, c), + punchy: punchy, ConntrackCacheTimeout: conntrackCacheTimeout, l: l, } - switch ifConfig.Cipher { - case "aes": - noiseEndianness = binary.BigEndian - case "chachapoly": - noiseEndianness = binary.LittleEndian - default: - return nil, fmt.Errorf("unknown cipher: %v", ifConfig.Cipher) - } - var ifce *Interface if !configTest { ifce, err = NewInterface(ctx, ifConfig) @@ -311,22 +241,21 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg return nil, fmt.Errorf("failed to initialize interface: %s", err) } - // TODO: Better way to attach these, probably want a new interface in InterfaceConfig - // I don't want to make this initial commit too far-reaching though ifce.writers = udpConns lightHouse.ifce = ifce ifce.RegisterConfigChangeCallbacks(c) ifce.reloadDisconnectInvalid(c) ifce.reloadSendRecvError(c) + ifce.reloadAcceptRecvError(c) handshakeManager.f = ifce go handshakeManager.Run(ctx) + + punchy.Start(ctx, ifce, hostMap, lightHouse) } - // TODO - stats third-party modules start uncancellable goroutines. Update those libs to accept - // a context so that they can exit when the context is Done. - statsStart, err := startStats(l, c, buildVersion, configTest) + stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest) if err != nil { return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err) } @@ -335,26 +264,38 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg return nil, nil } - //TODO: check if we _should_ be emitting stats go ifce.emitStats(ctx, c.GetDuration("stats.interval", time.Second*10)) attachCommands(l, c, ssh, ifce) - // Start DNS server last to allow using the nebula IP as lighthouse.dns.host - var dnsStart func() - if lightHouse.amLighthouse && serveDns { - l.Debugln("Starting dns server") - dnsStart = dnsMain(l, hostMap, c) - } + networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c) return &Control{ - ifce, - l, - ctx, - cancel, - sshStart, - statsStart, - dnsStart, - lightHouse.StartUpdateWorker, + state: StateReady, + f: ifce, + l: l, + ctx: ctx, + cancel: cancel, + sshStart: sshStart, + statsStart: stats.Start, + dnsStart: ds.Start, + lighthouseStart: lightHouse.StartUpdateWorker, + networkChangeStart: networkChanges.Start, + connectionManagerStart: connManager.Start, }, nil } + +func moduleVersion() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "" + } + + for _, dep := range info.Deps { + if dep.Path == "github.com/slackhq/nebula" { + return strings.TrimPrefix(dep.Version, "v") + } + } + + return "" +} diff --git a/message_metrics.go b/message_metrics.go index 94bb02fe..45de9a5c 100644 --- a/message_metrics.go +++ b/message_metrics.go @@ -7,14 +7,14 @@ import ( "github.com/slackhq/nebula/header" ) -//TODO: this can probably move into the header package - type MessageMetrics struct { rx [][]metrics.Counter tx [][]metrics.Counter rxUnknown metrics.Counter txUnknown metrics.Counter + + rxInvalid metrics.Counter } func (m *MessageMetrics) Rx(t header.MessageType, s header.MessageSubType, i int64) { @@ -35,6 +35,11 @@ func (m *MessageMetrics) Tx(t header.MessageType, s header.MessageSubType, i int } } } +func (m *MessageMetrics) RxInvalid(i int64) { + if m != nil && m.rxInvalid != nil { + m.rxInvalid.Inc(i) + } +} func newMessageMetrics() *MessageMetrics { gen := func(t string) [][]metrics.Counter { @@ -58,6 +63,7 @@ func newMessageMetrics() *MessageMetrics { rxUnknown: metrics.GetOrRegisterCounter("messages.rx.other", nil), txUnknown: metrics.GetOrRegisterCounter("messages.tx.other", nil), + rxInvalid: metrics.GetOrRegisterCounter("messages.rx.invalid", nil), } } diff --git a/metadata.go b/metadata.go deleted file mode 100644 index 6a023abf..00000000 --- a/metadata.go +++ /dev/null @@ -1,18 +0,0 @@ -package nebula - -/* - -import ( - proto "google.golang.org/protobuf/proto" -) - -func HandleMetaProto(p []byte) { - m := &NebulaMeta{} - err := proto.Unmarshal(p, m) - if err != nil { - l.Debugf("problem unmarshaling meta message: %s", err) - } - //fmt.Println(m) -} - -*/ diff --git a/nebula.pb.go b/nebula.pb.go index b3c723a4..94a4ebe2 100644 --- a/nebula.pb.go +++ b/nebula.pb.go @@ -96,7 +96,7 @@ func (x NebulaPing_MessageType) String() string { } func (NebulaPing_MessageType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_2d65afa7693df5ef, []int{4, 0} + return fileDescriptor_2d65afa7693df5ef, []int{5, 0} } type NebulaControl_MessageType int32 @@ -124,7 +124,7 @@ func (x NebulaControl_MessageType) String() string { } func (NebulaControl_MessageType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_2d65afa7693df5ef, []int{7, 0} + return fileDescriptor_2d65afa7693df5ef, []int{6, 0} } type NebulaMeta struct { @@ -180,11 +180,13 @@ func (m *NebulaMeta) GetDetails() *NebulaMetaDetails { } type NebulaMetaDetails struct { - VpnIp uint32 `protobuf:"varint,1,opt,name=VpnIp,proto3" json:"VpnIp,omitempty"` - Ip4AndPorts []*Ip4AndPort `protobuf:"bytes,2,rep,name=Ip4AndPorts,proto3" json:"Ip4AndPorts,omitempty"` - Ip6AndPorts []*Ip6AndPort `protobuf:"bytes,4,rep,name=Ip6AndPorts,proto3" json:"Ip6AndPorts,omitempty"` - RelayVpnIp []uint32 `protobuf:"varint,5,rep,packed,name=RelayVpnIp,proto3" json:"RelayVpnIp,omitempty"` - Counter uint32 `protobuf:"varint,3,opt,name=counter,proto3" json:"counter,omitempty"` + OldVpnAddr uint32 `protobuf:"varint,1,opt,name=OldVpnAddr,proto3" json:"OldVpnAddr,omitempty"` // Deprecated: Do not use. + VpnAddr *Addr `protobuf:"bytes,6,opt,name=VpnAddr,proto3" json:"VpnAddr,omitempty"` + OldRelayVpnAddrs []uint32 `protobuf:"varint,5,rep,packed,name=OldRelayVpnAddrs,proto3" json:"OldRelayVpnAddrs,omitempty"` // Deprecated: Do not use. + RelayVpnAddrs []*Addr `protobuf:"bytes,7,rep,name=RelayVpnAddrs,proto3" json:"RelayVpnAddrs,omitempty"` + V4AddrPorts []*V4AddrPort `protobuf:"bytes,2,rep,name=V4AddrPorts,proto3" json:"V4AddrPorts,omitempty"` + V6AddrPorts []*V6AddrPort `protobuf:"bytes,4,rep,name=V6AddrPorts,proto3" json:"V6AddrPorts,omitempty"` + Counter uint32 `protobuf:"varint,3,opt,name=counter,proto3" json:"counter,omitempty"` } func (m *NebulaMetaDetails) Reset() { *m = NebulaMetaDetails{} } @@ -220,30 +222,46 @@ func (m *NebulaMetaDetails) XXX_DiscardUnknown() { var xxx_messageInfo_NebulaMetaDetails proto.InternalMessageInfo -func (m *NebulaMetaDetails) GetVpnIp() uint32 { +// Deprecated: Do not use. +func (m *NebulaMetaDetails) GetOldVpnAddr() uint32 { if m != nil { - return m.VpnIp + return m.OldVpnAddr } return 0 } -func (m *NebulaMetaDetails) GetIp4AndPorts() []*Ip4AndPort { +func (m *NebulaMetaDetails) GetVpnAddr() *Addr { if m != nil { - return m.Ip4AndPorts + return m.VpnAddr } return nil } -func (m *NebulaMetaDetails) GetIp6AndPorts() []*Ip6AndPort { +// Deprecated: Do not use. +func (m *NebulaMetaDetails) GetOldRelayVpnAddrs() []uint32 { if m != nil { - return m.Ip6AndPorts + return m.OldRelayVpnAddrs } return nil } -func (m *NebulaMetaDetails) GetRelayVpnIp() []uint32 { +func (m *NebulaMetaDetails) GetRelayVpnAddrs() []*Addr { if m != nil { - return m.RelayVpnIp + return m.RelayVpnAddrs + } + return nil +} + +func (m *NebulaMetaDetails) GetV4AddrPorts() []*V4AddrPort { + if m != nil { + return m.V4AddrPorts + } + return nil +} + +func (m *NebulaMetaDetails) GetV6AddrPorts() []*V6AddrPort { + if m != nil { + return m.V6AddrPorts } return nil } @@ -255,23 +273,23 @@ func (m *NebulaMetaDetails) GetCounter() uint32 { return 0 } -type Ip4AndPort struct { - Ip uint32 `protobuf:"varint,1,opt,name=Ip,proto3" json:"Ip,omitempty"` - Port uint32 `protobuf:"varint,2,opt,name=Port,proto3" json:"Port,omitempty"` +type Addr struct { + Hi uint64 `protobuf:"varint,1,opt,name=Hi,proto3" json:"Hi,omitempty"` + Lo uint64 `protobuf:"varint,2,opt,name=Lo,proto3" json:"Lo,omitempty"` } -func (m *Ip4AndPort) Reset() { *m = Ip4AndPort{} } -func (m *Ip4AndPort) String() string { return proto.CompactTextString(m) } -func (*Ip4AndPort) ProtoMessage() {} -func (*Ip4AndPort) Descriptor() ([]byte, []int) { +func (m *Addr) Reset() { *m = Addr{} } +func (m *Addr) String() string { return proto.CompactTextString(m) } +func (*Addr) ProtoMessage() {} +func (*Addr) Descriptor() ([]byte, []int) { return fileDescriptor_2d65afa7693df5ef, []int{2} } -func (m *Ip4AndPort) XXX_Unmarshal(b []byte) error { +func (m *Addr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *Ip4AndPort) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *Addr) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_Ip4AndPort.Marshal(b, m, deterministic) + return xxx_messageInfo_Addr.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -281,50 +299,49 @@ func (m *Ip4AndPort) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *Ip4AndPort) XXX_Merge(src proto.Message) { - xxx_messageInfo_Ip4AndPort.Merge(m, src) +func (m *Addr) XXX_Merge(src proto.Message) { + xxx_messageInfo_Addr.Merge(m, src) } -func (m *Ip4AndPort) XXX_Size() int { +func (m *Addr) XXX_Size() int { return m.Size() } -func (m *Ip4AndPort) XXX_DiscardUnknown() { - xxx_messageInfo_Ip4AndPort.DiscardUnknown(m) +func (m *Addr) XXX_DiscardUnknown() { + xxx_messageInfo_Addr.DiscardUnknown(m) } -var xxx_messageInfo_Ip4AndPort proto.InternalMessageInfo +var xxx_messageInfo_Addr proto.InternalMessageInfo -func (m *Ip4AndPort) GetIp() uint32 { +func (m *Addr) GetHi() uint64 { if m != nil { - return m.Ip + return m.Hi } return 0 } -func (m *Ip4AndPort) GetPort() uint32 { +func (m *Addr) GetLo() uint64 { if m != nil { - return m.Port + return m.Lo } return 0 } -type Ip6AndPort struct { - Hi uint64 `protobuf:"varint,1,opt,name=Hi,proto3" json:"Hi,omitempty"` - Lo uint64 `protobuf:"varint,2,opt,name=Lo,proto3" json:"Lo,omitempty"` - Port uint32 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` +type V4AddrPort struct { + Addr uint32 `protobuf:"varint,1,opt,name=Addr,proto3" json:"Addr,omitempty"` + Port uint32 `protobuf:"varint,2,opt,name=Port,proto3" json:"Port,omitempty"` } -func (m *Ip6AndPort) Reset() { *m = Ip6AndPort{} } -func (m *Ip6AndPort) String() string { return proto.CompactTextString(m) } -func (*Ip6AndPort) ProtoMessage() {} -func (*Ip6AndPort) Descriptor() ([]byte, []int) { +func (m *V4AddrPort) Reset() { *m = V4AddrPort{} } +func (m *V4AddrPort) String() string { return proto.CompactTextString(m) } +func (*V4AddrPort) ProtoMessage() {} +func (*V4AddrPort) Descriptor() ([]byte, []int) { return fileDescriptor_2d65afa7693df5ef, []int{3} } -func (m *Ip6AndPort) XXX_Unmarshal(b []byte) error { +func (m *V4AddrPort) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *Ip6AndPort) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *V4AddrPort) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_Ip6AndPort.Marshal(b, m, deterministic) + return xxx_messageInfo_V4AddrPort.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -334,56 +351,50 @@ func (m *Ip6AndPort) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *Ip6AndPort) XXX_Merge(src proto.Message) { - xxx_messageInfo_Ip6AndPort.Merge(m, src) +func (m *V4AddrPort) XXX_Merge(src proto.Message) { + xxx_messageInfo_V4AddrPort.Merge(m, src) } -func (m *Ip6AndPort) XXX_Size() int { +func (m *V4AddrPort) XXX_Size() int { return m.Size() } -func (m *Ip6AndPort) XXX_DiscardUnknown() { - xxx_messageInfo_Ip6AndPort.DiscardUnknown(m) +func (m *V4AddrPort) XXX_DiscardUnknown() { + xxx_messageInfo_V4AddrPort.DiscardUnknown(m) } -var xxx_messageInfo_Ip6AndPort proto.InternalMessageInfo - -func (m *Ip6AndPort) GetHi() uint64 { - if m != nil { - return m.Hi - } - return 0 -} +var xxx_messageInfo_V4AddrPort proto.InternalMessageInfo -func (m *Ip6AndPort) GetLo() uint64 { +func (m *V4AddrPort) GetAddr() uint32 { if m != nil { - return m.Lo + return m.Addr } return 0 } -func (m *Ip6AndPort) GetPort() uint32 { +func (m *V4AddrPort) GetPort() uint32 { if m != nil { return m.Port } return 0 } -type NebulaPing struct { - Type NebulaPing_MessageType `protobuf:"varint,1,opt,name=Type,proto3,enum=nebula.NebulaPing_MessageType" json:"Type,omitempty"` - Time uint64 `protobuf:"varint,2,opt,name=Time,proto3" json:"Time,omitempty"` +type V6AddrPort struct { + Hi uint64 `protobuf:"varint,1,opt,name=Hi,proto3" json:"Hi,omitempty"` + Lo uint64 `protobuf:"varint,2,opt,name=Lo,proto3" json:"Lo,omitempty"` + Port uint32 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` } -func (m *NebulaPing) Reset() { *m = NebulaPing{} } -func (m *NebulaPing) String() string { return proto.CompactTextString(m) } -func (*NebulaPing) ProtoMessage() {} -func (*NebulaPing) Descriptor() ([]byte, []int) { +func (m *V6AddrPort) Reset() { *m = V6AddrPort{} } +func (m *V6AddrPort) String() string { return proto.CompactTextString(m) } +func (*V6AddrPort) ProtoMessage() {} +func (*V6AddrPort) Descriptor() ([]byte, []int) { return fileDescriptor_2d65afa7693df5ef, []int{4} } -func (m *NebulaPing) XXX_Unmarshal(b []byte) error { +func (m *V6AddrPort) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *NebulaPing) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *V6AddrPort) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_NebulaPing.Marshal(b, m, deterministic) + return xxx_messageInfo_V6AddrPort.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -393,104 +404,56 @@ func (m *NebulaPing) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *NebulaPing) XXX_Merge(src proto.Message) { - xxx_messageInfo_NebulaPing.Merge(m, src) +func (m *V6AddrPort) XXX_Merge(src proto.Message) { + xxx_messageInfo_V6AddrPort.Merge(m, src) } -func (m *NebulaPing) XXX_Size() int { +func (m *V6AddrPort) XXX_Size() int { return m.Size() } -func (m *NebulaPing) XXX_DiscardUnknown() { - xxx_messageInfo_NebulaPing.DiscardUnknown(m) +func (m *V6AddrPort) XXX_DiscardUnknown() { + xxx_messageInfo_V6AddrPort.DiscardUnknown(m) } -var xxx_messageInfo_NebulaPing proto.InternalMessageInfo - -func (m *NebulaPing) GetType() NebulaPing_MessageType { - if m != nil { - return m.Type - } - return NebulaPing_Ping -} +var xxx_messageInfo_V6AddrPort proto.InternalMessageInfo -func (m *NebulaPing) GetTime() uint64 { +func (m *V6AddrPort) GetHi() uint64 { if m != nil { - return m.Time + return m.Hi } return 0 } -type NebulaHandshake struct { - Details *NebulaHandshakeDetails `protobuf:"bytes,1,opt,name=Details,proto3" json:"Details,omitempty"` - Hmac []byte `protobuf:"bytes,2,opt,name=Hmac,proto3" json:"Hmac,omitempty"` -} - -func (m *NebulaHandshake) Reset() { *m = NebulaHandshake{} } -func (m *NebulaHandshake) String() string { return proto.CompactTextString(m) } -func (*NebulaHandshake) ProtoMessage() {} -func (*NebulaHandshake) Descriptor() ([]byte, []int) { - return fileDescriptor_2d65afa7693df5ef, []int{5} -} -func (m *NebulaHandshake) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NebulaHandshake) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NebulaHandshake.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *NebulaHandshake) XXX_Merge(src proto.Message) { - xxx_messageInfo_NebulaHandshake.Merge(m, src) -} -func (m *NebulaHandshake) XXX_Size() int { - return m.Size() -} -func (m *NebulaHandshake) XXX_DiscardUnknown() { - xxx_messageInfo_NebulaHandshake.DiscardUnknown(m) -} - -var xxx_messageInfo_NebulaHandshake proto.InternalMessageInfo - -func (m *NebulaHandshake) GetDetails() *NebulaHandshakeDetails { +func (m *V6AddrPort) GetLo() uint64 { if m != nil { - return m.Details + return m.Lo } - return nil + return 0 } -func (m *NebulaHandshake) GetHmac() []byte { +func (m *V6AddrPort) GetPort() uint32 { if m != nil { - return m.Hmac + return m.Port } - return nil + return 0 } -type NebulaHandshakeDetails struct { - Cert []byte `protobuf:"bytes,1,opt,name=Cert,proto3" json:"Cert,omitempty"` - InitiatorIndex uint32 `protobuf:"varint,2,opt,name=InitiatorIndex,proto3" json:"InitiatorIndex,omitempty"` - ResponderIndex uint32 `protobuf:"varint,3,opt,name=ResponderIndex,proto3" json:"ResponderIndex,omitempty"` - Cookie uint64 `protobuf:"varint,4,opt,name=Cookie,proto3" json:"Cookie,omitempty"` - Time uint64 `protobuf:"varint,5,opt,name=Time,proto3" json:"Time,omitempty"` +type NebulaPing struct { + Type NebulaPing_MessageType `protobuf:"varint,1,opt,name=Type,proto3,enum=nebula.NebulaPing_MessageType" json:"Type,omitempty"` + Time uint64 `protobuf:"varint,2,opt,name=Time,proto3" json:"Time,omitempty"` } -func (m *NebulaHandshakeDetails) Reset() { *m = NebulaHandshakeDetails{} } -func (m *NebulaHandshakeDetails) String() string { return proto.CompactTextString(m) } -func (*NebulaHandshakeDetails) ProtoMessage() {} -func (*NebulaHandshakeDetails) Descriptor() ([]byte, []int) { - return fileDescriptor_2d65afa7693df5ef, []int{6} +func (m *NebulaPing) Reset() { *m = NebulaPing{} } +func (m *NebulaPing) String() string { return proto.CompactTextString(m) } +func (*NebulaPing) ProtoMessage() {} +func (*NebulaPing) Descriptor() ([]byte, []int) { + return fileDescriptor_2d65afa7693df5ef, []int{5} } -func (m *NebulaHandshakeDetails) XXX_Unmarshal(b []byte) error { +func (m *NebulaPing) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *NebulaHandshakeDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *NebulaPing) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_NebulaHandshakeDetails.Marshal(b, m, deterministic) + return xxx_messageInfo_NebulaPing.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -500,47 +463,26 @@ func (m *NebulaHandshakeDetails) XXX_Marshal(b []byte, deterministic bool) ([]by return b[:n], nil } } -func (m *NebulaHandshakeDetails) XXX_Merge(src proto.Message) { - xxx_messageInfo_NebulaHandshakeDetails.Merge(m, src) +func (m *NebulaPing) XXX_Merge(src proto.Message) { + xxx_messageInfo_NebulaPing.Merge(m, src) } -func (m *NebulaHandshakeDetails) XXX_Size() int { +func (m *NebulaPing) XXX_Size() int { return m.Size() } -func (m *NebulaHandshakeDetails) XXX_DiscardUnknown() { - xxx_messageInfo_NebulaHandshakeDetails.DiscardUnknown(m) -} - -var xxx_messageInfo_NebulaHandshakeDetails proto.InternalMessageInfo - -func (m *NebulaHandshakeDetails) GetCert() []byte { - if m != nil { - return m.Cert - } - return nil -} - -func (m *NebulaHandshakeDetails) GetInitiatorIndex() uint32 { - if m != nil { - return m.InitiatorIndex - } - return 0 +func (m *NebulaPing) XXX_DiscardUnknown() { + xxx_messageInfo_NebulaPing.DiscardUnknown(m) } -func (m *NebulaHandshakeDetails) GetResponderIndex() uint32 { - if m != nil { - return m.ResponderIndex - } - return 0 -} +var xxx_messageInfo_NebulaPing proto.InternalMessageInfo -func (m *NebulaHandshakeDetails) GetCookie() uint64 { +func (m *NebulaPing) GetType() NebulaPing_MessageType { if m != nil { - return m.Cookie + return m.Type } - return 0 + return NebulaPing_Ping } -func (m *NebulaHandshakeDetails) GetTime() uint64 { +func (m *NebulaPing) GetTime() uint64 { if m != nil { return m.Time } @@ -551,15 +493,17 @@ type NebulaControl struct { Type NebulaControl_MessageType `protobuf:"varint,1,opt,name=Type,proto3,enum=nebula.NebulaControl_MessageType" json:"Type,omitempty"` InitiatorRelayIndex uint32 `protobuf:"varint,2,opt,name=InitiatorRelayIndex,proto3" json:"InitiatorRelayIndex,omitempty"` ResponderRelayIndex uint32 `protobuf:"varint,3,opt,name=ResponderRelayIndex,proto3" json:"ResponderRelayIndex,omitempty"` - RelayToIp uint32 `protobuf:"varint,4,opt,name=RelayToIp,proto3" json:"RelayToIp,omitempty"` - RelayFromIp uint32 `protobuf:"varint,5,opt,name=RelayFromIp,proto3" json:"RelayFromIp,omitempty"` + OldRelayToAddr uint32 `protobuf:"varint,4,opt,name=OldRelayToAddr,proto3" json:"OldRelayToAddr,omitempty"` // Deprecated: Do not use. + OldRelayFromAddr uint32 `protobuf:"varint,5,opt,name=OldRelayFromAddr,proto3" json:"OldRelayFromAddr,omitempty"` // Deprecated: Do not use. + RelayToAddr *Addr `protobuf:"bytes,6,opt,name=RelayToAddr,proto3" json:"RelayToAddr,omitempty"` + RelayFromAddr *Addr `protobuf:"bytes,7,opt,name=RelayFromAddr,proto3" json:"RelayFromAddr,omitempty"` } func (m *NebulaControl) Reset() { *m = NebulaControl{} } func (m *NebulaControl) String() string { return proto.CompactTextString(m) } func (*NebulaControl) ProtoMessage() {} func (*NebulaControl) Descriptor() ([]byte, []int) { - return fileDescriptor_2d65afa7693df5ef, []int{7} + return fileDescriptor_2d65afa7693df5ef, []int{6} } func (m *NebulaControl) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -609,83 +553,95 @@ func (m *NebulaControl) GetResponderRelayIndex() uint32 { return 0 } -func (m *NebulaControl) GetRelayToIp() uint32 { +// Deprecated: Do not use. +func (m *NebulaControl) GetOldRelayToAddr() uint32 { if m != nil { - return m.RelayToIp + return m.OldRelayToAddr } return 0 } -func (m *NebulaControl) GetRelayFromIp() uint32 { +// Deprecated: Do not use. +func (m *NebulaControl) GetOldRelayFromAddr() uint32 { if m != nil { - return m.RelayFromIp + return m.OldRelayFromAddr } return 0 } +func (m *NebulaControl) GetRelayToAddr() *Addr { + if m != nil { + return m.RelayToAddr + } + return nil +} + +func (m *NebulaControl) GetRelayFromAddr() *Addr { + if m != nil { + return m.RelayFromAddr + } + return nil +} + func init() { proto.RegisterEnum("nebula.NebulaMeta_MessageType", NebulaMeta_MessageType_name, NebulaMeta_MessageType_value) proto.RegisterEnum("nebula.NebulaPing_MessageType", NebulaPing_MessageType_name, NebulaPing_MessageType_value) proto.RegisterEnum("nebula.NebulaControl_MessageType", NebulaControl_MessageType_name, NebulaControl_MessageType_value) proto.RegisterType((*NebulaMeta)(nil), "nebula.NebulaMeta") proto.RegisterType((*NebulaMetaDetails)(nil), "nebula.NebulaMetaDetails") - proto.RegisterType((*Ip4AndPort)(nil), "nebula.Ip4AndPort") - proto.RegisterType((*Ip6AndPort)(nil), "nebula.Ip6AndPort") + proto.RegisterType((*Addr)(nil), "nebula.Addr") + proto.RegisterType((*V4AddrPort)(nil), "nebula.V4AddrPort") + proto.RegisterType((*V6AddrPort)(nil), "nebula.V6AddrPort") proto.RegisterType((*NebulaPing)(nil), "nebula.NebulaPing") - proto.RegisterType((*NebulaHandshake)(nil), "nebula.NebulaHandshake") - proto.RegisterType((*NebulaHandshakeDetails)(nil), "nebula.NebulaHandshakeDetails") proto.RegisterType((*NebulaControl)(nil), "nebula.NebulaControl") } func init() { proto.RegisterFile("nebula.proto", fileDescriptor_2d65afa7693df5ef) } var fileDescriptor_2d65afa7693df5ef = []byte{ - // 707 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x54, 0x4d, 0x6f, 0xda, 0x4a, - 0x14, 0xc5, 0xc6, 0x7c, 0x5d, 0x02, 0xf1, 0xbb, 0x79, 0x8f, 0x07, 0x4f, 0xaf, 0x16, 0xf5, 0xa2, - 0x62, 0x45, 0x22, 0x92, 0x46, 0x5d, 0x36, 0xa5, 0xaa, 0x20, 0x4a, 0x22, 0x3a, 0x4a, 0x5b, 0xa9, - 0x9b, 0x6a, 0x62, 0xa6, 0xc1, 0x02, 0x3c, 0x8e, 0x3d, 0x54, 0xe1, 0x5f, 0xf4, 0xc7, 0xe4, 0x47, - 0x74, 0xd7, 0x2c, 0xbb, 0xac, 0x92, 0x65, 0x97, 0xfd, 0x03, 0xd5, 0x8c, 0xc1, 0x36, 0x84, 0x76, - 0x37, 0xe7, 0xde, 0x73, 0x66, 0xce, 0x9c, 0xb9, 0x36, 0x6c, 0x79, 0xec, 0x62, 0x36, 0xa1, 0x6d, - 0x3f, 0xe0, 0x82, 0x63, 0x3e, 0x42, 0xf6, 0x0f, 0x1d, 0xe0, 0x4c, 0x2d, 0x4f, 0x99, 0xa0, 0xd8, - 0x01, 0xe3, 0x7c, 0xee, 0xb3, 0xba, 0xd6, 0xd4, 0x5a, 0xd5, 0x8e, 0xd5, 0x5e, 0x68, 0x12, 0x46, - 0xfb, 0x94, 0x85, 0x21, 0xbd, 0x64, 0x92, 0x45, 0x14, 0x17, 0xf7, 0xa1, 0xf0, 0x92, 0x09, 0xea, - 0x4e, 0xc2, 0xba, 0xde, 0xd4, 0x5a, 0xe5, 0x4e, 0xe3, 0xa1, 0x6c, 0x41, 0x20, 0x4b, 0xa6, 0xfd, - 0x53, 0x83, 0x72, 0x6a, 0x2b, 0x2c, 0x82, 0x71, 0xc6, 0x3d, 0x66, 0x66, 0xb0, 0x02, 0xa5, 0x1e, - 0x0f, 0xc5, 0xeb, 0x19, 0x0b, 0xe6, 0xa6, 0x86, 0x08, 0xd5, 0x18, 0x12, 0xe6, 0x4f, 0xe6, 0xa6, - 0x8e, 0xff, 0x41, 0x4d, 0xd6, 0xde, 0xf8, 0x43, 0x2a, 0xd8, 0x19, 0x17, 0xee, 0x47, 0xd7, 0xa1, - 0xc2, 0xe5, 0x9e, 0x99, 0xc5, 0x06, 0xfc, 0x23, 0x7b, 0xa7, 0xfc, 0x13, 0x1b, 0xae, 0xb4, 0x8c, - 0x65, 0x6b, 0x30, 0xf3, 0x9c, 0xd1, 0x4a, 0x2b, 0x87, 0x55, 0x00, 0xd9, 0x7a, 0x37, 0xe2, 0x74, - 0xea, 0x9a, 0x79, 0xdc, 0x81, 0xed, 0x04, 0x47, 0xc7, 0x16, 0xa4, 0xb3, 0x01, 0x15, 0xa3, 0xee, - 0x88, 0x39, 0x63, 0xb3, 0x28, 0x9d, 0xc5, 0x30, 0xa2, 0x94, 0xf0, 0x11, 0x34, 0x36, 0x3b, 0x3b, - 0x72, 0xc6, 0x26, 0xd8, 0x5f, 0x35, 0xf8, 0xeb, 0x41, 0x28, 0xf8, 0x37, 0xe4, 0xde, 0xfa, 0x5e, - 0xdf, 0x57, 0xa9, 0x57, 0x48, 0x04, 0xf0, 0x00, 0xca, 0x7d, 0xff, 0xe0, 0xc8, 0x1b, 0x0e, 0x78, - 0x20, 0x64, 0xb4, 0xd9, 0x56, 0xb9, 0x83, 0xcb, 0x68, 0x93, 0x16, 0x49, 0xd3, 0x22, 0xd5, 0x61, - 0xac, 0x32, 0xd6, 0x55, 0x87, 0x29, 0x55, 0x4c, 0x43, 0x0b, 0x80, 0xb0, 0x09, 0x9d, 0x47, 0x36, - 0x72, 0xcd, 0x6c, 0xab, 0x42, 0x52, 0x15, 0xac, 0x43, 0xc1, 0xe1, 0x33, 0x4f, 0xb0, 0xa0, 0x9e, - 0x55, 0x1e, 0x97, 0xd0, 0xde, 0x03, 0x48, 0x8e, 0xc7, 0x2a, 0xe8, 0xf1, 0x35, 0xf4, 0xbe, 0x8f, - 0x08, 0x86, 0xac, 0xab, 0xb9, 0xa8, 0x10, 0xb5, 0xb6, 0x9f, 0x4b, 0xc5, 0x61, 0x4a, 0xd1, 0x73, - 0x95, 0xc2, 0x20, 0x7a, 0xcf, 0x95, 0xf8, 0x84, 0x2b, 0xbe, 0x41, 0xf4, 0x13, 0x1e, 0xef, 0x90, - 0x4d, 0xed, 0x70, 0xbd, 0x1c, 0xd9, 0x81, 0xeb, 0x5d, 0xfe, 0x79, 0x64, 0x25, 0x63, 0xc3, 0xc8, - 0x22, 0x18, 0xe7, 0xee, 0x94, 0x2d, 0xce, 0x51, 0x6b, 0xdb, 0x7e, 0x30, 0x90, 0x52, 0x6c, 0x66, - 0xb0, 0x04, 0xb9, 0xe8, 0x79, 0x35, 0xfb, 0x03, 0x6c, 0x47, 0xfb, 0xf6, 0xa8, 0x37, 0x0c, 0x47, - 0x74, 0xcc, 0xf0, 0x59, 0x32, 0xfd, 0x9a, 0x9a, 0xfe, 0x35, 0x07, 0x31, 0x73, 0xfd, 0x13, 0x90, - 0x26, 0x7a, 0x53, 0xea, 0x28, 0x13, 0x5b, 0x44, 0xad, 0xed, 0x1b, 0x0d, 0x6a, 0x9b, 0x75, 0x92, - 0xde, 0x65, 0x81, 0x50, 0xa7, 0x6c, 0x11, 0xb5, 0xc6, 0x27, 0x50, 0xed, 0x7b, 0xae, 0x70, 0xa9, - 0xe0, 0x41, 0xdf, 0x1b, 0xb2, 0xeb, 0x45, 0xd2, 0x6b, 0x55, 0xc9, 0x23, 0x2c, 0xf4, 0xb9, 0x37, - 0x64, 0x0b, 0x5e, 0x94, 0xe7, 0x5a, 0x15, 0x6b, 0x90, 0xef, 0x72, 0x3e, 0x76, 0x59, 0xdd, 0x50, - 0xc9, 0x2c, 0x50, 0x9c, 0x57, 0x2e, 0xc9, 0xeb, 0xd8, 0x28, 0xe6, 0xcd, 0xc2, 0xb1, 0x51, 0x2c, - 0x98, 0x45, 0xfb, 0x46, 0x87, 0x4a, 0x64, 0xbb, 0xcb, 0x3d, 0x11, 0xf0, 0x09, 0x3e, 0x5d, 0x79, - 0x95, 0xc7, 0xab, 0x99, 0x2c, 0x48, 0x1b, 0x1e, 0x66, 0x0f, 0x76, 0x62, 0xeb, 0x6a, 0xfe, 0xd2, - 0xb7, 0xda, 0xd4, 0x92, 0x8a, 0xf8, 0x12, 0x29, 0x45, 0x74, 0xbf, 0x4d, 0x2d, 0xfc, 0x1f, 0x4a, - 0x0a, 0x9d, 0xf3, 0xbe, 0xaf, 0xee, 0x59, 0x21, 0x49, 0x01, 0x9b, 0x50, 0x56, 0xe0, 0x55, 0xc0, - 0xa7, 0xea, 0x5b, 0x90, 0xfd, 0x74, 0xc9, 0xee, 0xfd, 0xee, 0xcf, 0x55, 0x03, 0xec, 0x06, 0x8c, - 0x0a, 0xa6, 0xd8, 0x84, 0x5d, 0xcd, 0x58, 0x28, 0x4c, 0x0d, 0xff, 0x85, 0x9d, 0x95, 0xba, 0xb4, - 0x14, 0x32, 0x53, 0x7f, 0xb1, 0xff, 0xe5, 0xce, 0xd2, 0x6e, 0xef, 0x2c, 0xed, 0xfb, 0x9d, 0xa5, - 0x7d, 0xbe, 0xb7, 0x32, 0xb7, 0xf7, 0x56, 0xe6, 0xdb, 0xbd, 0x95, 0x79, 0xdf, 0xb8, 0x74, 0xc5, - 0x68, 0x76, 0xd1, 0x76, 0xf8, 0x74, 0x37, 0x9c, 0x50, 0x67, 0x3c, 0xba, 0xda, 0x8d, 0x22, 0xbc, - 0xc8, 0xab, 0x1f, 0xf8, 0xfe, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x17, 0x56, 0x28, 0x74, 0xd0, - 0x05, 0x00, 0x00, + // 665 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x54, 0xcd, 0x6e, 0xd3, 0x5c, + 0x10, 0x8d, 0x1d, 0x27, 0x69, 0x27, 0x4d, 0x3e, 0x7f, 0x53, 0x51, 0x12, 0x24, 0xac, 0xe0, 0x45, + 0x55, 0xb1, 0x48, 0x51, 0x5a, 0xba, 0xa6, 0x2d, 0x42, 0xa9, 0xd4, 0x9f, 0x70, 0x55, 0x8a, 0xc4, + 0xce, 0xb5, 0x2f, 0x8d, 0x55, 0xc7, 0x37, 0xb5, 0x6f, 0x50, 0xf3, 0x16, 0x3c, 0x0c, 0x0f, 0x01, + 0xbb, 0x2e, 0x59, 0xa2, 0x66, 0xc9, 0x92, 0x17, 0x40, 0xf7, 0xfa, 0xbf, 0x31, 0xb0, 0xbb, 0x33, + 0xe7, 0x9c, 0x99, 0xc9, 0xc9, 0x8c, 0x61, 0xcd, 0xa7, 0x97, 0x33, 0xcf, 0xea, 0x4f, 0x03, 0xc6, + 0x19, 0xd6, 0xa3, 0xc8, 0xfc, 0xa9, 0x02, 0x9c, 0xca, 0xe7, 0x09, 0xe5, 0x16, 0x0e, 0x40, 0x3b, + 0x9f, 0x4f, 0x69, 0x47, 0xe9, 0x29, 0x5b, 0xed, 0x81, 0xd1, 0x8f, 0x35, 0x19, 0xa3, 0x7f, 0x42, + 0xc3, 0xd0, 0xba, 0xa2, 0x82, 0x45, 0x24, 0x17, 0x77, 0xa0, 0xf1, 0x9a, 0x72, 0xcb, 0xf5, 0xc2, + 0x8e, 0xda, 0x53, 0xb6, 0x9a, 0x83, 0xee, 0xb2, 0x2c, 0x26, 0x90, 0x84, 0x69, 0xfe, 0x52, 0xa0, + 0x99, 0x2b, 0x85, 0x2b, 0xa0, 0x9d, 0x32, 0x9f, 0xea, 0x15, 0x6c, 0xc1, 0xea, 0x90, 0x85, 0xfc, + 0xed, 0x8c, 0x06, 0x73, 0x5d, 0x41, 0x84, 0x76, 0x1a, 0x12, 0x3a, 0xf5, 0xe6, 0xba, 0x8a, 0x4f, + 0x60, 0x43, 0xe4, 0xde, 0x4d, 0x1d, 0x8b, 0xd3, 0x53, 0xc6, 0xdd, 0x8f, 0xae, 0x6d, 0x71, 0x97, + 0xf9, 0x7a, 0x15, 0xbb, 0xf0, 0x48, 0x60, 0x27, 0xec, 0x13, 0x75, 0x0a, 0x90, 0x96, 0x40, 0xa3, + 0x99, 0x6f, 0x8f, 0x0b, 0x50, 0x0d, 0xdb, 0x00, 0x02, 0x7a, 0x3f, 0x66, 0xd6, 0xc4, 0xd5, 0xeb, + 0xb8, 0x0e, 0xff, 0x65, 0x71, 0xd4, 0xb6, 0x21, 0x26, 0x1b, 0x59, 0x7c, 0x7c, 0x38, 0xa6, 0xf6, + 0xb5, 0xbe, 0x22, 0x26, 0x4b, 0xc3, 0x88, 0xb2, 0x8a, 0x4f, 0xa1, 0x5b, 0x3e, 0xd9, 0xbe, 0x7d, + 0xad, 0x83, 0xf9, 0x4d, 0x85, 0xff, 0x97, 0x4c, 0x41, 0x13, 0xe0, 0xcc, 0x73, 0x2e, 0xa6, 0xfe, + 0xbe, 0xe3, 0x04, 0xd2, 0xfa, 0xd6, 0x81, 0xda, 0x51, 0x48, 0x2e, 0x8b, 0x9b, 0xd0, 0x48, 0x08, + 0x75, 0x69, 0xf2, 0x5a, 0x62, 0xb2, 0xc8, 0x91, 0x04, 0xc4, 0x3e, 0xe8, 0x67, 0x9e, 0x43, 0xa8, + 0x67, 0xcd, 0xe3, 0x54, 0xd8, 0xa9, 0xf5, 0xaa, 0x71, 0xc5, 0x25, 0x0c, 0x07, 0xd0, 0x2a, 0x92, + 0x1b, 0xbd, 0xea, 0x52, 0xf5, 0x22, 0x05, 0x77, 0xa1, 0x79, 0xb1, 0x2b, 0x9e, 0x23, 0x16, 0x70, + 0xf1, 0xa7, 0x0b, 0x05, 0x26, 0x8a, 0x0c, 0x22, 0x79, 0x9a, 0x54, 0xed, 0x65, 0x2a, 0xed, 0x81, + 0x6a, 0x2f, 0xa7, 0xca, 0x68, 0xd8, 0x81, 0x86, 0xcd, 0x66, 0x3e, 0xa7, 0x41, 0xa7, 0x2a, 0x8c, + 0x21, 0x49, 0x68, 0x6e, 0x82, 0x26, 0x7f, 0x71, 0x1b, 0xd4, 0xa1, 0x2b, 0x5d, 0xd3, 0x88, 0x3a, + 0x74, 0x45, 0x7c, 0xcc, 0xe4, 0x26, 0x6a, 0x44, 0x3d, 0x66, 0xe6, 0x2e, 0x40, 0x36, 0x06, 0x62, + 0xa4, 0x8a, 0x5c, 0x26, 0x51, 0x05, 0x04, 0x4d, 0x60, 0x52, 0xd3, 0x22, 0xf2, 0x6d, 0xbe, 0x02, + 0xc8, 0xc6, 0xf8, 0x57, 0x8f, 0xb4, 0x42, 0x35, 0x57, 0xe1, 0x36, 0x39, 0xac, 0x91, 0xeb, 0x5f, + 0xfd, 0xfd, 0xb0, 0x04, 0xa3, 0xe4, 0xb0, 0x10, 0xb4, 0x73, 0x77, 0x42, 0xe3, 0x3e, 0xf2, 0x6d, + 0x9a, 0x4b, 0x67, 0x23, 0xc4, 0x7a, 0x05, 0x57, 0xa1, 0x16, 0x2d, 0xa1, 0x62, 0x7e, 0xa9, 0x42, + 0x2b, 0x2a, 0x7c, 0xc8, 0x7c, 0x1e, 0x30, 0x0f, 0x5f, 0x16, 0xba, 0x3f, 0x2b, 0x76, 0x8f, 0x49, + 0x25, 0x03, 0xbc, 0x80, 0xf5, 0x23, 0xdf, 0xe5, 0xae, 0xc5, 0x59, 0x20, 0x57, 0xe0, 0xc8, 0x77, + 0xe8, 0x6d, 0xec, 0x53, 0x19, 0x24, 0x14, 0x84, 0x86, 0x53, 0xe6, 0x3b, 0x34, 0xaf, 0x88, 0x7c, + 0x29, 0x83, 0xf0, 0x39, 0xb4, 0x93, 0xa5, 0x3c, 0x67, 0xf2, 0xaf, 0xd1, 0xd2, 0x03, 0x78, 0x80, + 0xe4, 0x97, 0xfb, 0x4d, 0xc0, 0x26, 0x92, 0x5d, 0x4b, 0xd9, 0x4b, 0x18, 0xf6, 0xa1, 0x99, 0x2f, + 0x5c, 0x76, 0x38, 0x79, 0x42, 0x7a, 0x0c, 0x69, 0xf1, 0x46, 0x89, 0xa2, 0x48, 0x31, 0x87, 0x7f, + 0xfa, 0x8e, 0x6d, 0x00, 0x1e, 0x06, 0xd4, 0xe2, 0x54, 0xf2, 0x09, 0xbd, 0x99, 0xd1, 0x90, 0xeb, + 0x0a, 0x3e, 0x86, 0xf5, 0x42, 0x5e, 0x58, 0x12, 0x52, 0x5d, 0x3d, 0xd8, 0xf9, 0x7a, 0x6f, 0x28, + 0x77, 0xf7, 0x86, 0xf2, 0xe3, 0xde, 0x50, 0x3e, 0x2f, 0x8c, 0xca, 0xdd, 0xc2, 0xa8, 0x7c, 0x5f, + 0x18, 0x95, 0x0f, 0xdd, 0x2b, 0x97, 0x8f, 0x67, 0x97, 0x7d, 0x9b, 0x4d, 0xb6, 0x43, 0xcf, 0xb2, + 0xaf, 0xc7, 0x37, 0xdb, 0xd1, 0x48, 0x97, 0x75, 0xf9, 0x39, 0xdf, 0xf9, 0x1d, 0x00, 0x00, 0xff, + 0xff, 0x51, 0x0a, 0xe3, 0xd7, 0xde, 0x05, 0x00, 0x00, } func (m *NebulaMeta) Marshal() (dAtA []byte, err error) { @@ -748,28 +704,54 @@ func (m *NebulaMetaDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.RelayVpnIp) > 0 { - dAtA3 := make([]byte, len(m.RelayVpnIp)*10) - var j2 int - for _, num := range m.RelayVpnIp { + if len(m.RelayVpnAddrs) > 0 { + for iNdEx := len(m.RelayVpnAddrs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.RelayVpnAddrs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintNebula(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } + } + if m.VpnAddr != nil { + { + size, err := m.VpnAddr.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintNebula(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + if len(m.OldRelayVpnAddrs) > 0 { + dAtA4 := make([]byte, len(m.OldRelayVpnAddrs)*10) + var j3 int + for _, num := range m.OldRelayVpnAddrs { for num >= 1<<7 { - dAtA3[j2] = uint8(uint64(num)&0x7f | 0x80) + dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j2++ + j3++ } - dAtA3[j2] = uint8(num) - j2++ + dAtA4[j3] = uint8(num) + j3++ } - i -= j2 - copy(dAtA[i:], dAtA3[:j2]) - i = encodeVarintNebula(dAtA, i, uint64(j2)) + i -= j3 + copy(dAtA[i:], dAtA4[:j3]) + i = encodeVarintNebula(dAtA, i, uint64(j3)) i-- dAtA[i] = 0x2a } - if len(m.Ip6AndPorts) > 0 { - for iNdEx := len(m.Ip6AndPorts) - 1; iNdEx >= 0; iNdEx-- { + if len(m.V6AddrPorts) > 0 { + for iNdEx := len(m.V6AddrPorts) - 1; iNdEx >= 0; iNdEx-- { { - size, err := m.Ip6AndPorts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + size, err := m.V6AddrPorts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -785,10 +767,10 @@ func (m *NebulaMetaDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x18 } - if len(m.Ip4AndPorts) > 0 { - for iNdEx := len(m.Ip4AndPorts) - 1; iNdEx >= 0; iNdEx-- { + if len(m.V4AddrPorts) > 0 { + for iNdEx := len(m.V4AddrPorts) - 1; iNdEx >= 0; iNdEx-- { { - size, err := m.Ip4AndPorts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + size, err := m.V4AddrPorts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -799,48 +781,15 @@ func (m *NebulaMetaDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x12 } } - if m.VpnIp != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.VpnIp)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *Ip4AndPort) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Ip4AndPort) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Ip4AndPort) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Port != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.Port)) - i-- - dAtA[i] = 0x10 - } - if m.Ip != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.Ip)) + if m.OldVpnAddr != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.OldVpnAddr)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } -func (m *Ip6AndPort) Marshal() (dAtA []byte, err error) { +func (m *Addr) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -850,21 +799,16 @@ func (m *Ip6AndPort) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Ip6AndPort) MarshalTo(dAtA []byte) (int, error) { +func (m *Addr) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *Ip6AndPort) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *Addr) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if m.Port != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.Port)) - i-- - dAtA[i] = 0x18 - } if m.Lo != 0 { i = encodeVarintNebula(dAtA, i, uint64(m.Lo)) i-- @@ -878,7 +822,7 @@ func (m *Ip6AndPort) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *NebulaPing) Marshal() (dAtA []byte, err error) { +func (m *V4AddrPort) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -888,30 +832,30 @@ func (m *NebulaPing) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *NebulaPing) MarshalTo(dAtA []byte) (int, error) { +func (m *V4AddrPort) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *NebulaPing) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *V4AddrPort) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if m.Time != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.Time)) + if m.Port != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.Port)) i-- dAtA[i] = 0x10 } - if m.Type != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.Type)) + if m.Addr != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.Addr)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } -func (m *NebulaHandshake) Marshal() (dAtA []byte, err error) { +func (m *V6AddrPort) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -921,39 +865,35 @@ func (m *NebulaHandshake) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *NebulaHandshake) MarshalTo(dAtA []byte) (int, error) { +func (m *V6AddrPort) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *NebulaHandshake) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *V6AddrPort) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if len(m.Hmac) > 0 { - i -= len(m.Hmac) - copy(dAtA[i:], m.Hmac) - i = encodeVarintNebula(dAtA, i, uint64(len(m.Hmac))) + if m.Port != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.Port)) i-- - dAtA[i] = 0x12 + dAtA[i] = 0x18 } - if m.Details != nil { - { - size, err := m.Details.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintNebula(dAtA, i, uint64(size)) - } + if m.Lo != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.Lo)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x10 + } + if m.Hi != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.Hi)) + i-- + dAtA[i] = 0x8 } return len(dAtA) - i, nil } -func (m *NebulaHandshakeDetails) Marshal() (dAtA []byte, err error) { +func (m *NebulaPing) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -963,12 +903,12 @@ func (m *NebulaHandshakeDetails) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *NebulaHandshakeDetails) MarshalTo(dAtA []byte) (int, error) { +func (m *NebulaPing) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *NebulaHandshakeDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *NebulaPing) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -976,29 +916,12 @@ func (m *NebulaHandshakeDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) if m.Time != 0 { i = encodeVarintNebula(dAtA, i, uint64(m.Time)) i-- - dAtA[i] = 0x28 - } - if m.Cookie != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.Cookie)) - i-- - dAtA[i] = 0x20 - } - if m.ResponderIndex != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.ResponderIndex)) - i-- - dAtA[i] = 0x18 - } - if m.InitiatorIndex != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.InitiatorIndex)) - i-- dAtA[i] = 0x10 } - if len(m.Cert) > 0 { - i -= len(m.Cert) - copy(dAtA[i:], m.Cert) - i = encodeVarintNebula(dAtA, i, uint64(len(m.Cert))) + if m.Type != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.Type)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x8 } return len(dAtA) - i, nil } @@ -1023,13 +946,37 @@ func (m *NebulaControl) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.RelayFromIp != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.RelayFromIp)) + if m.RelayFromAddr != nil { + { + size, err := m.RelayFromAddr.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintNebula(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } + if m.RelayToAddr != nil { + { + size, err := m.RelayToAddr.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintNebula(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + if m.OldRelayFromAddr != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.OldRelayFromAddr)) i-- dAtA[i] = 0x28 } - if m.RelayToIp != 0 { - i = encodeVarintNebula(dAtA, i, uint64(m.RelayToIp)) + if m.OldRelayToAddr != 0 { + i = encodeVarintNebula(dAtA, i, uint64(m.OldRelayToAddr)) i-- dAtA[i] = 0x20 } @@ -1084,11 +1031,11 @@ func (m *NebulaMetaDetails) Size() (n int) { } var l int _ = l - if m.VpnIp != 0 { - n += 1 + sovNebula(uint64(m.VpnIp)) + if m.OldVpnAddr != 0 { + n += 1 + sovNebula(uint64(m.OldVpnAddr)) } - if len(m.Ip4AndPorts) > 0 { - for _, e := range m.Ip4AndPorts { + if len(m.V4AddrPorts) > 0 { + for _, e := range m.V4AddrPorts { l = e.Size() n += 1 + l + sovNebula(uint64(l)) } @@ -1096,48 +1043,55 @@ func (m *NebulaMetaDetails) Size() (n int) { if m.Counter != 0 { n += 1 + sovNebula(uint64(m.Counter)) } - if len(m.Ip6AndPorts) > 0 { - for _, e := range m.Ip6AndPorts { + if len(m.V6AddrPorts) > 0 { + for _, e := range m.V6AddrPorts { l = e.Size() n += 1 + l + sovNebula(uint64(l)) } } - if len(m.RelayVpnIp) > 0 { + if len(m.OldRelayVpnAddrs) > 0 { l = 0 - for _, e := range m.RelayVpnIp { + for _, e := range m.OldRelayVpnAddrs { l += sovNebula(uint64(e)) } n += 1 + sovNebula(uint64(l)) + l } + if m.VpnAddr != nil { + l = m.VpnAddr.Size() + n += 1 + l + sovNebula(uint64(l)) + } + if len(m.RelayVpnAddrs) > 0 { + for _, e := range m.RelayVpnAddrs { + l = e.Size() + n += 1 + l + sovNebula(uint64(l)) + } + } return n } -func (m *Ip4AndPort) Size() (n int) { +func (m *Addr) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.Ip != 0 { - n += 1 + sovNebula(uint64(m.Ip)) + if m.Hi != 0 { + n += 1 + sovNebula(uint64(m.Hi)) } - if m.Port != 0 { - n += 1 + sovNebula(uint64(m.Port)) + if m.Lo != 0 { + n += 1 + sovNebula(uint64(m.Lo)) } return n } -func (m *Ip6AndPort) Size() (n int) { +func (m *V4AddrPort) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.Hi != 0 { - n += 1 + sovNebula(uint64(m.Hi)) - } - if m.Lo != 0 { - n += 1 + sovNebula(uint64(m.Lo)) + if m.Addr != 0 { + n += 1 + sovNebula(uint64(m.Addr)) } if m.Port != 0 { n += 1 + sovNebula(uint64(m.Port)) @@ -1145,56 +1099,32 @@ func (m *Ip6AndPort) Size() (n int) { return n } -func (m *NebulaPing) Size() (n int) { +func (m *V6AddrPort) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.Type != 0 { - n += 1 + sovNebula(uint64(m.Type)) - } - if m.Time != 0 { - n += 1 + sovNebula(uint64(m.Time)) - } - return n -} - -func (m *NebulaHandshake) Size() (n int) { - if m == nil { - return 0 + if m.Hi != 0 { + n += 1 + sovNebula(uint64(m.Hi)) } - var l int - _ = l - if m.Details != nil { - l = m.Details.Size() - n += 1 + l + sovNebula(uint64(l)) + if m.Lo != 0 { + n += 1 + sovNebula(uint64(m.Lo)) } - l = len(m.Hmac) - if l > 0 { - n += 1 + l + sovNebula(uint64(l)) + if m.Port != 0 { + n += 1 + sovNebula(uint64(m.Port)) } return n } -func (m *NebulaHandshakeDetails) Size() (n int) { +func (m *NebulaPing) Size() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.Cert) - if l > 0 { - n += 1 + l + sovNebula(uint64(l)) - } - if m.InitiatorIndex != 0 { - n += 1 + sovNebula(uint64(m.InitiatorIndex)) - } - if m.ResponderIndex != 0 { - n += 1 + sovNebula(uint64(m.ResponderIndex)) - } - if m.Cookie != 0 { - n += 1 + sovNebula(uint64(m.Cookie)) + if m.Type != 0 { + n += 1 + sovNebula(uint64(m.Type)) } if m.Time != 0 { n += 1 + sovNebula(uint64(m.Time)) @@ -1217,11 +1147,19 @@ func (m *NebulaControl) Size() (n int) { if m.ResponderRelayIndex != 0 { n += 1 + sovNebula(uint64(m.ResponderRelayIndex)) } - if m.RelayToIp != 0 { - n += 1 + sovNebula(uint64(m.RelayToIp)) + if m.OldRelayToAddr != 0 { + n += 1 + sovNebula(uint64(m.OldRelayToAddr)) + } + if m.OldRelayFromAddr != 0 { + n += 1 + sovNebula(uint64(m.OldRelayFromAddr)) } - if m.RelayFromIp != 0 { - n += 1 + sovNebula(uint64(m.RelayFromIp)) + if m.RelayToAddr != nil { + l = m.RelayToAddr.Size() + n += 1 + l + sovNebula(uint64(l)) + } + if m.RelayFromAddr != nil { + l = m.RelayFromAddr.Size() + n += 1 + l + sovNebula(uint64(l)) } return n } @@ -1368,9 +1306,9 @@ func (m *NebulaMetaDetails) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field VpnIp", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field OldVpnAddr", wireType) } - m.VpnIp = 0 + m.OldVpnAddr = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -1380,14 +1318,14 @@ func (m *NebulaMetaDetails) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.VpnIp |= uint32(b&0x7F) << shift + m.OldVpnAddr |= uint32(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Ip4AndPorts", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field V4AddrPorts", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -1414,8 +1352,8 @@ func (m *NebulaMetaDetails) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Ip4AndPorts = append(m.Ip4AndPorts, &Ip4AndPort{}) - if err := m.Ip4AndPorts[len(m.Ip4AndPorts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.V4AddrPorts = append(m.V4AddrPorts, &V4AddrPort{}) + if err := m.V4AddrPorts[len(m.V4AddrPorts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -1440,7 +1378,7 @@ func (m *NebulaMetaDetails) Unmarshal(dAtA []byte) error { } case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Ip6AndPorts", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field V6AddrPorts", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -1467,8 +1405,8 @@ func (m *NebulaMetaDetails) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Ip6AndPorts = append(m.Ip6AndPorts, &Ip6AndPort{}) - if err := m.Ip6AndPorts[len(m.Ip6AndPorts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.V6AddrPorts = append(m.V6AddrPorts, &V6AddrPort{}) + if err := m.V6AddrPorts[len(m.V6AddrPorts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -1489,7 +1427,7 @@ func (m *NebulaMetaDetails) Unmarshal(dAtA []byte) error { break } } - m.RelayVpnIp = append(m.RelayVpnIp, v) + m.OldRelayVpnAddrs = append(m.OldRelayVpnAddrs, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { @@ -1524,8 +1462,8 @@ func (m *NebulaMetaDetails) Unmarshal(dAtA []byte) error { } } elementCount = count - if elementCount != 0 && len(m.RelayVpnIp) == 0 { - m.RelayVpnIp = make([]uint32, 0, elementCount) + if elementCount != 0 && len(m.OldRelayVpnAddrs) == 0 { + m.OldRelayVpnAddrs = make([]uint32, 0, elementCount) } for iNdEx < postIndex { var v uint32 @@ -1543,66 +1481,16 @@ func (m *NebulaMetaDetails) Unmarshal(dAtA []byte) error { break } } - m.RelayVpnIp = append(m.RelayVpnIp, v) + m.OldRelayVpnAddrs = append(m.OldRelayVpnAddrs, v) } } else { - return fmt.Errorf("proto: wrong wireType = %d for field RelayVpnIp", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field OldRelayVpnAddrs", wireType) } - default: - iNdEx = preIndex - skippy, err := skipNebula(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthNebula - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Ip4AndPort) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowNebula - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Ip4AndPort: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Ip4AndPort: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Ip", wireType) + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VpnAddr", wireType) } - m.Ip = 0 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -1612,16 +1500,33 @@ func (m *Ip4AndPort) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Ip |= uint32(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) + if msglen < 0 { + return ErrInvalidLengthNebula } - m.Port = 0 + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthNebula + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.VpnAddr == nil { + m.VpnAddr = &Addr{} + } + if err := m.VpnAddr.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RelayVpnAddrs", wireType) + } + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -1631,11 +1536,26 @@ func (m *Ip4AndPort) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Port |= uint32(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } + if msglen < 0 { + return ErrInvalidLengthNebula + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthNebula + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RelayVpnAddrs = append(m.RelayVpnAddrs, &Addr{}) + if err := m.RelayVpnAddrs[len(m.RelayVpnAddrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipNebula(dAtA[iNdEx:]) @@ -1657,7 +1577,7 @@ func (m *Ip4AndPort) Unmarshal(dAtA []byte) error { } return nil } -func (m *Ip6AndPort) Unmarshal(dAtA []byte) error { +func (m *Addr) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1680,10 +1600,10 @@ func (m *Ip6AndPort) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Ip6AndPort: wiretype end group for non-group") + return fmt.Errorf("proto: Addr: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Ip6AndPort: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Addr: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -1724,25 +1644,6 @@ func (m *Ip6AndPort) Unmarshal(dAtA []byte) error { break } } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) - } - m.Port = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowNebula - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Port |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } default: iNdEx = preIndex skippy, err := skipNebula(dAtA[iNdEx:]) @@ -1764,7 +1665,7 @@ func (m *Ip6AndPort) Unmarshal(dAtA []byte) error { } return nil } -func (m *NebulaPing) Unmarshal(dAtA []byte) error { +func (m *V4AddrPort) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1787,17 +1688,17 @@ func (m *NebulaPing) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: NebulaPing: wiretype end group for non-group") + return fmt.Errorf("proto: V4AddrPort: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: NebulaPing: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: V4AddrPort: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Addr", wireType) } - m.Type = 0 + m.Addr = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -1807,16 +1708,16 @@ func (m *NebulaPing) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Type |= NebulaPing_MessageType(b&0x7F) << shift + m.Addr |= uint32(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Time", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) } - m.Time = 0 + m.Port = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -1826,7 +1727,7 @@ func (m *NebulaPing) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Time |= uint64(b&0x7F) << shift + m.Port |= uint32(b&0x7F) << shift if b < 0x80 { break } @@ -1852,7 +1753,7 @@ func (m *NebulaPing) Unmarshal(dAtA []byte) error { } return nil } -func (m *NebulaHandshake) Unmarshal(dAtA []byte) error { +func (m *V6AddrPort) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1875,17 +1776,17 @@ func (m *NebulaHandshake) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: NebulaHandshake: wiretype end group for non-group") + return fmt.Errorf("proto: V6AddrPort: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: NebulaHandshake: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: V6AddrPort: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Details", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Hi", wireType) } - var msglen int + m.Hi = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -1895,33 +1796,16 @@ func (m *NebulaHandshake) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + m.Hi |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { - return ErrInvalidLengthNebula - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthNebula - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Details == nil { - m.Details = &NebulaHandshakeDetails{} - } - if err := m.Details.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Hmac", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Lo", wireType) } - var byteLen int + m.Lo = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -1931,26 +1815,30 @@ func (m *NebulaHandshake) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + m.Lo |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { - return ErrInvalidLengthNebula - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthNebula - } - if postIndex > l { - return io.ErrUnexpectedEOF + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) } - m.Hmac = append(m.Hmac[:0], dAtA[iNdEx:postIndex]...) - if m.Hmac == nil { - m.Hmac = []byte{} + m.Port = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Port |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipNebula(dAtA[iNdEx:]) @@ -1972,7 +1860,7 @@ func (m *NebulaHandshake) Unmarshal(dAtA []byte) error { } return nil } -func (m *NebulaHandshakeDetails) Unmarshal(dAtA []byte) error { +func (m *NebulaPing) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1995,89 +1883,17 @@ func (m *NebulaHandshakeDetails) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: NebulaHandshakeDetails: wiretype end group for non-group") + return fmt.Errorf("proto: NebulaPing: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: NebulaHandshakeDetails: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: NebulaPing: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Cert", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowNebula - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthNebula - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthNebula - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Cert = append(m.Cert[:0], dAtA[iNdEx:postIndex]...) - if m.Cert == nil { - m.Cert = []byte{} - } - iNdEx = postIndex - case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field InitiatorIndex", wireType) - } - m.InitiatorIndex = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowNebula - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.InitiatorIndex |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ResponderIndex", wireType) - } - m.ResponderIndex = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowNebula - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.ResponderIndex |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Cookie", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) } - m.Cookie = 0 + m.Type = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -2087,12 +1903,12 @@ func (m *NebulaHandshakeDetails) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Cookie |= uint64(b&0x7F) << shift + m.Type |= NebulaPing_MessageType(b&0x7F) << shift if b < 0x80 { break } } - case 5: + case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Time", wireType) } @@ -2220,9 +2036,9 @@ func (m *NebulaControl) Unmarshal(dAtA []byte) error { } case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RelayToIp", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field OldRelayToAddr", wireType) } - m.RelayToIp = 0 + m.OldRelayToAddr = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -2232,16 +2048,16 @@ func (m *NebulaControl) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.RelayToIp |= uint32(b&0x7F) << shift + m.OldRelayToAddr |= uint32(b&0x7F) << shift if b < 0x80 { break } } case 5: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RelayFromIp", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field OldRelayFromAddr", wireType) } - m.RelayFromIp = 0 + m.OldRelayFromAddr = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowNebula @@ -2251,11 +2067,83 @@ func (m *NebulaControl) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.RelayFromIp |= uint32(b&0x7F) << shift + m.OldRelayFromAddr |= uint32(b&0x7F) << shift if b < 0x80 { break } } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RelayToAddr", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthNebula + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthNebula + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.RelayToAddr == nil { + m.RelayToAddr = &Addr{} + } + if err := m.RelayToAddr.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RelayFromAddr", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowNebula + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthNebula + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthNebula + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.RelayFromAddr == nil { + m.RelayFromAddr = &Addr{} + } + if err := m.RelayFromAddr.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipNebula(dAtA[iNdEx:]) diff --git a/nebula.proto b/nebula.proto index 88e33b7e..7b44f473 100644 --- a/nebula.proto +++ b/nebula.proto @@ -23,19 +23,28 @@ message NebulaMeta { } message NebulaMetaDetails { - uint32 VpnIp = 1; - repeated Ip4AndPort Ip4AndPorts = 2; - repeated Ip6AndPort Ip6AndPorts = 4; - repeated uint32 RelayVpnIp = 5; + uint32 OldVpnAddr = 1 [deprecated = true]; + Addr VpnAddr = 6; + + repeated uint32 OldRelayVpnAddrs = 5 [deprecated = true]; + repeated Addr RelayVpnAddrs = 7; + + repeated V4AddrPort V4AddrPorts = 2; + repeated V6AddrPort V6AddrPorts = 4; uint32 counter = 3; } -message Ip4AndPort { - uint32 Ip = 1; +message Addr { + uint64 Hi = 1; + uint64 Lo = 2; +} + +message V4AddrPort { + uint32 Addr = 1; uint32 Port = 2; } -message Ip6AndPort { +message V6AddrPort { uint64 Hi = 1; uint64 Lo = 2; uint32 Port = 3; @@ -51,20 +60,9 @@ message NebulaPing { uint64 Time = 2; } -message NebulaHandshake { - NebulaHandshakeDetails Details = 1; - bytes Hmac = 2; -} - -message NebulaHandshakeDetails { - bytes Cert = 1; - uint32 InitiatorIndex = 2; - uint32 ResponderIndex = 3; - uint64 Cookie = 4; - uint64 Time = 5; - // reserved for WIP multiport - reserved 6, 7; -} +// NebulaHandshake / NebulaHandshakeDetails moved to +// handshake/handshake.proto. The handshake package speaks that wire format +// directly via a hand-written encoder/decoder. message NebulaControl { enum MessageType { @@ -76,6 +74,10 @@ message NebulaControl { uint32 InitiatorRelayIndex = 2; uint32 ResponderRelayIndex = 3; - uint32 RelayToIp = 4; - uint32 RelayFromIp = 5; + + uint32 OldRelayToAddr = 4 [deprecated = true]; + uint32 OldRelayFromAddr = 5 [deprecated = true]; + + Addr RelayToAddr = 6; + Addr RelayFromAddr = 7; } diff --git a/noise.go b/noise.go deleted file mode 100644 index 91ad2c02..00000000 --- a/noise.go +++ /dev/null @@ -1,75 +0,0 @@ -package nebula - -import ( - "crypto/cipher" - "encoding/binary" - "errors" - - "github.com/flynn/noise" -) - -type endianness interface { - PutUint64(b []byte, v uint64) -} - -var noiseEndianness endianness = binary.BigEndian - -type NebulaCipherState struct { - c noise.Cipher - //k [32]byte - //n uint64 -} - -func NewNebulaCipherState(s *noise.CipherState) *NebulaCipherState { - return &NebulaCipherState{c: s.Cipher()} - -} - -// EncryptDanger encrypts and authenticates a given payload. -// -// out is a destination slice to hold the output of the EncryptDanger operation. -// - ad is additional data, which will be authenticated and appended to out, but not encrypted. -// - plaintext is encrypted, authenticated and appended to out. -// - n is a nonce value which must never be re-used with this key. -// - nb is a buffer used for temporary storage in the implementation of this call, which should -// be re-used by callers to minimize garbage collection. -func (s *NebulaCipherState) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) { - if s != nil { - // TODO: Is this okay now that we have made messageCounter atomic? - // Alternative may be to split the counter space into ranges - //if n <= s.n { - // return nil, errors.New("CRITICAL: a duplicate counter value was used") - //} - //s.n = n - nb[0] = 0 - nb[1] = 0 - nb[2] = 0 - nb[3] = 0 - noiseEndianness.PutUint64(nb[4:], n) - out = s.c.(cipher.AEAD).Seal(out, nb, plaintext, ad) - //l.Debugf("Encryption: outlen: %d, nonce: %d, ad: %s, plainlen %d", len(out), n, ad, len(plaintext)) - return out, nil - } else { - return nil, errors.New("no cipher state available to encrypt") - } -} - -func (s *NebulaCipherState) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) { - if s != nil { - nb[0] = 0 - nb[1] = 0 - nb[2] = 0 - nb[3] = 0 - noiseEndianness.PutUint64(nb[4:], n) - return s.c.(cipher.AEAD).Open(out, nb, ciphertext, ad) - } else { - return []byte{}, nil - } -} - -func (s *NebulaCipherState) Overhead() int { - if s != nil { - return s.c.(cipher.AEAD).Overhead() - } - return 0 -} diff --git a/noiseutil/aesgcm.go b/noiseutil/aesgcm.go new file mode 100644 index 00000000..dcbd5693 --- /dev/null +++ b/noiseutil/aesgcm.go @@ -0,0 +1,53 @@ +package noiseutil + +import ( + "crypto/cipher" + "encoding/binary" + "errors" + + "github.com/flynn/noise" +) + +// CipherStateAESGCM is the data-plane wrapper for the AES-GCM AEAD cipher. +// AES-GCM uses big-endian nonce encoding per the Noise spec. +type CipherStateAESGCM struct { + c cipher.AEAD +} + +// NewCipherStateAESGCM extracts the underlying AEAD from the post-handshake noise.CipherState. +// The caller is responsible for ensuring the noise cipher is actually AES-GCM, +// otherwise the type assertion still succeeds but the nonce endianness will be wrong on the wire. +func NewCipherStateAESGCM(s *noise.CipherState) *CipherStateAESGCM { + return &CipherStateAESGCM{c: s.Cipher().(cipher.AEAD)} +} + +func (s *CipherStateAESGCM) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) { + if s == nil { + return nil, errors.New("no cipher state available to encrypt") + } + nb[0] = 0 + nb[1] = 0 + nb[2] = 0 + nb[3] = 0 + binary.BigEndian.PutUint64(nb[4:], n) + return s.c.Seal(out, nb, plaintext, ad), nil +} + +func (s *CipherStateAESGCM) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) { + if s == nil { + return []byte{}, nil + } + nb[0] = 0 + nb[1] = 0 + nb[2] = 0 + nb[3] = 0 + binary.BigEndian.PutUint64(nb[4:], n) + return s.c.Open(out, nb, ciphertext, ad) +} + +func (s *CipherStateAESGCM) Overhead() int { + if s == nil { + return 0 + } + return s.c.Overhead() +} diff --git a/noiseutil/boring.go b/noiseutil/boring.go index e9ad19bb..2129af71 100644 --- a/noiseutil/boring.go +++ b/noiseutil/boring.go @@ -22,7 +22,7 @@ const EncryptLockNeeded = true // NewGCMTLS is no longer exposed in go1.19+, so we need to link it in // See: https://github.com/golang/go/issues/56326 // -// NewGCMTLS is the internal method used with boringcrypto that provices a +// NewGCMTLS is the internal method used with boringcrypto that provides a // validated mode of AES-GCM which enforces the nonce is strictly // monotonically increasing. This is the TLS 1.2 specification for nonce // generation (which also matches the method used by the Noise Protocol) diff --git a/noiseutil/chachapoly.go b/noiseutil/chachapoly.go new file mode 100644 index 00000000..31ab3bfe --- /dev/null +++ b/noiseutil/chachapoly.go @@ -0,0 +1,52 @@ +package noiseutil + +import ( + "crypto/cipher" + "encoding/binary" + "errors" + + "github.com/flynn/noise" +) + +// CipherStateChaChaPoly is the data-plane wrapper for the ChaCha20-Poly1305 AEAD cipher. +// ChaCha20-Poly1305 uses little-endian nonce encoding per the Noise spec. +type CipherStateChaChaPoly struct { + c cipher.AEAD +} + +// NewCipherStateChaChaPoly extracts the underlying AEAD from the post-handshake noise.CipherState. +// The caller is responsible for ensuring the noise cipher is actually ChaCha20-Poly1305. +func NewCipherStateChaChaPoly(s *noise.CipherState) *CipherStateChaChaPoly { + return &CipherStateChaChaPoly{c: s.Cipher().(cipher.AEAD)} +} + +func (s *CipherStateChaChaPoly) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) { + if s == nil { + return nil, errors.New("no cipher state available to encrypt") + } + nb[0] = 0 + nb[1] = 0 + nb[2] = 0 + nb[3] = 0 + binary.LittleEndian.PutUint64(nb[4:], n) + return s.c.Seal(out, nb, plaintext, ad), nil +} + +func (s *CipherStateChaChaPoly) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) { + if s == nil { + return []byte{}, nil + } + nb[0] = 0 + nb[1] = 0 + nb[2] = 0 + nb[3] = 0 + binary.LittleEndian.PutUint64(nb[4:], n) + return s.c.Open(out, nb, ciphertext, ad) +} + +func (s *CipherStateChaChaPoly) Overhead() int { + if s == nil { + return 0 + } + return s.c.Overhead() +} diff --git a/noiseutil/cipher_state.go b/noiseutil/cipher_state.go new file mode 100644 index 00000000..bb316385 --- /dev/null +++ b/noiseutil/cipher_state.go @@ -0,0 +1,40 @@ +package noiseutil + +import ( + "fmt" + + "github.com/flynn/noise" +) + +// CipherState is the post-handshake AEAD cipher used for the data plane. +// Each supported cipher has its own concrete implementation in this package with the nonce endianness hardcoded, +// so the encrypt/decrypt fast path avoids interface dispatch on the byte order. +type CipherState interface { + // EncryptDanger encrypts and authenticates a given payload. + // + // out is a destination slice to hold the output of the EncryptDanger operation. + // - ad is additional data, which will be authenticated and appended to out, but not encrypted. + // - plaintext is encrypted, authenticated and appended to out. + // - n is a nonce value which must never be re-used with this key. + // - nb is a scratch buffer used to assemble the nonce. + EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) + + // DecryptDanger authenticates and decrypts a given payload, with the same argument shape as EncryptDanger. + DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) + + // Overhead returns the AEAD tag size, or 0 if the receiver is nil. + Overhead() int +} + +// NewCipherState wraps the post-handshake noise.CipherState in the per-cipher type that matches cipherFunc. +// cipherFunc must be the same cipher used to build the noise CipherSuite that produced s. +func NewCipherState(s *noise.CipherState, cipherFunc noise.CipherFunc) CipherState { + switch cipherFunc.CipherName() { + case CipherAESGCM.CipherName(): + return NewCipherStateAESGCM(s) + case noise.CipherChaChaPoly.CipherName(): + return NewCipherStateChaChaPoly(s) + default: + panic(fmt.Sprintf("noiseutil: unsupported cipher %q", cipherFunc.CipherName())) + } +} diff --git a/noiseutil/cipher_state_test.go b/noiseutil/cipher_state_test.go new file mode 100644 index 00000000..a4df01e9 --- /dev/null +++ b/noiseutil/cipher_state_test.go @@ -0,0 +1,166 @@ +package noiseutil + +import ( + "testing" + + "github.com/flynn/noise" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCipherStateAESGCMRoundtrip(t *testing.T) { + enc, dec := buildCipherStates(t, CipherAESGCM) + roundtrip(t, NewCipherStateAESGCM(enc), NewCipherStateAESGCM(dec)) +} + +func TestCipherStateChaChaPolyRoundtrip(t *testing.T) { + enc, dec := buildCipherStates(t, noise.CipherChaChaPoly) + roundtrip(t, NewCipherStateChaChaPoly(enc), NewCipherStateChaChaPoly(dec)) +} + +func TestNewCipherStateDispatch(t *testing.T) { + encA, _ := buildCipherStates(t, CipherAESGCM) + encC, _ := buildCipherStates(t, noise.CipherChaChaPoly) + + assert.IsType(t, &CipherStateAESGCM{}, NewCipherState(encA, CipherAESGCM)) + assert.IsType(t, &CipherStateChaChaPoly{}, NewCipherState(encC, noise.CipherChaChaPoly)) +} + +func TestNewCipherStateUnsupportedPanics(t *testing.T) { + enc, _ := buildCipherStates(t, CipherAESGCM) + assert.Panics(t, func() { + NewCipherState(enc, fakeCipher{}) + }) +} + +type fakeCipher struct{} + +func (fakeCipher) Cipher(k [32]byte) noise.Cipher { return nil } +func (fakeCipher) CipherName() string { return "Fake" } + +// buildCipherStates runs an in-memory NN handshake with the requested cipher +// to produce a pair of post-handshake CipherStates that share keys. +func buildCipherStates(t *testing.T, c noise.CipherFunc) (*noise.CipherState, *noise.CipherState) { + t.Helper() + suite := noise.NewCipherSuite(noise.DH25519, c, noise.HashSHA256) + cfg := noise.Config{CipherSuite: suite, Pattern: noise.HandshakeNN} + cfg.Initiator = true + hsI, err := noise.NewHandshakeState(cfg) + require.NoError(t, err) + cfg.Initiator = false + hsR, err := noise.NewHandshakeState(cfg) + require.NoError(t, err) + + msg, _, _, err := hsI.WriteMessage(nil, nil) + require.NoError(t, err) + _, _, _, err = hsR.ReadMessage(nil, msg) + require.NoError(t, err) + + msg, dR, _, err := hsR.WriteMessage(nil, nil) + require.NoError(t, err) + _, eI, _, err := hsI.ReadMessage(nil, msg) + require.NoError(t, err) + require.NotNil(t, eI) + require.NotNil(t, dR) + + // noise returns (cs1, cs2) where cs1 is the initiator->responder cipher. + return eI, dR +} + +func roundtrip(t *testing.T, enc, dec CipherState) { + t.Helper() + plaintext := []byte("nebula cipher state roundtrip") + ad := []byte("aad") + nb := make([]byte, 12) + + ct, err := enc.EncryptDanger(nil, ad, plaintext, 1, nb) + require.NoError(t, err) + assert.NotEqual(t, plaintext, ct) + + pt, err := dec.DecryptDanger(nil, ad, ct, 1, nb) + require.NoError(t, err) + assert.Equal(t, plaintext, pt) + + // Wrong nonce must fail authentication. + _, err = dec.DecryptDanger(nil, ad, ct, 2, nb) + require.Error(t, err) + + assert.Equal(t, enc.Overhead(), dec.Overhead()) + assert.Equal(t, 16, enc.Overhead()) +} + +func BenchmarkCipherStateEncryptAESGCM(b *testing.B) { + enc, _ := buildCipherStatesB(b, CipherAESGCM) + benchEncryptCipherState(b, NewCipherState(enc, CipherAESGCM)) +} + +func BenchmarkCipherStateEncryptChaChaPoly(b *testing.B) { + enc, _ := buildCipherStatesB(b, noise.CipherChaChaPoly) + benchEncryptCipherState(b, NewCipherState(enc, noise.CipherChaChaPoly)) +} + +func benchEncryptCipherState(b *testing.B, cs CipherState) { + plaintext := make([]byte, 1280) + ad := make([]byte, 16) + nb := make([]byte, 12) + out := make([]byte, 0, len(plaintext)+cs.Overhead()) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var err error + out, err = cs.EncryptDanger(out[:0], ad, plaintext, uint64(i+1), nb) + if err != nil { + b.Fatal(err) + } + } +} + +func buildCipherStatesB(b *testing.B, c noise.CipherFunc) (*noise.CipherState, *noise.CipherState) { + b.Helper() + suite := noise.NewCipherSuite(noise.DH25519, c, noise.HashSHA256) + cfg := noise.Config{CipherSuite: suite, Pattern: noise.HandshakeNN} + cfg.Initiator = true + hsI, err := noise.NewHandshakeState(cfg) + if err != nil { + b.Fatal(err) + } + cfg.Initiator = false + hsR, err := noise.NewHandshakeState(cfg) + if err != nil { + b.Fatal(err) + } + msg, _, _, err := hsI.WriteMessage(nil, nil) + if err != nil { + b.Fatal(err) + } + if _, _, _, err := hsR.ReadMessage(nil, msg); err != nil { + b.Fatal(err) + } + msg, dR, _, err := hsR.WriteMessage(nil, nil) + if err != nil { + b.Fatal(err) + } + _, eI, _, err := hsI.ReadMessage(nil, msg) + if err != nil { + b.Fatal(err) + } + return eI, dR +} + +func TestCipherStateNilSafety(t *testing.T) { + var aes *CipherStateAESGCM + _, err := aes.EncryptDanger(nil, nil, nil, 0, make([]byte, 12)) + require.Error(t, err) + out, err := aes.DecryptDanger(nil, nil, nil, 0, make([]byte, 12)) + require.NoError(t, err) + assert.Empty(t, out) + assert.Equal(t, 0, aes.Overhead()) + + var cc *CipherStateChaChaPoly + _, err = cc.EncryptDanger(nil, nil, nil, 0, make([]byte, 12)) + require.Error(t, err) + out, err = cc.DecryptDanger(nil, nil, nil, 0, make([]byte, 12)) + require.NoError(t, err) + assert.Empty(t, out) + assert.Equal(t, 0, cc.Overhead()) +} diff --git a/noiseutil/nist.go b/noiseutil/nist.go index 90e77abc..976a2742 100644 --- a/noiseutil/nist.go +++ b/noiseutil/nist.go @@ -48,7 +48,7 @@ func (c nistCurve) DH(privkey, pubkey []byte) ([]byte, error) { } ecdhPrivKey, err := c.curve.NewPrivateKey(privkey) if err != nil { - return nil, fmt.Errorf("unable to unmarshal pubkey: %w", err) + return nil, fmt.Errorf("unable to unmarshal private key: %w", err) } return ecdhPrivKey.ECDH(ecdhPubKey) diff --git a/noiseutil/pkcs11.go b/noiseutil/pkcs11.go new file mode 100644 index 00000000..d1c7ba91 --- /dev/null +++ b/noiseutil/pkcs11.go @@ -0,0 +1,50 @@ +package noiseutil + +import ( + "crypto/ecdh" + "fmt" + "strings" + + "github.com/slackhq/nebula/pkclient" + + "github.com/flynn/noise" +) + +// DHP256PKCS11 is the NIST P-256 ECDH function +var DHP256PKCS11 noise.DHFunc = newNISTP11Curve("P256", ecdh.P256(), 32) + +type nistP11Curve struct { + nistCurve +} + +func newNISTP11Curve(name string, curve ecdh.Curve, byteLen int) nistP11Curve { + return nistP11Curve{ + newNISTCurve(name, curve, byteLen), + } +} + +func (c nistP11Curve) DH(privkey, pubkey []byte) ([]byte, error) { + //for this function "privkey" is actually a pkcs11 URI + pkStr := string(privkey) + + //to set up a handshake, we need to also do non-pkcs11-DH. Handle that here. + if !strings.HasPrefix(pkStr, "pkcs11:") { + return DHP256.DH(privkey, pubkey) + } + ecdhPubKey, err := c.curve.NewPublicKey(pubkey) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal pubkey: %w", err) + } + + //this is not the most performant way to do this (a long-lived client would be better) + //but, it works, and helps avoid problems with stale sessions and HSMs used by multiple users. + client, err := pkclient.FromUrl(pkStr) + if err != nil { + return nil, err + } + defer func(client *pkclient.PKClient) { + _ = client.Close() + }(client) + + return client.DeriveNoise(ecdhPubKey.Bytes()) +} diff --git a/notboring.go b/notboring.go index c86b0bc3..f138a0a6 100644 --- a/notboring.go +++ b/notboring.go @@ -1,5 +1,4 @@ //go:build !boringcrypto -// +build !boringcrypto package nebula diff --git a/outside.go b/outside.go index 29189110..cf56bb4a 100644 --- a/outside.go +++ b/outside.go @@ -1,261 +1,258 @@ package nebula import ( + "context" "encoding/binary" "errors" - "fmt" + "log/slog" + "net/netip" "time" - "github.com/flynn/noise" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cert" + "github.com/google/gopacket/layers" + "golang.org/x/net/ipv6" + "github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" - "github.com/slackhq/nebula/udp" "golang.org/x/net/ipv4" - "google.golang.org/protobuf/proto" ) const ( minFwPacketLen = 4 ) -func readOutsidePackets(f *Interface) udp.EncReader { - return func( - addr *udp.Addr, - out []byte, - packet []byte, - header *header.H, - fwPacket *firewall.Packet, - lhh udp.LightHouseHandlerFunc, - nb []byte, - q int, - localCache firewall.ConntrackCache, - ) { - f.readOutsidePackets(addr, nil, out, packet, header, fwPacket, lhh, nb, q, localCache) - } -} +var ErrOutOfWindow = errors.New("out of window packet") -func (f *Interface) readOutsidePackets(addr *udp.Addr, via *ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf udp.LightHouseHandlerFunc, nb []byte, q int, localCache firewall.ConntrackCache) { +func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) { err := h.Parse(packet) if err != nil { - // TODO: best if we return this and let caller log - // TODO: Might be better to send the literal []byte("holepunch") packet and ignore that? // Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors + // TODO: record metrics for rx holepunch/punchy packets? if len(packet) > 1 { - f.l.WithField("packet", packet).Infof("Error while parsing inbound packet from %s: %s", addr, err) + f.messageMetrics.RxInvalid(1) + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("Error while parsing inbound packet", + "from", via, + "error", err, + "packet", packet, + ) + } } return } - //l.Error("in packet ", header, packet[HeaderLen:]) - if addr != nil { - if ip4 := addr.IP.To4(); ip4 != nil { - if ipMaskContains(f.lightHouse.myVpnIp, f.lightHouse.myVpnZeros, iputil.VpnIp(binary.BigEndian.Uint32(ip4))) { - if f.l.Level >= logrus.DebugLevel { - f.l.WithField("udpAddr", addr).Debug("Refusing to process double encrypted packet") - } - return + if h.Version != header.Version { + f.messageMetrics.RxInvalid(1) + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("Unexpected header version received", "from", via) + } + return + } + + // Check before processing to see if this is a expected type/subtype + if !h.IsValidSubType() { + f.messageMetrics.RxInvalid(1) + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("Unexpected packet received", "from", via) + } + return + } + + if !via.IsRelayed { + if f.myVpnNetworksTable.Contains(via.UdpAddr.Addr()) { + f.messageMetrics.RxInvalid(1) + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("Refusing to process double encrypted packet", "from", via) } + return } } + // don't keep Rx metrics for message type, since you can see those in the tun metrics + if h.Type != header.Message { + f.messageMetrics.Rx(h.Type, h.Subtype, 1) + } + + // Unencrypted packets + switch h.Type { + case header.Handshake: + f.handshakeManager.HandleIncoming(via, packet, h) + return + + case header.RecvError: + f.handleRecvError(via.UdpAddr, h) + return + } + + // Relay packets are special + isMessageRelay := (h.Type == header.Message && h.Subtype == header.MessageRelay) + var hostinfo *HostInfo - // verify if we've seen this index before, otherwise respond to the handshake initiation - if h.Type == header.Message && h.Subtype == header.MessageRelay { + if isMessageRelay { hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex) } else { hostinfo = f.hostMap.QueryIndex(h.RemoteIndex) } - var ci *ConnectionState - if hostinfo != nil { - ci = hostinfo.ConnectionState + // At this point we should have a valid existing tunnel, verify and send + // recvError if necessary + if hostinfo == nil || hostinfo.ConnectionState == nil { + if !via.IsRelayed { + f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex) + } + return } - switch h.Type { - case header.Message: - // TODO handleEncrypted sends directly to addr on error. Handle this in the tunneling case. - if !f.handleEncrypted(ci, addr, h) { - return + if len(packet) < header.Len+hostinfo.ConnectionState.dKey.Overhead() { + f.messageMetrics.RxInvalid(1) + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("packet too small", "from", via, "length", len(packet)) } + return + } - switch h.Subtype { - case header.MessageNone: - if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, nb, q, localCache) { - return - } - case header.MessageRelay: - // The entire body is sent as AD, not encrypted. - // The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value. - // The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's - // otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice - // which will gracefully fail in the DecryptDanger call. - signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()] - signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():] - out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb) - if err != nil { - return - } - // Successfully validated the thing. Get rid of the Relay header. - signedPayload = signedPayload[header.Len:] - // Pull the Roaming parts up here, and return in all call paths. - f.handleHostRoaming(hostinfo, addr) - // Track usage of both the HostInfo and the Relay for the received & authenticated packet - f.connectionManager.In(hostinfo.localIndexId) - f.connectionManager.RelayUsed(h.RemoteIndex) - - relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex) - if !ok { - // The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing - // its internal mapping. This should never happen. - hostinfo.logger(f.l).WithFields(logrus.Fields{"vpnIp": hostinfo.vpnIp, "remoteIndex": h.RemoteIndex}).Error("HostInfo missing remote relay index") - return - } - - switch relay.Type { - case TerminalType: - // If I am the target of this relay, process the unwrapped packet - // From this recursive point, all these variables are 'burned'. We shouldn't rely on them again. - f.readOutsidePackets(nil, &ViaSender{relayHI: hostinfo, remoteIdx: relay.RemoteIndex, relay: relay}, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache) - return - case ForwardingType: - // Find the target HostInfo relay object - targetHI, targetRelay, err := f.hostMap.QueryVpnIpRelayFor(hostinfo.vpnIp, relay.PeerIp) - if err != nil { - hostinfo.logger(f.l).WithField("relayTo", relay.PeerIp).WithError(err).Info("Failed to find target host info by ip") - return - } - - // If that relay is Established, forward the payload through it - if targetRelay.State == Established { - switch targetRelay.Type { - case ForwardingType: - // Forward this packet through the relay tunnel - // Find the target HostInfo - f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false) - return - case TerminalType: - hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal") - } - } else { - hostinfo.logger(f.l).WithFields(logrus.Fields{"relayTo": relay.PeerIp, "relayFrom": hostinfo.vpnIp, "targetRelayState": targetRelay.State}).Info("Unexpected target relay state") - return - } + // All remaining packets are encrypted + if isMessageRelay { + // Relay packets are special, this branch should always early-return + if err = hostinfo.ConnectionState.VerifyRelay(f.l, h.MessageCounter, packet, nb); err != nil { + if f.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(f.l).Debug("Failed to verify relay packet", "error", err, "from", via, "header", h) } - } - - case header.LightHouse: - f.messageMetrics.Rx(h.Type, h.Subtype, 1) - if !f.handleEncrypted(ci, addr, h) { return } + f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache) + return + } - d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb) - if err != nil { - hostinfo.logger(f.l).WithError(err).WithField("udpAddr", addr). - WithField("packet", packet). - Error("Failed to decrypt lighthouse packet") - - //TODO: maybe after build 64 is out? 06/14/2018 - NB - //f.sendRecvError(net.Addr(addr), header.RemoteIndex) - return + out, err = hostinfo.ConnectionState.Decrypt(f.l, h.MessageCounter, out, packet, nb) + if err != nil { + if f.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "header", h) } + return + } - lhf(addr, hostinfo.vpnIp, d) - - // Fallthrough to the bottom to record incoming traffic + // Roam before we respond + f.handleHostRoaming(hostinfo, via) + f.connectionManager.In(hostinfo) - case header.Test: - f.messageMetrics.Rx(h.Type, h.Subtype, 1) - if !f.handleEncrypted(ci, addr, h) { + switch h.Type { + case header.Message: + switch h.Subtype { + case header.MessageNone: + f.handleOutsideMessagePacket(hostinfo, out, packet, fwPacket, nb, q, localCache) + default: + hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected message subtype seen", "from", via, "header", h) return } - d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb) - if err != nil { - hostinfo.logger(f.l).WithError(err).WithField("udpAddr", addr). - WithField("packet", packet). - Error("Failed to decrypt test packet") + case header.LightHouse: + //TODO: assert via is not relayed + lhf.HandleRequest(via.UdpAddr, hostinfo.vpnAddrs, out, f) - //TODO: maybe after build 64 is out? 06/14/2018 - NB - //f.sendRecvError(net.Addr(addr), header.RemoteIndex) + case header.Test: + switch h.Subtype { + case header.TestReply: + // No-op, useful for the Roaming and connectionManager side-effects above + case header.TestRequest: + //recycle the input packet ciphertext as our output buffer + f.send(header.Test, header.TestReply, hostinfo.ConnectionState, hostinfo, out, nb, packet) + default: + hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected test subtype seen", "from", via, "header", h) return } - if h.Subtype == header.TestRequest { - // This testRequest might be from TryPromoteBest, so we should roam - // to the new IP address before responding - f.handleHostRoaming(hostinfo, addr) - f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out) - } + case header.CloseTunnel: + hostinfo.logger(f.l).Info("Close tunnel received, tearing down.", "from", via) + f.closeTunnel(hostinfo) - // Fallthrough to the bottom to record incoming traffic + case header.Control: + f.relayManager.HandleControlMsg(hostinfo, out, f) - // Non encrypted messages below here, they should not fall through to avoid tracking incoming traffic since they - // are unauthenticated + default: + hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected message type seen", "from", via, "header", h) + } +} - case header.Handshake: - f.messageMetrics.Rx(h.Type, h.Subtype, 1) - f.handshakeManager.HandleIncoming(addr, via, packet, h) - return - - case header.RecvError: - f.messageMetrics.Rx(h.Type, h.Subtype, 1) - f.handleRecvError(addr, h) +func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) { + // Successfully validated the thing. Get rid of the Relay header and the AEAD tag + signedPayload := packet[header.Len : len(packet)-hostinfo.ConnectionState.dKey.Overhead()] + // Pull the Roaming parts up here, and return in all call paths. + f.handleHostRoaming(hostinfo, via) + // Track usage of both the HostInfo and the Relay for the received & authenticated packet + f.connectionManager.In(hostinfo) + f.connectionManager.RelayUsed(h.RemoteIndex) + + relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex) + if !ok { + // The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing + // its internal mapping. This should never happen. + hostinfo.logger(f.l).Error("HostInfo missing remote relay index", + "relayRemoteIndex", h.RemoteIndex, + ) return + } - case header.CloseTunnel: - f.messageMetrics.Rx(h.Type, h.Subtype, 1) - if !f.handleEncrypted(ci, addr, h) { - return + switch relay.Type { + case TerminalType: + // If I am the target of this relay, process the unwrapped packet + // From this recursive point, all these variables are 'burned'. We shouldn't rely on them again. + via = ViaSender{ + UdpAddr: via.UdpAddr, + relayHI: hostinfo, + relay: relay, + IsRelayed: true, } - - hostinfo.logger(f.l).WithField("udpAddr", addr). - Info("Close tunnel received, tearing down.") - - f.closeTunnel(hostinfo) - return - - case header.Control: - if !f.handleEncrypted(ci, addr, h) { + f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache) + case ForwardingType: + // Find the target HostInfo relay object + targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr) + if err != nil { + hostinfo.logger(f.l).Info("Failed to find target host info by ip", + "relayTo", relay.PeerAddr, + "relayFrom", hostinfo.vpnAddrs[0], + "error", err, + ) return } - d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb) - if err != nil { - hostinfo.logger(f.l).WithError(err).WithField("udpAddr", addr). - WithField("packet", packet). - Error("Failed to decrypt Control packet") + // If that relay is Established, forward the payload through it + if targetRelay.State == Established { + switch targetRelay.Type { + case ForwardingType: + // Forward this packet through the relay tunnel, rebuilding it in place. + // Encode overwrites the old outer header, and the new AEAD tag lands where the old one was + fwdBuf := packet[:0:len(packet)] // Cap to len(packet) to protect memory from a larger parent buffer + f.SendVia(targetHI, targetRelay, signedPayload, nb, fwdBuf, true) + case TerminalType: + hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal") + return + default: + if f.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(f.l).Debug("Unexpected targetRelay Type", "from", via, "relayType", targetRelay.Type) + } + return + } + } else { + hostinfo.logger(f.l).Info("Unexpected target relay state", + "relayTo", relay.PeerAddr, + "relayFrom", hostinfo.vpnAddrs[0], + "targetRelayState", targetRelay.State, + ) return } - m := &NebulaControl{} - err = m.Unmarshal(d) - if err != nil { - hostinfo.logger(f.l).WithError(err).Error("Failed to unmarshal control message") - break - } - - f.relayManager.HandleControlMsg(hostinfo, m, f) - default: - f.messageMetrics.Rx(h.Type, h.Subtype, 1) - hostinfo.logger(f.l).Debugf("Unexpected packet received from %s", addr) - return + if f.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(f.l).Debug("Unexpected relay type", "from", via, "relayType", relay.Type) + } } - - f.handleHostRoaming(hostinfo, addr) - - f.connectionManager.In(hostinfo.localIndexId) } // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote func (f *Interface) closeTunnel(hostInfo *HostInfo) { final := f.hostMap.DeleteHostInfo(hostInfo) if final { - // We no longer have any tunnels with this vpn ip, clear learned lighthouse state to lower memory usage - f.lightHouse.DeleteVpnIp(hostInfo.vpnIp) + // We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage + f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs) } } @@ -264,62 +261,186 @@ func (f *Interface) sendCloseTunnel(h *HostInfo) { f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu)) } -func (f *Interface) handleHostRoaming(hostinfo *HostInfo, addr *udp.Addr) { - if addr != nil && !hostinfo.remote.Equals(addr) { - if !f.lightHouse.GetRemoteAllowList().Allow(hostinfo.vpnIp, addr.IP) { - hostinfo.logger(f.l).WithField("newAddr", addr).Debug("lighthouse.remote_allow_list denied roaming") +func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) { + curRemote := hostinfo.GetRemote() + if !via.IsRelayed && curRemote != via.UdpAddr { + if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) { + if f.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(f.l).Debug("lighthouse.remote_allow_list denied roaming", "newAddr", via.UdpAddr) + } return } - if !hostinfo.lastRoam.IsZero() && addr.Equals(hostinfo.lastRoamRemote) && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second { - if f.l.Level >= logrus.DebugLevel { - hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", addr). - Debugf("Suppressing roam back to previous remote for %d seconds", RoamingSuppressSeconds) + + if !hostinfo.lastRoam.IsZero() && via.UdpAddr == hostinfo.lastRoamRemote && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second { + if f.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(f.l).Debug("Suppressing roam back to previous remote", + "suppressSeconds", RoamingSuppressSeconds, + "udpAddr", curRemote, + "newAddr", via.UdpAddr, + ) } return } - hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", addr). - Info("Host roamed to new udp ip/port.") + hostinfo.logger(f.l).Info("Host roamed to new udp ip/port.", + "udpAddr", curRemote, + "newAddr", via.UdpAddr, + ) hostinfo.lastRoam = time.Now() - hostinfo.lastRoamRemote = hostinfo.remote - hostinfo.SetRemote(addr) + hostinfo.lastRoamRemote = curRemote + hostinfo.SetRemote(via.UdpAddr) } } -func (f *Interface) handleEncrypted(ci *ConnectionState, addr *udp.Addr, h *header.H) bool { - // If connectionstate exists and the replay protector allows, process packet - // Else, send recv errors for 300 seconds after a restart to allow fast reconnection. - if ci == nil || !ci.window.Check(f.l, h.MessageCounter) { - if addr != nil { - f.maybeSendRecvError(addr, h.RemoteIndex) - return false - } else { - return false +var ( + ErrPacketTooShort = errors.New("packet is too short") + ErrUnknownIPVersion = errors.New("packet is an unknown ip version") + ErrIPv4InvalidHeaderLength = errors.New("invalid ipv4 header length") + ErrIPv4PacketTooShort = errors.New("ipv4 packet is too short") + ErrIPv6PacketTooShort = errors.New("ipv6 packet is too short") + ErrIPv6CouldNotFindPayload = errors.New("could not find payload in ipv6 packet") +) + +// newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers +func newPacket(data []byte, incoming bool, fp *firewall.Packet) error { + if len(data) < 1 { + return ErrPacketTooShort + } + + version := int((data[0] >> 4) & 0x0f) + switch version { + case ipv4.Version: + return parseV4(data, incoming, fp) + case ipv6.Version: + return parseV6(data, incoming, fp) + } + return ErrUnknownIPVersion +} + +func parseV6(data []byte, incoming bool, fp *firewall.Packet) error { + dataLen := len(data) + if dataLen < ipv6.HeaderLen { + return ErrIPv6PacketTooShort + } + + if incoming { + fp.RemoteAddr, _ = netip.AddrFromSlice(data[8:24]) + fp.LocalAddr, _ = netip.AddrFromSlice(data[24:40]) + } else { + fp.LocalAddr, _ = netip.AddrFromSlice(data[8:24]) + fp.RemoteAddr, _ = netip.AddrFromSlice(data[24:40]) + } + + protoAt := 6 // NextHeader is at 6 bytes into the ipv6 header + offset := ipv6.HeaderLen // Start at the end of the ipv6 header + next := 0 + for { + if protoAt >= dataLen { + break + } + proto := layers.IPProtocol(data[protoAt]) + + switch proto { + case layers.IPProtocolESP, layers.IPProtocolNoNextHeader: + fp.Protocol = uint8(proto) + fp.RemotePort = 0 + fp.LocalPort = 0 + fp.Fragment = false + return nil + + case layers.IPProtocolICMPv6: + if dataLen < offset+6 { + return ErrIPv6PacketTooShort + } + fp.Protocol = uint8(proto) + fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6 + icmptype := data[offset+1] + switch icmptype { + case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply: + fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier + default: + fp.RemotePort = 0 + } + fp.Fragment = false + return nil + + case layers.IPProtocolTCP, layers.IPProtocolUDP: + if dataLen < offset+4 { + return ErrIPv6PacketTooShort + } + + fp.Protocol = uint8(proto) + if incoming { + fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2]) + fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4]) + } else { + fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2]) + fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4]) + } + + fp.Fragment = false + return nil + + case layers.IPProtocolIPv6Fragment: + // Fragment header is 8 bytes, need at least offset+4 to read the offset field + if dataLen < offset+8 { + return ErrIPv6PacketTooShort + } + + // Check if this is the first fragment + fragmentOffset := binary.BigEndian.Uint16(data[offset+2:offset+4]) &^ uint16(0x7) // Remove the reserved and M flag bits + if fragmentOffset != 0 { + // Non-first fragment, use what we have now and stop processing + fp.Protocol = data[offset] + fp.Fragment = true + fp.RemotePort = 0 + fp.LocalPort = 0 + return nil + } + + // The next loop should be the transport layer since we are the first fragment + next = 8 // Fragment headers are always 8 bytes + + case layers.IPProtocolAH: + // Auth headers, used by IPSec, have a different meaning for header length + if dataLen <= offset+1 { + break + } + next = (int(data[offset+1]) + 2) << 2 + + default: + // Normal ipv6 header length processing + if dataLen <= offset+1 { + break + } + next = (int(data[offset+1]) + 1) << 3 + } + + if next <= 0 { + // Safety check, each ipv6 header has to be at least 8 bytes + next = 8 } + + protoAt = offset + offset = offset + next } - return true + return ErrIPv6CouldNotFindPayload } -// newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers -func newPacket(data []byte, incoming bool, fp *firewall.Packet) error { +func parseV4(data []byte, incoming bool, fp *firewall.Packet) error { // Do we at least have an ipv4 header worth of data? if len(data) < ipv4.HeaderLen { - return fmt.Errorf("packet is less than %v bytes", ipv4.HeaderLen) - } - - // Is it an ipv4 packet? - if int((data[0]>>4)&0x0f) != 4 { - return fmt.Errorf("packet is not ipv4, type: %v", int((data[0]>>4)&0x0f)) + return ErrIPv4PacketTooShort } // Adjust our start position based on the advertised ip header length ihl := int(data[0]&0x0f) << 2 - // Well formed ip header length? + // Well-formed ip header length? if ihl < ipv4.HeaderLen { - return fmt.Errorf("packet had an invalid header length: %v", ihl) + return ErrIPv4InvalidHeaderLength } // Check if this is the second or further fragment of a fragmented packet. @@ -331,138 +452,120 @@ func newPacket(data []byte, incoming bool, fp *firewall.Packet) error { // Accounting for a variable header length, do we have enough data for our src/dst tuples? minLen := ihl - if !fp.Fragment && fp.Protocol != firewall.ProtoICMP { - minLen += minFwPacketLen - } - if len(data) < minLen { - return fmt.Errorf("packet is less than %v bytes, ip header len: %v", minLen, ihl) - } - - // Firewall packets are locally oriented - if incoming { - fp.RemoteIP = iputil.Ip2VpnIp(data[12:16]) - fp.LocalIP = iputil.Ip2VpnIp(data[16:20]) - if fp.Fragment || fp.Protocol == firewall.ProtoICMP { - fp.RemotePort = 0 - fp.LocalPort = 0 + if !fp.Fragment { + if fp.Protocol == firewall.ProtoICMP { + minLen += minFwPacketLen + 2 } else { - fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2]) - fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) - } - } else { - fp.LocalIP = iputil.Ip2VpnIp(data[12:16]) - fp.RemoteIP = iputil.Ip2VpnIp(data[16:20]) - if fp.Fragment || fp.Protocol == firewall.ProtoICMP { - fp.RemotePort = 0 - fp.LocalPort = 0 - } else { - fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2]) - fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) + minLen += minFwPacketLen } } - return nil -} - -func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) { - var err error - out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb) - if err != nil { - return nil, err + if len(data) < minLen { + return ErrIPv4InvalidHeaderLength } - if !hostinfo.ConnectionState.window.Update(f.l, mc) { - hostinfo.logger(f.l).WithField("header", h). - Debugln("dropping out of window packet") - return nil, errors.New("out of window packet") + if incoming { // Firewall packets are locally oriented + fp.RemoteAddr, _ = netip.AddrFromSlice(data[12:16]) + fp.LocalAddr, _ = netip.AddrFromSlice(data[16:20]) + } else { + fp.LocalAddr, _ = netip.AddrFromSlice(data[12:16]) + fp.RemoteAddr, _ = netip.AddrFromSlice(data[16:20]) + } + + if fp.Fragment { + fp.RemotePort = 0 + fp.LocalPort = 0 + } else if fp.Protocol == firewall.ProtoICMP { //note that orientation doesn't matter on ICMP + fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier + fp.LocalPort = 0 //code would be uint16(data[ihl+1]) + } else if incoming { + fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port + fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port + } else { + fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2]) //src port + fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port } - return out, nil + return nil } -func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) bool { - var err error - - out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb) - if err != nil { - hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet") - //TODO: maybe after build 64 is out? 06/14/2018 - NB - //f.sendRecvError(hostinfo.remote, header.RemoteIndex) - return false - } - - err = newPacket(out, true, fwPacket) +func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) { + err := newPacket(out, true, fwPacket) if err != nil { - hostinfo.logger(f.l).WithError(err).WithField("packet", out). - Warnf("Error while validating inbound packet") - return false - } - - if !hostinfo.ConnectionState.window.Update(f.l, messageCounter) { - hostinfo.logger(f.l).WithField("fwPacket", fwPacket). - Debugln("dropping out of window packet") - return false + hostinfo.logger(f.l).Warn("Error while validating inbound packet", + "error", err, + "packet", out, + ) + return } - dropReason := f.firewall.Drop(out, *fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache) + dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache) if dropReason != nil { // NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore // This gives us a buffer to build the reject packet in f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q) - if f.l.Level >= logrus.DebugLevel { - hostinfo.logger(f.l).WithField("fwPacket", fwPacket). - WithField("reason", dropReason). - Debugln("dropping inbound packet") + if f.l.Enabled(context.Background(), slog.LevelDebug) { + hostinfo.logger(f.l).Debug("dropping inbound packet", + "fwPacket", fwPacket, + "reason", dropReason, + ) } - return false + return } - f.connectionManager.In(hostinfo.localIndexId) _, err = f.readers[q].Write(out) if err != nil { - f.l.WithError(err).Error("Failed to write to tun") + f.l.Error("Failed to write to tun", "error", err) } - return true } -func (f *Interface) maybeSendRecvError(endpoint *udp.Addr, index uint32) { - if f.sendRecvErrorConfig.ShouldSendRecvError(endpoint.IP) { +func (f *Interface) maybeSendRecvError(endpoint netip.AddrPort, index uint32) { + if f.sendRecvErrorConfig.ShouldRecvError(endpoint) { f.sendRecvError(endpoint, index) } } -func (f *Interface) sendRecvError(endpoint *udp.Addr, index uint32) { +func (f *Interface) sendRecvError(endpoint netip.AddrPort, index uint32) { f.messageMetrics.Tx(header.RecvError, 0, 1) - //TODO: this should be a signed message so we can trust that we should drop the index b := header.Encode(make([]byte, header.Len), header.Version, header.RecvError, 0, index, 0) - f.outside.WriteTo(b, endpoint) - if f.l.Level >= logrus.DebugLevel { - f.l.WithField("index", index). - WithField("udpAddr", endpoint). - Debug("Recv error sent") + _ = f.outside.WriteTo(b, endpoint) + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("Recv error sent", + "index", index, + "udpAddr", endpoint, + ) } } -func (f *Interface) handleRecvError(addr *udp.Addr, h *header.H) { - if f.l.Level >= logrus.DebugLevel { - f.l.WithField("index", h.RemoteIndex). - WithField("udpAddr", addr). - Debug("Recv error received") +func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) { + if !f.acceptRecvErrorConfig.ShouldRecvError(addr) { + f.l.Debug("Recv error received, ignoring", + "index", h.RemoteIndex, + "udpAddr", addr, + ) + return } - hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex) - if hostinfo == nil { - f.l.WithField("remoteIndex", h.RemoteIndex).Debugln("Did not find remote index in main hostmap") - return + if f.l.Enabled(context.Background(), slog.LevelDebug) { + f.l.Debug("Recv error received", + "index", h.RemoteIndex, + "udpAddr", addr, + ) } - if !hostinfo.RecvErrorExceeded() { + hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex) + if hostinfo == nil { + f.l.Debug("Did not find remote index in main hostmap", "remoteIndex", h.RemoteIndex) return } - if hostinfo.remote != nil && !hostinfo.remote.Equals(addr) { - f.l.Infoln("Someone spoofing recv_errors? ", addr, hostinfo.remote) + hr := hostinfo.GetRemote() + if hr.IsValid() && hr != addr { + f.l.Info("Someone spoofing recv_errors?", + "addr", addr, + "hostinfoRemote", hr, + ) return } @@ -470,65 +573,3 @@ func (f *Interface) handleRecvError(addr *udp.Addr, h *header.H) { // We also delete it from pending hostmap to allow for fast reconnect. f.handshakeManager.DeleteHostInfo(hostinfo) } - -/* -func (f *Interface) sendMeta(ci *ConnectionState, endpoint *net.UDPAddr, meta *NebulaMeta) { - if ci.eKey != nil { - //TODO: log error? - return - } - - msg, err := proto.Marshal(meta) - if err != nil { - l.Debugln("failed to encode header") - } - - c := ci.messageCounter - b := HeaderEncode(nil, Version, uint8(metadata), 0, hostinfo.remoteIndexId, c) - ci.messageCounter++ - - msg := ci.eKey.EncryptDanger(b, nil, msg, c) - //msg := ci.eKey.EncryptDanger(b, nil, []byte(fmt.Sprintf("%d", counter)), c) - f.outside.WriteTo(msg, endpoint) -} -*/ - -func RecombineCertAndValidate(h *noise.HandshakeState, rawCertBytes []byte, caPool *cert.NebulaCAPool) (*cert.NebulaCertificate, error) { - pk := h.PeerStatic() - - if pk == nil { - return nil, errors.New("no peer static key was present") - } - - if rawCertBytes == nil { - return nil, errors.New("provided payload was empty") - } - - r := &cert.RawNebulaCertificate{} - err := proto.Unmarshal(rawCertBytes, r) - if err != nil { - return nil, fmt.Errorf("error unmarshaling cert: %s", err) - } - - // If the Details are nil, just exit to avoid crashing - if r.Details == nil { - return nil, fmt.Errorf("certificate did not contain any details") - } - - r.Details.PublicKey = pk - recombined, err := proto.Marshal(r) - if err != nil { - return nil, fmt.Errorf("error while recombining certificate: %s", err) - } - - c, _ := cert.UnmarshalNebulaCertificate(recombined) - isValid, err := c.Verify(time.Now(), caPool) - if err != nil { - return c, fmt.Errorf("certificate validation failed: %s", err) - } else if !isValid { - // This case should never happen but here's to defensive programming! - return c, errors.New("certificate validation failed but did not return an error") - } - - return c, nil -} diff --git a/outside_test.go b/outside_test.go index 682107bb..4a24cae5 100644 --- a/outside_test.go +++ b/outside_test.go @@ -1,21 +1,33 @@ package nebula import ( + "bytes" + "encoding/binary" "net" + "net/netip" "testing" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/slackhq/nebula/firewall" - "github.com/slackhq/nebula/iputil" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/net/ipv4" ) func Test_newPacket(t *testing.T) { p := &firewall.Packet{} - // length fail - err := newPacket([]byte{0, 1}, true, p) - assert.EqualError(t, err, "packet is less than 20 bytes") + // length fails + err := newPacket([]byte{}, true, p) + require.ErrorIs(t, err, ErrPacketTooShort) + + err = newPacket([]byte{0x40}, true, p) + require.ErrorIs(t, err, ErrIPv4PacketTooShort) + + err = newPacket([]byte{0x60}, true, p) + require.ErrorIs(t, err, ErrIPv6PacketTooShort) // length fail with ip options h := ipv4.Header{ @@ -28,16 +40,15 @@ func Test_newPacket(t *testing.T) { b, _ := h.Marshal() err = newPacket(b, true, p) - - assert.EqualError(t, err, "packet is less than 28 bytes, ip header len: 24") + require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength) // not an ipv4 packet err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p) - assert.EqualError(t, err, "packet is not ipv4, type: 0") + require.ErrorIs(t, err, ErrUnknownIPVersion) // invalid ihl err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p) - assert.EqualError(t, err, "packet had an invalid header length: 8") + require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength) // account for variable ip header length - incoming h = ipv4.Header{ @@ -53,12 +64,13 @@ func Test_newPacket(t *testing.T) { b = append(b, []byte{0, 3, 0, 4}...) err = newPacket(b, true, p) - assert.Nil(t, err) - assert.Equal(t, p.Protocol, uint8(firewall.ProtoTCP)) - assert.Equal(t, p.LocalIP, iputil.Ip2VpnIp(net.IPv4(10, 0, 0, 2))) - assert.Equal(t, p.RemoteIP, iputil.Ip2VpnIp(net.IPv4(10, 0, 0, 1))) - assert.Equal(t, p.RemotePort, uint16(3)) - assert.Equal(t, p.LocalPort, uint16(4)) + require.NoError(t, err) + assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol) + assert.Equal(t, netip.MustParseAddr("10.0.0.2"), p.LocalAddr) + assert.Equal(t, netip.MustParseAddr("10.0.0.1"), p.RemoteAddr) + assert.Equal(t, uint16(3), p.RemotePort) + assert.Equal(t, uint16(4), p.LocalPort) + assert.False(t, p.Fragment) // account for variable ip header length - outgoing h = ipv4.Header{ @@ -74,10 +86,592 @@ func Test_newPacket(t *testing.T) { b = append(b, []byte{0, 5, 0, 6}...) err = newPacket(b, false, p) - assert.Nil(t, err) - assert.Equal(t, p.Protocol, uint8(2)) - assert.Equal(t, p.LocalIP, iputil.Ip2VpnIp(net.IPv4(10, 0, 0, 1))) - assert.Equal(t, p.RemoteIP, iputil.Ip2VpnIp(net.IPv4(10, 0, 0, 2))) - assert.Equal(t, p.RemotePort, uint16(6)) - assert.Equal(t, p.LocalPort, uint16(5)) + require.NoError(t, err) + assert.Equal(t, uint8(2), p.Protocol) + assert.Equal(t, netip.MustParseAddr("10.0.0.1"), p.LocalAddr) + assert.Equal(t, netip.MustParseAddr("10.0.0.2"), p.RemoteAddr) + assert.Equal(t, uint16(6), p.RemotePort) + assert.Equal(t, uint16(5), p.LocalPort) + assert.False(t, p.Fragment) +} + +func Test_newPacket_v6(t *testing.T) { + p := &firewall.Packet{} + + // invalid ipv6 + ip := layers.IPv6{ + Version: 6, + HopLimit: 128, + SrcIP: net.IPv6linklocalallrouters, + DstIP: net.IPv6linklocalallnodes, + } + + buffer := gopacket.NewSerializeBuffer() + opt := gopacket.SerializeOptions{ + ComputeChecksums: false, + FixLengths: false, + } + err := gopacket.SerializeLayers(buffer, opt, &ip) + require.NoError(t, err) + + err = newPacket(buffer.Bytes(), true, p) + require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload) + + // A v6 packet with a hop-by-hop extension + // ICMPv6 Payload (Echo Request) + icmpLayer := layers.ICMPv6{ + TypeCode: layers.ICMPv6TypeEchoRequest, + } + // Hop-by-Hop Extension Header + hopOption := layers.IPv6HopByHopOption{} + hopOption.OptionData = []byte{0, 0, 0, 0} + hopByHop := layers.IPv6HopByHop{} + hopByHop.Options = append(hopByHop.Options, &hopOption) + + ip = layers.IPv6{ + Version: 6, + HopLimit: 128, + NextHeader: layers.IPProtocolIPv6Destination, + SrcIP: net.IPv6linklocalallrouters, + DstIP: net.IPv6linklocalallnodes, + } + + buffer.Clear() + err = gopacket.SerializeLayers(buffer, gopacket.SerializeOptions{ + ComputeChecksums: false, + FixLengths: true, + }, &ip, &hopByHop, &icmpLayer) + if err != nil { + panic(err) + } + // Ensure buffer length checks during parsing with the next 2 tests. + + // A full IPv6 header and 1 byte in the first extension, but missing + // the length byte. + err = newPacket(buffer.Bytes()[:41], true, p) + require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload) + + // A full IPv6 header plus 1 full extension, but only 1 byte of the + // next layer, missing length byte + err = newPacket(buffer.Bytes()[:49], true, p) + require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload) + err = nil + + // A good ICMP packet + ip = layers.IPv6{ + Version: 6, + NextHeader: layers.IPProtocolICMPv6, + HopLimit: 128, + SrcIP: net.IPv6linklocalallrouters, + DstIP: net.IPv6linklocalallnodes, + } + + icmp := layers.ICMPv6{ + TypeCode: layers.ICMPv6TypeEchoRequest, + Checksum: 0x1234, + } + + buffer.Clear() + require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp)) + require.Error(t, newPacket(buffer.Bytes(), true, p)) + + buffer.Clear() + echo := layers.ICMPv6Echo{ + Identifier: 0xabcd, + SeqNumber: 1234, + } + require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp, &echo)) + require.NoError(t, newPacket(buffer.Bytes(), true, p)) + assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr) + assert.Equal(t, uint16(0xabcd), p.RemotePort) + assert.Equal(t, uint16(0), p.LocalPort) + assert.False(t, p.Fragment) + + // A good ESP packet + b := buffer.Bytes() + b[6] = byte(layers.IPProtocolESP) + err = newPacket(b, true, p) + require.NoError(t, err) + assert.Equal(t, uint8(layers.IPProtocolESP), p.Protocol) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr) + assert.Equal(t, uint16(0), p.RemotePort) + assert.Equal(t, uint16(0), p.LocalPort) + assert.False(t, p.Fragment) + + // A good None packet + b = buffer.Bytes() + b[6] = byte(layers.IPProtocolNoNextHeader) + err = newPacket(b, true, p) + require.NoError(t, err) + assert.Equal(t, uint8(layers.IPProtocolNoNextHeader), p.Protocol) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr) + assert.Equal(t, uint16(0), p.RemotePort) + assert.Equal(t, uint16(0), p.LocalPort) + assert.False(t, p.Fragment) + + // An unknown protocol packet + b = buffer.Bytes() + b[6] = 255 // 255 is a reserved protocol number + err = newPacket(b, true, p) + require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload) + + // A good UDP packet + ip = layers.IPv6{ + Version: 6, + NextHeader: firewall.ProtoUDP, + HopLimit: 128, + SrcIP: net.IPv6linklocalallrouters, + DstIP: net.IPv6linklocalallnodes, + } + + udp := layers.UDP{ + SrcPort: layers.UDPPort(36123), + DstPort: layers.UDPPort(22), + } + err = udp.SetNetworkLayerForChecksum(&ip) + require.NoError(t, err) + + buffer.Clear() + err = gopacket.SerializeLayers(buffer, opt, &ip, &udp, gopacket.Payload([]byte{0xde, 0xad, 0xbe, 0xef})) + if err != nil { + panic(err) + } + b = buffer.Bytes() + + // incoming + err = newPacket(b, true, p) + require.NoError(t, err) + assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr) + assert.Equal(t, uint16(36123), p.RemotePort) + assert.Equal(t, uint16(22), p.LocalPort) + assert.False(t, p.Fragment) + + // outgoing + err = newPacket(b, false, p) + require.NoError(t, err) + assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr) + assert.Equal(t, uint16(36123), p.LocalPort) + assert.Equal(t, uint16(22), p.RemotePort) + assert.False(t, p.Fragment) + + // Too short UDP packet + err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes + require.ErrorIs(t, err, ErrIPv6PacketTooShort) + + // A good TCP packet + b[6] = byte(layers.IPProtocolTCP) + + // incoming + err = newPacket(b, true, p) + require.NoError(t, err) + assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr) + assert.Equal(t, uint16(36123), p.RemotePort) + assert.Equal(t, uint16(22), p.LocalPort) + assert.False(t, p.Fragment) + + // outgoing + err = newPacket(b, false, p) + require.NoError(t, err) + assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr) + assert.Equal(t, uint16(36123), p.LocalPort) + assert.Equal(t, uint16(22), p.RemotePort) + assert.False(t, p.Fragment) + + // Too short TCP packet + err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes + require.ErrorIs(t, err, ErrIPv6PacketTooShort) + + // A good UDP packet with an AH header + ip = layers.IPv6{ + Version: 6, + NextHeader: layers.IPProtocolAH, + HopLimit: 128, + SrcIP: net.IPv6linklocalallrouters, + DstIP: net.IPv6linklocalallnodes, + } + + ah := layers.IPSecAH{ + AuthenticationData: []byte{0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef}, + } + ah.NextHeader = layers.IPProtocolUDP + + udpHeader := []byte{ + 0x8d, 0x1b, // Source port 36123 + 0x00, 0x16, // Destination port 22 + 0x00, 0x00, // Length + 0x00, 0x00, // Checksum + } + + buffer.Clear() + err = ip.SerializeTo(buffer, opt) + if err != nil { + panic(err) + } + + b = buffer.Bytes() + ahb := serializeAH(&ah) + b = append(b, ahb...) + b = append(b, udpHeader...) + + err = newPacket(b, true, p) + require.NoError(t, err) + assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr) + assert.Equal(t, uint16(36123), p.RemotePort) + assert.Equal(t, uint16(22), p.LocalPort) + assert.False(t, p.Fragment) + + // Ensure buffer bounds checking during processing + err = newPacket(b[:41], true, p) + require.ErrorIs(t, err, ErrIPv6PacketTooShort) + + // Invalid AH header + b = buffer.Bytes() + err = newPacket(b, true, p) + require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload) +} + +func Test_newPacket_ipv6Fragment(t *testing.T) { + p := &firewall.Packet{} + + ip := &layers.IPv6{ + Version: 6, + NextHeader: layers.IPProtocolIPv6Fragment, + HopLimit: 64, + SrcIP: net.IPv6linklocalallrouters, + DstIP: net.IPv6linklocalallnodes, + } + + // First fragment + fragHeader1 := []byte{ + uint8(layers.IPProtocolUDP), // Next Header (UDP) + 0x00, // Reserved + 0x00, // Fragment Offset high byte (0) + 0x01, // Fragment Offset low byte & flags (M=1) + 0x00, 0x00, 0x00, 0x01, // Identification + } + + udpHeader := []byte{ + 0x8d, 0x1b, // Source port 36123 + 0x00, 0x16, // Destination port 22 + 0x00, 0x00, // Length + 0x00, 0x00, // Checksum + } + + buffer := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + err := ip.SerializeTo(buffer, opts) + if err != nil { + t.Fatal(err) + } + + firstFrag := buffer.Bytes() + firstFrag = append(firstFrag, fragHeader1...) + firstFrag = append(firstFrag, udpHeader...) + firstFrag = append(firstFrag, []byte{0xde, 0xad, 0xbe, 0xef}...) + + // Test first fragment incoming + err = newPacket(firstFrag, true, p) + require.NoError(t, err) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr) + assert.Equal(t, uint8(layers.IPProtocolUDP), p.Protocol) + assert.Equal(t, uint16(36123), p.RemotePort) + assert.Equal(t, uint16(22), p.LocalPort) + assert.False(t, p.Fragment) + + // Test first fragment outgoing + err = newPacket(firstFrag, false, p) + require.NoError(t, err) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr) + assert.Equal(t, uint8(layers.IPProtocolUDP), p.Protocol) + assert.Equal(t, uint16(36123), p.LocalPort) + assert.Equal(t, uint16(22), p.RemotePort) + assert.False(t, p.Fragment) + + // Second fragment + fragHeader2 := []byte{ + uint8(layers.IPProtocolUDP), // Next Header (UDP) + 0x00, // Reserved + 0xb9, // Fragment Offset high byte (185) + 0x01, // Fragment Offset low byte & flags (M=1) + 0x00, 0x00, 0x00, 0x01, // Identification + } + + buffer.Clear() + err = ip.SerializeTo(buffer, opts) + if err != nil { + t.Fatal(err) + } + + secondFrag := buffer.Bytes() + secondFrag = append(secondFrag, fragHeader2...) + secondFrag = append(secondFrag, []byte{0xde, 0xad, 0xbe, 0xef}...) + + // Test second fragment incoming + err = newPacket(secondFrag, true, p) + require.NoError(t, err) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr) + assert.Equal(t, uint8(layers.IPProtocolUDP), p.Protocol) + assert.Equal(t, uint16(0), p.RemotePort) + assert.Equal(t, uint16(0), p.LocalPort) + assert.True(t, p.Fragment) + + // Test second fragment outgoing + err = newPacket(secondFrag, false, p) + require.NoError(t, err) + assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr) + assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr) + assert.Equal(t, uint8(layers.IPProtocolUDP), p.Protocol) + assert.Equal(t, uint16(0), p.LocalPort) + assert.Equal(t, uint16(0), p.RemotePort) + assert.True(t, p.Fragment) + + // Too short of a fragment packet + err = newPacket(secondFrag[:len(secondFrag)-10], false, p) + require.ErrorIs(t, err, ErrIPv6PacketTooShort) +} + +func BenchmarkParseV6(b *testing.B) { + // Regular UDP packet + ip := &layers.IPv6{ + Version: 6, + NextHeader: layers.IPProtocolUDP, + HopLimit: 64, + SrcIP: net.IPv6linklocalallrouters, + DstIP: net.IPv6linklocalallnodes, + } + + udp := &layers.UDP{ + SrcPort: layers.UDPPort(36123), + DstPort: layers.UDPPort(22), + } + + buffer := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ + ComputeChecksums: false, + FixLengths: true, + } + + err := gopacket.SerializeLayers(buffer, opts, ip, udp) + if err != nil { + b.Fatal(err) + } + normalPacket := buffer.Bytes() + + // First Fragment packet + ipFrag := &layers.IPv6{ + Version: 6, + NextHeader: layers.IPProtocolIPv6Fragment, + HopLimit: 64, + SrcIP: net.IPv6linklocalallrouters, + DstIP: net.IPv6linklocalallnodes, + } + + fragHeader := []byte{ + uint8(layers.IPProtocolUDP), // Next Header (UDP) + 0x00, // Reserved + 0x00, // Fragment Offset high byte (0) + 0x01, // Fragment Offset low byte & flags (M=1) + 0x00, 0x00, 0x00, 0x01, // Identification + } + + udpHeader := []byte{ + 0x8d, 0x7b, // Source port 36123 + 0x00, 0x16, // Destination port 22 + 0x00, 0x00, // Length + 0x00, 0x00, // Checksum + } + + buffer.Clear() + err = ipFrag.SerializeTo(buffer, opts) + if err != nil { + b.Fatal(err) + } + + firstFrag := buffer.Bytes() + firstFrag = append(firstFrag, fragHeader...) + firstFrag = append(firstFrag, udpHeader...) + firstFrag = append(firstFrag, []byte{0xde, 0xad, 0xbe, 0xef}...) + + // Second Fragment packet + fragHeader[2] = 0xb9 // offset 185 + buffer.Clear() + err = ipFrag.SerializeTo(buffer, opts) + if err != nil { + b.Fatal(err) + } + + secondFrag := buffer.Bytes() + secondFrag = append(secondFrag, fragHeader...) + secondFrag = append(secondFrag, []byte{0xde, 0xad, 0xbe, 0xef}...) + + fp := &firewall.Packet{} + + b.Run("Normal", func(b *testing.B) { + for i := 0; i < b.N; i++ { + if err = parseV6(normalPacket, true, fp); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("FirstFragment", func(b *testing.B) { + for i := 0; i < b.N; i++ { + if err = parseV6(firstFrag, true, fp); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("SecondFragment", func(b *testing.B) { + for i := 0; i < b.N; i++ { + if err = parseV6(secondFrag, true, fp); err != nil { + b.Fatal(err) + } + } + }) + + // Evil packet + evilPacket := &layers.IPv6{ + Version: 6, + NextHeader: layers.IPProtocolIPv6HopByHop, + HopLimit: 64, + SrcIP: net.IPv6linklocalallrouters, + DstIP: net.IPv6linklocalallnodes, + } + + hopHeader := []byte{ + uint8(layers.IPProtocolIPv6HopByHop), // Next Header (HopByHop) + 0x00, // Length + 0x00, 0x00, // Options and padding + 0x00, 0x00, 0x00, 0x00, // More options and padding + } + + lastHopHeader := []byte{ + uint8(layers.IPProtocolUDP), // Next Header (UDP) + 0x00, // Length + 0x00, 0x00, // Options and padding + 0x00, 0x00, 0x00, 0x00, // More options and padding + } + + buffer.Clear() + err = evilPacket.SerializeTo(buffer, opts) + if err != nil { + b.Fatal(err) + } + + evilBytes := buffer.Bytes() + for range 200 { + evilBytes = append(evilBytes, hopHeader...) + } + evilBytes = append(evilBytes, lastHopHeader...) + evilBytes = append(evilBytes, udpHeader...) + evilBytes = append(evilBytes, []byte{0xde, 0xad, 0xbe, 0xef}...) + + b.Run("200 HopByHop headers", func(b *testing.B) { + for i := 0; i < b.N; i++ { + if err = parseV6(evilBytes, false, fp); err != nil { + b.Fatal(err) + } + } + }) +} + +// Ensure authentication data is a multiple of 8 bytes by padding if necessary +func padAuthData(authData []byte) []byte { + // Length of Authentication Data must be a multiple of 8 bytes + paddingLength := (8 - (len(authData) % 8)) % 8 // Only pad if necessary + if paddingLength > 0 { + authData = append(authData, make([]byte, paddingLength)...) + } + return authData +} + +// Custom function to manually serialize IPSecAH for both IPv4 and IPv6 +func serializeAH(ah *layers.IPSecAH) []byte { + buf := new(bytes.Buffer) + + // Ensure Authentication Data is a multiple of 8 bytes + ah.AuthenticationData = padAuthData(ah.AuthenticationData) + // Calculate Payload Length (in 32-bit words, minus 2) + payloadLen := uint8((12+len(ah.AuthenticationData))/4) - 2 + + // Serialize fields + if err := binary.Write(buf, binary.BigEndian, ah.NextHeader); err != nil { + panic(err) + } + if err := binary.Write(buf, binary.BigEndian, payloadLen); err != nil { + panic(err) + } + if err := binary.Write(buf, binary.BigEndian, ah.Reserved); err != nil { + panic(err) + } + if err := binary.Write(buf, binary.BigEndian, ah.SPI); err != nil { + panic(err) + } + if err := binary.Write(buf, binary.BigEndian, ah.Seq); err != nil { + panic(err) + } + if len(ah.AuthenticationData) > 0 { + if err := binary.Write(buf, binary.BigEndian, ah.AuthenticationData); err != nil { + panic(err) + } + } + + return buf.Bytes() +} + +// Test_newPacket_v6ExtHeaderOverflow is a regression test for the IPv6 extension-header +// length uint8 overflow in parseV6. A Destination-Options header with HdrExtLen=255 spans +// (255+1)*8 = 2048 bytes, so the real transport header sits at offset 2088. Before the fix +// the advance was computed in uint8 and wrapped to 0 (then clamped to 8), so the firewall +// read the transport header ~2KB too early from attacker-controlled option bytes while the +// host OS parses the real header, a firewall port/proto bypass. The fix makes parseV6 land +// on the same offset the host does. +func Test_newPacket_v6ExtHeaderOverflow(t *testing.T) { + p := &firewall.Packet{} + + const ( + hdrLen = 40 // IPv6 header + extLen = 2048 // (255+1)*8, the true Destination-Options header size + realTCPAt = hdrLen + extLen // 2088, where the host reads the transport header + forgedTCPAt = hdrLen + 8 // 48, where the pre-fix wrapped+clamped walk landed + ) + + pkt := make([]byte, realTCPAt+4) + pkt[0] = 0x60 // version 6 + pkt[6] = byte(layers.IPProtocolIPv6Destination) // NextHeader -> Destination Options + pkt[40] = byte(firewall.ProtoTCP) // Dest-Options NextHeader -> TCP + pkt[41] = 255 // HdrExtLen = 255 + + // Forged transport header at the pre-fix (wrong) offset: dst port 443. + binary.BigEndian.PutUint16(pkt[forgedTCPAt+2:forgedTCPAt+4], 443) + // Real transport header at the offset the host actually uses: dst port 22. + binary.BigEndian.PutUint16(pkt[realTCPAt+2:realTCPAt+4], 22) + + require.NoError(t, newPacket(pkt, true, p)) + assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol) + // LocalPort is the destination port for incoming traffic. It must be the real port (22) + // the host delivers to, not the forged 443 at the overflowed offset. + assert.Equal(t, uint16(22), p.LocalPort, "firewall must parse the real transport header, not the overflowed offset") } diff --git a/overlay/device.go b/overlay/device.go index 3f3f2eb4..b6077aba 100644 --- a/overlay/device.go +++ b/overlay/device.go @@ -2,16 +2,17 @@ package overlay import ( "io" - "net" + "net/netip" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" ) type Device interface { io.ReadWriteCloser Activate() error - Cidr() *net.IPNet + Networks() []netip.Prefix Name() string - RouteFor(iputil.VpnIp) iputil.VpnIp + RoutesFor(netip.Addr) routing.Gateways + SupportsMultiqueue() bool NewMultiQueueReader() (io.ReadWriteCloser, error) } diff --git a/overlay/network_category_windows.go b/overlay/network_category_windows.go new file mode 100644 index 00000000..cbf87f00 --- /dev/null +++ b/overlay/network_category_windows.go @@ -0,0 +1,358 @@ +//go:build !e2e_testing +// +build !e2e_testing + +package overlay + +import ( + "errors" + "fmt" + "log/slog" + "runtime" + "strings" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +// networkCategory mirrors NLM_NETWORK_CATEGORY from netlistmgr.h. +type networkCategory int32 + +const ( + networkCategoryPublic networkCategory = 0 + networkCategoryPrivate networkCategory = 1 + networkCategoryDomainAuthenticated networkCategory = 2 +) + +func (c networkCategory) String() string { + switch c { + case networkCategoryPublic: + return "public" + case networkCategoryPrivate: + return "private" + case networkCategoryDomainAuthenticated: + return "domain" + } + return fmt.Sprintf("unknown(%d)", c) +} + +// parseNetworkCategory accepts the user-supplied tun.network_category. A +// second return of false means "leave the category alone". +func parseNetworkCategory(s string) (networkCategory, bool, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "", "unset": + return 0, false, nil + case "public": + return networkCategoryPublic, true, nil + case "private": + return networkCategoryPrivate, true, nil + case "domain", "domainauthenticated": + return networkCategoryDomainAuthenticated, true, nil + } + return 0, false, fmt.Errorf("unknown tun.network_category %q (expected public, private, domain, or unset)", s) +} + +// CLSID_NetworkListManager {DCB00C01-570F-4A9B-8D69-199FDBA5723B} +var clsidNetworkListManager = windows.GUID{ + Data1: 0xDCB00C01, Data2: 0x570F, Data3: 0x4A9B, + Data4: [8]byte{0x8D, 0x69, 0x19, 0x9F, 0xDB, 0xA5, 0x72, 0x3B}, +} + +// IID_INetworkListManager {DCB00000-570F-4A9B-8D69-199FDBA5723B} +var iidINetworkListManager = windows.GUID{ + Data1: 0xDCB00000, Data2: 0x570F, Data3: 0x4A9B, + Data4: [8]byte{0x8D, 0x69, 0x19, 0x9F, 0xDB, 0xA5, 0x72, 0x3B}, +} + +// x/sys/windows doesn't expose CoCreateInstance, so we bind it ourselves. +var procCoCreateInstance = windows.NewLazySystemDLL("ole32.dll").NewProc("CoCreateInstance") + +const clsCtxAll = windows.CLSCTX_INPROC_SERVER | windows.CLSCTX_INPROC_HANDLER | + windows.CLSCTX_LOCAL_SERVER | windows.CLSCTX_REMOTE_SERVER + +const ( + hrSFALSE = 0x00000001 + hrRPCEChangedMode = 0x80010106 +) + +type hresult uint32 + +func (h hresult) failed() bool { return int32(h) < 0 } +func (h hresult) String() string { + return fmt.Sprintf("HRESULT 0x%08x", uint32(h)) +} + +var errAdapterNotFound = errors.New("adapter not present in network connections enumeration") + +// Vtable layouts. Slot order must match the declaration order in netlistmgr.h. +// All NLM interfaces here derive from IDispatch, which derives from IUnknown. + +type iUnknownVtbl struct { + QueryInterface uintptr + AddRef uintptr + Release uintptr +} + +type iDispatchVtbl struct { + iUnknownVtbl + GetTypeInfoCount uintptr + GetTypeInfo uintptr + GetIDsOfNames uintptr + Invoke uintptr +} + +type iNetworkListManagerVtbl struct { + iDispatchVtbl + GetNetworks uintptr + GetNetwork uintptr + GetNetworkConnections uintptr + GetNetworkConnection uintptr + IsConnectedToInternet uintptr + IsConnected uintptr + GetConnectivity uintptr +} + +type iNetworkListManager struct{ Vtbl *iNetworkListManagerVtbl } + +func (n *iNetworkListManager) Release() { + syscall.SyscallN(n.Vtbl.Release, uintptr(unsafe.Pointer(n))) +} + +func (n *iNetworkListManager) GetNetworkConnections() (*iEnumNetworkConnections, error) { + var enum *iEnumNetworkConnections + r1, _, _ := syscall.SyscallN(n.Vtbl.GetNetworkConnections, + uintptr(unsafe.Pointer(n)), uintptr(unsafe.Pointer(&enum)), + ) + if hr := hresult(r1); hr.failed() { + return nil, fmt.Errorf("INetworkListManager.GetNetworkConnections: %s", hr) + } + return enum, nil +} + +type iEnumNetworkConnectionsVtbl struct { + iDispatchVtbl + NewEnum uintptr + Next uintptr + Skip uintptr + Reset uintptr + Clone uintptr +} + +type iEnumNetworkConnections struct{ Vtbl *iEnumNetworkConnectionsVtbl } + +func (e *iEnumNetworkConnections) Release() { + syscall.SyscallN(e.Vtbl.Release, uintptr(unsafe.Pointer(e))) +} + +// Next returns the next connection, or (nil, nil) at the end of the enumeration. +func (e *iEnumNetworkConnections) Next() (*iNetworkConnection, error) { + var conn *iNetworkConnection + var fetched uint32 + r1, _, _ := syscall.SyscallN(e.Vtbl.Next, + uintptr(unsafe.Pointer(e)), 1, + uintptr(unsafe.Pointer(&conn)), uintptr(unsafe.Pointer(&fetched)), + ) + if hr := hresult(r1); hr.failed() { + return nil, fmt.Errorf("IEnumNetworkConnections.Next: %s", hr) + } + if fetched == 0 { + return nil, nil + } + return conn, nil +} + +type iNetworkConnectionVtbl struct { + iDispatchVtbl + GetNetwork uintptr + IsConnectedToInternet uintptr + IsConnected uintptr + GetConnectivity uintptr + GetConnectionId uintptr + GetAdapterId uintptr + GetDomainType uintptr +} + +type iNetworkConnection struct{ Vtbl *iNetworkConnectionVtbl } + +func (c *iNetworkConnection) Release() { + syscall.SyscallN(c.Vtbl.Release, uintptr(unsafe.Pointer(c))) +} + +func (c *iNetworkConnection) GetAdapterId() (windows.GUID, error) { + var g windows.GUID + r1, _, _ := syscall.SyscallN(c.Vtbl.GetAdapterId, + uintptr(unsafe.Pointer(c)), uintptr(unsafe.Pointer(&g)), + ) + if hr := hresult(r1); hr.failed() { + return windows.GUID{}, fmt.Errorf("INetworkConnection.GetAdapterId: %s", hr) + } + return g, nil +} + +func (c *iNetworkConnection) GetNetwork() (*iNetwork, error) { + var net *iNetwork + r1, _, _ := syscall.SyscallN(c.Vtbl.GetNetwork, + uintptr(unsafe.Pointer(c)), uintptr(unsafe.Pointer(&net)), + ) + if hr := hresult(r1); hr.failed() { + return nil, fmt.Errorf("INetworkConnection.GetNetwork: %s", hr) + } + return net, nil +} + +type iNetworkVtbl struct { + iDispatchVtbl + GetName uintptr + SetName uintptr + GetDescription uintptr + SetDescription uintptr + GetNetworkId uintptr + GetDomainType uintptr + GetNetworkConnections uintptr + GetTimeCreatedAndConnected uintptr + IsConnectedToInternet uintptr + IsConnected uintptr + GetConnectivity uintptr + GetCategory uintptr + SetCategory uintptr +} + +type iNetwork struct{ Vtbl *iNetworkVtbl } + +func (n *iNetwork) Release() { + syscall.SyscallN(n.Vtbl.Release, uintptr(unsafe.Pointer(n))) +} + +func (n *iNetwork) GetCategory() (networkCategory, error) { + var c networkCategory + r1, _, _ := syscall.SyscallN(n.Vtbl.GetCategory, + uintptr(unsafe.Pointer(n)), uintptr(unsafe.Pointer(&c)), + ) + if hr := hresult(r1); hr.failed() { + return 0, fmt.Errorf("INetwork.GetCategory: %s", hr) + } + return c, nil +} + +func (n *iNetwork) SetCategory(c networkCategory) error { + r1, _, _ := syscall.SyscallN(n.Vtbl.SetCategory, + uintptr(unsafe.Pointer(n)), uintptr(int32(c)), + ) + if hr := hresult(r1); hr.failed() { + return fmt.Errorf("INetwork.SetCategory: %s", hr) + } + return nil +} + +// coInit initializes COM for the current OS thread. The returned function must +// be deferred to balance a successful init. RPC_E_CHANGED_MODE means COM is +// already initialized in a different mode on this thread, which is still fine +// for our calls but we must not Uninitialize in that case. +func coInit() (func(), error) { + err := windows.CoInitializeEx(0, windows.COINIT_MULTITHREADED) + if err == nil { + return windows.CoUninitialize, nil + } + if e, ok := err.(syscall.Errno); ok { + switch uint32(e) { + case hrSFALSE: + return windows.CoUninitialize, nil + case hrRPCEChangedMode: + return func() {}, nil + } + } + return nil, fmt.Errorf("CoInitializeEx: %w", err) +} + +func createNetworkListManager() (*iNetworkListManager, error) { + var nlm *iNetworkListManager + r1, _, _ := procCoCreateInstance.Call( + uintptr(unsafe.Pointer(&clsidNetworkListManager)), + 0, + uintptr(clsCtxAll), + uintptr(unsafe.Pointer(&iidINetworkListManager)), + uintptr(unsafe.Pointer(&nlm)), + ) + if hr := hresult(r1); hr.failed() { + return nil, fmt.Errorf("CoCreateInstance(NetworkListManager): %s", hr) + } + return nlm, nil +} + +// setNetworkCategory locates the network connection bound to adapterGUID and +// sets the category of its parent network. Returns errAdapterNotFound if the +// adapter is not yet visible in the NLM enumeration. +func setNetworkCategory(adapterGUID windows.GUID, cat networkCategory) error { + deinit, err := coInit() + if err != nil { + return err + } + defer deinit() + + nlm, err := createNetworkListManager() + if err != nil { + return err + } + defer nlm.Release() + + enum, err := nlm.GetNetworkConnections() + if err != nil { + return err + } + defer enum.Release() + + for { + conn, err := enum.Next() + if err != nil { + return err + } + if conn == nil { + return errAdapterNotFound + } + + guid, err := conn.GetAdapterId() + if err != nil || guid != adapterGUID { + conn.Release() + continue + } + + net, err := conn.GetNetwork() + conn.Release() + if err != nil { + return err + } + err = net.SetCategory(cat) + net.Release() + return err + } +} + +// applyNetworkCategory polls until the wintun adapter shows up in the NLM +// enumeration, then sets the category. Intended to run in its own goroutine. +func applyNetworkCategory(l *slog.Logger, adapterGUID windows.GUID, cat networkCategory) { + // COM Init/Uninit must be paired on the same OS thread. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + const ( + attempts = 30 + interval = 500 * time.Millisecond + ) + for i := 0; i < attempts; i++ { + err := setNetworkCategory(adapterGUID, cat) + if err == nil { + l.Info("Set Windows network category", "category", cat.String()) + return + } + if !errors.Is(err, errAdapterNotFound) { + l.Warn("Failed to set Windows network category", "error", err, "category", cat.String()) + return + } + time.Sleep(interval) + } + l.Warn("Gave up waiting for adapter to appear in NLM enumeration; network category not set", + "category", cat.String(), + "waited", time.Duration(attempts)*interval, + ) +} diff --git a/overlay/network_category_windows_test.go b/overlay/network_category_windows_test.go new file mode 100644 index 00000000..c679f8c4 --- /dev/null +++ b/overlay/network_category_windows_test.go @@ -0,0 +1,109 @@ +//go:build !e2e_testing +// +build !e2e_testing + +package overlay + +import ( + "testing" +) + +func Test_parseNetworkCategory(t *testing.T) { + cases := []struct { + in string + wantCat networkCategory + wantApply bool + wantErr bool + }{ + {"", 0, false, false}, + {"unset", 0, false, false}, + {" UNSET ", 0, false, false}, + {"private", networkCategoryPrivate, true, false}, + {"Private", networkCategoryPrivate, true, false}, + {" PRIVATE ", networkCategoryPrivate, true, false}, + {"public", networkCategoryPublic, true, false}, + {"PUBLIC", networkCategoryPublic, true, false}, + {"domain", networkCategoryDomainAuthenticated, true, false}, + {"DomainAuthenticated", networkCategoryDomainAuthenticated, true, false}, + {"garbage", 0, false, true}, + {"privates", 0, false, true}, + } + for _, tc := range cases { + cat, apply, err := parseNetworkCategory(tc.in) + if (err != nil) != tc.wantErr { + t.Errorf("parseNetworkCategory(%q) err=%v, wantErr=%v", tc.in, err, tc.wantErr) + continue + } + if cat != tc.wantCat || apply != tc.wantApply { + t.Errorf("parseNetworkCategory(%q) = (%v, %v), want (%v, %v)", tc.in, cat, apply, tc.wantCat, tc.wantApply) + } + } +} + +// Test_NLM_round_trip exercises every COM call path used by setNetworkCategory +// without mutating the host's network state. It validates the CLSID/IID +// constants and every vtable index by enumerating connections, fetching the +// adapter id and parent network, reading the current category, and writing it +// back unchanged. +// +// Requires Windows but does not require admin or the wintun driver. Skips if +// no network connections are available (unlikely outside of an isolated +// container). +func Test_NLM_round_trip(t *testing.T) { + deinit, err := coInit() + if err != nil { + t.Fatalf("coInit: %v", err) + } + defer deinit() + + nlm, err := createNetworkListManager() + if err != nil { + t.Fatalf("createNetworkListManager: %v", err) + } + defer nlm.Release() + + enum, err := nlm.GetNetworkConnections() + if err != nil { + t.Fatalf("GetNetworkConnections: %v", err) + } + defer enum.Release() + + saw := 0 + for { + conn, err := enum.Next() + if err != nil { + t.Fatalf("EnumNetworkConnections.Next: %v", err) + } + if conn == nil { + break + } + saw++ + + if _, err := conn.GetAdapterId(); err != nil { + conn.Release() + t.Fatalf("INetworkConnection.GetAdapterId: %v", err) + } + + net, err := conn.GetNetwork() + conn.Release() + if err != nil { + t.Fatalf("INetworkConnection.GetNetwork: %v", err) + } + + cat, err := net.GetCategory() + if err != nil { + net.Release() + t.Fatalf("INetwork.GetCategory: %v", err) + } + // Set to the current value so the host's NLM state is unchanged but + // SetCategory's vtable slot is still validated end-to-end. + if err := net.SetCategory(cat); err != nil { + net.Release() + t.Fatalf("INetwork.SetCategory(%v): %v", cat, err) + } + net.Release() + } + + if saw == 0 { + t.Skip("no NLM network connections available; skipping round-trip") + } +} diff --git a/overlay/overlaytest/noop.go b/overlay/overlaytest/noop.go new file mode 100644 index 00000000..956da7dd --- /dev/null +++ b/overlay/overlaytest/noop.go @@ -0,0 +1,52 @@ +// Package overlaytest provides fakes of overlay.Device for tests that do +// not want to touch a real tun device or route table. +package overlaytest + +import ( + "errors" + "io" + "net/netip" + + "github.com/slackhq/nebula/routing" +) + +// NoopTun is an overlay.Device that silently discards every read and write. +// Useful in tests that need to construct a nebula Interface but do not +// exercise the datapath. +type NoopTun struct{} + +func (NoopTun) RoutesFor(addr netip.Addr) routing.Gateways { + return routing.Gateways{} +} + +func (NoopTun) Activate() error { + return nil +} + +func (NoopTun) Networks() []netip.Prefix { + return []netip.Prefix{} +} + +func (NoopTun) Name() string { + return "noop" +} + +func (NoopTun) Read([]byte) (int, error) { + return 0, nil +} + +func (NoopTun) Write([]byte) (int, error) { + return 0, nil +} + +func (NoopTun) SupportsMultiqueue() bool { + return false +} + +func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) { + return nil, errors.New("unsupported") +} + +func (NoopTun) Close() error { + return nil +} diff --git a/overlay/route.go b/overlay/route.go index 64c624c7..c6403f91 100644 --- a/overlay/route.go +++ b/overlay/route.go @@ -1,34 +1,31 @@ package overlay import ( - "bytes" "fmt" + "log/slog" "math" "net" + "net/netip" "runtime" "strconv" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" ) type Route struct { MTU int Metric int - Cidr *net.IPNet - Via *iputil.VpnIp + Cidr netip.Prefix + Via routing.Gateways Install bool } // Equal determines if a route that could be installed in the system route table is equal to another // Via is ignored since that is only consumed within nebula itself func (r Route) Equal(t Route) bool { - if !r.Cidr.IP.Equal(t.Cidr.IP) { - return false - } - if !bytes.Equal(r.Cidr.Mask, t.Cidr.Mask) { + if r.Cidr != t.Cidr { return false } if r.Metric != t.Metric { @@ -51,21 +48,26 @@ func (r Route) String() string { return s } -func makeRouteTree(l *logrus.Logger, routes []Route, allowMTU bool) (*cidr.Tree4[iputil.VpnIp], error) { - routeTree := cidr.NewTree4[iputil.VpnIp]() +func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) { + routeTree := new(bart.Table[routing.Gateways]) for _, r := range routes { if !allowMTU && r.MTU > 0 { - l.WithField("route", r).Warnf("route MTU is not supported in %s", runtime.GOOS) + l.Warn("route MTU is not supported on this platform", + "goos", runtime.GOOS, + "route", r, + ) } - if r.Via != nil { - routeTree.AddCIDR(r.Cidr, *r.Via) + gateways := r.Via + if len(gateways) > 0 { + routing.CalculateBucketsForGateways(gateways) + routeTree.Insert(r.Cidr, gateways) } } return routeTree, nil } -func parseRoutes(c *config.C, network *net.IPNet) ([]Route, error) { +func parseRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) { var err error r := c.Get("tun.routes") @@ -73,7 +75,7 @@ func parseRoutes(c *config.C, network *net.IPNet) ([]Route, error) { return []Route{}, nil } - rawRoutes, ok := r.([]interface{}) + rawRoutes, ok := r.([]any) if !ok { return nil, fmt.Errorf("tun.routes is not an array") } @@ -84,7 +86,7 @@ func parseRoutes(c *config.C, network *net.IPNet) ([]Route, error) { routes := make([]Route, len(rawRoutes)) for i, r := range rawRoutes { - m, ok := r.(map[interface{}]interface{}) + m, ok := r.(map[string]any) if !ok { return nil, fmt.Errorf("entry %v in tun.routes is invalid", i+1) } @@ -116,17 +118,25 @@ func parseRoutes(c *config.C, network *net.IPNet) ([]Route, error) { MTU: mtu, } - _, r.Cidr, err = net.ParseCIDR(fmt.Sprintf("%v", rRoute)) + r.Cidr, err = netip.ParsePrefix(fmt.Sprintf("%v", rRoute)) if err != nil { return nil, fmt.Errorf("entry %v.route in tun.routes failed to parse: %v", i+1, err) } - if !ipWithin(network, r.Cidr) { + found := false + for _, network := range networks { + if network.Contains(r.Cidr.Addr()) && r.Cidr.Bits() >= network.Bits() { + found = true + break + } + } + + if !found { return nil, fmt.Errorf( - "entry %v.route in tun.routes is not contained within the network attached to the certificate; route: %v, network: %v", + "entry %v.route in tun.routes is not contained within the configured vpn networks; route: %v, networks: %v", i+1, r.Cidr.String(), - network.String(), + networks, ) } @@ -136,7 +146,7 @@ func parseRoutes(c *config.C, network *net.IPNet) ([]Route, error) { return routes, nil } -func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]Route, error) { +func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) { var err error r := c.Get("tun.unsafe_routes") @@ -144,7 +154,7 @@ func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]Route, error) { return []Route{}, nil } - rawRoutes, ok := r.([]interface{}) + rawRoutes, ok := r.([]any) if !ok { return nil, fmt.Errorf("tun.unsafe_routes is not an array") } @@ -155,7 +165,7 @@ func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]Route, error) { routes := make([]Route, len(rawRoutes)) for i, r := range rawRoutes { - m, ok := r.(map[interface{}]interface{}) + m, ok := r.(map[string]any) if !ok { return nil, fmt.Errorf("entry %v in tun.unsafe_routes is invalid", i+1) } @@ -197,14 +207,63 @@ func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]Route, error) { return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not present", i+1) } - via, ok := rVia.(string) - if !ok { - return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not a string: found %T", i+1, rVia) - } + var gateways routing.Gateways + + switch via := rVia.(type) { + case string: + viaIp, err := netip.ParseAddr(via) + if err != nil { + return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes failed to parse address: %v", i+1, err) + } + + gateways = routing.Gateways{routing.NewGateway(viaIp, 1)} + + case []any: + gateways = make(routing.Gateways, len(via)) + for ig, v := range via { + gatewayMap, ok := v.(map[string]any) + if !ok { + return nil, fmt.Errorf("entry %v in tun.unsafe_routes[%v].via is invalid", i+1, ig+1) + } + + rGateway, ok := gatewayMap["gateway"] + if !ok { + return nil, fmt.Errorf("entry .gateway in tun.unsafe_routes[%v].via[%v] is not present", i+1, ig+1) + } + + parsedGateway, ok := rGateway.(string) + if !ok { + return nil, fmt.Errorf("entry .gateway in tun.unsafe_routes[%v].via[%v] is not a string", i+1, ig+1) + } - nVia := net.ParseIP(via) - if nVia == nil { - return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes failed to parse address: %v", i+1, via) + gatewayIp, err := netip.ParseAddr(parsedGateway) + if err != nil { + return nil, fmt.Errorf("entry .gateway in tun.unsafe_routes[%v].via[%v] failed to parse address: %v", i+1, ig+1, err) + } + + rGatewayWeight, ok := gatewayMap["weight"] + if !ok { + rGatewayWeight = 1 + } + + gatewayWeight, ok := rGatewayWeight.(int) + if !ok { + _, err = strconv.ParseInt(rGatewayWeight.(string), 10, 32) + if err != nil { + return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1) + } + } + + if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 { + return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not in range (1-%d) : %v", i+1, ig+1, math.MaxInt32, gatewayWeight) + } + + gateways[ig] = routing.NewGateway(gatewayIp, gatewayWeight) + + } + + default: + return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not a string or list of gateways: found %T", i+1, rVia) } rRoute, ok := m["route"] @@ -212,8 +271,6 @@ func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]Route, error) { return nil, fmt.Errorf("entry %v.route in tun.unsafe_routes is not present", i+1) } - viaVpnIp := iputil.Ip2VpnIp(nVia) - install := true rInstall, ok := m["install"] if ok { @@ -224,24 +281,26 @@ func parseUnsafeRoutes(c *config.C, network *net.IPNet) ([]Route, error) { } r := Route{ - Via: &viaVpnIp, + Via: gateways, MTU: mtu, Metric: metric, Install: install, } - _, r.Cidr, err = net.ParseCIDR(fmt.Sprintf("%v", rRoute)) + r.Cidr, err = netip.ParsePrefix(fmt.Sprintf("%v", rRoute)) if err != nil { return nil, fmt.Errorf("entry %v.route in tun.unsafe_routes failed to parse: %v", i+1, err) } - if ipWithin(network, r.Cidr) { - return nil, fmt.Errorf( - "entry %v.route in tun.unsafe_routes is contained within the network attached to the certificate; route: %v, network: %v", - i+1, - r.Cidr.String(), - network.String(), - ) + for _, network := range networks { + if network.Contains(r.Cidr.Addr()) { + return nil, fmt.Errorf( + "entry %v.route in tun.unsafe_routes is contained within the configured vpn networks; route: %v, network: %v", + i+1, + r.Cidr.String(), + network.String(), + ) + } } routes[i] = r diff --git a/overlay/route_test.go b/overlay/route_test.go index 46fb87ce..f9d9dcd9 100644 --- a/overlay/route_test.go +++ b/overlay/route_test.go @@ -2,92 +2,100 @@ package overlay import ( "fmt" - "net" + "net/netip" "testing" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_parseRoutes(t *testing.T) { l := test.NewLogger() c := config.NewC(l) - _, n, _ := net.ParseCIDR("10.0.0.0/24") + n, err := netip.ParsePrefix("10.0.0.0/24") + require.NoError(t, err) // test no routes config - routes, err := parseRoutes(c, n) - assert.Nil(t, err) - assert.Len(t, routes, 0) + routes, err := parseRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) + assert.Empty(t, routes) // not an array - c.Settings["tun"] = map[interface{}]interface{}{"routes": "hi"} - routes, err = parseRoutes(c, n) + c.Settings["tun"] = map[string]any{"routes": "hi"} + routes, err = parseRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "tun.routes is not an array") + require.EqualError(t, err, "tun.routes is not an array") // no routes - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{}} - routes, err = parseRoutes(c, n) - assert.Nil(t, err) - assert.Len(t, routes, 0) + c.Settings["tun"] = map[string]any{"routes": []any{}} + routes, err = parseRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) + assert.Empty(t, routes) // weird route - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{"asdf"}} - routes, err = parseRoutes(c, n) + c.Settings["tun"] = map[string]any{"routes": []any{"asdf"}} + routes, err = parseRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1 in tun.routes is invalid") + require.EqualError(t, err, "entry 1 in tun.routes is invalid") // no mtu - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{}}} - routes, err = parseRoutes(c, n) + c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{}}} + routes, err = parseRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.mtu in tun.routes is not present") + require.EqualError(t, err, "entry 1.mtu in tun.routes is not present") // bad mtu - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "nope"}}} - routes, err = parseRoutes(c, n) + c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"mtu": "nope"}}} + routes, err = parseRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.mtu in tun.routes is not an integer: strconv.Atoi: parsing \"nope\": invalid syntax") + require.EqualError(t, err, "entry 1.mtu in tun.routes is not an integer: strconv.Atoi: parsing \"nope\": invalid syntax") // low mtu - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "499"}}} - routes, err = parseRoutes(c, n) + c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"mtu": "499"}}} + routes, err = parseRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.mtu in tun.routes is below 500: 499") + require.EqualError(t, err, "entry 1.mtu in tun.routes is below 500: 499") // missing route - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "500"}}} - routes, err = parseRoutes(c, n) + c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"mtu": "500"}}} + routes, err = parseRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.route in tun.routes is not present") + require.EqualError(t, err, "entry 1.route in tun.routes is not present") // unparsable route - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "route": "nope"}}} - routes, err = parseRoutes(c, n) + c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"mtu": "500", "route": "nope"}}} + routes, err = parseRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.route in tun.routes failed to parse: invalid CIDR address: nope") + require.EqualError(t, err, "entry 1.route in tun.routes failed to parse: netip.ParsePrefix(\"nope\"): no '/'") // below network range - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "route": "1.0.0.0/8"}}} - routes, err = parseRoutes(c, n) + c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"mtu": "500", "route": "1.0.0.0/8"}}} + routes, err = parseRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.route in tun.routes is not contained within the network attached to the certificate; route: 1.0.0.0/8, network: 10.0.0.0/24") + require.EqualError(t, err, "entry 1.route in tun.routes is not contained within the configured vpn networks; route: 1.0.0.0/8, networks: [10.0.0.0/24]") // above network range - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "route": "10.0.1.0/24"}}} - routes, err = parseRoutes(c, n) + c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"mtu": "500", "route": "10.0.1.0/24"}}} + routes, err = parseRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.route in tun.routes is not contained within the network attached to the certificate; route: 10.0.1.0/24, network: 10.0.0.0/24") + require.EqualError(t, err, "entry 1.route in tun.routes is not contained within the configured vpn networks; route: 10.0.1.0/24, networks: [10.0.0.0/24]") + + // Not in multiple ranges + c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"mtu": "500", "route": "192.0.0.0/24"}}} + routes, err = parseRoutes(c, []netip.Prefix{n, netip.MustParsePrefix("192.1.0.0/24")}) + assert.Nil(t, routes) + require.EqualError(t, err, "entry 1.route in tun.routes is not contained within the configured vpn networks; route: 192.0.0.0/24, networks: [10.0.0.0/24 192.1.0.0/24]") // happy case - c.Settings["tun"] = map[interface{}]interface{}{"routes": []interface{}{ - map[interface{}]interface{}{"mtu": "9000", "route": "10.0.0.0/29"}, - map[interface{}]interface{}{"mtu": "8000", "route": "10.0.0.1/32"}, + c.Settings["tun"] = map[string]any{"routes": []any{ + map[string]any{"mtu": "9000", "route": "10.0.0.0/29"}, + map[string]any{"mtu": "8000", "route": "10.0.0.1/32"}, }} - routes, err = parseRoutes(c, n) - assert.Nil(t, err) + routes, err = parseRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) assert.Len(t, routes, 2) tested := 0 @@ -112,116 +120,141 @@ func Test_parseRoutes(t *testing.T) { func Test_parseUnsafeRoutes(t *testing.T) { l := test.NewLogger() c := config.NewC(l) - _, n, _ := net.ParseCIDR("10.0.0.0/24") + n, err := netip.ParsePrefix("10.0.0.0/24") + require.NoError(t, err) // test no routes config - routes, err := parseUnsafeRoutes(c, n) - assert.Nil(t, err) - assert.Len(t, routes, 0) + routes, err := parseUnsafeRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) + assert.Empty(t, routes) // not an array - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": "hi"} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": "hi"} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "tun.unsafe_routes is not an array") + require.EqualError(t, err, "tun.unsafe_routes is not an array") // no routes - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{}} - routes, err = parseUnsafeRoutes(c, n) - assert.Nil(t, err) - assert.Len(t, routes, 0) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) + assert.Empty(t, routes) // weird route - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{"asdf"}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{"asdf"}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1 in tun.unsafe_routes is invalid") + require.EqualError(t, err, "entry 1 in tun.unsafe_routes is invalid") // no via - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.via in tun.unsafe_routes is not present") + require.EqualError(t, err, "entry 1.via in tun.unsafe_routes is not present") // invalid via - for _, invalidValue := range []interface{}{ + for _, invalidValue := range []any{ 127, false, nil, 1.0, []string{"1", "2"}, } { - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": invalidValue}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": invalidValue}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, fmt.Sprintf("entry 1.via in tun.unsafe_routes is not a string: found %T", invalidValue)) + require.EqualError(t, err, fmt.Sprintf("entry 1.via in tun.unsafe_routes is not a string or list of gateways: found %T", invalidValue)) } + // Unparsable list of via + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": []string{"1", "2"}}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + assert.Nil(t, routes) + require.EqualError(t, err, "entry 1.via in tun.unsafe_routes is not a string or list of gateways: found []string") + // unparsable via - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "via": "nope"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"mtu": "500", "via": "nope"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + assert.Nil(t, routes) + require.EqualError(t, err, "entry 1.via in tun.unsafe_routes failed to parse address: ParseAddr(\"nope\"): unable to parse IP") + + // unparsable gateway + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"mtu": "500", "via": []any{map[string]any{"gateway": "1"}}}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + assert.Nil(t, routes) + require.EqualError(t, err, "entry .gateway in tun.unsafe_routes[1].via[1] failed to parse address: ParseAddr(\"1\"): unable to parse IP") + + // missing gateway element + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"mtu": "500", "via": []any{map[string]any{"weight": "1"}}}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.via in tun.unsafe_routes failed to parse address: nope") + require.EqualError(t, err, "entry .gateway in tun.unsafe_routes[1].via[1] is not present") + + // unparsable weight element + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"mtu": "500", "via": []any{map[string]any{"gateway": "10.0.0.1", "weight": "a"}}}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + assert.Nil(t, routes) + require.EqualError(t, err, "entry .weight in tun.unsafe_routes[1].via[1] is not an integer") // missing route - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "500"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "mtu": "500"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.route in tun.unsafe_routes is not present") + require.EqualError(t, err, "entry 1.route in tun.unsafe_routes is not present") // unparsable route - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "500", "route": "nope"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "mtu": "500", "route": "nope"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.route in tun.unsafe_routes failed to parse: invalid CIDR address: nope") + require.EqualError(t, err, "entry 1.route in tun.unsafe_routes failed to parse: netip.ParsePrefix(\"nope\"): no '/'") // within network range - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "route": "10.0.0.0/24"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "route": "10.0.0.0/24"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.route in tun.unsafe_routes is contained within the network attached to the certificate; route: 10.0.0.0/24, network: 10.0.0.0/24") + require.EqualError(t, err, "entry 1.route in tun.unsafe_routes is contained within the configured vpn networks; route: 10.0.0.0/24, network: 10.0.0.0/24") // below network range - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "route": "1.0.0.0/8"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "route": "1.0.0.0/8"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Len(t, routes, 1) - assert.Nil(t, err) + require.NoError(t, err) // above network range - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "route": "10.0.1.0/24"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "route": "10.0.1.0/24"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Len(t, routes, 1) - assert.Nil(t, err) + require.NoError(t, err) // no mtu - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "route": "1.0.0.0/8"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "route": "1.0.0.0/8"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Len(t, routes, 1) assert.Equal(t, 0, routes[0].MTU) // bad mtu - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "nope"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "mtu": "nope"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is not an integer: strconv.Atoi: parsing \"nope\": invalid syntax") + require.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is not an integer: strconv.Atoi: parsing \"nope\": invalid syntax") // low mtu - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "499"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "mtu": "499"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is below 500: 499") + require.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is below 500: 499") // bad install - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29", "install": "nope"}}} - routes, err = parseUnsafeRoutes(c, n) + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29", "install": "nope"}}} + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) assert.Nil(t, routes) - assert.EqualError(t, err, "entry 1.install in tun.unsafe_routes is not a boolean: strconv.ParseBool: parsing \"nope\": invalid syntax") + require.EqualError(t, err, "entry 1.install in tun.unsafe_routes is not a boolean: strconv.ParseBool: parsing \"nope\": invalid syntax") // happy case - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{ - map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29", "install": "t"}, - map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "8000", "route": "1.0.0.1/32", "install": 0}, - map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "1500", "metric": 1234, "route": "1.0.0.2/32", "install": 1}, - map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "1500", "metric": 1234, "route": "1.0.0.2/32"}, + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{ + map[string]any{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29", "install": "t"}, + map[string]any{"via": "127.0.0.1", "mtu": "8000", "route": "1.0.0.1/32", "install": 0}, + map[string]any{"via": "127.0.0.1", "mtu": "1500", "metric": 1234, "route": "1.0.0.2/32", "install": 1}, + map[string]any{"via": "127.0.0.1", "mtu": "1500", "metric": 1234, "route": "1.0.0.2/32"}, }} - routes, err = parseUnsafeRoutes(c, n) - assert.Nil(t, err) + routes, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) assert.Len(t, routes, 4) tested := 0 @@ -252,29 +285,120 @@ func Test_parseUnsafeRoutes(t *testing.T) { func Test_makeRouteTree(t *testing.T) { l := test.NewLogger() c := config.NewC(l) - _, n, _ := net.ParseCIDR("10.0.0.0/24") + n, err := netip.ParsePrefix("10.0.0.0/24") + require.NoError(t, err) - c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{ - map[interface{}]interface{}{"via": "192.168.0.1", "route": "1.0.0.0/28"}, - map[interface{}]interface{}{"via": "192.168.0.2", "route": "1.0.0.1/32"}, + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{ + map[string]any{"via": "192.168.0.1", "route": "1.0.0.0/28"}, + map[string]any{"via": "192.168.0.2", "route": "1.0.0.1/32"}, }} - routes, err := parseUnsafeRoutes(c, n) - assert.NoError(t, err) + routes, err := parseUnsafeRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) assert.Len(t, routes, 2) - routeTree, err := makeRouteTree(l, routes, true) - assert.NoError(t, err) + routeTree, err := makeRouteTree(test.NewLogger(), routes, true) + require.NoError(t, err) - ip := iputil.Ip2VpnIp(net.ParseIP("1.0.0.2")) - ok, r := routeTree.MostSpecificContains(ip) + ip, err := netip.ParseAddr("1.0.0.2") + require.NoError(t, err) + r, ok := routeTree.Lookup(ip) assert.True(t, ok) - assert.Equal(t, iputil.Ip2VpnIp(net.ParseIP("192.168.0.1")), r) - ip = iputil.Ip2VpnIp(net.ParseIP("1.0.0.1")) - ok, r = routeTree.MostSpecificContains(ip) + nip, err := netip.ParseAddr("192.168.0.1") + require.NoError(t, err) + assert.Equal(t, nip, r[0].Addr()) + + ip, err = netip.ParseAddr("1.0.0.1") + require.NoError(t, err) + r, ok = routeTree.Lookup(ip) assert.True(t, ok) - assert.Equal(t, iputil.Ip2VpnIp(net.ParseIP("192.168.0.2")), r) - ip = iputil.Ip2VpnIp(net.ParseIP("1.1.0.1")) - ok, r = routeTree.MostSpecificContains(ip) + nip, err = netip.ParseAddr("192.168.0.2") + require.NoError(t, err) + assert.Equal(t, nip, r[0].Addr()) + + ip, err = netip.ParseAddr("1.1.0.1") + require.NoError(t, err) + r, ok = routeTree.Lookup(ip) assert.False(t, ok) } + +func Test_makeMultipathUnsafeRouteTree(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + n, err := netip.ParsePrefix("10.0.0.0/24") + require.NoError(t, err) + + c.Settings["tun"] = map[string]any{ + "unsafe_routes": []any{ + map[string]any{ + "route": "192.168.86.0/24", + "via": "192.168.100.10", + }, + map[string]any{ + "route": "192.168.87.0/24", + "via": []any{ + map[string]any{ + "gateway": "10.0.0.1", + }, + map[string]any{ + "gateway": "10.0.0.2", + }, + map[string]any{ + "gateway": "10.0.0.3", + }, + }, + }, + map[string]any{ + "route": "192.168.89.0/24", + "via": []any{ + map[string]any{ + "gateway": "10.0.0.1", + "weight": 10, + }, + map[string]any{ + "gateway": "10.0.0.2", + "weight": 5, + }, + }, + }, + }, + } + + routes, err := parseUnsafeRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) + assert.Len(t, routes, 3) + routeTree, err := makeRouteTree(test.NewLogger(), routes, true) + require.NoError(t, err) + + ip, err := netip.ParseAddr("192.168.86.1") + require.NoError(t, err) + r, ok := routeTree.Lookup(ip) + assert.True(t, ok) + + nip, err := netip.ParseAddr("192.168.100.10") + require.NoError(t, err) + assert.Equal(t, nip, r[0].Addr()) + + ip, err = netip.ParseAddr("192.168.87.1") + require.NoError(t, err) + r, ok = routeTree.Lookup(ip) + assert.True(t, ok) + + expectedGateways := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.0.0.1"), 1), + routing.NewGateway(netip.MustParseAddr("10.0.0.2"), 1), + routing.NewGateway(netip.MustParseAddr("10.0.0.3"), 1)} + + routing.CalculateBucketsForGateways(expectedGateways) + assert.ElementsMatch(t, expectedGateways, r) + + ip, err = netip.ParseAddr("192.168.89.1") + require.NoError(t, err) + r, ok = routeTree.Lookup(ip) + assert.True(t, ok) + + expectedGateways = routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.0.0.1"), 10), + routing.NewGateway(netip.MustParseAddr("10.0.0.2"), 5)} + + routing.CalculateBucketsForGateways(expectedGateways) + assert.ElementsMatch(t, expectedGateways, r) +} diff --git a/overlay/tun.go b/overlay/tun.go index cedd7fe7..3af1e189 100644 --- a/overlay/tun.go +++ b/overlay/tun.go @@ -1,46 +1,57 @@ package overlay import ( + "fmt" + "log/slog" "net" + "net/netip" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/util" ) const DefaultMTU = 1300 +type NameError struct { + Name string + Underlying error +} + +func (e *NameError) Error() string { + return fmt.Sprintf("could not set tun device name: %s because %s", e.Name, e.Underlying) +} + // TODO: We may be able to remove routines -type DeviceFactory func(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error) +type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) -func NewDeviceFromConfig(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error) { +func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { switch { case c.GetBool("tun.disabled", false): - tun := newDisabledTun(tunCidr, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l) + tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l) return tun, nil default: - return newTun(c, l, tunCidr, routines > 1) + return newTun(c, l, vpnNetworks, routines > 1) } } func NewFdDeviceFromConfig(fd *int) DeviceFactory { - return func(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error) { - return newTunFromFd(c, l, *fd, tunCidr) + return func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { + return newTunFromFd(c, l, *fd, vpnNetworks) } } -func getAllRoutesFromConfig(c *config.C, cidr *net.IPNet, initial bool) (bool, []Route, error) { +func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) { if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") { return false, nil, nil } - routes, err := parseRoutes(c, cidr) + routes, err := parseRoutes(c, vpnNetworks) if err != nil { return true, nil, util.NewContextualError("Could not parse tun.routes", nil, err) } - unsafeRoutes, err := parseUnsafeRoutes(c, cidr) + unsafeRoutes, err := parseUnsafeRoutes(c, vpnNetworks) if err != nil { return true, nil, util.NewContextualError("Could not parse tun.unsafe_routes", nil, err) } @@ -70,3 +81,51 @@ func findRemovedRoutes(newRoutes, oldRoutes []Route) []Route { return removed } + +func prefixToMask(prefix netip.Prefix) netip.Addr { + pLen := 128 + if prefix.Addr().Is4() { + pLen = 32 + } + + addr, _ := netip.AddrFromSlice(net.CIDRMask(prefix.Bits(), pLen)) + return addr +} + +func flipBytes(b []byte) []byte { + for i := 0; i < len(b); i++ { + b[i] ^= 0xFF + } + return b +} +func orBytes(a []byte, b []byte) []byte { + ret := make([]byte, len(a)) + for i := 0; i < len(a); i++ { + ret[i] = a[i] | b[i] + } + return ret +} + +func getBroadcast(cidr netip.Prefix) netip.Addr { + broadcast, _ := netip.AddrFromSlice( + orBytes( + cidr.Addr().AsSlice(), + flipBytes(prefixToMask(cidr).AsSlice()), + ), + ) + return broadcast +} + +func selectGateway(dest netip.Prefix, gateways []netip.Prefix) (netip.Prefix, error) { + for _, gateway := range gateways { + if dest.Addr().Is4() && gateway.Addr().Is4() { + return gateway, nil + } + + if dest.Addr().Is6() && gateway.Addr().Is6() { + return gateway, nil + } + } + + return netip.Prefix{}, fmt.Errorf("no gateway found for %v in the list of vpn networks", dest) +} diff --git a/overlay/tun_android.go b/overlay/tun_android.go index c15827fe..e4080b41 100644 --- a/overlay/tun_android.go +++ b/overlay/tun_android.go @@ -6,27 +6,27 @@ package overlay import ( "fmt" "io" - "net" + "log/slog" + "net/netip" "os" "sync/atomic" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" ) type tun struct { io.ReadWriteCloser - fd int - cidr *net.IPNet - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] - l *logrus.Logger + fd int + vpnNetworks []netip.Prefix + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[bart.Table[routing.Gateways]] + l *slog.Logger } -func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) (*tun, error) { +func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) { // XXX Android returns an fd in non-blocking mode which is necessary for shutdown to work properly. // Be sure not to call file.Fd() as it will set the fd to blocking mode. file := os.NewFile(uintptr(deviceFd), "/dev/net/tun") @@ -34,12 +34,13 @@ func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) t := &tun{ ReadWriteCloser: file, fd: deviceFd, - cidr: cidr, + vpnNetworks: vpnNetworks, l: l, } err := t.reload(c, true) if err != nil { + _ = file.Close() return nil, err } @@ -53,12 +54,12 @@ func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) return t, nil } -func newTun(_ *config.C, _ *logrus.Logger, _ *net.IPNet, _ bool) (*tun, error) { +func newTun(_ *config.C, _ *slog.Logger, _ []netip.Prefix, _ bool) (*tun, error) { return nil, fmt.Errorf("newTun not supported in Android") } -func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.Load().MostSpecificContains(ip) +func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { + r, _ := t.routeTree.Load().Lookup(ip) return r } @@ -67,7 +68,7 @@ func (t tun) Activate() error { } func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) if err != nil { return err } @@ -87,14 +88,18 @@ func (t *tun) reload(c *config.C, initial bool) error { return nil } -func (t *tun) Cidr() *net.IPNet { - return t.cidr +func (t *tun) Networks() []netip.Prefix { + return t.vpnNetworks } func (t *tun) Name() string { return "android" } +func (t *tun) SupportsMultiqueue() bool { + return false +} + func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { return nil, fmt.Errorf("TODO: multiqueue not implemented for android") } diff --git a/overlay/tun_bypass_windows.go b/overlay/tun_bypass_windows.go new file mode 100644 index 00000000..1f62373c --- /dev/null +++ b/overlay/tun_bypass_windows.go @@ -0,0 +1,23 @@ +//go:build (amd64 || arm64) && !e2e_testing +// +build amd64 arm64 +// +build !e2e_testing + +package overlay + +import ( + "log/slog" + + "github.com/slackhq/nebula/wfp" +) + +// installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the +// nebula adapter bypasses Windows Defender Firewall. +func installInterfaceBypass(l *slog.Logger, luid uint64) closer { + s, err := wfp.PermitInterface(luid) + if err != nil { + l.Warn("Failed to install WFP bypass filters on nebula interface", "error", err) + return nil + } + l.Info("Installed WFP filters bypassing Windows Defender Firewall on nebula interface") + return s +} diff --git a/overlay/tun_bypass_windows_386.go b/overlay/tun_bypass_windows_386.go new file mode 100644 index 00000000..366430b0 --- /dev/null +++ b/overlay/tun_bypass_windows_386.go @@ -0,0 +1,11 @@ +//go:build !e2e_testing +// +build !e2e_testing + +package overlay + +import "log/slog" + +// installInterfaceBypass is a no-op on windows-386 because we don't currently build for it. +func installInterfaceBypass(_ *slog.Logger, _ uint64) closer { + return nil +} diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index 1c638282..d30148b9 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -7,73 +7,76 @@ import ( "errors" "fmt" "io" - "net" + "log/slog" + "net/netip" "os" "sync/atomic" "syscall" "unsafe" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" netroute "golang.org/x/net/route" "golang.org/x/sys/unix" ) type tun struct { - io.ReadWriteCloser - Device string - cidr *net.IPNet - DefaultMTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] - linkAddr *netroute.LinkAddr - l *logrus.Logger - - // cache out buffer since we need to prepend 4 bytes for tun metadata - out []byte -} - -type sockaddrCtl struct { - scLen uint8 - scFamily uint8 - ssSysaddr uint16 - scID uint32 - scUnit uint32 - scReserved [5]uint32 + f *os.File + Device string + vpnNetworks []netip.Prefix + DefaultMTU int + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[bart.Table[routing.Gateways]] + linkAddr *netroute.LinkAddr + l *slog.Logger } type ifReq struct { - Name [16]byte + Name [unix.IFNAMSIZ]byte Flags uint16 pad [8]byte } -var sockaddrCtlSize uintptr = 32 - const ( - _SYSPROTO_CONTROL = 2 //define SYSPROTO_CONTROL 2 /* kernel control protocol */ - _AF_SYS_CONTROL = 2 //#define AF_SYS_CONTROL 2 /* corresponding sub address type */ - _PF_SYSTEM = unix.AF_SYSTEM //#define PF_SYSTEM AF_SYSTEM - _CTLIOCGINFO = 3227799043 //#define CTLIOCGINFO _IOWR('N', 3, struct ctl_info) - utunControlName = "com.apple.net.utun_control" + _SIOCAIFADDR_IN6 = 2155899162 + _UTUN_OPT_IFNAME = 2 + _IN6_IFF_NODAD = 0x0020 + _IN6_IFF_SECURED = 0x0400 + utunControlName = "com.apple.net.utun_control" ) -type ifreqAddr struct { - Name [16]byte - Addr unix.RawSockaddrInet4 - pad [8]byte -} - type ifreqMTU struct { Name [16]byte MTU int32 pad [8]byte } -func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) { +type addrLifetime struct { + Expire float64 + Preferred float64 + Vltime uint32 + Pltime uint32 +} + +type ifreqAlias4 struct { + Name [unix.IFNAMSIZ]byte + Addr unix.RawSockaddrInet4 + DstAddr unix.RawSockaddrInet4 + MaskAddr unix.RawSockaddrInet4 +} + +type ifreqAlias6 struct { + Name [unix.IFNAMSIZ]byte + Addr unix.RawSockaddrInet6 + DstAddr unix.RawSockaddrInet6 + PrefixMask unix.RawSockaddrInet6 + Flags uint32 + Lifetime addrLifetime +} + +func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { name := c.GetString("tun.dev", "") ifIndex := -1 if name != "" && name != "utun" { @@ -86,68 +89,43 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error } } - fd, err := unix.Socket(_PF_SYSTEM, unix.SOCK_DGRAM, _SYSPROTO_CONTROL) + fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL) if err != nil { return nil, fmt.Errorf("system socket: %v", err) } - var ctlInfo = &struct { - ctlID uint32 - ctlName [96]byte - }{} - - copy(ctlInfo.ctlName[:], utunControlName) + var ctlInfo = &unix.CtlInfo{} + copy(ctlInfo.Name[:], utunControlName) - err = ioctl(uintptr(fd), uintptr(_CTLIOCGINFO), uintptr(unsafe.Pointer(ctlInfo))) + err = unix.IoctlCtlInfo(fd, ctlInfo) if err != nil { return nil, fmt.Errorf("CTLIOCGINFO: %v", err) } - sc := sockaddrCtl{ - scLen: uint8(sockaddrCtlSize), - scFamily: unix.AF_SYSTEM, - ssSysaddr: _AF_SYS_CONTROL, - scID: ctlInfo.ctlID, - scUnit: uint32(ifIndex) + 1, - } - - _, _, errno := unix.RawSyscall( - unix.SYS_CONNECT, - uintptr(fd), - uintptr(unsafe.Pointer(&sc)), - sockaddrCtlSize, - ) - if errno != 0 { - return nil, fmt.Errorf("SYS_CONNECT: %v", errno) + err = unix.Connect(fd, &unix.SockaddrCtl{ + ID: ctlInfo.Id, + Unit: uint32(ifIndex) + 1, + }) + if err != nil { + return nil, fmt.Errorf("SYS_CONNECT: %v", err) } - var ifName struct { - name [16]byte - } - ifNameSize := uintptr(len(ifName.name)) - _, _, errno = syscall.Syscall6(syscall.SYS_GETSOCKOPT, uintptr(fd), - 2, // SYSPROTO_CONTROL - 2, // UTUN_OPT_IFNAME - uintptr(unsafe.Pointer(&ifName)), - uintptr(unsafe.Pointer(&ifNameSize)), 0) - if errno != 0 { - return nil, fmt.Errorf("SYS_GETSOCKOPT: %v", errno) + name, err = unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME) + if err != nil { + return nil, fmt.Errorf("failed to retrieve tun name: %w", err) } - name = string(ifName.name[:ifNameSize-1]) - err = syscall.SetNonblock(fd, true) + err = unix.SetNonblock(fd, true) if err != nil { return nil, fmt.Errorf("SetNonblock: %v", err) } - file := os.NewFile(uintptr(fd), "") - t := &tun{ - ReadWriteCloser: file, - Device: name, - cidr: cidr, - DefaultMTU: c.GetInt("tun.mtu", DefaultMTU), - l: l, + f: os.NewFile(uintptr(fd), ""), + Device: name, + vpnNetworks: vpnNetworks, + DefaultMTU: c.GetInt("tun.mtu", DefaultMTU), + l: l, } err = t.reload(c, true) @@ -172,13 +150,13 @@ func (t *tun) deviceBytes() (o [16]byte) { return } -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) { +func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { return nil, fmt.Errorf("newTunFromFd not supported in Darwin") } func (t *tun) Close() error { - if t.ReadWriteCloser != nil { - return t.ReadWriteCloser.Close() + if t.f != nil { + return t.f.Close() } return nil } @@ -186,11 +164,6 @@ func (t *tun) Close() error { func (t *tun) Activate() error { devName := t.deviceBytes() - var addr, mask [4]byte - - copy(addr[:], t.cidr.IP.To4()) - copy(mask[:], t.cidr.Mask) - s, err := unix.Socket( unix.AF_INET, unix.SOCK_DGRAM, @@ -203,66 +176,18 @@ func (t *tun) Activate() error { fd := uintptr(s) - ifra := ifreqAddr{ - Name: devName, - Addr: unix.RawSockaddrInet4{ - Family: unix.AF_INET, - Addr: addr, - }, - } - - // Set the device ip address - if err = ioctl(fd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil { - return fmt.Errorf("failed to set tun address: %s", err) - } - - // Set the device network - ifra.Addr.Addr = mask - if err = ioctl(fd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil { - return fmt.Errorf("failed to set tun netmask: %s", err) - } - - // Set the device name - ifrf := ifReq{Name: devName} - if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { - return fmt.Errorf("failed to set tun device name: %s", err) - } - // Set the MTU on the device ifm := ifreqMTU{Name: devName, MTU: int32(t.DefaultMTU)} if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { return fmt.Errorf("failed to set tun mtu: %v", err) } - /* - // Set the transmit queue length - ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)} - if err = ioctl(fd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil { - // If we can't set the queue length nebula will still work but it may lead to packet loss - l.WithError(err).Error("Failed to set tun tx queue length") - } - */ - - // Bring up the interface - ifrf.Flags = ifrf.Flags | unix.IFF_UP - if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { - return fmt.Errorf("failed to bring the tun device up: %s", err) - } - - routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) - if err != nil { - return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + // Get the device flags + ifrf := ifReq{Name: devName} + if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + return fmt.Errorf("failed to get tun flags: %s", err) } - defer func() { - unix.Shutdown(routeSock, unix.SHUT_RDWR) - err := unix.Close(routeSock) - if err != nil { - t.l.WithError(err).Error("failed to close AF_ROUTE socket") - } - }() - routeAddr := &netroute.Inet4Addr{} - maskAddr := &netroute.Inet4Addr{} linkAddr, err := getLinkAddr(t.Device) if err != nil { return err @@ -272,14 +197,18 @@ func (t *tun) Activate() error { } t.linkAddr = linkAddr - copy(routeAddr.IP[:], addr[:]) - copy(maskAddr.IP[:], mask[:]) - err = addRoute(routeSock, routeAddr, maskAddr, linkAddr) - if err != nil { - if errors.Is(err, unix.EEXIST) { - err = fmt.Errorf("unable to add tun route, identical route already exists: %s", t.cidr) + for _, network := range t.vpnNetworks { + if network.Addr().Is4() { + err = t.activate4(network) + if err != nil { + return err + } + } else { + err = t.activate6(network) + if err != nil { + return err + } } - return err } // Run the interface @@ -292,8 +221,88 @@ func (t *tun) Activate() error { return t.addRoutes(false) } +func (t *tun) activate4(network netip.Prefix) error { + s, err := unix.Socket( + unix.AF_INET, + unix.SOCK_DGRAM, + unix.IPPROTO_IP, + ) + if err != nil { + return err + } + defer unix.Close(s) + + ifr := ifreqAlias4{ + Name: t.deviceBytes(), + Addr: unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: network.Addr().As4(), + }, + DstAddr: unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: network.Addr().As4(), + }, + MaskAddr: unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: prefixToMask(network).As4(), + }, + } + + if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { + return fmt.Errorf("failed to set tun v4 address: %s", err) + } + + err = addRoute(network, t.linkAddr) + if err != nil { + return err + } + + return nil +} + +func (t *tun) activate6(network netip.Prefix) error { + s, err := unix.Socket( + unix.AF_INET6, + unix.SOCK_DGRAM, + unix.IPPROTO_IP, + ) + if err != nil { + return err + } + defer unix.Close(s) + + ifr := ifreqAlias6{ + Name: t.deviceBytes(), + Addr: unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: network.Addr().As16(), + }, + PrefixMask: unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: prefixToMask(network).As16(), + }, + Lifetime: addrLifetime{ + // never expires + Vltime: 0xffffffff, + Pltime: 0xffffffff, + }, + Flags: _IN6_IFF_NODAD, + } + + if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { + return fmt.Errorf("failed to set tun address: %s", err) + } + + return nil +} + func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) if err != nil { return err } @@ -329,17 +338,16 @@ func (t *tun) reload(c *config.C, initial bool) error { return nil } -func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - ok, r := t.routeTree.Load().MostSpecificContains(ip) +func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { + r, ok := t.routeTree.Load().Lookup(ip) if ok { return r } - - return 0 + return routing.Gateways{} } // Get the LinkAddr for the interface of the given name -// TODO: Is there an easier way to fetch this when we create the interface? +// Is there an easier way to fetch this when we create the interface? // Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. func getLinkAddr(name string) (*netroute.LinkAddr, error) { rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0) @@ -367,38 +375,20 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) { } func (t *tun) addRoutes(logErrors bool) error { - routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) - if err != nil { - return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) - } - - defer func() { - unix.Shutdown(routeSock, unix.SHUT_RDWR) - err := unix.Close(routeSock) - if err != nil { - t.l.WithError(err).Error("failed to close AF_ROUTE socket") - } - }() - - routeAddr := &netroute.Inet4Addr{} - maskAddr := &netroute.Inet4Addr{} routes := *t.Routes.Load() + for _, r := range routes { - if r.Via == nil || !r.Install { + if len(r.Via) == 0 || !r.Install { // We don't allow route MTUs so only install routes with a via continue } - copy(routeAddr.IP[:], r.Cidr.IP.To4()) - copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4()) - - err := addRoute(routeSock, routeAddr, maskAddr, t.linkAddr) + err := addRoute(r.Cidr, t.linkAddr) if err != nil { if errors.Is(err, unix.EEXIST) { - t.l.WithField("route", r.Cidr). - Warnf("unable to add unsafe_route, identical route already exists") + t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr) } else { - retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err) + retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) } else { @@ -406,7 +396,7 @@ func (t *tun) addRoutes(logErrors bool) error { } } } else { - t.l.WithField("route", r).Info("Added route") + t.l.Info("Added route", "route", r) } } @@ -414,57 +404,54 @@ func (t *tun) addRoutes(logErrors bool) error { } func (t *tun) removeRoutes(routes []Route) error { - routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) - if err != nil { - return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) - } - - defer func() { - unix.Shutdown(routeSock, unix.SHUT_RDWR) - err := unix.Close(routeSock) - if err != nil { - t.l.WithError(err).Error("failed to close AF_ROUTE socket") - } - }() - - routeAddr := &netroute.Inet4Addr{} - maskAddr := &netroute.Inet4Addr{} - for _, r := range routes { if !r.Install { continue } - copy(routeAddr.IP[:], r.Cidr.IP.To4()) - copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4()) - - err := delRoute(routeSock, routeAddr, maskAddr, t.linkAddr) + err := delRoute(r.Cidr, t.linkAddr) if err != nil { - t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + t.l.Error("Failed to remove route", "error", err, "route", r) } else { - t.l.WithField("route", r).Info("Removed route") + t.l.Info("Removed route", "route", r) } } return nil } -func addRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) error { - r := netroute.RouteMessage{ +func addRoute(prefix netip.Prefix, gateway netroute.Addr) error { + sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + defer unix.Close(sock) + + route := &netroute.RouteMessage{ Version: unix.RTM_VERSION, Type: unix.RTM_ADD, Flags: unix.RTF_UP, Seq: 1, - Addrs: []netroute.Addr{ - unix.RTAX_DST: addr, - unix.RTAX_GATEWAY: link, - unix.RTAX_NETMASK: mask, - }, } - data, err := r.Marshal() + if prefix.Addr().Is4() { + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, + unix.RTAX_GATEWAY: gateway, + } + } else { + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, + unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, + unix.RTAX_GATEWAY: gateway, + } + } + + data, err := route.Marshal() if err != nil { return fmt.Errorf("failed to create route.RouteMessage: %w", err) } + _, err = unix.Write(sock, data[:]) if err != nil { return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) @@ -473,19 +460,34 @@ func addRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) return nil } -func delRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) error { - r := netroute.RouteMessage{ +func delRoute(prefix netip.Prefix, gateway netroute.Addr) error { + sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + defer unix.Close(sock) + + route := netroute.RouteMessage{ Version: unix.RTM_VERSION, Type: unix.RTM_DELETE, Seq: 1, - Addrs: []netroute.Addr{ - unix.RTAX_DST: addr, - unix.RTAX_GATEWAY: link, - unix.RTAX_NETMASK: mask, - }, } - data, err := r.Marshal() + if prefix.Addr().Is4() { + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, + unix.RTAX_GATEWAY: gateway, + } + } else { + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, + unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, + unix.RTAX_GATEWAY: gateway, + } + } + + data, err := route.Marshal() if err != nil { return fmt.Errorf("failed to create route.RouteMessage: %w", err) } @@ -497,53 +499,117 @@ func delRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) return nil } -func (t *tun) Read(to []byte) (int, error) { +// tunWritev and tunReadv are linkname'd to x/sys/unix's libc-routed writev/readv stubs so the +// calls go through libSystem's pinned trampoline. A raw syscall.Syscall(SYS_WRITEV/SYS_READV, ...) +// on darwin/arm64 emits an SVC #0x80 trap (see $GOROOT/src/syscall/asm_darwin_arm64.s), the path +// Apple keeps warning they will eventually disallow. We pull the low-level stubs instead of calling +// unix.Writev/unix.Readv because those take [][]byte and rebuild the []Iovec every call, which +// heap-allocates the header; linkname'ing the stubs lets us hand them our own stack-allocated +// iovecs. See golang/go#78049. - buf := make([]byte, len(to)+4) +//go:linkname tunWritev golang.org/x/sys/unix.writev +//go:noescape +func tunWritev(fd int, iovecs []unix.Iovec) (n int, err error) - n, err := t.ReadWriteCloser.Read(buf) +//go:linkname tunReadv golang.org/x/sys/unix.readv +//go:noescape +func tunReadv(fd int, iovecs []unix.Iovec) (n int, err error) - copy(to, buf[4:]) - return n - 4, err -} +// Read pulls one IP packet off the utun device, scattering the 4 byte protocol header away from +// the packet so the payload lands directly in to. +func (t *tun) Read(to []byte) (int, error) { + var head [4]byte -// Write is only valid for single threaded use -func (t *tun) Write(from []byte) (int, error) { - buf := t.out - if cap(buf) < len(from)+4 { - buf = make([]byte, len(from)+4) - t.out = buf + rc, err := t.f.SyscallConn() + if err != nil { + return 0, err + } + + var n int + var callErr error + err = rc.Read(func(fd uintptr) bool { + iovecs := []unix.Iovec{ + {Base: &head[0], Len: 4}, + {Base: &to[0], Len: uint64(len(to))}, + } + n, callErr = tunReadv(int(fd), iovecs) + if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() { + return false + } + return true + }) + if err != nil { + return 0, err + } + if callErr != nil { + return 0, callErr } - buf = buf[:len(from)+4] + if n < 4 { + return 0, nil + } + return n - 4, nil +} +// Write pushes one IP packet onto the utun device. +func (t *tun) Write(from []byte) (int, error) { if len(from) == 0 { return 0, syscall.EIO } - // Determine the IP Family for the NULL L2 Header ipVer := from[0] >> 4 - if ipVer == 4 { - buf[3] = syscall.AF_INET - } else if ipVer == 6 { - buf[3] = syscall.AF_INET6 - } else { + var head [4]byte + switch ipVer { + case 4: + head[3] = syscall.AF_INET + case 6: + head[3] = syscall.AF_INET6 + default: return 0, fmt.Errorf("unable to determine IP version from packet") } - copy(buf[4:], from) + // Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack. + rc, err := t.f.SyscallConn() + if err != nil { + return 0, err + } + + var n int + var callErr error + err = rc.Write(func(fd uintptr) bool { + iovecs := []unix.Iovec{ + {Base: &head[0], Len: 4}, + {Base: &from[0], Len: uint64(len(from))}, + } + n, callErr = tunWritev(int(fd), iovecs) + // Type-assert to syscall.Errno so the EAGAIN/EWOULDBLOCK/EINTR check doesn't box the errno + // constants into error interfaces on every call. + if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() { + return false + } + return true + }) + if err != nil { + return 0, err + } + if callErr != nil { + return 0, callErr + } - n, err := t.ReadWriteCloser.Write(buf) - return n - 4, err + return n - 4, nil } -func (t *tun) Cidr() *net.IPNet { - return t.cidr +func (t *tun) Networks() []netip.Prefix { + return t.vpnNetworks } func (t *tun) Name() string { return t.Device } +func (t *tun) SupportsMultiqueue() bool { + return false +} + func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin") } diff --git a/overlay/tun_disabled.go b/overlay/tun_disabled.go index e1e4ede6..f47880dd 100644 --- a/overlay/tun_disabled.go +++ b/overlay/tun_disabled.go @@ -1,31 +1,33 @@ package overlay import ( + "context" "fmt" "io" - "net" + "log/slog" + "net/netip" "strings" "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" ) type disabledTun struct { - read chan []byte - cidr *net.IPNet + read chan []byte + vpnNetworks []netip.Prefix // Track these metrics since we don't have the tun device to do it for us tx metrics.Counter rx metrics.Counter - l *logrus.Logger + l *slog.Logger } -func newDisabledTun(cidr *net.IPNet, queueLen int, metricsEnabled bool, l *logrus.Logger) *disabledTun { +func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *slog.Logger) *disabledTun { tun := &disabledTun{ - cidr: cidr, - read: make(chan []byte, queueLen), - l: l, + vpnNetworks: vpnNetworks, + read: make(chan []byte, queueLen), + l: l, } if metricsEnabled { @@ -43,12 +45,12 @@ func (*disabledTun) Activate() error { return nil } -func (*disabledTun) RouteFor(iputil.VpnIp) iputil.VpnIp { - return 0 +func (*disabledTun) RoutesFor(addr netip.Addr) routing.Gateways { + return routing.Gateways{} } -func (t *disabledTun) Cidr() *net.IPNet { - return t.cidr +func (t *disabledTun) Networks() []netip.Prefix { + return t.vpnNetworks } func (*disabledTun) Name() string { @@ -66,8 +68,8 @@ func (t *disabledTun) Read(b []byte) (int, error) { } t.tx.Inc(1) - if t.l.Level >= logrus.DebugLevel { - t.l.WithField("raw", prettyPacket(r)).Debugf("Write payload") + if t.l.Enabled(context.Background(), slog.LevelDebug) { + t.l.Debug("Write payload", "raw", prettyPacket(r)) } return copy(b, r), nil @@ -84,7 +86,7 @@ func (t *disabledTun) handleICMPEchoRequest(b []byte) bool { select { case t.read <- out: default: - t.l.Debugf("tun_disabled: dropped ICMP Echo Reply response") + t.l.Debug("tun_disabled: dropped ICMP Echo Reply response") } return true @@ -95,15 +97,19 @@ func (t *disabledTun) Write(b []byte) (int, error) { // Check for ICMP Echo Request before spending time doing the full parsing if t.handleICMPEchoRequest(b) { - if t.l.Level >= logrus.DebugLevel { - t.l.WithField("raw", prettyPacket(b)).Debugf("Disabled tun responded to ICMP Echo Request") + if t.l.Enabled(context.Background(), slog.LevelDebug) { + t.l.Debug("Disabled tun responded to ICMP Echo Request", "raw", prettyPacket(b)) } - } else if t.l.Level >= logrus.DebugLevel { - t.l.WithField("raw", prettyPacket(b)).Debugf("Disabled tun received unexpected payload") + } else if t.l.Enabled(context.Background(), slog.LevelDebug) { + t.l.Debug("Disabled tun received unexpected payload", "raw", prettyPacket(b)) } return len(b), nil } +func (t *disabledTun) SupportsMultiqueue() bool { + return true +} + func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) { return t, nil } diff --git a/overlay/tun_file_linux_test.go b/overlay/tun_file_linux_test.go new file mode 100644 index 00000000..5ab87e05 --- /dev/null +++ b/overlay/tun_file_linux_test.go @@ -0,0 +1,120 @@ +//go:build linux && !android && !e2e_testing +// +build linux,!android,!e2e_testing + +package overlay + +import ( + "errors" + "os" + "sync" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +// newReadPipe returns a read fd. The matching write fd is registered for cleanup. +// The caller takes ownership of the read fd (pass it to newTunFd / newFriend). +func newReadPipe(t *testing.T) int { + t.Helper() + var fds [2]int + if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil { + t.Fatalf("pipe2: %v", err) + } + t.Cleanup(func() { _ = unix.Close(fds[1]) }) + return fds[0] +} + +func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) { + tf, err := newTunFd(newReadPipe(t)) + if err != nil { + t.Fatalf("newTunFd: %v", err) + } + t.Cleanup(func() { _ = tf.Close() }) + + done := make(chan error, 1) + go func() { + _, err := tf.Read(make([]byte, 64)) + done <- err + }() + + // Verify Read is actually blocked in poll. + select { + case err := <-done: + t.Fatalf("Read returned before shutdown signal: %v", err) + case <-time.After(50 * time.Millisecond): + } + + if err := tf.wakeForShutdown(); err != nil { + t.Fatalf("wakeForShutdown: %v", err) + } + + select { + case err := <-done: + if !errors.Is(err, os.ErrClosed) { + t.Fatalf("expected os.ErrClosed, got %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Read did not wake on shutdown") + } +} + +func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) { + parent, err := newTunFd(newReadPipe(t)) + if err != nil { + t.Fatalf("newTunFd: %v", err) + } + friend, err := parent.newFriend(newReadPipe(t)) + if err != nil { + _ = parent.Close() + t.Fatalf("newFriend: %v", err) + } + t.Cleanup(func() { + _ = friend.Close() + _ = parent.Close() + }) + + readers := []*tunFile{parent, friend} + errs := make([]error, len(readers)) + var wg sync.WaitGroup + for i, r := range readers { + wg.Add(1) + go func(i int, r *tunFile) { + defer wg.Done() + _, errs[i] = r.Read(make([]byte, 64)) + }(i, r) + } + + time.Sleep(50 * time.Millisecond) + + if err := parent.wakeForShutdown(); err != nil { + t.Fatalf("wakeForShutdown: %v", err) + } + + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("readers did not wake") + } + + for i, err := range errs { + if !errors.Is(err, os.ErrClosed) { + t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err) + } + } +} + +func TestTunFile_Close_Idempotent(t *testing.T) { + tf, err := newTunFd(newReadPipe(t)) + if err != nil { + t.Fatalf("newTunFd: %v", err) + } + if err := tf.Close(); err != nil { + t.Fatalf("first Close: %v", err) + } + if err := tf.Close(); err != nil { + t.Fatalf("second Close should be a no-op, got %v", err) + } +} diff --git a/overlay/tun_freebsd.go b/overlay/tun_freebsd.go index 3b1b80f1..79f55697 100644 --- a/overlay/tun_freebsd.go +++ b/overlay/tun_freebsd.go @@ -9,25 +9,32 @@ import ( "fmt" "io" "io/fs" - "net" + "log/slog" + "net/netip" "os" - "os/exec" - "strconv" "sync/atomic" "syscall" + "time" "unsafe" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" + "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + + "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" + netroute "golang.org/x/net/route" + "golang.org/x/sys/unix" ) const ( // FIODGNAME is defined in sys/sys/filio.h on FreeBSD // For 32-bit systems, use FIODGNAME_32 (not defined in this file: 0x80086678) - FIODGNAME = 0x80106678 + FIODGNAME = 0x80106678 + TUNSIFMODE = 0x8004745e + TUNSIFHEAD = 0x80047460 + OSIOCAIFADDR_IN6 = 0x8088691b + IN6_IFF_NODAD = 0x0020 ) type fiodgnameArg struct { @@ -37,82 +44,302 @@ type fiodgnameArg struct { } type ifreqRename struct { - Name [16]byte + Name [unix.IFNAMSIZ]byte Data uintptr } type ifreqDestroy struct { - Name [16]byte + Name [unix.IFNAMSIZ]byte pad [16]byte } +type ifReq struct { + Name [unix.IFNAMSIZ]byte + Flags uint16 +} + +type ifreqMTU struct { + Name [unix.IFNAMSIZ]byte + MTU int32 +} + +type addrLifetime struct { + Expire uint64 + Preferred uint64 + Vltime uint32 + Pltime uint32 +} + +type ifreqAlias4 struct { + Name [unix.IFNAMSIZ]byte + Addr unix.RawSockaddrInet4 + DstAddr unix.RawSockaddrInet4 + MaskAddr unix.RawSockaddrInet4 + VHid uint32 +} + +type ifreqAlias6 struct { + Name [unix.IFNAMSIZ]byte + Addr unix.RawSockaddrInet6 + DstAddr unix.RawSockaddrInet6 + PrefixMask unix.RawSockaddrInet6 + Flags uint32 + Lifetime addrLifetime + VHid uint32 +} + type tun struct { - Device string - cidr *net.IPNet - MTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] - l *logrus.Logger - - io.ReadWriteCloser + Device string + vpnNetworks []netip.Prefix + MTU int + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[bart.Table[routing.Gateways]] + linkAddr *netroute.LinkAddr + l *slog.Logger + + fd int + shutdownR int // read end of the shutdown pipe; closing the write end wakes blocked polls + shutdownW int // write end of the shutdown pipe; closing this signals shutdown to any blocked reader/writer + readPoll [2]unix.PollFd + writePoll [2]unix.PollFd + closed atomic.Bool +} + +// blockOnRead waits until the tun fd is readable or shutdown has been signaled. +// Returns os.ErrClosed if Close was called. +func (t *tun) blockOnRead() error { + const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR + var err error + for { + _, err = unix.Poll(t.readPoll[:], -1) + if err != unix.EINTR { + break + } + } + tunEvents := t.readPoll[0].Revents + shutdownEvents := t.readPoll[1].Revents + t.readPoll[0].Revents = 0 + t.readPoll[1].Revents = 0 + if err != nil { + return err + } + if shutdownEvents&(unix.POLLIN|problemFlags) != 0 { + return os.ErrClosed + } + if tunEvents&problemFlags != 0 { + return os.ErrClosed + } + return nil +} + +func (t *tun) blockOnWrite() error { + const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR + var err error + for { + _, err = unix.Poll(t.writePoll[:], -1) + if err != unix.EINTR { + break + } + } + tunEvents := t.writePoll[0].Revents + shutdownEvents := t.writePoll[1].Revents + t.writePoll[0].Revents = 0 + t.writePoll[1].Revents = 0 + if err != nil { + return err + } + if shutdownEvents&(unix.POLLIN|problemFlags) != 0 { + return os.ErrClosed + } + if tunEvents&problemFlags != 0 { + return os.ErrClosed + } + return nil +} + +func (t *tun) Read(to []byte) (int, error) { + // first 4 bytes is protocol family, in network byte order + var head [4]byte + iovecs := [2]syscall.Iovec{ + {&head[0], 4}, + {&to[0], uint64(len(to))}, + } + for { + n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2) + if errno == 0 { + bytesRead := int(n) + if bytesRead < 4 { + return 0, nil + } + return bytesRead - 4, nil + } + switch errno { + case unix.EAGAIN: + if err := t.blockOnRead(); err != nil { + return 0, err + } + case unix.EINTR: + // retry + case unix.EBADF: + return 0, os.ErrClosed + default: + return 0, errno + } + } +} + +// Write is only valid for single threaded use +func (t *tun) Write(from []byte) (int, error) { + if len(from) <= 1 { + return 0, syscall.EIO + } + + ipVer := from[0] >> 4 + var head [4]byte + // first 4 bytes is protocol family, in network byte order + switch ipVer { + case 4: + head[3] = syscall.AF_INET + case 6: + head[3] = syscall.AF_INET6 + default: + return 0, fmt.Errorf("unable to determine IP version from packet") + } + + iovecs := [2]syscall.Iovec{ + {&head[0], 4}, + {&from[0], uint64(len(from))}, + } + for { + n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2) + if errno == 0 { + return int(n) - 4, nil + } + switch errno { + case unix.EAGAIN: + if err := t.blockOnWrite(); err != nil { + return 0, err + } + case unix.EINTR: + // retry + case unix.EBADF: + return 0, os.ErrClosed + default: + return 0, errno + } + } } func (t *tun) Close() error { - if t.ReadWriteCloser != nil { - if err := t.ReadWriteCloser.Close(); err != nil { - return err + if t.closed.Swap(true) { + return nil + } + + // Closing the write end of the shutdown pipe causes any blocked Poll to + // return with POLLHUP on the shutdown fd, so readers/writers wake up and + // exit with os.ErrClosed. + if t.shutdownW >= 0 { + _ = unix.Close(t.shutdownW) + t.shutdownW = -1 + } + + if t.fd >= 0 { + if err := unix.Close(t.fd); err != nil { + t.l.Error("Error closing device", "error", err) } + t.fd = -1 + } + if t.shutdownR >= 0 { + _ = unix.Close(t.shutdownR) + t.shutdownR = -1 + } + + c := make(chan struct{}) + go func() { + // destroying the interface can block if a read() is still pending. Do this asynchronously. + defer close(c) s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) + if err == nil { + defer syscall.Close(s) + ifreq := ifreqDestroy{Name: t.deviceBytes()} + err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq))) + } if err != nil { - return err + t.l.Error("Error destroying tunnel", "error", err) } - defer syscall.Close(s) + }() - ifreq := ifreqDestroy{Name: t.deviceBytes()} - - // Destroy the interface - err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq))) - return err + // wait up to 1 second so we start blocking at the ioctl + select { + case <-c: + case <-time.After(1 * time.Second): } return nil } -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) { +func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD") } -func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) { +func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { // Try to open existing tun device - var file *os.File + var fd int var err error deviceName := c.GetString("tun.dev", "") if deviceName != "" { - file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0) + fd, err = unix.Open("/dev/"+deviceName, os.O_RDWR, 0) } if errors.Is(err, fs.ErrNotExist) || deviceName == "" { // If the device doesn't already exist, request a new one and rename it - file, err = os.OpenFile("/dev/tun", os.O_RDWR, 0) + fd, err = unix.Open("/dev/tun", os.O_RDWR, 0) } if err != nil { return nil, err } - rawConn, err := file.SyscallConn() - if err != nil { - return nil, fmt.Errorf("SyscallConn: %v", err) + if err = unix.SetNonblock(fd, true); err != nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("failed to set tun device as nonblocking: %w", err) } + // Shutdown pipe lets Close wake any reader/writer blocked in Poll. + var pipeFds [2]int + if err = unix.Pipe2(pipeFds[:], unix.O_CLOEXEC|unix.O_NONBLOCK); err != nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("failed to create shutdown pipe: %w", err) + } + shutdownR, shutdownW := pipeFds[0], pipeFds[1] + + closeOnErr := true + defer func() { + if closeOnErr { + _ = unix.Close(fd) + _ = unix.Close(shutdownR) + _ = unix.Close(shutdownW) + } + }() + + // Read the name of the interface var name [16]byte - var ctrlErr error - rawConn.Control(func(fd uintptr) { - // Read the name of the interface - arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)} - ctrlErr = ioctl(fd, FIODGNAME, uintptr(unsafe.Pointer(&arg))) - }) + arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)} + ctrlErr := ioctl(uintptr(fd), FIODGNAME, uintptr(unsafe.Pointer(&arg))) + + if ctrlErr == nil { + // set broadcast mode and multicast + ifmode := uint32(unix.IFF_BROADCAST | unix.IFF_MULTICAST) + ctrlErr = ioctl(uintptr(fd), TUNSIFMODE, uintptr(unsafe.Pointer(&ifmode))) + } + + if ctrlErr == nil { + // turn on link-layer mode, to support ipv6 + ifhead := uint32(1) + ctrlErr = ioctl(uintptr(fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&ifhead))) + } + if ctrlErr != nil { - return nil, err + return nil, ctrlErr } ifName := string(bytes.TrimRight(name[:], "\x00")) @@ -122,18 +349,12 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error // If the name doesn't match the desired interface name, rename it now if ifName != deviceName { - s, err := syscall.Socket( - syscall.AF_INET, - syscall.SOCK_DGRAM, - syscall.IPPROTO_IP, - ) + s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) if err != nil { return nil, err } defer syscall.Close(s) - fd := uintptr(s) - var fromName [16]byte var toName [16]byte copy(fromName[:], ifName) @@ -145,21 +366,32 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error } // Set the device name - ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr))) + _ = ioctl(uintptr(s), syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr))) } t := &tun{ - ReadWriteCloser: file, - Device: deviceName, - cidr: cidr, - MTU: c.GetInt("tun.mtu", DefaultMTU), - l: l, + Device: deviceName, + vpnNetworks: vpnNetworks, + MTU: c.GetInt("tun.mtu", DefaultMTU), + l: l, + fd: fd, + shutdownR: shutdownR, + shutdownW: shutdownW, + readPoll: [2]unix.PollFd{ + {Fd: int32(fd), Events: unix.POLLIN}, + {Fd: int32(shutdownR), Events: unix.POLLIN}, + }, + writePoll: [2]unix.PollFd{ + {Fd: int32(fd), Events: unix.POLLOUT}, + {Fd: int32(shutdownR), Events: unix.POLLIN}, + }, } err = t.reload(c, true) if err != nil { return nil, err } + closeOnErr = false c.RegisterReloadCallback(func(c *config.C) { err := t.reload(c, false) @@ -171,33 +403,116 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error return t, nil } +func (t *tun) addIp(cidr netip.Prefix) error { + if cidr.Addr().Is4() { + ifr := ifreqAlias4{ + Name: t.deviceBytes(), + Addr: unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: cidr.Addr().As4(), + }, + DstAddr: unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: getBroadcast(cidr).As4(), + }, + MaskAddr: unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: prefixToMask(cidr).As4(), + }, + VHid: 0, + } + s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + // Note: unix.SIOCAIFADDR corresponds to FreeBSD's OSIOCAIFADDR + if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { + return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err) + } + return nil + } + + if cidr.Addr().Is6() { + ifr := ifreqAlias6{ + Name: t.deviceBytes(), + Addr: unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: cidr.Addr().As16(), + }, + PrefixMask: unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: prefixToMask(cidr).As16(), + }, + Lifetime: addrLifetime{ + Expire: 0, + Preferred: 0, + Vltime: 0xffffffff, + Pltime: 0xffffffff, + }, + Flags: IN6_IFF_NODAD, + } + s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + + if err := ioctl(uintptr(s), OSIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { + return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err) + } + return nil + } + + return fmt.Errorf("unknown address type %v", cidr) +} + func (t *tun) Activate() error { - var err error - // TODO use syscalls instead of exec.Command - cmd := exec.Command("/sbin/ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'ifconfig': %s", err) + // Setup our default MTU + err := t.setMTU() + if err != nil { + return err } - cmd = exec.Command("/sbin/route", "-n", "add", "-net", t.cidr.String(), "-interface", t.Device) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'route add': %s", err) + linkAddr, err := getLinkAddr(t.Device) + if err != nil { + return err + } + if linkAddr == nil { + return fmt.Errorf("unable to discover link_addr for tun interface") } + t.linkAddr = linkAddr - cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU)) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'ifconfig': %s", err) + for i := range t.vpnNetworks { + err := t.addIp(t.vpnNetworks[i]) + if err != nil { + return err + } } - // Unsafe path routes return t.addRoutes(false) } +func (t *tun) setMTU() error { + // Set the MTU on the device + s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + + ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MTU)} + err = ioctl(uintptr(s), unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))) + return err +} + func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) if err != nil { return err } @@ -233,19 +548,23 @@ func (t *tun) reload(c *config.C, initial bool) error { return nil } -func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.Load().MostSpecificContains(ip) +func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { + r, _ := t.routeTree.Load().Lookup(ip) return r } -func (t *tun) Cidr() *net.IPNet { - return t.cidr +func (t *tun) Networks() []netip.Prefix { + return t.vpnNetworks } func (t *tun) Name() string { return t.Device } +func (t *tun) SupportsMultiqueue() bool { + return false +} + func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd") } @@ -253,20 +572,21 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { func (t *tun) addRoutes(logErrors bool) error { routes := *t.Routes.Load() for _, r := range routes { - if r.Via == nil || !r.Install { + if len(r.Via) == 0 || !r.Install { // We don't allow route MTUs so only install routes with a via continue } - cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device) - t.l.Debug("command: ", cmd.String()) - if err := cmd.Run(); err != nil { - retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]interface{}{"route": r}, err) + err := addRoute(r.Cidr, t.linkAddr) + if err != nil { + retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) } else { return retErr } + } else { + t.l.Info("Added route", "route", r) } } @@ -279,12 +599,11 @@ func (t *tun) removeRoutes(routes []Route) error { continue } - cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), "-interface", t.Device) - t.l.Debug("command: ", cmd.String()) - if err := cmd.Run(); err != nil { - t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + err := delRoute(r.Cidr, t.linkAddr) + if err != nil { + t.l.Error("Failed to remove route", "error", err, "route", r) } else { - t.l.WithField("route", r).Info("Removed route") + t.l.Info("Removed route", "route", r) } } return nil @@ -296,3 +615,119 @@ func (t *tun) deviceBytes() (o [16]byte) { } return } + +func addRoute(prefix netip.Prefix, gateway netroute.Addr) error { + sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + defer unix.Close(sock) + + route := &netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_ADD, + Flags: unix.RTF_UP, + Seq: 1, + } + + if prefix.Addr().Is4() { + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, + unix.RTAX_GATEWAY: gateway, + } + } else { + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, + unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, + unix.RTAX_GATEWAY: gateway, + } + } + + data, err := route.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage: %w", err) + } + + _, err = unix.Write(sock, data[:]) + if err != nil { + if errors.Is(err, unix.EEXIST) { + // Try to do a change + route.Type = unix.RTM_CHANGE + data, err = route.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage for change: %w", err) + } + _, err = unix.Write(sock, data[:]) + return err + } + return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) + } + + return nil +} + +func delRoute(prefix netip.Prefix, gateway netroute.Addr) error { + sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + defer unix.Close(sock) + + route := netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_DELETE, + Seq: 1, + } + + if prefix.Addr().Is4() { + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, + unix.RTAX_GATEWAY: gateway, + } + } else { + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, + unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, + unix.RTAX_GATEWAY: gateway, + } + } + + data, err := route.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage: %w", err) + } + _, err = unix.Write(sock, data[:]) + if err != nil { + return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) + } + + return nil +} + +// getLinkAddr Gets the link address for the interface of the given name +func getLinkAddr(name string) (*netroute.LinkAddr, error) { + rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0) + if err != nil { + return nil, err + } + msgs, err := netroute.ParseRIB(unix.NET_RT_IFLIST, rib) + if err != nil { + return nil, err + } + + for _, m := range msgs { + switch m := m.(type) { + case *netroute.InterfaceMessage: + if m.Name == name { + sa, ok := m.Addrs[unix.RTAX_IFP].(*netroute.LinkAddr) + if ok { + return sa, nil + } + } + } + } + + return nil, nil +} diff --git a/overlay/tun_ios.go b/overlay/tun_ios.go index ba15d665..27bf558b 100644 --- a/overlay/tun_ios.go +++ b/overlay/tun_ios.go @@ -7,41 +7,49 @@ import ( "errors" "fmt" "io" - "net" + "log/slog" + "net/netip" "os" "sync" "sync/atomic" "syscall" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" + "golang.org/x/sys/unix" ) type tun struct { io.ReadWriteCloser - cidr *net.IPNet - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] - l *logrus.Logger + vpnNetworks []netip.Prefix + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[bart.Table[routing.Gateways]] + l *slog.Logger } -func newTun(_ *config.C, _ *logrus.Logger, _ *net.IPNet, _ bool) (*tun, error) { +func newTun(_ *config.C, _ *slog.Logger, _ []netip.Prefix, _ bool) (*tun, error) { return nil, fmt.Errorf("newTun not supported in iOS") } -func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) (*tun, error) { +func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) { + if err := unix.SetNonblock(deviceFd, true); err != nil { + // We own the fd from the moment it is handed to us, same as the reload error path below + _ = unix.Close(deviceFd) + return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err) + } + file := os.NewFile(uintptr(deviceFd), "/dev/tun") t := &tun{ - cidr: cidr, + vpnNetworks: vpnNetworks, ReadWriteCloser: &tunReadCloser{f: file}, l: l, } err := t.reload(c, true) if err != nil { + _ = file.Close() return nil, err } @@ -60,7 +68,7 @@ func (t *tun) Activate() error { } func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) if err != nil { return err } @@ -80,8 +88,8 @@ func (t *tun) reload(c *config.C, initial bool) error { return nil } -func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.Load().MostSpecificContains(ip) +func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { + r, _ := t.routeTree.Load().Lookup(ip) return r } @@ -143,14 +151,18 @@ func (tr *tunReadCloser) Close() error { return tr.f.Close() } -func (t *tun) Cidr() *net.IPNet { - return t.cidr +func (t *tun) Networks() []netip.Prefix { + return t.vpnNetworks } func (t *tun) Name() string { return "iOS" } +func (t *tun) SupportsMultiqueue() bool { + return false +} + func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { return nil, fmt.Errorf("TODO: multiqueue not implemented for ios") } diff --git a/overlay/tun_linux.go b/overlay/tun_linux.go index 1f6580ed..52633842 100644 --- a/overlay/tun_linux.go +++ b/overlay/tun_linux.go @@ -4,41 +4,221 @@ package overlay import ( - "bytes" + "encoding/binary" + "errors" "fmt" "io" + "log/slog" "net" + "net/netip" "os" "strings" + "sync" "sync/atomic" + "time" "unsafe" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) +// tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking. +// A shared eventfd allows Close to wake all readers blocked in poll. +type tunFile struct { + fd int + shutdownFd int + lastOne bool + readPoll [2]unix.PollFd + writePoll [2]unix.PollFd + closed bool +} + +// newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun +func (r *tunFile) newFriend(fd int) (*tunFile, error) { + if err := unix.SetNonblock(fd, true); err != nil { + return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err) + } + return &tunFile{ + fd: fd, + shutdownFd: r.shutdownFd, + readPoll: [2]unix.PollFd{ + {Fd: int32(fd), Events: unix.POLLIN}, + {Fd: int32(r.shutdownFd), Events: unix.POLLIN}, + }, + writePoll: [2]unix.PollFd{ + {Fd: int32(fd), Events: unix.POLLOUT}, + {Fd: int32(r.shutdownFd), Events: unix.POLLIN}, + }, + }, nil +} + +func newTunFd(fd int) (*tunFile, error) { + if err := unix.SetNonblock(fd, true); err != nil { + return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err) + } + + shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC) + if err != nil { + return nil, fmt.Errorf("failed to create eventfd: %w", err) + } + + out := &tunFile{ + fd: fd, + shutdownFd: shutdownFd, + lastOne: true, + readPoll: [2]unix.PollFd{ + {Fd: int32(fd), Events: unix.POLLIN}, + {Fd: int32(shutdownFd), Events: unix.POLLIN}, + }, + writePoll: [2]unix.PollFd{ + {Fd: int32(fd), Events: unix.POLLOUT}, + {Fd: int32(shutdownFd), Events: unix.POLLIN}, + }, + } + + return out, nil +} + +func (r *tunFile) blockOnRead() error { + const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR + var err error + for { + _, err = unix.Poll(r.readPoll[:], -1) + if err != unix.EINTR { + break + } + } + //always reset these! + tunEvents := r.readPoll[0].Revents + shutdownEvents := r.readPoll[1].Revents + r.readPoll[0].Revents = 0 + r.readPoll[1].Revents = 0 + //do the err check before trusting the potentially bogus bits we just got + if err != nil { + return err + } + if shutdownEvents&(unix.POLLIN|problemFlags) != 0 { + return os.ErrClosed + } else if tunEvents&problemFlags != 0 { + return os.ErrClosed + } + return nil +} + +func (r *tunFile) blockOnWrite() error { + const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR + var err error + for { + _, err = unix.Poll(r.writePoll[:], -1) + if err != unix.EINTR { + break + } + } + //always reset these! + tunEvents := r.writePoll[0].Revents + shutdownEvents := r.writePoll[1].Revents + r.writePoll[0].Revents = 0 + r.writePoll[1].Revents = 0 + //do the err check before trusting the potentially bogus bits we just got + if err != nil { + return err + } + if shutdownEvents&(unix.POLLIN|problemFlags) != 0 { + return os.ErrClosed + } else if tunEvents&problemFlags != 0 { + return os.ErrClosed + } + return nil +} + +func (r *tunFile) Read(buf []byte) (int, error) { + for { + if n, err := unix.Read(r.fd, buf); err == nil { + return n, nil + } else if err == unix.EAGAIN { + if err = r.blockOnRead(); err != nil { + return 0, err + } + continue + } else if err == unix.EINTR { + continue + } else if err == unix.EBADF { + return 0, os.ErrClosed + } else { + return 0, err + } + } +} + +func (r *tunFile) Write(buf []byte) (int, error) { + for { + if n, err := unix.Write(r.fd, buf); err == nil { + return n, nil + } else if err == unix.EAGAIN { + if err = r.blockOnWrite(); err != nil { + return 0, err + } + continue + } else if err == unix.EINTR { + continue + } else if err == unix.EBADF { + return 0, os.ErrClosed + } else { + return 0, err + } + } +} + +func (r *tunFile) wakeForShutdown() error { + var buf [8]byte + binary.NativeEndian.PutUint64(buf[:], 1) + _, err := unix.Write(int(r.readPoll[1].Fd), buf[:]) + return err +} + +func (r *tunFile) Close() error { + if r.closed { // avoid closing more than once. Technically a fd could get re-used, which would be a problem + return nil + } + r.closed = true + if r.lastOne { + _ = unix.Close(r.shutdownFd) + } + return unix.Close(r.fd) +} + type tun struct { - io.ReadWriteCloser - fd int + *tunFile + readers []*tunFile + closeLock sync.Mutex Device string - cidr *net.IPNet + vpnNetworks []netip.Prefix MaxMTU int DefaultMTU int TXQueueLen int deviceIndex int ioctlFd uintptr - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] - routeChan chan struct{} - useSystemRoutes bool + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[bart.Table[routing.Gateways]] + routeChan chan struct{} + useSystemRoutes bool + useSystemRoutesBufferSize int - l *logrus.Logger + // These are routes learned from `tun.use_system_route_table` + // stored here to make it easier to restore them after a reload + routesFromSystem map[netip.Prefix]routing.Gateways + routesFromSystemLock sync.Mutex + + l *slog.Logger +} + +func (t *tun) Networks() []netip.Prefix { + return t.vpnNetworks } type ifReq struct { @@ -47,12 +227,6 @@ type ifReq struct { pad [8]byte } -type ifreqAddr struct { - Name [16]byte - Addr unix.RawSockaddrInet4 - pad [8]byte -} - type ifreqMTU struct { Name [16]byte MTU int32 @@ -65,10 +239,8 @@ type ifreqQLEN struct { pad [8]byte } -func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) (*tun, error) { - file := os.NewFile(uintptr(deviceFd), "/dev/net/tun") - - t, err := newTunGeneric(c, l, file, cidr) +func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) { + t, err := newTunGeneric(c, l, deviceFd, vpnNetworks) if err != nil { return nil, err } @@ -78,10 +250,27 @@ func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) return t, nil } -func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, multiqueue bool) (*tun, error) { +func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) { fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0) if err != nil { - return nil, err + // If /dev/net/tun doesn't exist, try to create it (will happen in docker) + if os.IsNotExist(err) { + err = os.MkdirAll("/dev/net", 0755) + if err != nil { + return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err) + } + err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200))) + if err != nil { + return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err) + } + + fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err) + } + } else { + return nil, err + } } var req ifReq @@ -89,14 +278,18 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, multiqueue bool) (*t if multiqueue { req.Flags |= unix.IFF_MULTI_QUEUE } - copy(req.Name[:], c.GetString("tun.dev", "")) + nameStr := c.GetString("tun.dev", "") + copy(req.Name[:], nameStr) if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil { - return nil, err + _ = unix.Close(fd) + return nil, &NameError{ + Name: nameStr, + Underlying: err, + } } name := strings.Trim(string(req.Name[:]), "\x00") - file := os.NewFile(uintptr(fd), "/dev/net/tun") - t, err := newTunGeneric(c, l, file, cidr) + t, err := newTunGeneric(c, l, fd, vpnNetworks) if err != nil { return nil, err } @@ -106,18 +299,27 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, multiqueue bool) (*t return t, nil } -func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, cidr *net.IPNet) (*tun, error) { - t := &tun{ - ReadWriteCloser: file, - fd: int(file.Fd()), - cidr: cidr, - TXQueueLen: c.GetInt("tun.tx_queue", 500), - useSystemRoutes: c.GetBool("tun.use_system_route_table", false), - l: l, - } - - err := t.reload(c, true) +// newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error. +func newTunGeneric(c *config.C, l *slog.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) { + tfd, err := newTunFd(fd) if err != nil { + _ = unix.Close(fd) + return nil, err + } + t := &tun{ + tunFile: tfd, + readers: []*tunFile{tfd}, + closeLock: sync.Mutex{}, + vpnNetworks: vpnNetworks, + TXQueueLen: c.GetInt("tun.tx_queue", 500), + useSystemRoutes: c.GetBool("tun.use_system_route_table", false), + useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), + routesFromSystem: map[netip.Prefix]routing.Gateways{}, + l: l, + } + + if err = t.reload(c, true); err != nil { + _ = t.Close() return nil, err } @@ -132,7 +334,7 @@ func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, cidr *net.IPNet } func (t *tun) reload(c *config.C, initial bool) error { - routeChange, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) if err != nil { return err } @@ -146,6 +348,13 @@ func (t *tun) reload(c *config.C, initial bool) error { return err } + // Bring along any routes learned from the system route table on reload + t.routesFromSystemLock.Lock() + for dst, gw := range t.routesFromSystem { + routeTree.Insert(dst, gw) + } + t.routesFromSystemLock.Unlock() + oldDefaultMTU := t.DefaultMTU oldMaxMTU := t.MaxMTU newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU) @@ -170,15 +379,17 @@ func (t *tun) reload(c *config.C, initial bool) error { if !initial { if oldMaxMTU != newMaxMTU { t.setMTU() - t.l.Infof("Set max MTU to %v was %v", t.MaxMTU, oldMaxMTU) + t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU) } if oldDefaultMTU != newDefaultMTU { - err := t.setDefaultRoute() - if err != nil { - t.l.Warn(err) - } else { - t.l.Infof("Set default MTU to %v was %v", t.DefaultMTU, oldDefaultMTU) + for i := range t.vpnNetworks { + err := t.setDefaultRoute(t.vpnNetworks[i]) + if err != nil { + t.l.Warn(err.Error()) + } else { + t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU) + } } } @@ -196,7 +407,14 @@ func (t *tun) reload(c *config.C, initial bool) error { return nil } +func (t *tun) SupportsMultiqueue() bool { + return true +} + func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { + t.closeLock.Lock() + defer t.closeLock.Unlock() + fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0) if err != nil { return nil, err @@ -206,47 +424,91 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE) copy(req.Name[:], t.Device) if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil { + _ = unix.Close(fd) return nil, err } - file := os.NewFile(uintptr(fd), "/dev/net/tun") + out, err := t.tunFile.newFriend(fd) + if err != nil { + _ = unix.Close(fd) + return nil, err + } + + t.readers = append(t.readers, out) - return file, nil + return out, nil } -func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.Load().MostSpecificContains(ip) +func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { + r, _ := t.routeTree.Load().Lookup(ip) return r } -func (t *tun) Write(b []byte) (int, error) { - var nn int - max := len(b) +func (t *tun) deviceBytes() (o [16]byte) { + for i, c := range t.Device { + o[i] = byte(c) + } + return +} - for { - n, err := unix.Write(t.fd, b[nn:max]) - if n > 0 { - nn += n +func hasNetlinkAddr(al []*netlink.Addr, x netlink.Addr) bool { + for i := range al { + if al[i].Equal(x) { + return true } - if nn == len(b) { - return nn, err + } + return false +} + +// addIPs uses netlink to add all addresses that don't exist, then it removes ones that should not be there +func (t *tun) addIPs(link netlink.Link) error { + newAddrs := make([]*netlink.Addr, len(t.vpnNetworks)) + for i := range t.vpnNetworks { + newAddrs[i] = &netlink.Addr{ + IPNet: &net.IPNet{ + IP: t.vpnNetworks[i].Addr().AsSlice(), + Mask: net.CIDRMask(t.vpnNetworks[i].Bits(), t.vpnNetworks[i].Addr().BitLen()), + }, + Label: t.vpnNetworks[i].Addr().Zone(), } + } - if err != nil { - return nn, err + //add all new addresses + for i := range newAddrs { + //AddrReplace still adds new IPs, but if their properties change it will change them as well + if err := netlink.AddrReplace(link, newAddrs[i]); err != nil { + return err } + } - if n == 0 { - return nn, io.ErrUnexpectedEOF + //iterate over remainder, remove whoever shouldn't be there + al, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + //RTM_GETADDR dumps the whole system, so any concurrent address change + //interrupts it - including the kernel's async tentative->preferred + //flip of an IPv6 address the AddrReplace calls above just added, + //which makes this a race against our own setup. Partial results are + //still returned; the worst case is a stale address surviving until + //the next config reload, which beats failing startup over it. + if !errors.Is(err, netlink.ErrDumpInterrupted) { + return fmt.Errorf("failed to get tun address list: %s", err) } + t.l.Warn("tun address list dump was interrupted, stale addresses may remain") } -} -func (t *tun) deviceBytes() (o [16]byte) { - for i, c := range t.Device { - o[i] = byte(c) + for i := range al { + if hasNetlinkAddr(newAddrs, al[i]) { + continue + } + err = netlink.AddrDel(link, &al[i]) + if err != nil { + t.l.Error("failed to remove address from tun address list", "error", err) + } else { + t.l.Info("removed address not listed in cert(s)", "removed", al[i].String()) + } } - return + + return nil } func (t *tun) Activate() error { @@ -256,13 +518,8 @@ func (t *tun) Activate() error { t.watchRoutes() } - var addr, mask [4]byte - - copy(addr[:], t.cidr.IP.To4()) - copy(mask[:], t.cidr.Mask) - s, err := unix.Socket( - unix.AF_INET, + unix.AF_INET, //because everything we use t.ioctlFd for is address family independent, this is fine unix.SOCK_DGRAM, unix.IPPROTO_IP, ) @@ -271,31 +528,19 @@ func (t *tun) Activate() error { } t.ioctlFd = uintptr(s) - ifra := ifreqAddr{ - Name: devName, - Addr: unix.RawSockaddrInet4{ - Family: unix.AF_INET, - Addr: addr, - }, - } - - // Set the device ip address - if err = ioctl(t.ioctlFd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil { - return fmt.Errorf("failed to set tun address: %s", err) - } - - // Set the device network - ifra.Addr.Addr = mask - if err = ioctl(t.ioctlFd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil { - return fmt.Errorf("failed to set tun netmask: %s", err) - } - // Set the device name ifrf := ifReq{Name: devName} if err = ioctl(t.ioctlFd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { return fmt.Errorf("failed to set tun device name: %s", err) } + link, err := netlink.LinkByName(t.Device) + if err != nil { + return fmt.Errorf("failed to get tun device link: %s", err) + } + + t.deviceIndex = link.Attrs().Index + // Setup our default MTU t.setMTU() @@ -303,7 +548,16 @@ func (t *tun) Activate() error { ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)} if err = ioctl(t.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil { // If we can't set the queue length nebula will still work but it may lead to packet loss - t.l.WithError(err).Error("Failed to set tun tx queue length") + t.l.Error("Failed to set tun tx queue length", "error", err) + } + + const modeNone = 1 + if err = netlink.LinkSetIP6AddrGenMode(link, modeNone); err != nil { + t.l.Warn("Failed to disable link local address generation", "error", err) + } + + if err = t.addIPs(link); err != nil { + return err } // Bring up the interface @@ -312,14 +566,11 @@ func (t *tun) Activate() error { return fmt.Errorf("failed to bring the tun device up: %s", err) } - link, err := netlink.LinkByName(t.Device) - if err != nil { - return fmt.Errorf("failed to get tun device link: %s", err) - } - t.deviceIndex = link.Attrs().Index - - if err = t.setDefaultRoute(); err != nil { - return err + //set route MTU + for i := range t.vpnNetworks { + if err = t.setDefaultRoute(t.vpnNetworks[i]); err != nil { + return fmt.Errorf("failed to set default route MTU: %w", err) + } } // Set the routes @@ -341,27 +592,47 @@ func (t *tun) setMTU() { ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)} if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { // This is currently a non fatal condition because the route table must have the MTU set appropriately as well - t.l.WithError(err).Error("Failed to set tun mtu") + t.l.Error("Failed to set tun mtu", "error", err) } } -func (t *tun) setDefaultRoute() error { - // Default route - dr := &net.IPNet{IP: t.cidr.IP.Mask(t.cidr.Mask), Mask: t.cidr.Mask} +func (t *tun) setDefaultRoute(cidr netip.Prefix) error { + dr := &net.IPNet{ + IP: cidr.Masked().Addr().AsSlice(), + Mask: net.CIDRMask(cidr.Bits(), cidr.Addr().BitLen()), + } + nr := netlink.Route{ LinkIndex: t.deviceIndex, Dst: dr, MTU: t.DefaultMTU, AdvMSS: t.advMSS(Route{}), Scope: unix.RT_SCOPE_LINK, - Src: t.cidr.IP, + Src: net.IP(cidr.Addr().AsSlice()), Protocol: unix.RTPROT_KERNEL, Table: unix.RT_TABLE_MAIN, Type: unix.RTN_UNICAST, } err := netlink.RouteReplace(&nr) if err != nil { - return fmt.Errorf("failed to set mtu %v on the default route %v; %v", t.DefaultMTU, dr, err) + t.l.Warn("Failed to set default route MTU, retrying", "error", err, "cidr", cidr) + //retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument` + for i := 0; i < 2; i++ { + time.Sleep(100 * time.Millisecond) + err = netlink.RouteReplace(&nr) + if err == nil { + break + } else { + t.l.Warn("Failed to set default route MTU, retrying", + "error", err, + "cidr", cidr, + "mtu", t.DefaultMTU, + ) + } + } + if err != nil { + return fmt.Errorf("failed to set mtu %v on the default route %v; %v", t.DefaultMTU, dr, err) + } } return nil @@ -375,9 +646,14 @@ func (t *tun) addRoutes(logErrors bool) error { continue } + dr := &net.IPNet{ + IP: r.Cidr.Masked().Addr().AsSlice(), + Mask: net.CIDRMask(r.Cidr.Bits(), r.Cidr.Addr().BitLen()), + } + nr := netlink.Route{ LinkIndex: t.deviceIndex, - Dst: r.Cidr, + Dst: dr, MTU: r.MTU, AdvMSS: t.advMSS(r), Scope: unix.RT_SCOPE_LINK, @@ -389,14 +665,14 @@ func (t *tun) addRoutes(logErrors bool) error { err := netlink.RouteReplace(&nr) if err != nil { - retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err) + retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) } else { return retErr } } else { - t.l.WithField("route", r).Info("Added route") + t.l.Info("Added route", "route", r) } } @@ -409,9 +685,14 @@ func (t *tun) removeRoutes(routes []Route) { continue } + dr := &net.IPNet{ + IP: r.Cidr.Masked().Addr().AsSlice(), + Mask: net.CIDRMask(r.Cidr.Bits(), r.Cidr.Addr().BitLen()), + } + nr := netlink.Route{ LinkIndex: t.deviceIndex, - Dst: r.Cidr, + Dst: dr, MTU: r.MTU, AdvMSS: t.advMSS(r), Scope: unix.RT_SCOPE_LINK, @@ -423,17 +704,13 @@ func (t *tun) removeRoutes(routes []Route) { err := netlink.RouteDel(&nr) if err != nil { - t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + t.l.Error("Failed to remove route", "error", err, "route", r) } else { - t.l.WithField("route", r).Info("Removed route") + t.l.Info("Removed route", "route", r) } } } -func (t *tun) Cidr() *net.IPNet { - return t.cidr -} - func (t *tun) Name() string { return t.Device } @@ -455,8 +732,14 @@ func (t *tun) watchRoutes() { rch := make(chan netlink.RouteUpdate) doneChan := make(chan struct{}) - if err := netlink.RouteSubscribe(rch, doneChan); err != nil { - t.l.WithError(err).Errorf("failed to subscribe to system route changes") + netlinkOptions := netlink.RouteSubscribeOptions{ + ReceiveBufferSize: t.useSystemRoutesBufferSize, + ReceiveBufferForceSize: t.useSystemRoutesBufferSize != 0, + ErrorCallback: func(e error) { t.l.Error("netlink error", "error", e) }, + } + + if err := netlink.RouteSubscribeWithOptions(rch, doneChan, netlinkOptions); err != nil { + t.l.Error("failed to subscribe to system route changes", "error", err) return } @@ -465,8 +748,14 @@ func (t *tun) watchRoutes() { go func() { for { select { - case r := <-rch: - t.updateRoutes(r) + case r, ok := <-rch: + if ok { + t.updateRoutes(r) + } else { + // may be should do something here as + // netlink stops sending updates + return + } case <-doneChan: // netlink.RouteSubscriber will close the rch for us return @@ -475,62 +764,156 @@ func (t *tun) watchRoutes() { }() } +func (t *tun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool { + withinNetworks := false + for i := range t.vpnNetworks { + if t.vpnNetworks[i].Contains(gwAddr) { + withinNetworks = true + break + } + } + + return withinNetworks +} + +func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways { + var gateways routing.Gateways + + link, err := netlink.LinkByName(t.Device) + if err != nil { + t.l.Error("Ignoring route update: failed to get link by name", "deviceName", t.Device) + return gateways + } + + // If this route is relevant to our interface and there is a gateway then add it + if r.LinkIndex == link.Attrs().Index { + gwAddr, ok := getGatewayAddr(r.Gw, r.Via) + if ok { + if t.isGatewayInVpnNetworks(gwAddr) { + gateways = append(gateways, routing.NewGateway(gwAddr, 1)) + } else { + // Gateway isn't in our overlay network, ignore + t.l.Debug("Ignoring route update, gateway is not in our network", "route", r) + } + } else { + t.l.Debug("Ignoring route update, invalid gateway or via address", "route", r) + } + } + + for _, p := range r.MultiPath { + // If this route is relevant to our interface and there is a gateway then add it + if p.LinkIndex == link.Attrs().Index { + gwAddr, ok := getGatewayAddr(p.Gw, p.Via) + if ok { + if t.isGatewayInVpnNetworks(gwAddr) { + gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1)) + } else { + // Gateway isn't in our overlay network, ignore + t.l.Debug("Ignoring route update, gateway is not in our network", "route", r) + } + } else { + t.l.Debug("Ignoring route update, invalid gateway or via address", "route", r) + } + } + } + + routing.CalculateBucketsForGateways(gateways) + return gateways +} + +func getGatewayAddr(gw net.IP, via netlink.Destination) (netip.Addr, bool) { + // Try to use the old RTA_GATEWAY first + gwAddr, ok := netip.AddrFromSlice(gw) + if !ok { + // Fallback to the new RTA_VIA + rVia, ok := via.(*netlink.Via) + if ok { + gwAddr, ok = netip.AddrFromSlice(rVia.Addr) + } + } + + if gwAddr.IsValid() { + gwAddr = gwAddr.Unmap() + return gwAddr, true + } + + return netip.Addr{}, false +} + func (t *tun) updateRoutes(r netlink.RouteUpdate) { - if r.Gw == nil { - // Not a gateway route, ignore - t.l.WithField("route", r).Debug("Ignoring route update, not a gateway route") + gateways := t.getGatewaysFromRoute(&r.Route) + if len(gateways) == 0 { + // No gateways relevant to our network, no routing changes required. + t.l.Debug("Ignoring route update, no gateways", "route", r) return } - if !t.cidr.Contains(r.Gw) { - // Gateway isn't in our overlay network, ignore - t.l.WithField("route", r).Debug("Ignoring route update, not in our network") + if r.Dst == nil { + t.l.Debug("Ignoring route update, no destination address", "route", r) return } - if x := r.Dst.IP.To4(); x == nil { - // Nebula only handles ipv4 on the overlay currently - t.l.WithField("route", r).Debug("Ignoring route update, destination is not ipv4") + dstAddr, ok := netip.AddrFromSlice(r.Dst.IP) + if !ok { + t.l.Debug("Ignoring route update, invalid destination address", "route", r) return } - newTree := cidr.NewTree4[iputil.VpnIp]() - if r.Type == unix.RTM_NEWROUTE { - for _, oldR := range t.routeTree.Load().List() { - newTree.AddCIDR(oldR.CIDR, oldR.Value) - } + ones, _ := r.Dst.Mask.Size() + dst := netip.PrefixFrom(dstAddr, ones) - t.l.WithField("destination", r.Dst).WithField("via", r.Gw).Info("Adding route") - newTree.AddCIDR(r.Dst, iputil.Ip2VpnIp(r.Gw)) + newTree := t.routeTree.Load().Clone() - } else { - gw := iputil.Ip2VpnIp(r.Gw) - for _, oldR := range t.routeTree.Load().List() { - if bytes.Equal(oldR.CIDR.IP, r.Dst.IP) && bytes.Equal(oldR.CIDR.Mask, r.Dst.Mask) && oldR.Value == gw { - // This is the record to delete - t.l.WithField("destination", r.Dst).WithField("via", r.Gw).Info("Removing route") - continue - } + t.routesFromSystemLock.Lock() + if r.Type == unix.RTM_NEWROUTE { + t.l.Info("Adding route", "destination", dst, "via", gateways) + t.routesFromSystem[dst] = gateways + newTree.Insert(dst, gateways) - newTree.AddCIDR(oldR.CIDR, oldR.Value) - } + } else { + t.l.Info("Removing route", "destination", dst, "via", gateways) + delete(t.routesFromSystem, dst) + newTree.Delete(dst) } - + t.routesFromSystemLock.Unlock() t.routeTree.Store(newTree) } func (t *tun) Close() error { + t.closeLock.Lock() + defer t.closeLock.Unlock() + if t.routeChan != nil { close(t.routeChan) + t.routeChan = nil } - if t.ReadWriteCloser != nil { - t.ReadWriteCloser.Close() - } + // Signal all readers blocked in poll to wake up and exit + _ = t.tunFile.wakeForShutdown() if t.ioctlFd > 0 { - os.NewFile(t.ioctlFd, "ioctlFd").Close() + _ = unix.Close(int(t.ioctlFd)) + t.ioctlFd = 0 } - return nil + for i := range t.readers { + if i == 0 { + continue //we want to close the zeroth reader last + } + err := t.readers[i].Close() + if err != nil { + t.l.Error("error closing tun reader", "reader", i, "error", err) + } else { + t.l.Info("closed tun reader", "reader", i) + } + } + + //this is t.readers[0] too + err := t.tunFile.Close() + if err != nil { + t.l.Error("error closing tun reader", "reader", 0, "error", err) + } else { + t.l.Info("closed tun reader", "reader", 0) + } + return err } diff --git a/overlay/tun_netbsd.go b/overlay/tun_netbsd.go index cc0216fe..c971bb6e 100644 --- a/overlay/tun_netbsd.go +++ b/overlay/tun_netbsd.go @@ -4,70 +4,78 @@ package overlay import ( + "errors" "fmt" "io" - "net" + "log/slog" + "net/netip" "os" - "os/exec" "regexp" - "strconv" "sync/atomic" "syscall" "unsafe" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" + netroute "golang.org/x/net/route" + "golang.org/x/sys/unix" ) -type ifreqDestroy struct { - Name [16]byte - pad [16]byte -} - -type tun struct { - Device string - cidr *net.IPNet - MTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] - l *logrus.Logger +const ( + SIOCAIFADDR_IN6 = 0x8080696b + TUNSIFHEAD = 0x80047442 + TUNSIFMODE = 0x80047458 +) - io.ReadWriteCloser +type ifreqAlias4 struct { + Name [unix.IFNAMSIZ]byte + Addr unix.RawSockaddrInet4 + DstAddr unix.RawSockaddrInet4 + MaskAddr unix.RawSockaddrInet4 } -func (t *tun) Close() error { - if t.ReadWriteCloser != nil { - if err := t.ReadWriteCloser.Close(); err != nil { - return err - } - - s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) - if err != nil { - return err - } - defer syscall.Close(s) - - ifreq := ifreqDestroy{Name: t.deviceBytes()} +type ifreqAlias6 struct { + Name [unix.IFNAMSIZ]byte + Addr unix.RawSockaddrInet6 + DstAddr unix.RawSockaddrInet6 + PrefixMask unix.RawSockaddrInet6 + Flags uint32 + Lifetime addrLifetime +} - err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq))) +type ifreq struct { + Name [unix.IFNAMSIZ]byte + data int +} - return err - } - return nil +type addrLifetime struct { + Expire uint64 + Preferred uint64 + Vltime uint32 + Pltime uint32 } -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) { - return nil, fmt.Errorf("newTunFromFd not supported in NetBSD") +type tun struct { + Device string + vpnNetworks []netip.Prefix + MTU int + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[bart.Table[routing.Gateways]] + l *slog.Logger + f *os.File + fd int } var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`) -func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) { +func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { + return nil, fmt.Errorf("newTunFromFd not supported in NetBSD") +} + +func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { // Try to open tun device - var file *os.File var err error deviceName := c.GetString("tun.dev", "") if deviceName == "" { @@ -77,17 +85,23 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") } - file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0) + fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0) if err != nil { return nil, err } + err = unix.SetNonblock(fd, true) + if err != nil { + l.Warn("Failed to set the tun device as nonblocking", "error", err) + } + t := &tun{ - ReadWriteCloser: file, - Device: deviceName, - cidr: cidr, - MTU: c.GetInt("tun.mtu", DefaultMTU), - l: l, + f: os.NewFile(uintptr(fd), ""), + fd: fd, + Device: deviceName, + vpnNetworks: vpnNetworks, + MTU: c.GetInt("tun.mtu", DefaultMTU), + l: l, } err = t.reload(c, true) @@ -105,34 +119,229 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error return t, nil } +func (t *tun) Close() error { + if t.f != nil { + if err := t.f.Close(); err != nil { + return fmt.Errorf("error closing tun file: %w", err) + } + + // t.f.Close should have handled it for us but let's be extra sure + _ = unix.Close(t.fd) + + s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + + ifr := ifreq{Name: t.deviceBytes()} + err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifr))) + return err + } + return nil +} + +func (t *tun) Read(to []byte) (int, error) { + rc, err := t.f.SyscallConn() + if err != nil { + return 0, fmt.Errorf("failed to get syscall conn for tun: %w", err) + } + + var errno syscall.Errno + var n uintptr + err = rc.Read(func(fd uintptr) bool { + // first 4 bytes is protocol family, in network byte order + head := [4]byte{} + iovecs := []syscall.Iovec{ + {&head[0], 4}, + {&to[0], uint64(len(to))}, + } + + n, _, errno = syscall.Syscall(syscall.SYS_READV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2)) + if errno.Temporary() { + // We got an EAGAIN, EINTR, or EWOULDBLOCK, go again + return false + } + return true + }) + if err != nil { + if err == syscall.EBADF || err.Error() == "use of closed file" { + // Go doesn't export poll.ErrFileClosing but happily reports it to us so here we are + // https://github.com/golang/go/blob/master/src/internal/poll/fd_poll_runtime.go#L121 + return 0, os.ErrClosed + } + return 0, fmt.Errorf("failed to make read call for tun: %w", err) + } + + if errno != 0 { + return 0, fmt.Errorf("failed to make inner read call for tun: %w", errno) + } + + // fix bytes read number to exclude header + bytesRead := int(n) + if bytesRead < 0 { + return bytesRead, nil + } else if bytesRead < 4 { + return 0, nil + } else { + return bytesRead - 4, nil + } +} + +// Write is only valid for single threaded use +func (t *tun) Write(from []byte) (int, error) { + if len(from) <= 1 { + return 0, syscall.EIO + } + + ipVer := from[0] >> 4 + var head [4]byte + // first 4 bytes is protocol family, in network byte order + if ipVer == 4 { + head[3] = syscall.AF_INET + } else if ipVer == 6 { + head[3] = syscall.AF_INET6 + } else { + return 0, fmt.Errorf("unable to determine IP version from packet") + } + + rc, err := t.f.SyscallConn() + if err != nil { + return 0, err + } + + var errno syscall.Errno + var n uintptr + err = rc.Write(func(fd uintptr) bool { + iovecs := []syscall.Iovec{ + {&head[0], 4}, + {&from[0], uint64(len(from))}, + } + + n, _, errno = syscall.Syscall(syscall.SYS_WRITEV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2)) + // According to NetBSD documentation for TUN, writes will only return errors in which + // this packet will never be delivered so just go on living life. + return true + }) + if err != nil { + return 0, err + } + + if errno != 0 { + return 0, errno + } + + return int(n) - 4, err +} + +func (t *tun) addIp(cidr netip.Prefix) error { + if cidr.Addr().Is4() { + var req ifreqAlias4 + req.Name = t.deviceBytes() + req.Addr = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: cidr.Addr().As4(), + } + req.DstAddr = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: cidr.Addr().As4(), + } + req.MaskAddr = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: prefixToMask(cidr).As4(), + } + + s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + + if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&req))); err != nil { + return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr(), err) + } + + return nil + } + + if cidr.Addr().Is6() { + var req ifreqAlias6 + req.Name = t.deviceBytes() + req.Addr = unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: cidr.Addr().As16(), + } + req.PrefixMask = unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: prefixToMask(cidr).As16(), + } + req.Lifetime = addrLifetime{ + Vltime: 0xffffffff, + Pltime: 0xffffffff, + } + + s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + + if err := ioctl(uintptr(s), SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&req))); err != nil { + return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err) + } + return nil + } + + return fmt.Errorf("unknown address type %v", cidr) +} + func (t *tun) Activate() error { - var err error + mode := int32(unix.IFF_BROADCAST) + err := ioctl(uintptr(t.fd), TUNSIFMODE, uintptr(unsafe.Pointer(&mode))) + if err != nil { + return fmt.Errorf("failed to set tun device mode: %w", err) + } - // TODO use syscalls instead of exec.Command - cmd := exec.Command("/sbin/ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'ifconfig': %s", err) + v := 1 + err = ioctl(uintptr(t.fd), TUNSIFHEAD, uintptr(unsafe.Pointer(&v))) + if err != nil { + return fmt.Errorf("failed to set tun device head: %w", err) } - cmd = exec.Command("/sbin/route", "-n", "add", "-net", t.cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'route add': %s", err) + err = t.doIoctlByName(unix.SIOCSIFMTU, uint32(t.MTU)) + if err != nil { + return fmt.Errorf("failed to set tun mtu: %w", err) } - cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU)) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'ifconfig': %s", err) + for i := range t.vpnNetworks { + err = t.addIp(t.vpnNetworks[i]) + if err != nil { + return err + } } - // Unsafe path routes return t.addRoutes(false) } +func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { + s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + + ir := ifreq{Name: t.deviceBytes(), data: int(value)} + err = ioctl(uintptr(s), ctl, uintptr(unsafe.Pointer(&ir))) + return err +} + func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) if err != nil { return err } @@ -168,40 +377,46 @@ func (t *tun) reload(c *config.C, initial bool) error { return nil } -func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.Load().MostSpecificContains(ip) +func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { + r, _ := t.routeTree.Load().Lookup(ip) return r } -func (t *tun) Cidr() *net.IPNet { - return t.cidr +func (t *tun) Networks() []netip.Prefix { + return t.vpnNetworks } func (t *tun) Name() string { return t.Device } +func (t *tun) SupportsMultiqueue() bool { + return false +} + func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd") } func (t *tun) addRoutes(logErrors bool) error { routes := *t.Routes.Load() + for _, r := range routes { - if r.Via == nil || !r.Install { + if len(r.Via) == 0 || !r.Install { // We don't allow route MTUs so only install routes with a via continue } - cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err := cmd.Run(); err != nil { - retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]interface{}{"route": r}, err) + err := addRoute(r.Cidr, t.vpnNetworks) + if err != nil { + retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) } else { return retErr } + } else { + t.l.Info("Added route", "route", r) } } @@ -214,12 +429,11 @@ func (t *tun) removeRoutes(routes []Route) error { continue } - cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err := cmd.Run(); err != nil { - t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + err := delRoute(r.Cidr, t.vpnNetworks) + if err != nil { + t.l.Error("Failed to remove route", "error", err, "route", r) } else { - t.l.WithField("route", r).Info("Removed route") + t.l.Info("Removed route", "route", r) } } return nil @@ -231,3 +445,109 @@ func (t *tun) deviceBytes() (o [16]byte) { } return } + +func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error { + sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + defer unix.Close(sock) + + route := &netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_ADD, + Flags: unix.RTF_UP | unix.RTF_GATEWAY, + Seq: 1, + } + + if prefix.Addr().Is4() { + gw, err := selectGateway(prefix, gateways) + if err != nil { + return err + } + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, + unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()}, + } + } else { + gw, err := selectGateway(prefix, gateways) + if err != nil { + return err + } + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, + unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, + unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()}, + } + } + + data, err := route.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage: %w", err) + } + + _, err = unix.Write(sock, data[:]) + if err != nil { + if errors.Is(err, unix.EEXIST) { + // Try to do a change + route.Type = unix.RTM_CHANGE + data, err = route.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage for change: %w", err) + } + _, err = unix.Write(sock, data[:]) + return err + } + return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) + } + + return nil +} + +func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error { + sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + defer unix.Close(sock) + + route := netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_DELETE, + Seq: 1, + } + + if prefix.Addr().Is4() { + gw, err := selectGateway(prefix, gateways) + if err != nil { + return err + } + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, + unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()}, + } + } else { + gw, err := selectGateway(prefix, gateways) + if err != nil { + return err + } + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, + unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, + unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()}, + } + } + + data, err := route.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage: %w", err) + } + _, err = unix.Write(sock, data[:]) + if err != nil { + return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) + } + + return nil +} diff --git a/overlay/tun_openbsd.go b/overlay/tun_openbsd.go index 53f57b13..41224777 100644 --- a/overlay/tun_openbsd.go +++ b/overlay/tun_openbsd.go @@ -4,72 +4,95 @@ package overlay import ( + "errors" "fmt" "io" - "net" + "log/slog" + "net/netip" "os" - "os/exec" "regexp" - "strconv" "sync/atomic" "syscall" + "unsafe" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" + netroute "golang.org/x/net/route" + "golang.org/x/sys/unix" ) -type tun struct { - Device string - cidr *net.IPNet - MTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] - l *logrus.Logger - - io.ReadWriteCloser +const ( + SIOCAIFADDR_IN6 = 0x8080691a +) - // cache out buffer since we need to prepend 4 bytes for tun metadata - out []byte +type ifreqAlias4 struct { + Name [unix.IFNAMSIZ]byte + Addr unix.RawSockaddrInet4 + DstAddr unix.RawSockaddrInet4 + MaskAddr unix.RawSockaddrInet4 } -func (t *tun) Close() error { - if t.ReadWriteCloser != nil { - return t.ReadWriteCloser.Close() - } +type ifreqAlias6 struct { + Name [unix.IFNAMSIZ]byte + Addr unix.RawSockaddrInet6 + DstAddr unix.RawSockaddrInet6 + PrefixMask unix.RawSockaddrInet6 + Flags uint32 + Lifetime [2]uint32 +} - return nil +type ifreq struct { + Name [unix.IFNAMSIZ]byte + data int } -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) { - return nil, fmt.Errorf("newTunFromFd not supported in OpenBSD") +type tun struct { + Device string + vpnNetworks []netip.Prefix + MTU int + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[bart.Table[routing.Gateways]] + l *slog.Logger + f *os.File + fd int } var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`) -func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) { +func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { + return nil, fmt.Errorf("newTunFromFd not supported in openbsd") +} + +func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { + // Try to open tun device + var err error deviceName := c.GetString("tun.dev", "") if deviceName == "" { - return nil, fmt.Errorf("a device name in the format of tunN must be specified") + return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") } - if !deviceNameRE.MatchString(deviceName) { - return nil, fmt.Errorf("a device name in the format of tunN must be specified") + return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") } - file, err := os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0) + fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0) if err != nil { return nil, err } + err = unix.SetNonblock(fd, true) + if err != nil { + l.Warn("Failed to set the tun device as nonblocking", "error", err) + } + t := &tun{ - ReadWriteCloser: file, - Device: deviceName, - cidr: cidr, - MTU: c.GetInt("tun.mtu", DefaultMTU), - l: l, + f: os.NewFile(uintptr(fd), ""), + fd: fd, + Device: deviceName, + vpnNetworks: vpnNetworks, + MTU: c.GetInt("tun.mtu", DefaultMTU), + l: l, } err = t.reload(c, true) @@ -87,8 +110,217 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error return t, nil } +func (t *tun) Close() error { + if t.f != nil { + if err := t.f.Close(); err != nil { + return fmt.Errorf("error closing tun file: %w", err) + } + + // t.f.Close should have handled it for us but let's be extra sure + _ = unix.Close(t.fd) + } + return nil +} + +// tunWritev and tunReadv are linkname'd to x/sys/unix's libc-routed writev/readv stubs so the +// calls go through libc's pinned trampoline. OpenBSD's pinsyscall protection rejects a raw +// syscall.Syscall(SYS_WRITEV/SYS_READV, ...) because it doesn't originate from a libc-pinned +// address, so we can't use the syscall.Syscall pattern that freebsd / netbsd use. We pull the +// low-level stubs instead of calling unix.Writev/unix.Readv because those take [][]byte and rebuild +// the []Iovec every call, which heap-allocates the header; linkname'ing the stubs lets us hand them +// our own stack-allocated iovecs. See golang/go#78049. + +//go:linkname tunWritev golang.org/x/sys/unix.writev +//go:noescape +func tunWritev(fd int, iovecs []unix.Iovec) (n int, err error) + +//go:linkname tunReadv golang.org/x/sys/unix.readv +//go:noescape +func tunReadv(fd int, iovecs []unix.Iovec) (n int, err error) + +// Read pulls one IP packet off the tun device, scattering the 4 byte protocol header away from the +// packet so the payload lands directly in to. +func (t *tun) Read(to []byte) (int, error) { + var head [4]byte + + rc, err := t.f.SyscallConn() + if err != nil { + return 0, err + } + + var n int + var callErr error + err = rc.Read(func(fd uintptr) bool { + iovecs := []unix.Iovec{ + {Base: &head[0], Len: 4}, + {Base: &to[0], Len: uint64(len(to))}, + } + n, callErr = tunReadv(int(fd), iovecs) + if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() { + return false + } + return true + }) + if err != nil { + return 0, err + } + if callErr != nil { + return 0, callErr + } + if n < 4 { + return 0, nil + } + return n - 4, nil +} + +// Write pushes one IP packet onto the tun device. +func (t *tun) Write(from []byte) (int, error) { + if len(from) == 0 { + return 0, syscall.EIO + } + + ipVer := from[0] >> 4 + var head [4]byte + switch ipVer { + case 4: + head[3] = syscall.AF_INET + case 6: + head[3] = syscall.AF_INET6 + default: + return 0, fmt.Errorf("unable to determine IP version from packet") + } + + // Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack. + rc, err := t.f.SyscallConn() + if err != nil { + return 0, err + } + + var n int + var callErr error + err = rc.Write(func(fd uintptr) bool { + iovecs := []unix.Iovec{ + {Base: &head[0], Len: 4}, + {Base: &from[0], Len: uint64(len(from))}, + } + n, callErr = tunWritev(int(fd), iovecs) + // Type-assert to syscall.Errno so the EAGAIN/EWOULDBLOCK/EINTR check doesn't box the errno + // constants into error interfaces on every call. + if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() { + return false + } + return true + }) + if err != nil { + return 0, err + } + if callErr != nil { + return 0, callErr + } + + return n - 4, nil +} + +func (t *tun) addIp(cidr netip.Prefix) error { + if cidr.Addr().Is4() { + var req ifreqAlias4 + req.Name = t.deviceBytes() + req.Addr = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: cidr.Addr().As4(), + } + req.DstAddr = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: cidr.Addr().As4(), + } + req.MaskAddr = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Addr: prefixToMask(cidr).As4(), + } + + s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + + if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&req))); err != nil { + return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr(), err) + } + + err = addRoute(cidr, t.vpnNetworks) + if err != nil { + return fmt.Errorf("failed to set route for vpn network %v: %w", cidr, err) + } + + return nil + } + + if cidr.Addr().Is6() { + var req ifreqAlias6 + req.Name = t.deviceBytes() + req.Addr = unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: cidr.Addr().As16(), + } + req.PrefixMask = unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Addr: prefixToMask(cidr).As16(), + } + req.Lifetime[0] = 0xffffffff + req.Lifetime[1] = 0xffffffff + + s, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + + if err := ioctl(uintptr(s), SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&req))); err != nil { + return fmt.Errorf("failed to set tun address %s: %s", cidr.Addr().String(), err) + } + + return nil + } + + return fmt.Errorf("unknown address type %v", cidr) +} + +func (t *tun) Activate() error { + err := t.doIoctlByName(unix.SIOCSIFMTU, uint32(t.MTU)) + if err != nil { + return fmt.Errorf("failed to set tun mtu: %w", err) + } + + for i := range t.vpnNetworks { + err = t.addIp(t.vpnNetworks[i]) + if err != nil { + return err + } + } + + return t.addRoutes(false) +} + +func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { + s, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_IP) + if err != nil { + return err + } + defer syscall.Close(s) + + ir := ifreq{Name: t.deviceBytes(), data: int(value)} + err = ioctl(uintptr(s), ctl, uintptr(unsafe.Pointer(&ir))) + return err +} + func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) if err != nil { return err } @@ -124,53 +356,46 @@ func (t *tun) reload(c *config.C, initial bool) error { return nil } -func (t *tun) Activate() error { - var err error - // TODO use syscalls instead of exec.Command - cmd := exec.Command("/sbin/ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'ifconfig': %s", err) - } +func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { + r, _ := t.routeTree.Load().Lookup(ip) + return r +} - cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU)) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'ifconfig': %s", err) - } +func (t *tun) Networks() []netip.Prefix { + return t.vpnNetworks +} - cmd = exec.Command("/sbin/route", "-n", "add", "-inet", t.cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'route add': %s", err) - } +func (t *tun) Name() string { + return t.Device +} - // Unsafe path routes - return t.addRoutes(false) +func (t *tun) SupportsMultiqueue() bool { + return false } -func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.Load().MostSpecificContains(ip) - return r +func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { + return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd") } func (t *tun) addRoutes(logErrors bool) error { routes := *t.Routes.Load() + for _, r := range routes { - if r.Via == nil || !r.Install { + if len(r.Via) == 0 || !r.Install { // We don't allow route MTUs so only install routes with a via continue } - cmd := exec.Command("/sbin/route", "-n", "add", "-inet", r.Cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err := cmd.Run(); err != nil { - retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]interface{}{"route": r}, err) + err := addRoute(r.Cidr, t.vpnNetworks) + if err != nil { + retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) } else { return retErr } + } else { + t.l.Info("Added route", "route", r) } } @@ -183,63 +408,125 @@ func (t *tun) removeRoutes(routes []Route) error { continue } - cmd := exec.Command("/sbin/route", "-n", "delete", "-inet", r.Cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err := cmd.Run(); err != nil { - t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + err := delRoute(r.Cidr, t.vpnNetworks) + if err != nil { + t.l.Error("Failed to remove route", "error", err, "route", r) } else { - t.l.WithField("route", r).Info("Removed route") + t.l.Info("Removed route", "route", r) } } return nil } -func (t *tun) Cidr() *net.IPNet { - return t.cidr +func (t *tun) deviceBytes() (o [16]byte) { + for i, c := range t.Device { + o[i] = byte(c) + } + return } -func (t *tun) Name() string { - return t.Device -} +func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error { + sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + defer unix.Close(sock) -func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd") -} + route := &netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_ADD, + Flags: unix.RTF_UP | unix.RTF_GATEWAY, + Seq: 1, + } -func (t *tun) Read(to []byte) (int, error) { - buf := make([]byte, len(to)+4) + if prefix.Addr().Is4() { + gw, err := selectGateway(prefix, gateways) + if err != nil { + return err + } + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, + unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()}, + } + } else { + gw, err := selectGateway(prefix, gateways) + if err != nil { + return err + } + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, + unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, + unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()}, + } + } - n, err := t.ReadWriteCloser.Read(buf) + data, err := route.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage: %w", err) + } - copy(to, buf[4:]) - return n - 4, err + _, err = unix.Write(sock, data[:]) + if err != nil { + if errors.Is(err, unix.EEXIST) { + // Try to do a change + route.Type = unix.RTM_CHANGE + data, err = route.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage for change: %w", err) + } + _, err = unix.Write(sock, data[:]) + return err + } + return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) + } + + return nil } -// Write is only valid for single threaded use -func (t *tun) Write(from []byte) (int, error) { - buf := t.out - if cap(buf) < len(from)+4 { - buf = make([]byte, len(from)+4) - t.out = buf +func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error { + sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) } - buf = buf[:len(from)+4] + defer unix.Close(sock) - if len(from) == 0 { - return 0, syscall.EIO + route := netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_DELETE, + Seq: 1, } - // Determine the IP Family for the NULL L2 Header - ipVer := from[0] >> 4 - if ipVer == 4 { - buf[3] = syscall.AF_INET - } else if ipVer == 6 { - buf[3] = syscall.AF_INET6 + if prefix.Addr().Is4() { + gw, err := selectGateway(prefix, gateways) + if err != nil { + return err + } + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: prefix.Masked().Addr().As4()}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: prefixToMask(prefix).As4()}, + unix.RTAX_GATEWAY: &netroute.Inet4Addr{IP: gw.Addr().As4()}, + } } else { - return 0, fmt.Errorf("unable to determine IP version from packet") + gw, err := selectGateway(prefix, gateways) + if err != nil { + return err + } + route.Addrs = []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet6Addr{IP: prefix.Masked().Addr().As16()}, + unix.RTAX_NETMASK: &netroute.Inet6Addr{IP: prefixToMask(prefix).As16()}, + unix.RTAX_GATEWAY: &netroute.Inet6Addr{IP: gw.Addr().As16()}, + } } - copy(buf[4:], from) + data, err := route.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage: %w", err) + } + _, err = unix.Write(sock, data[:]) + if err != nil { + return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) + } - n, err := t.ReadWriteCloser.Write(buf) - return n - 4, err + return nil } diff --git a/overlay/tun_tester.go b/overlay/tun_tester.go index 38339832..8acd83f0 100644 --- a/overlay/tun_tester.go +++ b/overlay/tun_tester.go @@ -4,32 +4,34 @@ package overlay import ( + "context" "fmt" "io" - "net" + "log/slog" + "net/netip" "os" "sync/atomic" - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" + "github.com/slackhq/nebula/udp" ) type TestTun struct { - Device string - cidr *net.IPNet - Routes []Route - routeTree *cidr.Tree4[iputil.VpnIp] - l *logrus.Logger + Device string + vpnNetworks []netip.Prefix + Routes []Route + routeTree *bart.Table[routing.Gateways] + l *slog.Logger closed atomic.Bool rxPackets chan []byte // Packets to receive into nebula TxPackets chan []byte // Packets transmitted outside by nebula } -func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*TestTun, error) { - _, routes, err := getAllRoutesFromConfig(c, cidr, true) +func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) { + _, routes, err := getAllRoutesFromConfig(c, vpnNetworks, true) if err != nil { return nil, err } @@ -39,32 +41,37 @@ func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*TestTun, e } return &TestTun{ - Device: c.GetString("tun.dev", ""), - cidr: cidr, - Routes: routes, - routeTree: routeTree, - l: l, - rxPackets: make(chan []byte, 10), - TxPackets: make(chan []byte, 10), + Device: c.GetString("tun.dev", ""), + vpnNetworks: vpnNetworks, + Routes: routes, + routeTree: routeTree, + l: l, + rxPackets: make(chan []byte, 10), + TxPackets: make(chan []byte, 10), }, nil } -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*TestTun, error) { +func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*TestTun, error) { return nil, fmt.Errorf("newTunFromFd not supported") } -// Send will place a byte array onto the receive queue for nebula to consume +// Send will place a byte array onto the receive queue for nebula to consume. // These are unencrypted ip layer frames destined for another nebula node. -// packets should exit the udp side, capture them with udpConn.Get +// packets should exit the udp side, capture them with udpConn.Get. +// +// Send copies the input via the freelist, so the caller is free to mutate +// or reuse it after the call returns. func (t *TestTun) Send(packet []byte) { if t.closed.Load() { return } - if t.l.Level >= logrus.DebugLevel { - t.l.WithField("dataLen", len(packet)).Debug("Tun receiving injected packet") + if t.l.Enabled(context.Background(), slog.LevelDebug) { + t.l.Debug("Tun receiving injected packet", "dataLen", len(packet)) } - t.rxPackets <- packet + buf := acquireTunBuf(len(packet)) + copy(buf, packet) + t.rxPackets <- buf } // Get will pull an unencrypted ip layer frame from the transmit queue @@ -87,8 +94,8 @@ func (t *TestTun) Get(block bool) []byte { // Below this is boilerplate implementation to make nebula actually work //********************************************************************************************************************// -func (t *TestTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.MostSpecificContains(ip) +func (t *TestTun) RoutesFor(ip netip.Addr) routing.Gateways { + r, _ := t.routeTree.Lookup(ip) return r } @@ -96,8 +103,8 @@ func (t *TestTun) Activate() error { return nil } -func (t *TestTun) Cidr() *net.IPNet { - return t.cidr +func (t *TestTun) Networks() []netip.Prefix { + return t.vpnNetworks } func (t *TestTun) Name() string { @@ -109,12 +116,44 @@ func (t *TestTun) Write(b []byte) (n int, err error) { return 0, io.ErrClosedPipe } - packet := make([]byte, len(b), len(b)) + packet := acquireTunBuf(len(b)) copy(packet, b) t.TxPackets <- packet return len(b), nil } +// ReleaseTunBuf returns a slice from TxPackets to the harness freelist, don't use the bytes after the call. +// Channel-backed instead of sync.Pool because putting a []byte in a sync.Pool escapes the slice header to heap. +func ReleaseTunBuf(b []byte) { + if b == nil { + return + } + select { + case tunBufFreelist <- b: + default: + // Freelist full; drop the buffer for the GC. + } +} + +// tunBufFreelist retains the backing arrays for TestTun.Write so steady-state allocation drops to zero once the +// freelist has saturated for the current MTU. +var tunBufFreelist = make(chan []byte, 64) + +func acquireTunBuf(n int) []byte { + var b []byte + select { + case b = <-tunBufFreelist: + default: + b = make([]byte, 0, udp.MTU) + } + if cap(b) < n { + b = make([]byte, n) + } else { + b = b[:n] + } + return b +} + func (t *TestTun) Close() error { if t.closed.CompareAndSwap(false, true) { close(t.rxPackets) @@ -128,8 +167,18 @@ func (t *TestTun) Read(b []byte) (int, error) { if !ok { return 0, os.ErrClosed } + n := len(p) copy(b, p) - return len(p), nil + // Send always pushes a freelist-acquired slice, return it once we've copied the bytes into the caller's buffer. + select { + case tunBufFreelist <- p: + default: + } + return n, nil +} + +func (t *TestTun) SupportsMultiqueue() bool { + return false } func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) { diff --git a/overlay/tun_water_windows.go b/overlay/tun_water_windows.go deleted file mode 100644 index a1acd2b2..00000000 --- a/overlay/tun_water_windows.go +++ /dev/null @@ -1,208 +0,0 @@ -package overlay - -import ( - "fmt" - "io" - "net" - "os/exec" - "strconv" - "sync/atomic" - - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" - "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" - "github.com/slackhq/nebula/util" - "github.com/songgao/water" -) - -type waterTun struct { - Device string - cidr *net.IPNet - MTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] - l *logrus.Logger - f *net.Interface - *water.Interface -} - -func newWaterTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*waterTun, error) { - // NOTE: You cannot set the deviceName under Windows, so you must check tun.Device after calling .Activate() - t := &waterTun{ - cidr: cidr, - MTU: c.GetInt("tun.mtu", DefaultMTU), - l: l, - } - - err := t.reload(c, true) - if err != nil { - return nil, err - } - - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) - if err != nil { - util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) - } - }) - - return t, nil -} - -func (t *waterTun) Activate() error { - var err error - t.Interface, err = water.New(water.Config{ - DeviceType: water.TUN, - PlatformSpecificParams: water.PlatformSpecificParams{ - ComponentID: "tap0901", - Network: t.cidr.String(), - }, - }) - if err != nil { - return fmt.Errorf("activate failed: %v", err) - } - - t.Device = t.Interface.Name() - - // TODO use syscalls instead of exec.Command - err = exec.Command( - `C:\Windows\System32\netsh.exe`, "interface", "ipv4", "set", "address", - fmt.Sprintf("name=%s", t.Device), - "source=static", - fmt.Sprintf("addr=%s", t.cidr.IP), - fmt.Sprintf("mask=%s", net.IP(t.cidr.Mask)), - "gateway=none", - ).Run() - if err != nil { - return fmt.Errorf("failed to run 'netsh' to set address: %s", err) - } - err = exec.Command( - `C:\Windows\System32\netsh.exe`, "interface", "ipv4", "set", "interface", - t.Device, - fmt.Sprintf("mtu=%d", t.MTU), - ).Run() - if err != nil { - return fmt.Errorf("failed to run 'netsh' to set MTU: %s", err) - } - - t.f, err = net.InterfaceByName(t.Device) - if err != nil { - return fmt.Errorf("failed to find interface named %s: %v", t.Device, err) - } - - err = t.addRoutes(false) - if err != nil { - return err - } - - return nil -} - -func (t *waterTun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) - if err != nil { - return err - } - - if !initial && !change { - return nil - } - - routeTree, err := makeRouteTree(t.l, routes, false) - if err != nil { - return err - } - - // Teach nebula how to handle the routes before establishing them in the system table - oldRoutes := t.Routes.Swap(&routes) - t.routeTree.Store(routeTree) - - if !initial { - // Remove first, if the system removes a wanted route hopefully it will be re-added next - t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) - - // Ensure any routes we actually want are installed - err = t.addRoutes(true) - if err != nil { - // Catch any stray logs - util.LogWithContextIfNeeded("Failed to set routes", err, t.l) - } else { - for _, r := range findRemovedRoutes(routes, *oldRoutes) { - t.l.WithField("route", r).Info("Removed route") - } - } - } - - return nil -} - -func (t *waterTun) addRoutes(logErrors bool) error { - // Path routes - routes := *t.Routes.Load() - for _, r := range routes { - if r.Via == nil || !r.Install { - // We don't allow route MTUs so only install routes with a via - continue - } - - err := exec.Command( - "C:\\Windows\\System32\\route.exe", "add", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(t.f.Index), "METRIC", strconv.Itoa(r.Metric), - ).Run() - - if err != nil { - retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err) - if logErrors { - retErr.Log(t.l) - } else { - return retErr - } - } else { - t.l.WithField("route", r).Info("Added route") - } - } - - return nil -} - -func (t *waterTun) removeRoutes(routes []Route) { - for _, r := range routes { - if !r.Install { - continue - } - - err := exec.Command( - "C:\\Windows\\System32\\route.exe", "delete", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(t.f.Index), "METRIC", strconv.Itoa(r.Metric), - ).Run() - if err != nil { - t.l.WithError(err).WithField("route", r).Error("Failed to remove route") - } else { - t.l.WithField("route", r).Info("Removed route") - } - } -} - -func (t *waterTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.Load().MostSpecificContains(ip) - return r -} - -func (t *waterTun) Cidr() *net.IPNet { - return t.cidr -} - -func (t *waterTun) Name() string { - return t.Device -} - -func (t *waterTun) Close() error { - if t.Interface == nil { - return nil - } - - return t.Interface.Close() -} - -func (t *waterTun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - return nil, fmt.Errorf("TODO: multiqueue not implemented for windows") -} diff --git a/overlay/tun_windows.go b/overlay/tun_windows.go index f85ee9ce..cf01615f 100644 --- a/overlay/tun_windows.go +++ b/overlay/tun_windows.go @@ -4,41 +4,318 @@ package overlay import ( + "crypto" "fmt" - "net" + "io" + "log/slog" + "net/netip" "os" "path/filepath" "runtime" + "sync/atomic" "syscall" + "unsafe" - "github.com/sirupsen/logrus" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/routing" + "github.com/slackhq/nebula/util" + "github.com/slackhq/nebula/wintun" + "golang.org/x/sys/windows" + "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" ) -func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (Device, error) { +type closer interface { + Close() +} + +const tunGUIDLabel = "Fixed Nebula Windows GUID v1" + +type winTun struct { + Device string + vpnNetworks []netip.Prefix + MTU int + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[bart.Table[routing.Gateways]] + guid windows.GUID + networkCategory networkCategory + setCategory bool + bypassWDF bool + wdfBypass closer + l *slog.Logger + + tun *wintun.NativeTun +} + +func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (Device, error) { return nil, fmt.Errorf("newTunFromFd not supported in Windows") } -func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, multiqueue bool) (Device, error) { - useWintun := true - if err := checkWinTunExists(); err != nil { - l.WithError(err).Warn("Check Wintun driver failed, fallback to wintap driver") - useWintun = false +func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*winTun, error) { + err := checkWinTunExists() + if err != nil { + return nil, fmt.Errorf("can not load the wintun driver: %w", err) + } + + deviceName := c.GetString("tun.dev", "") + guid, err := generateGUIDByDeviceName(deviceName) + if err != nil { + return nil, fmt.Errorf("generate GUID failed: %w", err) + } + + cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private")) + if err != nil { + return nil, err + } + + t := &winTun{ + Device: deviceName, + vpnNetworks: vpnNetworks, + MTU: c.GetInt("tun.mtu", DefaultMTU), + guid: *guid, + networkCategory: cat, + setCategory: setCat, + bypassWDF: c.GetBool("tun.windows_bypass_wdf", true), + l: l, } - if useWintun { - device, err := newWinTun(c, l, cidr, multiqueue) + err = t.reload(c, true) + if err != nil { + return nil, err + } + + var tunDevice wintun.Device + tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) + if err != nil { + // Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device. + // Trying a second time resolves the issue. + l.Debug("Failed to create wintun device, retrying", "error", err) + tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) if err != nil { - return nil, fmt.Errorf("create Wintun interface failed, %w", err) + return nil, &NameError{ + Name: deviceName, + Underlying: fmt.Errorf("create TUN device failed: %w", err), + } } - return device, nil } + t.tun = tunDevice.(*wintun.NativeTun) + + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) - device, err := newWaterTun(c, l, cidr, multiqueue) + return t, nil +} + +func (t *winTun) reload(c *config.C, initial bool) error { + change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) if err != nil { - return nil, fmt.Errorf("create wintap driver failed, %w", err) + return err } - return device, nil + + if !initial && !change { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, false) + if err != nil { + return err + } + + // Teach nebula how to handle the routes before establishing them in the system table + oldRoutes := t.Routes.Swap(&routes) + t.routeTree.Store(routeTree) + + if !initial { + // Remove first, if the system removes a wanted route hopefully it will be re-added next + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + if err != nil { + util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) + } + + // Ensure any routes we actually want are installed + err = t.addRoutes(true) + if err != nil { + // Catch any stray logs + util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + } + } + + return nil +} + +func (t *winTun) Activate() error { + luid := winipcfg.LUID(t.tun.LUID()) + + err := luid.SetIPAddresses(t.vpnNetworks) + if err != nil { + return fmt.Errorf("failed to set address: %w", err) + } + + err = t.addRoutes(false) + if err != nil { + return err + } + + if t.setCategory { + // The wintun adapter takes a moment to register with the Network List + // Manager, so we apply the category in the background and retry until + // it shows up. + go applyNetworkCategory(t.l, t.guid, t.networkCategory) + } + + if t.bypassWDF { + t.wdfBypass = installInterfaceBypass(t.l, uint64(t.tun.LUID())) + } + + return nil +} + +func (t *winTun) addRoutes(logErrors bool) error { + luid := winipcfg.LUID(t.tun.LUID()) + routes := *t.Routes.Load() + foundDefault4 := false + + for _, r := range routes { + if len(r.Via) == 0 || !r.Install { + // We don't allow route MTUs so only install routes with a via + continue + } + + // Add our unsafe route as an on-link route to the nebula tun device. + err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric)) + if err != nil { + retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) + if logErrors { + retErr.Log(t.l) + continue + } else { + return retErr + } + } else { + t.l.Info("Added route", "route", r) + } + + if !foundDefault4 { + if r.Cidr.Bits() == 0 && r.Cidr.Addr().BitLen() == 32 { + foundDefault4 = true + } + } + } + + ipif, err := luid.IPInterface(windows.AF_INET) + if err != nil { + return fmt.Errorf("failed to get ip interface: %w", err) + } + + ipif.NLMTU = uint32(t.MTU) + if foundDefault4 { + ipif.UseAutomaticMetric = false + ipif.Metric = 0 + } + + if err := ipif.Set(); err != nil { + return fmt.Errorf("failed to set ip interface: %w", err) + } + return nil +} + +func (t *winTun) removeRoutes(routes []Route) error { + luid := winipcfg.LUID(t.tun.LUID()) + + for _, r := range routes { + if !r.Install { + continue + } + + // See comment on luid.AddRoute + err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr)) + if err != nil { + t.l.Error("Failed to remove route", "error", err, "route", r) + } else { + t.l.Info("Removed route", "route", r) + } + } + return nil +} + +func (t *winTun) RoutesFor(ip netip.Addr) routing.Gateways { + r, _ := t.routeTree.Load().Lookup(ip) + return r +} + +func (t *winTun) Networks() []netip.Prefix { + return t.vpnNetworks +} + +func (t *winTun) Name() string { + return t.Device +} + +func (t *winTun) Read(b []byte) (int, error) { + return t.tun.Read(b, 0) +} + +func (t *winTun) Write(b []byte) (int, error) { + return t.tun.Write(b, 0) +} + +func (t *winTun) SupportsMultiqueue() bool { + return false +} + +func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) { + return nil, fmt.Errorf("TODO: multiqueue not implemented for windows") +} + +func (t *winTun) Close() error { + // It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes, + // so to be certain, just remove everything before destroying. + luid := winipcfg.LUID(t.tun.LUID()) + _ = luid.FlushRoutes(windows.AF_INET) + _ = luid.FlushIPAddresses(windows.AF_INET) + + _ = luid.FlushRoutes(windows.AF_INET6) + _ = luid.FlushIPAddresses(windows.AF_INET6) + + _ = luid.FlushDNS(windows.AF_INET) + _ = luid.FlushDNS(windows.AF_INET6) + + if t.wdfBypass != nil { + t.wdfBypass.Close() + t.wdfBypass = nil + } + + return t.tun.Close() +} + +func unspecifiedNextHop(p netip.Prefix) netip.Addr { + if p.Addr().Is4() { + return netip.IPv4Unspecified() + } + return netip.IPv6Unspecified() +} + +func generateGUIDByDeviceName(name string) (*windows.GUID, error) { + // GUID is 128 bit + hash := crypto.MD5.New() + + _, err := hash.Write([]byte(tunGUIDLabel)) + if err != nil { + return nil, err + } + + _, err = hash.Write([]byte(name)) + if err != nil { + return nil, err + } + + sum := hash.Sum(nil) + + return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil } func checkWinTunExists() error { diff --git a/overlay/tun_wintun_windows.go b/overlay/tun_wintun_windows.go deleted file mode 100644 index 197e3a71..00000000 --- a/overlay/tun_wintun_windows.go +++ /dev/null @@ -1,278 +0,0 @@ -package overlay - -import ( - "crypto" - "fmt" - "io" - "net" - "net/netip" - "sync/atomic" - "unsafe" - - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/cidr" - "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" - "github.com/slackhq/nebula/util" - "github.com/slackhq/nebula/wintun" - "golang.org/x/sys/windows" - "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" -) - -const tunGUIDLabel = "Fixed Nebula Windows GUID v1" - -type winTun struct { - Device string - cidr *net.IPNet - prefix netip.Prefix - MTU int - Routes atomic.Pointer[[]Route] - routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] - l *logrus.Logger - - tun *wintun.NativeTun -} - -func generateGUIDByDeviceName(name string) (*windows.GUID, error) { - // GUID is 128 bit - hash := crypto.MD5.New() - - _, err := hash.Write([]byte(tunGUIDLabel)) - if err != nil { - return nil, err - } - - _, err = hash.Write([]byte(name)) - if err != nil { - return nil, err - } - - sum := hash.Sum(nil) - - return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil -} - -func newWinTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*winTun, error) { - deviceName := c.GetString("tun.dev", "") - guid, err := generateGUIDByDeviceName(deviceName) - if err != nil { - return nil, fmt.Errorf("generate GUID failed: %w", err) - } - - prefix, err := iputil.ToNetIpPrefix(*cidr) - if err != nil { - return nil, err - } - - t := &winTun{ - Device: deviceName, - cidr: cidr, - prefix: prefix, - MTU: c.GetInt("tun.mtu", DefaultMTU), - l: l, - } - - err = t.reload(c, true) - if err != nil { - return nil, err - } - - var tunDevice wintun.Device - tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) - if err != nil { - // Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device. - // Trying a second time resolves the issue. - l.WithError(err).Debug("Failed to create wintun device, retrying") - tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) - if err != nil { - return nil, fmt.Errorf("create TUN device failed: %w", err) - } - } - t.tun = tunDevice.(*wintun.NativeTun) - - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) - if err != nil { - util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) - } - }) - - return t, nil -} - -func (t *winTun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) - if err != nil { - return err - } - - if !initial && !change { - return nil - } - - routeTree, err := makeRouteTree(t.l, routes, false) - if err != nil { - return err - } - - // Teach nebula how to handle the routes before establishing them in the system table - oldRoutes := t.Routes.Swap(&routes) - t.routeTree.Store(routeTree) - - if !initial { - // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) - if err != nil { - util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) - } - - // Ensure any routes we actually want are installed - err = t.addRoutes(true) - if err != nil { - // Catch any stray logs - util.LogWithContextIfNeeded("Failed to add routes", err, t.l) - } - } - - return nil -} - -func (t *winTun) Activate() error { - luid := winipcfg.LUID(t.tun.LUID()) - - err := luid.SetIPAddresses([]netip.Prefix{t.prefix}) - if err != nil { - return fmt.Errorf("failed to set address: %w", err) - } - - err = t.addRoutes(false) - if err != nil { - return err - } - - return nil -} - -func (t *winTun) addRoutes(logErrors bool) error { - luid := winipcfg.LUID(t.tun.LUID()) - routes := *t.Routes.Load() - foundDefault4 := false - - for _, r := range routes { - if r.Via == nil || !r.Install { - // We don't allow route MTUs so only install routes with a via - continue - } - - prefix, err := iputil.ToNetIpPrefix(*r.Cidr) - if err != nil { - retErr := util.NewContextualError("Failed to parse cidr to netip prefix, ignoring route", map[string]interface{}{"route": r}, err) - if logErrors { - retErr.Log(t.l) - continue - } else { - return retErr - } - } - - // Add our unsafe route - err = luid.AddRoute(prefix, r.Via.ToNetIpAddr(), uint32(r.Metric)) - if err != nil { - retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err) - if logErrors { - retErr.Log(t.l) - continue - } else { - return retErr - } - } else { - t.l.WithField("route", r).Info("Added route") - } - - if !foundDefault4 { - if ones, bits := r.Cidr.Mask.Size(); ones == 0 && bits != 0 { - foundDefault4 = true - } - } - } - - ipif, err := luid.IPInterface(windows.AF_INET) - if err != nil { - return fmt.Errorf("failed to get ip interface: %w", err) - } - - ipif.NLMTU = uint32(t.MTU) - if foundDefault4 { - ipif.UseAutomaticMetric = false - ipif.Metric = 0 - } - - if err := ipif.Set(); err != nil { - return fmt.Errorf("failed to set ip interface: %w", err) - } - return nil -} - -func (t *winTun) removeRoutes(routes []Route) error { - luid := winipcfg.LUID(t.tun.LUID()) - - for _, r := range routes { - if !r.Install { - continue - } - - prefix, err := iputil.ToNetIpPrefix(*r.Cidr) - if err != nil { - t.l.WithError(err).WithField("route", r).Info("Failed to convert cidr to netip prefix") - continue - } - - err = luid.DeleteRoute(prefix, r.Via.ToNetIpAddr()) - if err != nil { - t.l.WithError(err).WithField("route", r).Error("Failed to remove route") - } else { - t.l.WithField("route", r).Info("Removed route") - } - } - return nil -} - -func (t *winTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.Load().MostSpecificContains(ip) - return r -} - -func (t *winTun) Cidr() *net.IPNet { - return t.cidr -} - -func (t *winTun) Name() string { - return t.Device -} - -func (t *winTun) Read(b []byte) (int, error) { - return t.tun.Read(b, 0) -} - -func (t *winTun) Write(b []byte) (int, error) { - return t.tun.Write(b, 0) -} - -func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - return nil, fmt.Errorf("TODO: multiqueue not implemented for windows") -} - -func (t *winTun) Close() error { - // It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes, - // so to be certain, just remove everything before destroying. - luid := winipcfg.LUID(t.tun.LUID()) - _ = luid.FlushRoutes(windows.AF_INET) - _ = luid.FlushIPAddresses(windows.AF_INET) - /* We don't support IPV6 yet - _ = luid.FlushRoutes(windows.AF_INET6) - _ = luid.FlushIPAddresses(windows.AF_INET6) - */ - _ = luid.FlushDNS(windows.AF_INET) - - return t.tun.Close() -} diff --git a/overlay/user.go b/overlay/user.go index 9d819ae9..e5f27f37 100644 --- a/overlay/user.go +++ b/overlay/user.go @@ -2,23 +2,23 @@ package overlay import ( "io" - "net" + "log/slog" + "net/netip" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/routing" ) -func NewUserDeviceFromConfig(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error) { - return NewUserDevice(tunCidr) +func NewUserDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { + return NewUserDevice(vpnNetworks) } -func NewUserDevice(tunCidr *net.IPNet) (Device, error) { +func NewUserDevice(vpnNetworks []netip.Prefix) (Device, error) { // these pipes guarantee each write/read will match 1:1 or, ow := io.Pipe() ir, iw := io.Pipe() return &UserDevice{ - tunCidr: tunCidr, + vpnNetworks: vpnNetworks, outboundReader: or, outboundWriter: ow, inboundReader: ir, @@ -27,7 +27,7 @@ func NewUserDevice(tunCidr *net.IPNet) (Device, error) { } type UserDevice struct { - tunCidr *net.IPNet + vpnNetworks []netip.Prefix outboundReader *io.PipeReader outboundWriter *io.PipeWriter @@ -39,9 +39,17 @@ type UserDevice struct { func (d *UserDevice) Activate() error { return nil } -func (d *UserDevice) Cidr() *net.IPNet { return d.tunCidr } -func (d *UserDevice) Name() string { return "faketun0" } -func (d *UserDevice) RouteFor(ip iputil.VpnIp) iputil.VpnIp { return ip } + +func (d *UserDevice) Networks() []netip.Prefix { return d.vpnNetworks } +func (d *UserDevice) Name() string { return "faketun0" } +func (d *UserDevice) RoutesFor(ip netip.Addr) routing.Gateways { + return routing.Gateways{routing.NewGateway(ip, 1)} +} + +func (d *UserDevice) SupportsMultiqueue() bool { + return true +} + func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) { return d, nil } diff --git a/pkclient/pkclient.go b/pkclient/pkclient.go new file mode 100644 index 00000000..7061de6a --- /dev/null +++ b/pkclient/pkclient.go @@ -0,0 +1,87 @@ +package pkclient + +import ( + "crypto/ecdsa" + "crypto/x509" + "fmt" + "io" + "strconv" + + "github.com/stefanberger/go-pkcs11uri" +) + +type Client interface { + io.Closer + GetPubKey() ([]byte, error) + DeriveNoise(peerPubKey []byte) ([]byte, error) + Test() error +} + +const NoiseKeySize = 32 + +func FromUrl(pkurl string) (*PKClient, error) { + uri := pkcs11uri.New() + uri.SetAllowAnyModule(true) //todo + err := uri.Parse(pkurl) + if err != nil { + return nil, err + } + + module, err := uri.GetModule() + if err != nil { + return nil, err + } + + slotid := 0 + slot, ok := uri.GetPathAttribute("slot-id", false) + if !ok { + slotid = 0 + } else { + slotid, err = strconv.Atoi(slot) + if err != nil { + return nil, err + } + } + + pin, _ := uri.GetPIN() + id, _ := uri.GetPathAttribute("id", false) + label, _ := uri.GetPathAttribute("object", false) + + return New(module, uint(slotid), pin, id, label) +} + +func ecKeyToArray(key *ecdsa.PublicKey) []byte { + x := make([]byte, 32) + y := make([]byte, 32) + key.X.FillBytes(x) + key.Y.FillBytes(y) + return append([]byte{0x04}, append(x, y...)...) +} + +func formatPubkeyFromPublicKeyInfoAttr(d []byte) ([]byte, error) { + e, err := x509.ParsePKIXPublicKey(d) + if err != nil { + return nil, err + } + switch t := e.(type) { + case *ecdsa.PublicKey: + return ecKeyToArray(e.(*ecdsa.PublicKey)), nil + default: + return nil, fmt.Errorf("unknown public key type: %T", t) + } +} + +func (c *PKClient) Test() error { + pub, err := c.GetPubKey() + if err != nil { + return fmt.Errorf("failed to get public key: %w", err) + } + out, err := c.DeriveNoise(pub) //do an ECDH with ourselves as a quick test + if err != nil { + return err + } + if len(out) != NoiseKeySize { + return fmt.Errorf("got a key of %d bytes, expected %d", len(out), NoiseKeySize) + } + return nil +} diff --git a/pkclient/pkclient_cgo.go b/pkclient/pkclient_cgo.go new file mode 100644 index 00000000..e6286093 --- /dev/null +++ b/pkclient/pkclient_cgo.go @@ -0,0 +1,230 @@ +//go:build cgo && pkcs11 + +package pkclient + +import ( + "encoding/asn1" + "errors" + "fmt" + "log" + "math/big" + + "github.com/miekg/pkcs11" + "github.com/miekg/pkcs11/p11" +) + +type PKClient struct { + module p11.Module + session p11.Session + id []byte + label []byte + privKeyObj p11.Object + pubKeyObj p11.Object +} + +type ecdsaSignature struct { + R, S *big.Int +} + +// New tries to open a session with the HSM, select the slot and login to it +func New(hsmPath string, slotId uint, pin string, id string, label string) (*PKClient, error) { + module, err := p11.OpenModule(hsmPath) + if err != nil { + return nil, fmt.Errorf("failed to load module library: %s", hsmPath) + } + + slots, err := module.Slots() + if err != nil { + module.Destroy() + return nil, err + } + + // Try to open a session on the slot + slotIdx := 0 + for i, slot := range slots { + if slot.ID() == slotId { + slotIdx = i + break + } + } + + client := &PKClient{ + module: module, + id: []byte(id), + label: []byte(label), + } + + client.session, err = slots[slotIdx].OpenWriteSession() + if err != nil { + module.Destroy() + return nil, fmt.Errorf("failed to open session on slot %d", slotId) + } + + if len(pin) != 0 { + err = client.session.Login(pin) + if err != nil { + // ignore "already logged in" + if !errors.Is(err, pkcs11.Error(256)) { + _ = client.session.Close() + return nil, fmt.Errorf("unable to login. error: %w", err) + } + } + } + + // Make sure the hsm has a private key for deriving + client.privKeyObj, err = client.findDeriveKey(client.id, client.label, true) + if err != nil { + _ = client.Close() //log out, close session, destroy module + return nil, fmt.Errorf("failed to find private key for deriving: %w", err) + } + + return client, nil +} + +// Close cleans up properly and logs out +func (c *PKClient) Close() error { + var err error = nil + if c.session != nil { + _ = c.session.Logout() //if logout fails, we still want to close + err = c.session.Close() + } + + c.module.Destroy() + return err +} + +// Try to find a suitable key on the hsm for key derivation +// parameter GET_PUB_KEY sets the search pattern for a public or private key +func (c *PKClient) findDeriveKey(id []byte, label []byte, private bool) (key p11.Object, err error) { + keyClass := pkcs11.CKO_PRIVATE_KEY + if !private { + keyClass = pkcs11.CKO_PUBLIC_KEY + } + keyAttrs := []*pkcs11.Attribute{ + //todo, not all HSMs seem to report this, even if its true: pkcs11.NewAttribute(pkcs11.CKA_DERIVE, true), + pkcs11.NewAttribute(pkcs11.CKA_CLASS, keyClass), + } + + if id != nil && len(id) != 0 { + keyAttrs = append(keyAttrs, pkcs11.NewAttribute(pkcs11.CKA_ID, id)) + } + if label != nil && len(label) != 0 { + keyAttrs = append(keyAttrs, pkcs11.NewAttribute(pkcs11.CKA_LABEL, label)) + } + + return c.session.FindObject(keyAttrs) +} + +func (c *PKClient) listDeriveKeys(id []byte, label []byte, private bool) { + keyClass := pkcs11.CKO_PRIVATE_KEY + if !private { + keyClass = pkcs11.CKO_PUBLIC_KEY + } + keyAttrs := []*pkcs11.Attribute{ + pkcs11.NewAttribute(pkcs11.CKA_CLASS, keyClass), + } + + if id != nil && len(id) != 0 { + keyAttrs = append(keyAttrs, pkcs11.NewAttribute(pkcs11.CKA_ID, id)) + } + if label != nil && len(label) != 0 { + keyAttrs = append(keyAttrs, pkcs11.NewAttribute(pkcs11.CKA_LABEL, label)) + } + + objects, err := c.session.FindObjects(keyAttrs) + if err != nil { + return + } + + for _, obj := range objects { + l, err := obj.Label() + log.Printf("%s, %v", l, err) + a, err := obj.Attribute(pkcs11.CKA_DERIVE) + log.Printf("DERIVE: %s %v, %v", l, a, err) + } +} + +// SignASN1 signs some data. Returns the ASN.1 encoded signature. +func (c *PKClient) SignASN1(data []byte) ([]byte, error) { + mech := pkcs11.NewMechanism(pkcs11.CKM_ECDSA_SHA256, nil) + sk := p11.PrivateKey(c.privKeyObj) + rawSig, err := sk.Sign(*mech, data) + if err != nil { + return nil, err + } + + // PKCS #11 Mechanisms v2.30: + // "The signature octets correspond to the concatenation of the ECDSA values r and s, + // both represented as an octet string of equal length of at most nLen with the most + // significant byte first. If r and s have different octet length, the shorter of both + // must be padded with leading zero octets such that both have the same octet length. + // Loosely spoken, the first half of the signature is r and the second half is s." + r := new(big.Int).SetBytes(rawSig[:len(rawSig)/2]) + s := new(big.Int).SetBytes(rawSig[len(rawSig)/2:]) + return asn1.Marshal(ecdsaSignature{r, s}) +} + +// DeriveNoise derives a shared secret using the input public key against the private key that was found during setup. +// Returns a fixed 32 byte array. +func (c *PKClient) DeriveNoise(peerPubKey []byte) ([]byte, error) { + // Before we call derive, we need to have an array of attributes which specify the type of + // key to be returned, in our case, it's the shared secret key, produced via deriving + // This template pulled from OpenSC pkclient-tool.c line 4038 + attrTemplate := []*pkcs11.Attribute{ + pkcs11.NewAttribute(pkcs11.CKA_TOKEN, false), + pkcs11.NewAttribute(pkcs11.CKA_CLASS, pkcs11.CKO_SECRET_KEY), + pkcs11.NewAttribute(pkcs11.CKA_KEY_TYPE, pkcs11.CKK_GENERIC_SECRET), + pkcs11.NewAttribute(pkcs11.CKA_SENSITIVE, false), + pkcs11.NewAttribute(pkcs11.CKA_EXTRACTABLE, true), + pkcs11.NewAttribute(pkcs11.CKA_ENCRYPT, true), + pkcs11.NewAttribute(pkcs11.CKA_DECRYPT, true), + pkcs11.NewAttribute(pkcs11.CKA_WRAP, true), + pkcs11.NewAttribute(pkcs11.CKA_UNWRAP, true), + pkcs11.NewAttribute(pkcs11.CKA_VALUE_LEN, NoiseKeySize), + } + + // Set up the parameters which include the peer's public key + ecdhParams := pkcs11.NewECDH1DeriveParams(pkcs11.CKD_NULL, nil, peerPubKey) + mech := pkcs11.NewMechanism(pkcs11.CKM_ECDH1_DERIVE, ecdhParams) + sk := p11.PrivateKey(c.privKeyObj) + + tmpKey, err := sk.Derive(*mech, attrTemplate) + if err != nil { + return nil, err + } + if tmpKey == nil || len(tmpKey) == 0 { + return nil, fmt.Errorf("got an empty secret key") + } + secret := make([]byte, NoiseKeySize) + copy(secret[:], tmpKey[:NoiseKeySize]) + return secret, nil +} + +func (c *PKClient) GetPubKey() ([]byte, error) { + d, err := c.privKeyObj.Attribute(pkcs11.CKA_PUBLIC_KEY_INFO) + if err != nil { + return nil, err + } + if d != nil && len(d) > 0 { + return formatPubkeyFromPublicKeyInfoAttr(d) + } + c.pubKeyObj, err = c.findDeriveKey(c.id, c.label, false) + if err != nil { + return nil, fmt.Errorf("pkcs11 module gave us a nil CKA_PUBLIC_KEY_INFO, and looking up the public key also failed: %w", err) + } + d, err = c.pubKeyObj.Attribute(pkcs11.CKA_EC_POINT) + if err != nil { + return nil, fmt.Errorf("pkcs11 module gave us a nil CKA_PUBLIC_KEY_INFO, and reading CKA_EC_POINT also failed: %w", err) + } + if d == nil || len(d) < 1 { + return nil, fmt.Errorf("pkcs11 module gave us a nil or empty CKA_EC_POINT") + } + switch len(d) { + case 65: //length of 0x04 + len(X) + len(Y) + return d, nil + case 67: //as above, DER-encoded IIRC? + return d[2:], nil + default: + return nil, fmt.Errorf("unknown public key length: %d", len(d)) + } +} diff --git a/pkclient/pkclient_stub.go b/pkclient/pkclient_stub.go new file mode 100644 index 00000000..36b0fc94 --- /dev/null +++ b/pkclient/pkclient_stub.go @@ -0,0 +1,30 @@ +//go:build !cgo || !pkcs11 + +package pkclient + +import "errors" + +type PKClient struct { +} + +var notImplemented = errors.New("not implemented") + +func New(hsmPath string, slotId uint, pin string, id string, label string) (*PKClient, error) { + return nil, notImplemented +} + +func (c *PKClient) Close() error { + return nil +} + +func (c *PKClient) SignASN1(data []byte) ([]byte, error) { + return nil, notImplemented +} + +func (c *PKClient) DeriveNoise(_ []byte) ([]byte, error) { + return nil, notImplemented +} + +func (c *PKClient) GetPubKey() ([]byte, error) { + return nil, notImplemented +} diff --git a/pki.go b/pki.go index 91478ce5..1bef5106 100644 --- a/pki.go +++ b/pki.go @@ -1,34 +1,55 @@ package nebula import ( + "encoding/binary" + "encoding/json" "errors" "fmt" + "io" + "log/slog" + "net" + "net/netip" "os" + "slices" "strings" "sync/atomic" "time" - "github.com/sirupsen/logrus" + "github.com/flynn/noise" + "github.com/gaissmai/bart" "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/handshake" + "github.com/slackhq/nebula/noiseutil" "github.com/slackhq/nebula/util" ) type PKI struct { cs atomic.Pointer[CertState] - caPool atomic.Pointer[cert.NebulaCAPool] - l *logrus.Logger + caPool atomic.Pointer[cert.CAPool] + l *slog.Logger } type CertState struct { - Certificate *cert.NebulaCertificate - RawCertificate []byte - RawCertificateNoKey []byte - PublicKey []byte - PrivateKey []byte + v1Cert cert.Certificate + v1Credential *handshake.Credential + + v2Cert cert.Certificate + v2Credential *handshake.Credential + + initiatingVersion cert.Version + privateKey []byte + pkcs11Backed bool + cipher string + + myVpnNetworks []netip.Prefix + myVpnNetworksTable *bart.Lite + myVpnAddrs []netip.Addr + myVpnAddrsTable *bart.Lite + myVpnBroadcastAddrsTable *bart.Lite } -func NewPKIFromConfig(l *logrus.Logger, c *config.C) (*PKI, error) { +func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) { pki := &PKI{l: l} err := pki.reload(c, true) if err != nil { @@ -45,16 +66,16 @@ func NewPKIFromConfig(l *logrus.Logger, c *config.C) (*PKI, error) { return pki, nil } -func (p *PKI) GetCertState() *CertState { - return p.cs.Load() +func (p *PKI) GetCAPool() *cert.CAPool { + return p.caPool.Load() } -func (p *PKI) GetCAPool() *cert.NebulaCAPool { - return p.caPool.Load() +func (p *PKI) getCertState() *CertState { + return p.cs.Load() } func (p *PKI) reload(c *config.C, initial bool) error { - err := p.reloadCert(c, initial) + err := p.reloadCerts(c, initial) if err != nil { if initial { return err @@ -73,31 +94,101 @@ func (p *PKI) reload(c *config.C, initial bool) error { return nil } -func (p *PKI) reloadCert(c *config.C, initial bool) *util.ContextualError { - cs, err := newCertStateFromConfig(c) +func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { + var cipher string + var currentState *CertState + if initial { + cipher = c.GetString("cipher", "aes") + switch cipher { + case "aes", "chachapoly": + // Each post-handshake CipherState in noiseutil hardcodes its own + // nonce endianness now, so there's nothing to set up here. + default: + return util.NewContextualError( + "unknown cipher", + m{"cipher": cipher}, + nil, + ) + } + } else { + // Cipher cant be hot swapped so just leave it at what it was before + currentState = p.cs.Load() + cipher = currentState.cipher + } + + newState, err := newCertStateFromConfig(c, cipher) if err != nil { return util.NewContextualError("Could not load client cert", nil, err) } - if !initial { - // did IP in cert change? if so, don't set - currentCert := p.cs.Load().Certificate - oldIPs := currentCert.Details.Ips - newIPs := cs.Certificate.Details.Ips - if len(oldIPs) > 0 && len(newIPs) > 0 && oldIPs[0].String() != newIPs[0].String() { - return util.NewContextualError( - "IP in new cert was different from old", - m{"new_ip": newIPs[0], "old_ip": oldIPs[0]}, - nil, - ) + if currentState != nil { + if newState.v1Cert != nil { + if currentState.v1Cert == nil { + //adding certs is fine, actually. Networks-in-common confirmed in newCertState(). + } else { + // did IP in cert change? if so, don't set + if !slices.Equal(currentState.v1Cert.Networks(), newState.v1Cert.Networks()) { + return util.NewContextualError( + "Networks in new cert was different from old", + m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1}, + nil, + ) + } + + if currentState.v1Cert.Curve() != newState.v1Cert.Curve() { + return util.NewContextualError( + "Curve in new v1 cert was different from old", + m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1}, + nil, + ) + } + } + } + + if newState.v2Cert != nil { + if currentState.v2Cert == nil { + //adding certs is fine, actually + } else { + // did IP in cert change? if so, don't set + if !slices.Equal(currentState.v2Cert.Networks(), newState.v2Cert.Networks()) { + return util.NewContextualError( + "Networks in new cert was different from old", + m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2}, + nil, + ) + } + + if currentState.v2Cert.Curve() != newState.v2Cert.Curve() { + return util.NewContextualError( + "Curve in new cert was different from old", + m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2}, + nil, + ) + } + } + + } else if currentState.v2Cert != nil { + //newState.v1Cert is non-nil bc empty certstates aren't permitted + if newState.v1Cert == nil { + return util.NewContextualError("v1 and v2 certs are nil, this should be impossible", nil, err) + } + //if we're going to v1-only, we need to make sure we didn't orphan any v2-cert vpnaddrs + if !slices.Equal(currentState.v2Cert.Networks(), newState.v1Cert.Networks()) { + return util.NewContextualError( + "Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert", + m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()}, + nil, + ) + } } } - p.cs.Store(cs) + p.cs.Store(newState) + if initial { - p.l.WithField("cert", cs.Certificate).Debug("Client nebula certificate") + p.l.Debug("Client nebula certificate(s)", "cert", newState) } else { - p.l.WithField("cert", cs.Certificate).Info("Client cert refreshed from disk") + p.l.Info("Client certificate(s) refreshed from disk", "cert", newState) } return nil } @@ -109,59 +200,104 @@ func (p *PKI) reloadCAPool(c *config.C) *util.ContextualError { } p.caPool.Store(caPool) - p.l.WithField("fingerprints", caPool.GetFingerprints()).Debug("Trusted CA fingerprints") + p.l.Debug("Trusted CA fingerprints", "fingerprints", caPool.GetFingerprints()) return nil } -func newCertState(certificate *cert.NebulaCertificate, privateKey []byte) (*CertState, error) { - // Marshal the certificate to ensure it is valid - rawCertificate, err := certificate.Marshal() - if err != nil { - return nil, fmt.Errorf("invalid nebula certificate on interface: %s", err) +func (cs *CertState) GetDefaultCertificate() cert.Certificate { + c := cs.getCertificate(cs.initiatingVersion) + if c == nil { + panic("No default certificate found") } + return c +} - publicKey := certificate.Details.PublicKey - cs := &CertState{ - RawCertificate: rawCertificate, - Certificate: certificate, - PrivateKey: privateKey, - PublicKey: publicKey, +// DefaultVersion returns the preferred cert version for initiating handshakes. +func (cs *CertState) DefaultVersion() cert.Version { return cs.initiatingVersion } + +// GetCredential returns the pre-computed handshake credential for the given version, or nil. +func (cs *CertState) GetCredential(v cert.Version) *handshake.Credential { + switch v { + case cert.Version1: + return cs.v1Credential + case cert.Version2: + return cs.v2Credential } + return nil +} - cs.Certificate.Details.PublicKey = nil - rawCertNoKey, err := cs.Certificate.Marshal() - if err != nil { - return nil, fmt.Errorf("error marshalling certificate no key: %s", err) +func (cs *CertState) getCertificate(v cert.Version) cert.Certificate { + switch v { + case cert.Version1: + return cs.v1Cert + case cert.Version2: + return cs.v2Cert } - cs.RawCertificateNoKey = rawCertNoKey - // put public key back - cs.Certificate.Details.PublicKey = cs.PublicKey - return cs, nil + + return nil } -func newCertStateFromConfig(c *config.C) (*CertState, error) { - var pemPrivateKey []byte - var err error +func newCipherSuite(curve cert.Curve, pkcs11backed bool, cipher string) (noise.CipherSuite, error) { + var dhFunc noise.DHFunc + switch curve { + case cert.Curve_CURVE25519: + dhFunc = noise.DH25519 + case cert.Curve_P256: + if pkcs11backed { + dhFunc = noiseutil.DHP256PKCS11 + } else { + dhFunc = noiseutil.DHP256 + } + default: + return nil, fmt.Errorf("unsupported curve: %s", curve) + } - privPathOrPEM := c.GetString("pki.key", "") - if privPathOrPEM == "" { - return nil, errors.New("no pki.key path or PEM data provided") + if cipher == "chachapoly" { + return noise.NewCipherSuite(dhFunc, noise.CipherChaChaPoly, noise.HashSHA256), nil } + return noise.NewCipherSuite(dhFunc, noiseutil.CipherAESGCM, noise.HashSHA256), nil +} - if strings.Contains(privPathOrPEM, "-----BEGIN") { - pemPrivateKey = []byte(privPathOrPEM) - privPathOrPEM = "" +func (cs *CertState) String() string { + b, err := cs.MarshalJSON() + if err != nil { + return fmt.Sprintf("error marshaling certificate state: %v", err) + } + return string(b) +} - } else { - pemPrivateKey, err = os.ReadFile(privPathOrPEM) +func (cs *CertState) MarshalJSON() ([]byte, error) { + msg := []json.RawMessage{} + if cs.v1Cert != nil { + b, err := cs.v1Cert.MarshalJSON() if err != nil { - return nil, fmt.Errorf("unable to read pki.key file %s: %s", privPathOrPEM, err) + return nil, err } + msg = append(msg, b) + } + + if cs.v2Cert != nil { + b, err := cs.v2Cert.MarshalJSON() + if err != nil { + return nil, err + } + msg = append(msg, b) + } + + return json.Marshal(msg) +} + +func newCertStateFromConfig(c *config.C, cipher string) (*CertState, error) { + var err error + + privPathOrPEM := c.GetString("pki.key", "") + if privPathOrPEM == "" { + return nil, errors.New("no pki.key path or PEM data provided") } - rawKey, _, curve, err := cert.UnmarshalPrivateKey(pemPrivateKey) + rawKey, curve, isPkcs11, err := loadPrivateKey(privPathOrPEM) if err != nil { - return nil, fmt.Errorf("error while unmarshaling pki.key %s: %s", privPathOrPEM, err) + return nil, err } var rawCert []byte @@ -182,52 +318,236 @@ func newCertStateFromConfig(c *config.C) (*CertState, error) { } } - nebulaCert, _, err := cert.UnmarshalNebulaCertificateFromPEM(rawCert) - if err != nil { - return nil, fmt.Errorf("error while unmarshaling pki.cert %s: %s", pubPathOrPEM, err) + var crt, v1, v2 cert.Certificate + for { + // Load the certificate + crt, rawCert, err = loadCertificate(rawCert) + if err != nil { + return nil, err + } + + switch crt.Version() { + case cert.Version1: + if v1 != nil { + return nil, fmt.Errorf("v1 certificate already found in pki.cert") + } + v1 = crt + case cert.Version2: + if v2 != nil { + return nil, fmt.Errorf("v2 certificate already found in pki.cert") + } + v2 = crt + default: + return nil, fmt.Errorf("unknown certificate version %v", crt.Version()) + } + + if len(rawCert) == 0 || strings.TrimSpace(string(rawCert)) == "" { + break + } } - if nebulaCert.Expired(time.Now()) { - return nil, fmt.Errorf("nebula certificate for this host is expired") + if v1 == nil && v2 == nil { + return nil, errors.New("no certificates found in pki.cert") } - if len(nebulaCert.Details.Ips) == 0 { - return nil, fmt.Errorf("no IPs encoded in certificate") + useInitiatingVersion := uint32(1) + if v1 == nil { + // The only condition that requires v2 as the default is if only a v2 certificate is present + // We do this to avoid having to configure it specifically in the config file + useInitiatingVersion = 2 } - if err = nebulaCert.VerifyPrivateKey(curve, rawKey); err != nil { - return nil, fmt.Errorf("private key is not a pair with public key in nebula cert") + rawInitiatingVersion := c.GetUint32("pki.initiating_version", useInitiatingVersion) + var initiatingVersion cert.Version + switch rawInitiatingVersion { + case 1: + if v1 == nil { + return nil, fmt.Errorf("can not use pki.initiating_version 1 without a v1 certificate in pki.cert") + } + initiatingVersion = cert.Version1 + case 2: + initiatingVersion = cert.Version2 + default: + return nil, fmt.Errorf("unknown pki.initiating_version: %v", rawInitiatingVersion) } - return newCertState(nebulaCert, rawKey) + return newCertState(initiatingVersion, v1, v2, isPkcs11, curve, rawKey, cipher) } -func loadCAPoolFromConfig(l *logrus.Logger, c *config.C) (*cert.NebulaCAPool, error) { - var rawCA []byte - var err error +func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, privateKeyCurve cert.Curve, privateKey []byte, cipher string) (*CertState, error) { + cs := CertState{ + privateKey: privateKey, + pkcs11Backed: pkcs11backed, + cipher: cipher, + myVpnNetworksTable: new(bart.Lite), + myVpnAddrsTable: new(bart.Lite), + myVpnBroadcastAddrsTable: new(bart.Lite), + } + + if v1 != nil && v2 != nil { + if !slices.Equal(v1.PublicKey(), v2.PublicKey()) { + return nil, util.NewContextualError("v1 and v2 public keys are not the same, ignoring", nil, nil) + } + + if v1.Curve() != v2.Curve() { + return nil, util.NewContextualError("v1 and v2 curve are not the same, ignoring", nil, nil) + } + + if v1.Networks()[0] != v2.Networks()[0] { + return nil, util.NewContextualError("v1 and v2 networks are not the same", nil, nil) + } + + cs.initiatingVersion = dv + } + + if v1 != nil { + if pkcs11backed { + //NOTE: We do not currently have a method to verify a public private key pair when the private key is in an hsm + } else { + if err := v1.VerifyPrivateKey(privateKeyCurve, privateKey); err != nil { + return nil, fmt.Errorf("private key is not a pair with public key in nebula cert") + } + } + + v1hs, err := v1.MarshalForHandshakes() + if err != nil { + return nil, fmt.Errorf("error marshalling v1 certificate for handshake: %w", err) + } + ncs, err := newCipherSuite(v1.Curve(), pkcs11backed, cipher) + if err != nil { + return nil, err + } + cs.v1Cert = v1 + cs.v1Credential = handshake.NewCredential(v1, v1hs, privateKey, ncs) + + if cs.initiatingVersion == 0 { + cs.initiatingVersion = cert.Version1 + } + } + + if v2 != nil { + if pkcs11backed { + //NOTE: We do not currently have a method to verify a public private key pair when the private key is in an hsm + } else { + if err := v2.VerifyPrivateKey(privateKeyCurve, privateKey); err != nil { + return nil, fmt.Errorf("private key is not a pair with public key in nebula cert") + } + } + + v2hs, err := v2.MarshalForHandshakes() + if err != nil { + return nil, fmt.Errorf("error marshalling v2 certificate for handshake: %w", err) + } + ncs, err := newCipherSuite(v2.Curve(), pkcs11backed, cipher) + if err != nil { + return nil, err + } + cs.v2Cert = v2 + cs.v2Credential = handshake.NewCredential(v2, v2hs, privateKey, ncs) + + if cs.initiatingVersion == 0 { + cs.initiatingVersion = cert.Version2 + } + } + + var crt cert.Certificate + crt = cs.getCertificate(cert.Version2) + if crt == nil { + // v2 certificates are a superset, only look at v1 if its all we have + crt = cs.getCertificate(cert.Version1) + } + + for _, network := range crt.Networks() { + cs.myVpnNetworks = append(cs.myVpnNetworks, network) + cs.myVpnNetworksTable.Insert(network) + + cs.myVpnAddrs = append(cs.myVpnAddrs, network.Addr()) + cs.myVpnAddrsTable.Insert(netip.PrefixFrom(network.Addr(), network.Addr().BitLen())) + + if network.Addr().Is4() { + addr := network.Masked().Addr().As4() + mask := net.CIDRMask(network.Bits(), network.Addr().BitLen()) + binary.BigEndian.PutUint32(addr[:], binary.BigEndian.Uint32(addr[:])|^binary.BigEndian.Uint32(mask)) + cs.myVpnBroadcastAddrsTable.Insert(netip.PrefixFrom(netip.AddrFrom4(addr), network.Addr().BitLen())) + } + } + + return &cs, nil +} + +func loadPrivateKey(privPathOrPEM string) (rawKey []byte, curve cert.Curve, isPkcs11 bool, err error) { + var pemPrivateKey []byte + if strings.Contains(privPathOrPEM, "-----BEGIN") { + pemPrivateKey = []byte(privPathOrPEM) + privPathOrPEM = "" + rawKey, _, curve, err = cert.UnmarshalPrivateKeyFromPEM(pemPrivateKey) + if err != nil { + return nil, curve, false, fmt.Errorf("error while unmarshaling pki.key %s: %s", privPathOrPEM, err) + } + } else if strings.HasPrefix(privPathOrPEM, "pkcs11:") { + rawKey = []byte(privPathOrPEM) + return rawKey, cert.Curve_P256, true, nil + } else { + pemPrivateKey, err = os.ReadFile(privPathOrPEM) + if err != nil { + return nil, curve, false, fmt.Errorf("unable to read pki.key file %s: %s", privPathOrPEM, err) + } + rawKey, _, curve, err = cert.UnmarshalPrivateKeyFromPEM(pemPrivateKey) + if err != nil { + return nil, curve, false, fmt.Errorf("error while unmarshaling pki.key %s: %s", privPathOrPEM, err) + } + } + return +} + +func loadCertificate(b []byte) (cert.Certificate, []byte, error) { + c, b, err := cert.UnmarshalCertificateFromPEM(b) + if err != nil { + return nil, b, fmt.Errorf("error while unmarshaling pki.cert: %w", err) + } + + if c.Expired(time.Now()) { + return nil, b, fmt.Errorf("nebula certificate for this host is expired") + } + + if len(c.Networks()) == 0 { + return nil, b, fmt.Errorf("no networks encoded in certificate") + } + + if c.IsCA() { + return nil, b, fmt.Errorf("host certificate is a CA certificate") + } + + return c, b, nil +} + +func loadCAPoolFromConfig(l *slog.Logger, c *config.C) (*cert.CAPool, error) { caPathOrPEM := c.GetString("pki.ca", "") if caPathOrPEM == "" { return nil, errors.New("no pki.ca path or PEM data provided") } - if strings.Contains(caPathOrPEM, "-----BEGIN") { - rawCA = []byte(caPathOrPEM) + var caReader io.ReadCloser + var err error + if strings.Contains(caPathOrPEM, "-----BEGIN") { + caReader = io.NopCloser(strings.NewReader(caPathOrPEM)) } else { - rawCA, err = os.ReadFile(caPathOrPEM) + caReader, err = os.Open(caPathOrPEM) if err != nil { return nil, fmt.Errorf("unable to read pki.ca file %s: %s", caPathOrPEM, err) } } + defer caReader.Close() - caPool, err := cert.NewCAPoolFromBytes(rawCA) + caPool, err := cert.NewCAPoolFromPEMReader(caReader) if errors.Is(err, cert.ErrExpired) { var expired int for _, crt := range caPool.CAs { - if crt.Expired(time.Now()) { + if crt.Certificate.Expired(time.Now()) { expired++ - l.WithField("cert", crt).Warn("expired certificate present in CA pool") + l.Warn("expired certificate present in CA pool", "cert", crt) } } @@ -239,9 +559,13 @@ func loadCAPoolFromConfig(l *logrus.Logger, c *config.C) (*cert.NebulaCAPool, er return nil, fmt.Errorf("error while adding CA certificate to CA trust store: %s", err) } - for _, fp := range c.GetStringSlice("pki.blocklist", []string{}) { - l.WithField("fingerprint", fp).Info("Blocklisting cert") - caPool.BlocklistFingerprint(fp) + bl := c.GetStringSlice("pki.blocklist", []string{}) + if len(bl) > 0 { + for _, fp := range bl { + caPool.BlocklistFingerprint(fp) + } + + l.Info("Blocklisted certificates", "fingerprintCount", len(bl)) } return caPool, nil diff --git a/pki_hup_benchmark_test.go b/pki_hup_benchmark_test.go new file mode 100644 index 00000000..bca23d78 --- /dev/null +++ b/pki_hup_benchmark_test.go @@ -0,0 +1,121 @@ +package nebula + +import ( + "bytes" + "fmt" + "net/netip" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/slackhq/nebula/cert" + cert_test "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/require" +) + +func BenchmarkReloadConfigWithCAs(b *testing.B) { + prevProcs := runtime.GOMAXPROCS(1) + b.Cleanup(func() { runtime.GOMAXPROCS(prevProcs) }) + + for _, size := range []int{100, 250, 500, 1000, 5000} { + b.Run(fmt.Sprintf("%dCAs", size), func(b *testing.B) { + l := test.NewLogger() + dir := b.TempDir() + + ca, caKey, caBundle := buildCABundle(b, size) + caPath, certPath, keyPath := writePKIFiles(b, dir, ca, caKey, caBundle) + + configBody := fmt.Sprintf(`pki: + ca: %s + cert: %s + key: %s +`, caPath, certPath, keyPath) + + configPath := filepath.Join(dir, "config.yml") + require.NoError(b, os.WriteFile(configPath, []byte(configBody), 0o600)) + + c := config.NewC(l) + require.NoError(b, c.Load(dir)) + + _, err := NewPKIFromConfig(test.NewLogger(), c) + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + c.ReloadConfig() + } + }) + } +} + +func buildCABundle(b *testing.B, count int) (cert.Certificate, []byte, []byte) { + b.Helper() + require.GreaterOrEqual(b, count, 1) + + before := time.Now().Add(-24 * time.Hour) + after := time.Now().Add(24 * time.Hour) + + ca, _, caKey, pem := cert_test.NewTestCaCert( + cert.Version2, + cert.Curve_CURVE25519, + before, + after, + nil, + nil, + nil, + ) + + buf := bytes.NewBuffer(pem) + buf.Write([]byte("\n# a comment!\n")) + + for i := 1; i < count; i++ { + _, _, _, extraPEM := cert_test.NewTestCaCert( + cert.Version2, + cert.Curve_CURVE25519, + time.Now(), + time.Now().Add(time.Hour), + nil, + nil, + nil, + ) + buf.Write([]byte("\n# a comment!\n")) + buf.Write(extraPEM) + } + + return ca, caKey, buf.Bytes() +} + +func writePKIFiles(b *testing.B, dir string, ca cert.Certificate, caKey []byte, caBundle []byte) (string, string, string) { + b.Helper() + + networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")} + + _, _, keyPEM, certPEM := cert_test.NewTestCert( + cert.Version2, + cert.Curve_CURVE25519, + ca, + caKey, + "reload-benchmark", + time.Now(), + time.Now().Add(time.Hour), + networks, + nil, + nil, + ) + + caPath := filepath.Join(dir, "ca.pem") + certPath := filepath.Join(dir, "cert.pem") + keyPath := filepath.Join(dir, "key.pem") + + require.NoError(b, os.WriteFile(caPath, caBundle, 0o600)) + require.NoError(b, os.WriteFile(certPath, certPEM, 0o600)) + require.NoError(b, os.WriteFile(keyPath, keyPEM, 0o600)) + + return caPath, certPath, keyPath +} diff --git a/punchy.go b/punchy.go index 2034405a..4bce4392 100644 --- a/punchy.go +++ b/punchy.go @@ -1,24 +1,70 @@ package nebula import ( + "context" + "log/slog" + "net/netip" "sync/atomic" "time" - "github.com/sirupsen/logrus" + "github.com/rcrowley/go-metrics" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/header" + "github.com/slackhq/nebula/udp" ) +// holepunchQueueSize buffers the channel that pending holepunchJobs land on after their delay timer fires. +const holepunchQueueSize = 64 + +// holepunchJob is one scheduled item delivered to the worker goroutine. +// - target valid -> send a UDP punch to target. vpnAddr, if set, is the peer's vpn addr carried for log context. +// - target invalid, vpnAddr valid -> send an encrypted test packet to vpnAddr (a "punchback"). +type holepunchJob struct { + target netip.AddrPort + vpnAddr netip.Addr +} + +// lighthouseChecker is the slice of LightHouse that Punchy actually needs. +// Defined here so Punchy doesn't take a *LightHouse dependency (LightHouse +// already holds a *Punchy, and the bidirectional pointer reference is awkward +// even within the same package). Tests can also substitute a fake. +type lighthouseChecker interface { + IsAnyLighthouseAddr(vpnAddrs []netip.Addr) bool +} + type Punchy struct { punch atomic.Bool respond atomic.Bool delay atomic.Int64 respondDelay atomic.Int64 punchEverything atomic.Bool - l *logrus.Logger + + sched *Scheduler[holepunchJob] + punchConn udp.Conn + metricHolepunchTx metrics.Counter + metricPunchyTx metrics.Counter + + ctx context.Context + ifce EncWriter + hm *HostMap + lh lighthouseChecker + + l *slog.Logger } -func NewPunchyFromConfig(l *logrus.Logger, c *config.C) *Punchy { - p := &Punchy{l: l} +func NewPunchyFromConfig(l *slog.Logger, c *config.C, punchConn udp.Conn) *Punchy { + p := &Punchy{ + l: l, + punchConn: punchConn, + sched: NewScheduler[holepunchJob](holepunchQueueSize), + metricPunchyTx: metrics.GetOrRegisterCounter("messages.tx.punchy", nil), + } + + if c.GetBool("stats.lighthouse_metrics", false) { + p.metricHolepunchTx = metrics.GetOrRegisterCounter("messages.tx.holepunch", nil) + } else { + p.metricHolepunchTx = metrics.NilCounter{} + } p.reload(c, true) c.RegisterReloadCallback(func(c *config.C) { @@ -29,7 +75,7 @@ func NewPunchyFromConfig(l *logrus.Logger, c *config.C) *Punchy { } func (p *Punchy) reload(c *config.C, initial bool) { - if initial { + if initial || c.HasChanged("punchy.punch") || c.HasChanged("punchy") { var yes bool if c.IsSet("punchy.punch") { yes = c.GetBool("punchy.punch", false) @@ -38,16 +84,15 @@ func (p *Punchy) reload(c *config.C, initial bool) { yes = c.GetBool("punchy", false) } - p.punch.Store(yes) - if yes { + old := p.punch.Swap(yes) + switch { + case initial && yes: p.l.Info("punchy enabled") - } else { + case initial: p.l.Info("punchy disabled") + case old != yes: + p.l.Info("punchy.punch changed", "punch", yes) } - - } else if c.HasChanged("punchy.punch") || c.HasChanged("punchy") { - //TODO: it should be relatively easy to support this, just need to be able to cancel the goroutine and boot it up from here - p.l.Warn("Changing punchy.punch with reload is not supported, ignoring.") } if initial || c.HasChanged("punchy.respond") || c.HasChanged("punch_back") { @@ -59,52 +104,132 @@ func (p *Punchy) reload(c *config.C, initial bool) { yes = c.GetBool("punch_back", false) } - p.respond.Store(yes) - - if !initial { - p.l.Infof("punchy.respond changed to %v", p.GetRespond()) + old := p.respond.Swap(yes) + if !initial && old != yes { + p.l.Info("punchy.respond changed", "respond", yes) } } //NOTE: this will not apply to any in progress operations, only the next one if initial || c.HasChanged("punchy.delay") { - p.delay.Store((int64)(c.GetDuration("punchy.delay", time.Second))) - if !initial { - p.l.Infof("punchy.delay changed to %s", p.GetDelay()) + newDelay := int64(c.GetDuration("punchy.delay", time.Second)) + old := p.delay.Swap(newDelay) + if !initial && old != newDelay { + p.l.Info("punchy.delay changed", "delay", time.Duration(newDelay)) } } if initial || c.HasChanged("punchy.target_all_remotes") { - p.punchEverything.Store(c.GetBool("punchy.target_all_remotes", false)) - if !initial { - p.l.WithField("target_all_remotes", p.GetTargetEverything()).Info("punchy.target_all_remotes changed") + yes := c.GetBool("punchy.target_all_remotes", false) + old := p.punchEverything.Swap(yes) + if !initial && old != yes { + p.l.Info("punchy.target_all_remotes changed", "target_all_remotes", yes) } } if initial || c.HasChanged("punchy.respond_delay") { - p.respondDelay.Store((int64)(c.GetDuration("punchy.respond_delay", 5*time.Second))) - if !initial { - p.l.Infof("punchy.respond_delay changed to %s", p.GetRespondDelay()) + newDelay := int64(c.GetDuration("punchy.respond_delay", 5*time.Second)) + old := p.respondDelay.Swap(newDelay) + if !initial && old != newDelay { + p.l.Info("punchy.respond_delay changed", "respond_delay", time.Duration(newDelay)) } } } -func (p *Punchy) GetPunch() bool { - return p.punch.Load() +// Schedule queues a punch packet to target, to be sent after the configured delay. +// vpnAddr is the peer's vpn addr, used for log context when the packet actually fires. +// No-op if target is not a valid AddrPort or if Start has not yet been called. Safe to call from any goroutine. +func (p *Punchy) Schedule(target netip.AddrPort, vpnAddr netip.Addr) { + if !target.IsValid() || p.ctx == nil { + return + } + p.scheduleJob(holepunchJob{target: target, vpnAddr: vpnAddr}, time.Duration(p.delay.Load())) +} + +// ScheduleRespond queues a punchback test packet to vpnAddr after the configured respond delay, +// gated on punchy.respond. No-op when respond is disabled or before Start has been called. +func (p *Punchy) ScheduleRespond(vpnAddr netip.Addr) { + if !p.respond.Load() || p.ctx == nil { + return + } + p.scheduleJob(holepunchJob{vpnAddr: vpnAddr}, time.Duration(p.respondDelay.Load())) } -func (p *Punchy) GetRespond() bool { - return p.respond.Load() +// scheduleJob delegates to the pooled Scheduler. +// The callback observes p.ctx so a job that becomes due after Stop is dropped instead of queued. +func (p *Punchy) scheduleJob(job holepunchJob, delay time.Duration) { + p.sched.Schedule(p.ctx, job, delay) } -func (p *Punchy) GetDelay() time.Duration { - return (time.Duration)(p.delay.Load()) +// SendPunch sends an immediate keepalive punch for an idle hostinfo. +// The configured punchy.target_all_remotes mode picks the targets. Gated on punchy.punch and the lighthouse-skip rule +// (lighthouses don't get keepalive punches because the regular update interval keeps their NAT state warm). +func (p *Punchy) SendPunch(hostinfo *HostInfo) { + if !p.punch.Load() { + return + } + if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) { + return + } + + if p.punchEverything.Load() { + p.sendPunchToAllRemotes(hostinfo) + } else if hr := hostinfo.GetRemote(); hr.IsValid() { + p.metricPunchyTx.Inc(1) + p.punchConn.WriteTo([]byte{1}, hr) + } } -func (p *Punchy) GetRespondDelay() time.Duration { - return (time.Duration)(p.respondDelay.Load()) +// SendPunchToAll punches every known remote for hostinfo, but only when punchy.target_all_remotes is enabled. +// The connection manager calls this during outbound-only traffic: the outbound traffic itself keeps the primary's +// NAT state warm, but non-primary remotes need separate refresh, so we fan out to all of them (the redundant +// primary punch is harmless). Gated on punchy.punch and the lighthouse-skip rule. +func (p *Punchy) SendPunchToAll(hostinfo *HostInfo) { + if !p.punchEverything.Load() { + return + } + if !p.punch.Load() { + return + } + if p.lh.IsAnyLighthouseAddr(hostinfo.vpnAddrs) { + return + } + p.sendPunchToAllRemotes(hostinfo) } -func (p *Punchy) GetTargetEverything() bool { - return p.punchEverything.Load() +func (p *Punchy) sendPunchToAllRemotes(hostinfo *HostInfo) { + hostinfo.remotes.ForEach(p.hm.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) { + p.metricPunchyTx.Inc(1) + p.punchConn.WriteTo([]byte{1}, addr) + }) +} + +// Start wires the runtime dependencies and spawns the scheduler worker. +func (p *Punchy) Start(ctx context.Context, ifce EncWriter, hm *HostMap, lh lighthouseChecker) { + p.ctx = ctx + p.ifce = ifce + p.hm = hm + p.lh = lh + + nb := make([]byte, 12, 12) + out := make([]byte, mtu) + empty := []byte{0} + + go p.sched.Run(ctx, func(job holepunchJob) { + switch { + case job.target.IsValid(): + if p.l.Enabled(context.Background(), slog.LevelDebug) { + p.l.Debug("Punching", "target", job.target, "vpnAddr", job.vpnAddr) + } + p.metricHolepunchTx.Inc(1) + p.punchConn.WriteTo(empty, job.target) + case job.vpnAddr.IsValid(): + // A nebula test packet to the host trying to contact us. + // In the case of a double nat or other difficult scenario, this may help establish a tunnel. + if p.l.Enabled(context.Background(), slog.LevelDebug) { + p.l.Debug("Sending a nebula test packet", "vpnAddr", job.vpnAddr) + } + p.ifce.SendMessageToVpnAddr(header.Test, header.TestRequest, job.vpnAddr, []byte(""), nb, out) + } + }) } diff --git a/punchy_test.go b/punchy_test.go index bedd2b26..e56f3eff 100644 --- a/punchy_test.go +++ b/punchy_test.go @@ -1,12 +1,15 @@ package nebula import ( + "context" + "log/slog" "testing" "time" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNewPunchyFromConfig(t *testing.T) { @@ -14,64 +17,218 @@ func TestNewPunchyFromConfig(t *testing.T) { c := config.NewC(l) // Test defaults - p := NewPunchyFromConfig(l, c) - assert.Equal(t, false, p.GetPunch()) - assert.Equal(t, false, p.GetRespond()) - assert.Equal(t, time.Second, p.GetDelay()) - assert.Equal(t, 5*time.Second, p.GetRespondDelay()) + p := NewPunchyFromConfig(test.NewLogger(), c, nil) + assert.False(t, p.punch.Load()) + assert.False(t, p.respond.Load()) + assert.Equal(t, time.Second, time.Duration(p.delay.Load())) + assert.Equal(t, 5*time.Second, time.Duration(p.respondDelay.Load())) // punchy deprecation c.Settings["punchy"] = true - p = NewPunchyFromConfig(l, c) - assert.Equal(t, true, p.GetPunch()) + p = NewPunchyFromConfig(test.NewLogger(), c, nil) + assert.True(t, p.punch.Load()) // punchy.punch - c.Settings["punchy"] = map[interface{}]interface{}{"punch": true} - p = NewPunchyFromConfig(l, c) - assert.Equal(t, true, p.GetPunch()) + c.Settings["punchy"] = map[string]any{"punch": true} + p = NewPunchyFromConfig(test.NewLogger(), c, nil) + assert.True(t, p.punch.Load()) // punch_back deprecation c.Settings["punch_back"] = true - p = NewPunchyFromConfig(l, c) - assert.Equal(t, true, p.GetRespond()) + p = NewPunchyFromConfig(test.NewLogger(), c, nil) + assert.True(t, p.respond.Load()) // punchy.respond - c.Settings["punchy"] = map[interface{}]interface{}{"respond": true} + c.Settings["punchy"] = map[string]any{"respond": true} c.Settings["punch_back"] = false - p = NewPunchyFromConfig(l, c) - assert.Equal(t, true, p.GetRespond()) + p = NewPunchyFromConfig(test.NewLogger(), c, nil) + assert.True(t, p.respond.Load()) // punchy.delay - c.Settings["punchy"] = map[interface{}]interface{}{"delay": "1m"} - p = NewPunchyFromConfig(l, c) - assert.Equal(t, time.Minute, p.GetDelay()) + c.Settings["punchy"] = map[string]any{"delay": "1m"} + p = NewPunchyFromConfig(test.NewLogger(), c, nil) + assert.Equal(t, time.Minute, time.Duration(p.delay.Load())) // punchy.respond_delay - c.Settings["punchy"] = map[interface{}]interface{}{"respond_delay": "1m"} - p = NewPunchyFromConfig(l, c) - assert.Equal(t, time.Minute, p.GetRespondDelay()) + c.Settings["punchy"] = map[string]any{"respond_delay": "1m"} + p = NewPunchyFromConfig(test.NewLogger(), c, nil) + assert.Equal(t, time.Minute, time.Duration(p.respondDelay.Load())) } func TestPunchy_reload(t *testing.T) { l := test.NewLogger() c := config.NewC(l) delay, _ := time.ParseDuration("1m") - assert.NoError(t, c.LoadString(` + require.NoError(t, c.LoadString(` punchy: + punch: false delay: 1m respond: false `)) - p := NewPunchyFromConfig(l, c) - assert.Equal(t, delay, p.GetDelay()) - assert.Equal(t, false, p.GetRespond()) + p := NewPunchyFromConfig(test.NewLogger(), c, nil) + assert.False(t, p.punch.Load()) + assert.Equal(t, delay, time.Duration(p.delay.Load())) + assert.False(t, p.respond.Load()) newDelay, _ := time.ParseDuration("10m") - assert.NoError(t, c.ReloadConfigString(` + require.NoError(t, c.ReloadConfigString(` punchy: + punch: true delay: 10m respond: true `)) p.reload(c, false) - assert.Equal(t, newDelay, p.GetDelay()) - assert.Equal(t, true, p.GetRespond()) + assert.True(t, p.punch.Load()) + assert.Equal(t, newDelay, time.Duration(p.delay.Load())) + assert.True(t, p.respond.Load()) +} + +// The tests below pin the shape of each log line Punchy produces so changes +// cannot silently break whatever operators are grepping for. The assertions +// are on the structured message + attrs (e.g. "punchy.respond changed" with +// a respond=true field) rather than a formatted string. Tests filter by +// message rather than asserting total entry counts so unrelated info lines +// are tolerated without being locked into the format. + +type capturedEntry struct { + Level slog.Level + Msg string + Attrs map[string]any +} + +// capturingHandler is a slog.Handler that records each Record it receives so +// tests can assert on the level, message, and attribute map of individual log +// lines without coupling to any specific text format. +type capturingHandler struct { + entries []capturedEntry +} + +func (h *capturingHandler) Enabled(_ context.Context, _ slog.Level) bool { return true } + +func (h *capturingHandler) Handle(_ context.Context, r slog.Record) error { + e := capturedEntry{ + Level: r.Level, + Msg: r.Message, + Attrs: make(map[string]any), + } + r.Attrs(func(a slog.Attr) bool { + e.Attrs[a.Key] = a.Value.Resolve().Any() + return true + }) + h.entries = append(h.entries, e) + return nil +} + +func (h *capturingHandler) WithAttrs(_ []slog.Attr) slog.Handler { return h } +func (h *capturingHandler) WithGroup(_ string) slog.Handler { return h } + +func newCapturingPunchyLogger(t *testing.T) (*slog.Logger, *capturingHandler) { + t.Helper() + hook := &capturingHandler{} + return slog.New(hook), hook +} + +func findEntry(t *testing.T, entries []capturedEntry, msg string) capturedEntry { + t.Helper() + for _, e := range entries { + if e.Msg == msg { + return e + } + } + t.Fatalf("no entry with message %q among %d entries", msg, len(entries)) + return capturedEntry{} +} + +func TestPunchy_LogFormat_InitialEnabled(t *testing.T) { + l, hook := newCapturingPunchyLogger(t) + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(`punchy: {punch: true}`)) + + NewPunchyFromConfig(l, c, nil) + + entry := findEntry(t, hook.entries, "punchy enabled") + assert.Equal(t, slog.LevelInfo, entry.Level) + assert.Empty(t, entry.Attrs) +} + +func TestPunchy_LogFormat_InitialDisabled(t *testing.T) { + l, hook := newCapturingPunchyLogger(t) + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(`punchy: {punch: false}`)) + + NewPunchyFromConfig(l, c, nil) + + entry := findEntry(t, hook.entries, "punchy disabled") + assert.Equal(t, slog.LevelInfo, entry.Level) + assert.Empty(t, entry.Attrs) +} + +func TestPunchy_LogFormat_ReloadPunch(t *testing.T) { + l, hook := newCapturingPunchyLogger(t) + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(`punchy: {punch: false}`)) + NewPunchyFromConfig(l, c, nil) + hook.entries = nil + + require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`)) + + entry := findEntry(t, hook.entries, "punchy.punch changed") + assert.Equal(t, slog.LevelInfo, entry.Level) + assert.Equal(t, map[string]any{"punch": true}, entry.Attrs) +} + +func TestPunchy_LogFormat_ReloadRespond(t *testing.T) { + l, hook := newCapturingPunchyLogger(t) + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(`punchy: {respond: false}`)) + NewPunchyFromConfig(l, c, nil) + hook.entries = nil + + require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`)) + + entry := findEntry(t, hook.entries, "punchy.respond changed") + assert.Equal(t, slog.LevelInfo, entry.Level) + assert.Equal(t, map[string]any{"respond": true}, entry.Attrs) +} + +func TestPunchy_LogFormat_ReloadDelay(t *testing.T) { + l, hook := newCapturingPunchyLogger(t) + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(`punchy: {delay: 1s}`)) + NewPunchyFromConfig(l, c, nil) + hook.entries = nil + + require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`)) + + entry := findEntry(t, hook.entries, "punchy.delay changed") + assert.Equal(t, slog.LevelInfo, entry.Level) + assert.Equal(t, map[string]any{"delay": 10 * time.Second}, entry.Attrs) +} + +func TestPunchy_LogFormat_ReloadTargetAllRemotes(t *testing.T) { + l, hook := newCapturingPunchyLogger(t) + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(`punchy: {target_all_remotes: false}`)) + NewPunchyFromConfig(l, c, nil) + hook.entries = nil + + require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`)) + + entry := findEntry(t, hook.entries, "punchy.target_all_remotes changed") + assert.Equal(t, slog.LevelInfo, entry.Level) + assert.Equal(t, map[string]any{"target_all_remotes": true}, entry.Attrs) +} + +func TestPunchy_LogFormat_ReloadRespondDelay(t *testing.T) { + l, hook := newCapturingPunchyLogger(t) + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(`punchy: {respond_delay: 5s}`)) + NewPunchyFromConfig(l, c, nil) + hook.entries = nil + + require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`)) + + entry := findEntry(t, hook.entries, "punchy.respond_delay changed") + assert.Equal(t, slog.LevelInfo, entry.Level) + assert.Equal(t, map[string]any{"respond_delay": 15 * time.Second}, entry.Attrs) } diff --git a/relay_manager.go b/relay_manager.go index 7aa06ccb..1ae382a3 100644 --- a/relay_manager.go +++ b/relay_manager.go @@ -2,23 +2,27 @@ package nebula import ( "context" + "encoding/binary" "errors" "fmt" + "log/slog" + "net/netip" + "slices" "sync/atomic" - "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" ) type relayManager struct { - l *logrus.Logger - hostmap *HostMap - amRelay atomic.Bool + l *slog.Logger + hostmap *HostMap + amRelay atomic.Bool + useRelays atomic.Bool } -func NewRelayManager(ctx context.Context, l *logrus.Logger, hostmap *HostMap, c *config.C) *relayManager { +func NewRelayManager(ctx context.Context, l *slog.Logger, hostmap *HostMap, c *config.C) *relayManager { rm := &relayManager{ l: l, hostmap: hostmap, @@ -27,15 +31,17 @@ func NewRelayManager(ctx context.Context, l *logrus.Logger, hostmap *HostMap, c c.RegisterReloadCallback(func(c *config.C) { err := rm.reload(c, false) if err != nil { - l.WithError(err).Error("Failed to reload relay_manager") + rm.l.Error("Failed to reload relay_manager", "error", err) } }) return rm } func (rm *relayManager) reload(c *config.C, initial bool) error { - if initial || c.HasChanged("relay.am_relay") { - rm.setAmRelay(c.GetBool("relay.am_relay", false)) + if initial || c.HasChanged("relay.am_relay") || c.HasChanged("relay.use_relays") { + amRelay := c.GetBool("relay.am_relay", false) + rm.amRelay.Store(amRelay) + rm.useRelays.Store(c.GetBool("relay.use_relays", true) && !amRelay) } return nil } @@ -44,16 +50,186 @@ func (rm *relayManager) GetAmRelay() bool { return rm.amRelay.Load() } -func (rm *relayManager) setAmRelay(v bool) { - rm.amRelay.Store(v) +func (rm *relayManager) GetUseRelays() bool { + return rm.useRelays.Load() +} + +// StartRelays drives the relay-establishment side of an outbound handshake attempt. +// For each candidate relay it either kicks off a handshake to the relay, sends a CreateRelayRequest, retransmits +// one that may have been lost, or, once the relay is Established, forwards the in-progress +// stage 0 handshake packet for vpnIp through it. +func (rm *relayManager) StartRelays(f *Interface, vpnIp netip.Addr, hh *HandshakeHostInfo, stage0 []byte) { + hostinfo := hh.hostinfo + if !rm.GetUseRelays() || len(hostinfo.remotes.relays) == 0 { + hh.lastRelays = nil + return + } + + relays := hostinfo.remotes.relays + listLevel := slog.LevelDebug + prior := hh.lastRelays + if !slices.Equal(relays, prior) { + listLevel = slog.LevelInfo + hh.lastRelays = slices.Clone(relays) + } + hl := hostinfo.logger(rm.l) + hl.Log(context.Background(), listLevel, "Attempt to relay through hosts", "relays", relays) + + // Send a RelayRequest to all known Relay IP's + for _, relay := range relays { + // Don't relay through the host I'm trying to connect to + if relay == vpnIp { + continue + } + + // Don't relay to myself + if f.myVpnAddrsTable.Contains(relay) { + continue + } + + // Each relay's per-attempt log fires at Info on the first time we hit it and Debug after that. + level := slog.LevelInfo + if slices.Contains(prior, relay) { + level = slog.LevelDebug + } + + relayHostInfo := rm.hostmap.QueryVpnAddr(relay) + if relayHostInfo == nil || !relayHostInfo.GetRemote().IsValid() { + hl.Log(context.Background(), level, "Establish tunnel to relay target", "relay", relay.String()) + f.Handshake(relay) + continue + } + + // Check the relay HostInfo to see if we already established a relay through + existingRelay, ok := relayHostInfo.relayState.QueryRelayForByIp(vpnIp) + if !ok { + // No relays exist or requested yet. + if relayHostInfo.GetRemote().IsValid() { + idx, err := AddRelay(rm.l, relayHostInfo, rm.hostmap, vpnIp, nil, TerminalType, Requested) + if err != nil { + // No local relay state was installed, so a CreateRelayRequest would hand the + // peer an index we could never resolve. Skip it. + hl.Info("Failed to add relay to hostmap", "relay", relay.String(), "error", err) + continue + } + + m := NebulaControl{ + Type: NebulaControl_CreateRelayRequest, + InitiatorRelayIndex: idx, + } + + switch relayHostInfo.GetCert().Certificate.Version() { + case cert.Version1: + if !f.myVpnAddrs[0].Is4() { + hl.Error("can not establish v1 relay with a v6 network because the relay is not running a current nebula version") + continue + } + + if !vpnIp.Is4() { + hl.Error("can not establish v1 relay with a v6 remote network because the relay is not running a current nebula version") + continue + } + + b := f.myVpnAddrs[0].As4() + m.OldRelayFromAddr = binary.BigEndian.Uint32(b[:]) + b = vpnIp.As4() + m.OldRelayToAddr = binary.BigEndian.Uint32(b[:]) + case cert.Version2: + m.RelayFromAddr = netAddrToProtoAddr(f.myVpnAddrs[0]) + m.RelayToAddr = netAddrToProtoAddr(vpnIp) + default: + hl.Error("Unknown certificate version found while creating relay") + continue + } + + msg, err := m.Marshal() + if err != nil { + hl.Error("Failed to marshal Control message to create relay", "error", err) + } else { + f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu)) + rm.l.Log(context.Background(), level, "send CreateRelayRequest", + "relayFrom", f.myVpnAddrs[0], + "relayTo", vpnIp, + "initiatorRelayIndex", idx, + "relay", relay, + ) + } + } + continue + } + + switch existingRelay.State { + case Established: + hl.Log(context.Background(), level, "Send handshake via relay", "relay", relay.String()) + f.SendVia(relayHostInfo, existingRelay, stage0, make([]byte, 12), make([]byte, mtu), false) + case Disestablished: + // Mark this relay as 'requested' + relayHostInfo.relayState.UpdateRelayForByIpState(vpnIp, Requested) + fallthrough + case Requested: + hl.Log(context.Background(), level, "Re-send CreateRelay request", "relay", relay.String()) + // Re-send the CreateRelay request, in case the previous one was lost. + m := NebulaControl{ + Type: NebulaControl_CreateRelayRequest, + InitiatorRelayIndex: existingRelay.LocalIndex, + } + + switch relayHostInfo.GetCert().Certificate.Version() { + case cert.Version1: + if !f.myVpnAddrs[0].Is4() { + hl.Error("can not establish v1 relay with a v6 network because the relay is not running a current nebula version") + continue + } + + if !vpnIp.Is4() { + hl.Error("can not establish v1 relay with a v6 remote network because the relay is not running a current nebula version") + continue + } + + b := f.myVpnAddrs[0].As4() + m.OldRelayFromAddr = binary.BigEndian.Uint32(b[:]) + b = vpnIp.As4() + m.OldRelayToAddr = binary.BigEndian.Uint32(b[:]) + case cert.Version2: + m.RelayFromAddr = netAddrToProtoAddr(f.myVpnAddrs[0]) + m.RelayToAddr = netAddrToProtoAddr(vpnIp) + default: + hl.Error("Unknown certificate version found while creating relay") + continue + } + msg, err := m.Marshal() + if err != nil { + hl.Error("Failed to marshal Control message to create relay", "error", err) + } else { + // This must send over the hostinfo, not over hm.Hosts[ip] + f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu)) + rm.l.Log(context.Background(), level, "send CreateRelayRequest", + "relayFrom", f.myVpnAddrs[0], + "relayTo", vpnIp, + "initiatorRelayIndex", existingRelay.LocalIndex, + "relay", relay, + ) + } + case PeerRequested: + // PeerRequested only occurs in Forwarding relays, not Terminal relays, and this is a Terminal relay case. + fallthrough + default: + hl.Error("Relay unexpected state", + "vpnIp", vpnIp, + "state", existingRelay.State, + "relay", relay, + ) + + } + } } // AddRelay finds an available relay index on the hostmap, and associates the relay info with it. // relayHostInfo is the Nebula peer which can be used as a relay to access the target vpnIp. -func AddRelay(l *logrus.Logger, relayHostInfo *HostInfo, hm *HostMap, vpnIp iputil.VpnIp, remoteIdx *uint32, relayType int, state int) (uint32, error) { +func AddRelay(l *slog.Logger, relayHostInfo *HostInfo, hm *HostMap, vpnIp netip.Addr, remoteIdx *uint32, relayType int, state int) (uint32, error) { hm.Lock() defer hm.Unlock() - for i := 0; i < 32; i++ { + for range 32 { index, err := generateIndex(l) if err != nil { return 0, err @@ -64,14 +240,19 @@ func AddRelay(l *logrus.Logger, relayHostInfo *HostInfo, hm *HostMap, vpnIp iput // Avoid standing up a relay that can't be used since only the primary hostinfo // will be pointed to by the relay logic //TODO: if there was an existing primary and it had relay state, should we merge? - hm.unlockedMakePrimary(relayHostInfo) + if !hm.unlockedMakePrimary(relayHostInfo) { + // The tunnel was torn down after the caller grabbed relayHostInfo. A relay standing + // on an unlinked hostinfo would never carry traffic, and its Relays entry could + // never be reclaimed since the delete-time cleanup has already run. + return 0, errors.New("relay hostinfo is no longer in the hostmap") + } hm.Relays[index] = relayHostInfo newRelay := Relay{ Type: relayType, State: state, LocalIndex: index, - PeerIp: vpnIp, + PeerAddr: vpnIp, } if remoteIdx != nil { @@ -90,40 +271,91 @@ func AddRelay(l *logrus.Logger, relayHostInfo *HostInfo, hm *HostMap, vpnIp iput func (rm *relayManager) EstablishRelay(relayHostInfo *HostInfo, m *NebulaControl) (*Relay, error) { relay, ok := relayHostInfo.relayState.CompleteRelayByIdx(m.InitiatorRelayIndex, m.ResponderRelayIndex) if !ok { - rm.l.WithFields(logrus.Fields{"relay": relayHostInfo.vpnIp, - "initiatorRelayIndex": m.InitiatorRelayIndex, - "relayFrom": m.RelayFromIp, - "relayTo": m.RelayToIp}).Info("relayManager failed to update relay") + var relayFrom, relayTo any + if m.RelayFromAddr == nil { + relayFrom = m.OldRelayFromAddr + } else { + relayFrom = m.RelayFromAddr + } + if m.RelayToAddr == nil { + relayTo = m.OldRelayToAddr + } else { + relayTo = m.RelayToAddr + } + + rm.l.Info("relayManager failed to update relay", + "relay", relayHostInfo.vpnAddrs[0], + "initiatorRelayIndex", m.InitiatorRelayIndex, + "relayFrom", relayFrom, + "relayTo", relayTo, + ) return nil, fmt.Errorf("unknown relay") } return relay, nil } -func (rm *relayManager) HandleControlMsg(h *HostInfo, m *NebulaControl, f *Interface) { +func (rm *relayManager) HandleControlMsg(h *HostInfo, d []byte, f *Interface) { + msg := &NebulaControl{} + err := msg.Unmarshal(d) + if err != nil { + h.logger(f.l).Error("Failed to unmarshal control message", "error", err) + return + } + + var v cert.Version + if msg.OldRelayFromAddr > 0 || msg.OldRelayToAddr > 0 { + v = cert.Version1 + + b := [4]byte{} + binary.BigEndian.PutUint32(b[:], msg.OldRelayFromAddr) + msg.RelayFromAddr = netAddrToProtoAddr(netip.AddrFrom4(b)) + + binary.BigEndian.PutUint32(b[:], msg.OldRelayToAddr) + msg.RelayToAddr = netAddrToProtoAddr(netip.AddrFrom4(b)) + } else { + v = cert.Version2 + } + + // validate: + switch msg.Type { + case NebulaControl_CreateRelayRequest, NebulaControl_CreateRelayResponse: + if msg.RelayFromAddr == nil { + if f.l.Enabled(context.Background(), slog.LevelDebug) { + h.logger(f.l).Debug("Control message received with nil RelayFromAddr", "type", msg.Type) + } + return + } else if msg.RelayToAddr == nil { + if f.l.Enabled(context.Background(), slog.LevelDebug) { + h.logger(f.l).Debug("Control message received with nil RelayToAddr", "type", msg.Type) + } + return + } + } - switch m.Type { + switch msg.Type { case NebulaControl_CreateRelayRequest: - rm.handleCreateRelayRequest(h, f, m) + rm.handleCreateRelayRequest(v, h, f, msg) case NebulaControl_CreateRelayResponse: - rm.handleCreateRelayResponse(h, f, m) + rm.handleCreateRelayResponse(v, h, f, msg) } - } -func (rm *relayManager) handleCreateRelayResponse(h *HostInfo, f *Interface, m *NebulaControl) { - rm.l.WithFields(logrus.Fields{ - "relayFrom": iputil.VpnIp(m.RelayFromIp), - "relayTo": iputil.VpnIp(m.RelayToIp), - "initiatorRelayIndex": m.InitiatorRelayIndex, - "responderRelayIndex": m.ResponderRelayIndex, - "vpnIp": h.vpnIp}). - Info("handleCreateRelayResponse") - target := iputil.VpnIp(m.RelayToIp) +func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f *Interface, m *NebulaControl) { + //nil-checks for protoAddrToNetAddr handled by caller + relayFrom := protoAddrToNetAddr(m.RelayFromAddr) + relayTo := protoAddrToNetAddr(m.RelayToAddr) + rm.l.Info("handleCreateRelayResponse", + "relayFrom", relayFrom, + "relayTo", relayTo, + "initiatorRelayIndex", m.InitiatorRelayIndex, + "responderRelayIndex", m.ResponderRelayIndex, + "vpnAddrs", h.vpnAddrs, + ) relay, err := rm.EstablishRelay(h, m) if err != nil { - rm.l.WithError(err).Error("Failed to update relay for relayTo") + rm.l.Error("Failed to update relay for relayTo", "error", err) return } // Do I need to complete the relays now? @@ -131,62 +363,88 @@ func (rm *relayManager) handleCreateRelayResponse(h *HostInfo, f *Interface, m * return } // I'm the middle man. Let the initiator know that the I've established the relay they requested. - peerHostInfo := rm.hostmap.QueryVpnIp(relay.PeerIp) + peerHostInfo := rm.hostmap.QueryVpnAddr(relay.PeerAddr) if peerHostInfo == nil { - rm.l.WithField("relayTo", relay.PeerIp).Error("Can't find a HostInfo for peer") + rm.l.Error("Can't find a HostInfo for peer", "relayTo", relay.PeerAddr) return } - peerRelay, ok := peerHostInfo.relayState.QueryRelayForByIp(target) + peerRelay, ok := peerHostInfo.relayState.QueryRelayForByIp(relayTo) if !ok { - rm.l.WithField("relayTo", peerHostInfo.vpnIp).Error("peerRelay does not have Relay state for relayTo") + rm.l.Error("peerRelay does not have Relay state for relayTo", "relayTo", peerHostInfo.vpnAddrs[0]) return } - if peerRelay.State == PeerRequested { - peerRelay.State = Established + switch peerRelay.State { + case Requested: + // I initiated the request to this peer, but haven't heard back from the peer yet. I must wait for this peer + // to respond to complete the connection. + case PeerRequested, Disestablished, Established: + peerHostInfo.relayState.UpdateRelayForByIpState(relayTo, Established) resp := NebulaControl{ Type: NebulaControl_CreateRelayResponse, ResponderRelayIndex: peerRelay.LocalIndex, InitiatorRelayIndex: peerRelay.RemoteIndex, - RelayFromIp: uint32(peerHostInfo.vpnIp), - RelayToIp: uint32(target), } + + peer := peerHostInfo.vpnAddrs[0] + if v == cert.Version1 { + if !peer.Is4() { + rm.l.Error("Refusing to CreateRelayResponse for a v1 relay with an ipv6 address", + "relayFrom", peer, + "relayTo", relayTo, + "initiatorRelayIndex", resp.InitiatorRelayIndex, + "responderRelayIndex", resp.ResponderRelayIndex, + "vpnAddrs", peerHostInfo.vpnAddrs, + ) + return + } + + b := peer.As4() + resp.OldRelayFromAddr = binary.BigEndian.Uint32(b[:]) + b = relayTo.As4() + resp.OldRelayToAddr = binary.BigEndian.Uint32(b[:]) + } else { + resp.RelayFromAddr = netAddrToProtoAddr(peer) + resp.RelayToAddr = m.RelayToAddr + } + msg, err := resp.Marshal() if err != nil { - rm.l. - WithError(err).Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay") - } else { - f.SendMessageToHostInfo(header.Control, 0, peerHostInfo, msg, make([]byte, 12), make([]byte, mtu)) - rm.l.WithFields(logrus.Fields{ - "relayFrom": iputil.VpnIp(resp.RelayFromIp), - "relayTo": iputil.VpnIp(resp.RelayToIp), - "initiatorRelayIndex": resp.InitiatorRelayIndex, - "responderRelayIndex": resp.ResponderRelayIndex, - "vpnIp": peerHostInfo.vpnIp}). - Info("send CreateRelayResponse") + rm.l.Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay", "error", err) + return } + f.SendMessageToHostInfo(header.Control, 0, peerHostInfo, msg, make([]byte, 12), make([]byte, mtu)) + rm.l.Info("send CreateRelayResponse", + "relayFrom", peer, + "relayTo", relayTo, + "initiatorRelayIndex", resp.InitiatorRelayIndex, + "responderRelayIndex", resp.ResponderRelayIndex, + "vpnAddrs", peerHostInfo.vpnAddrs, + ) } } -func (rm *relayManager) handleCreateRelayRequest(h *HostInfo, f *Interface, m *NebulaControl) { +func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f *Interface, m *NebulaControl) { + //nil-checks for protoAddrToNetAddr handled by caller + from := protoAddrToNetAddr(m.RelayFromAddr) + target := protoAddrToNetAddr(m.RelayToAddr) - from := iputil.VpnIp(m.RelayFromIp) - target := iputil.VpnIp(m.RelayToIp) - - logMsg := rm.l.WithFields(logrus.Fields{ - "relayFrom": from, - "relayTo": target, - "initiatorRelayIndex": m.InitiatorRelayIndex, - "vpnIp": h.vpnIp}) + logMsg := rm.l.With( + "relayFrom", from, + "relayTo", target, + "initiatorRelayIndex", m.InitiatorRelayIndex, + "vpnAddrs", h.vpnAddrs, + ) logMsg.Info("handleCreateRelayRequest") // Is the source of the relay me? This should never happen, but did happen due to // an issue migrating relays over to newly re-handshaked host info objects. - if from == f.myVpnIp { - logMsg.WithField("myIP", f.myVpnIp).Error("Discarding relay request from myself") + if f.myVpnAddrsTable.Contains(from) { + logMsg.Error("Discarding relay request from myself", "myIP", from) return } + // Is the target of the relay me? - if target == f.myVpnIp { + if f.myVpnAddrsTable.Contains(target) { existingRelay, ok := h.relayState.QueryRelayForByIp(from) if ok { switch existingRelay.State { @@ -200,22 +458,37 @@ func (rm *relayManager) handleCreateRelayRequest(h *HostInfo, f *Interface, m *N if existingRelay.RemoteIndex != m.InitiatorRelayIndex { // We got a brand new Relay request, because its index is different than what we saw before. // This should never happen. The peer should never change an index, once created. - logMsg.WithFields(logrus.Fields{ - "existingRemoteIndex": existingRelay.RemoteIndex}).Error("Existing relay mismatch with CreateRelayRequest") + logMsg.Error("Existing relay mismatch with CreateRelayRequest", + "existingRemoteIndex", existingRelay.RemoteIndex) return } + case Disestablished: + if existingRelay.RemoteIndex != m.InitiatorRelayIndex { + // We got a brand new Relay request, because its index is different than what we saw before. + // This should never happen. The peer should never change an index, once created. + logMsg.Error("Existing relay mismatch with CreateRelayRequest", + "existingRemoteIndex", existingRelay.RemoteIndex) + return + } + // Mark the relay as 'Established' because it's safe to use again + h.relayState.UpdateRelayForByIpState(from, Established) + case PeerRequested: + // I should never be in this state, because I am terminal, not forwarding. + logMsg.Error("Unexpected Relay State found", + "existingRemoteIndex", existingRelay.RemoteIndex, + "state", existingRelay.State) } } else { _, err := AddRelay(rm.l, h, f.hostMap, from, &m.InitiatorRelayIndex, TerminalType, Established) if err != nil { - logMsg.WithError(err).Error("Failed to add relay") + logMsg.Error("Failed to add relay", "error", err) return } } relay, ok := h.relayState.QueryRelayForByIp(from) if !ok { - logMsg.Error("Relay State not found") + logMsg.Error("Relay State not found", "from", from) return } @@ -223,22 +496,30 @@ func (rm *relayManager) handleCreateRelayRequest(h *HostInfo, f *Interface, m *N Type: NebulaControl_CreateRelayResponse, ResponderRelayIndex: relay.LocalIndex, InitiatorRelayIndex: relay.RemoteIndex, - RelayFromIp: uint32(from), - RelayToIp: uint32(target), } + + if v == cert.Version1 { + b := from.As4() + resp.OldRelayFromAddr = binary.BigEndian.Uint32(b[:]) + b = target.As4() + resp.OldRelayToAddr = binary.BigEndian.Uint32(b[:]) + } else { + resp.RelayFromAddr = netAddrToProtoAddr(from) + resp.RelayToAddr = netAddrToProtoAddr(target) + } + msg, err := resp.Marshal() if err != nil { - logMsg. - WithError(err).Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay") + logMsg.Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay", "error", err) } else { f.SendMessageToHostInfo(header.Control, 0, h, msg, make([]byte, 12), make([]byte, mtu)) - rm.l.WithFields(logrus.Fields{ - "relayFrom": iputil.VpnIp(resp.RelayFromIp), - "relayTo": iputil.VpnIp(resp.RelayToIp), - "initiatorRelayIndex": resp.InitiatorRelayIndex, - "responderRelayIndex": resp.ResponderRelayIndex, - "vpnIp": h.vpnIp}). - Info("send CreateRelayResponse") + rm.l.Info("send CreateRelayResponse", + "relayFrom", from, + "relayTo", target, + "initiatorRelayIndex", resp.InitiatorRelayIndex, + "responderRelayIndex", resp.ResponderRelayIndex, + "vpnAddrs", h.vpnAddrs, + ) } return } else { @@ -246,110 +527,79 @@ func (rm *relayManager) handleCreateRelayRequest(h *HostInfo, f *Interface, m *N if !rm.GetAmRelay() { return } - peer := rm.hostmap.QueryVpnIp(target) + peer := rm.hostmap.QueryVpnAddr(target) if peer == nil { // Try to establish a connection to this host. If we get a future relay request, // we'll be ready! f.Handshake(target) return } - if peer.remote == nil { + if !peer.GetRemote().IsValid() { // Only create relays to peers for whom I have a direct connection return } - sendCreateRequest := false var index uint32 var err error targetRelay, ok := peer.relayState.QueryRelayForByIp(from) if ok { index = targetRelay.LocalIndex - if targetRelay.State == Requested { - sendCreateRequest = true - } } else { // Allocate an index in the hostMap for this relay peer index, err = AddRelay(rm.l, peer, f.hostMap, from, nil, ForwardingType, Requested) if err != nil { return } - sendCreateRequest = true } - if sendCreateRequest { - // Send a CreateRelayRequest to the peer. - req := NebulaControl{ - Type: NebulaControl_CreateRelayRequest, - InitiatorRelayIndex: index, - RelayFromIp: uint32(h.vpnIp), - RelayToIp: uint32(target), - } - msg, err := req.Marshal() - if err != nil { - logMsg. - WithError(err).Error("relayManager Failed to marshal Control message to create relay") - } else { - f.SendMessageToHostInfo(header.Control, 0, peer, msg, make([]byte, 12), make([]byte, mtu)) - rm.l.WithFields(logrus.Fields{ - "relayFrom": iputil.VpnIp(req.RelayFromIp), - "relayTo": iputil.VpnIp(req.RelayToIp), - "initiatorRelayIndex": req.InitiatorRelayIndex, - "responderRelayIndex": req.ResponderRelayIndex, - "vpnIp": target}). - Info("send CreateRelayRequest") + peer.relayState.UpdateRelayForByIpState(from, Requested) + // Send a CreateRelayRequest to the peer. + req := NebulaControl{ + Type: NebulaControl_CreateRelayRequest, + InitiatorRelayIndex: index, + } + + if v == cert.Version1 { + if !h.vpnAddrs[0].Is4() { + rm.l.Error("Refusing to CreateRelayRequest for a v1 relay with an ipv6 address", + "relayFrom", h.vpnAddrs[0], + "relayTo", target, + "initiatorRelayIndex", req.InitiatorRelayIndex, + "responderRelayIndex", req.ResponderRelayIndex, + "vpnAddr", target, + ) + return } + + b := h.vpnAddrs[0].As4() + req.OldRelayFromAddr = binary.BigEndian.Uint32(b[:]) + b = target.As4() + req.OldRelayToAddr = binary.BigEndian.Uint32(b[:]) + } else { + req.RelayFromAddr = netAddrToProtoAddr(h.vpnAddrs[0]) + req.RelayToAddr = netAddrToProtoAddr(target) } + + msg, err := req.Marshal() + if err != nil { + logMsg.Error("relayManager Failed to marshal Control message to create relay", "error", err) + } else { + f.SendMessageToHostInfo(header.Control, 0, peer, msg, make([]byte, 12), make([]byte, mtu)) + rm.l.Info("send CreateRelayRequest", + "relayFrom", h.vpnAddrs[0], + "relayTo", target, + "initiatorRelayIndex", req.InitiatorRelayIndex, + "responderRelayIndex", req.ResponderRelayIndex, + "vpnAddr", target, + ) + } + // Also track the half-created Relay state just received - relay, ok := h.relayState.QueryRelayForByIp(target) + _, ok = h.relayState.QueryRelayForByIp(target) if !ok { - // Add the relay - state := PeerRequested - if targetRelay != nil && targetRelay.State == Established { - state = Established - } - _, err := AddRelay(rm.l, h, f.hostMap, target, &m.InitiatorRelayIndex, ForwardingType, state) + _, err := AddRelay(rm.l, h, f.hostMap, target, &m.InitiatorRelayIndex, ForwardingType, PeerRequested) if err != nil { - logMsg. - WithError(err).Error("relayManager Failed to allocate a local index for relay") + logMsg.Error("relayManager Failed to allocate a local index for relay", "error", err) return } - } else { - switch relay.State { - case Established: - if relay.RemoteIndex != m.InitiatorRelayIndex { - // We got a brand new Relay request, because its index is different than what we saw before. - // This should never happen. The peer should never change an index, once created. - logMsg.WithFields(logrus.Fields{ - "existingRemoteIndex": relay.RemoteIndex}).Error("Existing relay mismatch with CreateRelayRequest") - return - } - resp := NebulaControl{ - Type: NebulaControl_CreateRelayResponse, - ResponderRelayIndex: relay.LocalIndex, - InitiatorRelayIndex: relay.RemoteIndex, - RelayFromIp: uint32(h.vpnIp), - RelayToIp: uint32(target), - } - msg, err := resp.Marshal() - if err != nil { - rm.l. - WithError(err).Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay") - } else { - f.SendMessageToHostInfo(header.Control, 0, h, msg, make([]byte, 12), make([]byte, mtu)) - rm.l.WithFields(logrus.Fields{ - "relayFrom": iputil.VpnIp(resp.RelayFromIp), - "relayTo": iputil.VpnIp(resp.RelayToIp), - "initiatorRelayIndex": resp.InitiatorRelayIndex, - "responderRelayIndex": resp.ResponderRelayIndex, - "vpnIp": h.vpnIp}). - Info("send CreateRelayResponse") - } - - case Requested: - // Keep waiting for the other relay to complete - } } } } - -func (rm *relayManager) RemoveRelay(localIdx uint32) { - rm.hostmap.RemoveRelay(localIdx) -} diff --git a/relay_manager_test.go b/relay_manager_test.go new file mode 100644 index 00000000..8da38940 --- /dev/null +++ b/relay_manager_test.go @@ -0,0 +1,97 @@ +package nebula + +import ( + "bytes" + "log/slog" + "net/netip" + "testing" + + "github.com/gaissmai/bart" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" +) + +// TestStartRelaysLogDedupe verifies that repeated attempts with the same relay set drop the log +// chatter to Debug, mirroring how the normal handshake retry loop quiets down once it's already +// announced its targets. +func TestStartRelaysLogDedupe(t *testing.T) { + vpnIp := netip.MustParseAddr("100.64.99.4") + otherRelay := netip.MustParseAddr("100.64.99.5") + + newHH := func() *HandshakeHostInfo { + // Use the target's own vpnIp as the "relay" so the loop body skips it without + // touching any sender-side state. That isolates the test to the level-selection + // behavior of the top-level "Attempt to relay through hosts" log. + hostinfo := &HostInfo{ + vpnAddrs: []netip.Addr{vpnIp}, + localIndexId: 1, + remotes: NewRemoteList([]netip.Addr{vpnIp}, nil), + } + hostinfo.remotes.relays = []netip.Addr{vpnIp} + return &HandshakeHostInfo{hostinfo: hostinfo} + } + + // Park any extra relay addresses we'll introduce mid-test in myVpnAddrsTable so the loop + // body always skips before touching f.Handshake (which would need a real handshakeManager). + addrTable := new(bart.Lite) + addrTable.Insert(netip.PrefixFrom(otherRelay, otherRelay.BitLen())) + f := &Interface{myVpnAddrsTable: addrTable} + + newRM := func(buf *bytes.Buffer) *relayManager { + l := test.NewLoggerWithOutputAndLevel(buf, slog.LevelDebug) + rm := &relayManager{l: l, hostmap: newHostMap(l)} + rm.useRelays.Store(true) + return rm + } + + const msg = `msg="Attempt to relay through hosts"` + + t.Run("first attempt logs at Info", func(t *testing.T) { + var buf bytes.Buffer + rm := newRM(&buf) + hh := newHH() + rm.StartRelays(f, vpnIp, hh, nil) + assert.Equal(t, []netip.Addr{vpnIp}, hh.lastRelays, "lastRelays should record the relay set we just attempted") + assert.Contains(t, buf.String(), "level=INFO "+msg, "expected Info level on first attempt") + }) + + t.Run("repeat attempt with same relays drops to Debug", func(t *testing.T) { + var buf bytes.Buffer + rm := newRM(&buf) + hh := newHH() + rm.StartRelays(f, vpnIp, hh, nil) + first := append([]netip.Addr(nil), hh.lastRelays...) + buf.Reset() + rm.StartRelays(f, vpnIp, hh, nil) + assert.Equal(t, first, hh.lastRelays) + assert.Contains(t, buf.String(), "level=DEBUG "+msg, "expected Debug level on identical retry") + assert.NotContains(t, buf.String(), "level=INFO "+msg, "Info should not fire on identical retry") + }) + + t.Run("changed relay list bumps back to Info", func(t *testing.T) { + var buf bytes.Buffer + rm := newRM(&buf) + hh := newHH() + rm.StartRelays(f, vpnIp, hh, nil) + buf.Reset() + + // The lighthouse handed us a new set this round. + hh.hostinfo.remotes.relays = []netip.Addr{vpnIp, otherRelay} + + rm.StartRelays(f, vpnIp, hh, nil) + assert.Equal(t, []netip.Addr{vpnIp, otherRelay}, hh.lastRelays) + assert.Contains(t, buf.String(), "level=INFO "+msg, "expected Info when the relay list changes") + }) + + t.Run("disabled relays clears lastRelays and emits no Attempt log", func(t *testing.T) { + var buf bytes.Buffer + rm := newRM(&buf) + rm.useRelays.Store(false) + hh := newHH() + hh.lastRelays = []netip.Addr{vpnIp} + + rm.StartRelays(f, vpnIp, hh, nil) + assert.Nil(t, hh.lastRelays, "with relays disabled lastRelays should be cleared") + assert.NotContains(t, buf.String(), msg, "should not log when we shortcut out") + }) +} diff --git a/remote_list.go b/remote_list.go index 60a1afda..9d1b387e 100644 --- a/remote_list.go +++ b/remote_list.go @@ -1,27 +1,24 @@ package nebula import ( - "bytes" "context" + "log/slog" "net" "net/netip" + "slices" "sort" "strconv" "sync" "sync/atomic" "time" - - "github.com/sirupsen/logrus" - "github.com/slackhq/nebula/iputil" - "github.com/slackhq/nebula/udp" ) // forEachFunc is used to benefit folks that want to do work inside the lock -type forEachFunc func(addr *udp.Addr, preferred bool) +type forEachFunc func(addr netip.AddrPort, preferred bool) // The checkFuncs here are to simplify bulk importing LH query response logic into a single function (reset slice and iterate) -type checkFuncV4 func(vpnIp iputil.VpnIp, to *Ip4AndPort) bool -type checkFuncV6 func(vpnIp iputil.VpnIp, to *Ip6AndPort) bool +type checkFuncV4 func(vpnIp netip.Addr, to *V4AddrPort) bool +type checkFuncV6 func(vpnIp netip.Addr, to *V6AddrPort) bool // CacheMap is a struct that better represents the lighthouse cache for humans // The string key is the owners vpnIp @@ -30,14 +27,11 @@ type CacheMap map[string]*Cache // Cache is the other part of CacheMap to better represent the lighthouse cache for humans // We don't reason about ipv4 vs ipv6 here type Cache struct { - Learned []*udp.Addr `json:"learned,omitempty"` - Reported []*udp.Addr `json:"reported,omitempty"` - Relay []*net.IP `json:"relay"` + Learned []netip.AddrPort `json:"learned,omitempty"` + Reported []netip.AddrPort `json:"reported,omitempty"` + Relay []netip.Addr `json:"relay"` } -//TODO: Seems like we should plop static host entries in here too since the are protected by the lighthouse from deletion -// We will never clean learned/reported information for them as it stands today - // cache is an internal struct that splits v4 and v6 addresses inside the cache map type cache struct { v4 *cacheV4 @@ -46,19 +40,19 @@ type cache struct { } type cacheRelay struct { - relay []uint32 + relay []netip.Addr } // cacheV4 stores learned and reported ipv4 records under cache type cacheV4 struct { - learned *Ip4AndPort - reported []*Ip4AndPort + learned *V4AddrPort + reported []*V4AddrPort } // cacheV4 stores learned and reported ipv6 records under cache type cacheV6 struct { - learned *Ip6AndPort - reported []*Ip6AndPort + learned *V6AddrPort + reported []*V6AddrPort } type hostnamePort struct { @@ -71,11 +65,11 @@ type hostnamesResults struct { network string lookupTimeout time.Duration cancelFn func() - l *logrus.Logger + l *slog.Logger ips atomic.Pointer[map[netip.AddrPort]struct{}] } -func NewHostnameResults(ctx context.Context, l *logrus.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, onUpdate func()) (*hostnamesResults, error) { +func NewHostnameResults(ctx context.Context, l *slog.Logger, d time.Duration, network string, timeout time.Duration, hostPorts []string, onUpdate func()) (*hostnamesResults, error) { r := &hostnamesResults{ hostnames: make([]hostnamePort, len(hostPorts)), network: network, @@ -126,11 +120,15 @@ func NewHostnameResults(ctx context.Context, l *logrus.Logger, d time.Duration, addrs, err := net.DefaultResolver.LookupNetIP(timeoutCtx, r.network, hostPort.name) timeoutCancel() if err != nil { - l.WithFields(logrus.Fields{"hostname": hostPort.name, "network": r.network}).WithError(err).Error("DNS resolution failed for static_map host") + l.Error("DNS resolution failed for static_map host", + "hostname", hostPort.name, + "network", r.network, + "error", err, + ) continue } for _, a := range addrs { - netipAddrs[netip.AddrPortFrom(a, hostPort.port)] = struct{}{} + netipAddrs[netip.AddrPortFrom(a.Unmap(), hostPort.port)] = struct{}{} } } origSet := r.ips.Load() @@ -150,7 +148,10 @@ func NewHostnameResults(ctx context.Context, l *logrus.Logger, d time.Duration, } } if different { - l.WithFields(logrus.Fields{"origSet": origSet, "newSet": netipAddrs}).Info("DNS results changed for host list") + l.Info("DNS results changed for host list", + "origSet", origSet, + "newSet", netipAddrs, + ) r.ips.Store(&netipAddrs) onUpdate() } @@ -173,7 +174,7 @@ func (hr *hostnamesResults) Cancel() { } } -func (hr *hostnamesResults) GetIPs() []netip.AddrPort { +func (hr *hostnamesResults) GetAddrs() []netip.AddrPort { var retSlice []netip.AddrPort if hr != nil { p := hr.ips.Load() @@ -192,36 +193,44 @@ type RemoteList struct { // Every interaction with internals requires a lock! sync.RWMutex - // A deduplicated set of addresses. Any accessor should lock beforehand. - addrs []*udp.Addr + // The full list of vpn addresses assigned to this host + vpnAddrs []netip.Addr + + // A deduplicated set of underlay addresses. Any accessor should lock beforehand. + addrs []netip.AddrPort // A set of relay addresses. VpnIp addresses that the remote identified as relays. - relays []*iputil.VpnIp + relays []netip.Addr // These are maps to store v4 and v6 addresses per lighthouse // Map key is the vpnIp of the person that told us about this the cached entries underneath. // For learned addresses, this is the vpnIp that sent the packet - cache map[iputil.VpnIp]*cache + cache map[netip.Addr]*cache - hr *hostnamesResults - shouldAdd func(netip.Addr) bool + hr *hostnamesResults + + // shouldAdd is a nillable function that decides if x should be added to addrs. + shouldAdd func(vpnAddrs []netip.Addr, x netip.Addr) bool // This is a list of remotes that we have tried to handshake with and have returned from the wrong vpn ip. // They should not be tried again during a handshake - badRemotes []*udp.Addr + badRemotes []netip.AddrPort // A flag that the cache may have changed and addrs needs to be rebuilt shouldRebuild bool } // NewRemoteList creates a new empty RemoteList -func NewRemoteList(shouldAdd func(netip.Addr) bool) *RemoteList { - return &RemoteList{ - addrs: make([]*udp.Addr, 0), - relays: make([]*iputil.VpnIp, 0), - cache: make(map[iputil.VpnIp]*cache), +func NewRemoteList(vpnAddrs []netip.Addr, shouldAdd func([]netip.Addr, netip.Addr) bool) *RemoteList { + r := &RemoteList{ + vpnAddrs: make([]netip.Addr, len(vpnAddrs)), + addrs: make([]netip.AddrPort, 0), + relays: make([]netip.Addr, 0), + cache: make(map[netip.Addr]*cache), shouldAdd: shouldAdd, } + copy(r.vpnAddrs, vpnAddrs) + return r } func (r *RemoteList) unlockedSetHostnamesResults(hr *hostnamesResults) { @@ -230,9 +239,34 @@ func (r *RemoteList) unlockedSetHostnamesResults(hr *hostnamesResults) { r.hr = hr } +// ResetForOwner zeros the reported address slices for the given owner and marks the addrs list dirty. +// Any pending hostname resolution will be canceled. +func (r *RemoteList) ResetForOwner(ownerVpnAddr netip.Addr) { + r.Lock() + defer r.Unlock() + r.hr.Cancel() + if c, ok := r.cache[ownerVpnAddr]; ok { + if c.v4 != nil { + c.v4.reported = c.v4.reported[:0] + } + if c.v6 != nil { + c.v6.reported = c.v6.reported[:0] + } + } + r.shouldRebuild = true +} + +// ClearHostnameResults cancels the in-flight DNS resolver goroutine (if any) and drops the resolved IP cache. +func (r *RemoteList) ClearHostnameResults() { + r.Lock() + defer r.Unlock() + r.unlockedSetHostnamesResults(nil) + r.shouldRebuild = true +} + // Len locks and reports the size of the deduplicated address list // The deduplication work may need to occur here, so you must pass preferredRanges -func (r *RemoteList) Len(preferredRanges []*net.IPNet) int { +func (r *RemoteList) Len(preferredRanges []netip.Prefix) int { r.Rebuild(preferredRanges) r.RLock() defer r.RUnlock() @@ -241,18 +275,18 @@ func (r *RemoteList) Len(preferredRanges []*net.IPNet) int { // ForEach locks and will call the forEachFunc for every deduplicated address in the list // The deduplication work may need to occur here, so you must pass preferredRanges -func (r *RemoteList) ForEach(preferredRanges []*net.IPNet, forEach forEachFunc) { +func (r *RemoteList) ForEach(preferredRanges []netip.Prefix, forEach forEachFunc) { r.Rebuild(preferredRanges) r.RLock() for _, v := range r.addrs { - forEach(v, isPreferred(v.IP, preferredRanges)) + forEach(v, isPreferred(v.Addr(), preferredRanges)) } r.RUnlock() } // CopyAddrs locks and makes a deep copy of the deduplicated address list // The deduplication work may need to occur here, so you must pass preferredRanges -func (r *RemoteList) CopyAddrs(preferredRanges []*net.IPNet) []*udp.Addr { +func (r *RemoteList) CopyAddrs(preferredRanges []netip.Prefix) []netip.AddrPort { if r == nil { return nil } @@ -261,9 +295,9 @@ func (r *RemoteList) CopyAddrs(preferredRanges []*net.IPNet) []*udp.Addr { r.RLock() defer r.RUnlock() - c := make([]*udp.Addr, len(r.addrs)) + c := make([]netip.AddrPort, len(r.addrs)) for i, v := range r.addrs { - c[i] = v.Copy() + c[i] = v } return c } @@ -271,14 +305,13 @@ func (r *RemoteList) CopyAddrs(preferredRanges []*net.IPNet) []*udp.Addr { // LearnRemote locks and sets the learned slot for the owner vpn ip to the provided addr // Currently this is only needed when HostInfo.SetRemote is called as that should cover both handshaking and roaming. // It will mark the deduplicated address list as dirty, so do not call it unless new information is available -// TODO: this needs to support the allow list list -func (r *RemoteList) LearnRemote(ownerVpnIp iputil.VpnIp, addr *udp.Addr) { +func (r *RemoteList) LearnRemote(ownerVpnIp netip.Addr, remote netip.AddrPort) { r.Lock() defer r.Unlock() - if v4 := addr.IP.To4(); v4 != nil { - r.unlockedSetLearnedV4(ownerVpnIp, NewIp4AndPort(v4, uint32(addr.Port))) + if remote.Addr().Is4() { + r.unlockedSetLearnedV4(ownerVpnIp, netAddrToProtoV4AddrPort(remote.Addr(), remote.Port())) } else { - r.unlockedSetLearnedV6(ownerVpnIp, NewIp6AndPort(addr.IP, uint32(addr.Port))) + r.unlockedSetLearnedV6(ownerVpnIp, netAddrToProtoV6AddrPort(remote.Addr(), remote.Port())) } } @@ -293,9 +326,9 @@ func (r *RemoteList) CopyCache() *CacheMap { c := cm[vpnIp] if c == nil { c = &Cache{ - Learned: make([]*udp.Addr, 0), - Reported: make([]*udp.Addr, 0), - Relay: make([]*net.IP, 0), + Learned: make([]netip.AddrPort, 0), + Reported: make([]netip.AddrPort, 0), + Relay: make([]netip.Addr, 0), } cm[vpnIp] = c } @@ -307,28 +340,33 @@ func (r *RemoteList) CopyCache() *CacheMap { if mc.v4 != nil { if mc.v4.learned != nil { - c.Learned = append(c.Learned, NewUDPAddrFromLH4(mc.v4.learned)) + c.Learned = append(c.Learned, protoV4AddrPortToNetAddrPort(mc.v4.learned)) } for _, a := range mc.v4.reported { - c.Reported = append(c.Reported, NewUDPAddrFromLH4(a)) + if a == nil { + continue + } + c.Reported = append(c.Reported, protoV4AddrPortToNetAddrPort(a)) } } if mc.v6 != nil { if mc.v6.learned != nil { - c.Learned = append(c.Learned, NewUDPAddrFromLH6(mc.v6.learned)) + c.Learned = append(c.Learned, protoV6AddrPortToNetAddrPort(mc.v6.learned)) } for _, a := range mc.v6.reported { - c.Reported = append(c.Reported, NewUDPAddrFromLH6(a)) + if a == nil { + continue + } + c.Reported = append(c.Reported, protoV6AddrPortToNetAddrPort(a)) } } if mc.relay != nil { for _, a := range mc.relay.relay { - nip := iputil.VpnIp(a).ToIP() - c.Relay = append(c.Relay, &nip) + c.Relay = append(c.Relay, a) } } } @@ -337,38 +375,47 @@ func (r *RemoteList) CopyCache() *CacheMap { } // BlockRemote locks and records the address as bad, it will be excluded from the deduplicated address list -func (r *RemoteList) BlockRemote(bad *udp.Addr) { - if bad == nil { - // relays can have nil udp Addrs +func (r *RemoteList) BlockRemote(bad ViaSender) { + if bad.IsRelayed { return } + r.Lock() defer r.Unlock() // Check if we already blocked this addr - if r.unlockedIsBad(bad) { + if r.unlockedIsBad(bad.UdpAddr) { return } // We copy here because we are taking something else's memory and we can't trust everything - r.badRemotes = append(r.badRemotes, bad.Copy()) + r.badRemotes = append(r.badRemotes, bad.UdpAddr) // Mark the next interaction must recollect/dedupe r.shouldRebuild = true } // CopyBlockedRemotes locks and makes a deep copy of the blocked remotes list -func (r *RemoteList) CopyBlockedRemotes() []*udp.Addr { +func (r *RemoteList) CopyBlockedRemotes() []netip.AddrPort { r.RLock() defer r.RUnlock() - c := make([]*udp.Addr, len(r.badRemotes)) + c := make([]netip.AddrPort, len(r.badRemotes)) for i, v := range r.badRemotes { - c[i] = v.Copy() + c[i] = v } return c } +// RefreshFromHandshake locks and updates the RemoteList to account for data learned upon a completed handshake +func (r *RemoteList) RefreshFromHandshake(vpnAddrs []netip.Addr) { + r.Lock() + r.badRemotes = nil + r.vpnAddrs = make([]netip.Addr, len(vpnAddrs)) + copy(r.vpnAddrs, vpnAddrs) + r.Unlock() +} + // ResetBlockedRemotes locks and clears the blocked remotes list func (r *RemoteList) ResetBlockedRemotes() { r.Lock() @@ -378,12 +425,11 @@ func (r *RemoteList) ResetBlockedRemotes() { // Rebuild locks and generates the deduplicated address list only if there is work to be done // There is generally no reason to call this directly but it is safe to do so -func (r *RemoteList) Rebuild(preferredRanges []*net.IPNet) { +func (r *RemoteList) Rebuild(preferredRanges []netip.Prefix) { r.Lock() defer r.Unlock() // Only rebuild if the cache changed - //TODO: shouldRebuild is probably pointless as we don't check for actual change when lighthouse updates come in if r.shouldRebuild { r.unlockedCollect() r.shouldRebuild = false @@ -394,25 +440,20 @@ func (r *RemoteList) Rebuild(preferredRanges []*net.IPNet) { } // unlockedIsBad assumes you have the write lock and checks if the remote matches any entry in the blocked address list -func (r *RemoteList) unlockedIsBad(remote *udp.Addr) bool { - for _, v := range r.badRemotes { - if v.Equals(remote) { - return true - } - } - return false +func (r *RemoteList) unlockedIsBad(remote netip.AddrPort) bool { + return slices.Contains(r.badRemotes, remote) } // unlockedSetLearnedV4 assumes you have the write lock and sets the current learned address for this owner and marks the // deduplicated address list as dirty -func (r *RemoteList) unlockedSetLearnedV4(ownerVpnIp iputil.VpnIp, to *Ip4AndPort) { +func (r *RemoteList) unlockedSetLearnedV4(ownerVpnIp netip.Addr, to *V4AddrPort) { r.shouldRebuild = true r.unlockedGetOrMakeV4(ownerVpnIp).learned = to } // unlockedSetV4 assumes you have the write lock and resets the reported list of ips for this owner to the list provided // and marks the deduplicated address list as dirty -func (r *RemoteList) unlockedSetV4(ownerVpnIp iputil.VpnIp, vpnIp iputil.VpnIp, to []*Ip4AndPort, check checkFuncV4) { +func (r *RemoteList) unlockedSetV4(ownerVpnIp, vpnIp netip.Addr, to []*V4AddrPort, check checkFuncV4) { r.shouldRebuild = true c := r.unlockedGetOrMakeV4(ownerVpnIp) @@ -427,7 +468,7 @@ func (r *RemoteList) unlockedSetV4(ownerVpnIp iputil.VpnIp, vpnIp iputil.VpnIp, } } -func (r *RemoteList) unlockedSetRelay(ownerVpnIp iputil.VpnIp, vpnIp iputil.VpnIp, to []uint32) { +func (r *RemoteList) unlockedSetRelay(ownerVpnIp netip.Addr, to []netip.Addr) { r.shouldRebuild = true c := r.unlockedGetOrMakeRelay(ownerVpnIp) @@ -440,12 +481,12 @@ func (r *RemoteList) unlockedSetRelay(ownerVpnIp iputil.VpnIp, vpnIp iputil.VpnI // unlockedPrependV4 assumes you have the write lock and prepends the address in the reported list for this owner // This is only useful for establishing static hosts -func (r *RemoteList) unlockedPrependV4(ownerVpnIp iputil.VpnIp, to *Ip4AndPort) { +func (r *RemoteList) unlockedPrependV4(ownerVpnIp netip.Addr, to *V4AddrPort) { r.shouldRebuild = true c := r.unlockedGetOrMakeV4(ownerVpnIp) // We are doing the easy append because this is rarely called - c.reported = append([]*Ip4AndPort{to}, c.reported...) + c.reported = append([]*V4AddrPort{to}, c.reported...) if len(c.reported) > MaxRemotes { c.reported = c.reported[:MaxRemotes] } @@ -453,14 +494,14 @@ func (r *RemoteList) unlockedPrependV4(ownerVpnIp iputil.VpnIp, to *Ip4AndPort) // unlockedSetLearnedV6 assumes you have the write lock and sets the current learned address for this owner and marks the // deduplicated address list as dirty -func (r *RemoteList) unlockedSetLearnedV6(ownerVpnIp iputil.VpnIp, to *Ip6AndPort) { +func (r *RemoteList) unlockedSetLearnedV6(ownerVpnIp netip.Addr, to *V6AddrPort) { r.shouldRebuild = true r.unlockedGetOrMakeV6(ownerVpnIp).learned = to } // unlockedSetV6 assumes you have the write lock and resets the reported list of ips for this owner to the list provided // and marks the deduplicated address list as dirty -func (r *RemoteList) unlockedSetV6(ownerVpnIp iputil.VpnIp, vpnIp iputil.VpnIp, to []*Ip6AndPort, check checkFuncV6) { +func (r *RemoteList) unlockedSetV6(ownerVpnIp, vpnIp netip.Addr, to []*V6AddrPort, check checkFuncV6) { r.shouldRebuild = true c := r.unlockedGetOrMakeV6(ownerVpnIp) @@ -477,18 +518,18 @@ func (r *RemoteList) unlockedSetV6(ownerVpnIp iputil.VpnIp, vpnIp iputil.VpnIp, // unlockedPrependV6 assumes you have the write lock and prepends the address in the reported list for this owner // This is only useful for establishing static hosts -func (r *RemoteList) unlockedPrependV6(ownerVpnIp iputil.VpnIp, to *Ip6AndPort) { +func (r *RemoteList) unlockedPrependV6(ownerVpnIp netip.Addr, to *V6AddrPort) { r.shouldRebuild = true c := r.unlockedGetOrMakeV6(ownerVpnIp) // We are doing the easy append because this is rarely called - c.reported = append([]*Ip6AndPort{to}, c.reported...) + c.reported = append([]*V6AddrPort{to}, c.reported...) if len(c.reported) > MaxRemotes { c.reported = c.reported[:MaxRemotes] } } -func (r *RemoteList) unlockedGetOrMakeRelay(ownerVpnIp iputil.VpnIp) *cacheRelay { +func (r *RemoteList) unlockedGetOrMakeRelay(ownerVpnIp netip.Addr) *cacheRelay { am := r.cache[ownerVpnIp] if am == nil { am = &cache{} @@ -503,7 +544,7 @@ func (r *RemoteList) unlockedGetOrMakeRelay(ownerVpnIp iputil.VpnIp) *cacheRelay // unlockedGetOrMakeV4 assumes you have the write lock and builds the cache and owner entry. Only the v4 pointer is established. // The caller must dirty the learned address cache if required -func (r *RemoteList) unlockedGetOrMakeV4(ownerVpnIp iputil.VpnIp) *cacheV4 { +func (r *RemoteList) unlockedGetOrMakeV4(ownerVpnIp netip.Addr) *cacheV4 { am := r.cache[ownerVpnIp] if am == nil { am = &cache{} @@ -518,7 +559,7 @@ func (r *RemoteList) unlockedGetOrMakeV4(ownerVpnIp iputil.VpnIp) *cacheV4 { // unlockedGetOrMakeV6 assumes you have the write lock and builds the cache and owner entry. Only the v6 pointer is established. // The caller must dirty the learned address cache if required -func (r *RemoteList) unlockedGetOrMakeV6(ownerVpnIp iputil.VpnIp) *cacheV6 { +func (r *RemoteList) unlockedGetOrMakeV6(ownerVpnIp netip.Addr) *cacheV6 { am := r.cache[ownerVpnIp] if am == nil { am = &cache{} @@ -540,14 +581,17 @@ func (r *RemoteList) unlockedCollect() { for _, c := range r.cache { if c.v4 != nil { if c.v4.learned != nil { - u := NewUDPAddrFromLH4(c.v4.learned) + u := protoV4AddrPortToNetAddrPort(c.v4.learned) if !r.unlockedIsBad(u) { addrs = append(addrs, u) } } for _, v := range c.v4.reported { - u := NewUDPAddrFromLH4(v) + if v == nil { + continue + } + u := protoV4AddrPortToNetAddrPort(v) if !r.unlockedIsBad(u) { addrs = append(addrs, u) } @@ -556,14 +600,17 @@ func (r *RemoteList) unlockedCollect() { if c.v6 != nil { if c.v6.learned != nil { - u := NewUDPAddrFromLH6(c.v6.learned) + u := protoV6AddrPortToNetAddrPort(c.v6.learned) if !r.unlockedIsBad(u) { addrs = append(addrs, u) } } for _, v := range c.v6.reported { - u := NewUDPAddrFromLH6(v) + if v == nil { + continue + } + u := protoV6AddrPortToNetAddrPort(v) if !r.unlockedIsBad(u) { addrs = append(addrs, u) } @@ -572,20 +619,17 @@ func (r *RemoteList) unlockedCollect() { if c.relay != nil { for _, v := range c.relay.relay { - ip := iputil.VpnIp(v) - relays = append(relays, &ip) + relays = append(relays, v) } } } - dnsAddrs := r.hr.GetIPs() + dnsAddrs := r.hr.GetAddrs() for _, addr := range dnsAddrs { - if r.shouldAdd == nil || r.shouldAdd(addr.Addr()) { - v6 := addr.Addr().As16() - addrs = append(addrs, &udp.Addr{ - IP: v6[:], - Port: addr.Port(), - }) + if r.shouldAdd == nil || r.shouldAdd(r.vpnAddrs, addr.Addr()) { + if !r.unlockedIsBad(addr) { + addrs = append(addrs, addr) + } } } @@ -595,7 +639,22 @@ func (r *RemoteList) unlockedCollect() { } // unlockedSort assumes you have the write lock and performs the deduping and sorting of the address list -func (r *RemoteList) unlockedSort(preferredRanges []*net.IPNet) { +func (r *RemoteList) unlockedSort(preferredRanges []netip.Prefix) { + // Use a map to deduplicate any relay addresses + dedupedRelays := map[netip.Addr]struct{}{} + for _, relay := range r.relays { + dedupedRelays[relay] = struct{}{} + } + r.relays = r.relays[:0] + for relay := range dedupedRelays { + r.relays = append(r.relays, relay) + } + // Put them in a somewhat consistent order after de-duplication + slices.SortFunc(r.relays, func(a, b netip.Addr) int { + return a.Compare(b) + }) + + // Now the addrs n := len(r.addrs) if n < 2 { return @@ -606,8 +665,8 @@ func (r *RemoteList) unlockedSort(preferredRanges []*net.IPNet) { b := r.addrs[j] // Preferred addresses first - aPref := isPreferred(a.IP, preferredRanges) - bPref := isPreferred(b.IP, preferredRanges) + aPref := isPreferred(a.Addr(), preferredRanges) + bPref := isPreferred(b.Addr(), preferredRanges) switch { case aPref && !bPref: // If i is preferred and j is not, i is less than j @@ -622,21 +681,21 @@ func (r *RemoteList) unlockedSort(preferredRanges []*net.IPNet) { } // ipv6 addresses 2nd - a4 := a.IP.To4() - b4 := b.IP.To4() + a4 := a.Addr().Is4() + b4 := b.Addr().Is4() switch { - case a4 == nil && b4 != nil: + case a4 == false && b4 == true: // If i is v6 and j is v4, i is less than j return true - case a4 != nil && b4 == nil: + case a4 == true && b4 == false: // If j is v6 and i is v4, i is not less than j return false - case a4 != nil && b4 != nil: - // Special case for ipv4, a4 and b4 are not nil - aPrivate := isPrivateIP(a4) - bPrivate := isPrivateIP(b4) + case a4 == true && b4 == true: + // i and j are both ipv4 + aPrivate := a.Addr().IsPrivate() + bPrivate := b.Addr().IsPrivate() switch { case !aPrivate && bPrivate: // If i is a public ip (not private) and j is a private ip, i is less then j @@ -655,10 +714,10 @@ func (r *RemoteList) unlockedSort(preferredRanges []*net.IPNet) { } // lexical order of ips 3rd - c := bytes.Compare(a.IP, b.IP) + c := a.Addr().Compare(b.Addr()) if c == 0 { // Ips are the same, Lexical order of ports 4th - return a.Port < b.Port + return a.Port() < b.Port() } // Ip wasn't the same @@ -671,7 +730,7 @@ func (r *RemoteList) unlockedSort(preferredRanges []*net.IPNet) { // Deduplicate a, b := 0, 1 for b < n { - if !r.addrs[a].Equals(r.addrs[b]) { + if r.addrs[a] != r.addrs[b] { a++ if a != b { r.addrs[a], r.addrs[b] = r.addrs[b], r.addrs[a] @@ -693,8 +752,7 @@ func minInt(a, b int) int { } // isPreferred returns true of the ip is contained in the preferredRanges list -func isPreferred(ip net.IP, preferredRanges []*net.IPNet) bool { - //TODO: this would be better in a CIDR6Tree +func isPreferred(ip netip.Addr, preferredRanges []netip.Prefix) bool { for _, p := range preferredRanges { if p.Contains(ip) { return true @@ -702,14 +760,3 @@ func isPreferred(ip net.IP, preferredRanges []*net.IPNet) bool { } return false } - -var _, private24BitBlock, _ = net.ParseCIDR("10.0.0.0/8") -var _, private20BitBlock, _ = net.ParseCIDR("172.16.0.0/12") -var _, private16BitBlock, _ = net.ParseCIDR("192.168.0.0/16") - -// isPrivateIP returns true if the ip is contained by a rfc 1918 private range -func isPrivateIP(ip net.IP) bool { - //TODO: another great cidrtree option - //TODO: Private for ipv6 or just let it ride? - return private24BitBlock.Contains(ip) || private20BitBlock.Contains(ip) || private16BitBlock.Contains(ip) -} diff --git a/remote_list_test.go b/remote_list_test.go index 49aa1719..0b5b7d5d 100644 --- a/remote_list_test.go +++ b/remote_list_test.go @@ -1,47 +1,71 @@ package nebula import ( - "net" + "encoding/binary" + "net/netip" "testing" - "github.com/slackhq/nebula/iputil" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// trackedHostnameResults builds a *hostnamesResults with a known cancel function and a +// pre-populated ips map so tests can assert cancellation and verify previously-resolved +// IPs survive a cancel without spinning up a real DNS resolver. +func trackedHostnameResults(cancelFn func(), addrs ...string) *hostnamesResults { + hr := &hostnamesResults{cancelFn: cancelFn} + ips := map[netip.AddrPort]struct{}{} + for _, a := range addrs { + ips[netip.MustParseAddrPort(a)] = struct{}{} + } + hr.ips.Store(&ips) + return hr +} + func TestRemoteList_Rebuild(t *testing.T) { - rl := NewRemoteList(nil) + rl := NewRemoteList([]netip.Addr{netip.MustParseAddr("0.0.0.0")}, nil) rl.unlockedSetV4( - 0, - 0, - []*Ip4AndPort{ - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("70.199.182.92"))), Port: 1475}, // this is duped - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.17.0.182"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.17.1.1"))), Port: 10101}, // this is duped - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.18.0.1"))), Port: 10101}, // this is duped - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.18.0.1"))), Port: 10101}, // this is a dupe - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.19.0.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.31.0.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.17.1.1"))), Port: 10101}, // this is a dupe - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("70.199.182.92"))), Port: 1476}, // almost dupe of 0 with a diff port - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("70.199.182.92"))), Port: 1475}, // this is a dupe + netip.MustParseAddr("0.0.0.0"), + netip.MustParseAddr("0.0.0.0"), + []*V4AddrPort{ + newIp4AndPortFromString("70.199.182.92:1475"), // this is duped + newIp4AndPortFromString("172.17.0.182:10101"), + newIp4AndPortFromString("172.17.1.1:10101"), // this is duped + newIp4AndPortFromString("172.18.0.1:10101"), // this is duped + newIp4AndPortFromString("172.18.0.1:10101"), // this is a dupe + newIp4AndPortFromString("172.19.0.1:10101"), + newIp4AndPortFromString("172.31.0.1:10101"), + newIp4AndPortFromString("172.17.1.1:10101"), // this is a dupe + newIp4AndPortFromString("70.199.182.92:1476"), // almost dupe of 0 with a diff port + newIp4AndPortFromString("70.199.182.92:1475"), // this is a dupe }, - func(iputil.VpnIp, *Ip4AndPort) bool { return true }, + func(netip.Addr, *V4AddrPort) bool { return true }, ) rl.unlockedSetV6( - 1, - 1, - []*Ip6AndPort{ - NewIp6AndPort(net.ParseIP("1::1"), 1), // this is duped - NewIp6AndPort(net.ParseIP("1::1"), 2), // almost dupe of 0 with a diff port, also gets duped - NewIp6AndPort(net.ParseIP("1:100::1"), 1), - NewIp6AndPort(net.ParseIP("1::1"), 1), // this is a dupe - NewIp6AndPort(net.ParseIP("1::1"), 2), // this is a dupe + netip.MustParseAddr("0.0.0.1"), + netip.MustParseAddr("0.0.0.1"), + []*V6AddrPort{ + newIp6AndPortFromString("[1::1]:1"), // this is duped + newIp6AndPortFromString("[1::1]:2"), // almost dupe of 0 with a diff port, also gets duped + newIp6AndPortFromString("[1:100::1]:1"), + newIp6AndPortFromString("[1::1]:1"), // this is a dupe + newIp6AndPortFromString("[1::1]:2"), // this is a dupe }, - func(iputil.VpnIp, *Ip6AndPort) bool { return true }, + func(netip.Addr, *V6AddrPort) bool { return true }, ) - rl.Rebuild([]*net.IPNet{}) + rl.unlockedSetRelay( + netip.MustParseAddr("0.0.0.1"), + []netip.Addr{ + netip.MustParseAddr("1::1"), + netip.MustParseAddr("1.2.3.4"), + netip.MustParseAddr("1.2.3.4"), + netip.MustParseAddr("1::1"), + }, + ) + + rl.Rebuild([]netip.Prefix{}) assert.Len(t, rl.addrs, 10, "addrs contains too many entries") // ipv6 first, sorted lexically within @@ -59,9 +83,7 @@ func TestRemoteList_Rebuild(t *testing.T) { assert.Equal(t, "172.31.0.1:10101", rl.addrs[9].String()) // Now ensure we can hoist ipv4 up - _, ipNet, err := net.ParseCIDR("0.0.0.0/0") - assert.NoError(t, err) - rl.Rebuild([]*net.IPNet{ipNet}) + rl.Rebuild([]netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}) assert.Len(t, rl.addrs, 10, "addrs contains too many entries") // ipv4 first, public then private, lexically within them @@ -78,10 +100,13 @@ func TestRemoteList_Rebuild(t *testing.T) { assert.Equal(t, "[1::1]:2", rl.addrs[8].String()) assert.Equal(t, "[1:100::1]:1", rl.addrs[9].String()) + // assert relay deduplicated + assert.Len(t, rl.relays, 2) + assert.Equal(t, "1.2.3.4", rl.relays[0].String()) + assert.Equal(t, "1::1", rl.relays[1].String()) + // Ensure we can hoist a specific ipv4 range over anything else - _, ipNet, err = net.ParseCIDR("172.17.0.0/16") - assert.NoError(t, err) - rl.Rebuild([]*net.IPNet{ipNet}) + rl.Rebuild([]netip.Prefix{netip.MustParsePrefix("172.17.0.0/16")}) assert.Len(t, rl.addrs, 10, "addrs contains too many entries") // Preferred ipv4 first @@ -101,135 +126,223 @@ func TestRemoteList_Rebuild(t *testing.T) { assert.Equal(t, "172.31.0.1:10101", rl.addrs[9].String()) } +func TestRemoteList_ResetForOwner(t *testing.T) { + ourselves := netip.MustParseAddr("10.0.0.1") + otherOwner := netip.MustParseAddr("10.0.0.2") + vpnAddr := netip.MustParseAddr("10.0.0.99") + + rl := NewRemoteList([]netip.Addr{vpnAddr}, nil) + rl.unlockedSetV4(ourselves, vpnAddr, + []*V4AddrPort{newIp4AndPortFromString("1.1.1.1:4242")}, + func(netip.Addr, *V4AddrPort) bool { return true }, + ) + rl.unlockedSetV6(ourselves, vpnAddr, + []*V6AddrPort{newIp6AndPortFromString("[1::1]:4242")}, + func(netip.Addr, *V6AddrPort) bool { return true }, + ) + rl.unlockedSetV4(otherOwner, vpnAddr, + []*V4AddrPort{newIp4AndPortFromString("2.2.2.2:4242")}, + func(netip.Addr, *V4AddrPort) bool { return true }, + ) + + canceled := 0 + hr := trackedHostnameResults(func() { canceled++ }, "3.3.3.3:4242") + rl.Lock() + rl.unlockedSetHostnamesResults(hr) + rl.Unlock() + + rl.ResetForOwner(ourselves) + + rl.RLock() + defer rl.RUnlock() + assert.Empty(t, rl.cache[ourselves].v4.reported, "our v4 reported should be cleared") + assert.Empty(t, rl.cache[ourselves].v6.reported, "our v6 reported should be cleared") + assert.Len(t, rl.cache[otherOwner].v4.reported, 1, "other owner's contribution must be preserved") + assert.Equal(t, "2.2.2.2:4242", protoV4AddrPortToNetAddrPort(rl.cache[otherOwner].v4.reported[0]).String()) + assert.Equal(t, 1, canceled, "DNS resolution goroutine should be canceled") + assert.Same(t, hr, rl.hr, "hostnamesResults must be preserved so DNS-resolved IPs keep feeding addrs until replaced") + assert.NotEmpty(t, rl.hr.GetAddrs(), "previously-resolved IPs should still be readable after cancel") + assert.True(t, rl.shouldRebuild, "shouldRebuild must be set so the next Rebuild recomputes addrs") +} + +func TestRemoteList_ResetForOwner_NoEntry(t *testing.T) { + // An owner with no cache entry must not panic; shouldRebuild is still set and any + // existing hostnamesResults is canceled. + rl := NewRemoteList([]netip.Addr{netip.MustParseAddr("10.0.0.99")}, nil) + canceled := 0 + rl.Lock() + rl.unlockedSetHostnamesResults(trackedHostnameResults(func() { canceled++ }, "3.3.3.3:4242")) + rl.Unlock() + + rl.ResetForOwner(netip.MustParseAddr("10.0.0.1")) + + rl.RLock() + defer rl.RUnlock() + assert.Equal(t, 1, canceled) + assert.True(t, rl.shouldRebuild) +} + +func TestRemoteList_ClearHostnameResults(t *testing.T) { + rl := NewRemoteList([]netip.Addr{netip.MustParseAddr("10.0.0.99")}, nil) + + canceled := 0 + hr := trackedHostnameResults(func() { canceled++ }, "3.3.3.3:4242") + rl.Lock() + rl.unlockedSetHostnamesResults(hr) + rl.Unlock() + require.NotEmpty(t, hr.GetAddrs(), "hostnamesResults should have its fastrack IPs populated") + + rl.ClearHostnameResults() + + rl.RLock() + defer rl.RUnlock() + assert.Equal(t, 1, canceled, "DNS resolution goroutine should be canceled") + assert.Nil(t, rl.hr, "hostnamesResults should be dropped") + assert.True(t, rl.shouldRebuild) +} + func BenchmarkFullRebuild(b *testing.B) { - rl := NewRemoteList(nil) + rl := NewRemoteList([]netip.Addr{netip.MustParseAddr("0.0.0.0")}, nil) rl.unlockedSetV4( - 0, - 0, - []*Ip4AndPort{ - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("70.199.182.92"))), Port: 1475}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.17.0.182"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.17.1.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.18.0.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.19.0.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.31.0.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.17.1.1"))), Port: 10101}, // this is a dupe - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("70.199.182.92"))), Port: 1476}, // dupe of 0 with a diff port + netip.MustParseAddr("0.0.0.0"), + netip.MustParseAddr("0.0.0.0"), + []*V4AddrPort{ + newIp4AndPortFromString("70.199.182.92:1475"), + newIp4AndPortFromString("172.17.0.182:10101"), + newIp4AndPortFromString("172.17.1.1:10101"), + newIp4AndPortFromString("172.18.0.1:10101"), + newIp4AndPortFromString("172.19.0.1:10101"), + newIp4AndPortFromString("172.31.0.1:10101"), + newIp4AndPortFromString("172.17.1.1:10101"), // this is a dupe + newIp4AndPortFromString("70.199.182.92:1476"), // dupe of 0 with a diff port }, - func(iputil.VpnIp, *Ip4AndPort) bool { return true }, + func(netip.Addr, *V4AddrPort) bool { return true }, ) rl.unlockedSetV6( - 0, - 0, - []*Ip6AndPort{ - NewIp6AndPort(net.ParseIP("1::1"), 1), - NewIp6AndPort(net.ParseIP("1::1"), 2), // dupe of 0 with a diff port - NewIp6AndPort(net.ParseIP("1:100::1"), 1), - NewIp6AndPort(net.ParseIP("1::1"), 1), // this is a dupe + netip.MustParseAddr("0.0.0.0"), + netip.MustParseAddr("0.0.0.0"), + []*V6AddrPort{ + newIp6AndPortFromString("[1::1]:1"), + newIp6AndPortFromString("[1::1]:2"), // dupe of 0 with a diff port + newIp6AndPortFromString("[1:100::1]:1"), + newIp6AndPortFromString("[1::1]:1"), // this is a dupe }, - func(iputil.VpnIp, *Ip6AndPort) bool { return true }, + func(netip.Addr, *V6AddrPort) bool { return true }, ) b.Run("no preferred", func(b *testing.B) { for i := 0; i < b.N; i++ { rl.shouldRebuild = true - rl.Rebuild([]*net.IPNet{}) + rl.Rebuild([]netip.Prefix{}) } }) - _, ipNet, err := net.ParseCIDR("172.17.0.0/16") - assert.NoError(b, err) + ipNet1 := netip.MustParsePrefix("172.17.0.0/16") b.Run("1 preferred", func(b *testing.B) { for i := 0; i < b.N; i++ { rl.shouldRebuild = true - rl.Rebuild([]*net.IPNet{ipNet}) + rl.Rebuild([]netip.Prefix{ipNet1}) } }) - _, ipNet2, err := net.ParseCIDR("70.0.0.0/8") - assert.NoError(b, err) + ipNet2 := netip.MustParsePrefix("70.0.0.0/8") b.Run("2 preferred", func(b *testing.B) { for i := 0; i < b.N; i++ { rl.shouldRebuild = true - rl.Rebuild([]*net.IPNet{ipNet, ipNet2}) + rl.Rebuild([]netip.Prefix{ipNet2}) } }) - _, ipNet3, err := net.ParseCIDR("0.0.0.0/0") - assert.NoError(b, err) + ipNet3 := netip.MustParsePrefix("0.0.0.0/0") b.Run("3 preferred", func(b *testing.B) { for i := 0; i < b.N; i++ { rl.shouldRebuild = true - rl.Rebuild([]*net.IPNet{ipNet, ipNet2, ipNet3}) + rl.Rebuild([]netip.Prefix{ipNet1, ipNet2, ipNet3}) } }) } func BenchmarkSortRebuild(b *testing.B) { - rl := NewRemoteList(nil) + rl := NewRemoteList([]netip.Addr{netip.MustParseAddr("0.0.0.0")}, nil) rl.unlockedSetV4( - 0, - 0, - []*Ip4AndPort{ - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("70.199.182.92"))), Port: 1475}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.17.0.182"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.17.1.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.18.0.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.19.0.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.31.0.1"))), Port: 10101}, - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("172.17.1.1"))), Port: 10101}, // this is a dupe - {Ip: uint32(iputil.Ip2VpnIp(net.ParseIP("70.199.182.92"))), Port: 1476}, // dupe of 0 with a diff port + netip.MustParseAddr("0.0.0.0"), + netip.MustParseAddr("0.0.0.0"), + []*V4AddrPort{ + newIp4AndPortFromString("70.199.182.92:1475"), + newIp4AndPortFromString("172.17.0.182:10101"), + newIp4AndPortFromString("172.17.1.1:10101"), + newIp4AndPortFromString("172.18.0.1:10101"), + newIp4AndPortFromString("172.19.0.1:10101"), + newIp4AndPortFromString("172.31.0.1:10101"), + newIp4AndPortFromString("172.17.1.1:10101"), // this is a dupe + newIp4AndPortFromString("70.199.182.92:1476"), // dupe of 0 with a diff port }, - func(iputil.VpnIp, *Ip4AndPort) bool { return true }, + func(netip.Addr, *V4AddrPort) bool { return true }, ) rl.unlockedSetV6( - 0, - 0, - []*Ip6AndPort{ - NewIp6AndPort(net.ParseIP("1::1"), 1), - NewIp6AndPort(net.ParseIP("1::1"), 2), // dupe of 0 with a diff port - NewIp6AndPort(net.ParseIP("1:100::1"), 1), - NewIp6AndPort(net.ParseIP("1::1"), 1), // this is a dupe + netip.MustParseAddr("0.0.0.0"), + netip.MustParseAddr("0.0.0.0"), + []*V6AddrPort{ + newIp6AndPortFromString("[1::1]:1"), + newIp6AndPortFromString("[1::1]:2"), // dupe of 0 with a diff port + newIp6AndPortFromString("[1:100::1]:1"), + newIp6AndPortFromString("[1::1]:1"), // this is a dupe }, - func(iputil.VpnIp, *Ip6AndPort) bool { return true }, + func(netip.Addr, *V6AddrPort) bool { return true }, ) b.Run("no preferred", func(b *testing.B) { for i := 0; i < b.N; i++ { rl.shouldRebuild = true - rl.Rebuild([]*net.IPNet{}) + rl.Rebuild([]netip.Prefix{}) } }) - _, ipNet, err := net.ParseCIDR("172.17.0.0/16") - rl.Rebuild([]*net.IPNet{ipNet}) + ipNet1 := netip.MustParsePrefix("172.17.0.0/16") + rl.Rebuild([]netip.Prefix{ipNet1}) - assert.NoError(b, err) b.Run("1 preferred", func(b *testing.B) { for i := 0; i < b.N; i++ { - rl.Rebuild([]*net.IPNet{ipNet}) + rl.Rebuild([]netip.Prefix{ipNet1}) } }) - _, ipNet2, err := net.ParseCIDR("70.0.0.0/8") - rl.Rebuild([]*net.IPNet{ipNet, ipNet2}) + ipNet2 := netip.MustParsePrefix("70.0.0.0/8") + rl.Rebuild([]netip.Prefix{ipNet1, ipNet2}) - assert.NoError(b, err) b.Run("2 preferred", func(b *testing.B) { for i := 0; i < b.N; i++ { - rl.Rebuild([]*net.IPNet{ipNet, ipNet2}) + rl.Rebuild([]netip.Prefix{ipNet1, ipNet2}) } }) - _, ipNet3, err := net.ParseCIDR("0.0.0.0/0") - rl.Rebuild([]*net.IPNet{ipNet, ipNet2, ipNet3}) + ipNet3 := netip.MustParsePrefix("0.0.0.0/0") + rl.Rebuild([]netip.Prefix{ipNet1, ipNet2, ipNet3}) - assert.NoError(b, err) b.Run("3 preferred", func(b *testing.B) { for i := 0; i < b.N; i++ { - rl.Rebuild([]*net.IPNet{ipNet, ipNet2, ipNet3}) + rl.Rebuild([]netip.Prefix{ipNet1, ipNet2, ipNet3}) } }) } + +func newIp4AndPortFromString(s string) *V4AddrPort { + a := netip.MustParseAddrPort(s) + v4Addr := a.Addr().As4() + return &V4AddrPort{ + Addr: binary.BigEndian.Uint32(v4Addr[:]), + Port: uint32(a.Port()), + } +} + +func newIp6AndPortFromString(s string) *V6AddrPort { + a := netip.MustParseAddrPort(s) + v6Addr := a.Addr().As16() + return &V6AddrPort{ + Hi: binary.BigEndian.Uint64(v6Addr[:8]), + Lo: binary.BigEndian.Uint64(v6Addr[8:]), + Port: uint32(a.Port()), + } +} diff --git a/routing/balance.go b/routing/balance.go new file mode 100644 index 00000000..6f524970 --- /dev/null +++ b/routing/balance.go @@ -0,0 +1,39 @@ +package routing + +import ( + "net/netip" + + "github.com/slackhq/nebula/firewall" +) + +// Hashes the packet source and destination port and always returns a positive integer +// Based on 'Prospecting for Hash Functions' +// - https://nullprogram.com/blog/2018/07/31/ +// - https://github.com/skeeto/hash-prospector +// [16 21f0aaad 15 d35a2d97 15] = 0.10760229515479501 +func hashPacket(p *firewall.Packet) int { + x := (uint32(p.LocalPort) << 16) | uint32(p.RemotePort) + x ^= x >> 16 + x *= 0x21f0aaad + x ^= x >> 15 + x *= 0xd35a2d97 + x ^= x >> 15 + + return int(x) & 0x7FFFFFFF +} + +// For this function to work correctly it requires that the buckets for the gateways have been calculated +// If the contract is violated balancing will not work properly and the second return value will return false +func BalancePacket(fwPacket *firewall.Packet, gateways []Gateway) (netip.Addr, bool) { + hash := hashPacket(fwPacket) + + for i := range gateways { + if hash <= gateways[i].BucketUpperBound() { + return gateways[i].Addr(), true + } + } + + // If you land here then the buckets for the gateways are not properly calculated + // Fallback to random routing and let the caller know + return gateways[hash%len(gateways)].Addr(), false +} diff --git a/routing/balance_test.go b/routing/balance_test.go new file mode 100644 index 00000000..bbfcb226 --- /dev/null +++ b/routing/balance_test.go @@ -0,0 +1,144 @@ +package routing + +import ( + "net/netip" + "testing" + + "github.com/slackhq/nebula/firewall" + "github.com/stretchr/testify/assert" +) + +func TestPacketsAreBalancedEqually(t *testing.T) { + + gateways := []Gateway{} + + gw1Addr := netip.MustParseAddr("1.0.0.1") + gw2Addr := netip.MustParseAddr("1.0.0.2") + gw3Addr := netip.MustParseAddr("1.0.0.3") + + gateways = append(gateways, NewGateway(gw1Addr, 1)) + gateways = append(gateways, NewGateway(gw2Addr, 1)) + gateways = append(gateways, NewGateway(gw3Addr, 1)) + + CalculateBucketsForGateways(gateways) + + gw1count := 0 + gw2count := 0 + gw3count := 0 + + iterationCount := uint16(65535) + for i := uint16(0); i < iterationCount; i++ { + packet := firewall.Packet{ + LocalAddr: netip.MustParseAddr("192.168.1.1"), + RemoteAddr: netip.MustParseAddr("10.0.0.1"), + LocalPort: i, + RemotePort: 65535 - i, + Protocol: 6, // TCP + Fragment: false, + } + + selectedGw, ok := BalancePacket(&packet, gateways) + assert.True(t, ok) + + switch selectedGw { + case gw1Addr: + gw1count += 1 + case gw2Addr: + gw2count += 1 + case gw3Addr: + gw3count += 1 + } + + } + + // Assert packets are balanced, allow variation of up to 100 packets per gateway + assert.InDeltaf(t, iterationCount/3, gw1count, 100, "Expected %d +/- 100, but got %d", iterationCount/3, gw1count) + assert.InDeltaf(t, iterationCount/3, gw2count, 100, "Expected %d +/- 100, but got %d", iterationCount/3, gw1count) + assert.InDeltaf(t, iterationCount/3, gw3count, 100, "Expected %d +/- 100, but got %d", iterationCount/3, gw1count) + +} + +func TestPacketsAreBalancedByPriority(t *testing.T) { + + gateways := []Gateway{} + + gw1Addr := netip.MustParseAddr("1.0.0.1") + gw2Addr := netip.MustParseAddr("1.0.0.2") + + gateways = append(gateways, NewGateway(gw1Addr, 10)) + gateways = append(gateways, NewGateway(gw2Addr, 5)) + + CalculateBucketsForGateways(gateways) + + gw1count := 0 + gw2count := 0 + + iterationCount := uint16(65535) + for i := uint16(0); i < iterationCount; i++ { + packet := firewall.Packet{ + LocalAddr: netip.MustParseAddr("192.168.1.1"), + RemoteAddr: netip.MustParseAddr("10.0.0.1"), + LocalPort: i, + RemotePort: 65535 - i, + Protocol: 6, // TCP + Fragment: false, + } + + selectedGw, ok := BalancePacket(&packet, gateways) + assert.True(t, ok) + + switch selectedGw { + case gw1Addr: + gw1count += 1 + case gw2Addr: + gw2count += 1 + } + + } + + iterationCountAsFloat := float32(iterationCount) + + assert.InDeltaf(t, iterationCountAsFloat*(2.0/3.0), gw1count, 100, "Expected %d +/- 100, but got %d", iterationCountAsFloat*(2.0/3.0), gw1count) + assert.InDeltaf(t, iterationCountAsFloat*(1.0/3.0), gw2count, 100, "Expected %d +/- 100, but got %d", iterationCountAsFloat*(1.0/3.0), gw2count) +} + +func TestBalancePacketDistributsRandomlyAndReturnsFalseIfBucketsNotCalculated(t *testing.T) { + gateways := []Gateway{} + + gw1Addr := netip.MustParseAddr("1.0.0.1") + gw2Addr := netip.MustParseAddr("1.0.0.2") + + gateways = append(gateways, NewGateway(gw1Addr, 10)) + gateways = append(gateways, NewGateway(gw2Addr, 5)) + + iterationCount := uint16(65535) + gw1count := 0 + gw2count := 0 + + for i := uint16(0); i < iterationCount; i++ { + packet := firewall.Packet{ + LocalAddr: netip.MustParseAddr("192.168.1.1"), + RemoteAddr: netip.MustParseAddr("10.0.0.1"), + LocalPort: i, + RemotePort: 65535 - i, + Protocol: 6, // TCP + Fragment: false, + } + + selectedGw, ok := BalancePacket(&packet, gateways) + assert.False(t, ok) + + switch selectedGw { + case gw1Addr: + gw1count += 1 + case gw2Addr: + gw2count += 1 + } + + } + + assert.Equal(t, int(iterationCount), (gw1count + gw2count)) + assert.NotEqual(t, 0, gw1count) + assert.NotEqual(t, 0, gw2count) + +} diff --git a/routing/gateway.go b/routing/gateway.go new file mode 100644 index 00000000..88cf0933 --- /dev/null +++ b/routing/gateway.go @@ -0,0 +1,70 @@ +package routing + +import ( + "fmt" + "net/netip" +) + +const ( + // Sentinel value + BucketNotCalculated = -1 +) + +type Gateways []Gateway + +func (g Gateways) String() string { + str := "" + for i, gw := range g { + str += gw.String() + if i < len(g)-1 { + str += ", " + } + } + return str +} + +type Gateway struct { + addr netip.Addr + weight int + bucketUpperBound int +} + +func NewGateway(addr netip.Addr, weight int) Gateway { + return Gateway{addr: addr, weight: weight, bucketUpperBound: BucketNotCalculated} +} + +func (g *Gateway) BucketUpperBound() int { + return g.bucketUpperBound +} + +func (g *Gateway) Addr() netip.Addr { + return g.addr +} + +func (g *Gateway) String() string { + return fmt.Sprintf("{addr: %s, weight: %d}", g.addr, g.weight) +} + +// Divide and round to nearest integer +func divideAndRound(v uint64, d uint64) uint64 { + var tmp uint64 = v + d/2 + return tmp / d +} + +// Implements Hash-Threshold mapping, equivalent to the implementation in the linux kernel. +// After this function returns each gateway will have a +// positive bucketUpperBound with a maximum value of 2147483647 (INT_MAX) +func CalculateBucketsForGateways(gateways []Gateway) { + + var totalWeight int = 0 + for i := range gateways { + totalWeight += gateways[i].weight + } + + var loopWeight int = 0 + for i := range gateways { + loopWeight += gateways[i].weight + gateways[i].bucketUpperBound = int(divideAndRound(uint64(loopWeight)<<31, uint64(totalWeight))) - 1 + } + +} diff --git a/routing/gateway_test.go b/routing/gateway_test.go new file mode 100644 index 00000000..8ae78f3c --- /dev/null +++ b/routing/gateway_test.go @@ -0,0 +1,34 @@ +package routing + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRebalance3_2Split(t *testing.T) { + gateways := []Gateway{} + + gateways = append(gateways, Gateway{addr: netip.Addr{}, weight: 10}) + gateways = append(gateways, Gateway{addr: netip.Addr{}, weight: 5}) + + CalculateBucketsForGateways(gateways) + + assert.Equal(t, 1431655764, gateways[0].bucketUpperBound) // INT_MAX/3*2 + assert.Equal(t, 2147483647, gateways[1].bucketUpperBound) // INT_MAX +} + +func TestRebalanceEqualSplit(t *testing.T) { + gateways := []Gateway{} + + gateways = append(gateways, Gateway{addr: netip.Addr{}, weight: 1}) + gateways = append(gateways, Gateway{addr: netip.Addr{}, weight: 1}) + gateways = append(gateways, Gateway{addr: netip.Addr{}, weight: 1}) + + CalculateBucketsForGateways(gateways) + + assert.Equal(t, 715827882, gateways[0].bucketUpperBound) // INT_MAX/3 + assert.Equal(t, 1431655764, gateways[1].bucketUpperBound) // INT_MAX/3*2 + assert.Equal(t, 2147483647, gateways[2].bucketUpperBound) // INT_MAX +} diff --git a/scheduler.go b/scheduler.go new file mode 100644 index 00000000..7733204a --- /dev/null +++ b/scheduler.go @@ -0,0 +1,84 @@ +package nebula + +import ( + "context" + "sync" + "time" +) + +// Scheduler is an allocation-conscious dispatch primitive for delayed work. +// Pending items are handed to time.AfterFunc, and ready items land on a worker +// channel for centralized dispatch in fire-time order. +// +// Pick a Scheduler when fire timing matters (exact deadlines, no bucketing) or when the scheduling +// rate is uneven enough that idle CPU matters. Each fire is a runtime-spawned goroutine running the callback before +// delivering to the worker, which is fine at sparse rates but adds up at line rate. +// +// Pick a TimerWheel when scheduling is high-rate and uniform: its O(1) insert, internal item cache, +// and bucket-batched dispatch are cheaper at scale. +// The caller drives the tick loop (Advance/Purge) and pays for fires at bucket boundaries rather than exact deadlines. +type Scheduler[T any] struct { + queue chan T + pool sync.Pool +} + +type schedItem[T any] struct { + val T + ctx context.Context + s *Scheduler[T] + timer *time.Timer + fire func() +} + +// NewScheduler builds a Scheduler whose worker channel is sized to queueSize. +// The buffer absorbs bursts of timers firing close together without +// blocking the runtime's callback goroutines on the worker. +func NewScheduler[T any](queueSize int) *Scheduler[T] { + s := &Scheduler[T]{ + queue: make(chan T, queueSize), + } + s.pool.New = func() any { + si := &schedItem[T]{s: s} + // fire is allocated exactly once per pool-resident item. + // The closure captures only `si`, which stays stable for the item's lifetime. + si.fire = func() { + select { + case si.s.queue <- si.val: + case <-si.ctx.Done(): + } + var zero T + si.val = zero + si.ctx = nil + si.s.pool.Put(si) + } + return si + } + return s +} + +// Schedule arranges item to be delivered to the worker after delay. +// The runtime's timer heap handles the wait, so the scheduler itself burns no CPU while idle. +// The callback observes ctx: if ctx is cancelled before the timer fires, the item is dropped instead of queued. +func (s *Scheduler[T]) Schedule(ctx context.Context, item T, delay time.Duration) { + si := s.pool.Get().(*schedItem[T]) + si.val = item + si.ctx = ctx + if si.timer == nil { + si.timer = time.AfterFunc(delay, si.fire) + } else { + si.timer.Reset(delay) + } +} + +// Run drains the worker queue, calling fn for each item. Returns when ctx is cancelled. +// Tests that want deterministic timing should drive the queue directly rather than going through Schedule + Run. +func (s *Scheduler[T]) Run(ctx context.Context, fn func(T)) { + for { + select { + case <-ctx.Done(): + return + case item := <-s.queue: + fn(item) + } + } +} diff --git a/scheduler_test.go b/scheduler_test.go new file mode 100644 index 00000000..085d523c --- /dev/null +++ b/scheduler_test.go @@ -0,0 +1,79 @@ +package nebula + +import ( + "context" + "testing" + "time" +) + +func TestScheduler_PooledReuse(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s := NewScheduler[int](16) + delivered := make(chan int, 256) + go s.Run(ctx, func(item int) { delivered <- item }) + + const N = 100 + for i := 0; i < N; i++ { + s.Schedule(ctx, i, time.Millisecond) + } + + deadline := time.After(2 * time.Second) + got := 0 + for got < N { + select { + case <-delivered: + got++ + case <-deadline: + t.Fatalf("only %d/%d items delivered", got, N) + } + } +} + +// BenchmarkScheduler_Schedule reports allocations per Schedule call. +// In steady state the Scheduler's sync.Pool means we should see zero allocs per op once the pool warms up. +func BenchmarkScheduler_Schedule(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s := NewScheduler[int](b.N) + go s.Run(ctx, func(int) {}) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Schedule(ctx, i, time.Microsecond) + } +} + +// BenchmarkBareAfterFunc is the comparison baseline. +// What we'd pay per Schedule if Punchy called time.AfterFunc directly without the pooled Scheduler. +// Allocates a *time.Timer plus a closure each call. +func BenchmarkBareAfterFunc(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + queue := make(chan int, b.N) + go func() { + for { + select { + case <-ctx.Done(): + return + case <-queue: + } + } + }() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + i := i + time.AfterFunc(time.Microsecond, func() { + select { + case queue <- i: + case <-ctx.Done(): + } + }) + } +} diff --git a/service/service.go b/service/service.go index 66ce8649..6610800d 100644 --- a/service/service.go +++ b/service/service.go @@ -8,16 +8,14 @@ import ( "log" "math" "net" - "os" + "net/netip" "strings" "sync" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula" - "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/overlay" "golang.org/x/sync/errgroup" - "gvisor.dev/gvisor/pkg/bufferv2" + "gvisor.dev/gvisor/pkg/buffer" "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" "gvisor.dev/gvisor/pkg/tcpip/header" @@ -45,15 +43,24 @@ type Service struct { } } -func New(config *config.C) (*Service, error) { - logger := logrus.New() - logger.Out = os.Stdout +func New(control *nebula.Control) (_ *Service, reterr error) { + // Check this before Start so a failure doesn't leave a running nebula + device, ok := control.Device().(*overlay.UserDevice) + if !ok { + return nil, errors.New("must be using user device") + } - control, err := nebula.Main(config, false, "custom-app", logger, overlay.NewUserDeviceFromConfig) + err := control.Start() if err != nil { return nil, err } - control.Start() + + // Anything that fails after a successful Start must tear nebula back down + defer func() { + if reterr != nil { + control.Stop() + } + }() ctx := control.Context() eg, ctx := errgroup.WithContext(ctx) @@ -63,11 +70,6 @@ func New(config *config.C) (*Service, error) { } s.mu.listeners = map[uint16]*tcpListener{} - device, ok := control.Device().(*overlay.UserDevice) - if !ok { - return nil, errors.New("must be using user device") - } - s.ipstack = stack.New(stack.Options{ NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol}, TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6}, @@ -81,7 +83,7 @@ func New(config *config.C) (*Service, error) { if tcpipProblem := s.ipstack.CreateNIC(nicID, linkEP); tcpipProblem != nil { return nil, fmt.Errorf("could not create netstack NIC: %v", tcpipProblem) } - ipv4Subnet, _ := tcpip.NewSubnet(tcpip.Address(strings.Repeat("\x00", 4)), tcpip.AddressMask(strings.Repeat("\x00", 4))) + ipv4Subnet, _ := tcpip.NewSubnet(tcpip.AddrFrom4([4]byte{0x00, 0x00, 0x00, 0x00}), tcpip.MaskFrom(strings.Repeat("\x00", 4))) s.ipstack.SetRouteTable([]tcpip.Route{ { Destination: ipv4Subnet, @@ -89,9 +91,9 @@ func New(config *config.C) (*Service, error) { }, }) - ipNet := device.Cidr() + ipNet := device.Networks() pa := tcpip.ProtocolAddress{ - AddressWithPrefix: tcpip.Address(ipNet.IP).WithPrefix(), + AddressWithPrefix: tcpip.AddrFromSlice(ipNet[0].Addr().AsSlice()).WithPrefix(), Protocol: ipv4.ProtocolNumber, } if err := s.ipstack.AddProtocolAddress(nicID, pa, stack.AddressProperties{ @@ -124,7 +126,7 @@ func New(config *config.C) (*Service, error) { return err } packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{ - Payload: bufferv2.MakeWithData(bytes.Clone(buf[:n])), + Payload: buffer.MakeWithData(bytes.Clone(buf[:n])), }) linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf) @@ -136,7 +138,7 @@ func New(config *config.C) (*Service, error) { eg.Go(func() error { for { packet := linkEP.ReadContext(ctx) - if packet.IsNil() { + if packet == nil { if err := ctx.Err(); err != nil { return err } @@ -150,27 +152,57 @@ func New(config *config.C) (*Service, error) { } }) + // Add the nebula wait function to the group so a fatal reader error + // propagates out through errgroup.Wait(). + eg.Go(func() error { + return control.Wait() + }) + return &s, nil } -// DialContext dials the provided address. Currently only TCP is supported. -func (s *Service) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - if network != "tcp" && network != "tcp4" { - return nil, errors.New("only tcp is supported") - } - - addr, err := net.ResolveTCPAddr(network, address) - if err != nil { - return nil, err +func getProtocolNumber(addr netip.Addr) tcpip.NetworkProtocolNumber { + if addr.Is6() { + return ipv6.ProtocolNumber } + return ipv4.ProtocolNumber +} - fullAddr := tcpip.FullAddress{ - NIC: nicID, - Addr: tcpip.Address(addr.IP), - Port: uint16(addr.Port), +// DialContext dials the provided address. +func (s *Service) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + switch network { + case "udp", "udp4", "udp6": + addr, err := net.ResolveUDPAddr(network, address) + if err != nil { + return nil, err + } + fullAddr := tcpip.FullAddress{ + NIC: nicID, + Addr: tcpip.AddrFromSlice(addr.IP), + Port: uint16(addr.Port), + } + num := getProtocolNumber(addr.AddrPort().Addr()) + return gonet.DialUDP(s.ipstack, nil, &fullAddr, num) + case "tcp", "tcp4", "tcp6": + addr, err := net.ResolveTCPAddr(network, address) + if err != nil { + return nil, err + } + fullAddr := tcpip.FullAddress{ + NIC: nicID, + Addr: tcpip.AddrFromSlice(addr.IP), + Port: uint16(addr.Port), + } + num := getProtocolNumber(addr.AddrPort().Addr()) + return gonet.DialContextTCP(ctx, s.ipstack, fullAddr, num) + default: + return nil, fmt.Errorf("unknown network type: %s", network) } +} - return gonet.DialContextTCP(ctx, s.ipstack, fullAddr, ipv4.ProtocolNumber) +// Dial dials the provided address +func (s *Service) Dial(network, address string) (net.Conn, error) { + return s.DialContext(context.Background(), network, address) } // Listen listens on the provided address. Currently only TCP with wildcard diff --git a/service/service_test.go b/service/service_test.go index d1909cd1..4bcc8437 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -4,27 +4,27 @@ import ( "bytes" "context" "errors" - "net" + "net/netip" + "os" "testing" "time" "dario.cat/mergo" + "github.com/slackhq/nebula" "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/e2e" + "github.com/slackhq/nebula/logging" + "github.com/slackhq/nebula/overlay" + "go.yaml.in/yaml/v3" "golang.org/x/sync/errgroup" - "gopkg.in/yaml.v2" ) -type m map[string]interface{} +type m = map[string]any -func newSimpleService(caCrt *cert.NebulaCertificate, caKey []byte, name string, udpIp net.IP, overrides m) *Service { - - vpnIpNet := &net.IPNet{IP: make([]byte, len(udpIp)), Mask: net.IPMask{255, 255, 255, 0}} - copy(vpnIpNet.IP, udpIp) - - _, _, myPrivKey, myPEM := e2e.NewTestCert(caCrt, caKey, "a", time.Now(), time.Now().Add(5*time.Minute), vpnIpNet, nil, []string{}) - caB, err := caCrt.MarshalToPEM() +func newSimpleService(caCrt cert.Certificate, caKey []byte, name string, udpIp netip.Addr, overrides m) *Service { + _, _, myPrivKey, myPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, caCrt, caKey, "a", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.PrefixFrom(udpIp, 24)}, nil, []string{}) + caB, err := caCrt.MarshalPEM() if err != nil { panic(err) } @@ -75,7 +75,14 @@ func newSimpleService(caCrt *cert.NebulaCertificate, caKey []byte, name string, panic(err) } - s, err := New(&c) + logger := logging.NewLogger(os.Stdout) + + control, err := nebula.Main(&c, false, "custom-app", logger, overlay.NewUserDeviceFromConfig) + if err != nil { + panic(err) + } + + s, err := New(control) if err != nil { panic(err) } @@ -83,8 +90,8 @@ func newSimpleService(caCrt *cert.NebulaCertificate, caKey []byte, name string, } func TestService(t *testing.T) { - ca, _, caKey, _ := e2e.NewTestCaCert(time.Now(), time.Now().Add(10*time.Minute), []*net.IPNet{}, []*net.IPNet{}, []string{}) - a := newSimpleService(ca, caKey, "a", net.IP{10, 0, 0, 1}, m{ + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{ "static_host_map": m{}, "lighthouse": m{ "am_lighthouse": true, @@ -94,7 +101,7 @@ func TestService(t *testing.T) { "port": 4243, }, }) - b := newSimpleService(ca, caKey, "b", net.IP{10, 0, 0, 2}, m{ + b := newSimpleService(ca, caKey, "b", netip.MustParseAddr("10.0.0.2"), m{ "static_host_map": m{ "10.0.0.1": []string{"localhost:4243"}, }, diff --git a/ssh.go b/ssh.go index e99205ca..3863b5ec 100644 --- a/ssh.go +++ b/ssh.go @@ -6,21 +6,22 @@ import ( "errors" "flag" "fmt" + "log/slog" + "maps" "net" + "net/netip" "os" - "reflect" + "path/filepath" "runtime" "runtime/pprof" "sort" "strconv" "strings" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/sshd" - "github.com/slackhq/nebula/udp" ) type sshListHostMapFlags struct { @@ -51,12 +52,17 @@ type sshCreateTunnelFlags struct { Address string } -func wireSSHReload(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) { +type sshDeviceInfoFlags struct { + Json bool + Pretty bool +} + +func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { c.RegisterReloadCallback(func(c *config.C) { if c.GetBool("sshd.enabled", false) { sshRun, err := configSSH(l, ssh, c) if err != nil { - l.WithError(err).Error("Failed to reconfigure the sshd") + l.Error("Failed to reconfigure the sshd", "error", err) ssh.Stop() } if sshRun != nil { @@ -72,10 +78,7 @@ func wireSSHReload(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) { // updates the passed-in SSHServer. On success, it returns a function // that callers may invoke to run the configured ssh server. On // failure, it returns nil, error. -func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) { - //TODO conntrack list - //TODO print firewall rules or hash? - +func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) { listen := c.GetString("sshd.listen", "") if listen == "" { return nil, fmt.Errorf("sshd.listen must be provided") @@ -89,7 +92,6 @@ func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), erro return nil, fmt.Errorf("sshd.listen can not use port 22") } - //TODO: no good way to reload this right now hostKeyPathOrKey := c.GetString("sshd.host_key", "") if hostKeyPathOrKey == "" { return nil, fmt.Errorf("sshd.host_key must be provided") @@ -110,19 +112,32 @@ func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), erro return nil, fmt.Errorf("error while adding sshd.host_key: %s", err) } + // Clear existing trusted CAs and authorized keys + ssh.ClearTrustedCAs() + ssh.ClearAuthorizedKeys() + + rawCAs := c.GetStringSlice("sshd.trusted_cas", []string{}) + for _, caAuthorizedKey := range rawCAs { + err := ssh.AddTrustedCA(caAuthorizedKey) + if err != nil { + l.Warn("SSH CA had an error, ignoring", "error", err, "sshCA", caAuthorizedKey) + continue + } + } + rawKeys := c.Get("sshd.authorized_users") - keys, ok := rawKeys.([]interface{}) + keys, ok := rawKeys.([]any) if ok { for _, rk := range keys { - kDef, ok := rk.(map[interface{}]interface{}) + kDef, ok := rk.(map[string]any) if !ok { - l.WithField("sshKeyConfig", rk).Warn("Authorized user had an error, ignoring") + l.Warn("Authorized user had an error, ignoring", "sshKeyConfig", rk) continue } user, ok := kDef["user"].(string) if !ok { - l.WithField("sshKeyConfig", rk).Warn("Authorized user is missing the user field") + l.Warn("Authorized user is missing the user field", "sshKeyConfig", rk) continue } @@ -131,27 +146,37 @@ func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), erro case string: err := ssh.AddAuthorizedKey(user, v) if err != nil { - l.WithError(err).WithField("sshKeyConfig", rk).WithField("sshKey", v).Warn("Failed to authorize key") + l.Warn("Failed to authorize key", + "error", err, + "sshKeyConfig", rk, + "sshKey", v, + ) continue } - case []interface{}: + case []any: for _, subK := range v { sk, ok := subK.(string) if !ok { - l.WithField("sshKeyConfig", rk).WithField("sshKey", subK).Warn("Did not understand ssh key") + l.Warn("Did not understand ssh key", + "sshKeyConfig", rk, + "sshKey", subK, + ) continue } err := ssh.AddAuthorizedKey(user, sk) if err != nil { - l.WithError(err).WithField("sshKeyConfig", sk).Warn("Failed to authorize key") + l.Warn("Failed to authorize key", + "error", err, + "sshKeyConfig", sk, + ) continue } } default: - l.WithField("sshKeyConfig", rk).Warn("Authorized user is missing the keys field or was not understood") + l.Warn("Authorized user is missing the keys field or was not understood", "sshKeyConfig", rk) } } } else { @@ -163,7 +188,7 @@ func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), erro ssh.Stop() runner = func() { if err := ssh.Run(listen); err != nil { - l.WithField("err", err).Warn("Failed to run the SSH server") + l.Warn("Failed to run the SSH server", "error", err) } } } else { @@ -173,11 +198,17 @@ func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), erro return runner, nil } -func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) { +func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) { + // sandboxDir defaults to a dir in temp. The intention is that end user will + // create this dir as needed. Overriding this config value to "" allows + // writing to anywhere in the system. + defaultDir := filepath.Join(os.TempDir(), "nebula-debug") + sandboxDir := c.GetString("sshd.sandbox_dir", defaultDir) + ssh.RegisterCommand(&sshd.Command{ Name: "list-hostmap", ShortDescription: "List all known previously connected hosts", - Flags: func() (*flag.FlagSet, interface{}) { + Flags: func() (*flag.FlagSet, any) { fl := flag.NewFlagSet("", flag.ContinueOnError) s := sshListHostMapFlags{} fl.BoolVar(&s.Json, "json", false, "outputs as json with more information") @@ -185,7 +216,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table") return fl, &s }, - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshListHostMap(f.hostMap, fs, w) }, }) @@ -193,7 +224,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "list-pending-hostmap", ShortDescription: "List all handshaking hosts", - Flags: func() (*flag.FlagSet, interface{}) { + Flags: func() (*flag.FlagSet, any) { fl := flag.NewFlagSet("", flag.ContinueOnError) s := sshListHostMapFlags{} fl.BoolVar(&s.Json, "json", false, "outputs as json with more information") @@ -201,7 +232,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter fl.BoolVar(&s.ByIndex, "by-index", false, "gets all hosts in the hostmap from the index table") return fl, &s }, - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshListHostMap(f.handshakeManager, fs, w) }, }) @@ -209,14 +240,14 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "list-lighthouse-addrmap", ShortDescription: "List all lighthouse map entries", - Flags: func() (*flag.FlagSet, interface{}) { + Flags: func() (*flag.FlagSet, any) { fl := flag.NewFlagSet("", flag.ContinueOnError) s := sshListHostMapFlags{} fl.BoolVar(&s.Json, "json", false, "outputs as json with more information") fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json") return fl, &s }, - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshListLighthouseMap(f.lightHouse, fs, w) }, }) @@ -224,21 +255,23 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "reload", ShortDescription: "Reloads configuration from disk, same as sending HUP to the process", - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshReload(c, w) }, }) ssh.RegisterCommand(&sshd.Command{ Name: "start-cpu-profile", - ShortDescription: "Starts a cpu profile and write output to the provided file", - Callback: sshStartCpuProfile, + ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`", + Callback: func(fs any, a []string, w sshd.StringWriter) error { + return sshStartCpuProfile(sandboxDir, fs, a, w) + }, }) ssh.RegisterCommand(&sshd.Command{ Name: "stop-cpu-profile", ShortDescription: "Stops a cpu profile and writes output to the previously provided file", - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { pprof.StopCPUProfile() return w.WriteLine("If a CPU profile was running it is now stopped") }, @@ -246,8 +279,10 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "save-heap-profile", - ShortDescription: "Saves a heap profile to the provided path", - Callback: sshGetHeapProfile, + ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`", + Callback: func(fs any, a []string, w sshd.StringWriter) error { + return sshGetHeapProfile(sandboxDir, fs, a, w) + }, }) ssh.RegisterCommand(&sshd.Command{ @@ -258,14 +293,16 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "save-mutex-profile", - ShortDescription: "Saves a mutex profile to the provided path", - Callback: sshGetMutexProfile, + ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`", + Callback: func(fs any, a []string, w sshd.StringWriter) error { + return sshGetMutexProfile(sandboxDir, fs, a, w) + }, }) ssh.RegisterCommand(&sshd.Command{ Name: "log-level", ShortDescription: "Gets or sets the current log level", - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshLogLevel(l, fs, a, w) }, }) @@ -273,7 +310,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "log-format", ShortDescription: "Gets or sets the current log format", - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshLogFormat(l, fs, a, w) }, }) @@ -281,15 +318,30 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "version", ShortDescription: "Prints the currently running version of nebula", - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshVersion(f, fs, a, w) }, }) + ssh.RegisterCommand(&sshd.Command{ + Name: "device-info", + ShortDescription: "Prints information about the network device.", + Flags: func() (*flag.FlagSet, any) { + fl := flag.NewFlagSet("", flag.ContinueOnError) + s := sshDeviceInfoFlags{} + fl.BoolVar(&s.Json, "json", false, "outputs as json with more information") + fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json") + return fl, &s + }, + Callback: func(fs any, a []string, w sshd.StringWriter) error { + return sshDeviceInfo(f, fs, w) + }, + }) + ssh.RegisterCommand(&sshd.Command{ Name: "print-cert", - ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn ip", - Flags: func() (*flag.FlagSet, interface{}) { + ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn addr", + Flags: func() (*flag.FlagSet, any) { fl := flag.NewFlagSet("", flag.ContinueOnError) s := sshPrintCertFlags{} fl.BoolVar(&s.Json, "json", false, "outputs as json") @@ -297,21 +349,21 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter fl.BoolVar(&s.Raw, "raw", false, "raw prints the PEM encoded certificate, not compatible with -json or -pretty") return fl, &s }, - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshPrintCert(f, fs, a, w) }, }) ssh.RegisterCommand(&sshd.Command{ Name: "print-tunnel", - ShortDescription: "Prints json details about a tunnel for the provided vpn ip", - Flags: func() (*flag.FlagSet, interface{}) { + ShortDescription: "Prints json details about a tunnel for the provided vpn addr", + Flags: func() (*flag.FlagSet, any) { fl := flag.NewFlagSet("", flag.ContinueOnError) s := sshPrintTunnelFlags{} fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json") return fl, &s }, - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshPrintTunnel(f, fs, a, w) }, }) @@ -319,74 +371,73 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "print-relays", ShortDescription: "Prints json details about all relay info", - Flags: func() (*flag.FlagSet, interface{}) { + Flags: func() (*flag.FlagSet, any) { fl := flag.NewFlagSet("", flag.ContinueOnError) s := sshPrintTunnelFlags{} fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json") return fl, &s }, - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshPrintRelays(f, fs, a, w) }, }) ssh.RegisterCommand(&sshd.Command{ Name: "change-remote", - ShortDescription: "Changes the remote address used in the tunnel for the provided vpn ip", - Flags: func() (*flag.FlagSet, interface{}) { + ShortDescription: "Changes the remote address used in the tunnel for the provided vpn addr", + Flags: func() (*flag.FlagSet, any) { fl := flag.NewFlagSet("", flag.ContinueOnError) s := sshChangeRemoteFlags{} fl.StringVar(&s.Address, "address", "", "The new remote address, ip:port") return fl, &s }, - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshChangeRemote(f, fs, a, w) }, }) ssh.RegisterCommand(&sshd.Command{ Name: "close-tunnel", - ShortDescription: "Closes a tunnel for the provided vpn ip", - Flags: func() (*flag.FlagSet, interface{}) { + ShortDescription: "Closes a tunnel for the provided vpn addr", + Flags: func() (*flag.FlagSet, any) { fl := flag.NewFlagSet("", flag.ContinueOnError) s := sshCloseTunnelFlags{} fl.BoolVar(&s.LocalOnly, "local-only", false, "Disables notifying the remote that the tunnel is shutting down") return fl, &s }, - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshCloseTunnel(f, fs, a, w) }, }) ssh.RegisterCommand(&sshd.Command{ Name: "create-tunnel", - ShortDescription: "Creates a tunnel for the provided vpn ip and address", + ShortDescription: "Creates a tunnel for the provided vpn address", Help: "The lighthouses will be queried for real addresses but you can provide one as well.", - Flags: func() (*flag.FlagSet, interface{}) { + Flags: func() (*flag.FlagSet, any) { fl := flag.NewFlagSet("", flag.ContinueOnError) s := sshCreateTunnelFlags{} fl.StringVar(&s.Address, "address", "", "Optionally provide a real remote address, ip:port ") return fl, &s }, - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshCreateTunnel(f, fs, a, w) }, }) ssh.RegisterCommand(&sshd.Command{ Name: "query-lighthouse", - ShortDescription: "Query the lighthouses for the provided vpn ip", - Help: "This command is asynchronous. Only currently known udp ips will be printed.", - Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + ShortDescription: "Query the lighthouses for the provided vpn address", + Help: "This command is asynchronous. Only currently known udp addresses will be printed.", + Callback: func(fs any, a []string, w sshd.StringWriter) error { return sshQueryLighthouse(f, fs, a, w) }, }) } -func sshListHostMap(hl controlHostLister, a interface{}, w sshd.StringWriter) error { +func sshListHostMap(hl controlHostLister, a any, w sshd.StringWriter) error { fs, ok := a.(*sshListHostMapFlags) if !ok { - //TODO: error return nil } @@ -398,7 +449,7 @@ func sshListHostMap(hl controlHostLister, a interface{}, w sshd.StringWriter) er } sort.Slice(hm, func(i, j int) bool { - return bytes.Compare(hm[i].VpnIp, hm[j].VpnIp) < 0 + return hm[i].VpnAddrs[0].Compare(hm[j].VpnAddrs[0]) < 0 }) if fs.Json || fs.Pretty { @@ -409,13 +460,12 @@ func sshListHostMap(hl controlHostLister, a interface{}, w sshd.StringWriter) er err := js.Encode(hm) if err != nil { - //TODO return nil } } else { for _, v := range hm { - err := w.WriteLine(fmt.Sprintf("%s: %s", v.VpnIp, v.RemoteAddrs)) + err := w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddrs, v.RemoteAddrs)) if err != nil { return err } @@ -425,16 +475,15 @@ func sshListHostMap(hl controlHostLister, a interface{}, w sshd.StringWriter) er return nil } -func sshListLighthouseMap(lightHouse *LightHouse, a interface{}, w sshd.StringWriter) error { +func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) error { fs, ok := a.(*sshListHostMapFlags) if !ok { - //TODO: error return nil } type lighthouseInfo struct { - VpnIp string `json:"vpnIp"` - Addrs *CacheMap `json:"addrs"` + VpnAddr string `json:"vpnAddr"` + Addrs *CacheMap `json:"addrs"` } lightHouse.RLock() @@ -442,15 +491,15 @@ func sshListLighthouseMap(lightHouse *LightHouse, a interface{}, w sshd.StringWr x := 0 for k, v := range lightHouse.addrMap { addrMap[x] = lighthouseInfo{ - VpnIp: k.String(), - Addrs: v.CopyCache(), + VpnAddr: k.String(), + Addrs: v.CopyCache(), } x++ } lightHouse.RUnlock() sort.Slice(addrMap, func(i, j int) bool { - return strings.Compare(addrMap[i].VpnIp, addrMap[j].VpnIp) < 0 + return strings.Compare(addrMap[i].VpnAddr, addrMap[j].VpnAddr) < 0 }) if fs.Json || fs.Pretty { @@ -461,7 +510,6 @@ func sshListLighthouseMap(lightHouse *LightHouse, a interface{}, w sshd.StringWr err := js.Encode(addrMap) if err != nil { - //TODO return nil } @@ -471,7 +519,7 @@ func sshListLighthouseMap(lightHouse *LightHouse, a interface{}, w sshd.StringWr if err != nil { return err } - err = w.WriteLine(fmt.Sprintf("%s: %s", v.VpnIp, string(b))) + err = w.WriteLine(fmt.Sprintf("%s: %s", v.VpnAddr, string(b))) if err != nil { return err } @@ -481,13 +529,43 @@ func sshListLighthouseMap(lightHouse *LightHouse, a interface{}, w sshd.StringWr return nil } -func sshStartCpuProfile(fs interface{}, a []string, w sshd.StringWriter) error { +// sshSanitizeFilePath validates that the given file path is within the sandbox directory. +// If sandboxDir is empty, the path is returned as-is for backwards compatibility. +func sshSanitizeFilePath(sandboxDir, filePath string) (string, error) { + if sandboxDir == "" { + return filePath, nil + } + + // Clean and resolve the path relative to the sandbox directory + if !filepath.IsAbs(filePath) { + filePath = filepath.Join(sandboxDir, filePath) + } + cleaned := filepath.Clean(filePath) + + // Ensure the resolved path is within the sandbox directory + cleanedSandbox := filepath.Clean(sandboxDir) + if cleaned == cleanedSandbox { + return "", fmt.Errorf("path %q resolves to the sandbox directory itself %q", filePath, sandboxDir) + } + if !strings.HasPrefix(cleaned, cleanedSandbox+string(filepath.Separator)) { + return "", fmt.Errorf("path %q is outside the sandbox directory %q", filePath, sandboxDir) + } + + return cleaned, nil +} + +func sshStartCpuProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error { if len(a) == 0 { err := w.WriteLine("No path to write profile provided") return err } - file, err := os.Create(a[0]) + filePath, err := sshSanitizeFilePath(sandboxDir, a[0]) + if err != nil { + return w.WriteLine(err.Error()) + } + + file, err := os.Create(filePath) if err != nil { err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err)) return err @@ -503,57 +581,54 @@ func sshStartCpuProfile(fs interface{}, a []string, w sshd.StringWriter) error { return err } -func sshVersion(ifce *Interface, fs interface{}, a []string, w sshd.StringWriter) error { +func sshVersion(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { return w.WriteLine(fmt.Sprintf("%s", ifce.version)) } -func sshQueryLighthouse(ifce *Interface, fs interface{}, a []string, w sshd.StringWriter) error { +func sshQueryLighthouse(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { if len(a) == 0 { - return w.WriteLine("No vpn ip was provided") + return w.WriteLine("No vpn address was provided") } - parsedIp := net.ParseIP(a[0]) - if parsedIp == nil { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + vpnAddr, err := netip.ParseAddr(a[0]) + if err != nil { + return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0])) } - vpnIp := iputil.Ip2VpnIp(parsedIp) - if vpnIp == 0 { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + if !vpnAddr.IsValid() { + return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0])) } var cm *CacheMap - rl := ifce.lightHouse.Query(vpnIp) + rl := ifce.lightHouse.Query(vpnAddr) if rl != nil { cm = rl.CopyCache() } return json.NewEncoder(w.GetWriter()).Encode(cm) } -func sshCloseTunnel(ifce *Interface, fs interface{}, a []string, w sshd.StringWriter) error { +func sshCloseTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { flags, ok := fs.(*sshCloseTunnelFlags) if !ok { - //TODO: error return nil } if len(a) == 0 { - return w.WriteLine("No vpn ip was provided") + return w.WriteLine("No vpn address was provided") } - parsedIp := net.ParseIP(a[0]) - if parsedIp == nil { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + vpnAddr, err := netip.ParseAddr(a[0]) + if err != nil { + return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0])) } - vpnIp := iputil.Ip2VpnIp(parsedIp) - if vpnIp == 0 { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + if !vpnAddr.IsValid() { + return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0])) } - hostInfo := ifce.hostMap.QueryVpnIp(vpnIp) + hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr) if hostInfo == nil { - return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn ip: %v", a[0])) + return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0])) } if !flags.LocalOnly { @@ -572,98 +647,99 @@ func sshCloseTunnel(ifce *Interface, fs interface{}, a []string, w sshd.StringWr return w.WriteLine("Closed") } -func sshCreateTunnel(ifce *Interface, fs interface{}, a []string, w sshd.StringWriter) error { +func sshCreateTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { flags, ok := fs.(*sshCreateTunnelFlags) if !ok { - //TODO: error return nil } if len(a) == 0 { - return w.WriteLine("No vpn ip was provided") + return w.WriteLine("No vpn address was provided") } - parsedIp := net.ParseIP(a[0]) - if parsedIp == nil { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + vpnAddr, err := netip.ParseAddr(a[0]) + if err != nil { + return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0])) } - vpnIp := iputil.Ip2VpnIp(parsedIp) - if vpnIp == 0 { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + if !vpnAddr.IsValid() { + return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0])) } - hostInfo := ifce.hostMap.QueryVpnIp(vpnIp) + hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr) if hostInfo != nil { return w.WriteLine(fmt.Sprintf("Tunnel already exists")) } - hostInfo = ifce.handshakeManager.QueryVpnIp(vpnIp) + hostInfo = ifce.handshakeManager.QueryVpnAddr(vpnAddr) if hostInfo != nil { return w.WriteLine(fmt.Sprintf("Tunnel already handshaking")) } - var addr *udp.Addr + var addr netip.AddrPort if flags.Address != "" { - addr = udp.NewAddrFromString(flags.Address) - if addr == nil { + addr, err = netip.ParseAddrPort(flags.Address) + if err != nil { return w.WriteLine("Address could not be parsed") } } - hostInfo = ifce.handshakeManager.StartHandshake(vpnIp, nil) - if addr != nil { + hostInfo = ifce.handshakeManager.StartHandshake(vpnAddr, nil) + if addr.IsValid() { hostInfo.SetRemote(addr) } return w.WriteLine("Created") } -func sshChangeRemote(ifce *Interface, fs interface{}, a []string, w sshd.StringWriter) error { +func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { flags, ok := fs.(*sshChangeRemoteFlags) if !ok { - //TODO: error return nil } if len(a) == 0 { - return w.WriteLine("No vpn ip was provided") + return w.WriteLine("No vpn address was provided") } if flags.Address == "" { return w.WriteLine("No address was provided") } - addr := udp.NewAddrFromString(flags.Address) - if addr == nil { + addr, err := netip.ParseAddrPort(flags.Address) + if err != nil { return w.WriteLine("Address could not be parsed") } - parsedIp := net.ParseIP(a[0]) - if parsedIp == nil { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + vpnAddr, err := netip.ParseAddr(a[0]) + if err != nil { + return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0])) } - vpnIp := iputil.Ip2VpnIp(parsedIp) - if vpnIp == 0 { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + if !vpnAddr.IsValid() { + return w.WriteLine(fmt.Sprintf("The provided vpn address could not be parsed: %s", a[0])) } - hostInfo := ifce.hostMap.QueryVpnIp(vpnIp) + hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr) if hostInfo == nil { - return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn ip: %v", a[0])) + return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn address: %v", a[0])) } hostInfo.SetRemote(addr) return w.WriteLine("Changed") } -func sshGetHeapProfile(fs interface{}, a []string, w sshd.StringWriter) error { +func sshGetHeapProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error { if len(a) == 0 { return w.WriteLine("No path to write profile provided") } - file, err := os.Create(a[0]) + filePath, err := sshSanitizeFilePath(sandboxDir, a[0]) + if err != nil { + return w.WriteLine(err.Error()) + } + + file, err := os.Create(filePath) if err != nil { err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err)) return err @@ -679,7 +755,7 @@ func sshGetHeapProfile(fs interface{}, a []string, w sshd.StringWriter) error { return err } -func sshMutexProfileFraction(fs interface{}, a []string, w sshd.StringWriter) error { +func sshMutexProfileFraction(fs any, a []string, w sshd.StringWriter) error { if len(a) == 0 { rate := runtime.SetMutexProfileFraction(-1) return w.WriteLine(fmt.Sprintf("Current value: %d", rate)) @@ -694,12 +770,17 @@ func sshMutexProfileFraction(fs interface{}, a []string, w sshd.StringWriter) er return w.WriteLine(fmt.Sprintf("New value: %d. Old value: %d", newRate, oldRate)) } -func sshGetMutexProfile(fs interface{}, a []string, w sshd.StringWriter) error { +func sshGetMutexProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error { if len(a) == 0 { return w.WriteLine("No path to write profile provided") } - file, err := os.Create(a[0]) + filePath, err := sshSanitizeFilePath(sandboxDir, a[0]) + if err != nil { + return w.WriteLine(err.Error()) + } + + file, err := os.Create(filePath) if err != nil { return w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err)) } @@ -718,69 +799,75 @@ func sshGetMutexProfile(fs interface{}, a []string, w sshd.StringWriter) error { return w.WriteLine(fmt.Sprintf("Mutex profile created at %s", a)) } -func sshLogLevel(l *logrus.Logger, fs interface{}, a []string, w sshd.StringWriter) error { +func sshLogLevel(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error { + ctrl, ok := l.Handler().(interface { + GetLevel() slog.Level + SetLevel(slog.Level) + }) + if !ok { + return w.WriteLine("Log level is not reconfigurable on this logger") + } + if len(a) == 0 { - return w.WriteLine(fmt.Sprintf("Log level is: %s", l.Level)) + return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel()))) } - level, err := logrus.ParseLevel(a[0]) + level, err := logging.ParseLevel(strings.ToLower(a[0])) if err != nil { - return w.WriteLine(fmt.Sprintf("Unknown log level %s. Possible log levels: %s", a, logrus.AllLevels)) + return w.WriteLine(fmt.Sprintf("Unknown log level %s. Possible log levels: trace, debug, info, warn, error", a)) } - l.SetLevel(level) - return w.WriteLine(fmt.Sprintf("Log level is: %s", l.Level)) + ctrl.SetLevel(level) + return w.WriteLine(fmt.Sprintf("Log level is: %s", logging.LevelName(ctrl.GetLevel()))) } -func sshLogFormat(l *logrus.Logger, fs interface{}, a []string, w sshd.StringWriter) error { - if len(a) == 0 { - return w.WriteLine(fmt.Sprintf("Log format is: %s", reflect.TypeOf(l.Formatter))) +func sshLogFormat(l *slog.Logger, fs any, a []string, w sshd.StringWriter) error { + ctrl, ok := l.Handler().(interface { + GetFormat() string + SetFormat(string) error + }) + if !ok { + return w.WriteLine("Log format is not reconfigurable on this logger") } - logFormat := strings.ToLower(a[0]) - switch logFormat { - case "text": - l.Formatter = &logrus.TextFormatter{} - case "json": - l.Formatter = &logrus.JSONFormatter{} - default: - return fmt.Errorf("unknown log format `%s`. possible formats: %s", logFormat, []string{"text", "json"}) + if len(a) == 0 { + return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat())) } - return w.WriteLine(fmt.Sprintf("Log format is: %s", reflect.TypeOf(l.Formatter))) + if err := ctrl.SetFormat(strings.ToLower(a[0])); err != nil { + return err + } + return w.WriteLine(fmt.Sprintf("Log format is: %s", ctrl.GetFormat())) } -func sshPrintCert(ifce *Interface, fs interface{}, a []string, w sshd.StringWriter) error { +func sshPrintCert(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { args, ok := fs.(*sshPrintCertFlags) if !ok { - //TODO: error return nil } - cert := ifce.pki.GetCertState().Certificate + cert := ifce.pki.getCertState().GetDefaultCertificate() if len(a) > 0 { - parsedIp := net.ParseIP(a[0]) - if parsedIp == nil { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + vpnAddr, err := netip.ParseAddr(a[0]) + if err != nil { + return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0])) } - vpnIp := iputil.Ip2VpnIp(parsedIp) - if vpnIp == 0 { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + if !vpnAddr.IsValid() { + return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0])) } - hostInfo := ifce.hostMap.QueryVpnIp(vpnIp) + hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr) if hostInfo == nil { - return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn ip: %v", a[0])) + return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0])) } - cert = hostInfo.GetCert() + cert = hostInfo.GetCert().Certificate } if args.Json || args.Pretty { b, err := cert.MarshalJSON() if err != nil { - //TODO: handle it return nil } @@ -789,7 +876,6 @@ func sshPrintCert(ifce *Interface, fs interface{}, a []string, w sshd.StringWrit err := json.Indent(buf, b, "", " ") b = buf.Bytes() if err != nil { - //TODO: handle it return nil } } @@ -798,9 +884,8 @@ func sshPrintCert(ifce *Interface, fs interface{}, a []string, w sshd.StringWrit } if args.Raw { - b, err := cert.MarshalToPEM() + b, err := cert.MarshalPEM() if err != nil { - //TODO: handle it return nil } @@ -810,34 +895,31 @@ func sshPrintCert(ifce *Interface, fs interface{}, a []string, w sshd.StringWrit return w.WriteLine(cert.String()) } -func sshPrintRelays(ifce *Interface, fs interface{}, a []string, w sshd.StringWriter) error { +func sshPrintRelays(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { args, ok := fs.(*sshPrintTunnelFlags) if !ok { - //TODO: error w.WriteLine(fmt.Sprintf("sshPrintRelays failed to convert args type")) return nil } relays := map[uint32]*HostInfo{} ifce.hostMap.Lock() - for k, v := range ifce.hostMap.Relays { - relays[k] = v - } + maps.Copy(relays, ifce.hostMap.Relays) ifce.hostMap.Unlock() type RelayFor struct { Error error Type string State string - PeerIp iputil.VpnIp + PeerAddr netip.Addr LocalIndex uint32 RemoteIndex uint32 - RelayedThrough []iputil.VpnIp + RelayedThrough []netip.Addr } type RelayOutput struct { - NebulaIp iputil.VpnIp - RelayForIps []RelayFor + NebulaAddr netip.Addr + RelayForAddrs []RelayFor } type CmdOutput struct { @@ -853,16 +935,16 @@ func sshPrintRelays(ifce *Interface, fs interface{}, a []string, w sshd.StringWr } for k, v := range relays { - ro := RelayOutput{NebulaIp: v.vpnIp} + ro := RelayOutput{NebulaAddr: v.vpnAddrs[0]} co.Relays = append(co.Relays, &ro) - relayHI := ifce.hostMap.QueryVpnIp(v.vpnIp) + relayHI := ifce.hostMap.QueryVpnAddr(v.vpnAddrs[0]) if relayHI == nil { - ro.RelayForIps = append(ro.RelayForIps, RelayFor{Error: errors.New("could not find hostinfo")}) + ro.RelayForAddrs = append(ro.RelayForAddrs, RelayFor{Error: errors.New("could not find hostinfo")}) continue } - for _, vpnIp := range relayHI.relayState.CopyRelayForIps() { + for _, vpnAddr := range relayHI.relayState.CopyRelayForIps() { rf := RelayFor{Error: nil} - r, ok := relayHI.relayState.GetRelayForByIp(vpnIp) + r, ok := relayHI.relayState.GetRelayForByAddr(vpnAddr) if ok { t := "" switch r.Type { @@ -886,19 +968,19 @@ func sshPrintRelays(ifce *Interface, fs interface{}, a []string, w sshd.StringWr rf.LocalIndex = r.LocalIndex rf.RemoteIndex = r.RemoteIndex - rf.PeerIp = r.PeerIp + rf.PeerAddr = r.PeerAddr rf.Type = t rf.State = s if rf.LocalIndex != k { rf.Error = fmt.Errorf("hostmap LocalIndex '%v' does not match RelayState LocalIndex", k) } } - relayedHI := ifce.hostMap.QueryVpnIp(vpnIp) + relayedHI := ifce.hostMap.QueryVpnAddr(vpnAddr) if relayedHI != nil { rf.RelayedThrough = append(rf.RelayedThrough, relayedHI.relayState.CopyRelayIps()...) } - ro.RelayForIps = append(ro.RelayForIps, rf) + ro.RelayForAddrs = append(ro.RelayForAddrs, rf) } } err := enc.Encode(co) @@ -908,30 +990,28 @@ func sshPrintRelays(ifce *Interface, fs interface{}, a []string, w sshd.StringWr return nil } -func sshPrintTunnel(ifce *Interface, fs interface{}, a []string, w sshd.StringWriter) error { +func sshPrintTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) error { args, ok := fs.(*sshPrintTunnelFlags) if !ok { - //TODO: error return nil } if len(a) == 0 { - return w.WriteLine("No vpn ip was provided") + return w.WriteLine("No vpn address was provided") } - parsedIp := net.ParseIP(a[0]) - if parsedIp == nil { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + vpnAddr, err := netip.ParseAddr(a[0]) + if err != nil { + return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0])) } - vpnIp := iputil.Ip2VpnIp(parsedIp) - if vpnIp == 0 { - return w.WriteLine(fmt.Sprintf("The provided vpn ip could not be parsed: %s", a[0])) + if !vpnAddr.IsValid() { + return w.WriteLine(fmt.Sprintf("The provided vpn addr could not be parsed: %s", a[0])) } - hostInfo := ifce.hostMap.QueryVpnIp(vpnIp) + hostInfo := ifce.hostMap.QueryVpnAddr(vpnAddr) if hostInfo == nil { - return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn ip: %v", a[0])) + return w.WriteLine(fmt.Sprintf("Could not find tunnel for vpn addr: %v", a[0])) } enc := json.NewEncoder(w.GetWriter()) @@ -939,7 +1019,36 @@ func sshPrintTunnel(ifce *Interface, fs interface{}, a []string, w sshd.StringWr enc.SetIndent("", " ") } - return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.preferredRanges)) + return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges())) +} + +func sshDeviceInfo(ifce *Interface, fs any, w sshd.StringWriter) error { + + data := struct { + Name string `json:"name"` + Cidr []netip.Prefix `json:"cidr"` + }{ + Name: ifce.inside.Name(), + Cidr: make([]netip.Prefix, len(ifce.inside.Networks())), + } + + copy(data.Cidr, ifce.inside.Networks()) + + flags, ok := fs.(*sshDeviceInfoFlags) + if !ok { + return fmt.Errorf("internal error: expected flags to be sshDeviceInfoFlags but was %+v", fs) + } + + if flags.Json || flags.Pretty { + js := json.NewEncoder(w.GetWriter()) + if flags.Pretty { + js.SetIndent("", " ") + } + + return js.Encode(data) + } else { + return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr)) + } } func sshReload(c *config.C, w sshd.StringWriter) error { diff --git a/sshd/command.go b/sshd/command.go index 900b01e5..7323d120 100644 --- a/sshd/command.go +++ b/sshd/command.go @@ -12,7 +12,7 @@ import ( // CommandFlags is a function called before help or command execution to parse command line flags // It should return a flag.FlagSet instance and a pointer to the struct that will contain parsed flags -type CommandFlags func() (*flag.FlagSet, interface{}) +type CommandFlags func() (*flag.FlagSet, any) // CommandCallback is the function called when your command should execute. // fs will be a a pointer to the struct provided by Command.Flags callback, if there was one. -h and -help are reserved @@ -21,7 +21,7 @@ type CommandFlags func() (*flag.FlagSet, interface{}) // w is the writer to use when sending messages back to the client. // If an error is returned by the callback it is logged locally, the callback should handle messaging errors to the user // where appropriate -type CommandCallback func(fs interface{}, a []string, w StringWriter) error +type CommandCallback func(fs any, a []string, w StringWriter) error type Command struct { Name string @@ -34,7 +34,7 @@ type Command struct { func execCommand(c *Command, args []string, w StringWriter) error { var ( fl *flag.FlagSet - fs interface{} + fs any ) if c.Flags != nil { @@ -57,7 +57,6 @@ func execCommand(c *Command, args []string, w StringWriter) error { func dumpCommands(c *radix.Tree, w StringWriter) { err := w.WriteLine("Available commands:") if err != nil { - //TODO: log return } @@ -67,10 +66,7 @@ func dumpCommands(c *radix.Tree, w StringWriter) { } sort.Strings(cmds) - err = w.Write(strings.Join(cmds, "\n") + "\n\n") - if err != nil { - //TODO: log - } + _ = w.Write(strings.Join(cmds, "\n") + "\n\n") } func lookupCommand(c *radix.Tree, sCmd string) (*Command, error) { @@ -89,7 +85,7 @@ func lookupCommand(c *radix.Tree, sCmd string) (*Command, error) { func matchCommand(c *radix.Tree, cmd string) []string { cmds := make([]string, 0) - c.WalkPrefix(cmd, func(found string, v interface{}) bool { + c.WalkPrefix(cmd, func(found string, v any) bool { cmds = append(cmds, found) return false }) @@ -99,7 +95,7 @@ func matchCommand(c *radix.Tree, cmd string) []string { func allCommands(c *radix.Tree) []*Command { cmds := make([]*Command, 0) - c.WalkPrefix("", func(found string, v interface{}) bool { + c.WalkPrefix("", func(found string, v any) bool { cmd, ok := v.(*Command) if ok { cmds = append(cmds, cmd) @@ -119,8 +115,6 @@ func helpCallback(commands *radix.Tree, a []string, w StringWriter) (err error) // We are printing a specific commands help text cmd, err := lookupCommand(commands, a[0]) if err != nil { - //TODO: handle error - //TODO: message the user return } diff --git a/sshd/server.go b/sshd/server.go index 4a78fdf0..e0ee9364 100644 --- a/sshd/server.go +++ b/sshd/server.go @@ -1,54 +1,99 @@ package sshd import ( + "bytes" + "context" "errors" "fmt" + "log/slog" "net" "sync" "github.com/armon/go-radix" - "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" ) type SSHServer struct { config *ssh.ServerConfig - l *logrus.Entry + l *slog.Logger + certChecker *ssh.CertChecker + + // authLock guards trustedKeys and trustedCAs + authLock sync.RWMutex // Map of user -> authorized keys trustedKeys map[string]map[string]bool + trustedCAs []ssh.PublicKey // List of available commands helpCommand *Command commands *radix.Tree listener net.Listener - // Locks the conns/counter to avoid concurrent map access - connsLock sync.Mutex - conns map[int]*session - counter int + // ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even + // across reloads, since each Run derives a fresh child rather than reusing this one directly. + ctx context.Context } -// NewSSHServer creates a new ssh server rigged with default commands and prepares to listen -func NewSSHServer(l *logrus.Entry) (*SSHServer, error) { +// NewSSHServer creates a new ssh server rigged with default commands and prepares to listen. +// The ssh server's context is parented off the supplied ctx so cancelling it +// (e.g. on Control.Stop) tears down active sessions and closes the listener. +func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) { s := &SSHServer{ trustedKeys: make(map[string]map[string]bool), l: l, commands: radix.New(), - conns: make(map[int]*session), + ctx: ctx, + } + + cc := ssh.CertChecker{ + IsUserAuthority: func(auth ssh.PublicKey) bool { + s.authLock.RLock() + defer s.authLock.RUnlock() + for _, ca := range s.trustedCAs { + if bytes.Equal(ca.Marshal(), auth.Marshal()) { + return true + } + } + + return false + }, + UserKeyFallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) { + pk := string(pubKey.Marshal()) + fp := ssh.FingerprintSHA256(pubKey) + + s.authLock.RLock() + defer s.authLock.RUnlock() + tk, ok := s.trustedKeys[c.User()] + if !ok { + return nil, fmt.Errorf("unknown user %s", c.User()) + } + + _, ok = tk[pk] + if !ok { + return nil, fmt.Errorf("unknown public key for %s (%s)", c.User(), fp) + } + + return &ssh.Permissions{ + // Record the public key used for authentication. + Extensions: map[string]string{ + "fp": fp, + "user": c.User(), + }, + }, nil + + }, } s.config = &ssh.ServerConfig{ - PublicKeyCallback: s.matchPubKey, - //TODO: AuthLogCallback: s.authAttempt, - //TODO: version string - ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"), + PublicKeyCallback: cc.Authenticate, + ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"), } s.RegisterCommand(&Command{ Name: "help", ShortDescription: "prints available commands or help for specific usage info", - Callback: func(a interface{}, args []string, w StringWriter) error { + Callback: func(a any, args []string, w StringWriter) error { return helpCallback(s.commands, args, w) }, }) @@ -66,8 +111,30 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error { return nil } +func (s *SSHServer) ClearTrustedCAs() { + s.authLock.Lock() + s.trustedCAs = []ssh.PublicKey{} + s.authLock.Unlock() +} + func (s *SSHServer) ClearAuthorizedKeys() { + s.authLock.Lock() s.trustedKeys = make(map[string]map[string]bool) + s.authLock.Unlock() +} + +// AddTrustedCA adds a trusted CA for user certificates +func (s *SSHServer) AddTrustedCA(pubKey string) error { + pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubKey)) + if err != nil { + return err + } + + s.authLock.Lock() + s.trustedCAs = append(s.trustedCAs, pk) + s.authLock.Unlock() + s.l.Info("Trusted CA key", "sshKey", pubKey) + return nil } // AddAuthorizedKey adds an ssh public key for a user @@ -77,6 +144,7 @@ func (s *SSHServer) AddAuthorizedKey(user, pubKey string) error { return err } + s.authLock.Lock() tk, ok := s.trustedKeys[user] if !ok { tk = make(map[string]bool) @@ -84,7 +152,11 @@ func (s *SSHServer) AddAuthorizedKey(user, pubKey string) error { } tk[string(pk.Marshal())] = true - s.l.WithField("sshKey", pubKey).WithField("sshUser", user).Info("Authorized ssh key") + s.authLock.Unlock() + s.l.Info("Authorized ssh key", + "sshKey", pubKey, + "sshUser", user, + ) return nil } @@ -93,111 +165,108 @@ func (s *SSHServer) RegisterCommand(c *Command) { s.commands.Insert(c.Name, c) } -// Run begins listening and accepting connections +// Run begins listening and accepting connections. Each invocation derives a fresh per-Run context +// from the constructor-supplied ctx so a Stop+Run sequence (used by config reload) starts clean +// rather than carrying a permanently-cancelled context across runs. func (s *SSHServer) Run(addr string) error { - var err error - s.listener, err = net.Listen("tcp", addr) + if s.ctx.Err() != nil { + return s.ctx.Err() + } + + listener, err := net.Listen("tcp", addr) if err != nil { return err } + // s.listener is the public handle Stop uses to interrupt the active run; listener (the local) is what + // this run owns. They start equal but a fast reload may overwrite s.listener with the next run's + // listener before this run's watcher fires, so each run must close its own listener via the local + // reference. + s.listener = listener + + runCtx, cancel := context.WithCancel(s.ctx) + defer cancel() + + // Close the listener when this run's context is cancelled. That can come from the parent + // (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling + // run cancels through Stop closing the listener. net.Listener.Close is idempotent so a duplicate + // close from Stop is benign. + go func() { + <-runCtx.Done() + if err := listener.Close(); err != nil && !errors.Is(err, net.ErrClosed) { + s.l.Warn("Failed to close the sshd listener", "error", err) + } + }() - s.l.WithField("sshListener", addr).Info("SSH server is listening") + s.l.Info("SSH server is listening", "sshListener", addr) // Run loops until there is an error - s.run() - s.closeSessions() + s.run(runCtx, listener) s.l.Info("SSH server stopped listening") // We don't return an error because run logs for us return nil } -func (s *SSHServer) run() { +func (s *SSHServer) run(ctx context.Context, listener net.Listener) { for { - c, err := s.listener.Accept() + c, err := listener.Accept() if err != nil { if !errors.Is(err, net.ErrClosed) { - s.l.WithError(err).Warn("Error in listener, shutting down") + s.l.Warn("Error in listener, shutting down", "error", err) } return } - - conn, chans, reqs, err := ssh.NewServerConn(c, s.config) - fp := "" - if conn != nil { - fp = conn.Permissions.Extensions["fp"] - } - - if err != nil { - l := s.l.WithError(err).WithField("remoteAddress", c.RemoteAddr()) + go func(c net.Conn) { + // NewServerConn may block while waiting for the client to complete the handshake. + // Ensure that a bad client doesn't hurt us by checking for the parent context + // cancellation before calling NewServerConn, and forcing the socket to close when + // the context is cancelled. + sessionContext, sessionCancel := context.WithCancel(ctx) + go func() { + <-sessionContext.Done() + c.Close() + }() + conn, chans, reqs, err := ssh.NewServerConn(c, s.config) + fp := "" if conn != nil { - l = l.WithField("sshUser", conn.User()) - conn.Close() + fp = conn.Permissions.Extensions["fp"] } - if fp != "" { - l = l.WithField("sshFingerprint", fp) + + if err != nil { + l := s.l.With( + "error", err, + "remoteAddress", c.RemoteAddr(), + ) + if conn != nil { + l = l.With("sshUser", conn.User()) + conn.Close() + } + if fp != "" { + l = l.With("sshFingerprint", fp) + } + l.Warn("failed to handshake") + sessionCancel() + return } - l.Warn("failed to handshake") - continue - } - l := s.l.WithField("sshUser", conn.User()) - l.WithField("remoteAddress", c.RemoteAddr()).WithField("sshFingerprint", fp).Info("ssh user logged in") - - session := NewSession(s.commands, conn, chans, l.WithField("subsystem", "sshd.session")) - s.connsLock.Lock() - s.counter++ - counter := s.counter - s.conns[counter] = session - s.connsLock.Unlock() - - go ssh.DiscardRequests(reqs) - go func() { - <-session.exitChan - s.l.WithField("id", counter).Debug("closing conn") - s.connsLock.Lock() - delete(s.conns, counter) - s.connsLock.Unlock() - }() + l := s.l.With("sshUser", conn.User()) + l.Info("ssh user logged in", + "remoteAddress", c.RemoteAddr(), + "sshFingerprint", fp, + ) + + NewSession(s.commands, conn, chans, sessionCancel, l.With("subsystem", "sshd.session")) + + go ssh.DiscardRequests(reqs) + + }(c) } } func (s *SSHServer) Stop() { - // Close the listener, this will cause all session to terminate as well, see SSHServer.Run if s.listener != nil { if err := s.listener.Close(); err != nil { - s.l.WithError(err).Warn("Failed to close the sshd listener") + s.l.Warn("Failed to close the sshd listener", "error", err) } } } - -func (s *SSHServer) closeSessions() { - s.connsLock.Lock() - for _, c := range s.conns { - c.Close() - } - s.connsLock.Unlock() -} - -func (s *SSHServer) matchPubKey(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) { - pk := string(pubKey.Marshal()) - fp := ssh.FingerprintSHA256(pubKey) - - tk, ok := s.trustedKeys[c.User()] - if !ok { - return nil, fmt.Errorf("unknown user %s", c.User()) - } - - _, ok = tk[pk] - if !ok { - return nil, fmt.Errorf("unknown public key for %s (%s)", c.User(), fp) - } - - return &ssh.Permissions{ - // Record the public key used for authentication. - Extensions: map[string]string{ - "fp": fp, - "user": c.User(), - }, - }, nil -} diff --git a/sshd/session.go b/sshd/session.go index bba2a55c..1c8e1a9b 100644 --- a/sshd/session.go +++ b/sshd/session.go @@ -2,36 +2,36 @@ package sshd import ( "fmt" + "log/slog" "sort" "strings" "github.com/anmitsu/go-shlex" "github.com/armon/go-radix" - "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) type session struct { - l *logrus.Entry + l *slog.Logger c *ssh.ServerConn - term *terminal.Terminal + term *term.Terminal commands *radix.Tree - exitChan chan bool + cancel func() } -func NewSession(commands *radix.Tree, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, l *logrus.Entry) *session { +func NewSession(commands *radix.Tree, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, cancel func(), l *slog.Logger) *session { s := &session{ commands: radix.NewFromMap(commands.ToMap()), l: l, c: conn, - exitChan: make(chan bool), + cancel: cancel, } s.commands.Insert("logout", &Command{ Name: "logout", ShortDescription: "Ends the current session", - Callback: func(a interface{}, args []string, w StringWriter) error { + Callback: func(a any, args []string, w StringWriter) error { s.Close() return nil }, @@ -42,16 +42,17 @@ func NewSession(commands *radix.Tree, conn *ssh.ServerConn, chans <-chan ssh.New } func (s *session) handleChannels(chans <-chan ssh.NewChannel) { + defer s.Close() for newChannel := range chans { if newChannel.ChannelType() != "session" { - s.l.WithField("sshChannelType", newChannel.ChannelType()).Error("unknown channel type") + s.l.Error("unknown channel type", "sshChannelType", newChannel.ChannelType()) newChannel.Reject(ssh.UnknownChannelType, "unknown channel type") continue } channel, requests, err := newChannel.Accept() if err != nil { - s.l.WithError(err).Warn("could not accept channel") + s.l.Warn("could not accept channel", "error", err) continue } @@ -62,7 +63,6 @@ func (s *session) handleChannels(chans <-chan ssh.NewChannel) { func (s *session) handleRequests(in <-chan *ssh.Request, channel ssh.Channel) { for req := range in { var err error - //TODO: maybe support window sizing? switch req.Type { case "shell": if s.term == nil { @@ -89,29 +89,25 @@ func (s *session) handleRequests(in <-chan *ssh.Request, channel ssh.Channel) { req.Reply(true, nil) s.dispatchCommand(payload.Value, &stringWriter{channel}) - //TODO: Fix error handling and report the proper status back status := struct{ Status uint32 }{uint32(0)} - //TODO: I think this is how we shut down a shell as well? channel.SendRequest("exit-status", false, ssh.Marshal(status)) channel.Close() return default: - s.l.WithField("sshRequest", req.Type).Debug("Rejected unknown request") + s.l.Debug("Rejected unknown request", "sshRequest", req.Type) err = req.Reply(false, nil) } if err != nil { - s.l.WithError(err).Info("Error handling ssh session requests") - s.Close() + s.l.Info("Error handling ssh session requests", "error", err) return } } } -func (s *session) createTerm(channel ssh.Channel) *terminal.Terminal { - //TODO: PS1 with nebula cert name - term := terminal.NewTerminal(channel, s.c.User()+"@nebula > ") +func (s *session) createTerm(channel ssh.Channel) *term.Terminal { + term := term.NewTerminal(channel, s.c.User()+"@nebula > ") term.AutoCompleteCallback = func(line string, pos int, key rune) (newLine string, newPos int, ok bool) { // key 9 is tab if key == 9 { @@ -127,17 +123,15 @@ func (s *session) createTerm(channel ssh.Channel) *terminal.Terminal { return "", 0, false } - go s.handleInput(channel) + go s.handleInput() return term } -func (s *session) handleInput(channel ssh.Channel) { - defer s.Close() +func (s *session) handleInput() { w := &stringWriter{w: s.term} for { line, err := s.term.ReadLine() if err != nil { - //TODO: log break } @@ -148,7 +142,6 @@ func (s *session) handleInput(channel ssh.Channel) { func (s *session) dispatchCommand(line string, w StringWriter) { args, err := shlex.Split(line, true) if err != nil { - //todo: LOG IT return } @@ -159,13 +152,11 @@ func (s *session) dispatchCommand(line string, w StringWriter) { c, err := lookupCommand(s.commands, args[0]) if err != nil { - //TODO: handle the error return } if c == nil { err := w.WriteLine(fmt.Sprintf("did not understand: %s", line)) - //TODO: log error _ = err dumpCommands(s.commands, w) @@ -177,14 +168,10 @@ func (s *session) dispatchCommand(line string, w StringWriter) { return } - err = execCommand(c, args[1:], w) - if err != nil { - //TODO: log the error - } - return + _ = execCommand(c, args[1:], w) } func (s *session) Close() { s.c.Close() - s.exitChan <- true + s.cancel() } diff --git a/stats.go b/stats.go index c88c45cc..98c2baf2 100644 --- a/stats.go +++ b/stats.go @@ -1,13 +1,16 @@ package nebula import ( + "context" "errors" "fmt" - "log" + "log/slog" "net" "net/http" "runtime" "strconv" + "sync" + "sync/atomic" "time" graphite "github.com/cyberdelia/go-metrics-graphite" @@ -15,113 +18,350 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" ) -// startStats initializes stats from config. On success, if any further work -// is needed to serve stats, it returns a func to handle that work. If no -// work is needed, it'll return nil. On failure, it returns nil, error. -func startStats(l *logrus.Logger, c *config.C, buildVersion string, configTest bool) (func(), error) { - mType := c.GetString("stats.type", "") - if mType == "" || mType == "none" { - return nil, nil - } +// statsServer owns nebula's stats subsystem: the periodic metric capture +// goroutine and (for prometheus) an HTTP listener. It mirrors the lifecycle +// shape of dnsServer: constructor wires the reload callback, reload records +// config, Start builds and runs the runtime, Stop tears it down. +type statsServer struct { + l *slog.Logger + ctx context.Context + buildVersion string + configTest bool + + // enabled mirrors "stats configured to a real backend". Start consults + // it so callers don't need to know the gating rules. + enabled atomic.Bool + + runMu sync.Mutex + runCfg *statsConfig + run *statsRuntime // non-nil while a runtime is live +} + +// statsRuntime is the live state owned by a single Start invocation. Start +// stashes a pointer under runMu; Stop and Start's own exit path use pointer +// equality to tell "my runtime" apart from one that replaced it after a +// reload. +type statsRuntime struct { + cancel context.CancelFunc + listener *http.Server // nil for graphite +} + +// statsConfig is the snapshot of stats-related config that drives the runtime. +// It is comparable with == so reload can detect "no change" cheaply. +type statsConfig struct { + typ string + interval time.Duration + graphite graphiteConfig + prom promConfig +} + +type graphiteConfig struct { + protocol string + host string + // resolvedAddr is the string form of host resolved at config-load time. + // Including it in the struct means a SIGHUP picks up DNS changes even + // when stats.host hasn't been edited. + resolvedAddr string + prefix string +} + +type promConfig struct { + listen string + path string + namespace string + subsystem string +} - interval := c.GetDuration("stats.interval", 0) - if interval == 0 { - return nil, fmt.Errorf("stats.interval was an invalid duration: %s", c.GetString("stats.interval", "")) +// newStatsServerFromConfig builds a statsServer, applies the initial config, +// and registers a reload callback. The reload callback is registered before +// the initial config is applied so a SIGHUP can later enable, fix, or disable +// stats even if the initial application failed. +// +// Start is safe to call unconditionally: it no-ops when stats are disabled. +// The returned pointer is always non-nil, even on error. +func newStatsServerFromConfig(ctx context.Context, l *slog.Logger, c *config.C, buildVersion string, configTest bool) (*statsServer, error) { + s := &statsServer{ + l: l, + ctx: ctx, + buildVersion: buildVersion, + configTest: configTest, } - var startFn func() - switch mType { - case "graphite": - err := startGraphiteStats(l, interval, c, configTest) - if err != nil { - return nil, err + c.RegisterReloadCallback(func(c *config.C) { + if err := s.reload(c, false); err != nil { + s.l.Error("Failed to reload stats from config", "error", err) } - case "prometheus": - var err error - startFn, err = startPrometheusStats(l, interval, c, buildVersion, configTest) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("stats.type was not understood: %s", mType) + }) + + if err := s.reload(c, true); err != nil { + return s, err } + return s, nil +} - metrics.RegisterDebugGCStats(metrics.DefaultRegistry) - metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) +// reload records the latest config. On the initial call it only records it; +// Control.Start is what launches the first runtime via statsStart. On later +// calls it reconciles the running runtime with the new config: +// +// - newly enabled -> spawn Start +// - newly disabled -> Stop the runtime +// - config changed (still enabled) -> Stop the old, Start the new +// - no change -> no-op +func (s *statsServer) reload(c *config.C, initial bool) error { + newCfg, err := loadStatsConfig(c) + if err != nil { + return err + } + enabled := newCfg.typ != "" && newCfg.typ != "none" + + s.runMu.Lock() + sameCfg := s.runCfg != nil && *s.runCfg == newCfg + s.runCfg = &newCfg + running := s.run != nil + s.runMu.Unlock() + + s.enabled.Store(enabled) - go metrics.CaptureDebugGCStats(metrics.DefaultRegistry, interval) - go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, interval) + if initial || sameCfg { + return nil + } - return startFn, nil + if running { + s.Stop() + } + if enabled && !s.configTest { + go s.Start() + } + return nil } -func startGraphiteStats(l *logrus.Logger, i time.Duration, c *config.C, configTest bool) error { - proto := c.GetString("stats.protocol", "tcp") - host := c.GetString("stats.host", "") - if host == "" { - return errors.New("stats.host can not be empty") +// Start builds the runtime from the latest config, spawns the capture loop, +// and blocks until Stop is called or ctx fires. For prometheus it also serves +// the HTTP listener. For graphite it blocks on the capture loop's context. +// Safe to call when stats are disabled or already running (both no-op). +func (s *statsServer) Start() { + if !s.enabled.Load() || s.configTest { + return } - prefix := c.GetString("stats.prefix", "nebula") - addr, err := net.ResolveTCPAddr(proto, host) - if err != nil { - return fmt.Errorf("error while setting up graphite sink: %s", err) + s.runMu.Lock() + if s.ctx.Err() != nil || s.run != nil || s.runCfg == nil { + s.runMu.Unlock() + return } + cfg := *s.runCfg + captureFns, listener := s.buildRuntime(cfg) + runCtx, cancel := context.WithCancel(s.ctx) + rt := &statsRuntime{cancel: cancel, listener: listener} + s.run = rt + s.runMu.Unlock() - if !configTest { - l.Infof("Starting graphite. Interval: %s, prefix: %s, addr: %s", i, prefix, addr) - go graphite.Graphite(metrics.DefaultRegistry, i, prefix, addr) + go captureStatsLoop(runCtx, cfg.interval, captureFns) + + cleanExit := true + if listener == nil { + // Graphite: no HTTP listener to serve; block until teardown. + <-runCtx.Done() + } else { + cleanExit = s.serveListener(listener) } - return nil + + // Clear our runtime only if nothing has replaced it. Stop races through + // here too but leaves s.run == nil, so the pointer check skips. + s.runMu.Lock() + if s.run == rt { + rt.cancel() + s.run = nil + // A listener that exited with an error (e.g., bind conflict) leaves + // runCfg cached as if it were applied. Drop it so a SIGHUP with the + // same config re-triggers Start once the user fixes the underlying + // problem. + if !cleanExit { + s.runCfg = nil + } + } + s.runMu.Unlock() } -func startPrometheusStats(l *logrus.Logger, i time.Duration, c *config.C, buildVersion string, configTest bool) (func(), error) { - namespace := c.GetString("stats.namespace", "") - subsystem := c.GetString("stats.subsystem", "") +// serveListener runs ListenAndServe and ensures ctx cancellation unblocks it. +// Returns true if the listener exited cleanly (Stop, ctx cancellation, or any +// other http.ErrServerClosed path), false on an unexpected error. +func (s *statsServer) serveListener(listener *http.Server) bool { + // Per-invocation watcher: ctx cancellation triggers a listener shutdown + // which in turn unblocks ListenAndServe. Closing `done` on exit keeps + // the watcher from outliving this call. + done := make(chan struct{}) + go func() { + select { + case <-s.ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := listener.Shutdown(shutdownCtx); err != nil { + s.l.Warn("Failed to shut down prometheus stats listener", "error", err) + } + case <-done: + } + }() + defer close(done) - listen := c.GetString("stats.listen", "") - if listen == "" { - return nil, fmt.Errorf("stats.listen should not be empty") + s.l.Info("Starting prometheus stats listener", "addr", listener.Addr) + err := listener.ListenAndServe() + if err == nil || errors.Is(err, http.ErrServerClosed) { + return true } + s.l.Error("Prometheus stats listener exited", "error", err) + return false +} - path := c.GetString("stats.path", "") - if path == "" { - return nil, fmt.Errorf("stats.path should not be empty") +// Stop tears down the active runtime, if any. Idempotent. +func (s *statsServer) Stop() { + s.runMu.Lock() + rt := s.run + s.run = nil + s.runMu.Unlock() + if rt == nil { + return + } + rt.cancel() + if rt.listener != nil { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := rt.listener.Shutdown(shutdownCtx); err != nil { + s.l.Warn("Failed to shut down prometheus stats listener", "error", err) + } + cancel() } +} + +// buildRuntime produces the capture functions and, for prometheus, an un-served +// http.Server from cfg. cfg has already been validated by loadStatsConfig. +func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) { + // rcrowley/go-metrics guards these registrations with a private sync.Once, + // so subsequent reloads are no-ops. + metrics.RegisterDebugGCStats(metrics.DefaultRegistry) + metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry) - pr := prometheus.NewRegistry() - pClient := mp.NewPrometheusProvider(metrics.DefaultRegistry, namespace, subsystem, pr, i) - if !configTest { - go pClient.UpdatePrometheusMetrics() + captureFns := []func(){ + func() { metrics.CaptureDebugGCStatsOnce(metrics.DefaultRegistry) }, + func() { metrics.CaptureRuntimeMemStatsOnce(metrics.DefaultRegistry) }, } - // Export our version information as labels on a static gauge - g := prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "info", - Help: "Version information for the Nebula binary", - ConstLabels: prometheus.Labels{ - "version": buildVersion, - "goversion": runtime.Version(), - "boringcrypto": strconv.FormatBool(boringEnabled()), - }, - }) - pr.MustRegister(g) - g.Set(1) - - var startFn func() - if !configTest { - startFn = func() { - l.Infof("Prometheus stats listening on %s at %s", listen, path) - http.Handle(path, promhttp.HandlerFor(pr, promhttp.HandlerOpts{ErrorLog: l})) - log.Fatal(http.ListenAndServe(listen, nil)) + switch cfg.typ { + case "graphite": + // loadStatsConfig already resolved and validated the address; re-parse + // the resolved form (no DNS lookup) to get a *net.TCPAddr. + addr, _ := net.ResolveTCPAddr(cfg.graphite.protocol, cfg.graphite.resolvedAddr) + gcfg := graphite.Config{ + Addr: addr, + Registry: metrics.DefaultRegistry, + FlushInterval: cfg.interval, + DurationUnit: time.Nanosecond, + Prefix: cfg.graphite.prefix, + Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999}, } + captureFns = append(captureFns, func() { + if err := graphite.Once(gcfg); err != nil { + s.l.Error("Graphite export failed", "error", err) + } + }) + s.l.Info("Starting graphite stats", + "interval", cfg.interval, + "prefix", cfg.graphite.prefix, + "addr", addr, + ) + return captureFns, nil + + case "prometheus": + pr := prometheus.NewRegistry() + pClient := mp.NewPrometheusProvider(metrics.DefaultRegistry, cfg.prom.namespace, cfg.prom.subsystem, pr, cfg.interval) + captureFns = append(captureFns, func() { + if err := pClient.UpdatePrometheusMetricsOnce(); err != nil { + s.l.Error("Prometheus metrics update failed", "error", err) + } + }) + + g := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: cfg.prom.namespace, + Subsystem: cfg.prom.subsystem, + Name: "info", + Help: "Version information for the Nebula binary", + ConstLabels: prometheus.Labels{ + "version": s.buildVersion, + "goversion": runtime.Version(), + "boringcrypto": strconv.FormatBool(boringEnabled()), + }, + }) + pr.MustRegister(g) + g.Set(1) + + // promhttp.HandlerOpts.ErrorLog needs a stdlib-shaped Println logger, + // so bridge our slog.Logger back to a *log.Logger that emits at Error. + errLog := slog.NewLogLogger(s.l.Handler(), slog.LevelError) + mux := http.NewServeMux() + mux.Handle(cfg.prom.path, promhttp.HandlerFor(pr, promhttp.HandlerOpts{ErrorLog: errLog})) + return captureFns, &http.Server{Addr: cfg.prom.listen, Handler: mux} + } + return captureFns, nil +} + +// captureStatsLoop runs each fn on every tick of d until ctx is cancelled. +func captureStatsLoop(ctx context.Context, d time.Duration, fns []func()) { + t := time.NewTicker(d) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + for _, fn := range fns { + fn() + } + } + } +} + +func loadStatsConfig(c *config.C) (statsConfig, error) { + cfg := statsConfig{ + typ: c.GetString("stats.type", ""), + } + if cfg.typ == "" || cfg.typ == "none" { + return cfg, nil + } + + cfg.interval = c.GetDuration("stats.interval", 0) + if cfg.interval <= 0 { + return cfg, fmt.Errorf("stats.interval was an invalid duration: %s", c.GetString("stats.interval", "")) + } + + switch cfg.typ { + case "graphite": + cfg.graphite.protocol = c.GetString("stats.protocol", "tcp") + cfg.graphite.host = c.GetString("stats.host", "") + if cfg.graphite.host == "" { + return cfg, errors.New("stats.host can not be empty") + } + addr, err := net.ResolveTCPAddr(cfg.graphite.protocol, cfg.graphite.host) + if err != nil { + return cfg, fmt.Errorf("error while setting up graphite sink: %s", err) + } + cfg.graphite.resolvedAddr = addr.String() + cfg.graphite.prefix = c.GetString("stats.prefix", "nebula") + case "prometheus": + cfg.prom.listen = c.GetString("stats.listen", "") + if cfg.prom.listen == "" { + return cfg, errors.New("stats.listen should not be empty") + } + cfg.prom.path = c.GetString("stats.path", "") + if cfg.prom.path == "" { + return cfg, errors.New("stats.path should not be empty") + } + cfg.prom.namespace = c.GetString("stats.namespace", "") + cfg.prom.subsystem = c.GetString("stats.subsystem", "") + default: + return cfg, fmt.Errorf("stats.type was not understood: %s", cfg.typ) } - return startFn, nil + return cfg, nil } diff --git a/stats_test.go b/stats_test.go new file mode 100644 index 00000000..20b17c0e --- /dev/null +++ b/stats_test.go @@ -0,0 +1,410 @@ +package nebula + +import ( + "context" + "io" + "log/slog" + "net" + "strconv" + "testing" + "time" + + "github.com/slackhq/nebula/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestStatsServer(t *testing.T) (*statsServer, *config.C) { + t.Helper() + l := slog.New(slog.DiscardHandler) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + return &statsServer{ + l: l, + ctx: ctx, + }, config.NewC(l) +} + +func setStatsConfig(c *config.C, m map[string]any) { + c.Settings["stats"] = m +} + +func currentRuntime(s *statsServer) *statsRuntime { + s.runMu.Lock() + defer s.runMu.Unlock() + return s.run +} + +func TestStatsServer_reload_initial_disabled(t *testing.T) { + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{"type": "none"}) + + require.NoError(t, s.reload(c, true)) + assert.False(t, s.enabled.Load()) + assert.Nil(t, currentRuntime(s)) +} + +func TestStatsServer_reload_initial_invalidInterval(t *testing.T) { + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "graphite", + "host": "127.0.0.1:0", + "prefix": "test", + }) + + err := s.reload(c, true) + require.Error(t, err) + assert.False(t, s.enabled.Load()) +} + +func TestStatsServer_reload_initial_unknownType(t *testing.T) { + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "carbon", + "interval": "1s", + }) + + err := s.reload(c, true) + require.Error(t, err) + assert.False(t, s.enabled.Load()) +} + +func TestStatsServer_reload_unchanged_noOp(t *testing.T) { + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{"type": "none"}) + + require.NoError(t, s.reload(c, true)) + require.NoError(t, s.reload(c, false)) + assert.False(t, s.enabled.Load()) +} + +func TestStatsServer_reload_initial_graphite(t *testing.T) { + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "graphite", + "interval": "1s", + "protocol": "tcp", + "host": "127.0.0.1:2003", + "prefix": "test", + }) + + require.NoError(t, s.reload(c, true)) + assert.True(t, s.enabled.Load()) + // reload only records config; Start builds the runtime. + assert.Nil(t, currentRuntime(s)) +} + +func TestStatsServer_reload_initial_prometheus(t *testing.T) { + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:0", + "path": "/metrics", + }) + + require.NoError(t, s.reload(c, true)) + assert.True(t, s.enabled.Load()) + // reload only records config; Start builds the runtime. + assert.Nil(t, currentRuntime(s)) +} + +func TestStatsServer_Start_graphite_blocksUntilStop(t *testing.T) { + sink := newGraphiteSink(t) + defer sink.Close() + + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "graphite", + "interval": "1s", + "protocol": "tcp", + "host": sink.Addr(), + "prefix": "test", + }) + require.NoError(t, s.reload(c, true)) + + done := make(chan struct{}) + go func() { + s.Start() + close(done) + }() + + // Wait for Start to publish runtime state. + waitFor(t, func() bool { return currentRuntime(s) != nil }) + rt := currentRuntime(s) + require.NotNil(t, rt) + assert.Nil(t, rt.listener, "graphite has no listener") + + s.Stop() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("graphite Start did not return after Stop") + } + assert.Nil(t, currentRuntime(s)) +} + +func TestStatsServer_StartStop_lifecycle(t *testing.T) { + port := freeTCPPort(t) + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:" + port, + "path": "/metrics", + }) + require.NoError(t, s.reload(c, true)) + + done := make(chan struct{}) + go func() { + s.Start() + close(done) + }() + + waitForListening(t, "127.0.0.1:"+port) + rt := currentRuntime(s) + require.NotNil(t, rt) + require.NotNil(t, rt.listener) + + s.Stop() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Start did not return after Stop") + } + assert.Nil(t, currentRuntime(s)) +} + +func TestStatsServer_reload_disable_stopsRunningRuntime(t *testing.T) { + port := freeTCPPort(t) + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:" + port, + "path": "/metrics", + }) + require.NoError(t, s.reload(c, true)) + + done := make(chan struct{}) + go func() { + s.Start() + close(done) + }() + waitForListening(t, "127.0.0.1:"+port) + + setStatsConfig(c, map[string]any{"type": "none"}) + require.NoError(t, s.reload(c, false)) + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Start did not return after reload disabled stats") + } + assert.False(t, s.enabled.Load()) + assert.Nil(t, currentRuntime(s)) +} + +func TestStatsServer_reload_changeListener_restartsListener(t *testing.T) { + port1 := freeTCPPort(t) + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:" + port1, + "path": "/metrics", + }) + require.NoError(t, s.reload(c, true)) + + firstDone := make(chan struct{}) + go func() { + s.Start() + close(firstDone) + }() + waitForListening(t, "127.0.0.1:"+port1) + first := currentRuntime(s) + require.NotNil(t, first) + + port2 := freeTCPPort(t) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:" + port2, + "path": "/metrics", + }) + require.NoError(t, s.reload(c, false)) + + select { + case <-firstDone: + case <-time.After(5 * time.Second): + t.Fatal("old Start did not return after reload") + } + + waitForListening(t, "127.0.0.1:"+port2) + second := currentRuntime(s) + require.NotNil(t, second) + assert.NotSame(t, first, second, "expected a new runtime after listen address change") + + s.Stop() +} + +func TestStatsServer_Stop_beforeStart_doesNotBlock(t *testing.T) { + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:0", + "path": "/metrics", + }) + require.NoError(t, s.reload(c, true)) + + stopped := make(chan struct{}) + go func() { + s.Stop() + close(stopped) + }() + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("Stop hung with no runtime started") + } +} + +func TestStatsServer_configTest_validatesWithoutSpawning(t *testing.T) { + s, c := newTestStatsServer(t) + s.configTest = true + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:0", + "path": "/metrics", + }) + + require.NoError(t, s.reload(c, true)) + s.Start() + assert.Nil(t, currentRuntime(s)) +} + +func TestStatsServer_ctxCancel_unblocksStart(t *testing.T) { + // Ensures ctx cancellation alone (no explicit Stop) tears down both + // graphite and prom Start invocations. + port := freeTCPPort(t) + l := slog.New(slog.DiscardHandler) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := &statsServer{l: l, ctx: ctx} + c := config.NewC(l) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:" + port, + "path": "/metrics", + }) + require.NoError(t, s.reload(c, true)) + + done := make(chan struct{}) + go func() { + s.Start() + close(done) + }() + waitForListening(t, "127.0.0.1:"+port) + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Start did not return after ctx cancel") + } +} + +func TestStatsServer_listenerBindFailure_sameCfgReloadRetries(t *testing.T) { + // Hold the port so ListenAndServe will fail on first Start. + blocker, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + port := strconv.Itoa(blocker.Addr().(*net.TCPAddr).Port) + + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:" + port, + "path": "/metrics", + }) + require.NoError(t, s.reload(c, true)) + + done := make(chan struct{}) + go func() { + s.Start() + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Start did not return after bind failure") + } + // Bind failure should have dropped the cached config so a same-cfg + // SIGHUP can retry. + s.runMu.Lock() + cfgAfterFailure := s.runCfg + s.runMu.Unlock() + assert.Nil(t, cfgAfterFailure) + + // Free the port and reload with the same config; Start should fire again. + require.NoError(t, blocker.Close()) + require.NoError(t, s.reload(c, false)) + + waitForListening(t, "127.0.0.1:"+port) + require.NotNil(t, currentRuntime(s)) + + s.Stop() +} + +func waitForListening(t *testing.T, addr string) { + t.Helper() + waitFor(t, func() bool { + conn, err := net.DialTimeout("tcp", addr, 200*time.Millisecond) + if err != nil { + return false + } + _ = conn.Close() + return true + }) +} + +// graphiteSink is a minimal TCP accept-and-discard server so graphite.Once +// calls in tests don't spam error logs or wedge on connection refused. +type graphiteSink struct { + ln net.Listener +} + +func newGraphiteSink(t *testing.T) *graphiteSink { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + g := &graphiteSink{ln: ln} + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func(c net.Conn) { + _, _ = io.Copy(io.Discard, c) + _ = c.Close() + }(conn) + } + }() + return g +} + +func (g *graphiteSink) Addr() string { return g.ln.Addr().String() } +func (g *graphiteSink) Close() { _ = g.ln.Close() } + +func freeTCPPort(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + port := ln.Addr().(*net.TCPAddr).Port + require.NoError(t, ln.Close()) + return strconv.Itoa(port) +} diff --git a/test/assert.go b/test/assert.go index 6c6c7951..1856877e 100644 --- a/test/assert.go +++ b/test/assert.go @@ -2,6 +2,7 @@ package test import ( "fmt" + "net/netip" "reflect" "testing" "time" @@ -12,7 +13,7 @@ import ( // AssertDeepCopyEqual checks to see if two variables have the same values but DO NOT share any memory // There is currently a special case for `time.loc` (as this code traverses into unexported fields) -func AssertDeepCopyEqual(t *testing.T, a interface{}, b interface{}) { +func AssertDeepCopyEqual(t *testing.T, a any, b any) { v1 := reflect.ValueOf(a) v2 := reflect.ValueOf(b) @@ -24,6 +25,11 @@ func AssertDeepCopyEqual(t *testing.T, a interface{}, b interface{}) { } func traverseDeepCopy(t *testing.T, v1 reflect.Value, v2 reflect.Value, name string) bool { + if v1.Type() == v2.Type() && v1.Type() == reflect.TypeOf(netip.Addr{}) { + // Ignore netip.Addr types since they reuse an interned global value + return false + } + switch v1.Kind() { case reflect.Array: for i := 0; i < v1.Len(); i++ { diff --git a/test/logger.go b/test/logger.go index b5a717d8..faab0b69 100644 --- a/test/logger.go +++ b/test/logger.go @@ -1,29 +1,73 @@ package test import ( + "context" "io" + "log/slog" "os" + "time" - "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/logging" ) -func NewLogger() *logrus.Logger { - l := logrus.New() - +// NewLogger returns a *slog.Logger suitable for use in tests. Output goes to +// io.Discard by default; set TEST_LOGS=1 (info), 2 (debug), or 3 (trace) to +// stream output to stderr for local debugging. +func NewLogger() *slog.Logger { v := os.Getenv("TEST_LOGS") if v == "" { - l.SetOutput(io.Discard) - return l + return slog.New(slog.DiscardHandler) } + level := slog.LevelInfo switch v { case "2": - l.SetLevel(logrus.DebugLevel) + level = slog.LevelDebug case "3": - l.SetLevel(logrus.TraceLevel) - default: - l.SetLevel(logrus.InfoLevel) + level = logging.LevelTrace } + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})) +} + +// NewLoggerWithOutput returns a *slog.Logger whose text output is captured by +// w. Timestamps are suppressed so tests can assert on exact output without +// baking the current time into expected strings. +func NewLoggerWithOutput(w io.Writer) *slog.Logger { + return slog.New(&stripTimeHandler{inner: slog.NewTextHandler(w, nil)}) +} + +// NewLoggerWithOutputAndLevel is NewLoggerWithOutput with an explicit level +// so tests can exercise Enabled-gated paths. +func NewLoggerWithOutputAndLevel(w io.Writer, level slog.Level) *slog.Logger { + return slog.New(&stripTimeHandler{inner: slog.NewTextHandler(w, &slog.HandlerOptions{Level: level})}) +} + +// NewJSONLoggerWithOutput returns a *slog.Logger emitting JSON to w with +// timestamps suppressed, for tests that pin the JSON shape. +func NewJSONLoggerWithOutput(w io.Writer, level slog.Level) *slog.Logger { + return slog.New(&stripTimeHandler{inner: slog.NewJSONHandler(w, &slog.HandlerOptions{Level: level})}) +} + +// stripTimeHandler zeros each record's time before delegating so slog's +// built-in handlers skip emitting the time attribute. Used to avoid +// timestamp-dependent assertions in tests without resorting to ReplaceAttr. +type stripTimeHandler struct { + inner slog.Handler +} + +func (h *stripTimeHandler) Enabled(ctx context.Context, l slog.Level) bool { + return h.inner.Enabled(ctx, l) +} + +func (h *stripTimeHandler) Handle(ctx context.Context, r slog.Record) error { + r.Time = time.Time{} + return h.inner.Handle(ctx, r) +} + +func (h *stripTimeHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &stripTimeHandler{inner: h.inner.WithAttrs(attrs)} +} - return l +func (h *stripTimeHandler) WithGroup(name string) slog.Handler { + return &stripTimeHandler{inner: h.inner.WithGroup(name)} } diff --git a/test/tun.go b/test/tun.go deleted file mode 100644 index 86656c92..00000000 --- a/test/tun.go +++ /dev/null @@ -1,43 +0,0 @@ -package test - -import ( - "errors" - "io" - "net" - - "github.com/slackhq/nebula/iputil" -) - -type NoopTun struct{} - -func (NoopTun) RouteFor(iputil.VpnIp) iputil.VpnIp { - return 0 -} - -func (NoopTun) Activate() error { - return nil -} - -func (NoopTun) Cidr() *net.IPNet { - return nil -} - -func (NoopTun) Name() string { - return "noop" -} - -func (NoopTun) Read([]byte) (int, error) { - return 0, nil -} - -func (NoopTun) Write([]byte) (int, error) { - return 0, nil -} - -func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) { - return nil, errors.New("unsupported") -} - -func (NoopTun) Close() error { - return nil -} diff --git a/timeout.go b/timeout.go index c1b4c398..96bf688b 100644 --- a/timeout.go +++ b/timeout.go @@ -8,6 +8,23 @@ import ( // How many timer objects should be cached const timerCacheMax = 50000 +// TimerWheel is a hashed timing wheel: a fixed slot array indexed by (now + delay) % wheelLen, +// with each slot a singly linked list of items due in that bucket. +// Adds are O(1), Purges return items in arrival-within-slot order, and an internal cache of TimeoutItems +// keeps steady-state inserts allocation-free. +// +// The TimerWheel does not handle concurrency or lifecycle on its own. +// Callers drive Advance/Purge from their own ticker loop, take their own locks (or use LockingTimerWheel), +// and decide whether to keep ticking when the wheel is empty. +// +// Pick a TimerWheel when scheduling is high-rate and uniform: line-rate conntrack inserts, +// per-tunnel traffic checks at fixed intervals. O(1) insert plus the item cache means the hot path doesn't allocate. +// Items added in the same tick are dispatched together when that slot rotates current, +// which amortizes the cost of waking the worker. +// +// Pick a Scheduler when delay precision matters or scheduling is sparse or uneven. +// The wheel rounds requested timeouts up to its tick resolution and clamps anything beyond its wheel duration; +// both are silent in this implementation. type TimerWheel[T any] struct { // Current tick current int diff --git a/timeout_test.go b/timeout_test.go index 3f81ff40..ffeecc55 100644 --- a/timeout_test.go +++ b/timeout_test.go @@ -1,6 +1,7 @@ package nebula import ( + "net/netip" "testing" "time" @@ -115,10 +116,10 @@ func TestTimerWheel_Purge(t *testing.T) { assert.Equal(t, 0, tw.current) fps := []firewall.Packet{ - {LocalIP: 1}, - {LocalIP: 2}, - {LocalIP: 3}, - {LocalIP: 4}, + {LocalAddr: netip.MustParseAddr("0.0.0.1")}, + {LocalAddr: netip.MustParseAddr("0.0.0.2")}, + {LocalAddr: netip.MustParseAddr("0.0.0.3")}, + {LocalAddr: netip.MustParseAddr("0.0.0.4")}, } tw.Add(fps[0], time.Second*1) @@ -133,7 +134,7 @@ func TestTimerWheel_Purge(t *testing.T) { assert.True(t, tw.lastTick.After(lastTick)) // Make sure we get all 4 packets back - for i := 0; i < 4; i++ { + for i := range 4 { p, has := tw.Purge() assert.True(t, has) assert.Equal(t, fps[i], p) @@ -148,7 +149,7 @@ func TestTimerWheel_Purge(t *testing.T) { // Make sure we cached the free'd items assert.Equal(t, 4, tw.itemsCached) ci := tw.itemCache - for i := 0; i < 4; i++ { + for range 4 { assert.NotNil(t, ci) ci = ci.Next } diff --git a/udp/conn.go b/udp/conn.go index a2c24a1f..30d89dec 100644 --- a/udp/conn.go +++ b/udp/conn.go @@ -1,31 +1,25 @@ package udp import ( + "net/netip" + "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/firewall" - "github.com/slackhq/nebula/header" ) const MTU = 9001 type EncReader func( - addr *Addr, - out []byte, - packet []byte, - header *header.H, - fwPacket *firewall.Packet, - lhh LightHouseHandlerFunc, - nb []byte, - q int, - localCache firewall.ConntrackCache, + addr netip.AddrPort, + payload []byte, ) type Conn interface { Rebind() error - LocalAddr() (*Addr, error) - ListenOut(r EncReader, lhf LightHouseHandlerFunc, cache *firewall.ConntrackCacheTicker, q int) - WriteTo(b []byte, addr *Addr) error + LocalAddr() (netip.AddrPort, error) + ListenOut(r EncReader) error + WriteTo(b []byte, addr netip.AddrPort) error ReloadConfig(c *config.C) + SupportsMultipleReaders() bool Close() error } @@ -34,13 +28,16 @@ type NoopConn struct{} func (NoopConn) Rebind() error { return nil } -func (NoopConn) LocalAddr() (*Addr, error) { - return nil, nil +func (NoopConn) LocalAddr() (netip.AddrPort, error) { + return netip.AddrPort{}, nil } -func (NoopConn) ListenOut(_ EncReader, _ LightHouseHandlerFunc, _ *firewall.ConntrackCacheTicker, _ int) { - return +func (NoopConn) ListenOut(_ EncReader) error { + return nil +} +func (NoopConn) SupportsMultipleReaders() bool { + return false } -func (NoopConn) WriteTo(_ []byte, _ *Addr) error { +func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil } func (NoopConn) ReloadConfig(_ *config.C) { diff --git a/udp/errors.go b/udp/errors.go new file mode 100644 index 00000000..12a84879 --- /dev/null +++ b/udp/errors.go @@ -0,0 +1,5 @@ +package udp + +import "errors" + +var ErrInvalidIPv6RemoteForSocket = errors.New("listener is IPv4, but writing to IPv6 remote") diff --git a/udp/netchange.go b/udp/netchange.go new file mode 100644 index 00000000..e7b21e6b --- /dev/null +++ b/udp/netchange.go @@ -0,0 +1,61 @@ +package udp + +import ( + "context" + "log/slog" + + "github.com/slackhq/nebula/config" +) + +// NetworkChangeMonitor rebinds the udp listener when the local network moves out from under it. +// +// Detection lives here in the udp package, next to the socket it concerns and the platform matrix that already knows +// which sockets go stale. What to do about a change — updating the lighthouse, requerying tunnels — is not the udp +// package's business, so Start takes the reaction as a plain function. Passing it at Start rather than holding it +// keeps this package from referencing whatever owns the rebind. +// +// On platforms whose sockets do not go stale, watchNetworkChanges hands back a nil channel and Start returns. +type NetworkChangeMonitor struct { + l *slog.Logger + ctx context.Context + enabled bool +} + +// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start +// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it. +func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor { + return &NetworkChangeMonitor{ + l: l, + ctx: ctx, + enabled: c.GetBool("listen.rebind_on_network_change", true), + } +} + +// Start watches for network changes until the context is cancelled, calling rebind once per settled change. It +// blocks, so callers run it in a goroutine, and it no-ops when disabled, unsupported, or with nothing to rebind. +func (m *NetworkChangeMonitor) Start(rebind func()) { + if !m.enabled || rebind == nil || m.ctx.Err() != nil { + return + } + + changes, err := watchNetworkChanges(m.ctx, m.l) + if err != nil { + // Not fatal. Everything else still works, we just won't notice a network change on our own. + m.l.Error("Failed to watch for network changes, will not rebind the udp listener when the network moves", + "error", err, + ) + return + } + + if changes == nil { + // This platform's sockets don't go stale, so there is nothing to watch for. + return + } + + m.l.Info("Watching for network changes to rebind the udp listener") + + for range changes { + m.l.Info("Local network changed, rebinding the udp listener") + rebind() + } +} diff --git a/udp/netchange_darwin.go b/udp/netchange_darwin.go new file mode 100644 index 00000000..483165e9 --- /dev/null +++ b/udp/netchange_darwin.go @@ -0,0 +1,164 @@ +//go:build darwin && !ios && !e2e_testing +// +build darwin,!ios,!e2e_testing + +package udp + +import ( + "context" + "encoding/binary" + "errors" + "log/slog" + "os" + "time" + + "golang.org/x/sys/unix" +) + +const ( + // netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A + // single network change is never a single message, it is a burst: the link drops, addresses go away, new ones + // arrive, routes get rewritten. Reporting part way through that just means reporting again. + netChangeSettleWindow = time.Second + + // netChangeReadBuffer is sized well past any rt_msghdr plus its addresses. A short read would be discarded by + // the kernel, so being generous here is how we avoid missing a message. + netChangeReadBuffer = 4096 +) + +// watchNetworkChanges reports when the local network moves out from under us, so the listener can be rebound. +// +// Darwin scopes a udp socket to whatever interface it came up on. Move between networks and we keep sending out an +// interface that no longer has a route, which surfaces as an instant "no route to host" with no packet ever leaving +// the box. Rebind clears that, but only if something notices the change and calls it. iOS has always been told by +// the host app off NWPathMonitor. This is the equivalent for everything else that runs on darwin. +// +// The returned channel is buffered and coalescing: a send is dropped if one is already pending, since both mean the +// same thing to a reader. It is closed when ctx is cancelled or the routing socket fails, so a caller can simply +// range over it. Platforms whose sockets do not need rebinding return a nil channel and no error. +func watchNetworkChanges(ctx context.Context, l *slog.Logger) (<-chan struct{}, error) { + sock, err := openRouteSocket() + if err != nil { + return nil, err + } + + changes := make(chan struct{}, 1) + + go func() { + defer close(changes) + defer func() { _ = sock.Close() }() + + // Closing the socket is what unblocks the read in watchRouteSocket, so this turns cancellation into a + // close. It is scoped to this call so it cannot outlive the watch it belongs to. + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-ctx.Done(): + _ = sock.Close() + case <-done: + } + }() + + watchRouteSocket(l, sock, changes) + }() + + return changes, nil +} + +// watchRouteSocket blocks reading the routing socket, reporting once per settled burst of changes. It returns when +// the socket is closed, which is how cancellation gets us out of here. +func watchRouteSocket(l *slog.Logger, sock *os.File, changes chan<- struct{}) { + buf := make([]byte, netChangeReadBuffer) + + for { + n, err := sock.Read(buf) + if err != nil { + logRouteSocketError(l, err) + return + } + + if !isNetworkChange(buf[:n]) { + continue + } + + // Swallow the rest of the burst. The deadline is absolute and not extended by what arrives, so this always + // ends after the settle window no matter how chatty the socket is. Changes that land after the window + // simply produce another report, which is the correct outcome anyway. + deadline := time.Now().Add(netChangeSettleWindow) + for { + if err = sock.SetReadDeadline(deadline); err != nil { + logRouteSocketError(l, err) + return + } + + if _, err = sock.Read(buf); err != nil { + if os.IsTimeout(err) { + break + } + logRouteSocketError(l, err) + return + } + } + + if err = sock.SetReadDeadline(time.Time{}); err != nil { + logRouteSocketError(l, err) + return + } + + select { + case changes <- struct{}{}: + default: + // One already pending, and a second "the network moved" tells the reader nothing new. + } + } +} + +// logRouteSocketError reports a routing socket failure unless it is just us shutting the socket down. +func logRouteSocketError(l *slog.Logger, err error) { + if errors.Is(err, os.ErrClosed) { + return + } + + l.Error("Error reading the routing socket, will no longer notice local network changes", "error", err) +} + +// openRouteSocket returns the routing socket as a non blocking os.File. Going through os.File puts reads on the go +// poller, which buys us both a working read deadline and a Close that unblocks a read in progress. +func openRouteSocket() (*os.File, error) { + fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return nil, err + } + + if err = unix.SetNonblock(fd, true); err != nil { + _ = unix.Close(fd) + return nil, err + } + + return os.NewFile(uintptr(fd), "route"), nil +} + +// isNetworkChange reports whether a routing message means our local addressing may have moved out from under us. +// +// We read the header instead of parsing the message because the type is the only part we need, and a full parse can +// fail on shapes we don't care about, which would turn "a message I can't parse" into "a change I missed". +// rt_msghdr, if_msghdr and ifa_msghdr all begin with the same three fields, so this is the same for every type. +func isNetworkChange(msg []byte) bool { + if len(msg) < 4 { + return false + } + + // u_short msglen, u_char version, u_char type + if int(binary.NativeEndian.Uint16(msg[0:2])) > len(msg) || msg[2] != unix.RTM_VERSION { + return false + } + + switch msg[3] { + case unix.RTM_NEWADDR, unix.RTM_DELADDR, unix.RTM_IFINFO: + // An address arrived or left, or a link changed state. Anything else on this socket is either a route + // churning underneath us, which a rebind doesn't help with, or unrelated traffic. + return true + default: + return false + } +} diff --git a/udp/netchange_darwin_test.go b/udp/netchange_darwin_test.go new file mode 100644 index 00000000..a6d25c83 --- /dev/null +++ b/udp/netchange_darwin_test.go @@ -0,0 +1,244 @@ +//go:build darwin && !ios && !e2e_testing +// +build darwin,!ios,!e2e_testing + +package udp + +import ( + "context" + "encoding/binary" + "os" + "testing" + "time" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + "golang.org/x/sys/unix" +) + +// routeMsg builds the first four bytes of a routing message, which is all isNetworkChange reads. +func routeMsg(msgType uint8, extra int) []byte { + msg := make([]byte, 4+extra) + binary.NativeEndian.PutUint16(msg[0:2], uint16(len(msg))) + msg[2] = unix.RTM_VERSION + msg[3] = msgType + return msg +} + +func TestIsNetworkChange(t *testing.T) { + // The three that mean our addressing may have moved + assert.True(t, isNetworkChange(routeMsg(unix.RTM_NEWADDR, 0))) + assert.True(t, isNetworkChange(routeMsg(unix.RTM_DELADDR, 0))) + assert.True(t, isNetworkChange(routeMsg(unix.RTM_IFINFO, 0))) + + // Route churn is not something a rebind helps with + assert.False(t, isNetworkChange(routeMsg(unix.RTM_ADD, 0))) + assert.False(t, isNetworkChange(routeMsg(unix.RTM_DELETE, 0))) + assert.False(t, isNetworkChange(routeMsg(unix.RTM_GET, 0))) + + // Garbage must not be mistaken for a change + assert.False(t, isNetworkChange(nil), "empty") + assert.False(t, isNetworkChange([]byte{0, 0, 0}), "short header") + + wrongVersion := routeMsg(unix.RTM_NEWADDR, 0) + wrongVersion[2] = unix.RTM_VERSION + 1 + assert.False(t, isNetworkChange(wrongVersion), "wrong rtm_version") + + lying := routeMsg(unix.RTM_NEWADDR, 0) + binary.NativeEndian.PutUint16(lying[0:2], 512) + assert.False(t, isNetworkChange(lying), "msglen longer than what we read") +} + +// socketPair returns a connected pair of datagram sockets, the first wrapped the same way the routing socket is. It +// stands in for the kernel so the watch loop can be driven with synthetic messages. +func socketPair(t *testing.T) (*os.File, int) { + t.Helper() + + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0) + require.NoError(t, err) + require.NoError(t, unix.SetNonblock(fds[0], true)) + + f := os.NewFile(uintptr(fds[0]), "route") + t.Cleanup(func() { + _ = f.Close() + _ = unix.Close(fds[1]) + }) + + return f, fds[1] +} + +func TestWatchRouteSocketCoalescesABurst(t *testing.T) { + sock, kernel := socketPair(t) + changes := make(chan struct{}, 1) + + done := make(chan struct{}) + go func() { + watchRouteSocket(test.NewLogger(), sock, changes) + close(done) + }() + + // One network change is a burst of messages. All of these land inside the settle window, so they must produce + // exactly one report rather than one apiece. + for range 5 { + _, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8)) + require.NoError(t, err) + } + // Uninteresting messages in the middle of a burst must not add a report of their own either. + _, err := unix.Write(kernel, routeMsg(unix.RTM_ADD, 8)) + require.NoError(t, err) + + select { + case <-changes: + case <-time.After(netChangeSettleWindow * 4): + t.Fatal("a burst should have reported a change") + } + + // Nothing more from that burst + select { + case <-changes: + t.Fatal("a burst should report exactly once") + case <-time.After(netChangeSettleWindow): + } + + // A change after the window has closed is a separate event and gets its own report. + _, err = unix.Write(kernel, routeMsg(unix.RTM_IFINFO, 8)) + require.NoError(t, err) + select { + case <-changes: + case <-time.After(netChangeSettleWindow * 4): + t.Fatal("a later change should report again") + } + + // Closing the socket is how the real thing shuts down + require.NoError(t, sock.Close()) + select { + case <-done: + case <-time.After(time.Second * 5): + t.Fatal("watchRouteSocket did not return after the socket was closed") + } +} + +func TestWatchRouteSocketIgnoresUninterestingMessages(t *testing.T) { + sock, kernel := socketPair(t) + changes := make(chan struct{}, 1) + + done := make(chan struct{}) + go func() { + watchRouteSocket(test.NewLogger(), sock, changes) + close(done) + }() + + for _, msgType := range []uint8{unix.RTM_ADD, unix.RTM_DELETE, unix.RTM_GET, unix.RTM_MISS} { + _, err := unix.Write(kernel, routeMsg(msgType, 8)) + require.NoError(t, err) + } + + select { + case <-changes: + t.Fatal("route churn alone must not report a change") + case <-time.After(netChangeSettleWindow * 2): + } + + require.NoError(t, sock.Close()) + select { + case <-done: + case <-time.After(time.Second * 5): + t.Fatal("watchRouteSocket did not return after the socket was closed") + } +} + +// TestWatchRouteSocketDropsRatherThanBlocks covers the coalescing send. A reader that is busy rebinding must not +// wedge the watcher, and a second pending "the network moved" tells it nothing new anyway. +func TestWatchRouteSocketDropsRatherThanBlocks(t *testing.T) { + sock, kernel := socketPair(t) + changes := make(chan struct{}, 1) + + done := make(chan struct{}) + go func() { + watchRouteSocket(test.NewLogger(), sock, changes) + close(done) + }() + + // Nobody is reading changes, so after the first report the buffer is full for the rest of this test + for range 3 { + _, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8)) + require.NoError(t, err) + time.Sleep(netChangeSettleWindow + time.Millisecond*250) + } + + // The watcher must still be alive and responsive to a close + require.NoError(t, sock.Close()) + select { + case <-done: + case <-time.After(time.Second * 5): + t.Fatal("watchRouteSocket wedged on a full channel") + } + + assert.Len(t, changes, 1, "the pending report should have coalesced, not queued") +} + +// TestWatchNetworkChangesStopsWithContext covers the detection path against a real routing socket, including that +// cancelling the context closes the channel so a ranging caller falls out of its loop. +func TestWatchNetworkChangesStopsWithContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + changes, err := watchNetworkChanges(ctx, test.NewLogger()) + require.NoError(t, err) + require.NotNil(t, changes, "darwin should support watching") + + drained := make(chan struct{}) + go func() { + for range changes { + } + close(drained) + }() + + cancel() + select { + case <-drained: + case <-time.After(time.Second * 5): + t.Fatal("cancelling the context should close the changes channel") + } +} + +// TestNetworkChangeMonitorStopsWithContext drives the whole monitor against a real routing socket: Start must block +// watching, and cancelling the context (which is all Control does on shutdown, it never stops the monitor directly) +// must return it and clean up the watch goroutines. +func TestNetworkChangeMonitorStopsWithContext(t *testing.T) { + // IgnoreCurrent because other tests in this package leave readers running; we only care about what this test + // leaks itself. + defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + + ctx, cancel := context.WithCancel(context.Background()) + + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.LoadString("listen:\n rebind_on_network_change: true\n")) + m := NewNetworkChangeMonitor(ctx, l, c) + + done := make(chan struct{}) + go func() { + m.Start(func() {}) + close(done) + }() + + // Start should be sitting on the routing socket, not have fallen out. If it returned early it either failed to + // watch or no-op'd, both of which we want to catch. + select { + case <-done: + t.Fatal("Start returned instead of watching") + case <-time.After(time.Millisecond * 250): + } + + cancel() + select { + case <-done: + case <-time.After(time.Second * 5): + t.Fatal("Start did not return after the context was cancelled") + } + + // Starting again after the context is dead must not open anything. + m.Start(func() {}) +} diff --git a/udp/netchange_generic.go b/udp/netchange_generic.go new file mode 100644 index 00000000..de5cc3a6 --- /dev/null +++ b/udp/netchange_generic.go @@ -0,0 +1,22 @@ +//go:build !darwin || ios || e2e_testing +// +build !darwin ios e2e_testing + +package udp + +import ( + "context" + "log/slog" +) + +// watchNetworkChanges is a no-op outside of darwin. +// +// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket +// goes stale when the local network changes. Everywhere else Rebind has nothing to do, so there is nothing to watch +// for. iOS is excluded on purpose even though it is darwin: the host app already drives the rebind off NWPathMonitor, +// and two things racing to rebind the same socket is worse than one. +// +// A nil channel means "not supported here", which callers must treat as "do not start a watcher" rather than +// selecting on it, since a receive from a nil channel blocks forever. +func watchNetworkChanges(_ context.Context, _ *slog.Logger) (<-chan struct{}, error) { + return nil, nil +} diff --git a/udp/netchange_test.go b/udp/netchange_test.go new file mode 100644 index 00000000..fe57a2e2 --- /dev/null +++ b/udp/netchange_test.go @@ -0,0 +1,39 @@ +package udp + +import ( + "context" + "testing" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newMonitor(t *testing.T, ctx context.Context, cfg string) *NetworkChangeMonitor { + t.Helper() + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.LoadString(cfg)) + return NewNetworkChangeMonitor(ctx, l, c) +} + +func TestNetworkChangeMonitorDefaultsOn(t *testing.T) { + // Says nothing about rebinding, so this covers the default. + m := newMonitor(t, context.Background(), "listen:\n host: 0.0.0.0\n") + assert.True(t, m.enabled, "should default to on") +} + +func TestNetworkChangeMonitorDisabledIsANoOp(t *testing.T) { + m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: false\n") + require.False(t, m.enabled) + + // Must return without opening a socket. If it watched anything this would block. + m.Start(func() {}) +} + +func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) { + // Nothing to rebind, so there is no point watching, on any platform. + m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: true\n") + m.Start(nil) +} diff --git a/udp/temp.go b/udp/temp.go deleted file mode 100644 index 2efe31d2..00000000 --- a/udp/temp.go +++ /dev/null @@ -1,9 +0,0 @@ -package udp - -import ( - "github.com/slackhq/nebula/iputil" -) - -//TODO: The items in this file belong in their own packages but doing that in a single PR is a nightmare - -type LightHouseHandlerFunc func(rAddr *Addr, vpnIp iputil.VpnIp, p []byte) diff --git a/udp/udp_all.go b/udp/udp_all.go deleted file mode 100644 index 093bf69c..00000000 --- a/udp/udp_all.go +++ /dev/null @@ -1,100 +0,0 @@ -package udp - -import ( - "encoding/json" - "fmt" - "net" - "strconv" -) - -type m map[string]interface{} - -type Addr struct { - IP net.IP - Port uint16 -} - -func NewAddr(ip net.IP, port uint16) *Addr { - addr := Addr{IP: make([]byte, net.IPv6len), Port: port} - copy(addr.IP, ip.To16()) - return &addr -} - -func NewAddrFromString(s string) *Addr { - ip, port, err := ParseIPAndPort(s) - //TODO: handle err - _ = err - return &Addr{IP: ip.To16(), Port: port} -} - -func (ua *Addr) Equals(t *Addr) bool { - if t == nil || ua == nil { - return t == nil && ua == nil - } - return ua.IP.Equal(t.IP) && ua.Port == t.Port -} - -func (ua *Addr) String() string { - if ua == nil { - return "" - } - - return net.JoinHostPort(ua.IP.String(), fmt.Sprintf("%v", ua.Port)) -} - -func (ua *Addr) MarshalJSON() ([]byte, error) { - if ua == nil { - return nil, nil - } - - return json.Marshal(m{"ip": ua.IP, "port": ua.Port}) -} - -func (ua *Addr) Copy() *Addr { - if ua == nil { - return nil - } - - nu := Addr{ - Port: ua.Port, - IP: make(net.IP, len(ua.IP)), - } - - copy(nu.IP, ua.IP) - return &nu -} - -type AddrSlice []*Addr - -func (a AddrSlice) Equal(b AddrSlice) bool { - if len(a) != len(b) { - return false - } - - for i := range a { - if !a[i].Equals(b[i]) { - return false - } - } - - return true -} - -func ParseIPAndPort(s string) (net.IP, uint16, error) { - rIp, sPort, err := net.SplitHostPort(s) - if err != nil { - return nil, 0, err - } - - addr, err := net.ResolveIPAddr("ip", rIp) - if err != nil { - return nil, 0, err - } - - iPort, err := strconv.Atoi(sPort) - if err != nil { - return nil, 0, err - } - - return addr.IP, uint16(iPort), nil -} diff --git a/udp/udp_android.go b/udp/udp_android.go index 8d690748..213ab422 100644 --- a/udp/udp_android.go +++ b/udp/udp_android.go @@ -5,14 +5,15 @@ package udp import ( "fmt" + "log/slog" "net" + "net/netip" "syscall" - "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) -func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (Conn, error) { +func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { return NewGenericListener(l, ip, port, multi, batch) } diff --git a/udp/udp_bsd.go b/udp/udp_bsd.go index 785aa6a7..31ae9c5a 100644 --- a/udp/udp_bsd.go +++ b/udp/udp_bsd.go @@ -8,14 +8,15 @@ package udp import ( "fmt" + "log/slog" "net" + "net/netip" "syscall" - "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) -func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (Conn, error) { +func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { return NewGenericListener(l, ip, port, multi, batch) } diff --git a/udp/udp_bypass_windows.go b/udp/udp_bypass_windows.go new file mode 100644 index 00000000..b8b06b1e --- /dev/null +++ b/udp/udp_bypass_windows.go @@ -0,0 +1,57 @@ +//go:build (amd64 || arm64) && !e2e_testing +// +build amd64 arm64 +// +build !e2e_testing + +package udp + +import ( + "log/slog" + "sync" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/wfp" +) + +// wrapWithWDFBypass wraps a Conn so that the first ReloadConfig consults listen.windows_bypass_wdf +// and installs a WFP PERMIT filter for the listener's bound UDP port. The session is released when Close runs. +func wrapWithWDFBypass(l *slog.Logger, conn Conn) Conn { + return &bypassConn{Conn: conn, l: l} +} + +type bypassConn struct { + Conn + + l *slog.Logger + installOnce sync.Once + session *wfp.Session +} + +func (b *bypassConn) ReloadConfig(c *config.C) { + b.installOnce.Do(func() { + if !c.GetBool("listen.windows_bypass_wdf", true) { + return + } + addr, err := b.Conn.LocalAddr() + if err != nil { + b.l.Warn("Failed to query listener port for WFP bypass", "error", err) + return + } + s, err := wfp.PermitUDPPort(addr.Port()) + if err != nil { + b.l.Warn("Failed to install WFP bypass filters for listener", "error", err) + return + } + b.l.Info("Installed WFP filters bypassing Windows Defender Firewall on UDP listener port", + "port", addr.Port()) + b.session = s + }) + b.Conn.ReloadConfig(c) +} + +func (b *bypassConn) Close() error { + if b.session != nil { + b.session.Close() + b.session = nil + } + return b.Conn.Close() +} diff --git a/udp/udp_bypass_windows_386.go b/udp/udp_bypass_windows_386.go new file mode 100644 index 00000000..fa5a6eec --- /dev/null +++ b/udp/udp_bypass_windows_386.go @@ -0,0 +1,11 @@ +//go:build !e2e_testing +// +build !e2e_testing + +package udp + +import "log/slog" + +// wrapWithWDFBypass is a no-op on windows-386 since we don't currently build for it. +func wrapWithWDFBypass(_ *slog.Logger, conn Conn) Conn { + return conn +} diff --git a/udp/udp_darwin.go b/udp/udp_darwin.go index 08e1b6a8..574e4494 100644 --- a/udp/udp_darwin.go +++ b/udp/udp_darwin.go @@ -3,19 +3,62 @@ package udp -// Darwin support is primarily implemented in udp_generic, besides NewListenConfig - import ( + "context" + "encoding/binary" + "errors" "fmt" + "log/slog" "net" + "net/netip" "syscall" + "unsafe" - "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/config" "golang.org/x/sys/unix" ) -func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (Conn, error) { - return NewGenericListener(l, ip, port, multi, batch) +type StdConn struct { + *net.UDPConn + isV4 bool + sysFd uintptr + l *slog.Logger +} + +var _ Conn = &StdConn{} + +func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { + lc := NewListenConfig(multi) + pc, err := lc.ListenPacket(context.TODO(), "udp", net.JoinHostPort(ip.String(), fmt.Sprintf("%v", port))) + if err != nil { + return nil, err + } + + if uc, ok := pc.(*net.UDPConn); ok { + c := &StdConn{UDPConn: uc, l: l} + + rc, err := uc.SyscallConn() + if err != nil { + return nil, fmt.Errorf("failed to open udp socket: %w", err) + } + + err = rc.Control(func(fd uintptr) { + c.sysFd = fd + }) + if err != nil { + return nil, fmt.Errorf("failed to get udp fd: %w", err) + } + + la, err := c.LocalAddr() + if err != nil { + return nil, err + } + c.isV4 = la.Addr().Is4() + + return c, nil + } + + return nil, fmt.Errorf("unexpected PacketConn: %T %#v", pc, pc) } func NewListenConfig(multi bool) net.ListenConfig { @@ -42,16 +85,118 @@ func NewListenConfig(multi bool) net.ListenConfig { } } -func (u *GenericConn) Rebind() error { - rc, err := u.UDPConn.SyscallConn() - if err != nil { - return err +//go:linkname sendto golang.org/x/sys/unix.sendto +//go:noescape +func sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen int32) (err error) + +func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error { + var sa unsafe.Pointer + var addrLen int32 + + if u.isV4 { + if ap.Addr().Is6() { + return ErrInvalidIPv6RemoteForSocket + } + + var rsa unix.RawSockaddrInet4 + rsa.Family = unix.AF_INET + rsa.Addr = ap.Addr().As4() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ap.Port()) + sa = unsafe.Pointer(&rsa) + addrLen = syscall.SizeofSockaddrInet4 + } else { + var rsa unix.RawSockaddrInet6 + rsa.Family = unix.AF_INET6 + rsa.Addr = ap.Addr().As16() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ap.Port()) + sa = unsafe.Pointer(&rsa) + addrLen = syscall.SizeofSockaddrInet6 + } + + // Golang stdlib doesn't handle EAGAIN correctly in some situations so we do writes ourselves + // See https://github.com/golang/go/issues/73919 + for { + //_, _, err := unix.Syscall6(unix.SYS_SENDTO, u.sysFd, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), 0, sa, addrLen) + err := sendto(int(u.sysFd), b, 0, sa, addrLen) + if err == nil { + // Written, get out before the error handling + return nil + } + + if errors.Is(err, syscall.EINTR) { + // Write was interrupted, retry + continue + } + + if errors.Is(err, syscall.EAGAIN) { + return &net.OpError{Op: "sendto", Err: unix.EWOULDBLOCK} + } + + if errors.Is(err, syscall.EBADF) { + return net.ErrClosed + } + + return &net.OpError{Op: "sendto", Err: err} } +} + +func (u *StdConn) LocalAddr() (netip.AddrPort, error) { + a := u.UDPConn.LocalAddr() + + switch v := a.(type) { + case *net.UDPAddr: + addr, ok := netip.AddrFromSlice(v.IP) + if !ok { + return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP) + } + return netip.AddrPortFrom(addr, uint16(v.Port)), nil + + default: + return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a) + } +} - return rc.Control(func(fd uintptr) { - err := syscall.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_BOUND_IF, 0) +func (u *StdConn) ReloadConfig(c *config.C) { + // TODO +} + +func NewUDPStatsEmitter(udpConns []Conn) func() { + // No UDP stats for non-linux + return func() {} +} + +func (u *StdConn) ListenOut(r EncReader) error { + buffer := make([]byte, MTU) + + for { + // Just read one packet at a time + n, rua, err := u.ReadFromUDPAddrPort(buffer) if err != nil { - u.l.WithError(err).Error("Failed to rebind udp socket") + if errors.Is(err, net.ErrClosed) { + return err + } + u.l.Error("unexpected udp socket receive error", "error", err) + continue } - }) + + r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n]) + } +} + +func (u *StdConn) SupportsMultipleReaders() bool { + return false +} + +// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current +// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets +// this way on its own, which is what strands us after the underlying network changes. +func (u *StdConn) Rebind() error { + var err error + if u.isV4 { + err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IP, syscall.IP_BOUND_IF, 0) + } else { + err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0) + } + + return err } diff --git a/udp/udp_generic.go b/udp/udp_generic.go index 1dd6d1de..131eb73b 100644 --- a/udp/udp_generic.go +++ b/udp/udp_generic.go @@ -1,6 +1,7 @@ -//go:build (!linux || android) && !e2e_testing +//go:build (!linux || android) && !e2e_testing && !darwin // +build !linux android // +build !e2e_testing +// +build !darwin // udp_generic implements the nebula UDP interface in pure Go stdlib. This // means it can be used on platforms like Darwin and Windows. @@ -9,23 +10,24 @@ package udp import ( "context" + "errors" "fmt" + "log/slog" "net" + "net/netip" + "time" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/firewall" - "github.com/slackhq/nebula/header" ) type GenericConn struct { *net.UDPConn - l *logrus.Logger + l *slog.Logger } var _ Conn = &GenericConn{} -func NewGenericListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (Conn, error) { +func NewGenericListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { lc := NewListenConfig(multi) pc, err := lc.ListenPacket(context.TODO(), "udp", net.JoinHostPort(ip.String(), fmt.Sprintf("%v", port))) if err != nil { @@ -37,28 +39,29 @@ func NewGenericListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch return nil, fmt.Errorf("Unexpected PacketConn: %T %#v", pc, pc) } -func (u *GenericConn) WriteTo(b []byte, addr *Addr) error { - _, err := u.UDPConn.WriteToUDP(b, &net.UDPAddr{IP: addr.IP, Port: int(addr.Port)}) +func (u *GenericConn) WriteTo(b []byte, addr netip.AddrPort) error { + _, err := u.UDPConn.WriteToUDPAddrPort(b, addr) return err } -func (u *GenericConn) LocalAddr() (*Addr, error) { +func (u *GenericConn) LocalAddr() (netip.AddrPort, error) { a := u.UDPConn.LocalAddr() switch v := a.(type) { case *net.UDPAddr: - addr := &Addr{IP: make([]byte, len(v.IP))} - copy(addr.IP, v.IP) - addr.Port = uint16(v.Port) - return addr, nil + addr, ok := netip.AddrFromSlice(v.IP) + if !ok { + return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP) + } + return netip.AddrPortFrom(addr, uint16(v.Port)), nil default: - return nil, fmt.Errorf("LocalAddr returned: %#v", a) + return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a) } } func (u *GenericConn) ReloadConfig(c *config.C) { - // TODO + } func NewUDPStatsEmitter(udpConns []Conn) func() { @@ -70,24 +73,30 @@ type rawMessage struct { Len uint32 } -func (u *GenericConn) ListenOut(r EncReader, lhf LightHouseHandlerFunc, cache *firewall.ConntrackCacheTicker, q int) { - plaintext := make([]byte, MTU) +func (u *GenericConn) ListenOut(r EncReader) error { buffer := make([]byte, MTU) - h := &header.H{} - fwPacket := &firewall.Packet{} - udpAddr := &Addr{IP: make([]byte, 16)} - nb := make([]byte, 12, 12) + + var lastRecvErr time.Time for { // Just read one packet at a time - n, rua, err := u.ReadFromUDP(buffer) + n, rua, err := u.ReadFromUDPAddrPort(buffer) if err != nil { - u.l.WithError(err).Debug("udp socket is closed, exiting read loop") - return + if errors.Is(err, net.ErrClosed) { + return err + } + // Dampen unexpected message warns to once per minute + if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute { + lastRecvErr = time.Now() + u.l.Warn("unexpected udp socket receive error", "error", err) + } + continue } - udpAddr.IP = rua.IP - udpAddr.Port = uint16(rua.Port) - r(udpAddr, plaintext[:0], buffer[:n], h, fwPacket, lhf, nb, q, cache.Get(u.l)) + r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n]) } } + +func (u *GenericConn) SupportsMultipleReaders() bool { + return false +} diff --git a/udp/udp_linux.go b/udp/udp_linux.go index 1151c890..3920342c 100644 --- a/udp/udp_linux.go +++ b/udp/udp_linux.go @@ -5,59 +5,31 @@ package udp import ( "encoding/binary" + "errors" "fmt" + "log/slog" "net" + "net/netip" + "sync/atomic" "syscall" "unsafe" "github.com/rcrowley/go-metrics" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/firewall" - "github.com/slackhq/nebula/header" "golang.org/x/sys/unix" ) -//TODO: make it support reload as best you can! - type StdConn struct { - sysFd int - isV4 bool - l *logrus.Logger - batch int -} - -var x int - -// From linux/sock_diag.h -const ( - _SK_MEMINFO_RMEM_ALLOC = iota - _SK_MEMINFO_RCVBUF - _SK_MEMINFO_WMEM_ALLOC - _SK_MEMINFO_SNDBUF - _SK_MEMINFO_FWD_ALLOC - _SK_MEMINFO_WMEM_QUEUED - _SK_MEMINFO_OPTMEM - _SK_MEMINFO_BACKLOG - _SK_MEMINFO_DROPS - - _SK_MEMINFO_VARS -) - -type _SK_MEMINFO [_SK_MEMINFO_VARS]uint32 - -func maybeIPV4(ip net.IP) (net.IP, bool) { - ip4 := ip.To4() - if ip4 != nil { - return ip4, true - } - return ip, false + sysFd int + closed atomic.Bool + isV4 bool + l *slog.Logger + batch int } -func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (Conn, error) { - ipV4, isV4 := maybeIPV4(ip) +func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { af := unix.AF_INET6 - if isV4 { + if ip.Is4() { af = unix.AF_INET } syscall.ForkLock.RLock() @@ -66,39 +38,37 @@ func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) ( unix.CloseOnExec(fd) } syscall.ForkLock.RUnlock() - if err != nil { - unix.Close(fd) - return nil, fmt.Errorf("unable to open socket: %s", err) + return nil, fmt.Errorf("unable to open socket: %w", err) } if multi { if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil { - return nil, fmt.Errorf("unable to set SO_REUSEPORT: %s", err) + _ = unix.Close(fd) + return nil, fmt.Errorf("unable to set SO_REUSEPORT: %w", err) } } - //TODO: support multiple listening IPs (for limiting ipv6) var sa unix.Sockaddr - if isV4 { + if ip.Is4() { sa4 := &unix.SockaddrInet4{Port: port} - copy(sa4.Addr[:], ipV4) + sa4.Addr = ip.As4() sa = sa4 } else { sa6 := &unix.SockaddrInet6{Port: port} - copy(sa6.Addr[:], ip.To16()) + sa6.Addr = ip.As16() sa = sa6 } if err = unix.Bind(fd, sa); err != nil { - return nil, fmt.Errorf("unable to bind to socket: %s", err) + _ = unix.Close(fd) + return nil, fmt.Errorf("unable to bind to socket: %w", err) } - //TODO: this may be useful for forcing threads into specific cores - //unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_INCOMING_CPU, x) - //v, err := unix.GetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_INCOMING_CPU) - //l.Println(v, err) + return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, nil +} - return &StdConn{sysFd: fd, isV4: isV4, l: l, batch: batch}, err +func (u *StdConn) SupportsMultipleReaders() bool { + return true } func (u *StdConn) Rebind() error { @@ -113,122 +83,128 @@ func (u *StdConn) SetSendBuffer(n int) error { return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n) } +func (u *StdConn) SetSoMark(mark int) error { + return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark) +} + func (u *StdConn) GetRecvBuffer() (int, error) { - return unix.GetsockoptInt(int(u.sysFd), unix.SOL_SOCKET, unix.SO_RCVBUF) + return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF) } func (u *StdConn) GetSendBuffer() (int, error) { - return unix.GetsockoptInt(int(u.sysFd), unix.SOL_SOCKET, unix.SO_SNDBUF) + return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF) } -func (u *StdConn) LocalAddr() (*Addr, error) { +func (u *StdConn) GetSoMark() (int, error) { + return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK) +} + +func (u *StdConn) LocalAddr() (netip.AddrPort, error) { sa, err := unix.Getsockname(u.sysFd) if err != nil { - return nil, err + return netip.AddrPort{}, err } - - addr := &Addr{} switch sa := sa.(type) { case *unix.SockaddrInet4: - addr.IP = net.IP{sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3]}.To16() - addr.Port = uint16(sa.Port) + return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil case *unix.SockaddrInet6: - addr.IP = sa.Addr[0:] - addr.Port = uint16(sa.Port) + return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil + default: + return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa) } +} - return addr, nil +// recvmmsg does one blocking recvmmsg (MSG_WAITFORONE), reading up to len(msgs) datagrams +func (u *StdConn) recvmmsg(msgs []rawMessage) (int, error) { + r, _, errno := unix.Syscall6( + unix.SYS_RECVMMSG, + uintptr(u.sysFd), + uintptr(unsafe.Pointer(&msgs[0])), + uintptr(len(msgs)), + unix.MSG_WAITFORONE, + 0, + 0, + ) + if errno != 0 { + if u.closed.Load() { + return 0, net.ErrClosed + } + return 0, &net.OpError{Op: "recvmmsg", Err: errno} + } + n := int(r) + if (n == 0 || msgs[0].Len == 0) && u.closed.Load() { + return 0, net.ErrClosed + } + return n, nil } -func (u *StdConn) ListenOut(r EncReader, lhf LightHouseHandlerFunc, cache *firewall.ConntrackCacheTicker, q int) { - plaintext := make([]byte, MTU) - h := &header.H{} - fwPacket := &firewall.Packet{} - udpAddr := &Addr{} - nb := make([]byte, 12, 12) +// recvmsg does one blocking recvmsg into msgs[0] +func (u *StdConn) recvmsg(msgs []rawMessage) (int, error) { + r, _, errno := unix.Syscall6( + unix.SYS_RECVMSG, + uintptr(u.sysFd), + uintptr(unsafe.Pointer(&msgs[0].Hdr)), + 0, + 0, + 0, + 0, + ) + if errno != 0 { + if u.closed.Load() { + return 0, net.ErrClosed + } + return 0, &net.OpError{Op: "recvmsg", Err: errno} + } + if r == 0 && u.closed.Load() { + return 0, net.ErrClosed + } + msgs[0].Len = uint32(r) + return 1, nil +} - //TODO: should we track this? - //metric := metrics.GetOrRegisterHistogram("test.batch_read", nil, metrics.NewExpDecaySample(1028, 0.015)) +func (u *StdConn) ListenOut(r EncReader) error { + var ip netip.Addr msgs, buffers, names := u.PrepareRawMessages(u.batch) - read := u.ReadMulti + read := u.recvmmsg if u.batch == 1 { - read = u.ReadSingle + read = u.recvmsg } for { n, err := read(msgs) if err != nil { - u.l.WithError(err).Debug("udp socket is closed, exiting read loop") - return + if errors.Is(err, unix.EINTR) { + continue // interrupted by a signal, retry the read + } + // net.ErrClosed after Close() is teardown, absorbed by the caller's + // closed flag like the other platforms; anything else is a real error. + return err } - //metric.Update(int64(n)) for i := 0; i < n; i++ { + // Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic if u.isV4 { - udpAddr.IP = names[i][4:8] + ip, _ = netip.AddrFromSlice(names[i][4:8]) } else { - udpAddr.IP = names[i][8:24] + ip, _ = netip.AddrFromSlice(names[i][8:24]) } - udpAddr.Port = binary.BigEndian.Uint16(names[i][2:4]) - r(udpAddr, plaintext[:0], buffers[i][:msgs[i].Len], h, fwPacket, lhf, nb, q, cache.Get(u.l)) + r(netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4])), buffers[i][:msgs[i].Len]) } } } -func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) { - for { - n, _, err := unix.Syscall6( - unix.SYS_RECVMSG, - uintptr(u.sysFd), - uintptr(unsafe.Pointer(&(msgs[0].Hdr))), - 0, - 0, - 0, - 0, - ) - - if err != 0 { - return 0, &net.OpError{Op: "recvmsg", Err: err} - } - - msgs[0].Len = uint32(n) - return 1, nil - } -} - -func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) { - for { - n, _, err := unix.Syscall6( - unix.SYS_RECVMMSG, - uintptr(u.sysFd), - uintptr(unsafe.Pointer(&msgs[0])), - uintptr(len(msgs)), - unix.MSG_WAITFORONE, - 0, - 0, - ) - - if err != 0 { - return 0, &net.OpError{Op: "recvmmsg", Err: err} - } - - return int(n), nil - } -} - -func (u *StdConn) WriteTo(b []byte, addr *Addr) error { +func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error { if u.isV4 { - return u.writeTo4(b, addr) + return u.writeTo4(b, ip) } - return u.writeTo6(b, addr) + return u.writeTo6(b, ip) } -func (u *StdConn) writeTo6(b []byte, addr *Addr) error { +func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error { var rsa unix.RawSockaddrInet6 rsa.Family = unix.AF_INET6 - // Little Endian -> Network Endian - rsa.Port = (addr.Port >> 8) | ((addr.Port & 0xff) << 8) - copy(rsa.Addr[:], addr.IP.To16()) + rsa.Addr = ip.Addr().As16() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port()) for { _, _, err := unix.Syscall6( @@ -240,28 +216,22 @@ func (u *StdConn) writeTo6(b []byte, addr *Addr) error { uintptr(unsafe.Pointer(&rsa)), uintptr(unix.SizeofSockaddrInet6), ) - if err != 0 { return &net.OpError{Op: "sendto", Err: err} } - - //TODO: handle incomplete writes - return nil } } -func (u *StdConn) writeTo4(b []byte, addr *Addr) error { - addrV4, isAddrV4 := maybeIPV4(addr.IP) - if !isAddrV4 { - return fmt.Errorf("Listener is IPv4, but writing to IPv6 remote") +func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error { + if !ip.Addr().Is4() { + return ErrInvalidIPv6RemoteForSocket } var rsa unix.RawSockaddrInet4 rsa.Family = unix.AF_INET - // Little Endian -> Network Endian - rsa.Port = (addr.Port >> 8) | ((addr.Port & 0xff) << 8) - copy(rsa.Addr[:], addrV4) + rsa.Addr = ip.Addr().As4() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port()) for { _, _, err := unix.Syscall6( @@ -273,13 +243,9 @@ func (u *StdConn) writeTo4(b []byte, addr *Addr) error { uintptr(unsafe.Pointer(&rsa)), uintptr(unix.SizeofSockaddrInet4), ) - if err != 0 { return &net.OpError{Op: "sendto", Err: err} } - - //TODO: handle incomplete writes - return nil } } @@ -287,37 +253,47 @@ func (u *StdConn) writeTo4(b []byte, addr *Addr) error { func (u *StdConn) ReloadConfig(c *config.C) { b := c.GetInt("listen.read_buffer", 0) if b > 0 { - err := u.SetRecvBuffer(b) - if err == nil { - s, err := u.GetRecvBuffer() - if err == nil { - u.l.WithField("size", s).Info("listen.read_buffer was set") + if err := u.SetRecvBuffer(b); err == nil { + if s, err := u.GetRecvBuffer(); err == nil { + u.l.Info("listen.read_buffer was set", "size", s) } else { - u.l.WithError(err).Warn("Failed to get listen.read_buffer") + u.l.Warn("Failed to get listen.read_buffer", "error", err) } } else { - u.l.WithError(err).Error("Failed to set listen.read_buffer") + u.l.Error("Failed to set listen.read_buffer", "error", err) } } b = c.GetInt("listen.write_buffer", 0) if b > 0 { - err := u.SetSendBuffer(b) - if err == nil { - s, err := u.GetSendBuffer() - if err == nil { - u.l.WithField("size", s).Info("listen.write_buffer was set") + if err := u.SetSendBuffer(b); err == nil { + if s, err := u.GetSendBuffer(); err == nil { + u.l.Info("listen.write_buffer was set", "size", s) + } else { + u.l.Warn("Failed to get listen.write_buffer", "error", err) + } + } else { + u.l.Error("Failed to set listen.write_buffer", "error", err) + } + } + + b = c.GetInt("listen.so_mark", 0) + s, err := u.GetSoMark() + if b > 0 || (err == nil && s != 0) { + if err := u.SetSoMark(b); err == nil { + if s, err := u.GetSoMark(); err == nil { + u.l.Info("listen.so_mark was set", "mark", s) } else { - u.l.WithError(err).Warn("Failed to get listen.write_buffer") + u.l.Warn("Failed to get listen.so_mark", "error", err) } } else { - u.l.WithError(err).Error("Failed to set listen.write_buffer") + u.l.Error("Failed to set listen.so_mark", "error", err) } } } -func (u *StdConn) getMemInfo(meminfo *_SK_MEMINFO) error { - var vallen uint32 = 4 * _SK_MEMINFO_VARS +func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error { + var vallen uint32 = 4 * unix.SK_MEMINFO_VARS _, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0) if err != 0 { return err @@ -326,18 +302,22 @@ func (u *StdConn) getMemInfo(meminfo *_SK_MEMINFO) error { } func (u *StdConn) Close() error { - //TODO: this will not interrupt the read loop - return syscall.Close(u.sysFd) + u.closed.Store(true) + // Wake the reader parked in recvmmsg/recvmsg. shutdown(2) on an unconnected socket + // returns ENOTCONN but still wakes it, so ignore the error. + // The reader then sees closed and stops touching the fd, making the Close below safe. + _ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR) + return unix.Close(u.sysFd) } func NewUDPStatsEmitter(udpConns []Conn) func() { // Check if our kernel supports SO_MEMINFO before registering the gauges - var udpGauges [][_SK_MEMINFO_VARS]metrics.Gauge - var meminfo _SK_MEMINFO + var udpGauges [][unix.SK_MEMINFO_VARS]metrics.Gauge + var meminfo [unix.SK_MEMINFO_VARS]uint32 if err := udpConns[0].(*StdConn).getMemInfo(&meminfo); err == nil { - udpGauges = make([][_SK_MEMINFO_VARS]metrics.Gauge, len(udpConns)) + udpGauges = make([][unix.SK_MEMINFO_VARS]metrics.Gauge, len(udpConns)) for i := range udpConns { - udpGauges[i] = [_SK_MEMINFO_VARS]metrics.Gauge{ + udpGauges[i] = [unix.SK_MEMINFO_VARS]metrics.Gauge{ metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rmem_alloc", i), nil), metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rcvbuf", i), nil), metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.wmem_alloc", i), nil), @@ -354,7 +334,7 @@ func NewUDPStatsEmitter(udpConns []Conn) func() { return func() { for i, gauges := range udpGauges { if err := udpConns[i].(*StdConn).getMemInfo(&meminfo); err == nil { - for j := 0; j < _SK_MEMINFO_VARS; j++ { + for j := 0; j < unix.SK_MEMINFO_VARS; j++ { gauges[j].Update(int64(meminfo[j])) } } diff --git a/udp/udp_linux_32.go b/udp/udp_linux_32.go index 523968c2..de8f1cdf 100644 --- a/udp/udp_linux_32.go +++ b/udp/udp_linux_32.go @@ -39,7 +39,6 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) { buffers[i] = make([]byte, MTU) names[i] = make([]byte, unix.SizeofSockaddrInet6) - //TODO: this is still silly, no need for an array vs := []iovec{ {Base: &buffers[i][0], Len: uint32(len(buffers[i]))}, } diff --git a/udp/udp_linux_64.go b/udp/udp_linux_64.go index a54f1dfd..48c5a978 100644 --- a/udp/udp_linux_64.go +++ b/udp/udp_linux_64.go @@ -1,6 +1,6 @@ -//go:build linux && (amd64 || arm64 || ppc64 || ppc64le || mips64 || mips64le || s390x || riscv64) && !android && !e2e_testing +//go:build linux && (amd64 || arm64 || ppc64 || ppc64le || mips64 || mips64le || s390x || riscv64 || loong64) && !android && !e2e_testing // +build linux -// +build amd64 arm64 ppc64 ppc64le mips64 mips64le s390x riscv64 +// +build amd64 arm64 ppc64 ppc64le mips64 mips64le s390x riscv64 loong64 // +build !android // +build !e2e_testing @@ -42,7 +42,6 @@ func (u *StdConn) PrepareRawMessages(n int) ([]rawMessage, [][]byte, [][]byte) { buffers[i] = make([]byte, MTU) names[i] = make([]byte, unix.SizeofSockaddrInet6) - //TODO: this is still silly, no need for an array vs := []iovec{ {Base: &buffers[i][0], Len: uint64(len(buffers[i]))}, } diff --git a/udp/udp_linux_test.go b/udp/udp_linux_test.go new file mode 100644 index 00000000..f9e7b3d8 --- /dev/null +++ b/udp/udp_linux_test.go @@ -0,0 +1,179 @@ +//go:build linux && !android && !e2e_testing + +package udp + +import ( + "errors" + "fmt" + "log/slog" + "net" + "net/netip" + "os" + "runtime" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func testLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) +} + +// TestShutdownWakesAfterRx_Mechanism exercises the kernel quirk our teardown +// relies on: once a socket has received a packet, shutdown(2) wakes a blocked +// recvmmsg with n>=1/Len==0 (not n==0). recvmmsg must turn that into net.ErrClosed +// once Close set closed, so a parked reader exits instead of spinning. +func TestShutdownWakesAfterRx_Mechanism(t *testing.T) { + c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, 64) + if err != nil { + t.Fatalf("NewListener: %v", err) + } + sc := c.(*StdConn) + addr, err := sc.LocalAddr() + if err != nil { + t.Fatalf("LocalAddr: %v", err) + } + msgs, _, _ := sc.PrepareRawMessages(sc.batch) + + // Receive a real packet so the socket has carried data. + send, err := net.Dial("udp", addr.String()) + if err != nil { + t.Fatalf("dial: %v", err) + } + if _, err := send.Write([]byte("hello")); err != nil { + t.Fatalf("write: %v", err) + } + time.Sleep(50 * time.Millisecond) + n, err := sc.recvmmsg(msgs) + t.Logf("drain of real packet: n=%d err=%v msgs[0].Len=%d", n, err, msgs[0].Len) + _ = send.Close() + + // Block a reader on the now-empty queue, then tear down as Close() does. + // recvmmsg must return net.ErrClosed (not hang, not spin) even post-rx. + done := make(chan error, 1) + go func() { + _, err := sc.recvmmsg(msgs) + done <- err + }() + time.Sleep(150 * time.Millisecond) // let it park in recvmmsg + + sc.closed.Store(true) + if serr := unix.Shutdown(sc.sysFd, unix.SHUT_RDWR); serr != nil { + t.Logf("shutdown returned %v (expected ENOTCONN on unconnected UDP)", serr) + } + + select { + case err := <-done: + if !errors.Is(err, net.ErrClosed) { + t.Errorf("recvmmsg after post-rx shutdown returned %v, want net.ErrClosed", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("HANG: recvmmsg did not return after shutdown following a received packet") + } + _ = unix.Close(sc.sysFd) +} + +// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking +// reader must tear down cleanly on Close() regardless of what the socket has +// carried. The three cases the report called out: +// +// no traffic ever -> works (shutdown wakes recvmmsg with n==0) +// ping once, then idle -> historically HUNG: once the socket has received a +// packet, shutdown(2) wakes recvmmsg with n>=1/Len==0, +// which an n==0-only teardown check misses +// continuous traffic -> works (a real packet is always arriving) +// +// All three must return within the deadline; a hang dumps goroutines so the +// stuck reader is visible. +func TestListenOutTeardown_TrafficPatterns(t *testing.T) { + cases := []struct { + name string + traffic func(send net.Conn, stop <-chan struct{}) + }{ + {"no_traffic_ever", func(net.Conn, <-chan struct{}) {}}, + {"ping_once_then_idle", func(send net.Conn, _ <-chan struct{}) { + _, _ = send.Write([]byte("hello")) + }}, + {"continuous", func(send net.Conn, stop <-chan struct{}) { + for { + select { + case <-stop: + return + default: + _, _ = send.Write([]byte("hello")) + time.Sleep(2 * time.Millisecond) + } + } + }}, + } + + // batch 1 exercises the recvmsg path, batch 64 the recvmmsg path; both must + // tear down cleanly. + for _, batch := range []int{1, 64} { + for _, tc := range cases { + t.Run(fmt.Sprintf("batch%d/%s", batch, tc.name), func(t *testing.T) { + runTeardownCase(t, batch, tc.name, tc.traffic) + }) + } + } +} + +func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net.Conn, stop <-chan struct{})) { + c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, batch) + if err != nil { + t.Fatalf("NewListener: %v", err) + } + sc := c.(*StdConn) + addr, err := sc.LocalAddr() + if err != nil { + t.Fatalf("LocalAddr: %v", err) + } + + var received atomic.Int64 + loopDone := make(chan error, 1) + go func() { + loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) { + received.Add(1) + }) + }() + + send, err := net.Dial("udp", addr.String()) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer send.Close() + + stop := make(chan struct{}) + trafficDone := make(chan struct{}) + go func() { + traffic(send, stop) + close(trafficDone) + }() + + // Let the pattern run and, for the idle case, the reader park again on an + // empty queue with the socket already having received a packet. + time.Sleep(500 * time.Millisecond) + + start := time.Now() + if err := sc.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + close(stop) + + select { + case err := <-loopDone: + // Clean teardown surfaces as net.ErrClosed (propagated like the other + // platforms); the caller absorbs it via its closed flag. + if err != nil && !errors.Is(err, net.ErrClosed) { + t.Fatalf("%s: ListenOut returned unexpected error on teardown: %v", name, err) + } + t.Logf("%s: closed in %v (received %d packets)", name, time.Since(start), received.Load()) + case <-time.After(3 * time.Second): + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n]) + } + <-trafficDone +} diff --git a/udp/udp_netbsd.go b/udp/udp_netbsd.go index 3c14face..b0c81393 100644 --- a/udp/udp_netbsd.go +++ b/udp/udp_netbsd.go @@ -7,14 +7,15 @@ package udp import ( "fmt" + "log/slog" "net" + "net/netip" "syscall" - "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) -func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (Conn, error) { +func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { return NewGenericListener(l, ip, port, multi, batch) } diff --git a/udp/udp_rio_windows.go b/udp/udp_rio_windows.go index 31c1a554..d110af19 100644 --- a/udp/udp_rio_windows.go +++ b/udp/udp_rio_windows.go @@ -9,17 +9,16 @@ import ( "errors" "fmt" "io" + "log/slog" "net" + "net/netip" "sync" "sync/atomic" "syscall" + "time" "unsafe" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/firewall" - "github.com/slackhq/nebula/header" - "golang.org/x/sys/windows" "golang.zx2c4.com/wireguard/conn/winrio" ) @@ -54,23 +53,21 @@ type ringBuffer struct { type RIOConn struct { isOpen atomic.Bool - l *logrus.Logger + l *slog.Logger sock windows.Handle rx, tx ringBuffer rq winrio.Rq results [packetsPerRing]winrio.Result } -func NewRIOListener(l *logrus.Logger, ip net.IP, port int) (*RIOConn, error) { +func NewRIOListener(l *slog.Logger, addr netip.Addr, port int) (*RIOConn, error) { if !winrio.Initialize() { return nil, errors.New("could not initialize winrio") } u := &RIOConn{l: l} - addr := [16]byte{} - copy(addr[:], ip.To16()) - err := u.bind(&windows.SockaddrInet6{Addr: addr, Port: port}) + err := u.bind(l, &windows.SockaddrInet6{Addr: addr.As16(), Port: port}) if err != nil { return nil, fmt.Errorf("bind: %w", err) } @@ -86,60 +83,85 @@ func NewRIOListener(l *logrus.Logger, ip net.IP, port int) (*RIOConn, error) { return u, nil } -func (u *RIOConn) bind(sa windows.Sockaddr) error { +func (u *RIOConn) bind(l *slog.Logger, sa windows.Sockaddr) error { var err error u.sock, err = winrio.Socket(windows.AF_INET6, windows.SOCK_DGRAM, windows.IPPROTO_UDP) if err != nil { - return err + return fmt.Errorf("winrio.Socket error: %w", err) } // Enable v4 for this socket syscall.SetsockoptInt(syscall.Handle(u.sock), syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0) + // Disable reporting of PORT_UNREACHABLE and NET_UNREACHABLE errors from the UDP socket receive call. + // These errors are returned on Windows during UDP receives based on the receipt of ICMP packets. Disable + // the UDP receive error returns with these ioctl calls. + ret := uint32(0) + flag := uint32(0) + size := uint32(unsafe.Sizeof(flag)) + err = syscall.WSAIoctl(syscall.Handle(u.sock), syscall.SIO_UDP_CONNRESET, (*byte)(unsafe.Pointer(&flag)), size, nil, 0, &ret, nil, 0) + if err != nil { + // This is a best-effort to prevent errors from being returned by the udp recv operation. + // Quietly log a failure and continue. + l.Debug("failed to set UDP_CONNRESET ioctl", "error", err) + } + + ret = 0 + flag = 0 + size = uint32(unsafe.Sizeof(flag)) + SIO_UDP_NETRESET := uint32(syscall.IOC_IN | syscall.IOC_VENDOR | 15) + err = syscall.WSAIoctl(syscall.Handle(u.sock), SIO_UDP_NETRESET, (*byte)(unsafe.Pointer(&flag)), size, nil, 0, &ret, nil, 0) + if err != nil { + // This is a best-effort to prevent errors from being returned by the udp recv operation. + // Quietly log a failure and continue. + l.Debug("failed to set UDP_NETRESET ioctl", "error", err) + } + err = u.rx.Open() if err != nil { - return err + return fmt.Errorf("error rx.Open(): %w", err) } err = u.tx.Open() if err != nil { - return err + return fmt.Errorf("error tx.Open(): %w", err) } u.rq, err = winrio.CreateRequestQueue(u.sock, packetsPerRing, 1, packetsPerRing, 1, u.rx.cq, u.tx.cq, 0) if err != nil { - return err + return fmt.Errorf("error CreateRequestQueue: %w", err) } err = windows.Bind(u.sock, sa) if err != nil { - return err + return fmt.Errorf("error windows.Bind(): %w", err) } return nil } -func (u *RIOConn) ListenOut(r EncReader, lhf LightHouseHandlerFunc, cache *firewall.ConntrackCacheTicker, q int) { - plaintext := make([]byte, MTU) +func (u *RIOConn) ListenOut(r EncReader) error { buffer := make([]byte, MTU) - h := &header.H{} - fwPacket := &firewall.Packet{} - udpAddr := &Addr{IP: make([]byte, 16)} - nb := make([]byte, 12, 12) + + var lastRecvErr time.Time for { // Just read one packet at a time n, rua, err := u.receive(buffer) + if err != nil { - u.l.WithError(err).Debug("udp socket is closed, exiting read loop") - return + if errors.Is(err, net.ErrClosed) { + return err + } + // Dampen unexpected message warns to once per minute + if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute { + lastRecvErr = time.Now() + u.l.Warn("unexpected udp socket receive error", "error", err) + } + continue } - udpAddr.IP = rua.Addr[:] - p := (*[2]byte)(unsafe.Pointer(&udpAddr.Port)) - p[0] = byte(rua.Port >> 8) - p[1] = byte(rua.Port) - r(udpAddr, plaintext[:0], buffer[:n], h, fwPacket, lhf, nb, q, cache.Get(u.l)) + r(netip.AddrPortFrom(netip.AddrFrom16(rua.Addr).Unmap(), (rua.Port>>8)|((rua.Port&0xff)<<8)), buffer[:n]) } } @@ -231,7 +253,7 @@ retry: return n, ep, nil } -func (u *RIOConn) WriteTo(buf []byte, addr *Addr) error { +func (u *RIOConn) WriteTo(buf []byte, ip netip.AddrPort) error { if !u.isOpen.Load() { return net.ErrClosed } @@ -274,10 +296,9 @@ func (u *RIOConn) WriteTo(buf []byte, addr *Addr) error { packet := u.tx.Push() packet.addr.Family = windows.AF_INET6 - p := (*[2]byte)(unsafe.Pointer(&packet.addr.Port)) - p[0] = byte(addr.Port >> 8) - p[1] = byte(addr.Port) - copy(packet.addr.Addr[:], addr.IP.To16()) + packet.addr.Addr = ip.Addr().As16() + port := ip.Port() + packet.addr.Port = (port >> 8) | ((port & 0xff) << 8) copy(packet.data[:], buf) dataBuffer := &winrio.Buffer{ @@ -295,17 +316,19 @@ func (u *RIOConn) WriteTo(buf []byte, addr *Addr) error { return winrio.SendEx(u.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0) } -func (u *RIOConn) LocalAddr() (*Addr, error) { +func (u *RIOConn) LocalAddr() (netip.AddrPort, error) { sa, err := windows.Getsockname(u.sock) if err != nil { - return nil, err + return netip.AddrPort{}, err } v6 := sa.(*windows.SockaddrInet6) - return &Addr{ - IP: v6.Addr[:], - Port: uint16(v6.Port), - }, nil + return netip.AddrPortFrom(netip.AddrFrom16(v6.Addr).Unmap(), uint16(v6.Port)), nil + +} + +func (u *RIOConn) SupportsMultipleReaders() bool { + return false } func (u *RIOConn) Rebind() error { diff --git a/udp/udp_tester.go b/udp/udp_tester.go index 55985f47..9c0d989f 100644 --- a/udp/udp_tester.go +++ b/udp/udp_tester.go @@ -4,78 +4,128 @@ package udp import ( - "fmt" + "context" "io" - "net" + "log/slog" + "net/netip" + "os" + "sync" "sync/atomic" - "github.com/sirupsen/logrus" "github.com/slackhq/nebula/config" - "github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/header" ) type Packet struct { - ToIp net.IP - ToPort uint16 - FromIp net.IP - FromPort uint16 - Data []byte + To netip.AddrPort + From netip.AddrPort + Data []byte } +// Copy returns a fresh *Packet (from the freelist) with a duplicate Data buffer. func (u *Packet) Copy() *Packet { - n := &Packet{ - ToIp: make(net.IP, len(u.ToIp)), - ToPort: u.ToPort, - FromIp: make(net.IP, len(u.FromIp)), - FromPort: u.FromPort, - Data: make([]byte, len(u.Data)), + n := acquirePacket() + n.To = u.To + n.From = u.From + if cap(n.Data) < len(u.Data) { + n.Data = make([]byte, len(u.Data)) + } else { + n.Data = n.Data[:len(u.Data)] } - - copy(n.ToIp, u.ToIp) - copy(n.FromIp, u.FromIp) copy(n.Data, u.Data) return n } +// Release returns p to the harness packet freelist. +// Callers that pull a *Packet from Get / TxPackets must Release when done. +// Channel-backed instead of sync.Pool because sync.Pool's per-P caches drain badly under cross-goroutine Get/Put, +// and putting a []byte in a Pool escapes the slice header to heap. +func (p *Packet) Release() { + if p == nil { + return + } + p.Data = p.Data[:0] + select { + case packetFreelist <- p: + default: + // Freelist full; drop the *Packet for the GC. + } +} + +// packetFreelist retains *Packet structs (and their backing Data arrays) so steady-state allocation drops to zero. +var packetFreelist = make(chan *Packet, 64) + +func acquirePacket() *Packet { + select { + case p := <-packetFreelist: + return p + default: + return &Packet{} + } +} + type TesterConn struct { - Addr *Addr + // addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can + // move it mid-run to simulate roaming, so it is atomic rather than a plain field. + addr atomic.Pointer[netip.AddrPort] RxPackets chan *Packet // Packets to receive into nebula TxPackets chan *Packet // Packets transmitted outside by nebula - closed atomic.Bool - l *logrus.Logger + // done is closed exactly once by Close. Senders select on it so they + // never race with a channel close; readers exit when it fires. The + // packet channels are intentionally never closed - that was the source + // of `send on closed channel` panics when a WriteTo/Send from another + // goroutine passed the close check and reached the send just after + // Close ran. + done chan struct{} + closeOnce sync.Once + + l *slog.Logger } -func NewListener(l *logrus.Logger, ip net.IP, port int, _ bool, _ int) (Conn, error) { - return &TesterConn{ - Addr: &Addr{ip, uint16(port)}, +func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) { + c := &TesterConn{ RxPackets: make(chan *Packet, 10), TxPackets: make(chan *Packet, 10), + done: make(chan struct{}), l: l, - }, nil + } + c.SetAddr(netip.AddrPortFrom(ip, uint16(port))) + return c, nil +} + +// GetAddr returns the underlay address this conn currently sends from. +func (u *TesterConn) GetAddr() netip.AddrPort { + return *u.addr.Load() +} + +// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network. +func (u *TesterConn) SetAddr(addr netip.AddrPort) { + u.addr.Store(&addr) } // Send will place a UdpPacket onto the receive queue for nebula to consume // this is an encrypted packet or a handshake message in most cases // packets were transmitted from another nebula node, you can send them with Tun.Send func (u *TesterConn) Send(packet *Packet) { - if u.closed.Load() { - return - } - - h := &header.H{} - if err := h.Parse(packet.Data); err != nil { - panic(err) + if u.l.Enabled(context.Background(), slog.LevelDebug) { + // Parse the header only under debug logging, otherwise the + // allocation would show up in every Send call. + var h header.H + if err := h.Parse(packet.Data); err != nil { + panic(err) + } + u.l.Debug("UDP receiving injected packet", + "header", &h, + "udpAddr", packet.From, + "dataLen", len(packet.Data), + ) } - if u.l.Level >= logrus.DebugLevel { - u.l.WithField("header", h). - WithField("udpAddr", fmt.Sprintf("%v:%v", packet.FromIp, packet.FromPort)). - WithField("dataLen", len(packet.Data)). - Debug("UDP receiving injected packet") + select { + case <-u.done: + case u.RxPackets <- packet: } - u.RxPackets <- packet } // Get will pull a UdpPacket from the transmit queue @@ -83,7 +133,12 @@ func (u *TesterConn) Send(packet *Packet) { // packets were ingested from the tun side (in most cases), you can send them with Tun.Send func (u *TesterConn) Get(block bool) *Packet { if block { - return <-u.TxPackets + select { + case <-u.done: + return nil + case p := <-u.TxPackets: + return p + } } select { @@ -98,42 +153,34 @@ func (u *TesterConn) Get(block bool) *Packet { // Below this is boilerplate implementation to make nebula actually work //********************************************************************************************************************// -func (u *TesterConn) WriteTo(b []byte, addr *Addr) error { - if u.closed.Load() { - return io.ErrClosedPipe +func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error { + p := acquirePacket() + if cap(p.Data) < len(b) { + p.Data = make([]byte, len(b)) + } else { + p.Data = p.Data[:len(b)] } - - p := &Packet{ - Data: make([]byte, len(b), len(b)), - FromIp: make([]byte, 16), - FromPort: u.Addr.Port, - ToIp: make([]byte, 16), - ToPort: addr.Port, - } - copy(p.Data, b) - copy(p.ToIp, addr.IP.To16()) - copy(p.FromIp, u.Addr.IP.To16()) - - u.TxPackets <- p - return nil + p.From = u.GetAddr() + p.To = addr + select { + case <-u.done: + p.Release() + return io.ErrClosedPipe + case u.TxPackets <- p: + return nil + } } -func (u *TesterConn) ListenOut(r EncReader, lhf LightHouseHandlerFunc, cache *firewall.ConntrackCacheTicker, q int) { - plaintext := make([]byte, MTU) - h := &header.H{} - fwPacket := &firewall.Packet{} - ua := &Addr{IP: make([]byte, 16)} - nb := make([]byte, 12, 12) - +func (u *TesterConn) ListenOut(r EncReader) error { for { - p, ok := <-u.RxPackets - if !ok { - return + select { + case <-u.done: + return os.ErrClosed + case p := <-u.RxPackets: + r(p.From, p.Data) + p.Release() } - ua.Port = p.FromPort - copy(ua.IP, p.FromIp.To16()) - r(ua, plaintext[:0], p.Data, h, fwPacket, lhf, nb, q, cache.Get(u.l)) } } @@ -144,8 +191,12 @@ func NewUDPStatsEmitter(_ []Conn) func() { return func() {} } -func (u *TesterConn) LocalAddr() (*Addr, error) { - return u.Addr, nil +func (u *TesterConn) LocalAddr() (netip.AddrPort, error) { + return u.GetAddr(), nil +} + +func (u *TesterConn) SupportsMultipleReaders() bool { + return false } func (u *TesterConn) Rebind() error { @@ -153,9 +204,8 @@ func (u *TesterConn) Rebind() error { } func (u *TesterConn) Close() error { - if u.closed.CompareAndSwap(false, true) { - close(u.RxPackets) - close(u.TxPackets) - } + u.closeOnce.Do(func() { + close(u.done) + }) return nil } diff --git a/udp/udp_windows.go b/udp/udp_windows.go index ebcace67..1f34f0bc 100644 --- a/udp/udp_windows.go +++ b/udp/udp_windows.go @@ -5,13 +5,13 @@ package udp import ( "fmt" + "log/slog" "net" + "net/netip" "syscall" - - "github.com/sirupsen/logrus" ) -func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (Conn, error) { +func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { if multi { //NOTE: Technically we can support it with RIO but it wouldn't be at the socket level // The udp stack would need to be reworked to hide away the implementation differences between @@ -19,13 +19,18 @@ func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) ( return nil, fmt.Errorf("multiple udp listeners not supported on windows") } + var conn Conn rc, err := NewRIOListener(l, ip, port) if err == nil { - return rc, nil + conn = rc + } else { + l.Error("Falling back to standard udp sockets", "error", err) + conn, err = NewGenericListener(l, ip, port, multi, batch) + if err != nil { + return nil, err + } } - - l.WithError(err).Error("Falling back to standard udp sockets") - return NewGenericListener(l, ip, port, multi, batch) + return wrapWithWDFBypass(l, conn), nil } func NewListenConfig(multi bool) net.ListenConfig { diff --git a/util/error.go b/util/error.go index d7710f9a..14371d3f 100644 --- a/util/error.go +++ b/util/error.go @@ -1,19 +1,19 @@ package util import ( + "context" "errors" "fmt" - - "github.com/sirupsen/logrus" + "log/slog" ) type ContextualError struct { RealError error - Fields map[string]interface{} + Fields map[string]any Context string } -func NewContextualError(msg string, fields map[string]interface{}, realError error) *ContextualError { +func NewContextualError(msg string, fields map[string]any, realError error) *ContextualError { return &ContextualError{Context: msg, Fields: fields, RealError: realError} } @@ -28,12 +28,12 @@ func ContextualizeIfNeeded(msg string, err error) error { } // LogWithContextIfNeeded is a helper function to log an error line for an error or ContextualError -func LogWithContextIfNeeded(msg string, err error, l *logrus.Logger) { +func LogWithContextIfNeeded(msg string, err error, l *slog.Logger) { switch v := err.(type) { case *ContextualError: v.Log(l) default: - l.WithError(err).Error(msg) + l.Error(msg, "error", err) } } @@ -51,10 +51,19 @@ func (ce *ContextualError) Unwrap() error { return ce.RealError } -func (ce *ContextualError) Log(lr *logrus.Logger) { +// Log emits ce as a single error-level log line with Fields and RealError +// promoted to top-level attributes, producing a flat shape callers can grep +// or parse without walking into a nested object. +func (ce *ContextualError) Log(l *slog.Logger) { + attrs := make([]slog.Attr, 0, len(ce.Fields)+1) + for k, v := range ce.Fields { + attrs = append(attrs, slog.Any(k, v)) + } if ce.RealError != nil { - lr.WithFields(ce.Fields).WithError(ce.RealError).Error(ce.Context) - } else { - lr.WithFields(ce.Fields).Error(ce.Context) + attrs = append(attrs, slog.Any("error", ce.RealError)) } + // LogAttrs is intentional: attrs is built from a map[string]any so it has + // no pair-form equivalent. + //nolint:sloglint + l.LogAttrs(context.Background(), slog.LevelError, ce.Context, attrs...) } diff --git a/util/error_test.go b/util/error_test.go index 5041f82c..30e39e33 100644 --- a/util/error_test.go +++ b/util/error_test.go @@ -1,95 +1,67 @@ package util import ( + "bytes" "errors" "fmt" "testing" - "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" ) -type m map[string]interface{} - -type TestLogWriter struct { - Logs []string -} - -func NewTestLogWriter() *TestLogWriter { - return &TestLogWriter{Logs: make([]string, 0)} -} - -func (tl *TestLogWriter) Write(p []byte) (n int, err error) { - tl.Logs = append(tl.Logs, string(p)) - return len(p), nil -} - -func (tl *TestLogWriter) Reset() { - tl.Logs = tl.Logs[:0] -} +type m = map[string]any func TestContextualError_Log(t *testing.T) { - l := logrus.New() - l.Formatter = &logrus.TextFormatter{ - DisableTimestamp: true, - DisableColors: true, - } - - tl := NewTestLogWriter() - l.Out = tl + buf := &bytes.Buffer{} + l := test.NewLoggerWithOutput(buf) // Test a full context line - tl.Reset() + buf.Reset() e := NewContextualError("test message", m{"field": "1"}, errors.New("error")) e.Log(l) - assert.Equal(t, []string{"level=error msg=\"test message\" error=error field=1\n"}, tl.Logs) + assert.Equal(t, "level=ERROR msg=\"test message\" field=1 error=error\n", buf.String()) // Test a line with an error and msg but no fields - tl.Reset() + buf.Reset() e = NewContextualError("test message", nil, errors.New("error")) e.Log(l) - assert.Equal(t, []string{"level=error msg=\"test message\" error=error\n"}, tl.Logs) + assert.Equal(t, "level=ERROR msg=\"test message\" error=error\n", buf.String()) // Test just a context and fields - tl.Reset() + buf.Reset() e = NewContextualError("test message", m{"field": "1"}, nil) e.Log(l) - assert.Equal(t, []string{"level=error msg=\"test message\" field=1\n"}, tl.Logs) + assert.Equal(t, "level=ERROR msg=\"test message\" field=1\n", buf.String()) // Test just a context - tl.Reset() + buf.Reset() e = NewContextualError("test message", nil, nil) e.Log(l) - assert.Equal(t, []string{"level=error msg=\"test message\"\n"}, tl.Logs) + assert.Equal(t, "level=ERROR msg=\"test message\"\n", buf.String()) // Test just an error - tl.Reset() + buf.Reset() e = NewContextualError("", nil, errors.New("error")) e.Log(l) - assert.Equal(t, []string{"level=error error=error\n"}, tl.Logs) + assert.Equal(t, "level=ERROR msg=\"\" error=error\n", buf.String()) } func TestLogWithContextIfNeeded(t *testing.T) { - l := logrus.New() - l.Formatter = &logrus.TextFormatter{ - DisableTimestamp: true, - DisableColors: true, - } - - tl := NewTestLogWriter() - l.Out = tl + buf := &bytes.Buffer{} + l := test.NewLoggerWithOutput(buf) // Test ignoring fallback context - tl.Reset() + buf.Reset() e := NewContextualError("test message", m{"field": "1"}, errors.New("error")) LogWithContextIfNeeded("This should get thrown away", e, l) - assert.Equal(t, []string{"level=error msg=\"test message\" error=error field=1\n"}, tl.Logs) + assert.Equal(t, "level=ERROR msg=\"test message\" field=1 error=error\n", buf.String()) // Test using fallback context - tl.Reset() + buf.Reset() err := fmt.Errorf("this is a normal error") LogWithContextIfNeeded("Fallback context woo", err, l) - assert.Equal(t, []string{"level=error msg=\"Fallback context woo\" error=\"this is a normal error\"\n"}, tl.Logs) + assert.Equal(t, "level=ERROR msg=\"Fallback context woo\" error=\"this is a normal error\"\n", buf.String()) } func TestContextualizeIfNeeded(t *testing.T) { diff --git a/wfp/wfp_windows.go b/wfp/wfp_windows.go new file mode 100644 index 00000000..22aa0565 --- /dev/null +++ b/wfp/wfp_windows.go @@ -0,0 +1,377 @@ +//go:build (amd64 || arm64) && !e2e_testing +// +build amd64 arm64 +// +build !e2e_testing + +// Package wfp installs Windows Filtering Platform (WFP) PERMIT filters in a dynamic, session-scoped sublayer. +// Because WFP sits below Windows Defender Firewall, a high-weight permit at FWPM_LAYER_ALE_AUTH_RECV_ACCEPT_V4/V6 lets +// the matching inbound traffic through regardless of WDF rules. +// +// Each Session owns its own engine handle. When the handle closes, every dynamic object added during the session +// is auto-deleted by Windows, so there are no orphaned filters. +// +// Type definitions and constants are derived from the wireguard-windows firewall package (MIT). +// Only the subset we exercise is reproduced. +package wfp + +import ( + "fmt" + "unsafe" + + "golang.org/x/sys/windows" +) + +// FWPM layer GUIDs (fwpmu.h). +// +// FWPM_LAYER_ALE_AUTH_RECV_ACCEPT_V4 = e1cd9fe7-f4b5-4273-96c0-592e487b8650 +// FWPM_LAYER_ALE_AUTH_RECV_ACCEPT_V6 = a3b42c97-9f04-4672-b87e-cee9c483257f +var ( + fwpmLayerAleAuthRecvAcceptV4 = windows.GUID{ + Data1: 0xe1cd9fe7, Data2: 0xf4b5, Data3: 0x4273, + Data4: [8]byte{0x96, 0xc0, 0x59, 0x2e, 0x48, 0x7b, 0x86, 0x50}, + } + fwpmLayerAleAuthRecvAcceptV6 = windows.GUID{ + Data1: 0xa3b42c97, Data2: 0x9f04, Data3: 0x4672, + Data4: [8]byte{0xb8, 0x7e, 0xce, 0xe9, 0xc4, 0x83, 0x25, 0x7f}, + } +) + +// FWPM_CONDITION_IP_LOCAL_INTERFACE = 4cd62a49-59c3-4969-b7f3-bda5d32890a4 +var fwpmConditionIPLocalInterface = windows.GUID{ + Data1: 0x4cd62a49, Data2: 0x59c3, Data3: 0x4969, + Data4: [8]byte{0xb7, 0xf3, 0xbd, 0xa5, 0xd3, 0x28, 0x90, 0xa4}, +} + +// FWPM_CONDITION_IP_PROTOCOL = 3971ef2b-623e-4f9a-8cb1-6e79b806b9a7 +var fwpmConditionIPProtocol = windows.GUID{ + Data1: 0x3971ef2b, Data2: 0x623e, Data3: 0x4f9a, + Data4: [8]byte{0x8c, 0xb1, 0x6e, 0x79, 0xb8, 0x06, 0xb9, 0xa7}, +} + +// FWPM_CONDITION_IP_LOCAL_PORT = 0c1ba1af-5765-453f-af22-a8f791ac775b +var fwpmConditionIPLocalPort = windows.GUID{ + Data1: 0x0c1ba1af, Data2: 0x5765, Data3: 0x453f, + Data4: [8]byte{0xaf, 0x22, 0xa8, 0xf7, 0x91, 0xac, 0x77, 0x5b}, +} + +// IPPROTO_UDP from in.h. +const ipprotoUDP uint8 = 17 + +// FWP_ACTION_TYPE values (fwptypes.h). PERMIT is terminating. +const fwpActionPermit uint32 = 0x00001002 // 0x2 | FWP_ACTION_FLAG_TERMINATING(0x1000) + +// FWP_DATA_TYPE values we use. +const ( + fwpEmpty uint32 = 0 + fwpUint8 uint32 = 1 + fwpUint16 uint32 = 2 + fwpUint64 uint32 = 4 +) + +// FWP_MATCH_TYPE values. +const fwpMatchEqual uint32 = 0 + +// FWPM_SESSION flags. +const fwpmSessionFlagDynamic uint32 = 0x1 + +// FWPM_FILTER_FLAG_CLEAR_ACTION_RIGHT prevents lower-priority filters in other sublayers, +// notably Windows Defender Firewall's MPSSVC_WF sublayer, which shares our 0xFFFF weight from overriding this PERMIT. +// Without it, a default WDF block at the same sublayer weight can still win arbitration. +const fwpmFilterFlagClearActionRight uint32 = 0x8 + +// RPC authentication. +// RPC_C_AUTHN_WINNT works on workgroup machines with no domain context +// RPC_C_AUTHN_DEFAULT falls back through a chain that can land on something WFP doesn't accept on a fresh box. +const rpcCAuthnWinNT uint32 = 10 + +// fwpByteBlob (FWP_BYTE_BLOB). 16 bytes on 64-bit. +type fwpByteBlob struct { + size uint32 + _ uint32 // padding + data *uint8 +} + +// fwpValue0 / FWP_CONDITION_VALUE0 layout. 16 bytes on 64-bit. +// The union is pointer-sized; types <= 32 bits (UINT8/16/32, INT8/16/32, float) live inline in the low bytes +// of `value`, while UINT64/INT64/double and aggregate types are stored *by pointer*, even on 64-bit, where the +// union member is declared as UINT64*. So when populating an FWP_UINT64 condition, pass +// uintptr(unsafe.Pointer(&luidVar)) instead of the LUID inline. +type fwpValue0 struct { + type_ uint32 + _ uint32 // padding before union to 8-byte alignment + value uintptr +} + +// fwpmDisplayData0 / FWPM_DISPLAY_DATA0. 16 bytes on 64-bit. +type fwpmDisplayData0 struct { + name *uint16 + description *uint16 +} + +// fwpmAction0 / FWPM_ACTION0. 20 bytes; no leading padding because actionType +// is uint32 and GUID's first field is uint32. +type fwpmAction0 struct { + actionType uint32 + filterType windows.GUID +} + +// fwpmFilterCondition0. 40 bytes on 64-bit. +type fwpmFilterCondition0 struct { + fieldKey windows.GUID // 16 + matchType uint32 // 4 + _ uint32 // 4 padding + conditionValue fwpValue0 // 16 +} + +// fwpmFilter0. 200 bytes on 64-bit. +type fwpmFilter0 struct { + filterKey windows.GUID + displayData fwpmDisplayData0 + flags uint32 + _ uint32 // padding before *GUID + providerKey *windows.GUID + providerData fwpByteBlob + layerKey windows.GUID + subLayerKey windows.GUID + weight fwpValue0 + numFilterConditions uint32 + _ uint32 // padding before pointer + filterCondition *fwpmFilterCondition0 + action fwpmAction0 + _ [4]byte // layout correction + providerContextKey windows.GUID + reserved *windows.GUID + filterID uint64 + effectiveWeight fwpValue0 +} + +// fwpmSublayer0. 72 bytes on 64-bit. +type fwpmSublayer0 struct { + subLayerKey windows.GUID + displayData fwpmDisplayData0 + flags uint32 + _ uint32 // padding before *GUID + providerKey *windows.GUID + providerData fwpByteBlob + weight uint16 + _ [6]byte // padding to 72 bytes +} + +// fwpmSession0. 72 bytes on 64-bit. +type fwpmSession0 struct { + sessionKey windows.GUID + displayData fwpmDisplayData0 + flags uint32 + txnWaitTimeoutInMSec uint32 + processId uint32 + _ uint32 // padding before *SID + sid *windows.SID + username *uint16 + kernelMode uint8 + _ [7]byte // tail padding +} + +// fwpuclnt.dll bindings. Only the calls we use. +var ( + modFwpuclnt = windows.NewLazySystemDLL("fwpuclnt.dll") + procFwpmEngineOpen0 = modFwpuclnt.NewProc("FwpmEngineOpen0") + procFwpmEngineClose0 = modFwpuclnt.NewProc("FwpmEngineClose0") + procFwpmSubLayerAdd0 = modFwpuclnt.NewProc("FwpmSubLayerAdd0") + procFwpmFilterAdd0 = modFwpuclnt.NewProc("FwpmFilterAdd0") +) + +// Session holds the WFP engine handle for a single bypass operation. The handle owns a dynamic session: +// when it is closed, every WFP object added during the session (sublayer + filters) is automatically deleted by +// Windows. That gives us correct cleanup even if the host process is killed hard between Permit* and Close. +type Session struct { + engine uintptr +} + +// Close releases the engine handle. Windows deletes every dynamic object (sublayer + filters) the session installed. +// Safe to call on a nil receiver. +func (s *Session) Close() { + if s == nil || s.engine == 0 { + return + } + procFwpmEngineClose0.Call(s.engine) + s.engine = 0 +} + +// PermitInterface installs PERMIT filters at FWPM_LAYER_ALE_AUTH_RECV_ACCEPT_V4 and _V6 scoped to the given network +// interface LUID. Inbound traffic on that interface bypasses Windows Defender Firewall. +func PermitInterface(luid uint64) (*Session, error) { + s, sublayerKey, err := newSession() + if err != nil { + return nil, err + } + + if err := addInterfaceFilter(s.engine, sublayerKey, fwpmLayerAleAuthRecvAcceptV4, luid); err != nil { + s.Close() + return nil, fmt.Errorf("add v4 filter: %w", err) + } + if err := addInterfaceFilter(s.engine, sublayerKey, fwpmLayerAleAuthRecvAcceptV6, luid); err != nil { + s.Close() + return nil, fmt.Errorf("add v6 filter: %w", err) + } + return s, nil +} + +// PermitUDPPort installs PERMIT filters at FWPM_LAYER_ALE_AUTH_RECV_ACCEPT_V4 and _V6 scoped to UDP traffic with the +// given local port. Inbound UDP to that port on any interface bypasses Windows Defender Firewall. +func PermitUDPPort(port uint16) (*Session, error) { + s, sublayerKey, err := newSession() + if err != nil { + return nil, err + } + + if err := addUDPPortFilter(s.engine, sublayerKey, fwpmLayerAleAuthRecvAcceptV4, port); err != nil { + s.Close() + return nil, fmt.Errorf("add v4 filter: %w", err) + } + if err := addUDPPortFilter(s.engine, sublayerKey, fwpmLayerAleAuthRecvAcceptV6, port); err != nil { + s.Close() + return nil, fmt.Errorf("add v6 filter: %w", err) + } + return s, nil +} + +func newSession() (*Session, windows.GUID, error) { + engine, err := openDynamicEngine() + if err != nil { + return nil, windows.GUID{}, err + } + sublayerKey, err := registerSublayer(engine) + if err != nil { + procFwpmEngineClose0.Call(engine) + return nil, windows.GUID{}, err + } + return &Session{engine: engine}, sublayerKey, nil +} + +func openDynamicEngine() (uintptr, error) { + session := fwpmSession0{flags: fwpmSessionFlagDynamic} + var engine uintptr + r1, _, _ := procFwpmEngineOpen0.Call( + 0, // serverName == NULL (local) + uintptr(rpcCAuthnWinNT), + 0, // authIdentity == NULL + uintptr(unsafe.Pointer(&session)), + uintptr(unsafe.Pointer(&engine)), + ) + if r1 != 0 { + return 0, fmt.Errorf("FwpmEngineOpen0: 0x%x", r1) + } + return engine, nil +} + +// registerSublayer adds a session-scoped sublayer with a freshly generated GUID, weight 0xFFFF so its filters arbitrate +// above WDF's default sublayer. The sublayer is dynamic (no PERSISTENT flag) and goes away when the engine handle closes. +func registerSublayer(engine uintptr) (windows.GUID, error) { + key, err := windows.GenerateGUID() + if err != nil { + return windows.GUID{}, fmt.Errorf("GenerateGUID for sublayer: %w", err) + } + + name, _ := windows.UTF16PtrFromString("Nebula WDF bypass sublayer") + desc, _ := windows.UTF16PtrFromString("Permit filters bypassing Windows Defender Firewall") + sl := fwpmSublayer0{ + subLayerKey: key, + displayData: fwpmDisplayData0{name: name, description: desc}, + weight: 0xFFFF, + } + r1, _, _ := procFwpmSubLayerAdd0.Call( + engine, + uintptr(unsafe.Pointer(&sl)), + 0, // sd == NULL + ) + if r1 != 0 { + return windows.GUID{}, fmt.Errorf("FwpmSubLayerAdd0: 0x%x", r1) + } + return key, nil +} + +func addInterfaceFilter(engine uintptr, sublayerKey, layer windows.GUID, luid uint64) error { + name, _ := windows.UTF16PtrFromString("Nebula allow interface inbound") + desc, _ := windows.UTF16PtrFromString("Permits inbound traffic on a nebula interface") + + // luid must remain addressable through the syscall -- FWP_UINT64 is stored + // by pointer in the FWP_VALUE0 union. + cond := fwpmFilterCondition0{ + fieldKey: fwpmConditionIPLocalInterface, + matchType: fwpMatchEqual, + conditionValue: fwpValue0{ + type_: fwpUint64, + value: uintptr(unsafe.Pointer(&luid)), + }, + } + + filter := fwpmFilter0{ + // filterKey left zero: WFP assigns one when the filter is added. + displayData: fwpmDisplayData0{name: name, description: desc}, + flags: fwpmFilterFlagClearActionRight, + layerKey: layer, + subLayerKey: sublayerKey, + weight: fwpValue0{type_: fwpUint8, value: uintptr(15)}, + numFilterConditions: 1, + filterCondition: &cond, + action: fwpmAction0{actionType: fwpActionPermit}, + } + + r1, _, _ := procFwpmFilterAdd0.Call( + engine, + uintptr(unsafe.Pointer(&filter)), + 0, // sd == NULL + 0, // id == NULL + ) + if r1 != 0 { + return fmt.Errorf("FwpmFilterAdd0: 0x%x", r1) + } + return nil +} + +// addUDPPortFilter installs a PERMIT filter that matches (IP_PROTOCOL == UDP) AND (IP_LOCAL_PORT == port). +// FWP_UINT8 and FWP_UINT16 are <= 32 bits so they live inline in the FWP_VALUE0 union. +func addUDPPortFilter(engine uintptr, sublayerKey, layer windows.GUID, port uint16) error { + name, _ := windows.UTF16PtrFromString("Nebula allow UDP port inbound") + desc, _ := windows.UTF16PtrFromString("Permits inbound UDP to a nebula listener port") + + conds := [2]fwpmFilterCondition0{ + { + fieldKey: fwpmConditionIPProtocol, + matchType: fwpMatchEqual, + conditionValue: fwpValue0{ + type_: fwpUint8, + value: uintptr(ipprotoUDP), + }, + }, + { + fieldKey: fwpmConditionIPLocalPort, + matchType: fwpMatchEqual, + conditionValue: fwpValue0{ + type_: fwpUint16, + value: uintptr(port), + }, + }, + } + + filter := fwpmFilter0{ + displayData: fwpmDisplayData0{name: name, description: desc}, + flags: fwpmFilterFlagClearActionRight, + layerKey: layer, + subLayerKey: sublayerKey, + weight: fwpValue0{type_: fwpUint8, value: uintptr(15)}, + numFilterConditions: 2, + filterCondition: &conds[0], + action: fwpmAction0{actionType: fwpActionPermit}, + } + + r1, _, _ := procFwpmFilterAdd0.Call( + engine, + uintptr(unsafe.Pointer(&filter)), + 0, + 0, + ) + if r1 != 0 { + return fmt.Errorf("FwpmFilterAdd0: 0x%x", r1) + } + return nil +}