diff --git a/base/cvd/MODULE.bazel b/base/cvd/MODULE.bazel index 62a7d351d7f..006c22cdbec 100644 --- a/base/cvd/MODULE.bazel +++ b/base/cvd/MODULE.bazel @@ -16,6 +16,7 @@ bazel_dep(name = "curl") bazel_dep(name = "cxx.rs", version = "1.0.194") bazel_dep(name = "depend_on_what_you_use", version = "1.0.0") bazel_dep(name = "fmt", version = "11.2.0.bcr.1") +bazel_dep(name = "gazelle", version = "0.46.0") bazel_dep(name = "gflags") bazel_dep(name = "googleapis", version = "0.0.0-20251003-2193a2bf") bazel_dep(name = "googleapis-cc", version = "1.0.0") @@ -42,6 +43,7 @@ bazel_dep(name = "rules_cc", version = "0.2.18") bazel_dep(name = "rules_cuda") # CUDA support for NVENC (hermetic headers via redist_json, runtime dlopen) bazel_dep(name = "rules_flex", version = "0.4") bazel_dep(name = "rules_foreign_cc", version = "0.15.1") # this is normally an indirect dependency, but for bazel 9 we need a higher version +bazel_dep(name = "rules_go", version = "0.60.0") bazel_dep(name = "rules_java", version = "8.16.1") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_nodejs", version = "6.7.3") # this is normally an indirect dependency, but for bazel 9 we need a higher version @@ -62,3 +64,21 @@ include("//cuttlefish/host/commands/append_squashfs_overlay:append_squashfs_over include("//cuttlefish/host/commands/vhost_user_input:vhost_user_input.MODULE.bazel") include("//cuttlefish/host/commands/vhost_user_media:vhost_user_media.MODULE.bazel") include("//toolchain:toolchain.MODULE.bazel") + +go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") +go_sdk.download(version = "1.24.12") + +go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//host_tests:go.mod") +use_repo( + go_deps, + "com_github_google_go_cmp", +) + +new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +new_local_repository( + name = "cuttlefish_debian", + build_file_content = 'exports_files(glob(["**"]))\n', + path = "../debian", +) diff --git a/base/cvd/host_tests/BUILD.bazel b/base/cvd/host_tests/BUILD.bazel new file mode 100644 index 00000000000..991e19f92b3 --- /dev/null +++ b/base/cvd/host_tests/BUILD.bazel @@ -0,0 +1,18 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exports_files([ + "go.mod", + "go.sum", +]) diff --git a/e2etests/host_resources/common/BUILD.bazel b/base/cvd/host_tests/common/BUILD.bazel similarity index 85% rename from e2etests/host_resources/common/BUILD.bazel rename to base/cvd/host_tests/common/BUILD.bazel index c342a290f0f..f8ab015fe07 100644 --- a/e2etests/host_resources/common/BUILD.bazel +++ b/base/cvd/host_tests/common/BUILD.bazel @@ -17,6 +17,8 @@ load("@rules_go//go:def.bzl", "go_library") go_library( name = "common", srcs = [ + "cvdalloc.go", + "dnsmasq.go", "init_filesystem_ns.go", "init_network_ns.go", "ip.go", @@ -25,7 +27,8 @@ go_library( "state.go", "static_resources.go", ], - importpath = "github.com/google/android-cuttlefish/e2etests/host_resources/common", + embedsrcs = ["dnsmasq_shim.sh"], + importpath = "github.com/google/android-cuttlefish/base/cvd/host_tests/common", visibility = ["//visibility:public"], deps = [ "@com_github_google_go_cmp//cmp", diff --git a/base/cvd/host_tests/common/cvdalloc.go b/base/cvd/host_tests/common/cvdalloc.go new file mode 100644 index 00000000000..5752eac89c9 --- /dev/null +++ b/base/cvd/host_tests/common/cvdalloc.go @@ -0,0 +1,229 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "bytes" + "fmt" + "log" + "net" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "syscall" + "testing" + "time" + + "github.com/bazelbuild/rules_go/go/runfiles" +) + +const cvdallocRunDir = "/var/tmp/cvd" + +const ( + cvdallocReadyTimeout = 30 * time.Second + cvdallocTeardownTimeout = 30 * time.Second + cvdallocSignalTimeout = 5 * time.Second +) + +type Cvdalloc struct { + sandbox *Sandbox + bin string +} + +// NewCvdalloc resolves the cvdalloc binary under test and prepares the sandbox +// to run it. +// +// Resolution order: +// 1. $CVDALLOC_BIN, if set, is used verbatim (an explicit override). +// 2. Otherwise the binary is located through the Bazel runfiles of the +// //cuttlefish/host/commands/cvdalloc:cvdalloc data dependency, whose +// runfiles path is passed in $CVDALLOC_RLOCATION. +func NewCvdalloc(t *testing.T, s *Sandbox) *Cvdalloc { + bin, err := resolveCvdallocBin() + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(bin); err != nil { + t.Fatalf("cvdalloc binary %q is not usable: %v", bin, err) + } + log.Printf("cvdalloc binary under test: %s", bin) + + // Prepare a tmpfs for the rundir. + if _, err := s.Run("sh", "-c", fmt.Sprintf("mkdir -p %s && mount -t tmpfs tmpfs %s", cvdallocRunDir, cvdallocRunDir)); err != nil { + t.Fatalf("failed to prepare cvdalloc run dir %q: %v", cvdallocRunDir, err) + } + + return &Cvdalloc{sandbox: s, bin: bin} +} + +func resolveCvdallocBin() (string, error) { + if bin := os.Getenv("CVDALLOC_BIN"); bin != "" { + fi, err := os.Stat(bin) + if err != nil { + return "", fmt.Errorf("CVDALLOC_BIN=%q is not usable: %v", bin, err) + } + if fi.IsDir() { + return "", fmt.Errorf("CVDALLOC_BIN=%q is a directory, expected a binary", bin) + } + return bin, nil + } + + rloc := os.Getenv("CVDALLOC_RLOCATION") + if rloc == "" { + return "", fmt.Errorf("CVDALLOC_RLOCATION is not set (expected the runfiles path " + + "of //cuttlefish/host/commands/cvdalloc:cvdalloc); or set CVDALLOC_BIN to a binary") + } + bin, err := runfiles.Rlocation(rloc) + if err != nil { + return "", fmt.Errorf("failed to locate cvdalloc runfile %q: %v", rloc, err) + } + return bin, nil +} + +func (c *Cvdalloc) Setup() error { + _, err := c.sandbox.Run(c.bin, "--setup") + return err +} + +func (c *Cvdalloc) Teardown() error { + _, err := c.sandbox.Run(c.bin, "--teardown") + return err +} + +type Instance struct { + ID int + conn net.Conn + cmd *exec.Cmd + stdout *bytes.Buffer + stderr *bytes.Buffer +} + +// StartInstance launches "cvdalloc --id= --socket=" inside the sandbox: +// 1. A socketpair is created +// 2. the peer end is passed to the child as fd 3 (inherited through nsenter) +// 3. this call blocks until the child signals that allocation is complete. +func (c *Cvdalloc) StartInstance(id int) (*Instance, error) { + fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0) + if err != nil { + return nil, fmt.Errorf("socketpair: %w", err) + } + ourEnd := os.NewFile(uintptr(fds[0]), "cvdalloc-ctl") + theirEnd := os.NewFile(uintptr(fds[1]), "cvdalloc-peer") + + // net.FileConn dups the fd and gives us deadline support; drop the original. + conn, err := net.FileConn(ourEnd) + ourEnd.Close() + if err != nil { + theirEnd.Close() + return nil, fmt.Errorf("wrapping control socket: %w", err) + } + + // ExtraFiles[0] becomes fd 3 in the child. nsenter preserves it across the + // setns/exec into the sandbox, so cvdalloc sees the socket at --socket=3. + args := c.sandbox.nsenterArgs(c.bin, fmt.Sprintf("--id=%d", id), "--socket=3") + cmd := exec.CommandContext(c.sandbox.ctx, args[0], args[1:]...) + cmd.ExtraFiles = []*os.File{theirEnd} + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Start(); err != nil { + conn.Close() + theirEnd.Close() + return nil, fmt.Errorf("starting cvdalloc: %w", err) + } + // The child owns its own dup of the peer end now. + theirEnd.Close() + + inst := &Instance{ID: id, conn: conn, cmd: cmd, stdout: &stdout, stderr: &stderr} + + // Wait for the child's readiness Post (a single byte). + if err := recvByte(conn, cvdallocReadyTimeout); err != nil { + inst.kill() + return nil, fmt.Errorf("waiting for cvdalloc id=%d to finish allocation: %w%s", id, err, inst.stderrSuffix()) + } + return inst, nil +} + +// StopInstance signals the instance to tear down (a single byte), waits for its +// acknowledging Post, and then reaps the process. +func (c *Cvdalloc) StopInstance(inst *Instance) error { + if err := sendByte(inst.conn, cvdallocSignalTimeout); err != nil { + inst.kill() + return fmt.Errorf("signaling teardown to cvdalloc id=%d: %w", inst.ID, err) + } + if err := recvByte(inst.conn, cvdallocTeardownTimeout); err != nil { + inst.kill() + return fmt.Errorf("waiting for cvdalloc id=%d teardown ack: %w%s", inst.ID, err, inst.stderrSuffix()) + } + inst.conn.Close() + if err := inst.cmd.Wait(); err != nil { + return fmt.Errorf("cvdalloc id=%d exited with error: %w%s", inst.ID, err, inst.stderrSuffix()) + } + return nil +} + +func (i *Instance) kill() { + i.conn.Close() + if i.cmd.Process != nil { + i.cmd.Process.Kill() + i.cmd.Wait() + } +} + +func (i *Instance) stderrSuffix() string { + s := strings.TrimSpace(i.stderr.String()) + if s == "" { + return "" + } + return "\n--- cvdalloc stderr ---\n" + s +} + +func recvByte(c net.Conn, timeout time.Duration) error { + if err := c.SetReadDeadline(time.Now().Add(timeout)); err != nil { + return err + } + buf := make([]byte, 1) + if _, err := c.Read(buf); err != nil { + return err + } + return nil +} + +func sendByte(c net.Conn, timeout time.Duration) error { + if err := c.SetWriteDeadline(time.Now().Add(timeout)); err != nil { + return err + } + _, err := c.Write([]byte{0}) + return err +} + +// CvdallocDnsmasqIfaces reports the interfaces for which cvdalloc started a +// dnsmasq, by inspecting the pidfiles it writes under CvdDir(). +func CvdallocDnsmasqIfaces(s *Sandbox) []string { + out, err := s.Run("sh", "-c", fmt.Sprintf("ls -1 %s/cuttlefish-dnsmasq-*.pid 2>/dev/null || true", cvdallocRunDir)) + if err != nil { + return nil + } + var ifaces []string + for _, line := range nonEmptyLines(out.Stdout) { + base := filepath.Base(line) + ifaces = append(ifaces, strings.TrimSuffix(strings.TrimPrefix(base, "cuttlefish-dnsmasq-"), ".pid")) + } + sort.Strings(ifaces) + return ifaces +} diff --git a/base/cvd/host_tests/common/dnsmasq.go b/base/cvd/host_tests/common/dnsmasq.go new file mode 100644 index 00000000000..620ca54542d --- /dev/null +++ b/base/cvd/host_tests/common/dnsmasq.go @@ -0,0 +1,36 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + _ "embed" + "os" + "path/filepath" + "testing" +) + +//go:embed dnsmasq_shim.sh +var dnsmasqShimScript []byte + +func installDnsmasqShim(t *testing.T, s *Sandbox) { + dir := filepath.Join(s.tempdir, "shim") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("creating dnsmasq shim dir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "dnsmasq"), dnsmasqShimScript, 0o755); err != nil { + t.Fatalf("writing dnsmasq shim: %v", err) + } + os.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} diff --git a/base/cvd/host_tests/common/dnsmasq_shim.sh b/base/cvd/host_tests/common/dnsmasq_shim.sh new file mode 100644 index 00000000000..30037582547 --- /dev/null +++ b/base/cvd/host_tests/common/dnsmasq_shim.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Stand-in for dnsmasq, used by both host_tests e2e tests (the init script +# in static_resources_init_test and cvdalloc in cvdalloc_test) via a PATH shim. +# +# The real dnsmasq cannot run inside the rootless user namespace the sandbox +# uses. This is because setgroups(2) cannot be called in an unprivileged +# userns. + +pidfile="" +for arg in "$@"; do + case "$arg" in + --pid-file=*) pidfile="${arg#--pid-file=}" ;; + esac +done + +setsid /bin/sh -c ' + pidfile="$1" + # Close descriptors above stderr (e.g. the caller control socket) so the + # daemon does not keep them open for its whole lifetime. + for fd in 3 4 5 6 7 8 9; do + eval "exec ${fd}>&-" 2>/dev/null || true + done + child="" + cleanup() { + [ -n "$pidfile" ] && rm -f "$pidfile" + [ -n "$child" ] && kill "$child" 2>/dev/null + exit 0 + } + trap cleanup TERM INT + [ -n "$pidfile" ] && printf "%s\n" "$$" >"$pidfile" + # Sleep until signalled (teardown) or until the sandbox pid namespace is + # torn down. 2147483647s stands in for "forever" portably. + sleep 2147483647 & + child=$! + wait "$child" + cleanup +' dnsmasq-shim "$pidfile" /dev/null 2>&1 & + +exit 0 diff --git a/e2etests/host_resources/common/init_filesystem_ns.go b/base/cvd/host_tests/common/init_filesystem_ns.go similarity index 100% rename from e2etests/host_resources/common/init_filesystem_ns.go rename to base/cvd/host_tests/common/init_filesystem_ns.go diff --git a/e2etests/host_resources/common/init_network_ns.go b/base/cvd/host_tests/common/init_network_ns.go similarity index 100% rename from e2etests/host_resources/common/init_network_ns.go rename to base/cvd/host_tests/common/init_network_ns.go diff --git a/e2etests/host_resources/common/ip.go b/base/cvd/host_tests/common/ip.go similarity index 100% rename from e2etests/host_resources/common/ip.go rename to base/cvd/host_tests/common/ip.go diff --git a/e2etests/host_resources/common/nft.go b/base/cvd/host_tests/common/nft.go similarity index 100% rename from e2etests/host_resources/common/nft.go rename to base/cvd/host_tests/common/nft.go diff --git a/e2etests/host_resources/common/sandbox.go b/base/cvd/host_tests/common/sandbox.go similarity index 99% rename from e2etests/host_resources/common/sandbox.go rename to base/cvd/host_tests/common/sandbox.go index 7821d60a789..5e1dde7ab7d 100644 --- a/e2etests/host_resources/common/sandbox.go +++ b/base/cvd/host_tests/common/sandbox.go @@ -68,6 +68,8 @@ func NewSandbox(t *testing.T) *Sandbox { t.Fatalf("failed to prepare network sandbox: %v", err) } + installDnsmasqShim(t, s) + return s } diff --git a/e2etests/host_resources/common/state.go b/base/cvd/host_tests/common/state.go similarity index 100% rename from e2etests/host_resources/common/state.go rename to base/cvd/host_tests/common/state.go diff --git a/e2etests/host_resources/common/static_resources.go b/base/cvd/host_tests/common/static_resources.go similarity index 100% rename from e2etests/host_resources/common/static_resources.go rename to base/cvd/host_tests/common/static_resources.go diff --git a/base/cvd/host_tests/go.mod b/base/cvd/host_tests/go.mod new file mode 100644 index 00000000000..81c201cbfa1 --- /dev/null +++ b/base/cvd/host_tests/go.mod @@ -0,0 +1,7 @@ +module github.com/google/android-cuttlefish/base/cvd/host_tests + +go 1.24.0 + +require github.com/google/go-cmp v0.6.0 + +require github.com/bazelbuild/rules_go v0.60.0 // indirect diff --git a/base/cvd/host_tests/go.sum b/base/cvd/host_tests/go.sum new file mode 100644 index 00000000000..d007be37f67 --- /dev/null +++ b/base/cvd/host_tests/go.sum @@ -0,0 +1,4 @@ +github.com/bazelbuild/rules_go v0.60.0 h1:apGSxTTrFUyLNvX9NQmF4CbntWAO0/S5eALeVgB/6Qk= +github.com/bazelbuild/rules_go v0.60.0/go.mod h1:CYcohJVxs4n7eftbC39GCqaEJm3E1EME+6QAkGguKoI= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= diff --git a/e2etests/host_resources/static_resources_init_test/BUILD.bazel b/base/cvd/host_tests/static_resources_init_test/BUILD.bazel similarity index 97% rename from e2etests/host_resources/static_resources_init_test/BUILD.bazel rename to base/cvd/host_tests/static_resources_init_test/BUILD.bazel index 6ba1a4021c7..8af4ce5a77d 100644 --- a/e2etests/host_resources/static_resources_init_test/BUILD.bazel +++ b/base/cvd/host_tests/static_resources_init_test/BUILD.bazel @@ -31,7 +31,7 @@ go_test( "no-sandbox", ], deps = [ - "//host_resources/common", + "//host_tests/common", "@com_github_google_go_cmp//cmp", "@com_github_google_go_cmp//cmp/cmpopts", ], diff --git a/e2etests/host_resources/static_resources_init_test/main_test.go b/base/cvd/host_tests/static_resources_init_test/main_test.go similarity index 98% rename from e2etests/host_resources/static_resources_init_test/main_test.go rename to base/cvd/host_tests/static_resources_init_test/main_test.go index baa33ca9301..1be421572d7 100644 --- a/e2etests/host_resources/static_resources_init_test/main_test.go +++ b/base/cvd/host_tests/static_resources_init_test/main_test.go @@ -17,7 +17,7 @@ package main import ( "testing" - "github.com/google/android-cuttlefish/e2etests/host_resources/common" + "github.com/google/android-cuttlefish/base/cvd/host_tests/common" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" ) diff --git a/e2etests/MODULE.bazel b/e2etests/MODULE.bazel index 11521274cea..b7c91966dda 100644 --- a/e2etests/MODULE.bazel +++ b/e2etests/MODULE.bazel @@ -20,14 +20,3 @@ local_path_override( module_name = "com_github_google_android_cuttlefish_frontend", path = "../frontend", ) - -# Expose the cuttlefish-host-resources init script (which lives in -# ../base/debian, outside this Bazel module) as a data dependency for the -# host_resources e2e tests. -new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") - -new_local_repository( - name = "cuttlefish_debian", - build_file_content = 'exports_files(glob(["**"]))\n', - path = "../base/debian", -)