From 8191b0af591171daa57cae7c76658ac988f7ac0d Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:23:10 +0000 Subject: [PATCH 01/20] Add an Fstat method to Fd --- base/cvd/cuttlefish/common/libs/fs/fd.cc | 9 +++++++++ base/cvd/cuttlefish/common/libs/fs/fd.h | 2 ++ 2 files changed, 11 insertions(+) diff --git a/base/cvd/cuttlefish/common/libs/fs/fd.cc b/base/cvd/cuttlefish/common/libs/fs/fd.cc index abfa3fc8d04..8c26ce19f07 100644 --- a/base/cvd/cuttlefish/common/libs/fs/fd.cc +++ b/base/cvd/cuttlefish/common/libs/fs/fd.cc @@ -699,6 +699,15 @@ Result Fd::Flock(int operation) { return {}; } +Result Fd::Fstat() { + LocalErrno record_errno(errno_); + + struct stat file_info = {}; + CF_EXPECT(TEMP_FAILURE_RETRY(fstat(fd_, &file_info)) == 0, + ::cuttlefish::StrError(errno)); + return file_info; +} + int Fd::GetSockName(struct sockaddr* addr, socklen_t* addrlen) { LocalErrno record_errno(errno_); diff --git a/base/cvd/cuttlefish/common/libs/fs/fd.h b/base/cvd/cuttlefish/common/libs/fs/fd.h index e9e93daaab3..5011d6cf97f 100644 --- a/base/cvd/cuttlefish/common/libs/fs/fd.h +++ b/base/cvd/cuttlefish/common/libs/fs/fd.h @@ -183,6 +183,8 @@ class Fd : public ReaderWriterSeeker { Result Flock(int operation); + Result Fstat(); + int GetErrno() const { return errno_; } int GetSockName(struct sockaddr* addr, socklen_t* addrlen); From e573482c830170a10b9b3c0e88b7faa13114ed1d Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:35 +0000 Subject: [PATCH 02/20] Parse gs:// and https:// URLs as build strings --- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 2 + .../host/libs/web/android_build_api.cpp | 13 +- .../host/libs/web/android_build_string.cpp | 136 ++++++++++++ .../host/libs/web/android_build_string.h | 29 ++- .../libs/web/android_build_string_tests.cpp | 194 ++++++++++++++++++ 5 files changed, 370 insertions(+), 4 deletions(-) diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index 71d1899fe48..0d496b2c474 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -64,6 +64,7 @@ cf_cc_library( hdrs = ["android_build_string.h"], deps = [ "//cuttlefish/flag_parser", + "//cuttlefish/host/libs/web/http_client:scrub_secrets", "//cuttlefish/result", "@abseil-cpp//absl/strings", "@fmt", @@ -76,6 +77,7 @@ cf_cc_test( deps = [ "//cuttlefish/flag_parser", "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/result", "//cuttlefish/result:result_matchers", ], ) diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp index ffe7165b5e7..16afa01b6ca 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp @@ -37,6 +37,7 @@ #include "absl/log/log.h" #include "android-base/file.h" #include "fmt/format.h" +#include "fmt/ostream.h" #include "json/value.h" #include "cuttlefish/common/libs/utils/contains.h" @@ -146,9 +147,15 @@ Result AndroidBuildApi::GetBuild( } Result AndroidBuildApi::GetBuild(const BuildString& build_string) { - Result result = - std::visit([this](auto&& arg) { return GetBuild(arg); }, build_string); - return CF_EXPECT(std::move(result)); + if (const auto* device = std::get_if(&build_string)) { + return CF_EXPECT(GetBuild(*device)); + } + if (const auto* directory = + std::get_if(&build_string)) { + return CF_EXPECT(GetBuild(*directory)); + } + return CF_ERRF("AndroidBuildApi cannot handle '{}'", + fmt::streamed(build_string)); } Result AndroidBuildApi::DownloadFile( diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp index 4fc3cc79419..d12864eac50 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp @@ -26,18 +26,70 @@ #include #include +#include "absl/strings/ascii.h" #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" +#include "absl/strings/strip.h" #include "fmt/ostream.h" #include "fmt/ranges.h" #include "cuttlefish/flag_parser/flag.h" +#include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" #include "cuttlefish/result/result.h" namespace cuttlefish { namespace { +// Returns the "" of a "://" build string, which is what +// separates a URL build source from a branch, a build id or a directory path. +std::optional UrlScheme(std::string_view build_string) { + constexpr std::string_view kSchemeCharacters = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-."; + // The allowed set also holds digits and "+-.", which cannot open a scheme, + // and an empty scheme has no opening character at all. + if (build_string.empty() || !absl::ascii_isalpha(build_string.front())) { + return std::nullopt; + } + const size_t separator = build_string.find_first_not_of(kSchemeCharacters); + if (separator == std::string_view::npos || + !build_string.substr(separator).starts_with("://")) { + return std::nullopt; + } + return build_string.substr(0, separator); +} + +// Returns the digest of a "#sha256=<64 hex digits>" fragment, or nullopt when +// `fragment` is anything else. +std::optional Sha256FragmentDigest( + std::string_view fragment) { + constexpr size_t kHexDigits = 64; + if (!absl::ConsumePrefix(&fragment, "#sha256=") || + fragment.size() != kHexDigits || + fragment.find_first_not_of("0123456789abcdefABCDEF") != + std::string_view::npos) { + return std::nullopt; + } + return fragment; +} + +// A fragment is a suffix by definition, so a '#' anywhere else is an error +// rather than something silently kept in the URL. +Result>> +ParseSha256Fragment(std::string_view build_string) { + const size_t fragment_start = build_string.find('#'); + if (fragment_start == std::string_view::npos) { + return {{build_string, std::nullopt}}; + } + const std::optional digest = + Sha256FragmentDigest(build_string.substr(fragment_start)); + CF_EXPECTF(digest.has_value(), + "Only a trailing '#sha256=<64 hex digits>' fragment is supported " + "in a URL build string. Input: '{}'", + build_string); + return {{build_string.substr(0, fragment_start), digest}}; +} + Result>> ParseFilepath( std::string_view build_string) { std::string_view remaining_build_string = build_string; @@ -99,6 +151,60 @@ Result ParseDirectoryBuildString( return result; } +Result ParseUrlBuildString(std::string_view scheme, + std::string_view build_string) { + CF_EXPECTF(scheme != "http", + "Cleartext 'http://' build sources are not supported, use " + "'https://' instead. Input: '{}'", + build_string); + CF_EXPECTF(scheme == "gs" || scheme == "https", + "Unsupported URL scheme '{}'. The supported URL schemes are " + "'gs://' and 'https://'. Input: '{}'", + scheme, build_string); + // The format reserves ',' because build strings travel in comma separated + // lists, where a URL holding one would be split into pieces that each parse + // as some other kind of build string. + CF_EXPECTF(build_string.find(',') == std::string_view::npos, + "URL build strings cannot contain a comma, which the format " + "reserves because build strings travel in comma separated " + "lists. Input: '{}'", + build_string); + + auto [without_fragment, sha256] = + CF_EXPECT(ParseSha256Fragment(build_string)); + size_t close_bracket = without_fragment.find('}'); + CF_EXPECTF(close_bracket == std::string_view::npos || + close_bracket + 1 == without_fragment.size(), + "A URL build string cannot have characters after the closing " + "curly bracket. Input: '{}'", + build_string); + auto [url, filepath] = CF_EXPECT(ParseFilepath(without_fragment)); + + const size_t query = url.find('?'); + const bool object_form = !url.substr(0, query).ends_with('/'); + // The directory form resolves an artifact by joining its name onto the URL, + // which would put the name after the query string. A signed query also + // authorizes exactly one resource, so it cannot cover a listing and every + // artifact under the prefix. + CF_EXPECTF(object_form || query == std::string::npos, + "Query strings are only supported on URLs naming an object, not " + "on a '/'-terminated directory. Input: '{}'", + build_string); + CF_EXPECTF(object_form || !sha256.has_value(), + "'#sha256=' is only supported on URLs naming an object, not on a " + "'/'-terminated directory. Input: '{}'", + build_string); + + std::optional digest; + if (sha256.has_value()) { + digest = std::string(*sha256); + } + if (scheme == "gs") { + return GcsBuildString{.url = url, .filepath = filepath, .sha256 = digest}; + } + return HttpBuildString{.url = url, .filepath = filepath, .sha256 = digest}; +} + } // namespace std::ostream& operator<<(std::ostream& out, @@ -117,6 +223,22 @@ std::ostream& operator<<(std::ostream& out, return out; } +std::ostream& operator<<(std::ostream& out, + const GcsBuildString& build_string) { + fmt::print(out, "(url=\"{}\", filepath=\"{}\", sha256=\"{}\")", + ScrubUrl(build_string.url), build_string.filepath.value_or(""), + build_string.sha256.value_or("")); + return out; +} + +std::ostream& operator<<(std::ostream& out, + const HttpBuildString& build_string) { + fmt::print(out, "(url=\"{}\", filepath=\"{}\", sha256=\"{}\")", + ScrubUrl(build_string.url), build_string.filepath.value_or(""), + build_string.sha256.value_or("")); + return out; +} + std::ostream& operator<<(std::ostream& out, const BuildString& build_string) { std::visit([&out](auto&& arg) { out << arg; }, build_string); return out; @@ -142,6 +264,12 @@ void SetFilepath(BuildString& build_string, const std::string& value) { Result ParseBuildString(std::string_view build_string) { CF_EXPECT(!build_string.empty(), "The given build string cannot be empty"); + // Checked before the ':' of a directory build string, which every URL also + // contains. + const std::optional scheme = UrlScheme(build_string); + if (scheme) { + return CF_EXPECT(ParseUrlBuildString(*scheme, build_string)); + } auto [remaining_build_string, filepath] = CF_EXPECT(ParseFilepath(build_string)); if (remaining_build_string.find(':') != std::string::npos) { @@ -207,6 +335,14 @@ struct WithFallbackTargetVisitor { const std::string&) { return build_string; } + + BuildString operator()(GcsBuildString build_string, const std::string&) { + return build_string; + } + + BuildString operator()(HttpBuildString build_string, const std::string&) { + return build_string; + } }; } // namespace diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_string.h b/base/cvd/cuttlefish/host/libs/web/android_build_string.h index d7614c8bc2e..7f68dc81e41 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_string.h +++ b/base/cvd/cuttlefish/host/libs/web/android_build_string.h @@ -49,7 +49,34 @@ struct DirectoryBuildString { std::ostream& operator<<(std::ostream& out, const DirectoryBuildString& build_string); -using BuildString = std::variant; +// Names either a single object or, with a trailing '/', the prefix holding a +// build's artifacts. +struct GcsBuildString { + std::string url; + std::optional filepath; + std::optional sha256; + + auto operator<=>(const GcsBuildString&) const = default; +}; + +std::ostream& operator<<(std::ostream& out, const GcsBuildString& build_string); + +// Names either a single file or, with a trailing '/', the prefix holding a +// build's artifacts, over `https://`. Any credential travels in the URL +// itself, as in a pre-signed URL. +struct HttpBuildString { + std::string url; + std::optional filepath; + std::optional sha256; + + auto operator<=>(const HttpBuildString&) const = default; +}; + +std::ostream& operator<<(std::ostream& out, + const HttpBuildString& build_string); + +using BuildString = std::variant; std::ostream& operator<<(std::ostream& out, const BuildString& build_string); diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_string_tests.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_string_tests.cpp index eedc5e9ab1c..7be79e36db8 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_string_tests.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_string_tests.cpp @@ -22,17 +22,23 @@ #include "cuttlefish/flag_parser/flag.h" #include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/result/result.h" #include "cuttlefish/result/result_matchers.h" namespace cuttlefish { +using ::testing::AllOf; using ::testing::ElementsAre; using ::testing::Eq; +using ::testing::HasSubstr; using ::testing::IsEmpty; using ::testing::Optional; using ::testing::SizeIs; using ::testing::VariantWith; +constexpr char kSha256[] = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + TEST(ParseBuildStringTests, DeviceBuildStringSuccess) { auto result = ParseBuildString("abcde/test_target"); EXPECT_THAT(result, IsOk()); @@ -202,4 +208,192 @@ TEST(BuildStringGflagsCompatFlagTests, MultiValueMixedWithEmptySuccess) { DeviceBuildString{.branch_or_id = "abcde", .target = "test_target"})); } +TEST(ParseBuildStringTests, GcsObjectSuccess) { + Result result = ParseBuildString("gs://bucket/path/file.zip"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), VariantWith(GcsBuildString{ + .url = "gs://bucket/path/file.zip"})); +} + +TEST(ParseBuildStringTests, GcsDirectorySuccess) { + Result result = ParseBuildString("gs://bucket/dist/"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), VariantWith( + GcsBuildString{.url = "gs://bucket/dist/"})); +} + +TEST(ParseBuildStringTests, GcsObjectFilepathSuccess) { + Result result = + ParseBuildString("gs://bucket/img.zip{boot.img}"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), + VariantWith(GcsBuildString{ + .url = "gs://bucket/img.zip", .filepath = "boot.img"})); +} + +TEST(ParseBuildStringTests, GcsDirectoryFilepathSuccess) { + Result result = ParseBuildString("gs://bucket/dist/{bzImage}"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), + VariantWith(GcsBuildString{ + .url = "gs://bucket/dist/", .filepath = "bzImage"})); +} + +TEST(ParseBuildStringTests, HttpObjectSuccess) { + Result result = + ParseBuildString("https://example.com/path/file.zip"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), VariantWith(HttpBuildString{ + .url = "https://example.com/path/file.zip"})); +} + +TEST(ParseBuildStringTests, HttpDirectorySuccess) { + Result result = ParseBuildString("https://example.com/dist/"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), VariantWith(HttpBuildString{ + .url = "https://example.com/dist/"})); +} + +TEST(ParseBuildStringTests, HttpObjectFilepathSuccess) { + Result result = + ParseBuildString("https://example.com/img.zip{boot.img}"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), VariantWith(HttpBuildString{ + .url = "https://example.com/img.zip", + .filepath = "boot.img"})); +} + +TEST(ParseBuildStringTests, HttpDirectoryFilepathSuccess) { + Result result = + ParseBuildString("https://example.com/dist/{mykernel}"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), + VariantWith(HttpBuildString{ + .url = "https://example.com/dist/", .filepath = "mykernel"})); +} + +TEST(ParseBuildStringTests, UrlIsNotADirectoryBuildStringSuccess) { + Result result = ParseBuildString("gs://bucket/a/b/c.zip"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), VariantWith(GcsBuildString{ + .url = "gs://bucket/a/b/c.zip"})); + + result = ParseBuildString("https://example.com:8443/c.zip"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), VariantWith(HttpBuildString{ + .url = "https://example.com:8443/c.zip"})); +} + +TEST(ParseBuildStringTests, CleartextHttpFail) { + EXPECT_THAT(ParseBuildString("http://example.com/file.zip"), + IsErrorAndMessage(HasSubstr("https://"))); +} + +TEST(ParseBuildStringTests, UnknownSchemeFail) { + EXPECT_THAT( + ParseBuildString("s3://bucket/file.zip"), + IsErrorAndMessage(AllOf(HasSubstr("gs://"), HasSubstr("https://")))); + EXPECT_THAT( + ParseBuildString("ftp://example.com/file.zip"), + IsErrorAndMessage(AllOf(HasSubstr("gs://"), HasSubstr("https://")))); +} + +TEST(ParseBuildStringTests, Sha256FragmentSuccess) { + Result result = + ParseBuildString(std::string("gs://bucket/file.zip#sha256=") + kSha256); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), + VariantWith(GcsBuildString{ + .url = "gs://bucket/file.zip", .sha256 = kSha256})); +} + +TEST(ParseBuildStringTests, Sha256FragmentAndFilepathSuccess) { + Result result = ParseBuildString( + std::string("https://example.com/img.zip{boot.img}#sha256=") + kSha256); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), VariantWith(HttpBuildString{ + .url = "https://example.com/img.zip", + .filepath = "boot.img", + .sha256 = kSha256})); +} + +TEST(ParseBuildStringTests, FragmentBeforeFilepathFail) { + EXPECT_THAT(ParseBuildString(std::string("gs://bucket/img.zip#sha256=") + + kSha256 + "{boot.img}"), + IsError()); +} + +TEST(ParseBuildStringTests, MalformedFragmentFail) { + EXPECT_THAT(ParseBuildString("gs://bucket/file.zip#sha256=abcdef"), + IsError()); + EXPECT_THAT(ParseBuildString("gs://bucket/file.zip#md5=abcdef"), IsError()); + EXPECT_THAT( + ParseBuildString(std::string("gs://bucket/fi#le.zip#sha256=") + kSha256), + IsError()); +} + +TEST(ParseBuildStringTests, Sha256FragmentOnDirectoryFail) { + EXPECT_THAT( + ParseBuildString(std::string("gs://bucket/dist/#sha256=") + kSha256), + IsError()); +} + +TEST(ParseBuildStringTests, CharactersAfterFilepathFail) { + EXPECT_THAT(ParseBuildString("gs://bucket/{boot.img}trailing"), IsError()); + EXPECT_THAT(ParseBuildString("https://example.com/{boot.img}/more"), + IsError()); +} + +TEST(ParseBuildStringTests, UrlWithCommaFail) { + EXPECT_THAT(ParseBuildString("gs://bucket/file,name.zip"), + IsErrorAndMessage(HasSubstr("cannot contain a comma"))); +} + +TEST(ParseBuildStringTests, QueryStringOnObjectSuccess) { + Result result = + ParseBuildString("https://example.com/file.zip?sig=abc"); + EXPECT_THAT(result, IsOk()); + EXPECT_THAT(result.value(), + VariantWith(HttpBuildString{ + .url = "https://example.com/file.zip?sig=abc"})); +} + +TEST(ParseBuildStringTests, QueryStringOnDirectoryFail) { + EXPECT_THAT(ParseBuildString("https://example.com/dist/?sig=abc"), IsError()); +} + +TEST(SingleBuildStringGflagsCompatFlagTests, GcsBuildStringSuccess) { + std::optional value; + Flag flag = GflagsCompatFlag("myflag", value); + + ASSERT_THAT(ConsumeFlags({flag}, {"--myflag=gs://bucket/image.zip"}), IsOk()); + ASSERT_THAT(value, Optional(VariantWith( + GcsBuildString{.url = "gs://bucket/image.zip"}))); +} + +TEST(SingleBuildStringGflagsCompatFlagTests, HttpBuildStringSuccess) { + std::optional value; + Flag flag = GflagsCompatFlag("myflag", value); + + ASSERT_THAT(ConsumeFlags({flag}, {"--myflag=https://example.com/dist/"}), + IsOk()); + ASSERT_THAT(value, Optional(VariantWith( + HttpBuildString{.url = "https://example.com/dist/"}))); +} + +TEST(BuildStringGflagsCompatFlagTests, UrlMultiValueSuccess) { + std::vector> value; + Flag flag = GflagsCompatFlag("myflag", value); + + ASSERT_THAT(ConsumeFlags({flag}, {"--myflag=gs://bucket/a.zip,https://" + "example.com/dist/"}), + IsOk()); + ASSERT_THAT(value, SizeIs(2)); + ASSERT_THAT(value, + ElementsAre(Optional(VariantWith( + GcsBuildString{.url = "gs://bucket/a.zip"})), + Optional(VariantWith(HttpBuildString{ + .url = "https://example.com/dist/"})))); +} + } // namespace cuttlefish From 8e90d727fcba8c1708e7be71726ad861924afb29 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:35 +0000 Subject: [PATCH 03/20] Resolve URL build artifact names without any I/O --- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 23 ++ .../host/libs/web/android_build_string.cpp | 20 +- .../host/libs/web/url_namespace.cpp | 144 ++++++++++++ .../cuttlefish/host/libs/web/url_namespace.h | 72 ++++++ .../host/libs/web/url_namespace_test.cpp | 212 ++++++++++++++++++ 5 files changed, 452 insertions(+), 19 deletions(-) create mode 100644 base/cvd/cuttlefish/host/libs/web/url_namespace.cpp create mode 100644 base/cvd/cuttlefish/host/libs/web/url_namespace.h create mode 100644 base/cvd/cuttlefish/host/libs/web/url_namespace_test.cpp diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index 0d496b2c474..46cfb04e627 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -64,6 +64,7 @@ cf_cc_library( hdrs = ["android_build_string.h"], deps = [ "//cuttlefish/flag_parser", + "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/web/http_client:scrub_secrets", "//cuttlefish/result", "@abseil-cpp//absl/strings", @@ -257,3 +258,25 @@ cf_cc_test( "//cuttlefish/result:result_matchers", ], ) + +cf_cc_library( + name = "url_namespace", + srcs = ["url_namespace.cpp"], + hdrs = ["url_namespace.h"], + deps = [ + "//cuttlefish/host/libs/web/http_client:scrub_secrets", + "//cuttlefish/result", + "@abseil-cpp//absl/strings", + "@fmt", + ], +) + +cf_cc_test( + name = "url_namespace_test", + srcs = ["url_namespace_test.cpp"], + deps = [ + "//cuttlefish/host/libs/web:url_namespace", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + ], +) diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp index d12864eac50..686d7b38171 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp @@ -26,7 +26,6 @@ #include #include -#include "absl/strings/ascii.h" #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" #include "absl/strings/strip.h" @@ -35,30 +34,13 @@ #include "cuttlefish/flag_parser/flag.h" #include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" +#include "cuttlefish/host/libs/web/url_namespace.h" #include "cuttlefish/result/result.h" namespace cuttlefish { namespace { -// Returns the "" of a "://" build string, which is what -// separates a URL build source from a branch, a build id or a directory path. -std::optional UrlScheme(std::string_view build_string) { - constexpr std::string_view kSchemeCharacters = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-."; - // The allowed set also holds digits and "+-.", which cannot open a scheme, - // and an empty scheme has no opening character at all. - if (build_string.empty() || !absl::ascii_isalpha(build_string.front())) { - return std::nullopt; - } - const size_t separator = build_string.find_first_not_of(kSchemeCharacters); - if (separator == std::string_view::npos || - !build_string.substr(separator).starts_with("://")) { - return std::nullopt; - } - return build_string.substr(0, separator); -} - // Returns the digest of a "#sha256=<64 hex digits>" fragment, or nullopt when // `fragment` is anything else. std::optional Sha256FragmentDigest( diff --git a/base/cvd/cuttlefish/host/libs/web/url_namespace.cpp b/base/cvd/cuttlefish/host/libs/web/url_namespace.cpp new file mode 100644 index 00000000000..edcbec9f6d4 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/url_namespace.cpp @@ -0,0 +1,144 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/url_namespace.h" + +#include + +#include +#include +#include +#include + +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/str_join.h" +#include "absl/strings/strip.h" +#include "fmt/format.h" + +#include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { +namespace { + +// Returns whether `name` names the `kind` archive of a build. Android Build +// names archives "--.zip"; a republished artifact set drops +// the build id, leaving "-.zip". +bool NameMatchesZipKind(std::string_view name, BuildZipKind kind) { + return absl::StrContains(name, fmt::format("-{}-", kind)) || + absl::EndsWith(name, fmt::format("-{}.zip", kind)); +} + +} // namespace + +std::optional UrlScheme(std::string_view url) { + constexpr std::string_view kSchemeCharacters = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-."; + // The allowed set also holds digits and "+-.", which cannot open a scheme, + // and an empty scheme has no opening character at all. + if (url.empty() || !absl::ascii_isalpha(url.front())) { + return std::nullopt; + } + const size_t separator = url.find_first_not_of(kSchemeCharacters); + if (separator == std::string_view::npos || + !url.substr(separator).starts_with("://")) { + return std::nullopt; + } + return url.substr(0, separator); +} + +Result ParseUrl(std::string_view url) { + const std::optional scheme = UrlScheme(url); + CF_EXPECTF(scheme.has_value(), "'{}' is not a URL.", ScrubUrl(url)); + const std::string_view authority_and_path = url.substr(scheme->size() + 3); + const size_t authority_end = authority_and_path.find('/'); + CF_EXPECTF(authority_end != std::string_view::npos, + "The URL '{}' has no '/' after its host or bucket.", + ScrubUrl(url)); + CF_EXPECTF(authority_end != 0, "The URL '{}' has no host or bucket.", + ScrubUrl(url)); + + std::string_view path = authority_and_path.substr(authority_end + 1); + std::string query; + const size_t query_start = path.find('?'); + if (query_start != std::string_view::npos) { + query = std::string(path.substr(query_start + 1)); + path = path.substr(0, query_start); + } + return ParsedUrl{ + .authority = std::string(authority_and_path.substr(0, authority_end)), + .path = std::string(path), + .query = query, + }; +} + +std::optional DeriveProduct(std::string_view basename) { + if (!absl::ConsumeSuffix(&basename, ".zip")) { + return std::nullopt; + } + const size_t infix = basename.find("-img-"); + if (infix != 0 && infix != std::string_view::npos) { + return std::string(basename.substr(0, infix)); + } + if (absl::ConsumeSuffix(&basename, "-img") && !basename.empty()) { + return std::string(basename); + } + return std::nullopt; +} + +std::string_view format_as(BuildZipKind kind) { + switch (kind) { + case BuildZipKind::kImages: + return "img"; + case BuildZipKind::kTargetFiles: + return "target_files"; + case BuildZipKind::kOtaTools: + return "otatools"; + } +} + +Result ResolveUrlZipName(std::string_view object, + BuildZipKind kind) { + // A build that is one archive has no other candidate, but its name must not + // claim to be a different kind of archive. target_files and otatools are the + // other archive kinds a fetch resolves, so a name claiming one of those + // fails here rather than downstream. + CF_EXPECTF( + NameMatchesZipKind(object, kind) || + (kind == BuildZipKind::kImages && absl::EndsWith(object, ".zip") && + !NameMatchesZipKind(object, BuildZipKind::kTargetFiles) && + !NameMatchesZipKind(object, BuildZipKind::kOtaTools)), + "The build holds only '{}', which is not a '{}' zip.", object, kind); + return std::string(object); +} + +Result ResolveUrlZipName(const std::vector& names, + BuildZipKind kind) { + std::vector matches; + for (const std::string& name : names) { + if (absl::EndsWith(name, ".zip") && NameMatchesZipKind(name, kind)) { + matches.push_back(name); + } + } + CF_EXPECTF(matches.size() == 1, + "Expected one '{}' zip in the build, found {}. The build " + "contains [{}]. Name the archive itself in the URL if it is " + "named some other way.", + kind, matches.size(), absl::StrJoin(names, ", ")); + return matches.front(); +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/url_namespace.h b/base/cvd/cuttlefish/host/libs/web/url_namespace.h new file mode 100644 index 00000000000..b5067c1ecd7 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/url_namespace.h @@ -0,0 +1,72 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "cuttlefish/result/result.h" + +namespace cuttlefish { + +// Returns the "" of a "://..." string, or nullopt when it +// does not begin with one. A scheme starts with a letter and continues with +// letters, digits, '+', '-' and '.'. +std::optional UrlScheme(std::string_view url); + +struct ParsedUrl { + std::string authority; // bucket or host + std::string path; // no leading '/', ends with '/' in directory form + std::string query; // without the '?' + + // Returns whether the URL names a prefix rather than a single object. A URL + // naming just the bucket or the host has an empty path and names a prefix. + bool IsDirectoryForm() const { return path.empty() || path.ends_with('/'); } +}; + +Result ParseUrl(std::string_view url); + +// Returns the product named by a "-img-.zip" or +// "-img.zip" artifact, or nullopt when the name does not follow the +// Android Build convention. +std::optional DeriveProduct(std::string_view basename); + +// The kinds of zip archive a fetch resolves by name. +enum class BuildZipKind { + kImages, + kTargetFiles, + kOtaTools, +}; + +// Returns how `kind` is spelled inside an artifact name, as in the "img" of +// "-img-.zip". +std::string_view format_as(BuildZipKind kind); + +// Returns the name of the `kind` zip of a URL build that holds `object` and +// nothing else. +Result ResolveUrlZipName(std::string_view object, + BuildZipKind kind); + +// Returns the name of the `kind` zip of a URL build whose complete object +// listing is `names`. A name states its kind either as Android Build's +// "--" infix or as a "-.zip" suffix. `{selector}` never names a +// zip, so it is not an input. +Result ResolveUrlZipName(const std::vector& names, + BuildZipKind kind); + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/url_namespace_test.cpp b/base/cvd/cuttlefish/host/libs/web/url_namespace_test.cpp new file mode 100644 index 00000000000..473be69f8ca --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/url_namespace_test.cpp @@ -0,0 +1,212 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/url_namespace.h" + +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::AllOf; +using ::testing::HasSubstr; +using ::testing::Not; + +TEST(UrlSchemeTests, SchemesSuccess) { + EXPECT_EQ(UrlScheme("gs://bucket/object.zip"), "gs"); + EXPECT_EQ(UrlScheme("https://example.com/f.zip"), "https"); + EXPECT_EQ(UrlScheme("s3+v4.1://bucket/f.zip"), "s3+v4.1"); +} + +TEST(UrlSchemeTests, NonSchemesFail) { + EXPECT_EQ(UrlScheme(""), std::nullopt); + EXPECT_EQ(UrlScheme("://example.com/f.zip"), std::nullopt); + EXPECT_EQ(UrlScheme("1s://example.com/f.zip"), std::nullopt); + EXPECT_EQ(UrlScheme("branch/target"), std::nullopt); + EXPECT_EQ(UrlScheme("gs:/bucket/object.zip"), std::nullopt); +} + +TEST(ParseUrlTests, DecompositionSuccess) { + struct TestCase { + std::string_view url; + std::string_view authority; + std::string_view path; + bool directory_form; + std::string_view query; + }; + const TestCase cases[] = { + {"gs://bucket/dist/", "bucket", "dist/", true, ""}, + {"gs://bucket/", "bucket", "", true, ""}, + {"gs://bucket/a/b/c.zip", "bucket", "a/b/c.zip", false, ""}, + {"https://example.com/dist/", "example.com", "dist/", true, ""}, + {"https://example.com:8443/f.zip", "example.com:8443", "f.zip", false, + ""}, + {"https://example.com/f.zip?sig=abc&x=1", "example.com", "f.zip", false, + "sig=abc&x=1"}, + }; + for (const TestCase& test_case : cases) { + Result result = ParseUrl(test_case.url); + ASSERT_THAT(result, IsOk()) << test_case.url; + EXPECT_EQ(result->authority, test_case.authority) << test_case.url; + EXPECT_EQ(result->path, test_case.path) << test_case.url; + EXPECT_EQ(result->IsDirectoryForm(), test_case.directory_form) + << test_case.url; + EXPECT_EQ(result->query, test_case.query) << test_case.url; + } +} + +TEST(ParseUrlTests, MalformedFail) { + EXPECT_THAT(ParseUrl("bucket/object.zip"), IsError()); + EXPECT_THAT(ParseUrl("gs://bucket"), IsError()); + EXPECT_THAT(ParseUrl("gs:///object.zip"), IsError()); +} + +TEST(ParseUrlTests, ErrorWithoutQueryFail) { + EXPECT_THAT(ParseUrl("https://example.com?sig=secret"), + IsErrorAndMessage(Not(HasSubstr("sig=secret")))); +} + +TEST(DeriveProductTests, ImageZipNameSuccess) { + EXPECT_EQ(DeriveProduct("aosp_cf_x86_64_phone-img-12345.zip"), + "aosp_cf_x86_64_phone"); +} + +TEST(DeriveProductTests, ImageZipNameWithoutABuildIdSuccess) { + EXPECT_EQ(DeriveProduct("aosp_cf_x86_64_auto-img.zip"), + "aosp_cf_x86_64_auto"); +} + +TEST(DeriveProductTests, OtherNamesFail) { + EXPECT_EQ(DeriveProduct("images.zip"), std::nullopt); + EXPECT_EQ(DeriveProduct("bzImage"), std::nullopt); + EXPECT_EQ(DeriveProduct("-img-12345.zip"), std::nullopt); + EXPECT_EQ(DeriveProduct("-img.zip"), std::nullopt); + EXPECT_EQ(DeriveProduct("phone-img-12345.tar.gz"), std::nullopt); +} + +TEST(ResolveUrlZipNameTests, ObjectNamingTheKindSuccess) { + EXPECT_THAT(ResolveUrlZipName("phone-img-1.zip", BuildZipKind::kImages), + IsOkAndValue("phone-img-1.zip")); + EXPECT_THAT( + ResolveUrlZipName("phone-target_files-1.zip", BuildZipKind::kTargetFiles), + IsOkAndValue("phone-target_files-1.zip")); + EXPECT_THAT(ResolveUrlZipName("phone-img.zip", BuildZipKind::kImages), + IsOkAndValue("phone-img.zip")); +} + +TEST(ResolveUrlZipNameTests, ObjectIsTheImageZipSuccess) { + EXPECT_THAT(ResolveUrlZipName("images.zip", BuildZipKind::kImages), + IsOkAndValue("images.zip")); +} + +TEST(ResolveUrlZipNameTests, ObjectForOtherKindFail) { + EXPECT_THAT(ResolveUrlZipName("images.zip", BuildZipKind::kTargetFiles), + IsErrorAndMessage( + AllOf(HasSubstr("images.zip"), HasSubstr("target_files")))); +} + +TEST(ResolveUrlZipNameTests, ObjectIsNotAZipFail) { + EXPECT_THAT(ResolveUrlZipName("bzImage", BuildZipKind::kImages), + IsErrorAndMessage(HasSubstr("bzImage"))); +} + +TEST(ResolveUrlZipNameTests, ObjectNamingAnotherKindFail) { + EXPECT_THAT( + ResolveUrlZipName("phone-target_files-1.zip", BuildZipKind::kImages), + IsError()); + EXPECT_THAT(ResolveUrlZipName("phone-otatools-1.zip", BuildZipKind::kImages), + IsError()); + EXPECT_THAT( + ResolveUrlZipName("phone-target_files.zip", BuildZipKind::kImages), + IsError()); +} + +TEST(ResolveUrlZipNameTests, ListingWithOneMatchSuccess) { + EXPECT_THAT(ResolveUrlZipName({"misc_info.txt", "phone-img-1.zip", + "phone-target_files-1.zip"}, + BuildZipKind::kImages), + IsOkAndValue("phone-img-1.zip")); +} + +TEST(ResolveUrlZipNameTests, ListingWithAZipWithoutABuildIdSuccess) { + EXPECT_THAT(ResolveUrlZipName({"misc_info.txt", "aosp_cf_x86_64_auto-img.zip", + "cvd-host_package.tar.gz"}, + BuildZipKind::kImages), + IsOkAndValue("aosp_cf_x86_64_auto-img.zip")); +} + +TEST(ResolveUrlZipNameTests, ListingMatchesTheKindAskedForSuccess) { + const std::vector names = {"phone-img.zip", + "phone-target_files.zip"}; + + EXPECT_THAT(ResolveUrlZipName(names, BuildZipKind::kImages), + IsOkAndValue("phone-img.zip")); + EXPECT_THAT(ResolveUrlZipName(names, BuildZipKind::kTargetFiles), + IsOkAndValue("phone-target_files.zip")); +} + +TEST(ResolveUrlZipNameTests, ListingCountsEachNameOnceSuccess) { + const std::vector names = {"phone-img-1-img.zip"}; + + EXPECT_THAT(ResolveUrlZipName(names, BuildZipKind::kImages), + IsOkAndValue("phone-img-1-img.zip")); +} + +TEST(ResolveUrlZipNameTests, ListingWithBothZipNamingsFail) { + const std::vector names = {"phone-img-1.zip", "phone-img.zip"}; + + EXPECT_THAT( + ResolveUrlZipName(names, BuildZipKind::kImages), + IsErrorAndMessage(AllOf(HasSubstr("found 2"), HasSubstr("phone-img.zip"), + HasSubstr("phone-img-1.zip")))); +} + +TEST(ResolveUrlZipNameTests, ListingWithoutAMatchFail) { + const std::vector names = {"misc_info.txt", + "cvd-host_package.tar.gz"}; + + EXPECT_THAT( + ResolveUrlZipName(names, BuildZipKind::kImages), + IsErrorAndMessage(AllOf(HasSubstr("img"), HasSubstr("misc_info.txt")))); +} + +TEST(ResolveUrlZipNameTests, ListingWithSeveralMatchesFail) { + const std::vector names = {"phone-img-1.zip", + "tablet-img-2.zip"}; + + EXPECT_THAT(ResolveUrlZipName(names, BuildZipKind::kImages), + IsErrorAndMessage(AllOf(HasSubstr("phone-img-1.zip"), + HasSubstr("tablet-img-2.zip")))); +} + +// The resolver takes no selector: a `{selector}` names the host package or an +// artifact to download, never the image zip. +TEST(ResolveUrlZipNameTests, SelectorCannotNameTheZipFail) { + const std::vector names = {"images.zip", + "cvd-host_package.tar.gz"}; + + EXPECT_THAT(ResolveUrlZipName(names, BuildZipKind::kImages), IsError()); +} + +} // namespace +} // namespace cuttlefish From ae0d43739315cf5a824e4c77b620cd50c66ab1b4 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:36 +0000 Subject: [PATCH 04/20] Add GcsBuild and HttpBuild to the resolved build variant --- .../host/commands/cvd/fetch/BUILD.bazel | 2 - .../host/commands/cvd/fetch/fetch_context.cc | 1 - .../host/commands/cvd/fetch/host_package.cc | 1 - base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 16 +- .../cuttlefish/host/libs/web/android_build.cc | 90 ++++++++++ .../cuttlefish/host/libs/web/android_build.h | 61 ++++++- .../host/libs/web/android_build_api.cpp | 56 +++---- .../host/libs/web/android_build_api.h | 8 - .../host/libs/web/android_build_test.cpp | 154 ++++++++++++++++++ .../host/libs/web/caching_build_api.cpp | 1 - .../metrics/conversion/fetch_conversion.cc | 16 +- 11 files changed, 354 insertions(+), 52 deletions(-) create mode 100644 base/cvd/cuttlefish/host/libs/web/android_build_test.cpp diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel index 16cd15e9a2b..6f84bbcf90c 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel @@ -193,7 +193,6 @@ cf_cc_library( "//cuttlefish/host/libs/config:fetcher_config", "//cuttlefish/host/libs/config:file_source", "//cuttlefish/host/libs/web:android_build", - "//cuttlefish/host/libs/web:android_build_api", "//cuttlefish/host/libs/web:build_api", "//cuttlefish/host/libs/web:build_api_zip", "//cuttlefish/host/libs/zip:zip_file", @@ -296,7 +295,6 @@ cf_cc_library( "//cuttlefish/host/commands/cvd/fetch:fetch_tracer", "//cuttlefish/host/commands/cvd/fetch:substitute", "//cuttlefish/host/libs/web:android_build", - "//cuttlefish/host/libs/web:android_build_api", "//cuttlefish/host/libs/web:build_api", "//cuttlefish/result", "//libbase", diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.cc index fd6e5d09188..dac9bf5a996 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.cc @@ -39,7 +39,6 @@ #include "cuttlefish/host/libs/config/fetcher_config.h" #include "cuttlefish/host/libs/config/file_source.h" #include "cuttlefish/host/libs/web/android_build.h" -#include "cuttlefish/host/libs/web/android_build_api.h" #include "cuttlefish/host/libs/web/build_api.h" #include "cuttlefish/host/libs/web/build_api_zip.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.cc index aabaa161634..50ac1808a50 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.cc @@ -29,7 +29,6 @@ #include "cuttlefish/host/commands/cvd/fetch/fetch_tracer.h" #include "cuttlefish/host/commands/cvd/fetch/substitute.h" #include "cuttlefish/host/libs/web/android_build.h" -#include "cuttlefish/host/libs/web/android_build_api.h" #include "cuttlefish/host/libs/web/build_api.h" #include "cuttlefish/result/result.h" diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index 46cfb04e627..41803104935 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -12,12 +12,27 @@ cf_cc_library( hdrs = ["android_build.h"], deps = [ "//cuttlefish/common/libs/utils:environment", + "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:android_build_url", + "//cuttlefish/host/libs/web:url_namespace", + "//cuttlefish/host/libs/web/http_client:scrub_secrets", + "//cuttlefish/result", "@abseil-cpp//absl/strings", "@fmt", ], ) +cf_cc_test( + name = "android_build_test", + srcs = ["android_build_test.cpp"], + deps = [ + "//cuttlefish/host/libs/web:android_build", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + ], +) + cf_cc_library( name = "android_build_api", srcs = ["android_build_api.cpp"], @@ -156,7 +171,6 @@ cf_cc_library( "//cuttlefish/files:file_exists", "//cuttlefish/files:link_or_copy", "//cuttlefish/host/libs/web:android_build", - "//cuttlefish/host/libs/web:android_build_api", "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:build_api", "//cuttlefish/host/libs/zip:cached_zip_source", diff --git a/base/cvd/cuttlefish/host/libs/web/android_build.cc b/base/cvd/cuttlefish/host/libs/web/android_build.cc index 9de59b8a47b..c76b6fa0286 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build.cc +++ b/base/cvd/cuttlefish/host/libs/web/android_build.cc @@ -15,9 +15,13 @@ #include "cuttlefish/host/libs/web/android_build.h" +#include + #include #include #include +#include +#include #include #include #include @@ -26,8 +30,26 @@ #include "fmt/format.h" #include "cuttlefish/common/libs/utils/environment.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" +#include "cuttlefish/host/libs/web/url_namespace.h" +#include "cuttlefish/result/result.h" namespace cuttlefish { +namespace { + +// URL builds have no Android Build target, product or branch, so they carry +// this name wherever one is expected. +constexpr char kUrlName[] = "url"; + +// Returns where the name after the last '/' of `path` begins. A parsed path +// has no leading '/', so a name at the root is the whole path. +size_t BasenameStart(std::string_view path) { + const size_t slash = path.rfind('/'); + return slash == std::string_view::npos ? 0 : slash + 1; +} + +} // namespace std::ostream& operator<<(std::ostream& out, const DeviceBuild& build) { return out << "(id=\"" << build.id << "\", branch=\"" << build.branch @@ -52,6 +74,53 @@ std::ostream& operator<<(std::ostream& out, const DirectoryBuild& build) { << "\", filepath=\"" << build.filepath.value_or("") << "\")"; } +Result GcsBuild::FromBuildString(const GcsBuildString& build_string) { + const ParsedUrl url = CF_EXPECT(ParseUrl(build_string.url)); + const size_t basename = BasenameStart(url.path); + std::optional object; + if (!url.IsDirectoryForm()) { + object = url.path.substr(basename); + } + return GcsBuild{ + .bucket = url.authority, + .prefix = url.path.substr(0, basename), + .object = object, + .sha256 = build_string.sha256, + .id = ScrubUrl(build_string.url), + .target = kUrlName, + .product = DeriveProduct(object.value_or("")).value_or(kUrlName), + .filepath = build_string.filepath, + }; +} + +std::ostream& operator<<(std::ostream& out, const GcsBuild& build) { + return out << "(url=\"" << build.id << "\", filepath=\"" + << build.filepath.value_or("") << "\")"; +} + +Result HttpBuild::FromBuildString( + const HttpBuildString& build_string) { + const ParsedUrl url = CF_EXPECT(ParseUrl(build_string.url)); + std::optional object; + if (!url.IsDirectoryForm()) { + object = url.path.substr(BasenameStart(url.path)); + } + return HttpBuild{ + .url = build_string.url, + .object = object, + .sha256 = build_string.sha256, + .id = ScrubUrl(build_string.url), + .target = kUrlName, + .product = DeriveProduct(object.value_or("")).value_or(kUrlName), + .filepath = build_string.filepath, + }; +} + +std::ostream& operator<<(std::ostream& out, const HttpBuild& build) { + return out << "(url=\"" << build.id << "\", filepath=\"" + << build.filepath.value_or("") << "\")"; +} + std::ostream& operator<<(std::ostream& out, const Build& build) { std::visit([&out](auto&& arg) { out << arg; }, build); return out; @@ -59,9 +128,30 @@ std::ostream& operator<<(std::ostream& out, const Build& build) { std::string FetchLabel(const Build& build) { // return std::visit((auto&& arg) { return FetchLabel(arg); }, build) + if (const GcsBuild* gcs = std::get_if(&build)) { + return gcs->id; + } + if (const HttpBuild* http = std::get_if(&build)) { + return http->id; + } return fmt::format("{}/{}", std::visit([](auto&& arg) { return arg.id; }, build), std::visit([](auto&& arg) { return arg.target; }, build)); } +std::tuple GetBuildIdAndTarget(const Build& build) { + std::string id = std::visit([](auto&& arg) { return arg.id; }, build); + std::string target = std::visit([](auto&& arg) { return arg.target; }, build); + return {id, target}; +} + +std::optional GetFilepath(const Build& build) { + return std::visit([](auto&& arg) { return arg.filepath; }, build); +} + +std::string ConstructTargetFilepath(const std::string& directory, + const std::string& filename) { + return directory + "/" + filename; +} + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/android_build.h b/base/cvd/cuttlefish/host/libs/web/android_build.h index 6aaec7de5cd..a7ce9871ca8 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build.h +++ b/base/cvd/cuttlefish/host/libs/web/android_build.h @@ -15,13 +15,17 @@ #pragma once +#include #include #include #include +#include #include #include +#include "cuttlefish/host/libs/web/android_build_string.h" #include "cuttlefish/host/libs/web/android_build_url.h" +#include "cuttlefish/result/result.h" namespace cuttlefish { @@ -53,10 +57,65 @@ struct DirectoryBuild { std::ostream& operator<<(std::ostream&, const DirectoryBuild&); -using Build = std::variant; +struct GcsObjectInfo { + std::optional generation; + std::optional md5; +}; + +// The objects under a `gs://` prefix, or the single object a `gs://` URL +// names. +struct GcsBuild { + static Result FromBuildString(const GcsBuildString& build_string); + + std::string bucket; + std::string prefix; // ends with '/', empty at the bucket root + std::optional object; // set in the object form only + // The listing of the directory form, by artifact name. + std::map contents; + std::optional generation; + std::optional md5; + std::optional sha256; + + // Derived from the URL for the code that handles every build alike. Never + // used to address the objects themselves. + std::string id; + std::string target; + std::string product; + std::optional filepath; +}; + +std::ostream& operator<<(std::ostream&, const GcsBuild&); + +// The same two forms over `https://`, where a pre-signed URL carries its +// credential in the query string of `url`. +struct HttpBuild { + static Result FromBuildString(const HttpBuildString& build_string); + + std::string url; // object, or directory ending in '/' + std::optional object; // set in the object form only + std::optional etag; + std::optional sha256; + + // `id` has no query string, so requests must go to `url`. + std::string id; + std::string target; + std::string product; + std::optional filepath; +}; + +std::ostream& operator<<(std::ostream&, const HttpBuild&); + +using Build = std::variant; std::ostream& operator<<(std::ostream&, const Build&); std::string FetchLabel(const Build& build); +std::tuple GetBuildIdAndTarget(const Build& build); + +std::optional GetFilepath(const Build& build); + +std::string ConstructTargetFilepath(const std::string& directory, + const std::string& filename); + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp index 16afa01b6ca..8e55b95cf8c 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -170,11 +169,13 @@ Result AndroidBuildApi::DownloadFile( Result AndroidBuildApi::FileReader( const Build& build, const std::string& artifact_name) { - Result res = - std::visit([this, &artifact_name]( - auto&& arg) { return FileReader(arg, artifact_name); }, - build); - return CF_EXPECT(std::move(res)); + if (const auto* device = std::get_if(&build)) { + return CF_EXPECT(FileReader(*device, artifact_name)); + } + if (const auto* directory = std::get_if(&build)) { + return CF_EXPECT(FileReader(*directory, artifact_name)); + } + return CF_ERRF("AndroidBuildApi cannot handle '{}'", FetchLabel(build)); } Result AndroidBuildApi::FileReader( @@ -382,11 +383,13 @@ Result> AndroidBuildApi::Artifacts( Result> AndroidBuildApi::Artifacts( const Build& build, const std::vector& artifact_filenames) { - auto res = - std::visit([this, &artifact_filenames]( - auto&& arg) { return Artifacts(arg, artifact_filenames); }, - build); - return CF_EXPECT(std::move(res)); + if (const DeviceBuild* device = std::get_if(&build)) { + return CF_EXPECT(Artifacts(*device, artifact_filenames)); + } + if (const DirectoryBuild* directory = std::get_if(&build)) { + return CF_EXPECT(Artifacts(*directory, artifact_filenames)); + } + return CF_ERRF("AndroidBuildApi cannot handle '{}'", FetchLabel(build)); } Result AndroidBuildApi::GetArtifactDownloadUrl( @@ -432,13 +435,15 @@ Result AndroidBuildApi::ArtifactToFile(const DirectoryBuild& build, Result AndroidBuildApi::ArtifactToFile(const Build& build, const std::string& artifact, const std::string& path) { - auto res = std::visit( - [this, &artifact, &path](auto&& arg) { - return ArtifactToFile(arg, artifact, path); - }, - build); - CF_EXPECT(std::move(res)); - return {}; + if (const DeviceBuild* device = std::get_if(&build)) { + CF_EXPECT(ArtifactToFile(*device, artifact, path)); + return {}; + } + if (const DirectoryBuild* directory = std::get_if(&build)) { + CF_EXPECT(ArtifactToFile(*directory, artifact, path)); + return {}; + } + return CF_ERRF("AndroidBuildApi cannot handle '{}'", FetchLabel(build)); } Result AndroidBuildApi::DownloadTargetFileFromCas( @@ -487,19 +492,4 @@ Result AndroidBuildApi::DownloadTargetFile( return {target_filepath}; } -std::tuple GetBuildIdAndTarget(const Build& build) { - auto id = std::visit([](auto&& arg) { return arg.id; }, build); - auto target = std::visit([](auto&& arg) { return arg.target; }, build); - return {id, target}; -} - -std::optional GetFilepath(const Build& build) { - return std::visit([](auto&& arg) { return arg.filepath; }, build); -} - -std::string ConstructTargetFilepath(const std::string& directory, - const std::string& filename) { - return directory + "/" + filename; -} - } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_api.h b/base/cvd/cuttlefish/host/libs/web/android_build_api.h index 78df5f22965..031c7b681cc 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_api.h +++ b/base/cvd/cuttlefish/host/libs/web/android_build_api.h @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -126,11 +125,4 @@ class AndroidBuildApi : public BuildApi { CasDownloader* cas_downloader_; }; -std::tuple GetBuildIdAndTarget(const Build& build); - -std::optional GetFilepath(const Build& build); - -std::string ConstructTargetFilepath(const std::string& directory, - const std::string& filename); - } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_test.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_test.cpp new file mode 100644 index 00000000000..c0387874113 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/android_build_test.cpp @@ -0,0 +1,154 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/android_build.h" + +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::AllOf; +using ::testing::HasSubstr; +using ::testing::Not; + +constexpr char kSignedUrl[] = + "https://example.com/dist/phone-img-1.zip?X-Goog-Signature=secret"; + +std::string Print(const Build& build) { + std::stringstream out; + out << build; + return out.str(); +} + +TEST(GcsBuildTests, DirectoryFormSuccess) { + Result build = + GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket/dist/"}); + + ASSERT_THAT(build, IsOk()); + EXPECT_EQ(build->bucket, "bucket"); + EXPECT_EQ(build->prefix, "dist/"); + EXPECT_EQ(build->object, std::nullopt); + EXPECT_EQ(build->id, "gs://bucket/dist/"); + EXPECT_EQ(build->target, "url"); + EXPECT_EQ(build->product, "url"); +} + +TEST(GcsBuildTests, ObjectFormSuccess) { + Result build = GcsBuild::FromBuildString(GcsBuildString{ + .url = "gs://bucket/dist/phone-img-1.zip", + .filepath = "boot.img", + .sha256 = std::string(64, 'a'), + }); + + ASSERT_THAT(build, IsOk()); + EXPECT_EQ(build->bucket, "bucket"); + EXPECT_EQ(build->prefix, "dist/"); + EXPECT_EQ(build->object, "phone-img-1.zip"); + EXPECT_EQ(build->product, "phone"); + EXPECT_EQ(build->filepath, "boot.img"); + EXPECT_EQ(build->sha256, std::string(64, 'a')); +} + +TEST(GcsBuildTests, BucketRootSuccess) { + Result build = + GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket/"}); + + ASSERT_THAT(build, IsOk()); + EXPECT_TRUE(build->prefix.empty()); + EXPECT_EQ(build->object, std::nullopt); +} + +TEST(GcsBuildTests, ObjectAtBucketRootSuccess) { + Result build = GcsBuild::FromBuildString( + GcsBuildString{.url = "gs://bucket/phone-img-1.zip"}); + + ASSERT_THAT(build, IsOk()); + EXPECT_TRUE(build->prefix.empty()); + EXPECT_EQ(build->object, "phone-img-1.zip"); +} + +TEST(GcsBuildTests, MalformedUrlFail) { + EXPECT_THAT(GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket"}), + IsError()); +} + +TEST(HttpBuildTests, ObjectFormKeepsTheQueryOutOfTheIdSuccess) { + Result build = + HttpBuild::FromBuildString(HttpBuildString{.url = kSignedUrl}); + + ASSERT_THAT(build, IsOk()); + EXPECT_EQ(build->url, kSignedUrl); + EXPECT_EQ(build->object, "phone-img-1.zip"); + EXPECT_EQ(build->id, "https://example.com/dist/phone-img-1.zip"); + EXPECT_EQ(build->target, "url"); + EXPECT_EQ(build->product, "phone"); +} + +TEST(HttpBuildTests, DirectoryFormSuccess) { + Result build = HttpBuild::FromBuildString( + HttpBuildString{.url = "https://example.com/dist/"}); + + ASSERT_THAT(build, IsOk()); + EXPECT_EQ(build->url, "https://example.com/dist/"); + EXPECT_EQ(build->object, std::nullopt); + EXPECT_EQ(build->id, "https://example.com/dist/"); + EXPECT_EQ(build->product, "url"); +} + +TEST(BuildPrintingTests, UrlBuildsWithoutTheQuerySuccess) { + Build gcs_build = + *GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket/dist/"}); + Build http_build = *HttpBuild::FromBuildString(HttpBuildString{ + .url = kSignedUrl, + .filepath = "boot.img", + }); + + EXPECT_THAT(Print(gcs_build), HasSubstr("gs://bucket/dist/")); + EXPECT_THAT(Print(http_build), + AllOf(HasSubstr("https://example.com/dist/phone-img-1.zip"), + HasSubstr("boot.img"), Not(HasSubstr("secret")))); +} + +TEST(FetchLabelTests, UrlBuildsAreTheBareUrlSuccess) { + Build gcs_build = + *GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket/dist/"}); + Build http_build = + *HttpBuild::FromBuildString(HttpBuildString{.url = kSignedUrl}); + + EXPECT_EQ(FetchLabel(gcs_build), "gs://bucket/dist/"); + EXPECT_EQ(FetchLabel(http_build), "https://example.com/dist/phone-img-1.zip"); +} + +TEST(FetchLabelTests, OtherBuildsAreIdAndTargetSuccess) { + Build device_build = DeviceBuild{.id = "123", .target = "test_target"}; + Build directory_build = + DirectoryBuild({"/tmp/build"}, "test_target", std::nullopt); + + EXPECT_EQ(FetchLabel(device_build), "123/test_target"); + EXPECT_EQ(FetchLabel(directory_build), "eng/test_target"); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/caching_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/caching_build_api.cpp index e44eaeacc69..85970bce03d 100644 --- a/base/cvd/cuttlefish/host/libs/web/caching_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/caching_build_api.cpp @@ -27,7 +27,6 @@ #include "cuttlefish/files/file_exists.h" #include "cuttlefish/files/link_or_copy.h" #include "cuttlefish/host/libs/web/android_build.h" -#include "cuttlefish/host/libs/web/android_build_api.h" #include "cuttlefish/host/libs/web/android_build_string.h" #include "cuttlefish/host/libs/web/build_api.h" #include "cuttlefish/host/libs/zip/cached_zip_source.h" diff --git a/base/cvd/cuttlefish/metrics/conversion/fetch_conversion.cc b/base/cvd/cuttlefish/metrics/conversion/fetch_conversion.cc index 018e6761c4f..99eab97f32e 100644 --- a/base/cvd/cuttlefish/metrics/conversion/fetch_conversion.cc +++ b/base/cvd/cuttlefish/metrics/conversion/fetch_conversion.cc @@ -16,6 +16,7 @@ #include "cuttlefish/metrics/conversion/fetch_conversion.h" +#include #include #include "cuttlefish/host/commands/cvd/fetch/builds.h" @@ -41,13 +42,20 @@ using logs::proto::wireless::android::cuttlefish::events::CuttlefishFetchStart; using logs::proto::wireless::android::cuttlefish::events::MetricsEventV2; void PopulateCuttlefishBuild(CuttlefishBuild& cf_build, const Build& build) { - cf_build.set_build_id(std::visit([](auto&& arg) { return arg.id; }, build)); - cf_build.set_target(std::visit([](auto&& arg) { return arg.target; }, build)); - cf_build.set_product( - std::visit([](auto&& arg) { return arg.product; }, build)); + std::string build_id = std::visit([](auto&& arg) { return arg.id; }, build); if (const DeviceBuild* device_build = std::get_if(&build)) { cf_build.set_branch(device_build->branch); + } else if (std::holds_alternative(build)) { + // A URL can name a private bucket or an internal host, so only its scheme + // is reported. The rest stays in the local fetcher config. + build_id = "gs"; + } else if (std::holds_alternative(build)) { + build_id = "https"; } + cf_build.set_build_id(build_id); + cf_build.set_target(std::visit([](auto&& arg) { return arg.target; }, build)); + cf_build.set_product( + std::visit([](auto&& arg) { return arg.product; }, build)); } void PopulateFetchBuilds(CuttlefishFetchBuilds& fetch_builds, From a1710eb212dbf1472d82b7706c76807e92a7b700 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:36 +0000 Subject: [PATCH 05/20] Read Cloud Storage builds through the JSON API --- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 67 ++++- .../host/libs/web/android_build_api.cpp | 23 +- .../cuttlefish/host/libs/web/build_api_zip.cc | 10 +- .../cuttlefish/host/libs/web/build_api_zip.h | 3 + .../host/libs/web/gcs_build_api.cpp | 239 ++++++++++++++++ .../cuttlefish/host/libs/web/gcs_build_api.h | 55 ++++ .../host/libs/web/gcs_build_api_test.cpp | 256 ++++++++++++++++++ .../host/libs/web/http_client/BUILD.bazel | 1 + .../host/libs/web/http_client/http_json.cc | 25 ++ .../host/libs/web/http_client/http_json.h | 18 ++ .../host/libs/web/zip_over_ranges.cpp | 97 +++++++ .../host/libs/web/zip_over_ranges.h | 48 ++++ 12 files changed, 819 insertions(+), 23 deletions(-) create mode 100644 base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp create mode 100644 base/cvd/cuttlefish/host/libs/web/gcs_build_api.h create mode 100644 base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp create mode 100644 base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp create mode 100644 base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index 41803104935..43204ead0e9 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -53,7 +53,6 @@ cf_cc_library( "//cuttlefish/host/libs/web/http_client", "//cuttlefish/host/libs/web/http_client:http_file", "//cuttlefish/host/libs/web/http_client:http_json", - "//cuttlefish/host/libs/web/http_client:scrub_secrets", "//cuttlefish/host/libs/zip:remote_zip", "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", "//cuttlefish/host/libs/zip/libzip_cc:writable_source", @@ -211,6 +210,57 @@ cf_cc_library( ], ) +cf_cc_library( + name = "gcs_build_api", + srcs = ["gcs_build_api.cpp"], + hdrs = ["gcs_build_api.h"], + deps = [ + "//cuttlefish/common/libs/utils:contains", + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/common/libs/utils:json", + "//cuttlefish/host/libs/web:android_build", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/host/libs/web:build_api_zip", + "//cuttlefish/host/libs/web:credential_source", + "//cuttlefish/host/libs/web/http_client", + "//cuttlefish/host/libs/web/http_client:http_file", + "//cuttlefish/host/libs/web/http_client:http_json", + "//cuttlefish/host/libs/web/http_client:url_escape", + "//cuttlefish/host/libs/zip:remote_zip", + "//cuttlefish/host/libs/zip:zip_file", + "//cuttlefish/host/libs/zip/libzip_cc:archive", + "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", + "//cuttlefish/result", + "@abseil-cpp//absl/log", + "@abseil-cpp//absl/strings", + "@fmt", + "@jsoncpp", + ], +) + +cf_cc_test( + name = "gcs_build_api_test", + srcs = ["gcs_build_api_test.cpp"], + deps = [ + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/host/libs/web:android_build", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/host/libs/web:credential_source", + "//cuttlefish/host/libs/web:gcs_build_api", + "//cuttlefish/host/libs/web:zip_over_ranges", + "//cuttlefish/host/libs/web/http_client", + "//cuttlefish/host/libs/web/http_client:fake_http_client", + "//cuttlefish/host/libs/zip/libzip_cc:archive", + "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", + "//cuttlefish/io", + "//cuttlefish/io:string", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "//libbase", + "@abseil-cpp//absl/strings", + ], +) + cf_cc_library( name = "luci_build_api", srcs = ["luci_build_api.cpp"], @@ -294,3 +344,18 @@ cf_cc_test( "//cuttlefish/result:result_matchers", ], ) + +cf_cc_library( + name = "zip_over_ranges", + testonly = True, + srcs = ["zip_over_ranges.cpp"], + hdrs = ["zip_over_ranges.h"], + deps = [ + "//cuttlefish/host/libs/web/http_client", + "//cuttlefish/host/libs/zip:zip_string", + "//cuttlefish/host/libs/zip/libzip_cc:archive", + "//cuttlefish/host/libs/zip/libzip_cc:writable_source", + "//cuttlefish/result", + "@abseil-cpp//absl/strings", + ], +) diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp index 8e55b95cf8c..13d406232ae 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include @@ -52,7 +51,6 @@ #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/host/libs/web/http_client/http_file.h" #include "cuttlefish/host/libs/web/http_client/http_json.h" -#include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" #include "cuttlefish/host/libs/web/parse_time.h" #include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" #include "cuttlefish/host/libs/zip/libzip_cc/writable_source.h" @@ -76,22 +74,11 @@ struct CloseDir { Result GetResponseJson(const HttpResponse& response, const bool allow_redirect = false) { - // debug information in error responses floods stderr with too much text - // logged at a level that still ends up in the log file - // the artifact response carries a `signedUrl` whose query string is the - // credential, so the body is scrubbed before it reaches the log file - VLOG(0) << "API response data:\n" - << ScrubSecrets(response.data.toStyledString()); - const bool response_code_allowed = - response.HttpSuccess() || (allow_redirect && response.HttpRedirect()); - CF_EXPECTF(std::move(response_code_allowed), - "Error response from Android Build API - {}:{}\nCheck log file " - "for full response", - response.http_code, response.StatusDescription()); - CF_EXPECT(!response.data.isMember("error"), - "Response was successful, but contains error information. Check " - "log file for full response."); - return response.data; + return JsonFromResponse(response, JsonResponseOptions{ + .source = "Android Build API", + .allow_redirect = allow_redirect, + .reject_error_member = true, + }); } } // namespace diff --git a/base/cvd/cuttlefish/host/libs/web/build_api_zip.cc b/base/cvd/cuttlefish/host/libs/web/build_api_zip.cc index ee630c91fc2..da0171a23ef 100644 --- a/base/cvd/cuttlefish/host/libs/web/build_api_zip.cc +++ b/base/cvd/cuttlefish/host/libs/web/build_api_zip.cc @@ -27,14 +27,16 @@ namespace cuttlefish { -Result OpenZip(BuildApi& build_api, const Build& build, - const std::string& name) { - SeekableZipSource source = CF_EXPECT(build_api.FileReader(build, name)); - +Result OpenZip(SeekableZipSource source) { SeekableZipSource buffered = CF_EXPECT(BufferZipSource(std::move(source), 1 << 26)); return CF_EXPECT(ReadableZip::FromSource(std::move(buffered))); } +Result OpenZip(BuildApi& build_api, const Build& build, + const std::string& name) { + return CF_EXPECT(OpenZip(CF_EXPECT(build_api.FileReader(build, name)))); +} + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/build_api_zip.h b/base/cvd/cuttlefish/host/libs/web/build_api_zip.h index 4f2b7511b5f..93c46bafb20 100644 --- a/base/cvd/cuttlefish/host/libs/web/build_api_zip.h +++ b/base/cvd/cuttlefish/host/libs/web/build_api_zip.h @@ -20,10 +20,13 @@ #include "cuttlefish/host/libs/web/android_build.h" #include "cuttlefish/host/libs/web/build_api.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" #include "cuttlefish/result/result.h" namespace cuttlefish { +Result OpenZip(SeekableZipSource source); + Result OpenZip(BuildApi&, const Build&, const std::string& name); } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp new file mode 100644 index 00000000000..036531885e5 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp @@ -0,0 +1,239 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/gcs_build_api.h" + +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/strip.h" +#include "fmt/format.h" +#include "json/value.h" + +#include "cuttlefish/common/libs/utils/contains.h" +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/common/libs/utils/json.h" +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/build_api_zip.h" +#include "cuttlefish/host/libs/web/credential_source.h" +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/host/libs/web/http_client/http_file.h" +#include "cuttlefish/host/libs/web/http_client/http_json.h" +#include "cuttlefish/host/libs/web/http_client/url_escape.h" +#include "cuttlefish/host/libs/zip/libzip_cc/archive.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/host/libs/zip/remote_zip.h" +#include "cuttlefish/host/libs/zip/zip_file.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { +namespace { + +constexpr char kStorageApiUrl[] = "https://storage.googleapis.com/storage/v1"; +constexpr long kUnauthorized = 401; +constexpr long kForbidden = 403; + +// `cvd login` does not consent to the storage scope by default, so a rejected +// anonymous read has to name the command that does. +constexpr char kAnonymousHint[] = + "\n\nThis fetch presented no credentials. For a bucket that is not " + "public, run `cvd login " + "--scopes=https://www.googleapis.com/auth/devstorage.read_only`"; + +std::string ObjectUrl(const std::string& bucket, const std::string& object) { + return fmt::format("{}/b/{}/o/{}", kStorageApiUrl, UrlEscape(bucket), + UrlEscape(object)); +} + +std::string MediaUrl(const std::string& bucket, const std::string& object) { + return fmt::format("{}?alt=media", ObjectUrl(bucket, object)); +} + +std::string ListUrl(const std::string& bucket, const std::string& prefix, + const std::string& page_token) { + std::string url = + fmt::format("{}/b/{}/o?delimiter=%2F&prefix={}", kStorageApiUrl, + UrlEscape(bucket), UrlEscape(prefix)); + if (!page_token.empty()) { + url += fmt::format("&pageToken={}", UrlEscape(page_token)); + } + return url; +} + +std::string ArtifactNames(const GcsBuild& build) { + return absl::StrJoin(build.contents, ", ", + [](std::string* out, const auto& entry) { + absl::StrAppend(out, entry.first); + }); +} + +// The object form names one archive, so `{selector}` names a member of it +// rather than a second artifact of the build. +bool IsArchiveMember(const GcsBuild& build, const std::string& artifact_name) { + return build.object.has_value() && artifact_name != *build.object && + absl::EndsWith(*build.object, ".zip") && + build.filepath == artifact_name; +} + +Result ObjectName(const GcsBuild& build, + const std::string& artifact_name) { + if (build.object.has_value()) { + CF_EXPECTF(artifact_name == *build.object, + "The build '{}' holds only '{}', so it has no '{}'.", build.id, + *build.object, artifact_name); + } else { + CF_EXPECTF(Contains(build.contents, artifact_name), + "The build '{}' has no '{}'. It holds [{}].", build.id, + artifact_name, ArtifactNames(build)); + } + return build.prefix + artifact_name; +} + +Result ResponseJson(const HttpResponse& response, + const GcsBuild& build, bool authenticated) { + std::string_view hint; + if (!authenticated && (response.http_code == kUnauthorized || + response.http_code == kForbidden)) { + hint = kAnonymousHint; + } + const std::string source = fmt::format("Cloud Storage for '{}'", build.id); + return JsonFromResponse(response, JsonResponseOptions{ + .source = source, + .hint = hint, + }); +} + +} // namespace + +GcsBuildApi::GcsBuildApi(HttpClient& http_client, + CredentialSource* credential_source) + : http_client_(http_client), credential_source_(credential_source) {} + +Result> GcsBuildApi::Headers() { + std::vector headers; + if (credential_source_ != nullptr) { + headers.emplace_back("Authorization: Bearer " + + CF_EXPECT(credential_source_->Credential())); + } + return headers; +} + +Result GcsBuildApi::GetBuild(const GcsBuildString& build_string) { + GcsBuild build = CF_EXPECT(GcsBuild::FromBuildString(build_string)); + if (build.object.has_value()) { + CF_EXPECT(ProbeObject(build)); + } else { + CF_EXPECT(ListContents(build)); + } + return build; +} + +Result GcsBuildApi::ListContents(GcsBuild& build) { + std::string page_token; + do { + const std::string url = ListUrl(build.bucket, build.prefix, page_token); + const HttpResponse response = + CF_EXPECT(HttpGetToJson(http_client_, url, CF_EXPECT(Headers()))); + const Json::Value json = + CF_EXPECT(ResponseJson(response, build, credential_source_ != nullptr)); + + for (const Json::Value& item : json["items"]) { + const std::string object = + CF_EXPECT(GetValue(item, {"name"})); + std::string_view name = object; + // The prefix itself is listed when a placeholder object created it. + if (!absl::ConsumePrefix(&name, build.prefix) || name.empty()) { + continue; + } + GcsObjectInfo info; + if (item.isMember("generation")) { + info.generation = item["generation"].asString(); + } + if (item.isMember("md5Hash")) { + info.md5 = item["md5Hash"].asString(); + } + build.contents.emplace(name, std::move(info)); + } + + if (json.isMember("nextPageToken")) { + page_token = json["nextPageToken"].asString(); + } else { + page_token = ""; + } + } while (!page_token.empty()); + + CF_EXPECTF(!build.contents.empty(), "The build '{}' holds no artifacts.", + build.id); + return {}; +} + +Result GcsBuildApi::ProbeObject(GcsBuild& build) { + const std::string url = ObjectUrl(build.bucket, build.prefix + *build.object); + const HttpResponse response = + CF_EXPECT(HttpGetToJson(http_client_, url, CF_EXPECT(Headers()))); + const Json::Value json = + CF_EXPECT(ResponseJson(response, build, credential_source_ != nullptr)); + + if (json.isMember("generation")) { + build.generation = json["generation"].asString(); + } + if (json.isMember("md5Hash")) { + build.md5 = json["md5Hash"].asString(); + } + VLOG(1) << build.id << " is " << json["size"].asString() << " bytes"; + return {}; +} + +Result GcsBuildApi::DownloadFile( + const GcsBuild& build, const std::string& target_directory, + const std::string& artifact_name) { + const std::string dest_path = + ConstructTargetFilepath(target_directory, artifact_name); + CF_EXPECT(EnsureDirectoryExists(target_directory)); + + if (IsArchiveMember(build, artifact_name)) { + SeekableZipSource source = CF_EXPECT(FileReader(build, *build.object)); + ReadableZip zip = CF_EXPECT(OpenZip(std::move(source))); + CF_EXPECTF(ExtractFile(zip, artifact_name, dest_path), + "Could not read '{}' out of '{}'.", artifact_name, build.id); + return dest_path; + } + + const std::string url = + MediaUrl(build.bucket, CF_EXPECT(ObjectName(build, artifact_name))); + HttpResponse response = CF_EXPECT( + HttpGetToFile(http_client_, url, dest_path, CF_EXPECT(Headers()))); + CF_EXPECTF(response.HttpSuccess(), + "Could not download '{}' from '{}' - {}:{}", artifact_name, + build.id, response.http_code, response.StatusDescription()); + return dest_path; +} + +Result GcsBuildApi::FileReader( + const GcsBuild& build, const std::string& artifact_name) { + const std::string url = + MediaUrl(build.bucket, CF_EXPECT(ObjectName(build, artifact_name))); + return CF_EXPECT(ZipSourceFromUrl(http_client_, url, CF_EXPECT(Headers()))); +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.h b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.h new file mode 100644 index 00000000000..9ab172c6790 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.h @@ -0,0 +1,55 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/credential_source.h" +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { + +// Reads builds held in Cloud Storage through the JSON API. The credential +// source is optional because public buckets are readable without one. +class GcsBuildApi { + public: + GcsBuildApi(HttpClient& http_client, CredentialSource* credential_source); + + Result GetBuild(const GcsBuildString& build_string); + + Result DownloadFile(const GcsBuild& build, + const std::string& target_directory, + const std::string& artifact_name); + + Result FileReader(const GcsBuild& build, + const std::string& artifact_name); + + private: + Result> Headers(); + + Result ListContents(GcsBuild& build); + Result ProbeObject(GcsBuild& build); + + HttpClient& http_client_; + CredentialSource* credential_source_; +}; + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp new file mode 100644 index 00000000000..65daf47851e --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp @@ -0,0 +1,256 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/gcs_build_api.h" + +#include +#include +#include + +#include "absl/strings/match.h" +#include "android-base/file.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/credential_source.h" +#include "cuttlefish/host/libs/web/http_client/fake_http_client.h" +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/host/libs/web/zip_over_ranges.h" +#include "cuttlefish/host/libs/zip/libzip_cc/archive.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/io/io.h" +#include "cuttlefish/io/string.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::AllOf; +using ::testing::HasSubstr; +using ::testing::SizeIs; + +constexpr char kObjectUrl[] = + "https://storage.googleapis.com/storage/v1/b/bucket/o/" + "dist%2Fphone-img-1.zip"; +constexpr char kMediaUrl[] = + "https://storage.googleapis.com/storage/v1/b/bucket/o/" + "dist%2Fphone-img-1.zip?alt=media"; +constexpr char kListUrl[] = + "https://storage.googleapis.com/storage/v1/b/bucket/o?delimiter=%2F&" + "prefix=dist%2F"; + +TEST(GcsBuildApiTests, GetBuildProbesObjectMetadataSuccess) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse( + R"({"size": "3", "generation": "17", "md5Hash": "m"})", kObjectUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/phone-img-1.zip"}; + Result build = api.GetBuild(build_string); + + ASSERT_THAT(build, IsOk()); + EXPECT_EQ(build->object, "phone-img-1.zip"); + EXPECT_EQ(build->generation, "17"); + EXPECT_EQ(build->md5, "m"); + EXPECT_TRUE(http_client.RequestMade(kObjectUrl)); + EXPECT_FALSE(http_client.RequestMade("alt=media")); +} + +TEST(GcsBuildApiTests, GetBuildAuthorizesWithACredentialSuccess) { + FakeHttpClient http_client; + std::unique_ptr credentials = + FixedCredentialSource::Make("test-token"); + GcsBuildApi api(http_client, credentials.get()); + + bool authorized = false; + http_client.SetResponse( + [&authorized](const HttpRequest& request) { + authorized = HasAuthorization(request.headers); + return HttpResponse{.data = "{}", .http_code = 200}; + }, + kObjectUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/phone-img-1.zip"}; + ASSERT_THAT(api.GetBuild(build_string), IsOk()); + EXPECT_TRUE(authorized); +} + +TEST(GcsBuildApiTests, GetBuildIsAnonymousWithoutACredentialSuccess) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + + bool authorized = false; + http_client.SetResponse( + [&authorized](const HttpRequest& request) { + authorized = HasAuthorization(request.headers); + return HttpResponse{.data = "{}", .http_code = 200}; + }, + kObjectUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/phone-img-1.zip"}; + ASSERT_THAT(api.GetBuild(build_string), IsOk()); + EXPECT_FALSE(authorized); +} + +TEST(GcsBuildApiTests, GetBuildMissingObjectFail) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse( + HttpResponse{.data = "{}", .http_code = 404}, kObjectUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/phone-img-1.zip"}; + EXPECT_THAT( + api.GetBuild(build_string), + IsErrorAndMessage(AllOf(HasSubstr("gs://bucket/dist/phone-img-1.zip"), + HasSubstr("404"), HasSubstr("Check log file")))); +} + +TEST(GcsBuildApiTests, GetBuildEmptyPrefixFail) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse(R"({"kind": "storage#objects"})", kListUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/"}; + EXPECT_THAT(api.GetBuild(build_string), + IsErrorAndMessage(HasSubstr("gs://bucket/dist/"))); +} + +TEST(GcsBuildApiTests, GetBuildListingFollowsPageTokensSuccess) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse( + [](const HttpRequest& request) { + std::string data = + R"({"items": [{"name": "dist/a.txt", "generation": "1", + "md5Hash": "ma"}], "nextPageToken": "page2"})"; + if (absl::StrContains(request.url, "pageToken=page2")) { + data = + R"({"items": [{"name": "dist/b.txt", "generation": "2", + "md5Hash": "mb"}, {"name": "dist/"}]})"; + } + return HttpResponse{.data = data, .http_code = 200}; + }, + kListUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/"}; + Result build = api.GetBuild(build_string); + + ASSERT_THAT(build, IsOk()); + EXPECT_THAT(build->contents, SizeIs(2)); + EXPECT_EQ(build->contents.at("a.txt").generation, "1"); + EXPECT_EQ(build->contents.at("b.txt").md5, "mb"); + EXPECT_TRUE(http_client.RequestMade("pageToken=page2")); +} + +TEST(GcsBuildApiTests, DownloadFileAbsentArtifactFail) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse( + R"({"items": [{"name": "dist/a.txt", "generation": "1"}]})", kListUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/"}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + EXPECT_THAT(api.DownloadFile(*build, target_directory.path, "b.txt"), + IsErrorAndMessage(AllOf(HasSubstr("b.txt"), HasSubstr("a.txt"), + HasSubstr("gs://bucket/dist/")))); + EXPECT_FALSE(http_client.RequestMade("alt=media")); +} + +TEST(GcsBuildApiTests, DownloadFileWritesTheArtifactSuccess) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse( + R"({"items": [{"name": "dist/misc_info.txt", "generation": "1"}]})", + kListUrl); + http_client.SetResponse("recovery_api_version=3", + "b/bucket/o/dist%2Fmisc_info.txt?alt=media"); + + GcsBuildString build_string = {.url = "gs://bucket/dist/"}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + std::string expected_path = + std::string(target_directory.path) + "/misc_info.txt"; + EXPECT_THAT(api.DownloadFile(*build, target_directory.path, "misc_info.txt"), + IsOkAndValue(expected_path)); + + EXPECT_THAT(ReadFileContents(expected_path), + IsOkAndValue("recovery_api_version=3")); +} + +TEST(GcsBuildApiTests, FileReaderReadsTheNamedArtifactSuccess) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse( + R"({"items": [{"name": "dist/phone-img-1.zip", "generation": "1"}]})", + kListUrl); + + Result zip_handler = + ZipOverRanges::Create({{"boot.img", "boot bytes"}}); + ASSERT_THAT(zip_handler, IsOk()); + http_client.SetResponse(*zip_handler, kMediaUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/"}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + Result source = api.FileReader(*build, "phone-img-1.zip"); + ASSERT_THAT(source, IsOk()); + Result zip = ReadableZip::FromSource(std::move(*source)); + ASSERT_THAT(zip, IsOk()); + Result> member = zip->OpenReadOnly("boot.img"); + ASSERT_THAT(member, IsOk()); + EXPECT_THAT(ReadToString(**member), IsOkAndValue("boot bytes")); + EXPECT_TRUE(http_client.RequestMade(kMediaUrl)); +} + +TEST(GcsBuildApiTests, DownloadFileExtractsAnArchiveMemberSuccess) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse(R"({"size": "3"})", kObjectUrl); + + Result zip_handler = ZipOverRanges::Create( + {{"cvd-host_package.tar.gz", "package bytes"}, {"boot.img", "boot"}}); + ASSERT_THAT(zip_handler, IsOk()); + http_client.SetResponse(*zip_handler, kMediaUrl); + + GcsBuildString build_string = { + .url = "gs://bucket/dist/phone-img-1.zip", + .filepath = "cvd-host_package.tar.gz", + }; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + std::string expected_path = + std::string(target_directory.path) + "/cvd-host_package.tar.gz"; + EXPECT_THAT(api.DownloadFile(*build, target_directory.path, + "cvd-host_package.tar.gz"), + IsOkAndValue(expected_path)); + + EXPECT_THAT(ReadFileContents(expected_path), IsOkAndValue("package bytes")); + EXPECT_TRUE(zip_handler->RangeRequestMade()); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/http_client/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/http_client/BUILD.bazel index 3dae675a1f9..cec6c482e2c 100644 --- a/base/cvd/cuttlefish/host/libs/web/http_client/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/http_client/BUILD.bazel @@ -86,6 +86,7 @@ cf_cc_library( "//cuttlefish/common/libs/utils:json", "//cuttlefish/host/libs/web/http_client", "//cuttlefish/host/libs/web/http_client:http_string", + "//cuttlefish/host/libs/web/http_client:scrub_secrets", "//cuttlefish/result", "@abseil-cpp//absl/log", "@jsoncpp", diff --git a/base/cvd/cuttlefish/host/libs/web/http_client/http_json.cc b/base/cvd/cuttlefish/host/libs/web/http_client/http_json.cc index dd48f8b3df4..79ec1a23422 100644 --- a/base/cvd/cuttlefish/host/libs/web/http_client/http_json.cc +++ b/base/cvd/cuttlefish/host/libs/web/http_client/http_json.cc @@ -26,6 +26,7 @@ #include "cuttlefish/common/libs/utils/json.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/host/libs/web/http_client/http_string.h" +#include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" #include "cuttlefish/result/result.h" namespace cuttlefish { @@ -70,4 +71,28 @@ Result> HttpGetToJson( return Parse(CF_EXPECT(HttpGetToString(http_client, url, headers))); } +Result JsonFromResponse(const HttpResponse& response, + const JsonResponseOptions& options) { + // debug information in error responses floods stderr with too much text + // logged at a level that still ends up in the log file + // the artifact response carries a `signedUrl` whose query string is the + // credential, so the body is scrubbed before it reaches the log file + VLOG(0) << "API response data:\n" + << ScrubSecrets(response.data.toStyledString()); + const bool response_code_allowed = + response.HttpSuccess() || + (options.allow_redirect && response.HttpRedirect()); + CF_EXPECTF(std::move(response_code_allowed), + "Error response from {} - {}:{}\nCheck log file for full " + "response{}", + options.source, response.http_code, response.StatusDescription(), + options.hint); + if (options.reject_error_member) { + CF_EXPECT(!response.data.isMember("error"), + "Response was successful, but contains error information. Check " + "log file for full response."); + } + return response.data; +} + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/http_client/http_json.h b/base/cvd/cuttlefish/host/libs/web/http_client/http_json.h index adfb256d499..f5b443a17ab 100644 --- a/base/cvd/cuttlefish/host/libs/web/http_client/http_json.h +++ b/base/cvd/cuttlefish/host/libs/web/http_client/http_json.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include "json/json.h" @@ -43,4 +44,21 @@ Result> HttpGetToJson( HttpClient&, const std::string& url, const std::vector& headers = {}); +// What differs between the APIs that answer a request with json. +struct JsonResponseOptions { + // Names the API in the error of a response that failed. + std::string_view source; + // Whether a redirect carries the json the caller asked for. + bool allow_redirect = false; + // Whether an "error" member of a successful response is itself a failure. + bool reject_error_member = false; + // Appended to the error of a response that failed. + std::string_view hint; +}; + +// Logs the body of `response` and returns its json, or an error naming the +// status the API answered with. +Result JsonFromResponse(const HttpResponse& response, + const JsonResponseOptions& options); + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp new file mode 100644 index 00000000000..bc6908bcfbe --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp @@ -0,0 +1,97 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/zip_over_ranges.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_split.h" + +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/host/libs/zip/libzip_cc/archive.h" +#include "cuttlefish/host/libs/zip/libzip_cc/writable_source.h" +#include "cuttlefish/host/libs/zip/zip_string.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { + +bool HasAuthorization(const std::vector& headers) { + for (const std::string& header : headers) { + if (absl::StartsWith(header, "Authorization:")) { + return true; + } + } + return false; +} + +Result ZipOverRanges::Create( + const std::map& contents) { + std::string buffer(4096, '\0'); + + WritableZipSource source = + CF_EXPECT(WritableZipSource::BorrowData(buffer.data(), buffer.size())); + WritableZip zip = CF_EXPECT(WritableZip::FromSource(std::move(source))); + for (const auto& [path, data] : contents) { + CF_EXPECT(AddStringAt(zip, data, path)); + } + source = CF_EXPECT(WritableZipSource::FromZip(std::move(zip))); + + return ZipOverRanges(CF_EXPECT(ReadToString(source))); +} + +HttpResponse ZipOverRanges::operator()( + const HttpRequest& request) { + static constexpr std::string_view kPrefix = "Range: bytes="; + size_t start = 0; + size_t end = data_.size(); + for (const std::string& header : request.headers) { + if (!absl::StartsWith(header, kPrefix)) { + continue; + } + *ranged_ = true; + const std::string range = header.substr(kPrefix.size()); + const std::vector parts = absl::StrSplit(range, '-'); + if (parts.size() == 2 && absl::SimpleAtoi(parts[0], &start) && + absl::SimpleAtoi(parts[1], &end)) { + end++; // HTTP ranges are inclusive at both ends + } + } + if (end > data_.size()) { + end = data_.size(); + } + return HttpResponse{ + .data = data_.substr(start, end - start), + .http_code = 200, + .headers = + { + {"content-length", std::to_string(end - start)}, + {"accept-ranges", "bytes"}, + }, + }; +} + +ZipOverRanges::ZipOverRanges(std::string data) + : data_(std::move(data)), ranged_(std::make_shared(false)) {} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h new file mode 100644 index 00000000000..ace9d73aa42 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h @@ -0,0 +1,48 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { + +// Returns whether `headers` carry an `Authorization` header. +bool HasAuthorization(const std::vector& headers); + +// Serves one archive over HTTP range requests, as an object store does. +class ZipOverRanges { + public: + static Result Create( + const std::map& contents); + + HttpResponse operator()(const HttpRequest& request); + + bool RangeRequestMade() const { return *ranged_; } + + private: + explicit ZipOverRanges(std::string data); + + std::string data_; + std::shared_ptr ranged_; +}; + +} // namespace cuttlefish From dc81cd29b88ab82cb2615b62196d7b5dbca5e349 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:36 +0000 Subject: [PATCH 06/20] Read builds from plain https:// URLs --- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 41 +++ .../cuttlefish/host/libs/web/android_build.h | 3 + .../host/libs/web/http_build_api.cpp | 148 ++++++++++ .../cuttlefish/host/libs/web/http_build_api.h | 50 ++++ .../host/libs/web/http_build_api_test.cpp | 260 ++++++++++++++++++ .../host/libs/web/zip_over_ranges.cpp | 28 +- .../host/libs/web/zip_over_ranges.h | 6 +- 7 files changed, 523 insertions(+), 13 deletions(-) create mode 100644 base/cvd/cuttlefish/host/libs/web/http_build_api.cpp create mode 100644 base/cvd/cuttlefish/host/libs/web/http_build_api.h create mode 100644 base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index 43204ead0e9..85de494f2c1 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -261,6 +261,47 @@ cf_cc_test( ], ) +cf_cc_library( + name = "http_build_api", + srcs = ["http_build_api.cpp"], + hdrs = ["http_build_api.h"], + deps = [ + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/host/libs/web:android_build", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/host/libs/web:build_api_zip", + "//cuttlefish/host/libs/web/http_client", + "//cuttlefish/host/libs/web/http_client:http_file", + "//cuttlefish/host/libs/zip:remote_zip", + "//cuttlefish/host/libs/zip:zip_file", + "//cuttlefish/host/libs/zip/libzip_cc:archive", + "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", + "//cuttlefish/result", + "@abseil-cpp//absl/strings", + ], +) + +cf_cc_test( + name = "http_build_api_test", + srcs = ["http_build_api_test.cpp"], + deps = [ + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/host/libs/web:android_build", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/host/libs/web:http_build_api", + "//cuttlefish/host/libs/web:zip_over_ranges", + "//cuttlefish/host/libs/web/http_client", + "//cuttlefish/host/libs/web/http_client:fake_http_client", + "//cuttlefish/host/libs/zip/libzip_cc:archive", + "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", + "//cuttlefish/io", + "//cuttlefish/io:string", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "//libbase", + ], +) + cf_cc_library( name = "luci_build_api", srcs = ["luci_build_api.cpp"], diff --git a/base/cvd/cuttlefish/host/libs/web/android_build.h b/base/cvd/cuttlefish/host/libs/web/android_build.h index a7ce9871ca8..144a46eeb92 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build.h +++ b/base/cvd/cuttlefish/host/libs/web/android_build.h @@ -94,6 +94,9 @@ struct HttpBuild { std::string url; // object, or directory ending in '/' std::optional object; // set in the object form only std::optional etag; + // Whether the probe found an origin that serves range requests, without + // which a member cannot be read out of an archive. + bool accept_ranges = false; std::optional sha256; // `id` has no query string, so requests must go to `url`. diff --git a/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp new file mode 100644 index 00000000000..28ce5e5af6e --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp @@ -0,0 +1,148 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/http_build_api.h" + +#include + +#include +#include +#include +#include + +#include "absl/strings/match.h" + +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/build_api_zip.h" +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/host/libs/web/http_client/http_file.h" +#include "cuttlefish/host/libs/zip/libzip_cc/archive.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/host/libs/zip/remote_zip.h" +#include "cuttlefish/host/libs/zip/zip_file.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { +namespace { + +constexpr long kPartialContent = 206; + +// The object form names one archive, so `{selector}` names a member of it +// rather than a second artifact of the build. +bool IsArchiveMember(const HttpBuild& build, const std::string& artifact_name) { + return build.object.has_value() && artifact_name != *build.object && + absl::EndsWith(*build.object, ".zip") && + build.filepath == artifact_name; +} + +Result ArtifactUrl(const HttpBuild& build, + const std::string& artifact_name) { + if (build.object.has_value()) { + CF_EXPECTF(artifact_name == *build.object, + "The build '{}' holds only '{}', so it has no '{}'.", build.id, + *build.object, artifact_name); + return build.url; + } + return build.url + artifact_name; +} + +bool ServesRanges(const HttpResponse& response) { + if (response.http_code == kPartialContent) { + return true; + } + const std::optional ranges = + HeaderValue(response.headers, "accept-ranges"); + return ranges.has_value() && absl::StrContains(*ranges, "bytes"); +} + +} // namespace + +HttpBuildApi::HttpBuildApi(HttpClient& http_client) + : http_client_(http_client) {} + +Result HttpBuildApi::GetBuild(const HttpBuildString& build_string) { + HttpBuild build = CF_EXPECT(HttpBuild::FromBuildString(build_string)); + // A directory of plain HTTPS URLs has nothing to list and nothing to probe, + // so its artifacts are only known to be absent when they answer 404. + if (build.object.has_value()) { + CF_EXPECT(ProbeObject(build)); + } + return build; +} + +Result HttpBuildApi::ProbeObject(HttpBuild& build) { + // A pre-signed URL signs the verb, so this is a GET of one byte rather than + // the HEAD the size alone would call for. + const HttpRequest request = { + .method = HttpMethod::kGet, + .url = build.url, + .headers = {"Range: bytes=0-0"}, + }; + auto discard = [](char*, size_t) { return true; }; + const HttpResponse response = + CF_EXPECT(http_client_.DownloadToCallback(request, discard)); + + CF_EXPECTF(!response.HttpRedirect(), + "'{}' redirects with {}, and redirects are not followed. Name " + "the URL it redirects to.", + build.id, response.http_code); + CF_EXPECTF(response.HttpSuccess(), "'{}' is missing or inaccessible - {}:{}", + build.id, response.http_code, response.StatusDescription()); + + if (std::optional etag = + HeaderValue(response.headers, "etag")) { + build.etag = std::string(*etag); + } + build.accept_ranges = ServesRanges(response); + return {}; +} + +Result HttpBuildApi::DownloadFile( + const HttpBuild& build, const std::string& target_directory, + const std::string& artifact_name) { + const std::string dest_path = + ConstructTargetFilepath(target_directory, artifact_name); + CF_EXPECT(EnsureDirectoryExists(target_directory)); + + if (IsArchiveMember(build, artifact_name)) { + CF_EXPECTF(build.accept_ranges == true, + "'{}' does not serve range requests, so '{}' cannot be read out " + "of it.", + build.id, artifact_name); + SeekableZipSource source = CF_EXPECT(FileReader(build, *build.object)); + ReadableZip zip = CF_EXPECT(OpenZip(std::move(source))); + CF_EXPECTF(ExtractFile(zip, artifact_name, dest_path), + "Could not read '{}' out of '{}'.", artifact_name, build.id); + return dest_path; + } + + const std::string url = CF_EXPECT(ArtifactUrl(build, artifact_name)); + HttpResponse response = + CF_EXPECT(HttpGetToFile(http_client_, url, dest_path)); + CF_EXPECTF(response.HttpSuccess(), + "Could not download '{}' from '{}' - {}:{}", artifact_name, + build.id, response.http_code, response.StatusDescription()); + return dest_path; +} + +Result HttpBuildApi::FileReader( + const HttpBuild& build, const std::string& artifact_name) { + const std::string url = CF_EXPECT(ArtifactUrl(build, artifact_name)); + return CF_EXPECT(ZipSourceFromUrl(http_client_, url, {})); +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/http_build_api.h b/base/cvd/cuttlefish/host/libs/web/http_build_api.h new file mode 100644 index 00000000000..8360b53cc30 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/http_build_api.h @@ -0,0 +1,50 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { + +// Reads builds held behind plain `https://` URLs. Holds no credential, so it +// cannot leak one: an origin that needs authorization is reached through a URL +// that carries its own, as a pre-signed URL does. +class HttpBuildApi { + public: + explicit HttpBuildApi(HttpClient& http_client); + + Result GetBuild(const HttpBuildString& build_string); + + Result DownloadFile(const HttpBuild& build, + const std::string& target_directory, + const std::string& artifact_name); + + Result FileReader(const HttpBuild& build, + const std::string& artifact_name); + + private: + Result ProbeObject(HttpBuild& build); + + HttpClient& http_client_; +}; + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp new file mode 100644 index 00000000000..c565ba08742 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp @@ -0,0 +1,260 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/http_build_api.h" + +#include +#include +#include +#include + +#include "android-base/file.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/http_client/fake_http_client.h" +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/host/libs/web/zip_over_ranges.h" +#include "cuttlefish/host/libs/zip/libzip_cc/archive.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/io/io.h" +#include "cuttlefish/io/string.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::AllOf; +using ::testing::Contains; +using ::testing::HasSubstr; +using ::testing::Not; + +constexpr char kObjectUrl[] = "https://example.com/dist/phone-img-1.zip"; +constexpr char kSignedUrl[] = + "https://example.com/dist/phone-img-1.zip?X-Goog-Signature=secret"; + +TEST(HttpBuildApiTests, GetBuildProbesWithARangedGetSuccess) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + + std::vector seen_headers; + http_client.SetResponse( + [&seen_headers](const HttpRequest& request) { + seen_headers = request.headers; + return HttpResponse{ + .data = "a", + .http_code = 206, + .headers = {{"etag", "\"v1\""}, {"accept-ranges", "bytes"}}, + }; + }, + kObjectUrl); + + HttpBuildString build_string = {.url = kSignedUrl}; + Result build = api.GetBuild(build_string); + + ASSERT_THAT(build, IsOk()); + EXPECT_EQ(build->url, kSignedUrl); + EXPECT_EQ(build->etag, "\"v1\""); + EXPECT_TRUE(build->accept_ranges); + // A pre-signed URL signs the verb, so the probe is a ranged GET. + EXPECT_THAT(seen_headers, Contains("Range: bytes=0-0")); + EXPECT_FALSE(HasAuthorization(seen_headers)); +} + +TEST(HttpBuildApiTests, GetBuildDoesNotProbeADirectorySuccess) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + + HttpBuildString build_string = {.url = "https://example.com/dist/"}; + Result build = api.GetBuild(build_string); + + ASSERT_THAT(build, IsOk()); + EXPECT_FALSE(http_client.RequestMade("https://example.com/dist/")); +} + +TEST(HttpBuildApiTests, GetBuildMissingUrlFail) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + http_client.SetResponse(HttpResponse{.http_code = 404}, + kObjectUrl); + + HttpBuildString build_string = {.url = kSignedUrl}; + EXPECT_THAT(api.GetBuild(build_string), + IsErrorAndMessage(AllOf(HasSubstr(kObjectUrl), HasSubstr("404"), + Not(HasSubstr("secret"))))); +} + +TEST(HttpBuildApiTests, GetBuildRedirectFail) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + http_client.SetResponse(HttpResponse{.http_code = 302}, + kObjectUrl); + + HttpBuildString build_string = {.url = kSignedUrl}; + EXPECT_THAT(api.GetBuild(build_string), + IsErrorAndMessage(AllOf(HasSubstr(kObjectUrl), HasSubstr("302"), + HasSubstr("redirect")))); +} + +TEST(HttpBuildApiTests, DownloadFileFromADirectorySuccess) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + + bool authorized = false; + http_client.SetResponse( + [&authorized](const HttpRequest& request) { + authorized = HasAuthorization(request.headers); + return HttpResponse{.data = "recovery_api_version=3", + .http_code = 200}; + }, + "https://example.com/dist/misc_info.txt"); + + HttpBuildString build_string = {.url = "https://example.com/dist/"}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + std::string expected_path = + std::string(target_directory.path) + "/misc_info.txt"; + EXPECT_THAT(api.DownloadFile(*build, target_directory.path, "misc_info.txt"), + IsOkAndValue(expected_path)); + + EXPECT_THAT(ReadFileContents(expected_path), + IsOkAndValue("recovery_api_version=3")); + EXPECT_FALSE(authorized); +} + +TEST(HttpBuildApiTests, DownloadFileAbsentFromADirectoryFail) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + + HttpBuildString build_string = {.url = "https://example.com/dist/"}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + EXPECT_THAT(api.DownloadFile(*build, target_directory.path, "absent.txt"), + IsErrorAndMessage(AllOf(HasSubstr("absent.txt"), + HasSubstr("https://example.com/dist/"), + HasSubstr("404")))); +} + +TEST(HttpBuildApiTests, DownloadFileAbsentFromAnObjectFail) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + http_client.SetResponse( + HttpResponse{.data = "a", .http_code = 206}, kObjectUrl); + + HttpBuildString build_string = {.url = kSignedUrl}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + EXPECT_THAT(api.DownloadFile(*build, target_directory.path, "absent.txt"), + IsErrorAndMessage(AllOf(HasSubstr("absent.txt"), + HasSubstr("phone-img-1.zip")))); +} + +TEST(HttpBuildApiTests, FileReaderReadsTheObjectSuccess) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + + Result zip_handler = + ZipOverRanges::Create({{"boot.img", "boot bytes"}}, + /*serve_ranges=*/true); + ASSERT_THAT(zip_handler, IsOk()); + + bool authorized = false; + http_client.SetResponse( + [&zip_handler, &authorized](const HttpRequest& request) { + authorized |= HasAuthorization(request.headers); + return (*zip_handler)(request); + }, + kObjectUrl); + + HttpBuildString build_string = {.url = kSignedUrl}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + Result source = api.FileReader(*build, "phone-img-1.zip"); + ASSERT_THAT(source, IsOk()); + Result zip = ReadableZip::FromSource(std::move(*source)); + ASSERT_THAT(zip, IsOk()); + Result> member = zip->OpenReadOnly("boot.img"); + ASSERT_THAT(member, IsOk()); + EXPECT_THAT(ReadToString(**member), IsOkAndValue("boot bytes")); + + EXPECT_TRUE(http_client.RequestMade(kSignedUrl)); + EXPECT_FALSE(authorized); +} + +TEST(HttpBuildApiTests, DownloadFileExtractsAnArchiveMemberSuccess) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + + Result zip_handler = ZipOverRanges::Create( + {{"cvd-host_package.tar.gz", "package bytes"}, {"boot.img", "boot"}}, + /*serve_ranges=*/true); + ASSERT_THAT(zip_handler, IsOk()); + http_client.SetResponse(*zip_handler, kObjectUrl); + + HttpBuildString build_string = { + .url = kSignedUrl, + .filepath = "cvd-host_package.tar.gz", + }; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + std::string expected_path = + std::string(target_directory.path) + "/cvd-host_package.tar.gz"; + EXPECT_THAT(api.DownloadFile(*build, target_directory.path, + "cvd-host_package.tar.gz"), + IsOkAndValue(expected_path)); + + EXPECT_THAT(ReadFileContents(expected_path), IsOkAndValue("package bytes")); + EXPECT_TRUE(zip_handler->RangeRequestMade()); +} + +TEST(HttpBuildApiTests, DownloadFileMemberWithoutRangeSupportFail) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + + Result zip_handler = ZipOverRanges::Create( + {{"cvd-host_package.tar.gz", "package bytes"}}, /*serve_ranges=*/false); + ASSERT_THAT(zip_handler, IsOk()); + http_client.SetResponse(*zip_handler, kObjectUrl); + + HttpBuildString build_string = { + .url = kSignedUrl, + .filepath = "cvd-host_package.tar.gz", + }; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + EXPECT_FALSE(build->accept_ranges); + + TemporaryDir target_directory; + EXPECT_THAT(api.DownloadFile(*build, target_directory.path, + "cvd-host_package.tar.gz"), + IsErrorAndMessage(AllOf(HasSubstr("range requests"), + HasSubstr("cvd-host_package.tar.gz")))); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp index bc6908bcfbe..ef58ad94069 100644 --- a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp +++ b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp @@ -27,6 +27,7 @@ #include "absl/strings/match.h" #include "absl/strings/numbers.h" #include "absl/strings/str_split.h" +#include "absl/strings/strip.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" @@ -46,7 +47,7 @@ bool HasAuthorization(const std::vector& headers) { } Result ZipOverRanges::Create( - const std::map& contents) { + const std::map& contents, bool serve_ranges) { std::string buffer(4096, '\0'); WritableZipSource source = @@ -57,7 +58,7 @@ Result ZipOverRanges::Create( } source = CF_EXPECT(WritableZipSource::FromZip(std::move(zip))); - return ZipOverRanges(CF_EXPECT(ReadToString(source))); + return ZipOverRanges(CF_EXPECT(ReadToString(source)), serve_ranges); } HttpResponse ZipOverRanges::operator()( @@ -66,11 +67,11 @@ HttpResponse ZipOverRanges::operator()( size_t start = 0; size_t end = data_.size(); for (const std::string& header : request.headers) { - if (!absl::StartsWith(header, kPrefix)) { + std::string_view range = header; + if (!absl::ConsumePrefix(&range, kPrefix) || !serve_ranges_) { continue; } *ranged_ = true; - const std::string range = header.substr(kPrefix.size()); const std::vector parts = absl::StrSplit(range, '-'); if (parts.size() == 2 && absl::SimpleAtoi(parts[0], &start) && absl::SimpleAtoi(parts[1], &end)) { @@ -80,18 +81,23 @@ HttpResponse ZipOverRanges::operator()( if (end > data_.size()) { end = data_.size(); } + std::vector headers = { + {"content-length", std::to_string(end - start)}, + {"etag", "\"abc\""}, + }; + if (serve_ranges_) { + headers.push_back({"accept-ranges", "bytes"}); + } return HttpResponse{ .data = data_.substr(start, end - start), .http_code = 200, - .headers = - { - {"content-length", std::to_string(end - start)}, - {"accept-ranges", "bytes"}, - }, + .headers = std::move(headers), }; } -ZipOverRanges::ZipOverRanges(std::string data) - : data_(std::move(data)), ranged_(std::make_shared(false)) {} +ZipOverRanges::ZipOverRanges(std::string data, bool serve_ranges) + : data_(std::move(data)), + serve_ranges_(serve_ranges), + ranged_(std::make_shared(false)) {} } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h index ace9d73aa42..c01970c68d3 100644 --- a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h +++ b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h @@ -32,16 +32,18 @@ bool HasAuthorization(const std::vector& headers); class ZipOverRanges { public: static Result Create( - const std::map& contents); + const std::map& contents, + bool serve_ranges = true); HttpResponse operator()(const HttpRequest& request); bool RangeRequestMade() const { return *ranged_; } private: - explicit ZipOverRanges(std::string data); + ZipOverRanges(std::string data, bool serve_ranges); std::string data_; + bool serve_ranges_; std::shared_ptr ranged_; }; From e885b57564d2e887a200cb6907f900411ab1338e Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:36 +0000 Subject: [PATCH 07/20] Read remote zips whose size the caller already knows --- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 1 + .../cuttlefish/host/libs/web/android_build.h | 5 +++ .../host/libs/web/gcs_build_api.cpp | 37 ++++++++++++++++++- .../host/libs/web/gcs_build_api_test.cpp | 19 +++++++--- .../host/libs/web/http_build_api.cpp | 29 +++++++++++++++ .../host/libs/web/http_build_api_test.cpp | 32 +++++++++++++++- .../host/libs/web/zip_over_ranges.cpp | 22 +++++++++-- .../host/libs/web/zip_over_ranges.h | 10 ++++- .../cuttlefish/host/libs/zip/remote_zip.cc | 8 ++++ .../cvd/cuttlefish/host/libs/zip/remote_zip.h | 9 +++++ 10 files changed, 158 insertions(+), 14 deletions(-) diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index 85de494f2c1..74eaeee6045 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -398,5 +398,6 @@ cf_cc_library( "//cuttlefish/host/libs/zip/libzip_cc:writable_source", "//cuttlefish/result", "@abseil-cpp//absl/strings", + "@fmt", ], ) diff --git a/base/cvd/cuttlefish/host/libs/web/android_build.h b/base/cvd/cuttlefish/host/libs/web/android_build.h index 144a46eeb92..2c5e5577cc4 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build.h +++ b/base/cvd/cuttlefish/host/libs/web/android_build.h @@ -15,6 +15,8 @@ #pragma once +#include + #include #include #include @@ -60,6 +62,7 @@ std::ostream& operator<<(std::ostream&, const DirectoryBuild&); struct GcsObjectInfo { std::optional generation; std::optional md5; + std::optional size; }; // The objects under a `gs://` prefix, or the single object a `gs://` URL @@ -74,6 +77,7 @@ struct GcsBuild { std::map contents; std::optional generation; std::optional md5; + std::optional size; std::optional sha256; // Derived from the URL for the code that handles every build alike. Never @@ -97,6 +101,7 @@ struct HttpBuild { // Whether the probe found an origin that serves range requests, without // which a member cannot be read out of an archive. bool accept_ranges = false; + std::optional size; std::optional sha256; // `id` has no query string, so requests must go to `url`. diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp index 036531885e5..674340ca353 100644 --- a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp @@ -15,7 +15,10 @@ #include "cuttlefish/host/libs/web/gcs_build_api.h" +#include + #include +#include #include #include #include @@ -23,6 +26,7 @@ #include "absl/log/log.h" #include "absl/strings/match.h" +#include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/strip.h" @@ -109,6 +113,29 @@ Result ObjectName(const GcsBuild& build, return build.prefix + artifact_name; } +std::optional ParseSize(const std::string& size) { + uint64_t parsed = 0; + if (!absl::SimpleAtoi(size, &parsed)) { + return std::nullopt; + } + return parsed; +} + +// The size the listing or the metadata probe already reported, which spares +// the zip reader a round trip to ask for it. +std::optional ArtifactSize(const GcsBuild& build, + const std::string& artifact_name) { + if (build.object.has_value()) { + return build.size; + } + const std::map::const_iterator entry = + build.contents.find(artifact_name); + if (entry == build.contents.end()) { + return std::nullopt; + } + return entry->second.size; +} + Result ResponseJson(const HttpResponse& response, const GcsBuild& build, bool authenticated) { std::string_view hint; @@ -172,6 +199,7 @@ Result GcsBuildApi::ListContents(GcsBuild& build) { if (item.isMember("md5Hash")) { info.md5 = item["md5Hash"].asString(); } + info.size = ParseSize(item["size"].asString()); build.contents.emplace(name, std::move(info)); } @@ -200,7 +228,7 @@ Result GcsBuildApi::ProbeObject(GcsBuild& build) { if (json.isMember("md5Hash")) { build.md5 = json["md5Hash"].asString(); } - VLOG(1) << build.id << " is " << json["size"].asString() << " bytes"; + build.size = ParseSize(json["size"].asString()); return {}; } @@ -233,7 +261,12 @@ Result GcsBuildApi::FileReader( const GcsBuild& build, const std::string& artifact_name) { const std::string url = MediaUrl(build.bucket, CF_EXPECT(ObjectName(build, artifact_name))); - return CF_EXPECT(ZipSourceFromUrl(http_client_, url, CF_EXPECT(Headers()))); + std::vector headers = CF_EXPECT(Headers()); + if (std::optional size = ArtifactSize(build, artifact_name)) { + return CF_EXPECT( + ZipSourceFromUrl(http_client_, url, std::move(headers), *size)); + } + return CF_EXPECT(ZipSourceFromUrl(http_client_, url, std::move(headers))); } } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp index 65daf47851e..0046a75d3a7 100644 --- a/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp @@ -20,6 +20,7 @@ #include #include "absl/strings/match.h" +#include "absl/strings/str_cat.h" #include "android-base/file.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -142,7 +143,8 @@ TEST(GcsBuildApiTests, GetBuildListingFollowsPageTokensSuccess) { if (absl::StrContains(request.url, "pageToken=page2")) { data = R"({"items": [{"name": "dist/b.txt", "generation": "2", - "md5Hash": "mb"}, {"name": "dist/"}]})"; + "md5Hash": "mb", "size": "12"}, + {"name": "dist/"}]})"; } return HttpResponse{.data = data, .http_code = 200}; }, @@ -155,6 +157,7 @@ TEST(GcsBuildApiTests, GetBuildListingFollowsPageTokensSuccess) { EXPECT_THAT(build->contents, SizeIs(2)); EXPECT_EQ(build->contents.at("a.txt").generation, "1"); EXPECT_EQ(build->contents.at("b.txt").md5, "mb"); + EXPECT_EQ(build->contents.at("b.txt").size, 12); EXPECT_TRUE(http_client.RequestMade("pageToken=page2")); } @@ -201,14 +204,16 @@ TEST(GcsBuildApiTests, DownloadFileWritesTheArtifactSuccess) { TEST(GcsBuildApiTests, FileReaderReadsTheNamedArtifactSuccess) { FakeHttpClient http_client; GcsBuildApi api(http_client, nullptr); - http_client.SetResponse( - R"({"items": [{"name": "dist/phone-img-1.zip", "generation": "1"}]})", - kListUrl); Result zip_handler = ZipOverRanges::Create({{"boot.img", "boot bytes"}}); ASSERT_THAT(zip_handler, IsOk()); http_client.SetResponse(*zip_handler, kMediaUrl); + http_client.SetResponse( + absl::StrCat(R"({"items": [{"name": "dist/phone-img-1.zip",)", + R"( "generation": "1", "size": ")", zip_handler->Size(), + R"("}]})"), + kListUrl); GcsBuildString build_string = {.url = "gs://bucket/dist/"}; Result build = api.GetBuild(build_string); @@ -222,17 +227,20 @@ TEST(GcsBuildApiTests, FileReaderReadsTheNamedArtifactSuccess) { ASSERT_THAT(member, IsOk()); EXPECT_THAT(ReadToString(**member), IsOkAndValue("boot bytes")); EXPECT_TRUE(http_client.RequestMade(kMediaUrl)); + // The listing reported the size, so the reader has nothing left to ask. + EXPECT_FALSE(zip_handler->HeadRequestMade()); } TEST(GcsBuildApiTests, DownloadFileExtractsAnArchiveMemberSuccess) { FakeHttpClient http_client; GcsBuildApi api(http_client, nullptr); - http_client.SetResponse(R"({"size": "3"})", kObjectUrl); Result zip_handler = ZipOverRanges::Create( {{"cvd-host_package.tar.gz", "package bytes"}, {"boot.img", "boot"}}); ASSERT_THAT(zip_handler, IsOk()); http_client.SetResponse(*zip_handler, kMediaUrl); + http_client.SetResponse( + absl::StrCat(R"({"size": ")", zip_handler->Size(), R"("})"), kObjectUrl); GcsBuildString build_string = { .url = "gs://bucket/dist/phone-img-1.zip", @@ -250,6 +258,7 @@ TEST(GcsBuildApiTests, DownloadFileExtractsAnArchiveMemberSuccess) { EXPECT_THAT(ReadFileContents(expected_path), IsOkAndValue("package bytes")); EXPECT_TRUE(zip_handler->RangeRequestMade()); + EXPECT_FALSE(zip_handler->HeadRequestMade()); } } // namespace diff --git a/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp index 28ce5e5af6e..5e88b45b08d 100644 --- a/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp @@ -16,6 +16,7 @@ #include "cuttlefish/host/libs/web/http_build_api.h" #include +#include #include #include @@ -23,6 +24,7 @@ #include #include "absl/strings/match.h" +#include "absl/strings/numbers.h" #include "cuttlefish/common/libs/utils/files.h" #include "cuttlefish/host/libs/web/android_build.h" @@ -69,6 +71,28 @@ bool ServesRanges(const HttpResponse& response) { return ranges.has_value() && absl::StrContains(*ranges, "bytes"); } +// The whole object's length: the total of a partial response's `Content-Range` +// or, where the origin answered the range request with the whole object, its +// `Content-Length`. +std::optional ProbedSize(const HttpResponse& response) { + uint64_t size = 0; + if (std::optional range = + HeaderValue(response.headers, "content-range")) { + const size_t total = range->rfind('/'); + if (total != std::string_view::npos && + absl::SimpleAtoi(range->substr(total + 1), &size)) { + return size; + } + } + const std::optional length = + HeaderValue(response.headers, "content-length"); + if (response.http_code != kPartialContent && length.has_value() && + absl::SimpleAtoi(*length, &size)) { + return size; + } + return std::nullopt; +} + } // namespace HttpBuildApi::HttpBuildApi(HttpClient& http_client) @@ -108,6 +132,7 @@ Result HttpBuildApi::ProbeObject(HttpBuild& build) { build.etag = std::string(*etag); } build.accept_ranges = ServesRanges(response); + build.size = ProbedSize(response); return {}; } @@ -142,6 +167,10 @@ Result HttpBuildApi::DownloadFile( Result HttpBuildApi::FileReader( const HttpBuild& build, const std::string& artifact_name) { const std::string url = CF_EXPECT(ArtifactUrl(build, artifact_name)); + if (build.size.has_value()) { + return CF_EXPECT(ZipSourceFromUrl(http_client_, url, {}, *build.size)); + } + // Only a directory reaches here, having had no probe to learn a size from. return CF_EXPECT(ZipSourceFromUrl(http_client_, url, {})); } diff --git a/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp index c565ba08742..0f394e70780 100644 --- a/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp @@ -60,7 +60,9 @@ TEST(HttpBuildApiTests, GetBuildProbesWithARangedGetSuccess) { return HttpResponse{ .data = "a", .http_code = 206, - .headers = {{"etag", "\"v1\""}, {"accept-ranges", "bytes"}}, + .headers = {{"etag", "\"v1\""}, + {"accept-ranges", "bytes"}, + {"content-range", "bytes 0-0/4096"}}, }; }, kObjectUrl); @@ -72,6 +74,7 @@ TEST(HttpBuildApiTests, GetBuildProbesWithARangedGetSuccess) { EXPECT_EQ(build->url, kSignedUrl); EXPECT_EQ(build->etag, "\"v1\""); EXPECT_TRUE(build->accept_ranges); + EXPECT_EQ(build->size, 4096); // A pre-signed URL signs the verb, so the probe is a ranged GET. EXPECT_THAT(seen_headers, Contains("Range: bytes=0-0")); EXPECT_FALSE(HasAuthorization(seen_headers)); @@ -202,6 +205,32 @@ TEST(HttpBuildApiTests, FileReaderReadsTheObjectSuccess) { EXPECT_TRUE(http_client.RequestMade(kSignedUrl)); EXPECT_FALSE(authorized); + // The probe reported the size, so the reader has nothing left to ask. + EXPECT_FALSE(zip_handler->HeadRequestMade()); +} + +TEST(HttpBuildApiTests, FileReaderReadsAnOriginThatRejectsHeadSuccess) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + + Result zip_handler = + ZipOverRanges::Create({{"boot.img", "boot bytes"}}, + /*serve_ranges=*/true, /*reject_head=*/true); + ASSERT_THAT(zip_handler, IsOk()); + http_client.SetResponse(*zip_handler, kObjectUrl); + + HttpBuildString build_string = {.url = kSignedUrl}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + Result source = api.FileReader(*build, "phone-img-1.zip"); + ASSERT_THAT(source, IsOk()); + Result zip = ReadableZip::FromSource(std::move(*source)); + ASSERT_THAT(zip, IsOk()); + Result> member = zip->OpenReadOnly("boot.img"); + ASSERT_THAT(member, IsOk()); + EXPECT_THAT(ReadToString(**member), IsOkAndValue("boot bytes")); + EXPECT_FALSE(zip_handler->HeadRequestMade()); } TEST(HttpBuildApiTests, DownloadFileExtractsAnArchiveMemberSuccess) { @@ -230,6 +259,7 @@ TEST(HttpBuildApiTests, DownloadFileExtractsAnArchiveMemberSuccess) { EXPECT_THAT(ReadFileContents(expected_path), IsOkAndValue("package bytes")); EXPECT_TRUE(zip_handler->RangeRequestMade()); + EXPECT_FALSE(zip_handler->HeadRequestMade()); } TEST(HttpBuildApiTests, DownloadFileMemberWithoutRangeSupportFail) { diff --git a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp index ef58ad94069..af06d78db23 100644 --- a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp +++ b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.cpp @@ -28,6 +28,7 @@ #include "absl/strings/numbers.h" #include "absl/strings/str_split.h" #include "absl/strings/strip.h" +#include "fmt/format.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" @@ -47,7 +48,8 @@ bool HasAuthorization(const std::vector& headers) { } Result ZipOverRanges::Create( - const std::map& contents, bool serve_ranges) { + const std::map& contents, bool serve_ranges, + bool reject_head) { std::string buffer(4096, '\0'); WritableZipSource source = @@ -58,12 +60,19 @@ Result ZipOverRanges::Create( } source = CF_EXPECT(WritableZipSource::FromZip(std::move(zip))); - return ZipOverRanges(CF_EXPECT(ReadToString(source)), serve_ranges); + return ZipOverRanges(CF_EXPECT(ReadToString(source)), serve_ranges, + reject_head); } HttpResponse ZipOverRanges::operator()( const HttpRequest& request) { static constexpr std::string_view kPrefix = "Range: bytes="; + if (request.method == HttpMethod::kHead) { + *head_ = true; + if (reject_head_) { + return HttpResponse{.http_code = 403}; + } + } size_t start = 0; size_t end = data_.size(); for (const std::string& header : request.headers) { @@ -87,6 +96,8 @@ HttpResponse ZipOverRanges::operator()( }; if (serve_ranges_) { headers.push_back({"accept-ranges", "bytes"}); + headers.push_back({"content-range", fmt::format("bytes {}-{}/{}", start, + end - 1, data_.size())}); } return HttpResponse{ .data = data_.substr(start, end - start), @@ -95,9 +106,12 @@ HttpResponse ZipOverRanges::operator()( }; } -ZipOverRanges::ZipOverRanges(std::string data, bool serve_ranges) +ZipOverRanges::ZipOverRanges(std::string data, bool serve_ranges, + bool reject_head) : data_(std::move(data)), serve_ranges_(serve_ranges), - ranged_(std::make_shared(false)) {} + reject_head_(reject_head), + ranged_(std::make_shared(false)), + head_(std::make_shared(false)) {} } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h index c01970c68d3..6c2342938c6 100644 --- a/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h +++ b/base/cvd/cuttlefish/host/libs/web/zip_over_ranges.h @@ -15,6 +15,8 @@ #pragma once +#include + #include #include #include @@ -33,18 +35,22 @@ class ZipOverRanges { public: static Result Create( const std::map& contents, - bool serve_ranges = true); + bool serve_ranges = true, bool reject_head = false); HttpResponse operator()(const HttpRequest& request); + uint64_t Size() const { return data_.size(); } bool RangeRequestMade() const { return *ranged_; } + bool HeadRequestMade() const { return *head_; } private: - ZipOverRanges(std::string data, bool serve_ranges); + ZipOverRanges(std::string data, bool serve_ranges, bool reject_head); std::string data_; bool serve_ranges_; + bool reject_head_; std::shared_ptr ranged_; + std::shared_ptr head_; }; } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/zip/remote_zip.cc b/base/cvd/cuttlefish/host/libs/zip/remote_zip.cc index 961a9eb19e1..ff7338f4835 100644 --- a/base/cvd/cuttlefish/host/libs/zip/remote_zip.cc +++ b/base/cvd/cuttlefish/host/libs/zip/remote_zip.cc @@ -140,6 +140,14 @@ Result ZipSourceFromUrl(HttpClient& http_client, uint64_t size = CF_EXPECT(GetSizeIfSupportsRangeRequests(http_client, url, headers)); + return CF_EXPECT( + ZipSourceFromUrl(http_client, url, std::move(headers), size)); +} + +Result ZipSourceFromUrl(HttpClient& http_client, + const std::string& url, + std::vector headers, + uint64_t size) { std::unique_ptr callbacks = std::make_unique(http_client, url, size, std::move(headers)); CF_EXPECT(callbacks.get()); diff --git a/base/cvd/cuttlefish/host/libs/zip/remote_zip.h b/base/cvd/cuttlefish/host/libs/zip/remote_zip.h index ef1cf8d1916..0cdc53da350 100644 --- a/base/cvd/cuttlefish/host/libs/zip/remote_zip.h +++ b/base/cvd/cuttlefish/host/libs/zip/remote_zip.h @@ -15,6 +15,8 @@ #pragma once +#include + #include #include @@ -30,4 +32,11 @@ namespace cuttlefish { * `HttpClient`. */ Result ZipSourceFromUrl(HttpClient&, const std::string& url, std::vector headers); + +/* The same, for a caller that already knows the object's `size` and that the + * origin serves ranges. Asks the origin neither question, so it also works + * where a HEAD request would be rejected. */ +Result ZipSourceFromUrl(HttpClient&, const std::string& url, + std::vector headers, + uint64_t size); } // namespace cuttlefish From fa20fe3e8491c9d11918a5f2e5c846054401362f Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:37 +0000 Subject: [PATCH 08/20] Route each build to the API that owns its source --- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 35 +++++ .../host/libs/web/android_build_api.h | 16 +- .../host/libs/web/composite_build_api.cpp | 147 ++++++++++++++++++ .../host/libs/web/composite_build_api.h | 56 +++++++ .../libs/web/composite_build_api_test.cpp | 139 +++++++++++++++++ 5 files changed, 385 insertions(+), 8 deletions(-) create mode 100644 base/cvd/cuttlefish/host/libs/web/composite_build_api.cpp create mode 100644 base/cvd/cuttlefish/host/libs/web/composite_build_api.h create mode 100644 base/cvd/cuttlefish/host/libs/web/composite_build_api_test.cpp diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index 74eaeee6045..e73b2dd43d3 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -193,6 +193,41 @@ cf_cc_library( ], ) +cf_cc_library( + name = "composite_build_api", + srcs = ["composite_build_api.cpp"], + hdrs = ["composite_build_api.h"], + deps = [ + "//cuttlefish/host/libs/web:android_build", + "//cuttlefish/host/libs/web:android_build_api", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/host/libs/web:build_api", + "//cuttlefish/host/libs/web:gcs_build_api", + "//cuttlefish/host/libs/web:http_build_api", + "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", + "//cuttlefish/result", + ], +) + +cf_cc_test( + name = "composite_build_api_test", + srcs = ["composite_build_api_test.cpp"], + deps = [ + "//cuttlefish/host/libs/web:android_build", + "//cuttlefish/host/libs/web:android_build_api", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/host/libs/web:android_build_url", + "//cuttlefish/host/libs/web:composite_build_api", + "//cuttlefish/host/libs/web:gcs_build_api", + "//cuttlefish/host/libs/web:http_build_api", + "//cuttlefish/host/libs/web/http_client:fake_http_client", + "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "//libbase", + ], +) + cf_cc_library( name = "credential_source", srcs = ["credential_source.cc"], diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_api.h b/base/cvd/cuttlefish/host/libs/web/android_build_api.h index 031c7b681cc..56fa9a1ccd5 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_api.h +++ b/base/cvd/cuttlefish/host/libs/web/android_build_api.h @@ -47,6 +47,9 @@ class AndroidBuildApi : public BuildApi { Result GetBuild(const BuildString& build_string) override; + Result GetBuild(const DeviceBuildString& build_string); + Result GetBuild(const DirectoryBuildString& build_string); + Result DownloadFile(const Build& build, const std::string& target_directory, const std::string& artifact_name) override; @@ -54,6 +57,11 @@ class AndroidBuildApi : public BuildApi { Result FileReader( const Build&, const std::string& artifact_name) override; + Result FileReader(const DeviceBuild&, + const std::string& artifact_name); + Result FileReader(const DirectoryBuild&, + const std::string& artifact_name); + private: struct BuildInfo { std::string branch; @@ -110,14 +118,6 @@ class AndroidBuildApi : public BuildApi { const Build& build, const std::string& target_directory, const std::string& artifact_name); - Result GetBuild(const DeviceBuildString& build_string); - Result GetBuild(const DirectoryBuildString& build_string); - - Result FileReader(const DeviceBuild&, - const std::string& artifact_name); - Result FileReader(const DirectoryBuild&, - const std::string& artifact_name); - HttpClient& http_client_; AndroidBuildUrl& android_build_url_; CredentialSource* credential_source_; diff --git a/base/cvd/cuttlefish/host/libs/web/composite_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/composite_build_api.cpp new file mode 100644 index 00000000000..3a29f8c7916 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/composite_build_api.cpp @@ -0,0 +1,147 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/composite_build_api.h" + +#include +#include +#include +#include + +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_api.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/gcs_build_api.h" +#include "cuttlefish/host/libs/web/http_build_api.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { +namespace { + +struct GetBuildVisitor { + AndroidBuildApi& android; + GcsBuildApi& gcs; + HttpBuildApi& http; + + Result operator()(const DeviceBuildString& build_string) { + return CF_EXPECT(android.GetBuild(build_string)); + } + + Result operator()(const DirectoryBuildString& build_string) { + return CF_EXPECT(android.GetBuild(build_string)); + } + + Result operator()(const GcsBuildString& build_string) { + return CF_EXPECT(gcs.GetBuild(build_string)); + } + + Result operator()(const HttpBuildString& build_string) { + return CF_EXPECT(http.GetBuild(build_string)); + } +}; + +struct DownloadFileVisitor { + AndroidBuildApi& android; + GcsBuildApi& gcs; + HttpBuildApi& http; + const std::string& target_directory; + const std::string& artifact_name; + + Result operator()(const DeviceBuild& build) { + return CF_EXPECT( + android.DownloadFile(build, target_directory, artifact_name)); + } + + Result operator()(const DirectoryBuild& build) { + return CF_EXPECT( + android.DownloadFile(build, target_directory, artifact_name)); + } + + Result operator()(const GcsBuild& build) { + return CF_EXPECT(gcs.DownloadFile(build, target_directory, artifact_name)); + } + + Result operator()(const HttpBuild& build) { + return CF_EXPECT(http.DownloadFile(build, target_directory, artifact_name)); + } +}; + +struct FileReaderVisitor { + AndroidBuildApi& android; + GcsBuildApi& gcs; + HttpBuildApi& http; + const std::string& artifact_name; + + Result operator()(const DeviceBuild& build) { + return CF_EXPECT(android.FileReader(build, artifact_name)); + } + + Result operator()(const DirectoryBuild& build) { + return CF_EXPECT(android.FileReader(build, artifact_name)); + } + + Result operator()(const GcsBuild& build) { + return CF_EXPECT(gcs.FileReader(build, artifact_name)); + } + + Result operator()(const HttpBuild& build) { + return CF_EXPECT(http.FileReader(build, artifact_name)); + } +}; + +} // namespace + +CompositeBuildApi::CompositeBuildApi(std::unique_ptr android, + std::unique_ptr gcs, + std::unique_ptr http) + : android_(std::move(android)), + gcs_(std::move(gcs)), + http_(std::move(http)) {} + +Result CompositeBuildApi::GetBuild(const BuildString& build_string) { + GetBuildVisitor visitor{ + .android = *android_, + .gcs = *gcs_, + .http = *http_, + }; + return CF_EXPECT(std::visit(visitor, build_string)); +} + +Result CompositeBuildApi::DownloadFile( + const Build& build, const std::string& target_directory, + const std::string& artifact_name) { + DownloadFileVisitor visitor{ + .android = *android_, + .gcs = *gcs_, + .http = *http_, + .target_directory = target_directory, + .artifact_name = artifact_name, + }; + return CF_EXPECT(std::visit(visitor, build)); +} + +Result CompositeBuildApi::FileReader( + const Build& build, const std::string& artifact_name) { + FileReaderVisitor visitor{ + .android = *android_, + .gcs = *gcs_, + .http = *http_, + .artifact_name = artifact_name, + }; + return CF_EXPECT(std::visit(visitor, build)); +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/composite_build_api.h b/base/cvd/cuttlefish/host/libs/web/composite_build_api.h new file mode 100644 index 00000000000..a954eca4a32 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/composite_build_api.h @@ -0,0 +1,56 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_api.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/build_api.h" +#include "cuttlefish/host/libs/web/gcs_build_api.h" +#include "cuttlefish/host/libs/web/http_build_api.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { + +// Sends every build to the API that owns the source it names. This is the +// only `BuildApi` the fetch code sees; each API it holds takes the build +// types it can serve and nothing else. +class CompositeBuildApi : public BuildApi { + public: + CompositeBuildApi(std::unique_ptr android, + std::unique_ptr gcs, + std::unique_ptr http); + + Result GetBuild(const BuildString& build_string) override; + + Result DownloadFile(const Build& build, + const std::string& target_directory, + const std::string& artifact_name) override; + + Result FileReader( + const Build& build, const std::string& artifact_name) override; + + private: + std::unique_ptr android_; + std::unique_ptr gcs_; + std::unique_ptr http_; +}; + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/composite_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/composite_build_api_test.cpp new file mode 100644 index 00000000000..0a649fa69ba --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/composite_build_api_test.cpp @@ -0,0 +1,139 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/composite_build_api.h" + +#include +#include +#include +#include + +#include "android-base/file.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_api.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/android_build_url.h" +#include "cuttlefish/host/libs/web/gcs_build_api.h" +#include "cuttlefish/host/libs/web/http_build_api.h" +#include "cuttlefish/host/libs/web/http_client/fake_http_client.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::_; +using ::testing::HasSubstr; +using ::testing::VariantWith; + +constexpr char kAndroidBuildHost[] = "androidbuild-pa.googleapis.com"; +constexpr char kStorageHost[] = "storage.googleapis.com"; +constexpr char kObjectUrl[] = "https://example.com/dist/img.zip"; +constexpr char kDirectoryUrl[] = "https://example.com/dist/"; + +// Every request reaches the same client, so the host each build string or +// build is sent to is what the routing assertions read. Nothing is set up to +// answer, so each routed call fails at the API it reached. +class CompositeBuildApiTests : public ::testing::Test { + protected: + GcsBuild ListedGcsBuild() { + GcsBuild build = + *GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket/dist/"}); + build.contents = {{"a.txt", GcsObjectInfo{.generation = "1"}}}; + return build; + } + + DirectoryBuild LocalBuild() { + return DirectoryBuild(std::vector{directory_.path}, "target", + std::nullopt); + } + + FakeHttpClient http_client_; + AndroidBuildUrl android_build_url_{ + "https://androidbuild-pa.googleapis.com/v4", "", ""}; + CompositeBuildApi api_{ + std::make_unique(http_client_, android_build_url_), + std::make_unique(http_client_, nullptr), + std::make_unique(http_client_)}; + TemporaryDir directory_; +}; + +TEST_F(CompositeBuildApiTests, GetBuildRoutesByStringAlternativeSuccess) { + Result device = api_.GetBuild( + DeviceBuildString{.branch_or_id = "aosp-main", .target = "target"}); + EXPECT_THAT(device, IsError()); + EXPECT_TRUE(http_client_.RequestMade(kAndroidBuildHost)); + + EXPECT_THAT(api_.GetBuild(DirectoryBuildString{ + .paths = std::vector{directory_.path}, + .target = "target"}), + IsOkAndValue(VariantWith(_))); + + Result gcs = api_.GetBuild(GcsBuildString{.url = "gs://bucket/dist/"}); + EXPECT_THAT(gcs, IsError()); + EXPECT_TRUE(http_client_.RequestMade(kStorageHost)); + + Result http = api_.GetBuild(HttpBuildString{.url = kObjectUrl}); + EXPECT_THAT(http, IsError()); + EXPECT_TRUE(http_client_.RequestMade(kObjectUrl)); +} + +TEST_F(CompositeBuildApiTests, DownloadFileRoutesByBuildAlternativeSuccess) { + Result device = api_.DownloadFile( + DeviceBuild{.id = "1", .target = "target"}, directory_.path, "a.txt"); + EXPECT_THAT(device, IsError()); + EXPECT_TRUE(http_client_.RequestMade(kAndroidBuildHost)); + + EXPECT_THAT(api_.DownloadFile(LocalBuild(), directory_.path, "a.txt"), + IsErrorAndMessage(HasSubstr("a.txt"))); + + Result gcs = + api_.DownloadFile(ListedGcsBuild(), directory_.path, "a.txt"); + EXPECT_THAT(gcs, IsError()); + EXPECT_TRUE(http_client_.RequestMade(kStorageHost)); + + Result http = api_.DownloadFile( + *HttpBuild::FromBuildString(HttpBuildString{.url = kDirectoryUrl}), + directory_.path, "a.txt"); + EXPECT_THAT(http, IsError()); + EXPECT_TRUE(http_client_.RequestMade("https://example.com/dist/a.txt")); +} + +TEST_F(CompositeBuildApiTests, FileReaderRoutesByBuildAlternativeSuccess) { + Result device = + api_.FileReader(DeviceBuild{.id = "1", .target = "target"}, "a.zip"); + EXPECT_THAT(device, IsError()); + EXPECT_TRUE(http_client_.RequestMade(kAndroidBuildHost)); + + EXPECT_THAT(api_.FileReader(LocalBuild(), "a.zip"), + IsErrorAndMessage(HasSubstr("a.zip"))); + + Result gcs = api_.FileReader(ListedGcsBuild(), "a.txt"); + EXPECT_THAT(gcs, IsError()); + EXPECT_TRUE(http_client_.RequestMade(kStorageHost)); + + Result http = api_.FileReader( + *HttpBuild::FromBuildString(HttpBuildString{.url = kObjectUrl}), + "img.zip"); + EXPECT_THAT(http, IsError()); + EXPECT_TRUE(http_client_.RequestMade(kObjectUrl)); +} + +} // namespace +} // namespace cuttlefish From 31c377df463674aedb1c1e43d0e0533124e933fb Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:37 +0000 Subject: [PATCH 09/20] Resolve Cloud Storage credentials on their own ladder --- .../host/commands/cvd/fetch/BUILD.bazel | 22 ++ .../cvd/fetch/build_api_credentials.cc | 79 +++++- .../cvd/fetch/build_api_credentials.h | 14 + .../cvd/fetch/build_api_credentials_test.cpp | 266 ++++++++++++++++++ .../host/libs/web/gcs_build_api_test.cpp | 14 + 5 files changed, 384 insertions(+), 11 deletions(-) create mode 100644 base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials_test.cpp diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel index 6f84bbcf90c..567e438933f 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel @@ -49,10 +49,32 @@ cf_cc_library( "//cuttlefish/host/libs/web/http_client", "//cuttlefish/result", "@abseil-cpp//absl/log", + "@abseil-cpp//absl/strings", "@jsoncpp", ], ) +cf_cc_test( + name = "build_api_credentials_test", + srcs = ["build_api_credentials_test.cpp"], + include_cleaner_enabled = False, + deps = [ + "//cuttlefish/common/libs/utils:base64", + "//cuttlefish/common/libs/utils:environment", + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/host/commands/cvd/fetch:build_api_credentials", + "//cuttlefish/host/commands/cvd/fetch:build_api_flags", + "//cuttlefish/host/libs/web:credential_source", + "//cuttlefish/host/libs/web/http_client", + "//cuttlefish/host/libs/web/http_client:fake_http_client", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "//libbase", + "@abseil-cpp//absl/strings", + "@fmt", + ], +) + cf_cc_library( name = "build_api_flags", srcs = ["build_api_flags.cc"], diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials.cc index 24949d4140f..1f676e5c3cb 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials.cc @@ -20,6 +20,7 @@ #include #include "absl/log/log.h" +#include "absl/strings/match.h" #include "json/reader.h" #include "json/value.h" @@ -46,8 +47,9 @@ std::unique_ptr TryParseServiceAccount( VLOG(0) << "Could not parse credential file as Service Account"; return {}; } - auto result = ServiceAccountOauthCredentialSource::FromJson(http_client, - content, scope); + Result> result = + ServiceAccountOauthCredentialSource::FromJson(http_client, content, + scope); if (!result.has_value()) { VLOG(0) << "Failed to load service account json file: \n" << result.error(); return {}; @@ -55,6 +57,21 @@ std::unique_ptr TryParseServiceAccount( return std::move(*result); } +// Loads the refresh token stored in an oauth2 client file, reporting a file +// that cannot be parsed as a null result rather than an error. +Result> LoadOauth2ClientFile( + HttpClient& http_client, const std::string& filepath) { + const std::string contents = CF_EXPECT(ReadFileContents(filepath)); + Result> credentials = + RefreshTokenCredentialSource::FromOauth2ClientFile(http_client, contents); + if (!credentials.has_value()) { + LOG(ERROR) << "Failed to load oauth credentials from \"" << filepath + << "\":" << credentials.error(); + return nullptr; + } + return std::move(*credentials); +} + Result> GetCredentialSourceLegacy( HttpClient& http_client, const std::string& credential_source, const std::string& oauth_filepath, const std::string& scope) { @@ -63,15 +80,11 @@ Result> GetCredentialSourceLegacy( result = GceMetadataCredentialSource::Make(http_client); } else if (credential_source.empty()) { if (FileExists(oauth_filepath)) { - std::string oauth_contents = CF_EXPECT(ReadFileContents(oauth_filepath)); - auto attempt_load = RefreshTokenCredentialSource::FromOauth2ClientFile( - http_client, oauth_contents); - if (attempt_load.has_value()) { - result = std::move(*attempt_load); + std::unique_ptr credentials = + CF_EXPECT(LoadOauth2ClientFile(http_client, oauth_filepath)); + if (credentials != nullptr) { + result = std::move(credentials); VLOG(0) << "Loaded credentials from '" << oauth_filepath << "'"; - } else { - LOG(ERROR) << "Failed to load oauth credentials from \"" - << oauth_filepath << "\":" << attempt_load.error(); } } else { VLOG(0) << "\"" << oauth_filepath @@ -89,7 +102,8 @@ Result> GetCredentialSourceLegacy( CF_EXPECTF(ReadFileContents(credential_source), "Failure getting credential file contents from file \"{}\"", credential_source); - if (auto crds = TryParseServiceAccount(http_client, file_content, scope)) { + if (std::unique_ptr crds = + TryParseServiceAccount(http_client, file_content, scope)) { result = std::move(crds); } else { result = FixedCredentialSource::Make(file_content); @@ -152,6 +166,49 @@ Result> GetCredentialSourceFromFlags( flags.credential_flags.service_account_filepath, scope)); } +bool IsRunningOnGce() { + const Result product_name = + ReadFileContents("/sys/class/dmi/id/product_name"); + return product_name.has_value() && + absl::StrContains(*product_name, "Google Compute Engine"); +} + +Result> GetStorageCredentialSource( + HttpClient& http_client, const BuildApiFlags& flags, bool running_on_gce) { + const std::string& service_account_filepath = + flags.credential_flags.service_account_filepath; + if (!service_account_filepath.empty()) { + const std::string contents = + CF_EXPECTF(ReadFileContents(service_account_filepath), + "Failure getting service account credential file contents " + "from file '{}'.", + service_account_filepath); + std::unique_ptr credentials = + TryParseServiceAccount(http_client, contents, kCloudStorageReadScope); + CF_EXPECTF(credentials != nullptr, + "Unable to parse service account credentials in file '{}'.", + service_account_filepath); + return credentials; + } + + const std::string boto_filepath = StringFromEnv("HOME", ".") + "/.boto"; + if (FileExists(boto_filepath)) { + std::unique_ptr credentials = + CF_EXPECTF(LoadOauth2ClientFile(http_client, boto_filepath), + "Failure getting credential file contents from file '{}'.", + boto_filepath); + if (credentials != nullptr) { + return credentials; + } + } + + if (flags.credential_flags.use_gce_metadata || + flags.credential_source == "gce" || running_on_gce) { + return GceMetadataCredentialSource::Make(http_client); + } + return nullptr; +} + std::string GetAcloudOauthFilepath() { return StringFromEnv("HOME", ".") + "/.acloud_oauth2.dat"; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials.h b/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials.h index 8541d66ba6d..be31b6e5a2c 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials.h +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials.h @@ -36,6 +36,20 @@ Result> GetCredentialSourceFromFlags( const std::string& oauth_filepath, const std::string& scope = kAndroidBuildApiScope); +// Returns whether the DMI product name identifies this machine as a GCE VM, +// where the metadata server issues credentials without any flag being given. +// Reading the product name costs no HTTP round trip on a machine that is not +// one. +bool IsRunningOnGce(); + +// Resolves credentials for Cloud Storage. Never opens the `credential_source` +// file, which the Android Build path may consume from a pipe, and never +// re-uses an Android Build token, which Cloud Storage rejects. A null result +// means anonymous access, which public buckets serve. The caller supplies +// `running_on_gce`, so the ladder can be exercised on any machine. +Result> GetStorageCredentialSource( + HttpClient& http_client, const BuildApiFlags& flags, bool running_on_gce); + std::string GetAcloudOauthFilepath(); } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials_test.cpp new file mode 100644 index 00000000000..5a3787a6cbf --- /dev/null +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/build_api_credentials_test.cpp @@ -0,0 +1,266 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/commands/cvd/fetch/build_api_credentials.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include "absl/strings/str_replace.h" +#include "absl/strings/str_split.h" +#include "android-base/file.h" +#include "fmt/format.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/common/libs/utils/base64.h" +#include "cuttlefish/common/libs/utils/environment.h" +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/host/commands/cvd/fetch/build_api_flags.h" +#include "cuttlefish/host/libs/web/credential_source.h" +#include "cuttlefish/host/libs/web/http_client/fake_http_client.h" +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::HasSubstr; +using ::testing::IsNull; +using ::testing::Not; +using ::testing::NotNull; + +// Generated for this test alone, and only so that the service account +// credential source has a key it can sign an assertion with. +constexpr char kTestPrivateKey[] = + "-----BEGIN PRIVATE KEY-----\\n" + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCr4jcMb0PlSMAl\\n" + "UO0oams84Ph8HnEYp14sSuNPuarrGaFEm4GPS0BMFBV4BSd7nX/MCqk3Jbd8kolL\\n" + "oaHa+7uKs5F7lrl3w0/JmA8P4/6D86CU0dCrwriIKL6BNvik3CHZEHCmKI6Ht8ur\\n" + "C+jQjEOp9bJVW3+NdyyKxChK29a7W6JjuBRp8TohHKeV5BQHCVj8krkylFCTO3VX\\n" + "KQQpFyy+H6jZyIJY2WSRMLBHg8sVkw+qjXRnGxRdK262SGuPfvzdUzaYbNxqLtXT\\n" + "IzQCV6kDyaBsTCU8fz7nOGg/cgCBuzocnpXgKTgICEwktUUVAC2PXbwrHHEYC05N\\n" + "Z5LOY00xAgMBAAECggEAAdP5E+fHCBQ6/uqaaxiepVobKm7Ecyesh7oQKtPlrnRq\\n" + "U6l3ukdpmqWICOu9HMJzDn96hzyec/O3BBfm+cY9m18HiBH1TQHFwnYciuW42jxo\\n" + "E80bdAgxIDmWtRcZk99HeOCE4i+CPI1G3D3XLwie25riV6gOdjmzPpKRfyJRaVKu\\n" + "spQQdjZ19puXbU4ApLL4QhO3RwjhVzECxhHMxmXWKuCdWp11+NQ+fFBJO6FM7ipz\\n" + "kYDJXRiKTMZK9wOloEhz3CYeukCe4P9HUDfq8TCK4xmEYLawFfSvYSA0Sb7SHrT2\\n" + "pNsyS2s9/74Y8e63Ea7o8Y0uT3hPqskAnG09ockS8QKBgQDdQEvHo+XJfPF0f1OE\\n" + "1LrZYJvIwjh9/PNO5W42CYnsJ+KLlgq+ZIrJdUObPQk1Cp3juPPWJuLMtC8oVz1J\\n" + "wT9dbubnlzVcprDb3NVRrFRSChIdwauew7piyyBP7ciBGAtMtigr9rdhKeZUB5Hj\\n" + "ZIP9X5XCJtriHAw+MENLyTaTdQKBgQDG4QgD8uHfK36PrtTk4mE+8wWAI+efI+0T\\n" + "UNSSjMdxipDFZfeU1b4lXzlHBeAegLeQmPQFIoE5lmdAYcHOdPfy88GQnNG2Q0Ao\\n" + "bNp/IPIMjB3O+k31hhNWHnNl1diPYTV9z5DNlpuOhHqZtbANbx2QnS/4TvRgq657\\n" + "NFXW/NXHTQKBgG6Ms9Ca+jQE8/iLrkWOrZX0CaL0OJnrC/997+WcOof/Hdk1LUUY\\n" + "o6gpqZAlnTYdierA/UUhxO0XkwCLJpp1rp2WzlUlXope17vjycq3WqJrWcX4gTIh\\n" + "Bj5a1FhbrXWjd/HqioP9EH/CGc4ewixmivTND90k4PVdolhocRerAFQJAoGAI3NF\\n" + "XH7U6FT2cGI3rLz1nKTxHBBKX0GmJsVHvv+9JW4PtEAiy7L1++9nZFOVyZokHnBF\\n" + "Pw0Rf9Rhf0Ztp4GOGQ5+OGrbruN58jrFD9gtjTMEtTpE3zkRBU7UPxjJS3WGdXCk\\n" + "XSE1hUf0GqYaRarC2F5MiLR6NykjJu8DRhk3ehkCgYEAkNLfUrtgW2C94lFeFIm8\\n" + "JEUXNNJx40TWKOtenS4CsB0i+xuemjRzAPCmIRpRVwSoFZvXI0kkGd1W09hDO9DE\\n" + "G4BLoOcpJEN9OFp1lwi2xFgb9ibrtmDUM/+SuSr3w4oxKmK9RfDk7uTSQtxM4BO3\\n" + "MeqOwdgF5vKL/MNnxi319Ic=\\n" + "-----END PRIVATE KEY-----\\n"; + +constexpr char kTokenResponse[] = + R"({"access_token": "a-minted-token", "expires_in": 3600})"; + +std::string ServiceAccountJson() { + return fmt::format( + R"({{"client_email": "test@example.com", "private_key": "{}"}})", + kTestPrivateKey); +} + +// The scope a service account asks for travels inside the base64url claim set +// of the assertion it signs. +std::string AssertionClaims(std::string_view body) { + size_t assertion = body.find("assertion="); + if (assertion == std::string_view::npos) { + return ""; + } + std::vector parts = + absl::StrSplit(body.substr(assertion), '.'); + if (parts.size() < 2) { + return ""; + } + std::string claims = + absl::StrReplaceAll(parts[1], {{"-", "+"}, {"_", "/"}, {"%3D", "="}}); + Result> decoded = DecodeBase64(claims); + if (!decoded.has_value()) { + return ""; + } + return std::string(decoded->begin(), decoded->end()); +} + +class StorageCredentialTests : public ::testing::Test { + protected: + void SetUp() override { + previous_home_ = StringFromEnv("HOME", ""); + // The ~/.boto credential source reads $HOME, which must not be the one of + // whoever runs the test. + setenv("HOME", temp_dir_.path, /* overwrite */ 1); + } + + void TearDown() override { + setenv("HOME", previous_home_.c_str(), /* overwrite */ 1); + } + + std::string WriteTempFile(const std::string& name, + const std::string& contents) { + std::string path = fmt::format("{}/{}", temp_dir_.path, name); + EXPECT_THAT(WriteNewFile(path, contents), IsOk()); + return path; + } + + TemporaryDir temp_dir_; + std::string previous_home_; + FakeHttpClient http_client_; +}; + +TEST_F(StorageCredentialTests, + CredentialSourceTokenIsNotPresentedToStorageSuccess) { + BuildApiFlags flags; + flags.credential_source = WriteTempFile("credential", "a-build-api-token"); + + Result> credentials = + GetStorageCredentialSource(http_client_, flags, + /*running_on_gce=*/false); + + ASSERT_THAT(credentials, IsOk()); + EXPECT_THAT(credentials->get(), IsNull()); + EXPECT_FALSE(http_client_.RequestMade("metadata.google.internal")); +} + +TEST_F(StorageCredentialTests, CredentialSourceFileIsNeverReadSuccess) { + BuildApiFlags flags; + // A directory cannot be read as a file, so a credential source that opened + // this path would fail here instead of reaching anonymous access. + flags.credential_source = temp_dir_.path; + + Result> credentials = + GetStorageCredentialSource(http_client_, flags, + /*running_on_gce=*/false); + + ASSERT_THAT(credentials, IsOk()); + EXPECT_THAT(credentials->get(), IsNull()); +} + +TEST_F(StorageCredentialTests, + CredentialFilepathIsNotPresentedToStorageSuccess) { + BuildApiFlags flags; + flags.credential_flags.credential_filepath = + WriteTempFile("credential", "a-build-api-token"); + + Result> credentials = + GetStorageCredentialSource(http_client_, flags, + /*running_on_gce=*/false); + + ASSERT_THAT(credentials, IsOk()); + EXPECT_THAT(credentials->get(), IsNull()); +} + +TEST_F(StorageCredentialTests, ServiceAccountMintsAStorageScopedTokenSuccess) { + BuildApiFlags flags; + flags.credential_flags.service_account_filepath = + WriteTempFile("service_account.json", ServiceAccountJson()); + std::string claims; + http_client_.SetResponse( + [&claims](const HttpRequest& request) { + claims = AssertionClaims(request.data_to_write); + return HttpResponse{.data = kTokenResponse, + .http_code = 200}; + }, + "oauth2.googleapis.com/token"); + + Result> credentials = + GetStorageCredentialSource(http_client_, flags, + /*running_on_gce=*/false); + + ASSERT_THAT(credentials, IsOk()); + ASSERT_THAT(credentials->get(), NotNull()); + EXPECT_THAT((*credentials)->Credential(), IsOkAndValue("a-minted-token")); + EXPECT_THAT(claims, HasSubstr("devstorage.read_only")); + EXPECT_THAT(claims, Not(HasSubstr("androidbuild.internal"))); +} + +TEST_F(StorageCredentialTests, BotoFileOutranksTheMetadataServerSuccess) { + BuildApiFlags flags; + WriteTempFile(".boto", + "[OAuth2]\nclient_id = an-id\nclient_secret = a-secret\n" + "gs_oauth2_refresh_token = a-refresh-token\n"); + http_client_.SetResponse(kTokenResponse, "oauth2.googleapis.com/token"); + + Result> credentials = + GetStorageCredentialSource(http_client_, flags, /*running_on_gce=*/true); + + ASSERT_THAT(credentials, IsOk()); + ASSERT_THAT(credentials->get(), NotNull()); + EXPECT_THAT((*credentials)->Credential(), IsOkAndValue("a-minted-token")); + EXPECT_FALSE(http_client_.RequestMade("metadata.google.internal")); +} + +TEST_F(StorageCredentialTests, GceHostsGetAnAmbientCredentialSuccess) { + BuildApiFlags flags; + http_client_.SetResponse(kTokenResponse, "metadata.google.internal"); + + Result> credentials = + GetStorageCredentialSource(http_client_, flags, /*running_on_gce=*/true); + + ASSERT_THAT(credentials, IsOk()); + ASSERT_THAT(credentials->get(), NotNull()); + EXPECT_THAT((*credentials)->Credential(), IsOkAndValue("a-minted-token")); + EXPECT_TRUE(http_client_.RequestMade("metadata.google.internal")); +} + +TEST_F(StorageCredentialTests, + UseGceMetadataFlagQueriesTheMetadataServerSuccess) { + BuildApiFlags flags; + flags.credential_flags.use_gce_metadata = true; + http_client_.SetResponse(kTokenResponse, "metadata.google.internal"); + + Result> credentials = + GetStorageCredentialSource(http_client_, flags, + /*running_on_gce=*/false); + + ASSERT_THAT(credentials, IsOk()); + ASSERT_THAT(credentials->get(), NotNull()); + EXPECT_THAT((*credentials)->Credential(), IsOkAndValue("a-minted-token")); + EXPECT_TRUE(http_client_.RequestMade("metadata.google.internal")); +} + +TEST_F(StorageCredentialTests, LegacyGceValueQueriesTheMetadataServerSuccess) { + BuildApiFlags flags; + flags.credential_source = "gce"; + http_client_.SetResponse(kTokenResponse, "metadata.google.internal"); + + Result> credentials = + GetStorageCredentialSource(http_client_, flags, + /*running_on_gce=*/false); + + ASSERT_THAT(credentials, IsOk()); + ASSERT_THAT(credentials->get(), NotNull()); + EXPECT_THAT((*credentials)->Credential(), IsOkAndValue("a-minted-token")); + EXPECT_TRUE(http_client_.RequestMade("metadata.google.internal")); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp index 0046a75d3a7..3ac4e55e593 100644 --- a/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp @@ -122,6 +122,20 @@ TEST(GcsBuildApiTests, GetBuildMissingObjectFail) { HasSubstr("404"), HasSubstr("Check log file")))); } +TEST(GcsBuildApiTests, GetBuildAnonymousForbiddenNamesTheLoginCommandFail) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse( + HttpResponse{.data = "{}", .http_code = 403}, kObjectUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/phone-img-1.zip"}; + EXPECT_THAT( + api.GetBuild(build_string), + IsErrorAndMessage(HasSubstr("cvd login " + "--scopes=https://www.googleapis.com/auth/" + "devstorage.read_only"))); +} + TEST(GcsBuildApiTests, GetBuildEmptyPrefixFail) { FakeHttpClient http_client; GcsBuildApi api(http_client, nullptr); From ba84a2d6a6c5aee9f3db4c8b269718e10b65fd20 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:37 +0000 Subject: [PATCH 10/20] Serve `cvd fetch` from every build source --- .../host/commands/cvd/fetch/BUILD.bazel | 3 ++ .../host/commands/cvd/fetch/downloaders.cc | 53 ++++++++++++++----- .../host/commands/cvd/fetch/downloaders.h | 2 +- .../host/commands/cvd/fetch/fetch_cvd.cc | 12 ++--- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 1 - .../host/libs/web/android_build_api.cpp | 39 +++++--------- .../host/libs/web/android_build_api.h | 22 ++++---- .../host/libs/web/android_build_api_test.cpp | 2 +- 8 files changed, 75 insertions(+), 59 deletions(-) diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel index 567e438933f..a29f9bb8118 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel @@ -161,7 +161,10 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build_url", "//cuttlefish/host/libs/web:build_api", "//cuttlefish/host/libs/web:caching_build_api", + "//cuttlefish/host/libs/web:composite_build_api", "//cuttlefish/host/libs/web:credential_source", + "//cuttlefish/host/libs/web:gcs_build_api", + "//cuttlefish/host/libs/web:http_build_api", "//cuttlefish/host/libs/web:luci_build_api", "//cuttlefish/host/libs/web:oauth2_consent", "//cuttlefish/host/libs/web/http_client", diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/downloaders.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/downloaders.cc index fff493f50e2..644a22e3714 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/downloaders.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/downloaders.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "cuttlefish/common/libs/utils/environment.h" @@ -27,7 +28,10 @@ #include "cuttlefish/host/libs/web/android_build_url.h" #include "cuttlefish/host/libs/web/build_api.h" #include "cuttlefish/host/libs/web/caching_build_api.h" +#include "cuttlefish/host/libs/web/composite_build_api.h" #include "cuttlefish/host/libs/web/credential_source.h" +#include "cuttlefish/host/libs/web/gcs_build_api.h" +#include "cuttlefish/host/libs/web/http_build_api.h" #include "cuttlefish/host/libs/web/http_client/curl_http_client.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/host/libs/web/http_client/retrying_http_client.h" @@ -43,11 +47,12 @@ struct Downloaders::Impl { std::unique_ptr android_creds_; std::unique_ptr android_build_url_; std::unique_ptr cas_downloader_; - std::unique_ptr android_build_api_; - std::unique_ptr caching_build_api_; std::unique_ptr luci_credential_source_; std::unique_ptr gsutil_credential_source_; std::unique_ptr luci_build_api_; + std::unique_ptr storage_credential_source_; + std::unique_ptr composite_build_api_; + std::unique_ptr caching_build_api_; }; Downloaders::Downloaders(std::unique_ptr impl) @@ -87,15 +92,11 @@ Result Downloaders::Create(const BuildApiFlags& flags, impl->cas_downloader_ = std::move(cas_downloader_result.value()); } - impl->android_build_api_ = std::make_unique( - *impl->retrying_http_client_, *impl->android_build_url_, - impl->android_creds_.get(), flags.wait_retry_period, - impl->cas_downloader_.get()); - - if (flags.enable_caching) { - impl->caching_build_api_ = std::make_unique( - *impl->android_build_api_, cache_base_path); - } + std::unique_ptr android_build_api = + std::make_unique( + *impl->retrying_http_client_, *impl->android_build_url_, + impl->android_creds_.get(), flags.wait_retry_period, + impl->cas_downloader_.get()); impl->luci_credential_source_ = CF_EXPECT(GetCredentialSourceFromFlags( *impl->retrying_http_client_, flags, @@ -108,15 +109,39 @@ Result Downloaders::Create(const BuildApiFlags& flags, *impl->retrying_http_client_, impl->luci_credential_source_.get(), impl->gsutil_credential_source_.get()); + // Cloud Storage builds resolve a credential of their own. The gsutil ladder + // above stays as it is because the Luci downloads also accept a bare + // storage-scoped token minted outside `cvd`. + Result> storage_creds = + CredentialForScopes(*impl->curl_, {kCloudStorageReadScope}); + + impl->storage_credential_source_ = + storage_creds.has_value() && storage_creds->get() + ? std::move(*storage_creds) + : CF_EXPECT(GetStorageCredentialSource(*impl->retrying_http_client_, + flags, IsRunningOnGce())); + + impl->composite_build_api_ = std::make_unique( + std::move(android_build_api), + std::make_unique(*impl->retrying_http_client_, + impl->storage_credential_source_.get()), + std::make_unique(*impl->retrying_http_client_)); + + // One cache in front of every source, so that each build keys its artifacts + // the way its own source versions them. + if (flags.enable_caching) { + impl->caching_build_api_ = std::make_unique( + *impl->composite_build_api_, cache_base_path); + } + return Downloaders(std::move(impl)); } -BuildApi& Downloaders::AndroidBuild() { +BuildApi& Downloaders::Builds() { if (impl_->caching_build_api_) { return *impl_->caching_build_api_; - } else { - return *impl_->android_build_api_; } + return *impl_->composite_build_api_; } LuciBuildApi& Downloaders::Luci() { return *impl_->luci_build_api_; } diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/downloaders.h b/base/cvd/cuttlefish/host/commands/cvd/fetch/downloaders.h index 2d34c98d68f..71cef7bf884 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/downloaders.h +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/downloaders.h @@ -34,7 +34,7 @@ class Downloaders { Downloaders(Downloaders&&); ~Downloaders(); - BuildApi& AndroidBuild(); + BuildApi& Builds(); LuciBuildApi& Luci(); private: diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc index 151067731f5..a1b699115e2 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc @@ -464,13 +464,13 @@ Result Fetch(const FetchFlags& flags, FetchTracer tracer; FetchTracer::Trace prefetch_trace = tracer.NewTrace("PreFetch actions"); - CF_EXPECT(UpdateTargetsWithBuilds(downloaders.AndroidBuild(), targets)); + CF_EXPECT(UpdateTargetsWithBuilds(downloaders.Builds(), targets)); std::optional fallback_host_build = std::nullopt; if (!targets.empty()) { fallback_host_build = targets[0].builds.default_build; } - const auto host_target_build = CF_EXPECT(GetHostBuild( - downloaders.AndroidBuild(), host_target, fallback_host_build)); + const Build host_target_build = CF_EXPECT( + GetHostBuild(downloaders.Builds(), host_target, fallback_host_build)); prefetch_trace.CompletePhase("GetBuilds"); std::future> host_package_future; @@ -480,8 +480,8 @@ Result Fetch(const FetchFlags& flags, std::cref(host_target.host_tools_directory)); } else { host_package_future = std::async( - std::launch::async, FetchHostPackage, - std::ref(downloaders.AndroidBuild()), std::cref(host_target_build), + std::launch::async, FetchHostPackage, std::ref(downloaders.Builds()), + std::cref(host_target_build), std::cref(host_target.host_tools_directory), std::cref(flags.keep_downloaded_archives), std::cref(flags.host_substitutions), tracer.NewTrace("Host Package")); @@ -490,7 +490,7 @@ Result Fetch(const FetchFlags& flags, FetchResult fetch_result; for (const auto& target : targets) { FetcherConfig config; - FetchContext fetch_context(downloaders.AndroidBuild(), target.directories, + FetchContext fetch_context(downloaders.Builds(), target.directories, target.builds, config, tracer); LOG(INFO) << "Starting fetch to \"" << target.directories.root << "\""; CF_EXPECT(FetchTarget(fetch_context, target.download_flags, diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index e73b2dd43d3..c6ee881ad82 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -46,7 +46,6 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build_api_key", "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:android_build_url", - "//cuttlefish/host/libs/web:build_api", "//cuttlefish/host/libs/web:credential_source", "//cuttlefish/host/libs/web:parse_time", "//cuttlefish/host/libs/web/cas:cas_downloader", diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp index 13d406232ae..7768754aa56 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_api.cpp @@ -35,7 +35,6 @@ #include "absl/log/log.h" #include "android-base/file.h" #include "fmt/format.h" -#include "fmt/ostream.h" #include "json/value.h" #include "cuttlefish/common/libs/utils/contains.h" @@ -94,7 +93,8 @@ AndroidBuildApi::AndroidBuildApi(HttpClient& http_client, retry_period_(retry_period), cas_downloader_(cas_downloader) {} -Result AndroidBuildApi::GetBuild(const DeviceBuildString& build_string) { +Result AndroidBuildApi::GetBuild( + const DeviceBuildString& build_string) { CF_EXPECT( build_string.target.has_value(), "Given build string must have a target with the branch or build id"); @@ -126,25 +126,25 @@ Result AndroidBuildApi::GetBuild(const DeviceBuildString& build_string) { }; } -Result AndroidBuildApi::GetBuild( +Result AndroidBuildApi::GetBuild( const DirectoryBuildString& build_string) { return DirectoryBuild(build_string.paths, build_string.target, build_string.filepath); } -Result AndroidBuildApi::GetBuild(const BuildString& build_string) { - if (const auto* device = std::get_if(&build_string)) { - return CF_EXPECT(GetBuild(*device)); - } - if (const auto* directory = - std::get_if(&build_string)) { - return CF_EXPECT(GetBuild(*directory)); - } - return CF_ERRF("AndroidBuildApi cannot handle '{}'", - fmt::streamed(build_string)); +Result AndroidBuildApi::DownloadFile( + const DeviceBuild& build, const std::string& target_directory, + const std::string& artifact_name) { + return CF_EXPECT(DownloadArtifact(build, target_directory, artifact_name)); } Result AndroidBuildApi::DownloadFile( + const DirectoryBuild& build, const std::string& target_directory, + const std::string& artifact_name) { + return CF_EXPECT(DownloadArtifact(build, target_directory, artifact_name)); +} + +Result AndroidBuildApi::DownloadArtifact( const Build& build, const std::string& target_directory, const std::string& artifact_name) { std::unordered_set artifacts = @@ -154,17 +154,6 @@ Result AndroidBuildApi::DownloadFile( return DownloadTargetFile(build, target_directory, artifact_name); } -Result AndroidBuildApi::FileReader( - const Build& build, const std::string& artifact_name) { - if (const auto* device = std::get_if(&build)) { - return CF_EXPECT(FileReader(*device, artifact_name)); - } - if (const auto* directory = std::get_if(&build)) { - return CF_EXPECT(FileReader(*directory, artifact_name)); - } - return CF_ERRF("AndroidBuildApi cannot handle '{}'", FetchLabel(build)); -} - Result AndroidBuildApi::FileReader( const DeviceBuild& build, const std::string& artifact_name) { std::string url = CF_EXPECT(GetArtifactDownloadUrl(build, artifact_name)); @@ -445,7 +434,7 @@ Result AndroidBuildApi::DownloadTargetFileFromCas( DigestsFetcher digests_fetcher = [&build, &target_directory, this](std::string filename) -> Result { - CF_EXPECTF(DownloadFile(build, target_directory, filename), + CF_EXPECTF(DownloadArtifact(build, target_directory, filename), "Failed to download '{}' from AB.", filename); return ConstructTargetFilepath(target_directory, filename); }; diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_api.h b/base/cvd/cuttlefish/host/libs/web/android_build_api.h index 56fa9a1ccd5..a98afd274d8 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_api.h +++ b/base/cvd/cuttlefish/host/libs/web/android_build_api.h @@ -25,7 +25,6 @@ #include "cuttlefish/host/libs/web/android_build.h" #include "cuttlefish/host/libs/web/android_build_string.h" #include "cuttlefish/host/libs/web/android_build_url.h" -#include "cuttlefish/host/libs/web/build_api.h" #include "cuttlefish/host/libs/web/cas/cas_downloader.h" #include "cuttlefish/host/libs/web/credential_source.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" @@ -34,7 +33,7 @@ namespace cuttlefish { -class AndroidBuildApi : public BuildApi { +class AndroidBuildApi { public: AndroidBuildApi() = delete; AndroidBuildApi(AndroidBuildApi&&) = delete; @@ -45,17 +44,15 @@ class AndroidBuildApi : public BuildApi { std::chrono::seconds retry_period = std::chrono::seconds::zero(), CasDownloader* cas_downloader = nullptr); - Result GetBuild(const BuildString& build_string) override; + Result GetBuild(const DeviceBuildString& build_string); + Result GetBuild(const DirectoryBuildString& build_string); - Result GetBuild(const DeviceBuildString& build_string); - Result GetBuild(const DirectoryBuildString& build_string); - - Result DownloadFile(const Build& build, + Result DownloadFile(const DeviceBuild& build, const std::string& target_directory, - const std::string& artifact_name) override; - - Result FileReader( - const Build&, const std::string& artifact_name) override; + const std::string& artifact_name); + Result DownloadFile(const DirectoryBuild& build, + const std::string& target_directory, + const std::string& artifact_name); Result FileReader(const DeviceBuild&, const std::string& artifact_name); @@ -111,6 +108,9 @@ class AndroidBuildApi : public BuildApi { Result ArtifactToFile(const Build& build, const std::string& artifact, const std::string& path); + Result DownloadArtifact(const Build& build, + const std::string& target_directory, + const std::string& artifact_name); Result DownloadTargetFile(const Build& build, const std::string& target_directory, const std::string& artifact_name); diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_api_test.cpp index 4b70d769df7..ae719a670b0 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_api_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_api_test.cpp @@ -50,7 +50,7 @@ TEST(AndroidBuildApiTest, FileReader) { }; http_client.SetResponse(res, "http://zip-url"); - Build build = DeviceBuild{.id = "123", .target = "test"}; + DeviceBuild build = {.id = "123", .target = "test"}; Result source = api.FileReader(build, "a.zip"); ASSERT_THAT(source, IsOk()); From ea780a35c4544345c6b8c198468d4ba0ee9975e5 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:38 +0000 Subject: [PATCH 11/20] Resolve build zip names from a URL build's namespace --- .../host/commands/cvd/fetch/BUILD.bazel | 21 +++ .../host/commands/cvd/fetch/fetch_context.cc | 24 ++- .../host/commands/cvd/fetch/fetch_context.h | 3 +- .../commands/cvd/fetch/fetch_context_test.cpp | 140 ++++++++++++++++++ .../host/commands/cvd/fetch/fetch_cvd.cc | 56 ++++++- .../host/commands/cvd/fetch/host_package.cc | 6 +- .../host/commands/cvd/fetch/host_package.h | 4 + base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 3 +- .../cuttlefish/host/libs/web/android_build.cc | 9 ++ .../cuttlefish/host/libs/web/android_build.h | 4 + .../host/libs/web/gcs_build_api.cpp | 23 +-- .../host/libs/web/http_build_api.cpp | 11 +- .../host/libs/web/url_namespace.cpp | 7 + .../cuttlefish/host/libs/web/url_namespace.h | 6 + .../host/libs/web/url_namespace_test.cpp | 12 ++ 15 files changed, 288 insertions(+), 41 deletions(-) create mode 100644 base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context_test.cpp diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel index a29f9bb8118..45c9849144d 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/BUILD.bazel @@ -220,6 +220,7 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build", "//cuttlefish/host/libs/web:build_api", "//cuttlefish/host/libs/web:build_api_zip", + "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/zip:zip_file", "//cuttlefish/host/libs/zip/libzip_cc:archive", "//cuttlefish/posix:remove", @@ -230,6 +231,25 @@ cf_cc_library( ], ) +cf_cc_test( + name = "fetch_context_test", + srcs = ["fetch_context_test.cpp"], + deps = [ + "//cuttlefish/host/commands/cvd/fetch:builds", + "//cuttlefish/host/commands/cvd/fetch:fetch_context", + "//cuttlefish/host/commands/cvd/fetch:fetch_tracer", + "//cuttlefish/host/commands/cvd/fetch:target_directories", + "//cuttlefish/host/libs/config:fetcher_config", + "//cuttlefish/host/libs/web:android_build", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/host/libs/web:build_api", + "//cuttlefish/host/libs/web:url_namespace", + "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + ], +) + cf_cc_library( name = "fetch_cvd", srcs = ["fetch_cvd.cc"], @@ -259,6 +279,7 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:chrome_os_build_string", "//cuttlefish/host/libs/web:luci_build_api", + "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/web/http_client:curl_global_init", "//cuttlefish/host/libs/zip/libzip_cc:archive", "//cuttlefish/io", diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.cc index dac9bf5a996..8130cd7076e 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.cc @@ -41,6 +41,7 @@ #include "cuttlefish/host/libs/web/android_build.h" #include "cuttlefish/host/libs/web/build_api.h" #include "cuttlefish/host/libs/web/build_api_zip.h" +#include "cuttlefish/host/libs/web/url_namespace.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" #include "cuttlefish/host/libs/zip/zip_file.h" #include "cuttlefish/posix/remove.h" @@ -182,11 +183,26 @@ FetchBuildContext::FetchBuildContext(FetchContext& fetch_context, const cuttlefish::Build& FetchBuildContext::Build() const { return build_; } -std::string FetchBuildContext::GetBuildZipName(const std::string& name) const { - std::string product = +Result FetchBuildContext::GetBuildZipName( + BuildZipKind kind) const { + if (const GcsBuild* gcs = std::get_if(&build_)) { + if (gcs->object.has_value()) { + return CF_EXPECT(ResolveUrlZipName(*gcs->object, kind)); + } + return CF_EXPECT(ResolveUrlZipName(GcsArtifactNames(*gcs), kind)); + } + if (const HttpBuild* http = std::get_if(&build_)) { + CF_EXPECTF(http->object.has_value(), + "Cannot discover the '{}' zip of a directory that has no " + "listing. Name the archive itself in the URL, or use a " + "'gs://' URL, which lists its contents.", + kind); + return CF_EXPECT(ResolveUrlZipName(*http->object, kind)); + } + const std::string product = std::visit([](auto&& arg) { return arg.product; }, build_); - std::string id = std::get<0>(GetBuildIdAndTarget(build_)); - return product + "-" + name + "-" + id + ".zip"; + const std::string id = std::get<0>(GetBuildIdAndTarget(build_)); + return fmt::format("{}-{}-{}.zip", product, kind, id); } std::optional FetchBuildContext::GetFilepath() const { diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.h b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.h index da667247cef..36835061483 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.h +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context.h @@ -28,6 +28,7 @@ #include "cuttlefish/host/libs/config/file_source.h" #include "cuttlefish/host/libs/web/android_build.h" #include "cuttlefish/host/libs/web/build_api.h" +#include "cuttlefish/host/libs/web/url_namespace.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" #include "cuttlefish/result/result.h" @@ -78,7 +79,7 @@ class FetchArtifact { class FetchBuildContext { public: const cuttlefish::Build& Build() const; - std::string GetBuildZipName(const std::string&) const; + Result GetBuildZipName(BuildZipKind kind) const; // The specific filepath the user requested for a particular build. Ignored // for some builds. std::optional GetFilepath() const; diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context_test.cpp b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context_test.cpp new file mode 100644 index 00000000000..9dbfe0707a2 --- /dev/null +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_context_test.cpp @@ -0,0 +1,140 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/commands/cvd/fetch/fetch_context.h" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/host/commands/cvd/fetch/builds.h" +#include "cuttlefish/host/commands/cvd/fetch/fetch_tracer.h" +#include "cuttlefish/host/commands/cvd/fetch/target_directories.h" +#include "cuttlefish/host/libs/config/fetcher_config.h" +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/build_api.h" +#include "cuttlefish/host/libs/web/url_namespace.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::HasSubstr; + +class MockBuildApi : public BuildApi { + public: + MOCK_METHOD(Result, GetBuild, (const BuildString&), (override)); + MOCK_METHOD(Result, DownloadFile, + (const Build&, const std::string&, const std::string&), + (override)); + MOCK_METHOD(Result, FileReader, + (const Build&, const std::string&), (override)); +}; + +TEST(FetchContextTest, GetBuildZipNameGcsObject) { + MockBuildApi mock_api; + TargetDirectories target_directories; + Builds builds = {.default_build = GcsBuild{.bucket = "bucket", + .object = "phone-img-1.zip"}}; + FetcherConfig fetcher_config; + FetchTracer tracer; + FetchContext fetch_context(mock_api, target_directories, builds, + fetcher_config, tracer); + + std::optional context = fetch_context.DefaultBuild(); + + ASSERT_TRUE(context.has_value()); + ASSERT_THAT(context->GetBuildZipName(BuildZipKind::kImages), + IsOkAndValue("phone-img-1.zip")); +} + +TEST(FetchContextTest, GetBuildZipNameGcsListing) { + MockBuildApi mock_api; + TargetDirectories target_directories; + Builds builds = { + .default_build = GcsBuild{ + .bucket = "bucket", + .contents = {{"misc_info.txt", {}}, {"phone-img-1.zip", {}}}}}; + FetcherConfig fetcher_config; + FetchTracer tracer; + FetchContext fetch_context(mock_api, target_directories, builds, + fetcher_config, tracer); + + std::optional context = fetch_context.DefaultBuild(); + + ASSERT_TRUE(context.has_value()); + ASSERT_THAT(context->GetBuildZipName(BuildZipKind::kImages), + IsOkAndValue("phone-img-1.zip")); +} + +TEST(FetchContextTest, GetBuildZipNameHttpObject) { + MockBuildApi mock_api; + TargetDirectories target_directories; + Builds builds = {.default_build = HttpBuild{ + .url = "https://example.com/dist/phone-img-1.zip", + .object = "phone-img-1.zip"}}; + FetcherConfig fetcher_config; + FetchTracer tracer; + FetchContext fetch_context(mock_api, target_directories, builds, + fetcher_config, tracer); + + std::optional context = fetch_context.DefaultBuild(); + + ASSERT_TRUE(context.has_value()); + ASSERT_THAT(context->GetBuildZipName(BuildZipKind::kImages), + IsOkAndValue("phone-img-1.zip")); +} + +TEST(FetchContextTest, GetBuildZipNameHttpDirectoryNoListing) { + MockBuildApi mock_api; + TargetDirectories target_directories; + Builds builds = {.default_build = + HttpBuild{.url = "https://example.com/dist/"}}; + FetcherConfig fetcher_config; + FetchTracer tracer; + FetchContext fetch_context(mock_api, target_directories, builds, + fetcher_config, tracer); + + std::optional context = fetch_context.DefaultBuild(); + + ASSERT_TRUE(context.has_value()); + ASSERT_THAT(context->GetBuildZipName(BuildZipKind::kImages), + IsErrorAndMessage(HasSubstr("gs://"))); +} + +TEST(FetchContextTest, GetBuildZipNameAndroidBuild) { + MockBuildApi mock_api; + TargetDirectories target_directories; + Builds builds = {.default_build = DeviceBuild{ + .id = "12345", .target = "test", .product = "phone"}}; + FetcherConfig fetcher_config; + FetchTracer tracer; + FetchContext fetch_context(mock_api, target_directories, builds, + fetcher_config, tracer); + + std::optional context = fetch_context.DefaultBuild(); + + ASSERT_TRUE(context.has_value()); + ASSERT_THAT(context->GetBuildZipName(BuildZipKind::kTargetFiles), + IsOkAndValue("phone-target_files-12345.zip")); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc index a1b699115e2..be65415e84e 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc @@ -28,6 +28,7 @@ #include #include "absl/log/log.h" +#include "absl/strings/str_join.h" #include "absl/strings/str_split.h" #include "cuttlefish/common/libs/utils/archive.h" @@ -53,6 +54,7 @@ #include "cuttlefish/host/libs/web/chrome_os_build_string.h" #include "cuttlefish/host/libs/web/http_client/curl_global_init.h" #include "cuttlefish/host/libs/web/luci_build_api.h" +#include "cuttlefish/host/libs/web/url_namespace.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" #include "cuttlefish/io/io.h" #include "cuttlefish/io/string.h" @@ -172,6 +174,43 @@ Result GetHostBuild(BuildApi& build_api, "aosp_cf_x86_64_only_phone-userdebug"); } +template +Result CheckObjectIsNamed(const UrlBuild& build, + const std::string& name) { + CF_EXPECTF(name == *build.object || + IsArchiveMember(build.object, build.filepath, name), + "The build '{}' holds only '{}', so it has no '{}'.", build.id, + *build.object, name); + return {}; +} + +// The host package is fetched asynchronously, so without this check a build +// that has none only says so after every target has been fetched. +Result CheckHostPackagePresent(const Build& build) { + const std::string name = HostPackageName(build); + if (const GcsBuild* gcs = std::get_if(&build)) { + if (gcs->object.has_value()) { + CF_EXPECT(CheckObjectIsNamed(*gcs, name), + "Name a build that has a host package with " + "`--host_package_build`."); + } else { + CF_EXPECTF(Contains(gcs->contents, name), + "The build '{}' has no host package '{}'. It holds [{}]. " + "Name a build that has one with `--host_package_build`.", + gcs->id, name, absl::StrJoin(GcsArtifactNames(*gcs), ", ")); + } + } else if (const HttpBuild* http = std::get_if(&build)) { + // A plain HTTPS directory has no listing, so its host package is only + // known to be absent when the download answers 404. + if (http->object.has_value()) { + CF_EXPECT(CheckObjectIsNamed(*http, name), + "Name a build that has a host package with " + "`--host_package_build`."); + } + } + return {}; +} + Result SaveConfig(FetcherConfig& config, const std::string& target_directory) { // Due to constraints of the build system, artifacts intentionally cannot @@ -204,7 +243,8 @@ Result FetchDefaultTarget(FetchBuildContext& context, } if (flags.download_img_zip) { LOG(INFO) << "Downloading image zip for " << context; - std::string img_zip_name = context.GetBuildZipName("img"); + const std::string img_zip_name = + CF_EXPECT(context.GetBuildZipName(BuildZipKind::kImages)); std::string img_zip_artifact_name = img_zip_name; if (IsSignedBuild(context.Build())) { img_zip_artifact_name = kSignedPrefix + img_zip_artifact_name; @@ -221,7 +261,8 @@ Result FetchDefaultTarget(FetchBuildContext& context, const bool download_target_files = has_system_build || flags.download_target_files_zip; if (download_target_files || flags.dynamic_super_image) { - std::string target_files_name = context.GetBuildZipName("target_files"); + const std::string target_files_name = + CF_EXPECT(context.GetBuildZipName(BuildZipKind::kTargetFiles)); FetchArtifact target_files = context.Artifact(target_files_name); if (download_target_files) { LOG(INFO) << "Downloading target files zip for " << context; @@ -259,7 +300,8 @@ Result FetchDefaultTarget(FetchBuildContext& context, Result FetchSystemTarget(FetchBuildContext& context, bool download_img_zip, const bool keep_downloaded_archives) { - std::string target_files_name = context.GetBuildZipName("target_files"); + const std::string target_files_name = + CF_EXPECT(context.GetBuildZipName(BuildZipKind::kTargetFiles)); FetchArtifact target_files = context.Artifact(target_files_name); CF_EXPECT( @@ -271,7 +313,8 @@ Result FetchSystemTarget(FetchBuildContext& context, .has_value()) { LOG(INFO) << "Unable to retrieve system.img from target files, falling " "back to system *-img-*.zip for system image"; - std::string system_img_zip_name = context.GetBuildZipName("img"); + const std::string system_img_zip_name = + CF_EXPECT(context.GetBuildZipName(BuildZipKind::kImages)); FetchArtifact system_files = context.Artifact(system_img_zip_name); CF_EXPECT(system_files.Download()); @@ -320,7 +363,9 @@ Result FetchBootTarget(FetchBuildContext& context, bool keep_downloaded_archives) { const std::optional filepath = context.GetFilepath(); const std::string to_download = - filepath.has_value() ? *filepath : context.GetBuildZipName("img"); + filepath.has_value() + ? *filepath + : CF_EXPECT(context.GetBuildZipName(BuildZipKind::kImages)); FetchArtifact artifact = context.Artifact(to_download); CF_EXPECT(artifact.Download()); @@ -479,6 +524,7 @@ Result Fetch(const FetchFlags& flags, std::async(std::launch::async, SymlinkHostPackage, std::cref(host_target.host_tools_directory)); } else { + CF_EXPECT(CheckHostPackagePresent(host_target_build)); host_package_future = std::async( std::launch::async, FetchHostPackage, std::ref(downloaders.Builds()), std::cref(host_target_build), diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.cc index 50ac1808a50..9d6c28d7d50 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.cc @@ -34,6 +34,10 @@ namespace cuttlefish { +std::string HostPackageName(const Build& build) { + return GetFilepath(build).value_or("cvd-host_package.tar.gz"); +} + Result FetchHostPackage( BuildApi& build_api, const Build& build, const std::string& target_dir, const bool keep_archives, @@ -46,7 +50,7 @@ Result FetchHostPackage( // The download time will still include time spent waiting for the mutex in // the build_api though. trace.CompletePhase("Async start delay"); - auto host_tools_name = GetFilepath(build).value_or("cvd-host_package.tar.gz"); + const std::string host_tools_name = HostPackageName(build); std::string host_tools_filepath = CF_EXPECT(build_api.DownloadFile(build, target_dir, host_tools_name)); trace.CompletePhase("Download", FileSize(host_tools_filepath)); diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.h b/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.h index 0331f3bf72d..6e10b07abb4 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.h +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/host_package.h @@ -25,6 +25,10 @@ namespace cuttlefish { +// Returns the artifact name the host package is downloaded from, which is the +// filepath of the build string when it names one. +std::string HostPackageName(const Build& build); + Result FetchHostPackage( BuildApi& build_api, const Build& build, const std::string& target_dir, bool keep_archives, const std::vector& host_substitutions, diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index c6ee881ad82..bba6afbbdeb 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -256,6 +256,7 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:build_api_zip", "//cuttlefish/host/libs/web:credential_source", + "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/web/http_client", "//cuttlefish/host/libs/web/http_client:http_file", "//cuttlefish/host/libs/web/http_client:http_json", @@ -265,7 +266,6 @@ cf_cc_library( "//cuttlefish/host/libs/zip/libzip_cc:archive", "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", "//cuttlefish/result", - "@abseil-cpp//absl/log", "@abseil-cpp//absl/strings", "@fmt", "@jsoncpp", @@ -304,6 +304,7 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build", "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:build_api_zip", + "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/web/http_client", "//cuttlefish/host/libs/web/http_client:http_file", "//cuttlefish/host/libs/zip:remote_zip", diff --git a/base/cvd/cuttlefish/host/libs/web/android_build.cc b/base/cvd/cuttlefish/host/libs/web/android_build.cc index c76b6fa0286..ddd7d13711e 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build.cc +++ b/base/cvd/cuttlefish/host/libs/web/android_build.cc @@ -98,6 +98,15 @@ std::ostream& operator<<(std::ostream& out, const GcsBuild& build) { << build.filepath.value_or("") << "\")"; } +std::vector GcsArtifactNames(const GcsBuild& build) { + std::vector names; + names.reserve(build.contents.size()); + for (const auto& entry : build.contents) { + names.push_back(entry.first); + } + return names; +} + Result HttpBuild::FromBuildString( const HttpBuildString& build_string) { const ParsedUrl url = CF_EXPECT(ParseUrl(build_string.url)); diff --git a/base/cvd/cuttlefish/host/libs/web/android_build.h b/base/cvd/cuttlefish/host/libs/web/android_build.h index 2c5e5577cc4..640783d4c98 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build.h +++ b/base/cvd/cuttlefish/host/libs/web/android_build.h @@ -90,6 +90,10 @@ struct GcsBuild { std::ostream& operator<<(std::ostream&, const GcsBuild&); +// The artifacts of a `gs://` directory build, in the order the listing keeps +// them. +std::vector GcsArtifactNames(const GcsBuild& build); + // The same two forms over `https://`, where a pre-signed URL carries its // credential in the query string of `url`. struct HttpBuild { diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp index 674340ca353..c06c32e78d0 100644 --- a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp @@ -24,10 +24,7 @@ #include #include -#include "absl/log/log.h" -#include "absl/strings/match.h" #include "absl/strings/numbers.h" -#include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/strip.h" #include "fmt/format.h" @@ -44,6 +41,7 @@ #include "cuttlefish/host/libs/web/http_client/http_file.h" #include "cuttlefish/host/libs/web/http_client/http_json.h" #include "cuttlefish/host/libs/web/http_client/url_escape.h" +#include "cuttlefish/host/libs/web/url_namespace.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" #include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" #include "cuttlefish/host/libs/zip/remote_zip.h" @@ -84,21 +82,6 @@ std::string ListUrl(const std::string& bucket, const std::string& prefix, return url; } -std::string ArtifactNames(const GcsBuild& build) { - return absl::StrJoin(build.contents, ", ", - [](std::string* out, const auto& entry) { - absl::StrAppend(out, entry.first); - }); -} - -// The object form names one archive, so `{selector}` names a member of it -// rather than a second artifact of the build. -bool IsArchiveMember(const GcsBuild& build, const std::string& artifact_name) { - return build.object.has_value() && artifact_name != *build.object && - absl::EndsWith(*build.object, ".zip") && - build.filepath == artifact_name; -} - Result ObjectName(const GcsBuild& build, const std::string& artifact_name) { if (build.object.has_value()) { @@ -108,7 +91,7 @@ Result ObjectName(const GcsBuild& build, } else { CF_EXPECTF(Contains(build.contents, artifact_name), "The build '{}' has no '{}'. It holds [{}].", build.id, - artifact_name, ArtifactNames(build)); + artifact_name, absl::StrJoin(GcsArtifactNames(build), ", ")); } return build.prefix + artifact_name; } @@ -239,7 +222,7 @@ Result GcsBuildApi::DownloadFile( ConstructTargetFilepath(target_directory, artifact_name); CF_EXPECT(EnsureDirectoryExists(target_directory)); - if (IsArchiveMember(build, artifact_name)) { + if (IsArchiveMember(build.object, build.filepath, artifact_name)) { SeekableZipSource source = CF_EXPECT(FileReader(build, *build.object)); ReadableZip zip = CF_EXPECT(OpenZip(std::move(source))); CF_EXPECTF(ExtractFile(zip, artifact_name, dest_path), diff --git a/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp index 5e88b45b08d..a554bb88a1c 100644 --- a/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp @@ -32,6 +32,7 @@ #include "cuttlefish/host/libs/web/build_api_zip.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/host/libs/web/http_client/http_file.h" +#include "cuttlefish/host/libs/web/url_namespace.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" #include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" #include "cuttlefish/host/libs/zip/remote_zip.h" @@ -43,14 +44,6 @@ namespace { constexpr long kPartialContent = 206; -// The object form names one archive, so `{selector}` names a member of it -// rather than a second artifact of the build. -bool IsArchiveMember(const HttpBuild& build, const std::string& artifact_name) { - return build.object.has_value() && artifact_name != *build.object && - absl::EndsWith(*build.object, ".zip") && - build.filepath == artifact_name; -} - Result ArtifactUrl(const HttpBuild& build, const std::string& artifact_name) { if (build.object.has_value()) { @@ -143,7 +136,7 @@ Result HttpBuildApi::DownloadFile( ConstructTargetFilepath(target_directory, artifact_name); CF_EXPECT(EnsureDirectoryExists(target_directory)); - if (IsArchiveMember(build, artifact_name)) { + if (IsArchiveMember(build.object, build.filepath, artifact_name)) { CF_EXPECTF(build.accept_ranges == true, "'{}' does not serve range requests, so '{}' cannot be read out " "of it.", diff --git a/base/cvd/cuttlefish/host/libs/web/url_namespace.cpp b/base/cvd/cuttlefish/host/libs/web/url_namespace.cpp index edcbec9f6d4..d0359b3ec11 100644 --- a/base/cvd/cuttlefish/host/libs/web/url_namespace.cpp +++ b/base/cvd/cuttlefish/host/libs/web/url_namespace.cpp @@ -125,6 +125,13 @@ Result ResolveUrlZipName(std::string_view object, return std::string(object); } +bool IsArchiveMember(const std::optional& object, + const std::optional& filepath, + const std::string& artifact_name) { + return object.has_value() && artifact_name != *object && + absl::EndsWith(*object, ".zip") && filepath == artifact_name; +} + Result ResolveUrlZipName(const std::vector& names, BuildZipKind kind) { std::vector matches; diff --git a/base/cvd/cuttlefish/host/libs/web/url_namespace.h b/base/cvd/cuttlefish/host/libs/web/url_namespace.h index b5067c1ecd7..78861242a22 100644 --- a/base/cvd/cuttlefish/host/libs/web/url_namespace.h +++ b/base/cvd/cuttlefish/host/libs/web/url_namespace.h @@ -46,6 +46,12 @@ Result ParseUrl(std::string_view url); // Android Build convention. std::optional DeriveProduct(std::string_view basename); +// Returns true when the object form names one archive, so that `{selector}` +// names a member of it rather than a second artifact of the build. +bool IsArchiveMember(const std::optional& object, + const std::optional& filepath, + const std::string& artifact_name); + // The kinds of zip archive a fetch resolves by name. enum class BuildZipKind { kImages, diff --git a/base/cvd/cuttlefish/host/libs/web/url_namespace_test.cpp b/base/cvd/cuttlefish/host/libs/web/url_namespace_test.cpp index 473be69f8ca..94c931bf5c1 100644 --- a/base/cvd/cuttlefish/host/libs/web/url_namespace_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/url_namespace_test.cpp @@ -16,6 +16,7 @@ #include "cuttlefish/host/libs/web/url_namespace.h" #include +#include #include #include @@ -104,6 +105,17 @@ TEST(DeriveProductTests, OtherNamesFail) { EXPECT_EQ(DeriveProduct("phone-img-12345.tar.gz"), std::nullopt); } +TEST(IsArchiveMemberTests, SelectorNamesAMemberSuccess) { + EXPECT_TRUE(IsArchiveMember("images.zip", "boot.img", "boot.img")); +} + +TEST(IsArchiveMemberTests, OtherArtifactsAreNotMembersSuccess) { + EXPECT_FALSE(IsArchiveMember("images.zip", std::nullopt, "boot.img")); + EXPECT_FALSE(IsArchiveMember("images.zip", "boot.img", "images.zip")); + EXPECT_FALSE(IsArchiveMember("host.tar.gz", "boot.img", "boot.img")); + EXPECT_FALSE(IsArchiveMember(std::nullopt, "boot.img", "boot.img")); +} + TEST(ResolveUrlZipNameTests, ObjectNamingTheKindSuccess) { EXPECT_THAT(ResolveUrlZipName("phone-img-1.zip", BuildZipKind::kImages), IsOkAndValue("phone-img-1.zip")); From a0153e1f0ecf52ec7d347e4d47f1feafbf16f21a Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:38 +0000 Subject: [PATCH 12/20] Pass URL build sources through `cvd load` --- .../cvd/cli/commands/load_configs.cpp | 18 +- .../host/commands/cvd/cli/parser/BUILD.bazel | 2 + .../cvd/cli/parser/fetch_config_parser.cpp | 31 ++- .../cli/parser/fetch_config_parser_test.cc | 254 +++++++++++++++++- 4 files changed, 295 insertions(+), 10 deletions(-) diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp index 8d139aa47b9..05821dc3bca 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp @@ -292,10 +292,14 @@ std::vector LoadConfigsCommand::CommonCommandDescription() { description.emplace_back( "While most config file properties are self explanatory, the build " "properties (default_build, kernel.build, bootloader.build, etc) require " - "more explanation. These properties support two types of values:"); + "more explanation. These properties support the following values:"); description.emplace_back(HelpParagraph::Raw( R"( - "@ab/[/[{}]]" + - "gs:////" + - "gs:///[{}]" + - "https:////" + - "https:///[{}]" - "")")); description.emplace_back( @@ -307,6 +311,18 @@ std::vector LoadConfigsCommand::CommonCommandDescription() { "braces. For more information on build fetching and caching operations " "refer to `cvd help fetch`."); + description.emplace_back( + "A \"gs://\" or \"https://\" value names a build outside the Android " + "build servers. A URL ending in '/' names the directory holding the " + "build's artifacts, which cvd picks from by name; a URL naming a single " + "object is a build of just that artifact, and then selects a " + "member of it if it is a zip. A \"gs://\" build is read with the " + "credentials `cvd fetch` would use for the build servers, while an " + "\"https://\" build is read as given, so any credential has to travel " + "in the URL itself as in a pre-signed URL. A directory URL over plain " + "\"https://\" cannot be listed, so a build named that way needs the " + "host package named separately."); + description.emplace_back( "Alternatively, the build value may point to an absolute path (starts " "with '/') in the filesystem where the Android source code has been " diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/BUILD.bazel b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/BUILD.bazel index bfd427a02b2..0b9f2115eee 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/BUILD.bazel +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/BUILD.bazel @@ -75,6 +75,8 @@ cf_cc_library( "//cuttlefish/host/commands/cvd/cli/parser:cf_configs_common", "//cuttlefish/host/commands/cvd/cli/parser:load_config_cc_proto", "//cuttlefish/host/commands/cvd/fetch:fetch_cvd_parser", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/host/libs/web/http_client:scrub_secrets", "//cuttlefish/result", "@abseil-cpp//absl/strings", ], diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser.cpp index 5ad0bec4967..db277cc522b 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser.cpp @@ -19,13 +19,17 @@ #include #include #include +#include #include +#include "absl/strings/match.h" #include "absl/strings/strip.h" #include "cuttlefish/host/commands/cvd/cli/parser/cf_configs_common.h" #include "cuttlefish/host/commands/cvd/cli/parser/load_config.pb.h" #include "cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" #include "cuttlefish/result/result.h" namespace cuttlefish { @@ -56,15 +60,26 @@ bool ShouldFetch(const Instance& instance) { Result GetFetchBuildString(const std::string& strVal) { std::string_view view = strVal; - if (!absl::ConsumePrefix(&view, kFetchPrefix)) { - // intentionally return an empty string when there are local, non-prefixed - // paths. Fetch does not process the local paths - return ""; + if (absl::ConsumePrefix(&view, kFetchPrefix)) { + CF_EXPECTF(!view.empty(), + "\"{}\" prefixed build string was not followed by a value", + kFetchPrefix); + CF_EXPECT(ParseBuildString(view)); + return std::string(view); } - CF_EXPECTF(!view.empty(), - "\"{}\" prefixed build string was not followed by a value", - kFetchPrefix); - return std::string(view); + if (absl::StrContains(strVal, "://")) { + const BuildString parsed = CF_EXPECT(ParseBuildString(strVal)); + // A local path can hold a "://" without naming a URL scheme, and parses + // as a directory build string. + CF_EXPECTF(std::holds_alternative(parsed) || + std::holds_alternative(parsed), + "'{}' contains '://' but is not a 'gs://' or 'https://' URL.", + ScrubUrl(strVal)); + return strVal; + } + // intentionally return an empty string when there are local, non-prefixed + // paths. Fetch does not process the local paths + return ""; } Result RemoveNonPrefixedBuildStrings(const Instance& instance) { diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser_test.cc b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser_test.cc index 84db0afbbac..a4964b2b0c4 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser_test.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser_test.cc @@ -40,6 +40,9 @@ Result> FetchCvdParserTester(const Json::Value& root) { } // namespace +using ::testing::AllOf; +using ::testing::HasSubstr; + TEST(FetchConfigParserTests, AndroidEfiLoaderBuildOnlySuccess) { const char* test_string = R""""( { @@ -105,9 +108,258 @@ TEST(FetchConfigParserTests, NoBuildStringsProducesNoFlagsSuccess) { std::string json_text(test_string); ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + Result> flags = FetchCvdParserTester(json_configs); + EXPECT_THAT(flags, IsOkAndValue(std::vector{})); +} + +TEST(FetchConfigParserTests, MalformedAbBuildStringFail) { + const char* test_string = R""""( +{ + "instances": [ + { + "disk": { + "default_build": "@ab/branch/target/extra" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + EXPECT_THAT(FetchCvdParserTester(json_configs), IsError()); +} + +TEST(FetchConfigParserTests, ObjectUrlBuildsSuccess) { + const char* test_string = R""""( +{ + "common": { + "host_package": "gs://bucket/dist/cvd-host_package.tar.gz" + }, + "instances": [ + { + "disk": { + "default_build": "gs://bucket/dist/phone-img-1.zip" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + Result> flags = FetchCvdParserTester(json_configs); ASSERT_THAT(flags, IsOk()); - EXPECT_TRUE(flags->empty()); + EXPECT_TRUE( + FindConfig(*flags, "--default_build=gs://bucket/dist/phone-img-1.zip")); + EXPECT_TRUE(FindConfig( + *flags, "--host_package_build=gs://bucket/dist/cvd-host_package.tar.gz")); +} + +TEST(FetchConfigParserTests, DirectoryUrlBuildsSuccess) { + const char* test_string = R""""( +{ + "common": { + "host_package": "gs://bucket/host/" + }, + "instances": [ + { + "disk": { + "default_build": "gs://bucket/dist/" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + Result> flags = FetchCvdParserTester(json_configs); + ASSERT_THAT(flags, IsOk()); + EXPECT_TRUE(FindConfig(*flags, "--default_build=gs://bucket/dist/")); + EXPECT_TRUE(FindConfig(*flags, "--host_package_build=gs://bucket/host/")); +} + +TEST(FetchConfigParserTests, HttpsDirectoryUrlBuildSuccess) { + const char* test_string = R""""( +{ + "instances": [ + { + "disk": { + "default_build": "https://example.com/dist/" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + Result> flags = FetchCvdParserTester(json_configs); + ASSERT_THAT(flags, IsOk()); + EXPECT_TRUE(FindConfig(*flags, "--default_build=https://example.com/dist/")); +} + +TEST(FetchConfigParserTests, OtaToolsUrlBuildSuccess) { + const char* test_string = R""""( +{ + "instances": [ + { + "disk": { + "otatools": "gs://bucket/dist/" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + Result> flags = FetchCvdParserTester(json_configs); + ASSERT_THAT(flags, IsOk()); + EXPECT_TRUE(FindConfig(*flags, "--otatools_build=gs://bucket/dist/")); +} + +TEST(FetchConfigParserTests, UrlFilepathSelectorSuccess) { + const char* test_string = R""""( +{ + "instances": [ + { + "boot": { + "build": "gs://bucket/dist/images.zip{boot.img}" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + Result> flags = FetchCvdParserTester(json_configs); + ASSERT_THAT(flags, IsOk()); + EXPECT_TRUE( + FindConfig(*flags, "--boot_build=gs://bucket/dist/images.zip{boot.img}")); +} + +TEST(FetchConfigParserTests, CleartextHttpUrlFail) { + const char* test_string = R""""( +{ + "instances": [ + { + "disk": { + "default_build": "http://example.com/dist/" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + EXPECT_THAT( + FetchCvdParserTester(json_configs), + IsErrorAndMessage(AllOf(HasSubstr("http://"), HasSubstr("https://")))); +} + +TEST(FetchConfigParserTests, UnsupportedUrlSchemeFail) { + const char* test_string = R""""( +{ + "instances": [ + { + "disk": { + "default_build": "s3://bucket/dist/" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + EXPECT_THAT(FetchCvdParserTester(json_configs), + IsErrorAndMessage(AllOf(HasSubstr("s3"), HasSubstr("gs://")))); +} + +// The URL build string format reserves ',', so a value holding one is +// refused even where nothing splits it. +TEST(FetchConfigParserTests, UrlWithCommaFail) { + const char* test_string = R""""( +{ + "instances": [ + { + "disk": { + "default_build": "https://example.com/d/img.zip?k=a,b" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + EXPECT_THAT(FetchCvdParserTester(json_configs), + IsErrorAndMessage(HasSubstr("comma"))); +} + +TEST(FetchConfigParserTests, LocalPathProducesNoFlagsSuccess) { + const char* test_string = R""""( +{ + "instances": [ + { + "disk": { + "default_build": "/home/user/local_images" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + Result> flags = FetchCvdParserTester(json_configs); + EXPECT_THAT(flags, IsOkAndValue(std::vector{})); +} + +TEST(FetchConfigParserTests, UnprefixedRelativeValuesProduceNoFlagsSuccess) { + const char* test_string = R""""( +{ + "instances": [ + { + "disk": { + "default_build": "out/target/product/vsoc_x86_64", + "otatools": "branch/target" + } + } + ] +} + )""""; + + Json::Value json_configs; + std::string json_text(test_string); + ASSERT_TRUE(ParseJsonString(json_text, json_configs)); + + Result> flags = FetchCvdParserTester(json_configs); + EXPECT_THAT(flags, IsOkAndValue(std::vector{})); } } // namespace cuttlefish From d6b4679b4cceb10736253086de1a800feaa486fb Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:38 +0000 Subject: [PATCH 13/20] Keep query strings out of URL build string parse errors --- .../host/libs/web/android_build_string.cpp | 47 +++++++++++-------- .../libs/web/android_build_string_tests.cpp | 27 +++++++++++ 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp index 686d7b38171..e8f98d97500 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_string.cpp @@ -55,8 +55,22 @@ std::optional Sha256FragmentDigest( return fragment; } +// A URL build string's '{selector}', when it has one, must be complete and +// must end the string. Checked before `ParseFilepath`, which allows a +// selector anywhere and discards whatever follows it. +bool HasSelectorSuffix(std::string_view build_string) { + const size_t open_bracket = build_string.find('{'); + const size_t close_bracket = build_string.find('}'); + if (open_bracket == std::string_view::npos) { + return close_bracket == std::string_view::npos; + } + return close_bracket + 1 == build_string.size() && + close_bracket > open_bracket + 1; +} + // A fragment is a suffix by definition, so a '#' anywhere else is an error -// rather than something silently kept in the URL. +// rather than something silently kept in the URL. What follows the '#' is not +// quoted back, as it can hold a query string. Result>> ParseSha256Fragment(std::string_view build_string) { const size_t fragment_start = build_string.find('#'); @@ -66,9 +80,9 @@ ParseSha256Fragment(std::string_view build_string) { const std::optional digest = Sha256FragmentDigest(build_string.substr(fragment_start)); CF_EXPECTF(digest.has_value(), - "Only a trailing '#sha256=<64 hex digits>' fragment is supported " - "in a URL build string. Input: '{}'", - build_string); + "The only fragment a URL build string may carry is a trailing " + "'#sha256=<64 hex digits>'. Input: '{}'", + ScrubUrl(build_string)); return {{build_string.substr(0, fragment_start), digest}}; } @@ -135,14 +149,11 @@ Result ParseDirectoryBuildString( Result ParseUrlBuildString(std::string_view scheme, std::string_view build_string) { - CF_EXPECTF(scheme != "http", - "Cleartext 'http://' build sources are not supported, use " - "'https://' instead. Input: '{}'", - build_string); CF_EXPECTF(scheme == "gs" || scheme == "https", "Unsupported URL scheme '{}'. The supported URL schemes are " - "'gs://' and 'https://'. Input: '{}'", - scheme, build_string); + "'gs://' and 'https://' (cleartext 'http://' is not). Input: " + "'{}'", + scheme, ScrubUrl(build_string)); // The format reserves ',' because build strings travel in comma separated // lists, where a URL holding one would be split into pieces that each parse // as some other kind of build string. @@ -150,16 +161,14 @@ Result ParseUrlBuildString(std::string_view scheme, "URL build strings cannot contain a comma, which the format " "reserves because build strings travel in comma separated " "lists. Input: '{}'", - build_string); + ScrubUrl(build_string)); auto [without_fragment, sha256] = CF_EXPECT(ParseSha256Fragment(build_string)); - size_t close_bracket = without_fragment.find('}'); - CF_EXPECTF(close_bracket == std::string_view::npos || - close_bracket + 1 == without_fragment.size(), - "A URL build string cannot have characters after the closing " - "curly bracket. Input: '{}'", - build_string); + CF_EXPECTF(HasSelectorSuffix(without_fragment), + "A URL build string ends either with its '{{selector}}' or with " + "the URL itself. Input: '{}'", + ScrubUrl(build_string)); auto [url, filepath] = CF_EXPECT(ParseFilepath(without_fragment)); const size_t query = url.find('?'); @@ -171,11 +180,11 @@ Result ParseUrlBuildString(std::string_view scheme, CF_EXPECTF(object_form || query == std::string::npos, "Query strings are only supported on URLs naming an object, not " "on a '/'-terminated directory. Input: '{}'", - build_string); + ScrubUrl(build_string)); CF_EXPECTF(object_form || !sha256.has_value(), "'#sha256=' is only supported on URLs naming an object, not on a " "'/'-terminated directory. Input: '{}'", - build_string); + ScrubUrl(build_string)); std::optional digest; if (sha256.has_value()) { diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_string_tests.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_string_tests.cpp index 7be79e36db8..bca2c128079 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_string_tests.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_string_tests.cpp @@ -32,6 +32,7 @@ using ::testing::ElementsAre; using ::testing::Eq; using ::testing::HasSubstr; using ::testing::IsEmpty; +using ::testing::Not; using ::testing::Optional; using ::testing::SizeIs; using ::testing::VariantWith; @@ -296,6 +297,9 @@ TEST(ParseBuildStringTests, UnknownSchemeFail) { EXPECT_THAT( ParseBuildString("ftp://example.com/file.zip"), IsErrorAndMessage(AllOf(HasSubstr("gs://"), HasSubstr("https://")))); + EXPECT_THAT( + ParseBuildString("s3://bucket/file.zip?X-Goog-Signature=SECRETVALUE"), + IsErrorAndMessage(Not(HasSubstr("SECRETVALUE")))); } TEST(ParseBuildStringTests, Sha256FragmentSuccess) { @@ -342,11 +346,34 @@ TEST(ParseBuildStringTests, CharactersAfterFilepathFail) { EXPECT_THAT(ParseBuildString("gs://bucket/{boot.img}trailing"), IsError()); EXPECT_THAT(ParseBuildString("https://example.com/{boot.img}/more"), IsError()); + EXPECT_THAT(ParseBuildString("gs://bucket/img.zip{}"), IsError()); + EXPECT_THAT(ParseBuildString("gs://bucket/img.zip{boot.img"), IsError()); } TEST(ParseBuildStringTests, UrlWithCommaFail) { EXPECT_THAT(ParseBuildString("gs://bucket/file,name.zip"), IsErrorAndMessage(HasSubstr("cannot contain a comma"))); + EXPECT_THAT( + ParseBuildString("gs://bucket/f,ile.zip?X-Goog-Signature=SECRETVALUE"), + IsErrorAndMessage(AllOf(HasSubstr("cannot contain a comma"), + Not(HasSubstr("SECRETVALUE"))))); +} + +TEST(ParseBuildStringTests, UrlErrorsWithoutQueryFail) { + const std::string secret = "?X-Goog-Signature=SECRETVALUE"; + const std::string inputs[] = { + "http://example.com/img.zip" + secret, + "gs://bucket/img.zip" + secret + "{}", + "gs://bucket/img.zip" + secret + "{boot.img}trailing", + "gs://bucket/img.zip#md5=abcdef" + secret, + "https://example.com/dist/" + secret, + std::string("https://example.com/dist/") + secret + "#sha256=" + kSha256, + }; + for (const std::string& input : inputs) { + EXPECT_THAT(ParseBuildString(input), + IsErrorAndMessage(Not(HasSubstr("SECRETVALUE")))) + << input; + } } TEST(ParseBuildStringTests, QueryStringOnObjectSuccess) { From 31cff9d5240f890fcac1fee8cb8bfaf02d63ace5 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:39 +0000 Subject: [PATCH 14/20] Pass an environment to `cvd create --config_file` --- e2etests/cvd/common/common.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2etests/cvd/common/common.go b/e2etests/cvd/common/common.go index b2c76a07ebb..92a6a964988 100644 --- a/e2etests/cvd/common/common.go +++ b/e2etests/cvd/common/common.go @@ -191,7 +191,7 @@ func (tc *TestContext) CVDFetch(args FetchArgs) (CommandOutput, error) { if credentialArg != "" { fetchCmd = append(fetchCmd, fmt.Sprintf("--credential_source=%s", credentialArg)) } - res, err := tc.RunCmd(fetchCmd...); + res, err := tc.RunCmd(fetchCmd...) if err != nil { log.Printf("Failed to fetch: %w", err) return res, err @@ -302,6 +302,7 @@ func (tc *TestContext) CVDPowerwash() error { // Common parameters for `cvd create --config_file`. type LoadArgs struct { LoadConfig string + Env map[string]string } // Performs `cvd create --config_file`. @@ -326,7 +327,7 @@ func (tc *TestContext) CVDCreateWithConfigFile(load LoadArgs) error { if credentialArg != "" { loadCmd = append(loadCmd, fmt.Sprintf("--credential_source=%s", credentialArg)) } - if _, err := tc.RunCmd(loadCmd...); err != nil { + if _, err := tc.RunCmdWithEnv(loadCmd, load.Env); err != nil { log.Printf("Failed to perform `cvd create --config_file`: %w", err) return err } From 60ec8166d83f01405adf0fdc75fea6e1a0974f78 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:39 +0000 Subject: [PATCH 15/20] Add e2e tests for https:// build sources --- e2etests/cvd/cvd_url_build_tests/BUILD.bazel | 54 +++++++ e2etests/cvd/cvd_url_build_tests/boot_test.go | 116 +++++++++++++ .../https_fixtures_test.go | 152 ++++++++++++++++++ e2etests/cvd/cvd_url_build_tests/main_test.go | 108 +++++++++++++ 4 files changed, 430 insertions(+) create mode 100644 e2etests/cvd/cvd_url_build_tests/BUILD.bazel create mode 100644 e2etests/cvd/cvd_url_build_tests/boot_test.go create mode 100644 e2etests/cvd/cvd_url_build_tests/https_fixtures_test.go create mode 100644 e2etests/cvd/cvd_url_build_tests/main_test.go diff --git a/e2etests/cvd/cvd_url_build_tests/BUILD.bazel b/e2etests/cvd/cvd_url_build_tests/BUILD.bazel new file mode 100644 index 00000000000..8159fa086a9 --- /dev/null +++ b/e2etests/cvd/cvd_url_build_tests/BUILD.bazel @@ -0,0 +1,54 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_go//go:def.bzl", "go_test") + +# Serves its own artifacts, so it needs neither the Android build servers nor +# a device. +go_test( + name = "cvd_url_build_tests", + size = "large", + srcs = [ + "https_fixtures_test.go", + "main_test.go", + ], + tags = [ + "exclusive", + "external", + "no-sandbox", + "supports-graceful-termination", + ], + deps = [ + "//cvd/common", + ], +) + +go_test( + name = "cvd_url_build_boot_tests", + size = "large", + srcs = [ + "boot_test.go", + "https_fixtures_test.go", + ], + tags = [ + "exclusive", + "external", + "no-sandbox", + "requires_ab", + "supports-graceful-termination", + ], + deps = [ + "//cvd/common", + ], +) diff --git a/e2etests/cvd/cvd_url_build_tests/boot_test.go b/e2etests/cvd/cvd_url_build_tests/boot_test.go new file mode 100644 index 00000000000..332ecf35c16 --- /dev/null +++ b/e2etests/cvd/cvd_url_build_tests/boot_test.go @@ -0,0 +1,116 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + "path" + "path/filepath" + "testing" + + "github.com/google/android-cuttlefish/e2etests/cvd/common" +) + +const bootBuild = "aosp-android-latest-release/aosp_cf_x86_64_only_phone-userdebug" + +const urlLoadConfig = ` +{ + "instances": [ + { + "name": "ins-1", + "disk": { + "default_build": "%s" + }, + "vm": { + "cpus": 4, + "memory_mb": 4096, + "setupwizard_mode": "REQUIRED" + }, + "graphics": { + "displays": [ + { + "width": 720, + "height": 1280, + "dpi": 140, + "refresh_rate_hertz": 60 + } + ], + "record_screen": false + } + } + ], + "netsim_bt": false, + "metrics": { + "enable": true + }, + "common": { + "host_package": "%s" + } +}` + +// Downloads the archives of an Android build server build without unpacking +// them, so that they can be served again as a plain https:// build. +func stageBuild(t *testing.T, c *e2etests.TestContext, directory string) string { + fetchCmd := []string{ + c.TargetBin(), + "fetch", + "--keep_downloaded_archives", + "--target_directory=" + directory, + "--default_build=" + bootBuild, + } + if credential := os.Getenv("CREDENTIAL_SOURCE"); credential != "" { + fetchCmd = append(fetchCmd, "--credential_source="+credential) + } + if _, err := c.RunCmd(fetchCmd...); err != nil { + t.Fatal(err) + } + + matches, err := filepath.Glob(path.Join(directory, "*-img-*.zip")) + if err != nil { + t.Fatal(err) + } + if len(matches) != 1 { + t.Fatalf("staged %d image zips in %s, want exactly one", len(matches), directory) + } + if !e2etests.FileExists(path.Join(directory, hostPackageName)) { + t.Fatalf("%s is missing from %s", hostPackageName, directory) + } + return path.Base(matches[0]) +} + +// Boots a device from the same artifacts the other tests in this suite take +// from the Android build servers, served over https:// instead. +func TestCvdLoadHttpsObjectBuild(t *testing.T) { + c := e2etests.TestContext{} + c.SetUp(t) + defer c.TearDown() + + staging := t.TempDir() + imgZip := stageBuild(t, &c, staging) + base, certpath := serveArtifacts(t, staging) + + config := fmt.Sprintf(urlLoadConfig, base+"/"+imgZip, base+"/"+hostPackageName) + if err := c.CVDCreateWithConfigFile(e2etests.LoadArgs{ + LoadConfig: config, + Env: map[string]string{"CURL_CA_BUNDLE": certpath}, + }); err != nil { + t.Fatal(err) + } + + if err := c.RunAdbWaitForDevice(); err != nil { + t.Fatal(err) + } +} diff --git a/e2etests/cvd/cvd_url_build_tests/https_fixtures_test.go b/e2etests/cvd/cvd_url_build_tests/https_fixtures_test.go new file mode 100644 index 00000000000..eb722c6a287 --- /dev/null +++ b/e2etests/cvd/cvd_url_build_tests/https_fixtures_test.go @@ -0,0 +1,152 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "encoding/json" + "encoding/pem" + "net/http" + "net/http/httptest" + "os" + "path" + "testing" +) + +const hostPackageName = "cvd-host_package.tar.gz" + +// An entry of `fetcher_config.json`. +type cvdFile struct { + Source string `json:"source"` + BuildID string `json:"build_id"` + BuildTarget string `json:"build_target"` +} + +type fetcherConfig struct { + CvdFiles map[string]cvdFile `json:"cvd_files"` +} + +// Serves dir over TLS and returns the base URL and the path of a PEM file +// holding the server certificate. The certificate is valid for 127.0.0.1, and +// `cvd` trusts it when the path is handed to it as `CURL_CA_BUNDLE`. +func serveArtifacts(t *testing.T, dir string) (base string, certpath string) { + server := httptest.NewTLSServer(http.FileServer(http.Dir(dir))) + t.Cleanup(server.Close) + + certpath = path.Join(t.TempDir(), "test_ca.pem") + certfile, err := os.Create(certpath) + if err != nil { + t.Fatalf("failed to create %s: %v", certpath, err) + } + defer certfile.Close() + + block := pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw} + if err := pem.Encode(certfile, &block); err != nil { + t.Fatalf("failed to write %s: %v", certpath, err) + } + + return server.URL, certpath +} + +func writeArtifact(t *testing.T, filepath string) { + if err := os.WriteFile(filepath, []byte(path.Base(filepath)+"\n"), 0644); err != nil { + t.Fatalf("failed to write %s: %v", filepath, err) + } +} + +func writeZip(t *testing.T, filepath string, members []string) { + file, err := os.Create(filepath) + if err != nil { + t.Fatalf("failed to create %s: %v", filepath, err) + } + defer file.Close() + + writer := zip.NewWriter(file) + for _, member := range members { + entry, err := writer.Create(member) + if err != nil { + t.Fatalf("failed to add %s to %s: %v", member, filepath, err) + } + if _, err := entry.Write([]byte(member + "\n")); err != nil { + t.Fatalf("failed to write %s in %s: %v", member, filepath, err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("failed to close %s: %v", filepath, err) + } +} + +func writeTarGz(t *testing.T, filepath string, members []string) { + file, err := os.Create(filepath) + if err != nil { + t.Fatalf("failed to create %s: %v", filepath, err) + } + defer file.Close() + + compressor := gzip.NewWriter(file) + writer := tar.NewWriter(compressor) + for _, member := range members { + contents := []byte(member + "\n") + header := tar.Header{Name: member, Mode: 0755, Size: int64(len(contents))} + if err := writer.WriteHeader(&header); err != nil { + t.Fatalf("failed to add %s to %s: %v", member, filepath, err) + } + if _, err := writer.Write(contents); err != nil { + t.Fatalf("failed to write %s in %s: %v", member, filepath, err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("failed to close the archive in %s: %v", filepath, err) + } + if err := compressor.Close(); err != nil { + t.Fatalf("failed to close %s: %v", filepath, err) + } +} + +func readFile(t *testing.T, filepath string) string { + contents, err := os.ReadFile(filepath) + if err != nil { + t.Fatalf("failed to read %s: %v", filepath, err) + } + return string(contents) +} + +func readFetcherConfig(t *testing.T, directory string) fetcherConfig { + filepath := path.Join(directory, "fetcher_config.json") + var config fetcherConfig + if err := json.Unmarshal([]byte(readFile(t, filepath)), &config); err != nil { + t.Fatalf("failed to parse %s: %v", filepath, err) + } + return config +} + +// Checks that `cvd fetch` recorded name as coming from the given URL build. +func checkProvenance(t *testing.T, config fetcherConfig, name string, source string, url string) { + entry, ok := config.CvdFiles[name] + if !ok { + t.Fatalf("fetcher_config.json has no entry for %s", name) + } + if entry.Source != source { + t.Errorf("%s source = %q, want %q", name, entry.Source, source) + } + if entry.BuildID != url { + t.Errorf("%s build_id = %q, want %q", name, entry.BuildID, url) + } + if entry.BuildTarget != "url" { + t.Errorf("%s build_target = %q, want \"url\"", name, entry.BuildTarget) + } +} diff --git a/e2etests/cvd/cvd_url_build_tests/main_test.go b/e2etests/cvd/cvd_url_build_tests/main_test.go new file mode 100644 index 00000000000..5362f62bdd2 --- /dev/null +++ b/e2etests/cvd/cvd_url_build_tests/main_test.go @@ -0,0 +1,108 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path" + "strings" + "testing" + + "github.com/google/android-cuttlefish/e2etests/cvd/common" +) + +var hostPackageFiles = []string{"bin/launch_cvd", "bin/stop_cvd"} + +func checkFetched(t *testing.T, directory string, names []string) { + for _, name := range names { + if !e2etests.FileExists(path.Join(directory, name)) { + t.Errorf("%s is missing from %s", name, directory) + } + } +} + +func fetch(t *testing.T, c *e2etests.TestContext, certpath string, args []string) { + command := append([]string{c.TargetBin(), "fetch", "--enable_caching=false"}, args...) + if _, err := c.RunCmdWithEnv(command, map[string]string{"CURL_CA_BUNDLE": certpath}); err != nil { + t.Fatal(err) + } +} + +// Fetches a build named as a single https:// object. The image zip is the +// build's only artifact, so the host package has to be named separately. +func TestFetchHttpsObjectBuild(t *testing.T) { + c := e2etests.TestContext{} + c.SetUp(t) + defer c.TearDown() + + const imgZip = "aosp_cf_x86_64_phone-img-13579246.zip" + const query = "?fake_signature=not-a-real-credential" + images := []string{"boot.img", "super.img", "vbmeta.img"} + + staging := t.TempDir() + writeZip(t, path.Join(staging, imgZip), images) + writeTarGz(t, path.Join(staging, hostPackageName), hostPackageFiles) + base, certpath := serveArtifacts(t, staging) + + target := t.TempDir() + fetch(t, &c, certpath, []string{ + "--target_directory=" + target, + "--default_build=" + base + "/" + imgZip + query, + "--host_package_build=" + base + "/" + hostPackageName, + }) + + checkFetched(t, target, images) + checkFetched(t, target, hostPackageFiles) + + config := readFetcherConfig(t, target) + for _, image := range images { + checkProvenance(t, config, image, "default_build", base+"/"+imgZip) + } + + // The query carries the credential of a pre-signed URL, so it belongs in + // no record of the fetch. + if strings.Contains(readFile(t, path.Join(target, "fetch.log")), query) { + t.Errorf("fetch.log holds the query string of the build URL") + } +} + +// Fetches individually named artifacts out of an https:// directory. The +// directory cannot be listed, so every artifact is named by the build string. +func TestFetchHttpsDirectoryBuild(t *testing.T) { + c := e2etests.TestContext{} + c.SetUp(t) + defer c.TearDown() + + staging := path.Join(t.TempDir(), "dist") + if err := os.MkdirAll(staging, 0755); err != nil { + t.Fatal(err) + } + writeArtifact(t, path.Join(staging, "boot.img")) + writeTarGz(t, path.Join(staging, hostPackageName), hostPackageFiles) + base, certpath := serveArtifacts(t, path.Dir(staging)) + directory := base + "/dist/" + + target := t.TempDir() + fetch(t, &c, certpath, []string{ + "--target_directory=" + target, + "--boot_build=" + directory + "{boot.img}", + "--host_package_build=" + directory, + }) + + checkFetched(t, target, []string{"boot.img"}) + checkFetched(t, target, hostPackageFiles) + + checkProvenance(t, readFetcherConfig(t, target), "boot.img", "boot_build", directory) +} From e0bc6f8a19c0a8736928b215a3aa249aa8c79bc1 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:39 +0000 Subject: [PATCH 16/20] Verify downloaded URL build artifacts against their digests --- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 27 +++++ base/cvd/cuttlefish/host/libs/web/digest.cpp | 93 +++++++++++++++ base/cvd/cuttlefish/host/libs/web/digest.h | 37 ++++++ .../cuttlefish/host/libs/web/digest_test.cpp | 111 ++++++++++++++++++ .../host/libs/web/gcs_build_api.cpp | 24 ++++ .../host/libs/web/gcs_build_api_test.cpp | 62 ++++++++++ .../host/libs/web/http_build_api.cpp | 4 + .../host/libs/web/http_build_api_test.cpp | 41 +++++++ 8 files changed, 399 insertions(+) create mode 100644 base/cvd/cuttlefish/host/libs/web/digest.cpp create mode 100644 base/cvd/cuttlefish/host/libs/web/digest.h create mode 100644 base/cvd/cuttlefish/host/libs/web/digest_test.cpp diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index bba6afbbdeb..21c1840871e 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -244,6 +244,31 @@ cf_cc_library( ], ) +cf_cc_library( + name = "digest", + srcs = ["digest.cpp"], + hdrs = ["digest.h"], + deps = [ + "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/common/libs/utils:base64", + "//cuttlefish/result", + "@abseil-cpp//absl/strings", + "@boringssl//:crypto", + ], +) + +cf_cc_test( + name = "digest_test", + srcs = ["digest_test.cpp"], + deps = [ + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/host/libs/web:digest", + "//cuttlefish/result:result_matchers", + "//libbase", + "@abseil-cpp//absl/strings", + ], +) + cf_cc_library( name = "gcs_build_api", srcs = ["gcs_build_api.cpp"], @@ -256,6 +281,7 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:build_api_zip", "//cuttlefish/host/libs/web:credential_source", + "//cuttlefish/host/libs/web:digest", "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/web/http_client", "//cuttlefish/host/libs/web/http_client:http_file", @@ -304,6 +330,7 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build", "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:build_api_zip", + "//cuttlefish/host/libs/web:digest", "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/web/http_client", "//cuttlefish/host/libs/web/http_client:http_file", diff --git a/base/cvd/cuttlefish/host/libs/web/digest.cpp b/base/cvd/cuttlefish/host/libs/web/digest.cpp new file mode 100644 index 00000000000..06d33e99e1a --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/digest.cpp @@ -0,0 +1,93 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/digest.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include "absl/strings/ascii.h" +#include "absl/strings/escaping.h" +#include "openssl/base.h" +#include "openssl/digest.h" + +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/common/libs/utils/base64.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { +namespace { + +constexpr size_t kReadSize = 1 << 16; + +Result> DigestFile(const std::string& path, + const EVP_MD* digest) { + Fd fd = CF_EXPECT(Fd::Open(path, O_RDONLY)); + + std::unique_ptr context(EVP_MD_CTX_new(), + EVP_MD_CTX_free); + CF_EXPECT(EVP_DigestInit_ex(context.get(), digest, nullptr)); + + std::vector buffer(kReadSize); + while (true) { + const uint64_t read = CF_EXPECTF(fd.Read(buffer.data(), buffer.size()), + "Could not read '{}'", path); + if (read == 0) { + break; + } + CF_EXPECT(EVP_DigestUpdate(context.get(), buffer.data(), read)); + } + + std::vector value(EVP_MAX_MD_SIZE); + unsigned int length = 0; + CF_EXPECT(EVP_DigestFinal_ex(context.get(), value.data(), &length)); + value.resize(length); + return value; +} + +} // namespace + +Result Sha256File(const std::string& path) { + const std::vector value = CF_EXPECT(DigestFile(path, EVP_sha256())); + return absl::BytesToHexString(std::string_view( + reinterpret_cast(value.data()), value.size())); +} + +Result VerifySha256(const std::string& path, std::string_view expected, + std::string_view artifact_name) { + const std::string actual = CF_EXPECT(Sha256File(path)); + CF_EXPECTF(actual == absl::AsciiStrToLower(expected), + "'{}' has sha256 '{}', but '{}' was expected.", artifact_name, + actual, expected); + return {}; +} + +Result VerifyMd5(const std::string& path, std::string_view expected, + std::string_view artifact_name) { + const std::vector value = CF_EXPECT(DigestFile(path, EVP_md5())); + const std::string actual = + CF_EXPECT(EncodeBase64(value.data(), value.size())); + CF_EXPECTF(actual == expected, "'{}' has md5 '{}', but '{}' was expected.", + artifact_name, actual, expected); + return {}; +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/digest.h b/base/cvd/cuttlefish/host/libs/web/digest.h new file mode 100644 index 00000000000..acde53e11a3 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/digest.h @@ -0,0 +1,37 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "cuttlefish/result/result.h" + +namespace cuttlefish { + +// The SHA-256 of the contents of `path`, in lowercase hexadecimal. +Result Sha256File(const std::string& path); + +// Fails unless `path` holds the hexadecimal SHA-256 `expected`, which is +// compared without regard to case. `artifact_name` names the file in the error. +Result VerifySha256(const std::string& path, std::string_view expected, + std::string_view artifact_name); + +// The same against the base64 MD5 that Cloud Storage reports for an object. +Result VerifyMd5(const std::string& path, std::string_view expected, + std::string_view artifact_name); + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/digest_test.cpp b/base/cvd/cuttlefish/host/libs/web/digest_test.cpp new file mode 100644 index 00000000000..818490f530b --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/digest_test.cpp @@ -0,0 +1,111 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/digest.h" + +#include +#include + +#include "absl/strings/ascii.h" +#include "android-base/file.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::AllOf; +using ::testing::HasSubstr; + +constexpr char kAbcSha256[] = + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; +constexpr char kAbcMd5[] = "kAFQmDzST7DWlj99KOF/cg=="; +constexpr char kEmptySha256[] = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +std::string WriteFile(const TemporaryDir& directory, + std::string_view contents) { + std::string path = std::string(directory.path) + "/artifact"; + EXPECT_THAT(WriteNewFile(path, contents), IsOk()); + return path; +} + +TEST(DigestTests, Sha256OfAFileSuccess) { + TemporaryDir directory; + EXPECT_THAT(Sha256File(WriteFile(directory, "abc")), + IsOkAndValue(kAbcSha256)); +} + +TEST(DigestTests, Sha256OfAnEmptyFileSuccess) { + TemporaryDir directory; + EXPECT_THAT(Sha256File(WriteFile(directory, "")), IsOkAndValue(kEmptySha256)); +} + +TEST(DigestTests, Sha256OfMoreThanOneReadSuccess) { + TemporaryDir directory; + + EXPECT_THAT( + Sha256File(WriteFile(directory, std::string(1 << 20, 'x'))), + IsOkAndValue( + "8f990ba0b577b51cf009ea049368c16bbda1b21e1b93be07a824758bb253c39b")); +} + +TEST(DigestTests, MissingFileFail) { + TemporaryDir directory; + EXPECT_THAT(Sha256File(std::string(directory.path) + "/absent"), + IsErrorAndMessage(HasSubstr("absent"))); +} + +TEST(DigestTests, VerifySha256IgnoresCaseSuccess) { + TemporaryDir directory; + std::string path = WriteFile(directory, "abc"); + + EXPECT_THAT(VerifySha256(path, kAbcSha256, "artifact"), IsOk()); + EXPECT_THAT(VerifySha256(path, absl::AsciiStrToUpper(kAbcSha256), "artifact"), + IsOk()); +} + +TEST(DigestTests, VerifySha256NamesBothDigestsFail) { + TemporaryDir directory; + std::string path = WriteFile(directory, "abc"); + std::string expected(64, 'a'); + + EXPECT_THAT( + VerifySha256(path, expected, "phone-img-1.zip"), + IsErrorAndMessage(AllOf(HasSubstr("phone-img-1.zip"), + HasSubstr(kAbcSha256), HasSubstr(expected)))); +} + +TEST(DigestTests, VerifyMd5OfABase64DigestSuccess) { + TemporaryDir directory; + EXPECT_THAT(VerifyMd5(WriteFile(directory, "abc"), kAbcMd5, "artifact"), + IsOk()); +} + +TEST(DigestTests, VerifyMd5NamesBothDigestsFail) { + TemporaryDir directory; + + EXPECT_THAT( + VerifyMd5(WriteFile(directory, "abc"), + "AAAAAAAAAAAAAAAAAAAAAA==", "phone-img-1.zip"), + IsErrorAndMessage(AllOf(HasSubstr("phone-img-1.zip"), HasSubstr(kAbcMd5), + HasSubstr("AAAAAAAAAAAAAAAAAAAAAA==")))); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp index c06c32e78d0..e8894fea62d 100644 --- a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp @@ -37,6 +37,7 @@ #include "cuttlefish/host/libs/web/android_build_string.h" #include "cuttlefish/host/libs/web/build_api_zip.h" #include "cuttlefish/host/libs/web/credential_source.h" +#include "cuttlefish/host/libs/web/digest.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/host/libs/web/http_client/http_file.h" #include "cuttlefish/host/libs/web/http_client/http_json.h" @@ -119,6 +120,28 @@ std::optional ArtifactSize(const GcsBuild& build, return entry->second.size; } +// The listing and the metadata probe both report an md5, so a `gs://` download +// is checked whether or not the build string carried a digest. +Result VerifyArtifact(const GcsBuild& build, + const std::string& artifact_name, + const std::string& path) { + if (build.object.has_value()) { + if (build.sha256.has_value()) { + CF_EXPECT(VerifySha256(path, *build.sha256, artifact_name)); + } + if (build.md5.has_value()) { + CF_EXPECT(VerifyMd5(path, *build.md5, artifact_name)); + } + return {}; + } + const std::map::const_iterator entry = + build.contents.find(artifact_name); + if (entry != build.contents.end() && entry->second.md5.has_value()) { + CF_EXPECT(VerifyMd5(path, *entry->second.md5, artifact_name)); + } + return {}; +} + Result ResponseJson(const HttpResponse& response, const GcsBuild& build, bool authenticated) { std::string_view hint; @@ -237,6 +260,7 @@ Result GcsBuildApi::DownloadFile( CF_EXPECTF(response.HttpSuccess(), "Could not download '{}' from '{}' - {}:{}", artifact_name, build.id, response.http_code, response.StatusDescription()); + CF_EXPECT(VerifyArtifact(build, artifact_name, dest_path)); return dest_path; } diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp index 3ac4e55e593..0b1dedf815f 100644 --- a/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp @@ -215,6 +215,68 @@ TEST(GcsBuildApiTests, DownloadFileWritesTheArtifactSuccess) { IsOkAndValue("recovery_api_version=3")); } +TEST(GcsBuildApiTests, DownloadFileChecksTheListedMd5Fail) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse( + R"({"items": [{"name": "dist/misc_info.txt", "generation": "1", + "md5Hash": "AAAAAAAAAAAAAAAAAAAAAA=="}]})", + kListUrl); + http_client.SetResponse("recovery_api_version=3", + "b/bucket/o/dist%2Fmisc_info.txt?alt=media"); + + GcsBuildString build_string = {.url = "gs://bucket/dist/"}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + EXPECT_THAT(api.DownloadFile(*build, target_directory.path, "misc_info.txt"), + IsErrorAndMessage(AllOf(HasSubstr("misc_info.txt"), + HasSubstr("iJ/GrhggATmj4tovMOdNDQ=="), + HasSubstr("AAAAAAAAAAAAAAAAAAAAAA==")))); +} + +TEST(GcsBuildApiTests, DownloadFileChecksTheProbedMd5Success) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse( + R"({"size": "22", "generation": "17", + "md5Hash": "iJ/GrhggATmj4tovMOdNDQ=="})", + kObjectUrl); + http_client.SetResponse("recovery_api_version=3", kMediaUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/phone-img-1.zip"}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + EXPECT_THAT( + api.DownloadFile(*build, target_directory.path, "phone-img-1.zip"), + IsOk()); +} + +TEST(GcsBuildApiTests, DownloadFileChecksTheRequestedSha256Fail) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + http_client.SetResponse(R"({"size": "22"})", kObjectUrl); + http_client.SetResponse("recovery_api_version=3", kMediaUrl); + + GcsBuildString build_string = { + .url = "gs://bucket/dist/phone-img-1.zip", + .sha256 = std::string(64, 'b'), + }; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + EXPECT_THAT( + api.DownloadFile(*build, target_directory.path, "phone-img-1.zip"), + IsErrorAndMessage(AllOf( + HasSubstr("phone-img-1.zip"), HasSubstr(std::string(64, 'b')), + HasSubstr("8e441a1db0c390234afe2970a82f888e5608304062d77df18622d872e" + "1328f5d")))); +} + TEST(GcsBuildApiTests, FileReaderReadsTheNamedArtifactSuccess) { FakeHttpClient http_client; GcsBuildApi api(http_client, nullptr); diff --git a/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp index a554bb88a1c..09387142f98 100644 --- a/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp @@ -30,6 +30,7 @@ #include "cuttlefish/host/libs/web/android_build.h" #include "cuttlefish/host/libs/web/android_build_string.h" #include "cuttlefish/host/libs/web/build_api_zip.h" +#include "cuttlefish/host/libs/web/digest.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/host/libs/web/http_client/http_file.h" #include "cuttlefish/host/libs/web/url_namespace.h" @@ -154,6 +155,9 @@ Result HttpBuildApi::DownloadFile( CF_EXPECTF(response.HttpSuccess(), "Could not download '{}' from '{}' - {}:{}", artifact_name, build.id, response.http_code, response.StatusDescription()); + if (build.sha256.has_value()) { + CF_EXPECT(VerifySha256(dest_path, *build.sha256, artifact_name)); + } return dest_path; } diff --git a/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp index 0f394e70780..5dea3290623 100644 --- a/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/http_build_api_test.cpp @@ -174,6 +174,47 @@ TEST(HttpBuildApiTests, DownloadFileAbsentFromAnObjectFail) { HasSubstr("phone-img-1.zip")))); } +TEST(HttpBuildApiTests, DownloadFileChecksTheRequestedSha256Success) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + http_client.SetResponse("recovery_api_version=3", kObjectUrl); + + HttpBuildString build_string = { + .url = kSignedUrl, + .sha256 = + "8E441A1DB0C390234AFE2970A82F888E5608304062D77DF18622D872E132" + "8F5D", + }; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + EXPECT_THAT( + api.DownloadFile(*build, target_directory.path, "phone-img-1.zip"), + IsOk()); +} + +TEST(HttpBuildApiTests, DownloadFileChecksTheRequestedSha256Fail) { + FakeHttpClient http_client; + HttpBuildApi api(http_client); + http_client.SetResponse("recovery_api_version=3", kObjectUrl); + + HttpBuildString build_string = { + .url = kSignedUrl, + .sha256 = std::string(64, 'b'), + }; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + TemporaryDir target_directory; + EXPECT_THAT( + api.DownloadFile(*build, target_directory.path, "phone-img-1.zip"), + IsErrorAndMessage(AllOf( + HasSubstr("phone-img-1.zip"), HasSubstr(std::string(64, 'b')), + HasSubstr("8e441a1db0c390234afe2970a82f888e5608304062d77df18622d872e" + "1328f5d")))); +} + TEST(HttpBuildApiTests, FileReaderReadsTheObjectSuccess) { FakeHttpClient http_client; HttpBuildApi api(http_client); From 20ecad5c5b53a5230da9cb3635e837369b6d2660 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:40 +0000 Subject: [PATCH 17/20] Key the artifact cache on the version the source reports --- .../host/commands/cvd/fetch/fetch_cvd.cc | 2 +- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 22 +++ .../cuttlefish/host/libs/web/android_build.cc | 73 +++++++ .../cuttlefish/host/libs/web/android_build.h | 37 ++-- .../host/libs/web/android_build_test.cpp | 103 ++++++++++ .../host/libs/web/caching_build_api.cpp | 46 ++++- .../host/libs/web/caching_build_api_test.cpp | 179 ++++++++++++++++++ base/cvd/cuttlefish/host/libs/web/digest.cpp | 15 +- base/cvd/cuttlefish/host/libs/web/digest.h | 7 +- .../cuttlefish/host/libs/web/digest_test.cpp | 5 + 10 files changed, 463 insertions(+), 26 deletions(-) create mode 100644 base/cvd/cuttlefish/host/libs/web/caching_build_api_test.cpp diff --git a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc index be65415e84e..e1320cc9bda 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc +++ b/base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc @@ -194,7 +194,7 @@ Result CheckHostPackagePresent(const Build& build) { "Name a build that has a host package with " "`--host_package_build`."); } else { - CF_EXPECTF(Contains(gcs->contents, name), + CF_EXPECTF(BuildHasArtifact(build, name), "The build '{}' has no host package '{}'. It holds [{}]. " "Name a build that has one with `--host_package_build`.", gcs->id, name, absl::StrJoin(GcsArtifactNames(*gcs), ", ")); diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index 21c1840871e..8344277ffed 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -11,9 +11,11 @@ cf_cc_library( srcs = ["android_build.cc"], hdrs = ["android_build.h"], deps = [ + "//cuttlefish/common/libs/utils:contains", "//cuttlefish/common/libs/utils:environment", "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:android_build_url", + "//cuttlefish/host/libs/web:digest", "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/web/http_client:scrub_secrets", "//cuttlefish/result", @@ -171,6 +173,7 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build", "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:build_api", + "//cuttlefish/host/libs/web:digest", "//cuttlefish/host/libs/zip:cached_zip_source", "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", "//cuttlefish/result", @@ -180,6 +183,25 @@ cf_cc_library( ], ) +cf_cc_test( + name = "caching_build_api_test", + srcs = ["caching_build_api_test.cpp"], + deps = [ + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/files:file_exists", + "//cuttlefish/files:is_directory_empty", + "//cuttlefish/host/libs/web:android_build", + "//cuttlefish/host/libs/web:android_build_string", + "//cuttlefish/host/libs/web:build_api", + "//cuttlefish/host/libs/web:caching_build_api", + "//cuttlefish/host/libs/zip/libzip_cc:seekable_source", + "//cuttlefish/host/libs/zip/libzip_cc:writable_source", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "//libbase", + ], +) + cf_cc_library( name = "chrome_os_build_string", srcs = ["chrome_os_build_string.cpp"], diff --git a/base/cvd/cuttlefish/host/libs/web/android_build.cc b/base/cvd/cuttlefish/host/libs/web/android_build.cc index ddd7d13711e..739fa48ea99 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build.cc +++ b/base/cvd/cuttlefish/host/libs/web/android_build.cc @@ -17,6 +17,7 @@ #include +#include #include #include #include @@ -26,11 +27,14 @@ #include #include +#include "absl/strings/ascii.h" #include "absl/strings/str_join.h" #include "fmt/format.h" +#include "cuttlefish/common/libs/utils/contains.h" #include "cuttlefish/common/libs/utils/environment.h" #include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/digest.h" #include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" #include "cuttlefish/host/libs/web/url_namespace.h" #include "cuttlefish/result/result.h" @@ -41,6 +45,29 @@ namespace { // URL builds have no Android Build target, product or branch, so they carry // this name wherever one is expected. constexpr char kUrlName[] = "url"; +constexpr size_t kUrlKeyLength = 12; + +// Cache keys are directory names and an ETag may hold anything, so a version +// that is not already path safe stands in as a digest of itself. +std::string PathSafeVersion(const std::string& version) { + for (char character : version) { + if (!absl::ascii_isalnum(character) && character != '.' && + character != '_' && character != '-') { + return Sha256Hex(version).substr(0, kUrlKeyLength); + } + } + return version; +} + +std::optional UrlCacheKey( + const std::string& id, const std::optional& version) { + if (!version.has_value()) { + return std::nullopt; + } + return fmt::format("{}/{}/{}", kUrlName, + Sha256Hex(id).substr(0, kUrlKeyLength), + PathSafeVersion(*version)); +} // Returns where the name after the last '/' of `path` begins. A parsed path // has no leading '/', so a name at the root is the whole path. @@ -148,6 +175,52 @@ std::string FetchLabel(const Build& build) { std::visit([](auto&& arg) { return arg.target; }, build)); } +std::optional ArtifactSha256(const Build& build, + const std::string& artifact_name) { + if (const GcsBuild* gcs = std::get_if(&build)) { + return gcs->object == artifact_name ? gcs->sha256 : std::nullopt; + } + if (const HttpBuild* http = std::get_if(&build)) { + return http->object == artifact_name ? http->sha256 : std::nullopt; + } + return std::nullopt; +} + +bool BuildHasArtifact(const Build& build, const std::string& artifact_name) { + const GcsBuild* gcs = std::get_if(&build); + if (gcs == nullptr || gcs->object.has_value()) { + return true; + } + return Contains(gcs->contents, artifact_name); +} + +std::optional BuildCacheKey(const Build& build, + const std::string& artifact_name) { + if (const GcsBuild* gcs = std::get_if(&build)) { + if (gcs->object.has_value()) { + return UrlCacheKey( + gcs->id, gcs->generation.has_value() ? gcs->generation : gcs->sha256); + } + const std::map::const_iterator entry = + gcs->contents.find(artifact_name); + if (entry == gcs->contents.end()) { + return std::nullopt; + } + return UrlCacheKey(gcs->id, entry->second.generation); + } + if (const HttpBuild* http = std::get_if(&build)) { + // A directory of plain HTTPS URLs reports nothing about its artifacts. + if (!http->object.has_value()) { + return std::nullopt; + } + return UrlCacheKey(http->id, + http->etag.has_value() ? http->etag : http->sha256); + } + return fmt::format("{}/{}", + std::visit([](auto&& arg) { return arg.id; }, build), + std::visit([](auto&& arg) { return arg.target; }, build)); +} + std::tuple GetBuildIdAndTarget(const Build& build) { std::string id = std::visit([](auto&& arg) { return arg.id; }, build); std::string target = std::visit([](auto&& arg) { return arg.target; }, build); diff --git a/base/cvd/cuttlefish/host/libs/web/android_build.h b/base/cvd/cuttlefish/host/libs/web/android_build.h index 640783d4c98..3369fe01863 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build.h +++ b/base/cvd/cuttlefish/host/libs/web/android_build.h @@ -65,23 +65,19 @@ struct GcsObjectInfo { std::optional size; }; -// The objects under a `gs://` prefix, or the single object a `gs://` URL -// names. +// Identifies the objects under a `gs://` prefix, or the single object a +// `gs://` URL names. struct GcsBuild { static Result FromBuildString(const GcsBuildString& build_string); std::string bucket; std::string prefix; // ends with '/', empty at the bucket root std::optional object; // set in the object form only - // The listing of the directory form, by artifact name. std::map contents; std::optional generation; std::optional md5; std::optional size; std::optional sha256; - - // Derived from the URL for the code that handles every build alike. Never - // used to address the objects themselves. std::string id; std::string target; std::string product; @@ -90,25 +86,21 @@ struct GcsBuild { std::ostream& operator<<(std::ostream&, const GcsBuild&); -// The artifacts of a `gs://` directory build, in the order the listing keeps -// them. +// Returns the artifacts of a `gs://` directory build, in name order. std::vector GcsArtifactNames(const GcsBuild& build); -// The same two forms over `https://`, where a pre-signed URL carries its -// credential in the query string of `url`. +// Identifies the same two forms over `https://`, where a pre-signed URL +// carries its credential in the query string of `url`. `id` drops that query +// string, so requests use `url`. struct HttpBuild { static Result FromBuildString(const HttpBuildString& build_string); std::string url; // object, or directory ending in '/' std::optional object; // set in the object form only std::optional etag; - // Whether the probe found an origin that serves range requests, without - // which a member cannot be read out of an archive. bool accept_ranges = false; std::optional size; std::optional sha256; - - // `id` has no query string, so requests must go to `url`. std::string id; std::string target; std::string product; @@ -123,6 +115,23 @@ std::ostream& operator<<(std::ostream&, const Build&); std::string FetchLabel(const Build& build); +// Returns the digest the build string asked `artifact_name` to have, if any. +std::optional ArtifactSha256(const Build& build, + const std::string& artifact_name); + +// Returns whether the build's namespace holds `artifact_name`. Only a listed +// namespace can answer "no"; the others answer by attempting the download. +bool BuildHasArtifact(const Build& build, const std::string& artifact_name); + +// Returns the path safe cache directory of one artifact of one build, +// relative to the cache root. Android Build and directory builds key on +// "{id}/{target}"; URL builds add the version the source reports for that one +// artifact, so that overwriting an object never serves the bytes it replaced. +// Returns nullopt when the source reports no version, which means the +// artifact must not be cached. +std::optional BuildCacheKey(const Build& build, + const std::string& artifact_name); + std::tuple GetBuildIdAndTarget(const Build& build); std::optional GetFilepath(const Build& build); diff --git a/base/cvd/cuttlefish/host/libs/web/android_build_test.cpp b/base/cvd/cuttlefish/host/libs/web/android_build_test.cpp index c0387874113..7ad0386472a 100644 --- a/base/cvd/cuttlefish/host/libs/web/android_build_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/android_build_test.cpp @@ -15,6 +15,7 @@ #include "cuttlefish/host/libs/web/android_build.h" +#include #include #include #include @@ -30,8 +31,11 @@ namespace cuttlefish { namespace { using ::testing::AllOf; +using ::testing::EndsWith; using ::testing::HasSubstr; using ::testing::Not; +using ::testing::Optional; +using ::testing::StartsWith; constexpr char kSignedUrl[] = "https://example.com/dist/phone-img-1.zip?X-Goog-Signature=secret"; @@ -150,5 +154,104 @@ TEST(FetchLabelTests, OtherBuildsAreIdAndTargetSuccess) { EXPECT_EQ(FetchLabel(directory_build), "eng/test_target"); } +TEST(ArtifactSha256Tests, OnlyTheNamedObjectSuccess) { + Build build = *GcsBuild::FromBuildString(GcsBuildString{ + .url = "gs://bucket/dist/phone-img-1.zip", + .filepath = "boot.img", + .sha256 = std::string(64, 'a'), + }); + + EXPECT_EQ(ArtifactSha256(build, "phone-img-1.zip"), std::string(64, 'a')); + EXPECT_EQ(ArtifactSha256(build, "boot.img"), std::nullopt); + EXPECT_EQ(ArtifactSha256(DeviceBuild{.id = "123"}, "img.zip"), std::nullopt); +} + +TEST(BuildHasArtifactTests, ListedArtifactsSuccess) { + GcsBuild build = + *GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket/dist/"}); + build.contents = {{"a.txt", GcsObjectInfo{.generation = "1"}}}; + + EXPECT_TRUE(BuildHasArtifact(build, "a.txt")); + EXPECT_FALSE(BuildHasArtifact(build, "misc_info.txt")); +} + +TEST(BuildHasArtifactTests, UnlistedNamespacesAnswerYesSuccess) { + Build http_directory = *HttpBuild::FromBuildString( + HttpBuildString{.url = "https://example.com/dist/"}); + Build gcs_object = *GcsBuild::FromBuildString( + GcsBuildString{.url = "gs://bucket/dist/phone-img-1.zip"}); + + EXPECT_TRUE(BuildHasArtifact(http_directory, "misc_info.txt")); + EXPECT_TRUE(BuildHasArtifact(gcs_object, "phone-img-1.zip")); + EXPECT_TRUE(BuildHasArtifact(DeviceBuild{.id = "123"}, "misc_info.txt")); +} + +TEST(BuildCacheKeyTests, AndroidBuildsAreTheIdAndTargetSuccess) { + Build device_build = DeviceBuild{.id = "123", .target = "test_target"}; + Build directory_build = + DirectoryBuild({"/tmp/build"}, "test_target", std::nullopt); + + EXPECT_EQ(BuildCacheKey(device_build, "img.zip"), "123/test_target"); + EXPECT_EQ(BuildCacheKey(directory_build, "img.zip"), "eng/test_target"); +} + +TEST(BuildCacheKeyTests, GcsObjectGenerationsDifferSuccess) { + GcsBuild build = *GcsBuild::FromBuildString( + GcsBuildString{.url = "gs://bucket/dist/phone-img-1.zip"}); + build.generation = "17"; + std::string first = *BuildCacheKey(build, "phone-img-1.zip"); + build.generation = "18"; + std::string second = *BuildCacheKey(build, "phone-img-1.zip"); + + EXPECT_THAT(first, EndsWith("/17")); + EXPECT_THAT(second, EndsWith("/18")); + EXPECT_THAT(first, StartsWith("url/")); + // The URL is what the two keys share. + EXPECT_EQ(first.substr(0, first.rfind('/')), + second.substr(0, second.rfind('/'))); +} + +TEST(BuildCacheKeyTests, GcsDirectoryArtifactsAreKeyedApartSuccess) { + GcsBuild build = + *GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket/dist/"}); + build.contents = { + {"a.txt", GcsObjectInfo{.generation = "1"}}, + {"b.txt", GcsObjectInfo{.generation = "2"}}, + }; + + EXPECT_THAT(BuildCacheKey(build, "a.txt"), Optional(EndsWith("/1"))); + EXPECT_THAT(BuildCacheKey(build, "b.txt"), Optional(EndsWith("/2"))); + EXPECT_EQ(BuildCacheKey(build, "absent.txt"), std::nullopt); +} + +TEST(BuildCacheKeyTests, HttpObjectsUseTheEtagSuccess) { + HttpBuild build = + *HttpBuild::FromBuildString(HttpBuildString{.url = kSignedUrl}); + build.etag = "W/\"a/b\""; + + std::string key = *BuildCacheKey(build, "phone-img-1.zip"); + EXPECT_THAT(key, StartsWith("url/")); + EXPECT_THAT(key, Not(HasSubstr("\""))); + EXPECT_THAT(key, Not(HasSubstr("secret"))); + EXPECT_EQ(std::count(key.begin(), key.end(), '/'), 2); +} + +TEST(BuildCacheKeyTests, HttpObjectsFallBackToTheDigestSuccess) { + HttpBuild build = *HttpBuild::FromBuildString(HttpBuildString{ + .url = "https://example.com/dist/phone-img-1.zip", + .sha256 = std::string(64, 'a'), + }); + + EXPECT_THAT(BuildCacheKey(build, "phone-img-1.zip"), + Optional(EndsWith(std::string(64, 'a')))); +} + +TEST(BuildCacheKeyTests, HttpDirectoriesHaveNoKeySuccess) { + HttpBuild build = *HttpBuild::FromBuildString( + HttpBuildString{.url = "https://example.com/dist/"}); + + EXPECT_EQ(BuildCacheKey(build, "misc_info.txt"), std::nullopt); +} + } // namespace } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/caching_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/caching_build_api.cpp index 85970bce03d..af353fda9cb 100644 --- a/base/cvd/cuttlefish/host/libs/web/caching_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/caching_build_api.cpp @@ -15,6 +15,7 @@ #include "cuttlefish/host/libs/web/caching_build_api.h" +#include #include #include @@ -29,6 +30,7 @@ #include "cuttlefish/host/libs/web/android_build.h" #include "cuttlefish/host/libs/web/android_build_string.h" #include "cuttlefish/host/libs/web/build_api.h" +#include "cuttlefish/host/libs/web/digest.h" #include "cuttlefish/host/libs/zip/cached_zip_source.h" #include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" #include "cuttlefish/result/result.h" @@ -47,12 +49,11 @@ struct CachingPaths { }; Result ConstructCachePaths( - const std::string& cache_base, const Build& build, + const std::string& cache_base, const std::string& build_key, const std::string& target_directory, const std::string& artifact, const std::string& backup_artifact = "") { - const auto [id, target] = GetBuildIdAndTarget(build); auto result = CachingPaths{ - .build_cache = fmt::format("{}/{}/{}", cache_base, id, target), + .build_cache = fmt::format("{}/{}", cache_base, build_key), .target_artifact = ConstructTargetFilepath(target_directory, artifact), }; result.cache_artifact = ConstructTargetFilepath(result.build_cache, artifact); @@ -80,6 +81,16 @@ bool IsInCache(const std::string& filepath) { return exists; } +void WarnUnversioned(const Build& build, const std::string& artifact) { + // An artifact the build does not hold at all is absent, not unversioned; + // the download that follows reports it. + if (!BuildHasArtifact(build, artifact)) { + return; + } + VLOG(0) << "Not caching \"" << artifact << "\" of " << FetchLabel(build) + << ", which reports no version for it"; +} + } // namespace CachingBuildApi::CachingBuildApi(BuildApi& build_api, @@ -93,9 +104,22 @@ Result CachingBuildApi::GetBuild(const BuildString& build_string) { Result CachingBuildApi::DownloadFile( const Build& build, const std::string& target_directory, const std::string& artifact_name) { + const std::optional build_key = + BuildCacheKey(build, artifact_name); + if (!build_key.has_value()) { + WarnUnversioned(build, artifact_name); + return CF_EXPECT( + build_api_.DownloadFile(build, target_directory, artifact_name)); + } const auto paths = CF_EXPECT(ConstructCachePaths( - cache_base_path_, build, target_directory, artifact_name)); - if (!IsInCache(paths.cache_artifact)) { + cache_base_path_, *build_key, target_directory, artifact_name)); + if (IsInCache(paths.cache_artifact)) { + const std::optional sha256 = + ArtifactSha256(build, artifact_name); + if (sha256.has_value()) { + CF_EXPECT(VerifySha256(paths.cache_artifact, *sha256, artifact_name)); + } + } else { CF_EXPECT(build_api_.DownloadFile(build, paths.build_cache, artifact_name)); } return CF_EXPECT(LinkOrCopy(paths.cache_artifact, paths.target_artifact, @@ -105,8 +129,16 @@ Result CachingBuildApi::DownloadFile( Result CachingBuildApi::FileReader( const Build& build, const std::string& artifact) { SeekableZipSource source = CF_EXPECT(build_api_.FileReader(build, artifact)); - std::string cache_path = fmt::format("{}/{}", cache_base_path_, artifact); - return CF_EXPECT(CacheZipSource(std::move(source), cache_path)); + const std::optional build_key = BuildCacheKey(build, artifact); + if (!build_key.has_value()) { + WarnUnversioned(build, artifact); + return source; + } + const std::string build_cache = + fmt::format("{}/{}", cache_base_path_, *build_key); + CF_EXPECT(EnsureDirectoryExists(build_cache)); + return CF_EXPECT(CacheZipSource( + std::move(source), ConstructTargetFilepath(build_cache, artifact))); } } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/caching_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/caching_build_api_test.cpp new file mode 100644 index 00000000000..d4589a9156e --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/caching_build_api_test.cpp @@ -0,0 +1,179 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/caching_build_api.h" + +#include +#include + +#include "android-base/file.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/files/file_exists.h" +#include "cuttlefish/files/is_directory_empty.h" +#include "cuttlefish/host/libs/web/android_build.h" +#include "cuttlefish/host/libs/web/android_build_string.h" +#include "cuttlefish/host/libs/web/build_api.h" +#include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" +#include "cuttlefish/host/libs/zip/libzip_cc/writable_source.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::ElementsAre; +using ::testing::HasSubstr; +using ::testing::SizeIs; + +constexpr char kContents[] = "artifact bytes"; +constexpr char kContentsSha256[] = + "4659fc0570122b0e0aa14f4ff7c261b1fe51795a01ba79963f462ebf40d7520d"; + +// Records where the caching layer asked for each download and writes the bytes +// a real API would have written. +class RecordingBuildApi : public BuildApi { + public: + Result GetBuild(const BuildString&) override { + return CF_ERR("not used"); + } + + Result DownloadFile(const Build&, + const std::string& target_directory, + const std::string& artifact_name) override { + directories.push_back(target_directory); + CF_EXPECT(EnsureDirectoryExists(target_directory)); + std::string path = target_directory + "/" + artifact_name; + CF_EXPECT(WriteNewFile(path, kContents)); + return path; + } + + Result FileReader(const Build&, + const std::string&) override { + return CF_EXPECT( + WritableZipSource::BorrowData(contents.data(), contents.size())); + } + + std::vector directories; + std::string contents = kContents; +}; + +class CachingBuildApiTests : public ::testing::Test { + protected: + std::string CachePath(const std::string& relative) const { + return std::string(cache_.path) + "/" + relative; + } + + RecordingBuildApi inner_; + TemporaryDir cache_; + TemporaryDir target_; + CachingBuildApi api_{inner_, std::string(cache_.path)}; +}; + +TEST_F(CachingBuildApiTests, DownloadFileKeysOnTheBuildIdAndTargetSuccess) { + Build build = DeviceBuild{.id = "123", .target = "test_target"}; + + EXPECT_THAT(api_.DownloadFile(build, target_.path, "img.zip"), + IsOkAndValue(std::string(target_.path) + "/img.zip")); + EXPECT_THAT(inner_.directories, ElementsAre(CachePath("123/test_target"))); +} + +TEST_F(CachingBuildApiTests, DownloadFileServesASecondCallFromTheCacheSuccess) { + Build build = DeviceBuild{.id = "123", .target = "test_target"}; + + ASSERT_THAT(api_.DownloadFile(build, target_.path, "img.zip"), IsOk()); + ASSERT_THAT(api_.DownloadFile(build, target_.path, "img.zip"), IsOk()); + EXPECT_THAT(inner_.directories, SizeIs(1)); +} + +TEST_F(CachingBuildApiTests, FileReaderKeysOnTheBuildSuccess) { + Build build = DeviceBuild{.id = "123", .target = "test_target"}; + + EXPECT_THAT(api_.FileReader(build, "img.zip"), IsOk()); + EXPECT_TRUE(FileExists(CachePath("123/test_target/img.zip"))); +} + +TEST_F(CachingBuildApiTests, DownloadFileKeysOnTheObjectGenerationSuccess) { + GcsBuild build = *GcsBuild::FromBuildString( + GcsBuildString{.url = "gs://bucket/dist/phone-img-1.zip"}); + build.generation = "17"; + ASSERT_THAT(api_.DownloadFile(build, target_.path, "phone-img-1.zip"), + IsOk()); + build.generation = "18"; + ASSERT_THAT(api_.DownloadFile(build, target_.path, "phone-img-1.zip"), + IsOk()); + + EXPECT_THAT(inner_.directories, SizeIs(2)); + EXPECT_NE(inner_.directories[0], inner_.directories[1]); + EXPECT_THAT(inner_.directories[0], HasSubstr("/url/")); +} + +TEST_F(CachingBuildApiTests, DownloadFileKeysEachListedObjectSuccess) { + GcsBuild build = + *GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket/dist/"}); + build.contents = { + {"a.txt", GcsObjectInfo{.generation = "1"}}, + {"b.txt", GcsObjectInfo{.generation = "2"}}, + }; + + ASSERT_THAT(api_.DownloadFile(build, target_.path, "a.txt"), IsOk()); + ASSERT_THAT(api_.DownloadFile(build, target_.path, "b.txt"), IsOk()); + + EXPECT_THAT(inner_.directories, SizeIs(2)); + EXPECT_NE(inner_.directories[0], inner_.directories[1]); +} + +TEST_F(CachingBuildApiTests, + DownloadFileOfAnUnversionedArtifactSkipsTheCacheSuccess) { + Build build = *HttpBuild::FromBuildString( + HttpBuildString{.url = "https://example.com/dist/"}); + + EXPECT_THAT(api_.DownloadFile(build, target_.path, "misc_info.txt"), + IsOkAndValue(std::string(target_.path) + "/misc_info.txt")); + EXPECT_THAT(inner_.directories, ElementsAre(std::string(target_.path))); + EXPECT_THAT(IsDirectoryEmpty(cache_.path), IsOkAndValue(true)); +} + +// The listing answers that the build does not hold the artifact, so the +// caching layer has nothing to say about it and the download reports it. +TEST_F(CachingBuildApiTests, + DownloadFileOfAnUnlistedArtifactSkipsTheCacheSuccess) { + GcsBuild build = + *GcsBuild::FromBuildString(GcsBuildString{.url = "gs://bucket/dist/"}); + build.contents = {{"a.txt", GcsObjectInfo{.generation = "1"}}}; + + EXPECT_THAT(api_.DownloadFile(build, target_.path, "misc_info.txt"), + IsOkAndValue(std::string(target_.path) + "/misc_info.txt")); + EXPECT_THAT(inner_.directories, ElementsAre(std::string(target_.path))); + EXPECT_THAT(IsDirectoryEmpty(cache_.path), IsOkAndValue(true)); +} + +TEST_F(CachingBuildApiTests, DownloadFileChecksTheCachedDigestFail) { + GcsBuild build = *GcsBuild::FromBuildString(GcsBuildString{ + .url = "gs://bucket/dist/phone-img-1.zip", + .sha256 = std::string(64, 'a'), + }); + build.generation = "17"; + + ASSERT_THAT(api_.DownloadFile(build, target_.path, "phone-img-1.zip"), + IsOk()); + EXPECT_THAT(api_.DownloadFile(build, target_.path, "phone-img-1.zip"), + IsErrorAndMessage(HasSubstr(kContentsSha256))); +} + +} // namespace +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/digest.cpp b/base/cvd/cuttlefish/host/libs/web/digest.cpp index 06d33e99e1a..88b5687e225 100644 --- a/base/cvd/cuttlefish/host/libs/web/digest.cpp +++ b/base/cvd/cuttlefish/host/libs/web/digest.cpp @@ -28,6 +28,7 @@ #include "absl/strings/escaping.h" #include "openssl/base.h" #include "openssl/digest.h" +#include "openssl/sha.h" #include "cuttlefish/common/libs/fs/fd.h" #include "cuttlefish/common/libs/utils/base64.h" @@ -63,12 +64,22 @@ Result> DigestFile(const std::string& path, return value; } +std::string HexDigest(const uint8_t* value, size_t size) { + return absl::BytesToHexString( + std::string_view(reinterpret_cast(value), size)); +} + } // namespace Result Sha256File(const std::string& path) { const std::vector value = CF_EXPECT(DigestFile(path, EVP_sha256())); - return absl::BytesToHexString(std::string_view( - reinterpret_cast(value.data()), value.size())); + return HexDigest(value.data(), value.size()); +} + +std::string Sha256Hex(std::string_view data) { + uint8_t value[SHA256_DIGEST_LENGTH]; + SHA256(reinterpret_cast(data.data()), data.size(), value); + return HexDigest(value, sizeof(value)); } Result VerifySha256(const std::string& path, std::string_view expected, diff --git a/base/cvd/cuttlefish/host/libs/web/digest.h b/base/cvd/cuttlefish/host/libs/web/digest.h index acde53e11a3..27cd12e7fd9 100644 --- a/base/cvd/cuttlefish/host/libs/web/digest.h +++ b/base/cvd/cuttlefish/host/libs/web/digest.h @@ -22,15 +22,18 @@ namespace cuttlefish { -// The SHA-256 of the contents of `path`, in lowercase hexadecimal. +// Returns the SHA-256 of the contents of `path`, in lowercase hexadecimal. Result Sha256File(const std::string& path); +// Returns the SHA-256 of `data`, in lowercase hexadecimal. +std::string Sha256Hex(std::string_view data); + // Fails unless `path` holds the hexadecimal SHA-256 `expected`, which is // compared without regard to case. `artifact_name` names the file in the error. Result VerifySha256(const std::string& path, std::string_view expected, std::string_view artifact_name); -// The same against the base64 MD5 that Cloud Storage reports for an object. +// Checks the same against the base64 MD5 Cloud Storage reports for an object. Result VerifyMd5(const std::string& path, std::string_view expected, std::string_view artifact_name); diff --git a/base/cvd/cuttlefish/host/libs/web/digest_test.cpp b/base/cvd/cuttlefish/host/libs/web/digest_test.cpp index 818490f530b..19dd7fae5f3 100644 --- a/base/cvd/cuttlefish/host/libs/web/digest_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/digest_test.cpp @@ -65,6 +65,11 @@ TEST(DigestTests, Sha256OfMoreThanOneReadSuccess) { "8f990ba0b577b51cf009ea049368c16bbda1b21e1b93be07a824758bb253c39b")); } +TEST(DigestTests, Sha256OfAStringSuccess) { + EXPECT_EQ(Sha256Hex("abc"), kAbcSha256); + EXPECT_EQ(Sha256Hex(""), kEmptySha256); +} + TEST(DigestTests, MissingFileFail) { TemporaryDir directory; EXPECT_THAT(Sha256File(std::string(directory.path) + "/absent"), From 2685ae379824386013fa2ade91169a8d7b5a71c7 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:40 +0000 Subject: [PATCH 18/20] Resume an interrupted URL artifact download --- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 39 +++- .../host/libs/web/gcs_build_api.cpp | 38 +++- .../host/libs/web/http_build_api.cpp | 16 +- .../cuttlefish/host/libs/web/url_download.cpp | 143 ++++++++++++++ .../cuttlefish/host/libs/web/url_download.h | 49 +++++ .../host/libs/web/url_download_test.cpp | 178 ++++++++++++++++++ 6 files changed, 447 insertions(+), 16 deletions(-) create mode 100644 base/cvd/cuttlefish/host/libs/web/url_download.cpp create mode 100644 base/cvd/cuttlefish/host/libs/web/url_download.h create mode 100644 base/cvd/cuttlefish/host/libs/web/url_download_test.cpp diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index 8344277ffed..f367ee9d55a 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -304,9 +304,9 @@ cf_cc_library( "//cuttlefish/host/libs/web:build_api_zip", "//cuttlefish/host/libs/web:credential_source", "//cuttlefish/host/libs/web:digest", + "//cuttlefish/host/libs/web:url_download", "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/web/http_client", - "//cuttlefish/host/libs/web/http_client:http_file", "//cuttlefish/host/libs/web/http_client:http_json", "//cuttlefish/host/libs/web/http_client:url_escape", "//cuttlefish/host/libs/zip:remote_zip", @@ -353,9 +353,9 @@ cf_cc_library( "//cuttlefish/host/libs/web:android_build_string", "//cuttlefish/host/libs/web:build_api_zip", "//cuttlefish/host/libs/web:digest", + "//cuttlefish/host/libs/web:url_download", "//cuttlefish/host/libs/web:url_namespace", "//cuttlefish/host/libs/web/http_client", - "//cuttlefish/host/libs/web/http_client:http_file", "//cuttlefish/host/libs/zip:remote_zip", "//cuttlefish/host/libs/zip:zip_file", "//cuttlefish/host/libs/zip/libzip_cc:archive", @@ -448,6 +448,41 @@ cf_cc_test( ], ) +cf_cc_library( + name = "url_download", + srcs = ["url_download.cpp"], + hdrs = ["url_download.h"], + deps = [ + "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/host/libs/web/http_client", + "//cuttlefish/host/libs/web/http_client:http_file", + "//cuttlefish/host/libs/web/http_client:scrub_secrets", + "//cuttlefish/io:write_exact", + "//cuttlefish/posix:remove", + "//cuttlefish/posix:rename", + "//cuttlefish/result", + "@abseil-cpp//absl/log", + "@fmt", + ], +) + +cf_cc_test( + name = "url_download_test", + srcs = ["url_download_test.cpp"], + deps = [ + "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/files:file_exists", + "//cuttlefish/host/libs/web:url_download", + "//cuttlefish/host/libs/web/http_client", + "//cuttlefish/host/libs/web/http_client:fake_http_client", + "//cuttlefish/result", + "//cuttlefish/result:result_matchers", + "//libbase", + "@abseil-cpp//absl/strings", + ], +) + cf_cc_library( name = "url_namespace", srcs = ["url_namespace.cpp"], diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp index e8894fea62d..8789ba1876d 100644 --- a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp @@ -39,9 +39,9 @@ #include "cuttlefish/host/libs/web/credential_source.h" #include "cuttlefish/host/libs/web/digest.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" -#include "cuttlefish/host/libs/web/http_client/http_file.h" #include "cuttlefish/host/libs/web/http_client/http_json.h" #include "cuttlefish/host/libs/web/http_client/url_escape.h" +#include "cuttlefish/host/libs/web/url_download.h" #include "cuttlefish/host/libs/web/url_namespace.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" #include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" @@ -120,6 +120,19 @@ std::optional ArtifactSize(const GcsBuild& build, return entry->second.size; } +std::optional ArtifactGeneration( + const GcsBuild& build, const std::string& artifact_name) { + if (build.object.has_value()) { + return build.generation; + } + const std::map::const_iterator entry = + build.contents.find(artifact_name); + if (entry == build.contents.end()) { + return std::nullopt; + } + return entry->second.generation; +} + // The listing and the metadata probe both report an md5, so a `gs://` download // is checked whether or not the build string carried a digest. Result VerifyArtifact(const GcsBuild& build, @@ -253,13 +266,24 @@ Result GcsBuildApi::DownloadFile( return dest_path; } - const std::string url = + std::string url = MediaUrl(build.bucket, CF_EXPECT(ObjectName(build, artifact_name))); - HttpResponse response = CF_EXPECT( - HttpGetToFile(http_client_, url, dest_path, CF_EXPECT(Headers()))); - CF_EXPECTF(response.HttpSuccess(), - "Could not download '{}' from '{}' - {}:{}", artifact_name, - build.id, response.http_code, response.StatusDescription()); + std::optional generation = + ArtifactGeneration(build, artifact_name); + if (generation.has_value()) { + // Naming the generation fixes the bytes the URL serves, which is what a + // resumed download needs of it. + absl::StrAppend(&url, "&generation=", *generation); + } + + const UrlDownload download = { + .url = url, + .headers = CF_EXPECT(Headers()), + .resumable = generation.has_value(), + .size = ArtifactSize(build, artifact_name), + }; + CF_EXPECTF(DownloadUrlToFile(http_client_, download, dest_path), + "Could not download '{}' from '{}'", artifact_name, build.id); CF_EXPECT(VerifyArtifact(build, artifact_name, dest_path)); return dest_path; } diff --git a/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp index 09387142f98..03b4f6778e6 100644 --- a/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/http_build_api.cpp @@ -32,7 +32,7 @@ #include "cuttlefish/host/libs/web/build_api_zip.h" #include "cuttlefish/host/libs/web/digest.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" -#include "cuttlefish/host/libs/web/http_client/http_file.h" +#include "cuttlefish/host/libs/web/url_download.h" #include "cuttlefish/host/libs/web/url_namespace.h" #include "cuttlefish/host/libs/zip/libzip_cc/archive.h" #include "cuttlefish/host/libs/zip/libzip_cc/seekable_source.h" @@ -149,12 +149,14 @@ Result HttpBuildApi::DownloadFile( return dest_path; } - const std::string url = CF_EXPECT(ArtifactUrl(build, artifact_name)); - HttpResponse response = - CF_EXPECT(HttpGetToFile(http_client_, url, dest_path)); - CF_EXPECTF(response.HttpSuccess(), - "Could not download '{}' from '{}' - {}:{}", artifact_name, - build.id, response.http_code, response.StatusDescription()); + const UrlDownload download = { + .url = CF_EXPECT(ArtifactUrl(build, artifact_name)), + .if_range = build.etag, + .resumable = build.accept_ranges && build.etag.has_value(), + .size = build.size, + }; + CF_EXPECTF(DownloadUrlToFile(http_client_, download, dest_path), + "Could not download '{}' from '{}'", artifact_name, build.id); if (build.sha256.has_value()) { CF_EXPECT(VerifySha256(dest_path, *build.sha256, artifact_name)); } diff --git a/base/cvd/cuttlefish/host/libs/web/url_download.cpp b/base/cvd/cuttlefish/host/libs/web/url_download.cpp new file mode 100644 index 00000000000..2680f3c6c5f --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/url_download.cpp @@ -0,0 +1,143 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/url_download.h" + +#include +#include +#include +#include + +#include + +#include "absl/log/log.h" +#include "fmt/format.h" + +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/host/libs/web/http_client/http_file.h" +#include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" +#include "cuttlefish/io/write_exact.h" +#include "cuttlefish/posix/remove.h" +#include "cuttlefish/posix/rename.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { +namespace { + +Result FullDownload(HttpClient& http_client, const UrlDownload& download, + const std::string& path) { + const HttpResponse response = CF_EXPECT( + HttpGetToFile(http_client, download.url, path, download.headers)); + CF_EXPECTF(response.HttpSuccess(), "'{}' - {}:{}", ScrubUrl(download.url), + response.http_code, response.StatusDescription()); + return {}; +} + +} // namespace + +Result DownloadUrlToFile(HttpClient& http_client, + const UrlDownload& download, + const std::string& path) { + // Without something to resume against, a unique temporary file per attempt + // keeps concurrent downloads of the same artifact out of each other's way. + if (!download.resumable) { + CF_EXPECT(FullDownload(http_client, download, path)); + return {}; + } + + // Resuming keeps the file HttpGetToFile would hide in a temporary: the + // offset an interrupted attempt left off at comes from that file, and the + // lock that serializes other `cvd` invocations sits on its descriptor. + const std::string part_path = fmt::format("{}.part", path); + Fd part = CF_EXPECT(Fd::Open(part_path, O_RDWR | O_CREAT, 0644)); + CF_EXPECTF(part.Flock(LOCK_EX), "Could not lock '{}'", part_path); + + const uint64_t have = + CF_EXPECTF(part.SeekEnd(0), "Could not measure '{}'", part_path); + + uint64_t offset = 0; + if (download.size.has_value() && have < *download.size) { + offset = have; + } + + uint64_t written = 0; + uint64_t last_log = 0; + while (true) { + HttpRequest request = { + .method = HttpMethod::kGet, + .url = download.url, + .headers = download.headers, + }; + if (offset > 0) { + request.headers.push_back(fmt::format("Range: bytes={}-", offset)); + if (download.if_range.has_value()) { + request.headers.push_back( + fmt::format("If-Range: {}", *download.if_range)); + } + } + + auto callback = [&part, &part_path, offset, &written, &last_log]( + char* data, size_t size) -> bool { + // A retry inside the client starts the response over from the range that + // was asked for. + if (data == nullptr) { + written = 0; + last_log = 0; + if (Result truncated = part.Truncate(offset); + !truncated.has_value()) { + LOG(ERROR) << truncated.error(); + return false; + } + return part.SeekSet(offset).has_value(); + } + if (Result written_data = WriteExact(part, data, size); + !written_data.has_value()) { + LOG(ERROR) << "Could not write '" << part_path + << "': " << written_data.error(); + return false; + } + written += size; + if (written / 2 >= last_log) { + VLOG(0) << "Downloaded " << offset + written << " bytes"; + last_log = written; + } + return true; + }; + + const HttpResponse response = + CF_EXPECT(http_client.DownloadToCallback(request, callback)); + if (!response.HttpSuccess()) { + // The body of an error response is not the artifact. + CF_EXPECT(RemoveFile(part_path)); + return CF_ERRF("'{}' - {}:{}", ScrubUrl(download.url), response.http_code, + response.StatusDescription()); + } + if (offset == 0 || !download.size.has_value() || + offset + written == *download.size) { + break; + } + LOG(WARNING) << "'" << ScrubUrl(download.url) + << "' answered a resumed request with the whole object"; + offset = 0; + } + + VLOG(0) << "Downloaded '" << offset + written << "' total bytes from '" + << ScrubUrl(download.url) << "' to '" << path << "'."; + CF_EXPECT(Rename(part_path, path)); + return {}; +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/url_download.h b/base/cvd/cuttlefish/host/libs/web/url_download.h new file mode 100644 index 00000000000..d522b1915e7 --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/url_download.h @@ -0,0 +1,49 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/result/result.h" + +namespace cuttlefish { + +struct UrlDownload { + std::string url; + std::vector headers; + // Sent as `If-Range` on a resumed request, so an origin whose object changed + // answers with the whole of it rather than the tail of something else. Unset + // for a URL that already names one version of the object. + std::optional if_range; + // Whether the probe found that `url` serves ranges and names bytes that do + // not change under it. Only then is a partial download worth keeping. + bool resumable = false; + std::optional size; +}; + +// Writes `download` to `path`, picking up where an interrupted earlier attempt +// left off when the origin allows it. This is the sequential whole-file path; +// random access into a remote zip lives in host/libs/zip/remote_zip.h. +Result DownloadUrlToFile(HttpClient& http_client, + const UrlDownload& download, + const std::string& path); + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/url_download_test.cpp b/base/cvd/cuttlefish/host/libs/web/url_download_test.cpp new file mode 100644 index 00000000000..0b72a6b29cd --- /dev/null +++ b/base/cvd/cuttlefish/host/libs/web/url_download_test.cpp @@ -0,0 +1,178 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cuttlefish/host/libs/web/url_download.h" + +#include +#include +#include + +#include +#include +#include + +#include "absl/strings/numbers.h" +#include "absl/strings/strip.h" +#include "android-base/file.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/files/file_exists.h" +#include "cuttlefish/host/libs/web/http_client/fake_http_client.h" +#include "cuttlefish/host/libs/web/http_client/http_client.h" +#include "cuttlefish/result/result.h" +#include "cuttlefish/result/result_matchers.h" + +namespace cuttlefish { +namespace { + +using ::testing::AllOf; +using ::testing::Contains; +using ::testing::HasSubstr; +using ::testing::Not; +using ::testing::SizeIs; + +constexpr char kUrl[] = "https://example.com/dist/phone-img-1.zip"; +constexpr char kContents[] = "0123456789ABCDEF"; +constexpr char kRangePrefix[] = "Range: bytes="; +constexpr size_t kSize = 16; + +size_t RangeStart(const std::vector& headers) { + size_t start = 0; + for (const std::string& header : headers) { + std::string_view range = header; + if (!absl::ConsumePrefix(&range, kRangePrefix)) { + continue; + } + EXPECT_TRUE(absl::SimpleAtoi(range.substr(0, range.find('-')), &start)); + } + return start; +} + +class UrlDownloadTests : public ::testing::Test { + protected: + std::string Path() const { return std::string(directory_.path) + "/img.zip"; } + std::string PartPath() const { return Path() + ".part"; } + + void WritePart(const std::string& contents) { + ASSERT_THAT(WriteNewFile(PartPath(), contents), IsOk()); + } + + // Serves whatever range is asked for, unless it is told to ignore ranges as + // an origin does when `If-Range` does not match what it holds. + void ServeContents(bool honor_ranges = true) { + http_client_.SetResponse( + [this, honor_ranges](const HttpRequest& request) { + requests_.push_back(request.headers); + size_t start = honor_ranges ? RangeStart(request.headers) : 0; + return HttpResponse{ + .data = std::string(kContents).substr(start), + .http_code = start > 0 ? 206 : 200, + }; + }, + kUrl); + } + + FakeHttpClient http_client_; + TemporaryDir directory_; + std::vector> requests_; +}; + +TEST_F(UrlDownloadTests, WithoutAValidatorTheWholeObjectIsDownloadedSuccess) { + ServeContents(); + UrlDownload download = {.url = kUrl, .size = kSize}; + + EXPECT_THAT(DownloadUrlToFile(http_client_, download, Path()), IsOk()); + EXPECT_THAT(ReadFileContents(Path()), IsOkAndValue(kContents)); + EXPECT_THAT(requests_, SizeIs(1)); + EXPECT_THAT(requests_[0], Not(Contains(HasSubstr("Range:")))); + EXPECT_FALSE(FileExists(PartPath())); +} + +TEST_F(UrlDownloadTests, APartialFileIsResumedSuccess) { + ServeContents(); + WritePart("012345"); + UrlDownload download = { + .url = kUrl, + .if_range = "\"v1\"", + .resumable = true, + .size = kSize, + }; + + EXPECT_THAT(DownloadUrlToFile(http_client_, download, Path()), IsOk()); + EXPECT_THAT(ReadFileContents(Path()), IsOkAndValue(kContents)); + EXPECT_THAT(requests_, SizeIs(1)); + EXPECT_THAT(requests_[0], + AllOf(Contains("Range: bytes=6-"), Contains("If-Range: \"v1\""))); + EXPECT_FALSE(FileExists(PartPath())); +} + +TEST_F(UrlDownloadTests, AVersionedUrlResumesWithoutIfRangeSuccess) { + ServeContents(); + WritePart("012345"); + UrlDownload download = {.url = kUrl, .resumable = true, .size = kSize}; + + EXPECT_THAT(DownloadUrlToFile(http_client_, download, Path()), IsOk()); + EXPECT_THAT(ReadFileContents(Path()), IsOkAndValue(kContents)); + EXPECT_THAT(requests_[0], AllOf(Contains("Range: bytes=6-"), + Not(Contains(HasSubstr("If-Range:"))))); +} + +TEST_F(UrlDownloadTests, AChangedObjectIsDownloadedAgainSuccess) { + ServeContents(/*honor_ranges=*/false); + WritePart("xxxxxx"); + UrlDownload download = { + .url = kUrl, + .if_range = "\"v1\"", + .resumable = true, + .size = kSize, + }; + + EXPECT_THAT(DownloadUrlToFile(http_client_, download, Path()), IsOk()); + EXPECT_THAT(ReadFileContents(Path()), IsOkAndValue(kContents)); + EXPECT_THAT(requests_, SizeIs(2)); + EXPECT_THAT(requests_[1], Not(Contains(HasSubstr("Range:")))); + EXPECT_FALSE(FileExists(PartPath())); +} + +TEST_F(UrlDownloadTests, APartialFileIsHeldUnderALockSuccess) { + bool locked_out = false; + http_client_.SetResponse( + [this, &locked_out](const HttpRequest&) { + Result other = Fd::Open(PartPath(), O_RDWR); + locked_out = + other.has_value() && !other->Flock(LOCK_EX | LOCK_NB).has_value(); + return HttpResponse{.data = kContents, .http_code = 200}; + }, + kUrl); + UrlDownload download = {.url = kUrl, .resumable = true, .size = kSize}; + + EXPECT_THAT(DownloadUrlToFile(http_client_, download, Path()), IsOk()); + EXPECT_TRUE(locked_out); +} + +TEST_F(UrlDownloadTests, AMissingObjectLeavesNoPartialFileFail) { + UrlDownload download = {.url = kUrl, .resumable = true, .size = kSize}; + + EXPECT_THAT(DownloadUrlToFile(http_client_, download, Path()), + IsErrorAndMessage(AllOf(HasSubstr(kUrl), HasSubstr("404")))); + EXPECT_FALSE(FileExists(PartPath())); + EXPECT_FALSE(FileExists(Path())); +} + +} // namespace +} // namespace cuttlefish From 8739df450c3626c44812f25b239ce8031120fd31 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:41 +0000 Subject: [PATCH 19/20] Pin ranged Cloud Storage reads to one object generation --- .../host/libs/web/gcs_build_api.cpp | 32 +++++++++++-------- .../host/libs/web/gcs_build_api_test.cpp | 31 ++++++++++++++++++ 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp index 8789ba1876d..90d4ef13e6a 100644 --- a/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api.cpp @@ -133,6 +133,19 @@ std::optional ArtifactGeneration( return entry->second.generation; } +// The media URL of one artifact, naming the generation wherever it is known so +// that every request against the URL reads one version of the object. +Result ArtifactUrl(const GcsBuild& build, + const std::string& artifact_name) { + std::string url = + MediaUrl(build.bucket, CF_EXPECT(ObjectName(build, artifact_name))); + if (std::optional generation = + ArtifactGeneration(build, artifact_name)) { + url += fmt::format("&generation={}", *generation); + } + return url; +} + // The listing and the metadata probe both report an md5, so a `gs://` download // is checked whether or not the build string carried a digest. Result VerifyArtifact(const GcsBuild& build, @@ -266,20 +279,12 @@ Result GcsBuildApi::DownloadFile( return dest_path; } - std::string url = - MediaUrl(build.bucket, CF_EXPECT(ObjectName(build, artifact_name))); - std::optional generation = - ArtifactGeneration(build, artifact_name); - if (generation.has_value()) { - // Naming the generation fixes the bytes the URL serves, which is what a - // resumed download needs of it. - absl::StrAppend(&url, "&generation=", *generation); - } - + // A generation makes the bytes the URL serves fixed, which is what a resumed + // download needs of it. const UrlDownload download = { - .url = url, + .url = CF_EXPECT(ArtifactUrl(build, artifact_name)), .headers = CF_EXPECT(Headers()), - .resumable = generation.has_value(), + .resumable = ArtifactGeneration(build, artifact_name).has_value(), .size = ArtifactSize(build, artifact_name), }; CF_EXPECTF(DownloadUrlToFile(http_client_, download, dest_path), @@ -290,8 +295,7 @@ Result GcsBuildApi::DownloadFile( Result GcsBuildApi::FileReader( const GcsBuild& build, const std::string& artifact_name) { - const std::string url = - MediaUrl(build.bucket, CF_EXPECT(ObjectName(build, artifact_name))); + const std::string url = CF_EXPECT(ArtifactUrl(build, artifact_name)); std::vector headers = CF_EXPECT(Headers()); if (std::optional size = ArtifactSize(build, artifact_name)) { return CF_EXPECT( diff --git a/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp index 0b1dedf815f..cf028c08344 100644 --- a/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/gcs_build_api_test.cpp @@ -303,10 +303,39 @@ TEST(GcsBuildApiTests, FileReaderReadsTheNamedArtifactSuccess) { ASSERT_THAT(member, IsOk()); EXPECT_THAT(ReadToString(**member), IsOkAndValue("boot bytes")); EXPECT_TRUE(http_client.RequestMade(kMediaUrl)); + EXPECT_TRUE(http_client.RequestMade("&generation=1")); // The listing reported the size, so the reader has nothing left to ask. EXPECT_FALSE(zip_handler->HeadRequestMade()); } +TEST(GcsBuildApiTests, FileReaderReadsOneGenerationOfAnObjectSuccess) { + FakeHttpClient http_client; + GcsBuildApi api(http_client, nullptr); + + Result zip_handler = + ZipOverRanges::Create({{"boot.img", "boot bytes"}}); + ASSERT_THAT(zip_handler, IsOk()); + http_client.SetResponse(*zip_handler, kMediaUrl); + http_client.SetResponse(absl::StrCat(R"({"generation": "17", "size": ")", + zip_handler->Size(), R"("})"), + kObjectUrl); + + GcsBuildString build_string = {.url = "gs://bucket/dist/phone-img-1.zip"}; + Result build = api.GetBuild(build_string); + ASSERT_THAT(build, IsOk()); + + Result source = api.FileReader(*build, "phone-img-1.zip"); + ASSERT_THAT(source, IsOk()); + Result zip = ReadableZip::FromSource(std::move(*source)); + ASSERT_THAT(zip, IsOk()); + Result> member = zip->OpenReadOnly("boot.img"); + ASSERT_THAT(member, IsOk()); + EXPECT_THAT(ReadToString(**member), IsOkAndValue("boot bytes")); + // Every range of the read names the version the probe reported. + EXPECT_TRUE(zip_handler->RangeRequestMade()); + EXPECT_TRUE(http_client.RequestMade("&generation=17")); +} + TEST(GcsBuildApiTests, DownloadFileExtractsAnArchiveMemberSuccess) { FakeHttpClient http_client; GcsBuildApi api(http_client, nullptr); @@ -335,6 +364,8 @@ TEST(GcsBuildApiTests, DownloadFileExtractsAnArchiveMemberSuccess) { EXPECT_THAT(ReadFileContents(expected_path), IsOkAndValue("package bytes")); EXPECT_TRUE(zip_handler->RangeRequestMade()); EXPECT_FALSE(zip_handler->HeadRequestMade()); + // The probe reported no generation, so there is no version to name. + EXPECT_FALSE(http_client.RequestMade("generation=")); } } // namespace From 7418bc7a247e7c06283a0b349a5a030c7473d257 Mon Sep 17 00:00:00 2001 From: Lars Ershammar Date: Fri, 21 Aug 2026 07:27:41 +0000 Subject: [PATCH 20/20] Start a partial download from the offset it will be written at --- base/cvd/cuttlefish/host/libs/web/BUILD.bazel | 1 + .../cuttlefish/host/libs/web/url_download.cpp | 76 ++++++++++++++----- .../cuttlefish/host/libs/web/url_download.h | 15 ++-- .../host/libs/web/url_download_test.cpp | 56 ++++++++++++++ 4 files changed, 122 insertions(+), 26 deletions(-) diff --git a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel index f367ee9d55a..448e277ae0f 100644 --- a/base/cvd/cuttlefish/host/libs/web/BUILD.bazel +++ b/base/cvd/cuttlefish/host/libs/web/BUILD.bazel @@ -454,6 +454,7 @@ cf_cc_library( hdrs = ["url_download.h"], deps = [ "//cuttlefish/common/libs/fs:fd", + "//cuttlefish/files:file_exists", "//cuttlefish/host/libs/web/http_client", "//cuttlefish/host/libs/web/http_client:http_file", "//cuttlefish/host/libs/web/http_client:scrub_secrets", diff --git a/base/cvd/cuttlefish/host/libs/web/url_download.cpp b/base/cvd/cuttlefish/host/libs/web/url_download.cpp index 2680f3c6c5f..16d0bfdac8c 100644 --- a/base/cvd/cuttlefish/host/libs/web/url_download.cpp +++ b/base/cvd/cuttlefish/host/libs/web/url_download.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include @@ -26,6 +27,7 @@ #include "fmt/format.h" #include "cuttlefish/common/libs/fs/fd.h" +#include "cuttlefish/files/file_exists.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/host/libs/web/http_client/http_file.h" #include "cuttlefish/host/libs/web/http_client/scrub_secrets.h" @@ -37,6 +39,8 @@ namespace cuttlefish { namespace { +constexpr int kLockAttempts = 4; + Result FullDownload(HttpClient& http_client, const UrlDownload& download, const std::string& path) { const HttpResponse response = CF_EXPECT( @@ -46,25 +50,13 @@ Result FullDownload(HttpClient& http_client, const UrlDownload& download, return {}; } -} // namespace - -Result DownloadUrlToFile(HttpClient& http_client, - const UrlDownload& download, - const std::string& path) { - // Without something to resume against, a unique temporary file per attempt - // keeps concurrent downloads of the same artifact out of each other's way. - if (!download.resumable) { - CF_EXPECT(FullDownload(http_client, download, path)); - return {}; - } - - // Resuming keeps the file HttpGetToFile would hide in a temporary: the - // offset an interrupted attempt left off at comes from that file, and the - // lock that serializes other `cvd` invocations sits on its descriptor. - const std::string part_path = fmt::format("{}.part", path); - Fd part = CF_EXPECT(Fd::Open(part_path, O_RDWR | O_CREAT, 0644)); - CF_EXPECTF(part.Flock(LOCK_EX), "Could not lock '{}'", part_path); - +// Resuming keeps the file HttpGetToFile would hide in a temporary: the offset +// an interrupted attempt left off at comes from that file, and the lock that +// serializes other `cvd` invocations sits on its descriptor. +Result ResumeDownload(HttpClient& http_client, + const UrlDownload& download, Fd& part, + const std::string& part_path, + const std::string& path) { const uint64_t have = CF_EXPECTF(part.SeekEnd(0), "Could not measure '{}'", part_path); @@ -72,6 +64,9 @@ Result DownloadUrlToFile(HttpClient& http_client, if (download.size.has_value() && have < *download.size) { offset = have; } + // Anything past the offset belongs to a download of something else. + CF_EXPECTF(part.Truncate(offset), "Could not truncate '{}'", part_path); + CF_EXPECTF(part.SeekSet(offset), "Could not seek '{}'", part_path); uint64_t written = 0; uint64_t last_log = 0; @@ -140,4 +135,47 @@ Result DownloadUrlToFile(HttpClient& http_client, return {}; } +} // namespace + +Result HoldsFileAt(Fd& fd, const std::string& path) { + // The descriptor is open before the lock says whose file it is, so comparing + // two paths would race with the rename that ends another download. + struct stat by_path = {}; + if (stat(path.c_str(), &by_path) != 0) { + return false; + } + const struct stat by_fd = CF_EXPECTF(fd.Fstat(), "Could not read '{}'", path); + return by_path.st_dev == by_fd.st_dev && by_path.st_ino == by_fd.st_ino; +} + +Result DownloadUrlToFile(HttpClient& http_client, + const UrlDownload& download, + const std::string& path) { + // Without something to resume against, a unique temporary file per attempt + // keeps concurrent downloads of the same artifact out of each other's way. + if (!download.resumable) { + CF_EXPECT(FullDownload(http_client, download, path)); + return {}; + } + + const std::string part_path = fmt::format("{}.part", path); + // The lock serializes other `cvd` invocations downloading this artifact into + // the shared generation-keyed cache; a fetch itself is single-threaded. + for (int attempt = 0; attempt < kLockAttempts; attempt++) { + Fd part = CF_EXPECT(Fd::Open(part_path, O_RDWR | O_CREAT, 0644)); + CF_EXPECTF(part.Flock(LOCK_EX), "Could not lock '{}'", part_path); + + if (CF_EXPECT(HoldsFileAt(part, part_path))) { + CF_EXPECT(ResumeDownload(http_client, download, part, part_path, path)); + return {}; + } + // Another download of this artifact renamed the partial file away while + // this one waited for its lock. + if (FileExists(path)) { + return {}; + } + } + return CF_ERRF("Gave up waiting for another download of '{}'", part_path); +} + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/libs/web/url_download.h b/base/cvd/cuttlefish/host/libs/web/url_download.h index d522b1915e7..987d9b83651 100644 --- a/base/cvd/cuttlefish/host/libs/web/url_download.h +++ b/base/cvd/cuttlefish/host/libs/web/url_download.h @@ -21,6 +21,7 @@ #include #include +#include "cuttlefish/common/libs/fs/fd.h" #include "cuttlefish/host/libs/web/http_client/http_client.h" #include "cuttlefish/result/result.h" @@ -29,16 +30,16 @@ namespace cuttlefish { struct UrlDownload { std::string url; std::vector headers; - // Sent as `If-Range` on a resumed request, so an origin whose object changed - // answers with the whole of it rather than the tail of something else. Unset - // for a URL that already names one version of the object. - std::optional if_range; - // Whether the probe found that `url` serves ranges and names bytes that do - // not change under it. Only then is a partial download worth keeping. - bool resumable = false; + std::optional if_range; // unset when `url` pins one version + bool resumable = false; // whether a partial file is worth keeping std::optional size; }; +// Returns whether `path` still names the file `fd` holds open. A download +// that another process finished renames its partial file away, so whoever was +// waiting for the lock on it wakes up holding a file that is gone. +Result HoldsFileAt(Fd& fd, const std::string& path); + // Writes `download` to `path`, picking up where an interrupted earlier attempt // left off when the origin allows it. This is the sequential whole-file path; // random access into a remote zip lives in host/libs/zip/remote_zip.h. diff --git a/base/cvd/cuttlefish/host/libs/web/url_download_test.cpp b/base/cvd/cuttlefish/host/libs/web/url_download_test.cpp index 0b72a6b29cd..3a52ee4c1c3 100644 --- a/base/cvd/cuttlefish/host/libs/web/url_download_test.cpp +++ b/base/cvd/cuttlefish/host/libs/web/url_download_test.cpp @@ -132,6 +132,35 @@ TEST_F(UrlDownloadTests, AVersionedUrlResumesWithoutIfRangeSuccess) { Not(Contains(HasSubstr("If-Range:"))))); } +TEST_F(UrlDownloadTests, APartialFileLongerThanTheObjectIsDiscardedSuccess) { + ServeContents(); + WritePart(std::string(kSize + 8, 'z')); + UrlDownload download = { + .url = kUrl, + .if_range = "\"v1\"", + .resumable = true, + .size = kSize, + }; + + EXPECT_THAT(DownloadUrlToFile(http_client_, download, Path()), IsOk()); + EXPECT_THAT(ReadFileContents(Path()), IsOkAndValue(kContents)); + EXPECT_THAT(requests_[0], Not(Contains(HasSubstr("Range:")))); +} + +TEST_F(UrlDownloadTests, APartialFileOfAnUnmeasuredObjectIsDiscardedSuccess) { + ServeContents(); + WritePart("zzzz"); + UrlDownload download = { + .url = kUrl, + .if_range = "\"v1\"", + .resumable = true, + }; + + EXPECT_THAT(DownloadUrlToFile(http_client_, download, Path()), IsOk()); + EXPECT_THAT(ReadFileContents(Path()), IsOkAndValue(kContents)); + EXPECT_THAT(requests_[0], Not(Contains(HasSubstr("Range:")))); +} + TEST_F(UrlDownloadTests, AChangedObjectIsDownloadedAgainSuccess) { ServeContents(/*honor_ranges=*/false); WritePart("xxxxxx"); @@ -165,6 +194,33 @@ TEST_F(UrlDownloadTests, APartialFileIsHeldUnderALockSuccess) { EXPECT_TRUE(locked_out); } +TEST_F(UrlDownloadTests, AnOpenFileIsFoundAtItsPathSuccess) { + WritePart("012345"); + Result part = Fd::Open(PartPath(), O_RDWR); + ASSERT_THAT(part, IsOk()); + + EXPECT_THAT(HoldsFileAt(*part, PartPath()), IsOkAndValue(true)); +} + +TEST_F(UrlDownloadTests, AnOpenFileRenamedAwayIsNotFoundSuccess) { + WritePart("012345"); + Result part = Fd::Open(PartPath(), O_RDWR); + ASSERT_THAT(part, IsOk()); + ASSERT_THAT(RenameFile(PartPath(), Path()), IsOk()); + + EXPECT_THAT(HoldsFileAt(*part, PartPath()), IsOkAndValue(false)); +} + +TEST_F(UrlDownloadTests, AnOpenFileReplacedAtItsPathIsNotFoundSuccess) { + WritePart("012345"); + Result part = Fd::Open(PartPath(), O_RDWR); + ASSERT_THAT(part, IsOk()); + ASSERT_THAT(RenameFile(PartPath(), Path()), IsOk()); + WritePart("6789"); + + EXPECT_THAT(HoldsFileAt(*part, PartPath()), IsOkAndValue(false)); +} + TEST_F(UrlDownloadTests, AMissingObjectLeavesNoPartialFileFail) { UrlDownload download = {.url = kUrl, .resumable = true, .size = kSize};