Support gs:// and https:// build sources in cvd fetch and cvd load

base/cvd/cuttlefish/common/libs/fs/fd.cc

@@ -699,6 +699,15 @@ Result<void> Fd::Flock(int operation) {
   return {};
 }
 
+Result<struct stat> Fd::Fstat() {
+  LocalErrno record_errno(errno_);
+
+  struct stat file_info = {};
+  CF_EXPECT(TEMP_FAILURE_RETRY(fstat(fd_, &file_info)) == 0,
+            ::cuttlefish::StrError(errno));
+  return file_info;
+}
+
 int Fd::GetSockName(struct sockaddr* addr, socklen_t* addrlen) {
   LocalErrno record_errno(errno_);
 

base/cvd/cuttlefish/common/libs/fs/fd.h

@@ -183,6 +183,8 @@ class Fd : public ReaderWriterSeeker {
 
   Result<void> Flock(int operation);
 
+  Result<struct stat> Fstat();
+
   int GetErrno() const { return errno_; }
   int GetSockName(struct sockaddr* addr, socklen_t* addrlen);
 

base/cvd/cuttlefish/host/commands/cvd/cli/commands/load_configs.cpp

@@ -292,10 +292,14 @@ std::vector<HelpParagraph> LoadConfigsCommand::CommonCommandDescription() {
   description.emplace_back(
       "While most config file properties are self explanatory, the build "
       "properties (default_build, kernel.build, bootloader.build, etc) require "
-      "more explanation. These properties support two types of values:");
+      "more explanation. These properties support the following values:");
 
   description.emplace_back(HelpParagraph::Raw(
       R"( - "@ab/<branch_or_build_id>[/<target>[{<filepath>}]]"
+ - "gs://<bucket>/<prefix>/"
+ - "gs://<bucket>/<object>[{<filepath>}]"
+ - "https://<host>/<directory>/"
+ - "https://<host>/<object>[{<filepath>}]"
  - "<absolute_path>")"));
 
   description.emplace_back(
@@ -307,6 +311,18 @@ std::vector<HelpParagraph> LoadConfigsCommand::CommonCommandDescription() {
       "braces. For more information on build fetching and caching operations "
       "refer to `cvd help fetch`.");
 
+  description.emplace_back(
+      "A \"gs://\" or \"https://\" value names a build outside the Android "
+      "build servers. A URL ending in '/' names the directory holding the "
+      "build's artifacts, which cvd picks from by name; a URL naming a single "
+      "object is a build of just that artifact, and <filepath> then selects a "
+      "member of it if it is a zip. A \"gs://\" build is read with the "
+      "credentials `cvd fetch` would use for the build servers, while an "
+      "\"https://\" build is read as given, so any credential has to travel "
+      "in the URL itself as in a pre-signed URL. A directory URL over plain "
+      "\"https://\" cannot be listed, so a build named that way needs the "
+      "host package named separately.");
+
   description.emplace_back(
       "Alternatively, the build value may point to an absolute path (starts "
       "with '/') in the filesystem where the Android source code has been "

base/cvd/cuttlefish/host/commands/cvd/cli/parser/BUILD.bazel

@@ -75,6 +75,8 @@ cf_cc_library(
         "//cuttlefish/host/commands/cvd/cli/parser:cf_configs_common",
         "//cuttlefish/host/commands/cvd/cli/parser:load_config_cc_proto",
         "//cuttlefish/host/commands/cvd/fetch:fetch_cvd_parser",
+        "//cuttlefish/host/libs/web:android_build_string",
+        "//cuttlefish/host/libs/web/http_client:scrub_secrets",
         "//cuttlefish/result",
         "@abseil-cpp//absl/strings",
     ],

base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser.cpp

@@ -19,13 +19,17 @@
 #include <string>
 #include <string_view>
 #include <utility>
+#include <variant>
 #include <vector>
 
+#include "absl/strings/match.h"
 #include "absl/strings/strip.h"
 
 #include "cuttlefish/host/commands/cvd/cli/parser/cf_configs_common.h"
 #include "cuttlefish/host/commands/cvd/cli/parser/load_config.pb.h"
 #include "cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.h"
+#include "cuttlefish/host/libs/web/android_build_string.h"
+#include "cuttlefish/host/libs/web/http_client/scrub_secrets.h"
 #include "cuttlefish/result/result.h"
 
 namespace cuttlefish {
@@ -56,15 +60,26 @@ bool ShouldFetch(const Instance& instance) {
 
 Result<std::string> GetFetchBuildString(const std::string& strVal) {
   std::string_view view = strVal;
-  if (!absl::ConsumePrefix(&view, kFetchPrefix)) {
-    // intentionally return an empty string when there are local, non-prefixed
-    // paths.  Fetch does not process the local paths
-    return "";
+  if (absl::ConsumePrefix(&view, kFetchPrefix)) {
+    CF_EXPECTF(!view.empty(),
+               "\"{}\" prefixed build string was not followed by a value",
+               kFetchPrefix);
+    CF_EXPECT(ParseBuildString(view));
+    return std::string(view);
   }
-  CF_EXPECTF(!view.empty(),
-             "\"{}\" prefixed build string was not followed by a value",
-             kFetchPrefix);
-  return std::string(view);
+  if (absl::StrContains(strVal, "://")) {
+    const BuildString parsed = CF_EXPECT(ParseBuildString(strVal));
+    // A local path can hold a "://" without naming a URL scheme, and parses
+    // as a directory build string.
+    CF_EXPECTF(std::holds_alternative<GcsBuildString>(parsed) ||
+                   std::holds_alternative<HttpBuildString>(parsed),
+               "'{}' contains '://' but is not a 'gs://' or 'https://' URL.",
+               ScrubUrl(strVal));
+    return strVal;
+  }
+  // intentionally return an empty string when there are local, non-prefixed
+  // paths.  Fetch does not process the local paths
+  return "";
 }
 
 Result<Instance> RemoveNonPrefixedBuildStrings(const Instance& instance) {

base/cvd/cuttlefish/host/commands/cvd/cli/parser/fetch_config_parser_test.cc

@@ -40,6 +40,9 @@ Result<std::vector<std::string>> FetchCvdParserTester(const Json::Value& root) {
 
 }  // namespace
 
+using ::testing::AllOf;
+using ::testing::HasSubstr;
+
 TEST(FetchConfigParserTests, AndroidEfiLoaderBuildOnlySuccess) {
   const char* test_string = R""""(
 {
@@ -105,9 +108,258 @@ TEST(FetchConfigParserTests, NoBuildStringsProducesNoFlagsSuccess) {
   std::string json_text(test_string);
   ASSERT_TRUE(ParseJsonString(json_text, json_configs));
 
+  Result<std::vector<std::string>> flags = FetchCvdParserTester(json_configs);
+  EXPECT_THAT(flags, IsOkAndValue(std::vector<std::string>{}));
+}
+
+TEST(FetchConfigParserTests, MalformedAbBuildStringFail) {
+  const char* test_string = R""""(
+{
+    "instances": [
+        {
+            "disk": {
+                "default_build": "@ab/branch/target/extra"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  EXPECT_THAT(FetchCvdParserTester(json_configs), IsError());
+}
+
+TEST(FetchConfigParserTests, ObjectUrlBuildsSuccess) {
+  const char* test_string = R""""(
+{
+    "common": {
+        "host_package": "gs://bucket/dist/cvd-host_package.tar.gz"
+    },
+    "instances": [
+        {
+            "disk": {
+                "default_build": "gs://bucket/dist/phone-img-1.zip"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
   Result<std::vector<std::string>> flags = FetchCvdParserTester(json_configs);
   ASSERT_THAT(flags, IsOk());
-  EXPECT_TRUE(flags->empty());
+  EXPECT_TRUE(
+      FindConfig(*flags, "--default_build=gs://bucket/dist/phone-img-1.zip"));
+  EXPECT_TRUE(FindConfig(
+      *flags, "--host_package_build=gs://bucket/dist/cvd-host_package.tar.gz"));
+}
+
+TEST(FetchConfigParserTests, DirectoryUrlBuildsSuccess) {
+  const char* test_string = R""""(
+{
+    "common": {
+        "host_package": "gs://bucket/host/"
+    },
+    "instances": [
+        {
+            "disk": {
+                "default_build": "gs://bucket/dist/"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  Result<std::vector<std::string>> flags = FetchCvdParserTester(json_configs);
+  ASSERT_THAT(flags, IsOk());
+  EXPECT_TRUE(FindConfig(*flags, "--default_build=gs://bucket/dist/"));
+  EXPECT_TRUE(FindConfig(*flags, "--host_package_build=gs://bucket/host/"));
+}
+
+TEST(FetchConfigParserTests, HttpsDirectoryUrlBuildSuccess) {
+  const char* test_string = R""""(
+{
+    "instances": [
+        {
+            "disk": {
+                "default_build": "https://example.com/dist/"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  Result<std::vector<std::string>> flags = FetchCvdParserTester(json_configs);
+  ASSERT_THAT(flags, IsOk());
+  EXPECT_TRUE(FindConfig(*flags, "--default_build=https://example.com/dist/"));
+}
+
+TEST(FetchConfigParserTests, OtaToolsUrlBuildSuccess) {
+  const char* test_string = R""""(
+{
+    "instances": [
+        {
+            "disk": {
+                "otatools": "gs://bucket/dist/"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  Result<std::vector<std::string>> flags = FetchCvdParserTester(json_configs);
+  ASSERT_THAT(flags, IsOk());
+  EXPECT_TRUE(FindConfig(*flags, "--otatools_build=gs://bucket/dist/"));
+}
+
+TEST(FetchConfigParserTests, UrlFilepathSelectorSuccess) {
+  const char* test_string = R""""(
+{
+    "instances": [
+        {
+            "boot": {
+                "build": "gs://bucket/dist/images.zip{boot.img}"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  Result<std::vector<std::string>> flags = FetchCvdParserTester(json_configs);
+  ASSERT_THAT(flags, IsOk());
+  EXPECT_TRUE(
+      FindConfig(*flags, "--boot_build=gs://bucket/dist/images.zip{boot.img}"));
+}
+
+TEST(FetchConfigParserTests, CleartextHttpUrlFail) {
+  const char* test_string = R""""(
+{
+    "instances": [
+        {
+            "disk": {
+                "default_build": "http://example.com/dist/"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  EXPECT_THAT(
+      FetchCvdParserTester(json_configs),
+      IsErrorAndMessage(AllOf(HasSubstr("http://"), HasSubstr("https://"))));
+}
+
+TEST(FetchConfigParserTests, UnsupportedUrlSchemeFail) {
+  const char* test_string = R""""(
+{
+    "instances": [
+        {
+            "disk": {
+                "default_build": "s3://bucket/dist/"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  EXPECT_THAT(FetchCvdParserTester(json_configs),
+              IsErrorAndMessage(AllOf(HasSubstr("s3"), HasSubstr("gs://"))));
+}
+
+// The URL build string format reserves ',', so a value holding one is
+// refused even where nothing splits it.
+TEST(FetchConfigParserTests, UrlWithCommaFail) {
+  const char* test_string = R""""(
+{
+    "instances": [
+        {
+            "disk": {
+                "default_build": "https://example.com/d/img.zip?k=a,b"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  EXPECT_THAT(FetchCvdParserTester(json_configs),
+              IsErrorAndMessage(HasSubstr("comma")));
+}
+
+TEST(FetchConfigParserTests, LocalPathProducesNoFlagsSuccess) {
+  const char* test_string = R""""(
+{
+    "instances": [
+        {
+            "disk": {
+                "default_build": "/home/user/local_images"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  Result<std::vector<std::string>> flags = FetchCvdParserTester(json_configs);
+  EXPECT_THAT(flags, IsOkAndValue(std::vector<std::string>{}));
+}
+
+TEST(FetchConfigParserTests, UnprefixedRelativeValuesProduceNoFlagsSuccess) {
+  const char* test_string = R""""(
+{
+    "instances": [
+        {
+            "disk": {
+                "default_build": "out/target/product/vsoc_x86_64",
+                "otatools": "branch/target"
+            }
+        }
+    ]
+}
+  )"""";
+
+  Json::Value json_configs;
+  std::string json_text(test_string);
+  ASSERT_TRUE(ParseJsonString(json_text, json_configs));
+
+  Result<std::vector<std::string>> flags = FetchCvdParserTester(json_configs);
+  EXPECT_THAT(flags, IsOkAndValue(std::vector<std::string>{}));
 }
 
 }  // namespace cuttlefish