feat(objectstore): expose checksum provenance

zig/build.zig

@@ -2672,6 +2672,14 @@ pub fn build(b: *std.Build) void {
     const lib_httpx_test_step = b.step("lib-httpx-test", "Run standalone lib/httpx tests");
     lib_httpx_test_step.dependOn(&run_httpx_tests.step);
 
+    const objectstore_tests = b.addTest(.{
+        .root_module = objectstore_mod,
+        .filters = selectTestFilters(b, &.{}),
+    });
+    const run_objectstore_tests = b.addRunArtifact(objectstore_tests);
+    const lib_objectstore_test_step = b.step("lib-objectstore-test", "Run standalone lib/objectstore tests");
+    lib_objectstore_test_step.dependOn(&run_objectstore_tests.step);
+
     const common_http_test_mod = b.createModule(.{
         .root_source_file = b.path("pkg/antfly/src/common_http_test_root.zig"),
         .target = target,

zig/lib/objectstore/src/filesystem.zig

@@ -279,6 +279,10 @@ pub const FilesystemClient = struct {
             .bucket = try alloc.dupe(u8, bucket),
             .key = try alloc.dupe(u8, key),
             .etag = try alloc.dupe(u8, &header.etag),
+            .checksum = .{
+                .algorithm = .sha256_hex,
+                .value = try alloc.dupe(u8, &header.etag),
+            },
             .content_length = header.content_length,
             .content_type = if (header.content_type.len == 0) null else try alloc.dupe(u8, header.content_type),
             .last_modified_unix_ms = file_stat.mtime.toMilliseconds(),
@@ -1034,6 +1038,8 @@ test "filesystem client supports bucket/object lifecycle and file helpers" {
     defer got.deinit(alloc);
     try std.testing.expectEqualStrings("alpha", got.body);
     try std.testing.expectEqualStrings("text/plain", got.metadata.content_type.?);
+    try std.testing.expectEqual(types.ObjectChecksumAlgorithm.sha256_hex, got.metadata.checksum.?.algorithm);
+    try std.testing.expectEqualStrings(put.etag.?, got.metadata.checksum.?.value);
 
     var ranged = try client.getObject("docs", "nested/a.txt", .{ .range = .{ .offset = 1, .length = 3 } });
     defer ranged.deinit(alloc);

zig/lib/objectstore/src/gcs.zig

@@ -454,15 +454,27 @@ pub const JsonApiClient = struct {
                 .key = owned_key,
                 .content_length = 0,
             };
-        } else try self.statObject(alloc, bucket, key);
+        } else try self.statObjectVersion(alloc, bucket, key, opts.version_id);
         errdefer meta.deinit(alloc);
 
-        const url = try objectMediaUrlWithGenerationAlloc(alloc, self.cfg, bucket, key, opts.version_id);
+        const effective_generation: ?[]const u8 = if (opts.version_id) |value|
+            value
+        else if (!opts.skip_metadata_probe)
+            meta.version_id
+        else
+            null;
+        const url = try objectMediaUrlWithGenerationAlloc(alloc, self.cfg, bucket, key, effective_generation);
         defer alloc.free(url);
 
         var headers = std.ArrayListUnmanaged(HeaderPair).empty;
         defer headers.deinit(alloc);
-        try appendConditionalHeaders(alloc, &headers, opts.if_match_etag, false);
+        const effective_if_match: ?[]const u8 = if (opts.if_match_etag) |value|
+            value
+        else if (!opts.skip_metadata_probe and effective_generation == null)
+            meta.etag
+        else
+            null;
+        try appendConditionalHeaders(alloc, &headers, effective_if_match, false);
         if (opts.range) |range| {
             const value = try byteRangeHeaderAlloc(alloc, range);
             errdefer alloc.free(value);
@@ -533,7 +545,11 @@ pub const JsonApiClient = struct {
     }
 
     fn statObject(self: *JsonApiClient, alloc: Allocator, bucket: []const u8, key: []const u8) !types.ObjectMetadata {
-        const url = try objectMetadataUrlWithGenerationAlloc(alloc, self.cfg, bucket, key, null);
+        return self.statObjectVersion(alloc, bucket, key, null);
+    }
+
+    fn statObjectVersion(self: *JsonApiClient, alloc: Allocator, bucket: []const u8, key: []const u8, generation: ?[]const u8) !types.ObjectMetadata {
+        const url = try objectMetadataUrlWithGenerationAlloc(alloc, self.cfg, bucket, key, generation);
         defer alloc.free(url);
 
         var response = try self.perform(.GET, url, &.{}, null, null);
@@ -1043,6 +1059,8 @@ fn parseObjectMetadataResponse(alloc: Allocator, bucket: []const u8, body: []con
         generation: ?[]const u8 = null,
         size: ?[]const u8 = null,
         contentType: ?[]const u8 = null,
+        md5Hash: ?[]const u8 = null,
+        crc32c: ?[]const u8 = null,
     };
 
     var parsed = try std.json.parseFromSlice(Parsed, alloc, body, .{ .ignore_unknown_fields = true });
@@ -1058,11 +1076,21 @@ fn parseObjectMetadataResponse(alloc: Allocator, bucket: []const u8, body: []con
     const version_id = if (parsed.value.generation) |value| try alloc.dupe(u8, value) else null;
     errdefer if (version_id) |value| alloc.free(value);
     const content_type = if (parsed.value.contentType) |value| try alloc.dupe(u8, value) else null;
+    errdefer if (content_type) |value| alloc.free(value);
+    var checksum: ?types.ObjectChecksum = if (parsed.value.md5Hash) |value| .{
+        .algorithm = .md5_base64,
+        .value = try alloc.dupe(u8, value),
+    } else if (parsed.value.crc32c) |value| .{
+        .algorithm = .crc32c_base64,
+        .value = try alloc.dupe(u8, value),
+    } else null;
+    errdefer if (checksum) |*value| value.deinit(alloc);
     return .{
         .bucket = owned_bucket,
         .key = key,
         .etag = etag,
         .version_id = version_id,
+        .checksum = checksum,
         .content_length = content_length,
         .content_type = content_type,
         .last_modified_unix_ms = null,
@@ -1319,7 +1347,7 @@ test "gcs local grpc reference path can be discovered when present" {
     }
 }
 
-test "json api client get object uses metadata then media with auth and range" {
+test "json api client pins media reads to the generation returned by metadata" {
     const alloc = std.testing.allocator;
     const State = struct {
         calls: usize = 0,
@@ -1347,15 +1375,14 @@ test "json api client get object uses metadata then media with auth and range" {
                     try expectHeader(headers, "Authorization", "Bearer token-123");
                     return .{
                         .status = 200,
-                        .body = try request_alloc.dupe(u8, "{\"bucket\":\"bucket\",\"name\":\"folder/doc.txt\",\"etag\":\"etag-1\",\"generation\":\"42\",\"size\":\"11\",\"contentType\":\"text/plain\"}"),
+                        .body = try request_alloc.dupe(u8, "{\"bucket\":\"bucket\",\"name\":\"folder/doc.txt\",\"etag\":\"etag-1\",\"generation\":\"42\",\"size\":\"11\",\"contentType\":\"text/plain\",\"md5Hash\":\"md5-body\"}"),
                     };
                 },
                 1 => {
                     try std.testing.expectEqual(@as(?usize, null), max_response_size);
                     try std.testing.expectEqual(HttpMethod.GET, method);
                     try std.testing.expectEqualStrings("https://storage.googleapis.com/storage/v1/b/bucket/o/folder%2Fdoc.txt?generation=42&alt=media", url);
                     try expectHeader(headers, "Authorization", "Bearer token-123");
-                    try expectHeader(headers, "If-Match", "etag-1");
                     try expectHeader(headers, "Range", "bytes=2-5");
                     return .{
                         .status = 206,
@@ -1390,9 +1417,7 @@ test "json api client get object uses metadata then media with auth and range" {
     defer client.deinit();
 
     var result = try client.getObject("bucket", "folder/doc.txt", .{
-        .version_id = "42",
         .range = .{ .offset = 2, .length = 4 },
-        .if_match_etag = "etag-1",
     });
     defer result.deinit(alloc);
 
@@ -1401,6 +1426,9 @@ test "json api client get object uses metadata then media with auth and range" {
     try std.testing.expectEqualStrings("folder/doc.txt", result.metadata.key);
     try std.testing.expectEqualStrings("etag-1", result.metadata.etag.?);
     try std.testing.expectEqualStrings("42", result.metadata.version_id.?);
+    try std.testing.expectEqual(types.ObjectChecksumAlgorithm.md5_base64, result.metadata.checksum.?.algorithm);
+    try std.testing.expectEqualStrings("md5-body", result.metadata.checksum.?.value);
+    try std.testing.expectEqual(types.ObjectChecksumType.full_object, result.metadata.checksum.?.checksum_type);
     try std.testing.expectEqualStrings("text/plain", result.metadata.content_type.?);
 
     var direct = try client.getObject("bucket", "folder/doc.txt", .{
@@ -1419,6 +1447,20 @@ test "json api client get object uses metadata then media with auth and range" {
     try std.testing.expectEqual(@as(usize, 3), state.calls);
 }
 
+test "json api metadata falls back to the always-available crc32c checksum" {
+    const alloc = std.testing.allocator;
+    var meta = try parseObjectMetadataResponse(
+        alloc,
+        "bucket",
+        "{\"name\":\"composite\",\"generation\":\"9\",\"size\":\"4\",\"crc32c\":\"crc-body\"}",
+    );
+    defer meta.deinit(alloc);
+
+    try std.testing.expectEqual(types.ObjectChecksumAlgorithm.crc32c_base64, meta.checksum.?.algorithm);
+    try std.testing.expectEqualStrings("crc-body", meta.checksum.?.value);
+    try std.testing.expectEqual(types.ObjectChecksumType.full_object, meta.checksum.?.checksum_type);
+}
+
 test "json api client put object encodes upload url and returns etag" {
     const alloc = std.testing.allocator;
     const State = struct {

zig/lib/objectstore/src/memory.zig

@@ -148,6 +148,10 @@ pub const MemoryClient = struct {
                 .bucket = try alloc.dupe(u8, bucket),
                 .key = try alloc.dupe(u8, key),
                 .etag = try alloc.dupe(u8, object.etag),
+                .checksum = .{
+                    .algorithm = .sha256_hex,
+                    .value = try alloc.dupe(u8, object.etag),
+                },
                 .content_length = @intCast(object.body.len),
                 .content_type = if (object.content_type) |value| try alloc.dupe(u8, value) else null,
             },
@@ -179,6 +183,10 @@ pub const MemoryClient = struct {
             .bucket = try alloc.dupe(u8, bucket),
             .key = try alloc.dupe(u8, key),
             .etag = try alloc.dupe(u8, object.etag),
+            .checksum = .{
+                .algorithm = .sha256_hex,
+                .value = try alloc.dupe(u8, object.etag),
+            },
             .content_length = @intCast(object.body.len),
             .content_type = if (object.content_type) |value| try alloc.dupe(u8, value) else null,
         };
@@ -373,6 +381,8 @@ test "memory client supports put get stat list and delete" {
     var got = try client.getObject("bucket", "a/one", .{});
     defer got.deinit(alloc);
     try std.testing.expectEqualStrings("alpha", got.body);
+    try std.testing.expectEqual(types.ObjectChecksumAlgorithm.sha256_hex, got.metadata.checksum.?.algorithm);
+    try std.testing.expectEqualStrings(put.etag.?, got.metadata.checksum.?.value);
     try std.testing.expectError(
         error.ResponseTooLarge,
         client.getObject("bucket", "a/one", .{ .max_response_bytes = 4 }),
@@ -387,6 +397,8 @@ test "memory client supports put get stat list and delete" {
     var meta = try client.statObject("bucket", "a/one");
     defer meta.deinit(alloc);
     try std.testing.expectEqual(@as(u64, 5), meta.content_length);
+    try std.testing.expectEqual(types.ObjectChecksumAlgorithm.sha256_hex, meta.checksum.?.algorithm);
+    try std.testing.expectEqualStrings(put.etag.?, meta.checksum.?.value);
 
     var listed = try client.listObjects("bucket", .{ .prefix = "a/" });
     defer listed.deinit(alloc);

zig/lib/objectstore/src/root.zig

@@ -21,6 +21,10 @@ pub const gcs = @import("gcs.zig");
 pub const s3_compat = @import("s3_compat.zig");
 
 pub const ObjectMetadata = types.ObjectMetadata;
+pub const ObjectChecksum = types.ObjectChecksum;
+pub const ObjectChecksumAlgorithm = types.ObjectChecksumAlgorithm;
+pub const ObjectChecksumScope = types.ObjectChecksumScope;
+pub const ObjectChecksumType = types.ObjectChecksumType;
 pub const PutOptions = types.PutOptions;
 pub const GetOptions = types.GetOptions;
 pub const DeleteOptions = types.DeleteOptions;