From d5ea47f8458e668d6bf1752d93045d99f17391c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:28:36 +0700 Subject: [PATCH 01/22] feat(display-format): decode igbinary and Avro values igbinary decodes into the PHP serialized tree, numbered as serialize() numbers it. Avro decodes with a schema file chosen per column, stored by a hash of its text. --- .../Formatting/BinaryValues/AvroDecoder.swift | 291 +++++++++++++++ .../Formatting/BinaryValues/AvroSchema.swift | 214 +++++++++++ .../BinaryValues/BinaryValueDecoding.swift | 40 ++- .../BinaryValues/BinaryValueReader.swift | 19 + .../BinaryValues/IgbinaryDecoder.swift | 333 ++++++++++++++++++ .../BinaryValues/PhpSerializeWriter.swift | 82 +++++ .../Formatting/PhpSerializeParser.swift | 4 +- .../Formatting/ValueDisplayFormat.swift | 67 +++- .../Formatting/ValueDisplayFormatter.swift | 4 +- .../Core/Storage/AvroSchemaLibrary.swift | 61 ++++ .../TablePro/Resources/Localizable.xcstrings | 83 +++++ .../DataGridViewController+HeaderMenu.swift | 50 +++ .../Views/Results/BinaryValueViewerView.swift | 6 + .../Results/CellInteractionResolver.swift | 2 +- .../FieldEditors/FieldEditorResolver.swift | 2 +- .../BinaryValues/AvroDecoderTests.swift | 178 ++++++++++ .../BinaryDisplayFormatTests.swift | 47 +++ .../BinaryValues/IgbinaryDecoderTests.swift | 182 ++++++++++ .../Formatting/ValueDisplayFormatTests.swift | 23 +- 19 files changed, 1674 insertions(+), 14 deletions(-) create mode 100644 apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift create mode 100644 apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift create mode 100644 apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift create mode 100644 apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift create mode 100644 apple/TablePro/Core/Storage/AvroSchemaLibrary.swift create mode 100644 apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift create mode 100644 apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift new file mode 100644 index 0000000000..3a69bff31f --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift @@ -0,0 +1,291 @@ +// +// AvroDecoder.swift +// TablePro +// + +import Foundation + +internal struct AvroMessage: Equatable, Sendable { + let value: MessagePackValue + let schemaID: UInt32? +} + +// Values land in MessagePack's model, which already has the tree, the JSON writer and the timestamp badge. +// A union shows the branch it holds, as Avro's own readers return it. +internal enum AvroDecoder { + static let maxDepth = 256 + static let maxZeroByteItems: Int64 = 1_000_000 + // Long division is quadratic in the width; past this a decimal shows as its bytes. + static let maxDecimalBytes = 512 + + // A Kafka value carries the Confluent prefix: magic byte 0 and a four-byte schema ID. Zero also starts + // plenty of plain datums, so the prefix is only tried once the value fails to read without it. + static func decode(_ data: Data, schema: AvroSchema, maxValues: Int = .max) throws -> AvroMessage { + do { + return AvroMessage(value: try datum(data, schema: schema, maxValues: maxValues), schemaID: nil) + } catch let plainError as BinaryValueDecodeError { + guard data.count >= 5, data.first == 0 else { throw plainError } + var header = BinaryValueReader(data) + _ = try header.readByte() + let schemaID = try header.readUInt32() + guard let value = try? datum(data.dropFirst(5), schema: schema, maxValues: maxValues) else { throw plainError } + return AvroMessage(value: value, schemaID: schemaID) + } + } + + private static func datum(_ data: Data, schema: AvroSchema, maxValues: Int) throws -> MessagePackValue { + var parser = Parser(reader: BinaryValueReader(Data(data)), schema: schema, budget: maxValues) + let value = try parser.value(schema.root, depth: 0) + if !parser.isTruncated, !parser.reader.isAtEnd { + throw parser.reader.error(.trailingBytes(parser.reader.remaining)) + } + return value + } + + private struct Parser { + var reader: BinaryValueReader + let schema: AvroSchema + var budget: Int + var isTruncated = false + private var zeroByteItems: Int64 = 0 + + init(reader: BinaryValueReader, schema: AvroSchema, budget: Int) { + self.reader = reader + self.schema = schema + self.budget = budget + } + + mutating func value(_ type: AvroType, depth: Int) throws -> MessagePackValue { + guard depth <= AvroDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } + guard budget > 0 else { + isTruncated = true + return .truncated + } + budget -= 1 + switch schema.resolve(type) { + case .null: + return .null + case .boolean: + let start = reader.offset + let byte = try reader.readByte() + guard byte <= 1 else { throw BinaryValueDecodeError(kind: .invalidByte(byte), offset: start) } + return .bool(byte == 1) + case .int: + return .int(try int()) + case .long: + return .int(try long()) + case .float: + return .float(Float(bitPattern: try reader.readUInt32LittleEndian())) + case .double: + return .double(Double(bitPattern: try reader.readUInt64LittleEndian())) + case .bytes: + return .binary(try bytes()) + case .string: + return try string() + case .record(_, let fields): + var pairs: [MessagePackPair] = [] + pairs.reserveCapacity(fields.count) + for field in fields { + let item = try value(field.type, depth: depth + 1) + pairs.append(MessagePackPair(key: .string(field.name), value: item)) + if isTruncated { break } + } + return .map(pairs) + case .enumeration(_, let symbols): + return .string(symbols[try index(below: symbols.count)]) + case .array(let items): + var values: [MessagePackValue] = [] + try blocks(takesNoBytes: schema.takesNoBytes(items)) { parser in + values.append(try parser.value(items, depth: depth + 1)) + } + return .array(values) + case .map(let valueType): + var pairs: [MessagePackPair] = [] + try blocks(takesNoBytes: false) { parser in + let key = try parser.string() + pairs.append(MessagePackPair(key: key, value: try parser.value(valueType, depth: depth + 1))) + } + return .map(pairs) + case .union(let branches): + return try value(branches[try index(below: branches.count)], depth: depth) + case .fixed(_, let size): + return .binary(try reader.readBytes(size)) + case .named: + throw reader.error(.invalidField) + case .logical(let logical, let base): + return AvroLogicalRendering.render(try value(base, depth: depth), as: logical) + } + } + + // Each block is a count, then that many items; a negative count is followed by the block's byte size. + private mutating func blocks(takesNoBytes: Bool, _ item: (inout Parser) throws -> Void) throws { + while true { + let start = reader.offset + var count = try long() + if count == 0 { return } + if count < 0 { + guard count != .min else { throw BinaryValueDecodeError(kind: .integerOutOfRange, offset: start) } + count = -count + _ = try long() + } + // A count past the bytes left is forged, unless the items take no bytes; then the count alone bounds the loop. + if count > Int64(reader.remaining) { + guard takesNoBytes else { throw BinaryValueDecodeError(kind: .unexpectedEnd, offset: reader.end) } + zeroByteItems += count + guard zeroByteItems <= AvroDecoder.maxZeroByteItems else { + throw BinaryValueDecodeError(kind: .tooLarge, offset: start) + } + } + for _ in 0.. Int { + let start = reader.offset + let index = try long() + guard index >= 0, index < count else { + throw BinaryValueDecodeError(kind: .indexOutOfRange(Int(clamping: index)), offset: start) + } + return Int(index) + } + + private mutating func bytes() throws -> Data { + let start = reader.offset + let count = try long() + guard count >= 0, count <= Int64(reader.remaining) else { + throw BinaryValueDecodeError(kind: count < 0 ? .invalidLength : .unexpectedEnd, offset: start) + } + return try reader.readBytes(Int(count)) + } + + // Writers that ignore the encoding put any bytes in a string, so those show as bytes. + mutating func string() throws -> MessagePackValue { + let data = try bytes() + guard let text = String(data: data, encoding: .utf8) else { return .binary(data) } + return .string(text) + } + + private mutating func int() throws -> Int64 { + let start = reader.offset + let value = try long() + guard let narrowed = Int32(exactly: value) else { + throw BinaryValueDecodeError(kind: .integerOutOfRange, offset: start) + } + return Int64(narrowed) + } + + private mutating func long() throws -> Int64 { + let raw = try reader.readVarint() + return Int64(bitPattern: raw >> 1) ^ -Int64(bitPattern: raw & 1) + } + } +} + +internal enum AvroLogicalRendering { + static func render(_ value: MessagePackValue, as logical: AvroLogicalType) -> MessagePackValue { + switch (logical, value) { + case (.decimal(let scale), .binary(let data)) where data.count <= AvroDecoder.maxDecimalBytes: + return .string(decimalText(unscaled: [UInt8](data), scale: scale)) + case (.uuid, .string): + return value + case (.uuid, .binary(let data)) where data.count == 16: + return .string(UUID(uuid: uuidTuple([UInt8](data))).uuidString.lowercased()) + case (.date, .int(let days)): + return .string(dateText(days: days)) + case (.timeMillis, .int(let millis)): + return timeText(units: millis, perSecond: 1_000).map(MessagePackValue.string) ?? value + case (.timeMicros, .int(let micros)): + return timeText(units: micros, perSecond: 1_000_000).map(MessagePackValue.string) ?? value + case (.timestampMillis, .int(let millis)): + return timestamp(units: millis, perSecond: 1_000) + case (.timestampMicros, .int(let micros)): + return timestamp(units: micros, perSecond: 1_000_000) + case (.localTimestampMillis, .int(let millis)): + return .string(localText(timestamp(units: millis, perSecond: 1_000))) + case (.localTimestampMicros, .int(let micros)): + return .string(localText(timestamp(units: micros, perSecond: 1_000_000))) + default: + return value + } + } + + private static func timestamp(units: Int64, perSecond: Int64) -> MessagePackValue { + let (seconds, remainder) = floorDivide(units, perSecond) + return .timestamp(seconds: seconds, nanoseconds: UInt32(remainder * (1_000_000_000 / perSecond))) + } + + private static func localText(_ timestamp: MessagePackValue) -> String { + guard case .timestamp(let seconds, let nanoseconds) = timestamp else { return "" } + return String(RFC3339Timestamp.format(seconds: seconds, nanoseconds: nanoseconds).dropLast()) + } + + private static func dateText(days: Int64) -> String { + let (seconds, overflow) = days.multipliedReportingOverflow(by: 86_400) + guard !overflow else { return String(days) } + let text = RFC3339Timestamp.format(seconds: seconds, nanoseconds: 0) + return text.firstIndex(of: "T").map { String(text[..<$0]) } ?? text + } + + private static func timeText(units: Int64, perSecond: Int64) -> String? { + guard units >= 0, units < 86_400 * perSecond else { return nil } + let seconds = units / perSecond + var text = String(format: "%02lld:%02lld:%02lld", seconds / 3_600, seconds % 3_600 / 60, seconds % 60) + let fraction = units % perSecond + let digits = perSecond == 1_000 ? 3 : 6 + if fraction > 0 { + text += "." + String(format: "%0\(digits)lld", fraction) + } + return text + } + + private static func floorDivide(_ value: Int64, _ divisor: Int64) -> (Int64, Int64) { + let quotient = value / divisor + let remainder = value % divisor + return remainder < 0 ? (quotient - 1, remainder + divisor) : (quotient, remainder) + } + + // Big-endian two's complement of any width, converted to base 10 by long division. + static func decimalText(unscaled bytes: [UInt8], scale: Int) -> String { + guard !bytes.isEmpty else { return "0" } + let isNegative = bytes[0] & 0x80 != 0 + var magnitude = bytes + if isNegative { + var carry = true + for index in magnitude.indices.reversed() { + magnitude[index] = ~magnitude[index] + if carry { + let (sum, overflow) = magnitude[index].addingReportingOverflow(1) + magnitude[index] = sum + carry = overflow + } + } + } + var digits: [UInt8] = [] + while magnitude.contains(where: { $0 != 0 }) { + var remainder = 0 + for index in magnitude.indices { + let current = remainder << 8 | Int(magnitude[index]) + magnitude[index] = UInt8(current / 10) + remainder = current % 10 + } + digits.append(UInt8(remainder)) + } + var text = String(digits.reversed().map { Character(String($0)) }) + if text.isEmpty { text = "0" } + if scale > 0 { + if text.count <= scale { + text = String(repeating: "0", count: scale - text.count + 1) + text + } + text.insert(".", at: text.index(text.endIndex, offsetBy: -scale)) + } + return isNegative ? "-" + text : text + } + + private static func uuidTuple(_ bytes: [UInt8]) -> uuid_t { + (bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]) + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift new file mode 100644 index 0000000000..9178e646e0 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift @@ -0,0 +1,214 @@ +// +// AvroSchema.swift +// TablePro +// + +import Foundation + +internal enum AvroLogicalType: Equatable, Sendable { + case decimal(scale: Int) + case uuid + case date + case timeMillis + case timeMicros + case timestampMillis + case timestampMicros + case localTimestampMillis + case localTimestampMicros +} + +internal struct AvroField: Equatable, Sendable { + let name: String + let type: AvroType +} + +internal indirect enum AvroType: Equatable, Sendable { + case null + case boolean + case int + case long + case float + case double + case bytes + case string + case record(name: String, fields: [AvroField]) + case enumeration(name: String, symbols: [String]) + case array(AvroType) + case map(AvroType) + case union([AvroType]) + case fixed(name: String, size: Int) + case named(String) + case logical(AvroLogicalType, AvroType) +} + +internal struct AvroSchemaError: Error, Equatable, Sendable { + let reason: String +} + +// https://avro.apache.org/docs/1.12.0/specification/ +// A named type is stored once and referenced by full name, so a recursive record parses. +internal struct AvroSchema: Equatable, Sendable { + let root: AvroType + let namedTypes: [String: AvroType] + + static func parse(_ text: String) throws -> AvroSchema { + let json: Any + do { + json = try JSONSerialization.jsonObject(with: Data(text.utf8), options: [.fragmentsAllowed]) + } catch { + throw AvroSchemaError(reason: String(localized: "The schema is not valid JSON.")) + } + var parser = Parser() + let root = try parser.type(json, namespace: nil) + return AvroSchema(root: root, namedTypes: parser.namedTypes) + } + + // Far past any engine's decimal; a larger one would only let a schema file pad a cell with zeros. + static let maxDecimalPrecision = 1_000 + + func resolve(_ type: AvroType) -> AvroType { + if case .named(let name) = type, let defined = namedTypes[name] { return defined } + return type + } + + // Items of these types take no bytes, so an array of them can hold more items than bytes remain. + func takesNoBytes(_ type: AvroType, visiting: Set = []) -> Bool { + switch resolve(type) { + case .null: return true + case .fixed(_, let size): return size == 0 + case .logical(_, let base): return takesNoBytes(base, visiting: visiting) + case .record(let name, let fields): + guard !visiting.contains(name) else { return false } + return fields.allSatisfy { takesNoBytes($0.type, visiting: visiting.union([name])) } + default: return false + } + } + + private struct Parser { + var namedTypes: [String: AvroType] = [:] + + mutating func type(_ json: Any, namespace: String?) throws -> AvroType { + if let name = json as? String { + return try reference(name, namespace: namespace) + } + if let branches = json as? [Any] { + return .union(try branches.map { try type($0, namespace: namespace) }) + } + guard let object = json as? [String: Any] else { + throw AvroSchemaError(reason: String(localized: "The schema has a type that is not a name, an object or a union.")) + } + let base = try complex(object, namespace: namespace) + guard let logicalName = object["logicalType"] as? String, + let logical = Self.logical(logicalName, object: object, base: base) else { return base } + return .logical(logical, base) + } + + private mutating func complex(_ object: [String: Any], namespace: String?) throws -> AvroType { + guard let typeName = object["type"] else { + throw AvroSchemaError(reason: String(localized: "A schema object has no \"type\".")) + } + guard let kind = typeName as? String else { return try type(typeName, namespace: namespace) } + switch kind { + case "record", "error": + let name = try fullName(object, namespace: namespace) + namedTypes[name] = .record(name: name, fields: []) + let inner = Self.namespace(of: name) + guard let rawFields = object["fields"] as? [[String: Any]] else { + throw AvroSchemaError(reason: String(format: String(localized: "Record %@ has no fields list."), name)) + } + let fields = try rawFields.map { field -> AvroField in + guard let fieldName = field["name"] as? String, let fieldType = field["type"] else { + throw AvroSchemaError(reason: String(format: String(localized: "A field of record %@ has no name or type."), name)) + } + return AvroField(name: fieldName, type: try type(fieldType, namespace: inner)) + } + let record = AvroType.record(name: name, fields: fields) + namedTypes[name] = record + return record + case "enum": + let name = try fullName(object, namespace: namespace) + guard let symbols = object["symbols"] as? [String] else { + throw AvroSchemaError(reason: String(format: String(localized: "Enum %@ has no symbols."), name)) + } + let enumeration = AvroType.enumeration(name: name, symbols: symbols) + namedTypes[name] = enumeration + return enumeration + case "fixed": + let name = try fullName(object, namespace: namespace) + guard let size = object["size"] as? Int, size >= 0 else { + throw AvroSchemaError(reason: String(format: String(localized: "Fixed type %@ has no size."), name)) + } + let fixed = AvroType.fixed(name: name, size: size) + namedTypes[name] = fixed + return fixed + case "array": + guard let items = object["items"] else { + throw AvroSchemaError(reason: String(localized: "An array has no \"items\".")) + } + return .array(try type(items, namespace: namespace)) + case "map": + guard let values = object["values"] else { + throw AvroSchemaError(reason: String(localized: "A map has no \"values\".")) + } + return .map(try type(values, namespace: namespace)) + default: + return try reference(kind, namespace: namespace) + } + } + + private func reference(_ name: String, namespace: String?) throws -> AvroType { + switch name { + case "null": return .null + case "boolean": return .boolean + case "int": return .int + case "long": return .long + case "float": return .float + case "double": return .double + case "bytes": return .bytes + case "string": return .string + default: break + } + if !name.contains("."), let namespace, namedTypes["\(namespace).\(name)"] != nil { + return .named("\(namespace).\(name)") + } + guard namedTypes[name] != nil else { + throw AvroSchemaError(reason: String(format: String(localized: "The schema names type %@ before defining it."), name)) + } + return .named(name) + } + + private func fullName(_ object: [String: Any], namespace: String?) throws -> String { + guard let name = object["name"] as? String, !name.isEmpty else { + throw AvroSchemaError(reason: String(localized: "A record, enum or fixed type has no name.")) + } + if name.contains(".") { return name } + let space = object["namespace"] as? String ?? namespace + guard let space, !space.isEmpty else { return name } + return "\(space).\(name)" + } + + private static func namespace(of fullName: String) -> String? { + guard let dot = fullName.lastIndex(of: ".") else { return nil } + return String(fullName[.. AvroLogicalType? { + switch (name, base) { + case ("decimal", .bytes), ("decimal", .fixed): + guard let precision = object["precision"] as? Int, (1...AvroSchema.maxDecimalPrecision).contains(precision) else { return nil } + let scale = object["scale"] as? Int ?? 0 + return (0...precision).contains(scale) ? .decimal(scale: scale) : nil + case ("uuid", .string), ("uuid", .fixed(_, 16)): return .uuid + case ("date", .int): return .date + case ("time-millis", .int): return .timeMillis + case ("time-micros", .long): return .timeMicros + case ("timestamp-millis", .long): return .timestampMillis + case ("timestamp-micros", .long): return .timestampMicros + case ("local-timestamp-millis", .long): return .localTimestampMillis + case ("local-timestamp-micros", .long): return .localTimestampMicros + default: return nil + } + } + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift index 2daceec8b2..ba2ee33002 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift @@ -5,14 +5,18 @@ import Foundation -internal enum BinaryValueFormat: String, Equatable, Sendable { +internal enum BinaryValueFormat: Hashable, Sendable { case messagePack case protobuf + case igbinary + case avro(schemaID: String) var displayName: String { switch self { case .messagePack: return "MessagePack" case .protobuf: return "Protobuf" + case .igbinary: return "igbinary" + case .avro: return "Avro" } } } @@ -20,6 +24,7 @@ internal enum BinaryValueFormat: String, Equatable, Sendable { internal enum BinaryValueTree: Sendable { case json(JSONTreeNode) case protobuf(ProtobufTreeNode) + case php(PhpTreeNode) } internal enum BinaryValueDocument: Sendable { @@ -48,6 +53,12 @@ internal enum BinaryValueDecoding { case .protobuf: let fields = try ProtobufWireDecoder.decode(data, maxValues: budget).fields text = ProtobufTextWriter.text(fields, multiLine: false, limit: limit) + case .igbinary: + text = PhpSerializeWriter.text(try IgbinaryDecoder.decode(data, maxValues: budget), limit: limit) + case .avro(let schemaID): + guard let schema = AvroSchemaLibrary.shared.schema(id: schemaID) else { return nil } + let value = try AvroDecoder.decode(data, schema: schema, maxValues: budget).value + text = MessagePackTextWriter.json(value, pretty: false, limit: limit) } } catch { return nil @@ -92,6 +103,29 @@ internal enum BinaryValueDecoding { ), note: message.framing.map(framingNote) ) + case .igbinary: + let value = try IgbinaryDecoder.decode(data) + return .decoded( + tree: .php(PhpTreeBuilder.build(from: value)), + text: capped( + PhpSerializeWriter.text(value, limit: writerLimit(forCap: maxViewerTextLength)), + at: maxViewerTextLength + ), + note: nil + ) + case .avro(let schemaID): + guard let schema = AvroSchemaLibrary.shared.schema(id: schemaID) else { + return .failed(BinaryValueDecodeError(kind: .schemaMissing, offset: 0)) + } + let message = try AvroDecoder.decode(data, schema: schema) + return .decoded( + tree: .json(MessagePackTreeBuilder.build(from: message.value)), + text: capped( + MessagePackTextWriter.json(message.value, pretty: true, limit: writerLimit(forCap: maxViewerTextLength)), + at: maxViewerTextLength + ), + note: message.schemaID.map(schemaIDNote) + ) } } catch let error as BinaryValueDecodeError { return .failed(error) @@ -100,6 +134,10 @@ internal enum BinaryValueDecoding { } } + private static func schemaIDNote(_ schemaID: UInt32) -> String { + String(format: String(localized: "Schema Registry framing: schema ID %lld"), Int(schemaID)) + } + private static func framingNote(_ framing: ProtobufFraming) -> String { let indexes = framing.messageIndexes.map(String.init).joined(separator: ", ") return String( diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift index a33a2bf9f1..9a0a87a7f1 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift @@ -15,6 +15,11 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { case integerOutOfRange case invalidField case unmatchedGroupEnd + case unknownVersion(UInt32) + case unknownReference(Int) + case indexOutOfRange(Int) + case invalidLength + case schemaMissing } let kind: Kind @@ -42,6 +47,20 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { return String(format: String(localized: "Byte %lld does not start a valid field."), offset) case .unmatchedGroupEnd: return String(format: String(localized: "The group end at byte %lld matches no group start."), offset) + case .unknownVersion(let version): + return String(format: String(localized: "The header names version %lld, which this format does not have."), Int(version)) + case .unknownReference(let index): + return String( + format: String(localized: "Byte %lld refers back to item %lld, which comes later or not at all."), + offset, + index + ) + case .indexOutOfRange(let index): + return String(format: String(localized: "Byte %lld picks entry %lld, which the schema does not have."), offset, index) + case .invalidLength: + return String(format: String(localized: "The length at byte %lld is negative."), offset) + case .schemaMissing: + return String(localized: "The Avro schema for this column is missing. Choose the schema file again.") } } } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift new file mode 100644 index 0000000000..fb7749de2d --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift @@ -0,0 +1,333 @@ +// +// IgbinaryDecoder.swift +// TablePro +// + +import Foundation + +// https://github.com/igbinary/igbinary/blob/master/src/php7/igbinary.c +// References carry the number PHP's serialize() would give them, so `r:`/`R:` read the same in both viewers. +internal enum IgbinaryDecoder { + static let maxDepth = PhpSerializeParser.depthCap + // Arrays repeated by value share storage, so a few bytes can stand for billions of values; past this it is refused. + static let maxSlots = 10_000_000 + // A two-byte string ID can repeat a long string; the tree and Find would each walk every copy. + static let maxRepeatedStringBytes = 20_000_000 + + static func decode(_ data: Data, maxValues: Int = .max) throws -> PhpValue { + var reader = BinaryValueReader(data) + let version = try reader.readUInt32() + guard version == 1 || version == 2 else { + throw BinaryValueDecodeError(kind: .unknownVersion(version), offset: 0) + } + var parser = Parser(reader: reader, budget: maxValues) + let value = try parser.value(depth: 0) + if !parser.isTruncated, !parser.reader.isAtEnd { + throw parser.reader.error(.trailingBytes(parser.reader.remaining)) + } + return value + } + + private enum Tag: UInt8 { + case null = 0x00 + case ref8, ref16, ref32 + case boolFalse, boolTrue + case long8p, long8n, long16p, long16n, long32p, long32n + case double + case stringEmpty + case stringID8, stringID16, stringID32 + case string8, string16, string32 + case array8, array16, array32 + case object8, object16, object32 + case objectID8, objectID16, objectID32 + case objectSer8, objectSer16, objectSer32 + case long64p, long64n + case objref8, objref16, objref32 + case reference + case string64 + case enumCase + } + + // What serialize() writes where igbinary refers back by value: arrays in full, objects as `r:`. + private struct Copy { + let value: PhpValue + let slots: Int + } + + private struct Decoded { + let value: PhpValue + let copy: Copy + } + + private struct Parser { + var reader: BinaryValueReader + var budget: Int + var isTruncated = false + private var strings: [String] = [] + // Indexed like igbinary's reference table: each array, object and referenced scalar, as serialize() numbers it. + private var references: [(slot: Int, copy: Copy?)] = [] + private var slot = 0 + private var repeatedStringBytes = 0 + + init(reader: BinaryValueReader, budget: Int) { + self.reader = reader + self.budget = budget + } + + mutating func value(depth: Int) throws -> PhpValue { + try decoded(depth: depth).value + } + + private mutating func decoded(depth: Int) throws -> Decoded { + guard depth <= IgbinaryDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } + guard budget > 0 else { + isTruncated = true + return Decoded(value: .tooLarge, copy: Copy(value: .tooLarge, slots: 1)) + } + budget -= 1 + slot += 1 + return try decoded(tag: try readTag(), depth: depth, isReference: false) + } + + private mutating func decoded(tag: Tag, depth: Int, isReference: Bool) throws -> Decoded { + let ownSlot = slot + switch tag { + case .reference: + guard !isReference else { throw invalid(tag, at: reader.offset - 1) } + let result = try decoded(tag: try readTag(), depth: depth, isReference: true) + switch result.value { + case .null, .bool, .int, .float, .string: references.append((ownSlot, result.copy)) + default: break + } + return result + case .ref8, .ref16, .ref32: + let target = try reference(width: tag) + if isReference { + // A PHP reference to a value already written takes no number of its own, as `R:` in serialize(). + slot -= 1 + let value = PhpValue.reference(id: target.slot) + return Decoded(value: value, copy: Copy(value: value, slots: 0)) + } + // Not wrapped as a reference: the same array again by value, which serialize() writes out in full. + guard let copy = target.copy else { + throw BinaryValueDecodeError(kind: .unknownReference(target.id), offset: target.offset) + } + slot += copy.slots - 1 + guard slot <= IgbinaryDecoder.maxSlots else { throw reader.error(.tooLarge) } + return Decoded(value: copy.value, copy: copy) + case .objref8, .objref16, .objref32: + let value = PhpValue.reference(id: try reference(width: tag).slot) + return Decoded(value: value, copy: Copy(value: value, slots: 1)) + case .null: + return Self.scalar(.null) + case .boolFalse: + return Self.scalar(.bool(false)) + case .boolTrue: + return Self.scalar(.bool(true)) + case .long8p, .long8n, .long16p, .long16n, .long32p, .long32n, .long64p, .long64n: + return Self.scalar(.int(try integer(tag))) + case .double: + return Self.scalar(.float(Double(bitPattern: try reader.readUInt64()))) + case .stringEmpty: + return Self.scalar(.string("")) + case .stringID8, .stringID16, .stringID32: + return Self.scalar(.string(try storedString(width: tag))) + case .string8, .string16, .string32, .string64: + return Self.scalar(.string(try newString(tag))) + case .array8, .array16, .array32: + let count = try length(tag) + let index = references.count + references.append((ownSlot, nil)) + let children = try entries(count: count, depth: depth) + let copy = Copy( + value: .array(children.map { PhpKeyValue(key: $0.key, value: $0.copy.value) }), + slots: children.reduce(1) { $0 + $1.copy.slots } + ) + references[index].copy = copy + return Decoded(value: .array(children.map { PhpKeyValue(key: $0.key, value: $0.value) }), copy: copy) + case .object8, .object16, .object32: + return try object(className: try newString(tag), depth: depth, slot: ownSlot) + case .objectID8, .objectID16, .objectID32: + return try object(className: try storedString(width: tag), depth: depth, slot: ownSlot) + case .objectSer8, .objectSer16, .objectSer32, .enumCase: + throw invalid(tag, at: reader.offset - 1) + } + } + + private static func scalar(_ value: PhpValue) -> Decoded { + Decoded(value: value, copy: Copy(value: value, slots: 1)) + } + + private mutating func object(className: String, depth: Int, slot ownSlot: Int) throws -> Decoded { + let objectCopy = Copy(value: .reference(id: ownSlot), slots: 1) + references.append((ownSlot, objectCopy)) + let tag = try readTag() + let value: PhpValue + switch tag { + case .array8, .array16, .array32: + let count = try length(tag) + let properties = try entries(count: count, depth: depth).map { entry -> PhpProperty in + let property = decodePhpPropertyKey(Self.keyText(entry.key)) + return PhpProperty(name: property.name, visibility: property.visibility, value: entry.value) + } + value = .object(className: className, properties: properties) + case .objectSer8, .objectSer16, .objectSer32: + let payload = try reader.readBytes(try length(tag)) + value = .serializable(className: className, rawPayload: Self.text(payload)) + case .enumCase: + let caseTag = try readTag() + let caseName: String + switch caseTag { + case .string8, .string16, .string32, .string64: caseName = try newString(caseTag) + case .stringID8, .stringID16, .stringID32: caseName = try storedString(width: caseTag) + default: throw invalid(caseTag, at: reader.offset - 1) + } + value = .enumCase(className: className, caseName: caseName) + default: + throw invalid(tag, at: reader.offset - 1) + } + return Decoded(value: value, copy: objectCopy) + } + + // A count past the bytes left is forged; checking first keeps it from reserving gigabytes. + private mutating func entries(count: Int, depth: Int) throws -> [(key: PhpValue, value: PhpValue, copy: Copy)] { + guard count <= reader.remaining else { throw reader.error(.unexpectedEnd) } + var entries: [(key: PhpValue, value: PhpValue, copy: Copy)] = [] + entries.reserveCapacity(min(count, 1_024)) + for _ in 0.. String { + String(bytes: bytes, encoding: .utf8) ?? String(bytes: bytes, encoding: .isoLatin1) ?? "" + } + + private static func keyText(_ key: PhpValue) -> String { + if case .int(let number) = key { return String(number) } + if case .string(let text) = key { return text } + return "" + } + + private mutating func integer(_ tag: Tag) throws -> Int { + let start = reader.offset + let magnitude: UInt64 + let isNegative: Bool + switch tag { + case .long8p, .long8n: + magnitude = UInt64(try reader.readByte()) + isNegative = tag == .long8n + case .long16p, .long16n: + magnitude = UInt64(try reader.readUInt16()) + isNegative = tag == .long16n + case .long32p, .long32n: + magnitude = UInt64(try reader.readUInt32()) + isNegative = tag == .long32n + default: + magnitude = try reader.readUInt64() + isNegative = tag == .long64n + } + if isNegative { + guard magnitude <= UInt64(Int64.max) + 1 else { + throw BinaryValueDecodeError(kind: .integerOutOfRange, offset: start) + } + return Int(Int64(bitPattern: 0 &- magnitude)) + } + guard magnitude <= UInt64(Int64.max) else { + throw BinaryValueDecodeError(kind: .integerOutOfRange, offset: start) + } + return Int(magnitude) + } + + private mutating func length(_ tag: Tag) throws -> Int { + switch tag { + case .string8, .array8, .object8, .objectSer8: + return Int(try reader.readByte()) + case .string16, .array16, .object16, .objectSer16: + return Int(try reader.readUInt16()) + case .string64: + let start = reader.offset + let count = try reader.readUInt64() + guard count <= UInt64(reader.remaining) else { + throw BinaryValueDecodeError(kind: .unexpectedEnd, offset: start) + } + return Int(count) + default: + return Int(try reader.readUInt32()) + } + } + + private mutating func index(width tag: Tag) throws -> Int { + switch tag { + case .ref8, .objref8, .stringID8, .objectID8: + return Int(try reader.readByte()) + case .ref16, .objref16, .stringID16, .objectID16: + return Int(try reader.readUInt16()) + default: + return Int(try reader.readUInt32()) + } + } + + // Every string igbinary writes in full, keys and class names included, joins the table it refers back to. + private mutating func newString(_ tag: Tag) throws -> String { + let text = Self.text(try reader.readBytes(try length(tag))) + strings.append(text) + return text + } + + private mutating func storedString(width tag: Tag) throws -> String { + let start = reader.offset + let id = try index(width: tag) + guard id < strings.count else { + throw BinaryValueDecodeError(kind: .unknownReference(id), offset: start) + } + repeatedStringBytes += strings[id].utf8.count + guard repeatedStringBytes <= IgbinaryDecoder.maxRepeatedStringBytes else { + throw BinaryValueDecodeError(kind: .tooLarge, offset: start) + } + return strings[id] + } + + private mutating func reference(width tag: Tag) throws -> (id: Int, offset: Int, slot: Int, copy: Copy?) { + let start = reader.offset + let id = try index(width: tag) + guard id < references.count else { + throw BinaryValueDecodeError(kind: .unknownReference(id), offset: start) + } + return (id, start, references[id].slot, references[id].copy) + } + + private mutating func readTag() throws -> Tag { + let start = reader.offset + let byte = try reader.readByte() + guard let tag = Tag(rawValue: byte) else { + throw BinaryValueDecodeError(kind: .invalidByte(byte), offset: start) + } + return tag + } + + private func invalid(_ tag: Tag, at offset: Int) -> BinaryValueDecodeError { + BinaryValueDecodeError(kind: .invalidByte(tag.rawValue), offset: offset) + } + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift new file mode 100644 index 0000000000..993bbaa171 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift @@ -0,0 +1,82 @@ +// +// PhpSerializeWriter.swift +// TablePro +// + +import Foundation + +// Writes what PHP's serialize() writes, so the decoded text reads back in the PHP Serialized viewer. +internal enum PhpSerializeWriter { + static func text(_ value: PhpValue, limit: Int = .max) -> String { + var output = "" + write(value, into: &output, limit: limit) + return output + } + + private static func write(_ value: PhpValue, into output: inout String, limit: Int) { + guard output.utf8.count < limit else { return } + switch value { + case .null: + output += "N;" + case .bool(let flag): + output += flag ? "b:1;" : "b:0;" + case .int(let number): + output += "i:\(number);" + case .float(let number): + output += "d:\(floatText(number));" + case .string(let text): + output += quoted("s", text) + ";" + case .array(let entries): + output += "a:\(entries.count):{" + for entry in entries { + guard output.utf8.count < limit else { return } + write(entry.key, into: &output, limit: limit) + write(entry.value, into: &output, limit: limit) + } + output += "}" + case .object(let className, let properties): + output += quoted("O", className) + ":\(properties.count):{" + for property in properties { + guard output.utf8.count < limit else { return } + output += quoted("s", mangled(property)) + ";" + write(property.value, into: &output, limit: limit) + } + output += "}" + case .serializable(let className, let rawPayload): + output += quoted("C", className) + ":\(rawPayload.utf8.count):{\(rawPayload)}" + case .enumCase(let className, let caseName): + output += quoted("E", "\(className):\(caseName)") + ";" + case .reference(let id): + output += "R:\(id);" + case .unsupported(let token): + output += token + case .depthExceeded, .tooLarge: + output += "…" + } + } + + private static func quoted(_ token: String, _ text: String) -> String { + "\(token):\(text.utf8.count):\"\(text)\"" + } + + private static func mangled(_ property: PhpProperty) -> String { + switch property.visibility { + case .publicVisibility: return property.name + case .protectedVisibility: return "\u{0}*\u{0}\(property.name)" + case .privateVisibility(let className): return "\u{0}\(className)\u{0}\(property.name)" + } + } + + // serialize_precision -1: the shortest text that reads back, `1.0E+25` past 15 digits. + static func floatText(_ value: Double) -> String { + if value.isNaN { return "NAN" } + if value.isInfinite { return value > 0 ? "INF" : "-INF" } + if value == value.rounded(), abs(value) < 1e15 { return String(Int64(value)) } + let text = "\(value)" + guard let exponentIndex = text.firstIndex(of: "e") else { return text } + var mantissa = String(text[.. (name: String, visibility: PhpVisibility) { +internal func decodePhpPropertyKey(_ raw: String) -> (name: String, visibility: PhpVisibility) { let scalars = Array(raw.unicodeScalars) guard scalars.first?.value == 0 else { return (raw, .publicVisibility) diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift index 6676c6f15d..bfdc5b83b8 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift @@ -9,7 +9,7 @@ import Foundation -enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { +enum ValueDisplayFormat: Hashable, Codable, CaseIterable, Identifiable, RawRepresentable { case raw case text case uuid @@ -20,6 +20,58 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { case phpSerialized case messagePack case protobuf + case igbinary + // The schema lives in AvroSchemaLibrary; the format carries its ID so every per-column path can decode. + case avro(schemaID: String) + + private static let avroPrefix = "avro:" + + static let allCases: [ValueDisplayFormat] = [ + .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized, + .messagePack, .protobuf, .igbinary + ] + + init?(rawValue: String) { + if rawValue.hasPrefix(Self.avroPrefix) { + let schemaID = String(rawValue.dropFirst(Self.avroPrefix.count)) + guard !schemaID.isEmpty else { return nil } + self = .avro(schemaID: schemaID) + return + } + guard let format = Self.allCases.first(where: { $0.rawValue == rawValue }) else { return nil } + self = format + } + + var rawValue: String { + switch self { + case .raw: return "raw" + case .text: return "text" + case .uuid: return "uuid" + case .uuidSwappedTime: return "uuidSwappedTime" + case .unixTimestamp: return "unixTimestamp" + case .unixTimestampMillis: return "unixTimestampMillis" + case .json: return "json" + case .phpSerialized: return "phpSerialized" + case .messagePack: return "messagePack" + case .protobuf: return "protobuf" + case .igbinary: return "igbinary" + case .avro(let schemaID): return Self.avroPrefix + schemaID + } + } + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let rawValue = try container.decode(String.self) + guard let format = Self(rawValue: rawValue) else { + throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unknown display format \(rawValue)") + } + self = format + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(rawValue) + } var id: String { rawValue } @@ -33,7 +85,10 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { case .unixTimestampMillis: return String(localized: "Unix Timestamp (milliseconds)") case .json: return String(localized: "JSON") case .phpSerialized: return String(localized: "PHP Serialized") - case .messagePack, .protobuf: return binaryValueFormat?.displayName ?? rawValue + case .messagePack, .protobuf, .igbinary: return binaryValueFormat?.displayName ?? rawValue + case .avro(let schemaID): + guard let name = AvroSchemaLibrary.shared.name(id: schemaID) else { return "Avro" } + return String(format: String(localized: "Avro (%@)"), name) } } @@ -41,6 +96,8 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { switch self { case .messagePack: return .messagePack case .protobuf: return .protobuf + case .igbinary: return .igbinary + case .avro(let schemaID): return .avro(schemaID: schemaID) case .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: return nil } @@ -51,7 +108,7 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { switch self { case .raw: return [] - case .text, .uuidSwappedTime, .messagePack, .protobuf: + case .text, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro: return ["blob"] case .uuid: return ["blob", "text"] @@ -84,7 +141,7 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { /// searching exactly when one of these is on it; hex is not what anyone types into Find. var rendersBinaryAsText: Bool { switch self { - case .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf: + case .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro: return true case .raw, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: return false @@ -92,7 +149,7 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { } /// Returns applicable formats for a given column type. - /// Always includes `.raw` as the first option. + /// Always includes `.raw` as the first option. Avro is not listed: it needs a schema chosen first. static func applicableFormats( for columnType: ColumnType?, databaseType: DatabaseType? = nil diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift index 8e5cd8ed53..23780cd542 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift @@ -36,7 +36,7 @@ enum ValueDisplayFormatter { return formatAsTimestamp(rawValue, divideBy: 1_000) case .json, .phpSerialized: return rawValue - case .messagePack, .protobuf: + case .messagePack, .protobuf, .igbinary, .avro: guard let data = rawValue.data(using: .isoLatin1), let decoder = format.binaryValueFormat else { return nil } return BinaryValueDecoding.text(data, format: decoder) } @@ -54,7 +54,7 @@ enum ValueDisplayFormatter { guard rawValue.count == 16 else { return nil } let order: UUIDByteOrder = format == .uuidSwappedTime ? .swappedTime : .stored return insertUuidHyphens(order.uuidBytes(fromStored: [UInt8](rawValue)).hexEncoded) - case .messagePack, .protobuf: + case .messagePack, .protobuf, .igbinary, .avro: guard let decoder = format.binaryValueFormat else { return nil } return BinaryValueDecoding.text(rawValue, format: decoder) case .raw, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: diff --git a/apple/TablePro/Core/Storage/AvroSchemaLibrary.swift b/apple/TablePro/Core/Storage/AvroSchemaLibrary.swift new file mode 100644 index 0000000000..cf45750665 --- /dev/null +++ b/apple/TablePro/Core/Storage/AvroSchemaLibrary.swift @@ -0,0 +1,61 @@ +// +// AvroSchemaLibrary.swift +// TablePro +// + +import CryptoKit +import Foundation + +// A column's Avro format names its schema by a hash of the file's text, so the same file chosen twice is +// stored once. Read from the grid on the main actor and from the viewer off it. +internal final class AvroSchemaLibrary: @unchecked Sendable { + static let shared = AvroSchemaLibrary() + + static let keyPrefix = "com.TablePro.avroSchemas." + + private struct Entry: Codable { + let name: String + let text: String + } + + private let store: KeyValueStore + private let lock = NSLock() + private var parsed: [String: AvroSchema] = [:] + + init(store: KeyValueStore = AppStorageEnvironment.shared.defaults) { + self.store = store + } + + func register(text: String, name: String) throws -> String { + let schema = try AvroSchema.parse(text) + let id = SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() + guard let data = try? JSONEncoder().encode(Entry(name: name, text: text)) else { + throw AvroSchemaError(reason: String(localized: "The schema could not be saved.")) + } + lock.lock() + defer { lock.unlock() } + store.setDataValue(data, forKey: Self.keyPrefix + id) + parsed[id] = schema + return id + } + + func schema(id: String) -> AvroSchema? { + lock.lock() + defer { lock.unlock() } + if let schema = parsed[id] { return schema } + guard let entry = entry(id: id), let schema = try? AvroSchema.parse(entry.text) else { return nil } + parsed[id] = schema + return schema + } + + func name(id: String) -> String? { + lock.lock() + defer { lock.unlock() } + return entry(id: id)?.name + } + + private func entry(id: String) -> Entry? { + guard let data = store.dataValue(forKey: Self.keyPrefix + id) else { return nil } + return try? JSONDecoder().decode(Entry.self, from: data) + } +} diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 4b999ea5d9..9c55e9ec34 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -4011,6 +4011,9 @@ } } } + }, + "%@ is not a usable Avro schema" : { + }, "%@ is not a Vault address. Use a URL such as https://vault.example.com:8200." : { @@ -15526,6 +15529,9 @@ }, "A field needs a type." : { + }, + "A field of record %@ has no name or type." : { + }, "A file" : { "localizations" : { @@ -15829,6 +15835,9 @@ } } } + }, + "A map has no \"values\"." : { + }, "A map needs a geometry or geography column. This result has none." : { "localizations" : { @@ -16286,6 +16295,9 @@ } } } + }, + "A record, enum or fixed type has no name." : { + }, "A record's id cannot be edited." : { "extractionState" : "manual", @@ -16478,6 +16490,9 @@ }, "A saved query with this name exists." : { + }, + "A schema object has no \"type\"." : { + }, "A schema with this name already exists." : { "localizations" : { @@ -24019,6 +24034,9 @@ } } } + }, + "An array has no \"items\"." : { + }, "An attribute value must name exactly one type" : { "extractionState" : "manual", @@ -29696,6 +29714,12 @@ } } } + }, + "Avro (%@)" : { + + }, + "Avro Schema…" : { + }, "AWS answered without the Redshift user and password." : { @@ -32409,6 +32433,26 @@ }, "Byte %lld does not start a valid field." : { + }, + "Byte %lld picks entry %lld, which the schema does not have." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Byte %1$lld picks entry %2$lld, which the schema does not have." + } + } + } + }, + "Byte %lld refers back to item %lld, which comes later or not at all." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Byte %1$lld refers back to item %2$lld, which comes later or not at all." + } + } + } }, "Byte 0x%@ at offset %lld is not valid here." : { "localizations" : { @@ -38359,6 +38403,12 @@ } } } + }, + "Choose Schema" : { + + }, + "Choose the Avro schema the values in this column were written with." : { + }, "Choose the base type." : { @@ -72526,6 +72576,9 @@ } } } + }, + "Enum %@ has no symbols." : { + }, "Enum Type" : { "localizations" : { @@ -83405,6 +83458,9 @@ } } } + }, + "Fixed type %@ has no size." : { + }, "Flag" : { "extractionState" : "manual", @@ -143422,6 +143478,9 @@ } } } + }, + "Record %@ has no fields list." : { + }, "Record Access" : { "extractionState" : "manual", @@ -157129,6 +157188,9 @@ } } } + }, + "Schema Registry framing: schema ID %lld" : { + }, "Schema Registry framing: schema ID %lld, message index %@" : { "localizations" : { @@ -183294,6 +183356,9 @@ } } } + }, + "The Avro schema for this column is missing. Choose the schema file again." : { + }, "The AWS credentials for this profile have expired. Sign in again and retry." : { "localizations" : { @@ -186711,6 +186776,9 @@ }, "The group end at byte %lld matches no group start." : { + }, + "The header names version %lld, which this format does not have." : { + }, "The host cannot be passed to FreeTDS. Use a host name or an IP address, without spaces, brackets, “=”, “;” or “#”." : { "extractionState" : "manual", @@ -187008,6 +187076,9 @@ }, "The left result stopped at the row cap and is compared on the %@ rows it holds. A row past the cap can show as only on the right." : { + }, + "The length at byte %lld is negative." : { + }, "The license has been suspended." : { "localizations" : { @@ -189909,6 +189980,18 @@ }, "The schema changes staged in this diagram will be lost." : { + }, + "The schema could not be saved." : { + + }, + "The schema has a type that is not a name, an object or a union." : { + + }, + "The schema is not valid JSON." : { + + }, + "The schema names type %@ before defining it." : { + }, "The script already ran. Compare again to see where the target stands." : { "localizations" : { diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift index 1b2d3456ff..d10e5b55fd 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift @@ -4,6 +4,7 @@ // import AppKit +import UniformTypeIdentifiers struct DataGridDisplayFormatChoice { let columnID: DataGridColumnID @@ -100,6 +101,19 @@ extension DataGridViewController { item.state = format == current ? .on : .off submenu.addItem(item) } + let databaseType = model.configuration.traits.databaseType + if ValueDisplayFormat.avro(schemaID: "").isApplicable(to: column.type, databaseType: databaseType) { + if case .avro = current { + let item = menuItem( + current.displayName, + #selector(setDisplayFormatFromMenu(_:)), + DataGridDisplayFormatChoice(columnID: column.id, format: current) + ) + item.state = .on + submenu.addItem(item) + } + submenu.addItem(menuItem(String(localized: "Avro Schema…"), #selector(chooseAvroSchemaFromMenu(_:)), column.id)) + } let item = NSMenuItem(title: String(localized: "Display As"), action: nil, keyEquivalent: "") item.submenu = submenu return item @@ -142,6 +156,7 @@ extension DataGridViewController { #selector(DataGridViewController.clearValueFilterFromMenu(_:)), #selector(DataGridViewController.clearAllValueFiltersFromMenu(_:)), #selector(DataGridViewController.setDisplayFormatFromMenu(_:)), + #selector(DataGridViewController.chooseAvroSchemaFromMenu(_:)), #selector(DataGridViewController.sizeColumnToFitFromMenu(_:)), #selector(DataGridViewController.sizeAllColumnsToFitFromMenu(_:)), #selector(DataGridViewController.hideColumnFromMenu(_:)), @@ -237,6 +252,41 @@ extension DataGridViewController { actions.gridConfirmDisplayOrderChange(then: apply) } + // The schema is read once and kept, so the column still decodes after the file moves. + @objc func chooseAvroSchemaFromMenu(_ sender: Any?) { + guard let columnID = menuColumn(sender)?.id, let window = view.window else { return } + let panel = NSOpenPanel() + panel.canChooseFiles = true + panel.canChooseDirectories = false + panel.allowsMultipleSelection = false + panel.allowedContentTypes = [UTType(filenameExtension: "avsc") ?? .json, .json] + panel.prompt = String(localized: "Choose Schema") + panel.message = String(localized: "Choose the Avro schema the values in this column were written with.") + Task { [weak self] in + guard await panel.presentAsSheet(for: window) == .OK, let url = panel.url else { return } + self?.applyAvroSchema(at: url, columnID: columnID, window: window) + } + } + + private func applyAvroSchema(at url: URL, columnID: DataGridColumnID, window: NSWindow) { + let schemaID: String + do { + let text = try String(contentsOf: url, encoding: .utf8) + schemaID = try AvroSchemaLibrary.shared.register(text: text, name: url.lastPathComponent) + } catch { + let reason = (error as? AvroSchemaError)?.reason ?? error.localizedDescription + let alert = NSAlert() + alert.alertStyle = .warning + alert.messageText = String(format: String(localized: "%@ is not a usable Avro schema"), url.lastPathComponent) + alert.informativeText = reason + alert.beginSheetModal(for: window) + return + } + let item = NSMenuItem() + item.representedObject = DataGridDisplayFormatChoice(columnID: columnID, format: .avro(schemaID: schemaID)) + setDisplayFormatFromMenu(item) + } + @objc func sizeColumnToFitFromMenu(_ sender: Any?) { guard let column = menuColumn(sender) else { return } sizeColumnToFit(column) diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index 43d6f59ed2..75fdb52f26 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -129,6 +129,12 @@ internal struct BinaryValueViewerView: View { searchText: $searchText, fullValueModeName: String(localized: "Text") ) + case .php(let root): + FilterableTreeView( + rootNode: root, + searchText: $searchText, + fullValueModeName: String(localized: "Text") + ) } } diff --git a/apple/TablePro/Views/Results/CellInteractionResolver.swift b/apple/TablePro/Views/Results/CellInteractionResolver.swift index aa64b5e819..5c9be4a713 100644 --- a/apple/TablePro/Views/Results/CellInteractionResolver.swift +++ b/apple/TablePro/Views/Results/CellInteractionResolver.swift @@ -99,7 +99,7 @@ internal struct CellInteractionResolver { case .phpSerialized: return .viewPhpSerialized case .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .messagePack, .protobuf, - .none: + .igbinary, .avro, .none: return plainText(for: context, isReadOnly: isReadOnly) } } diff --git a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift index 2976e16b9f..0ebe813d53 100644 --- a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift +++ b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift @@ -51,7 +51,7 @@ internal enum FieldEditorResolver { return .phpSerialized case .json: return .json - case .messagePack, .protobuf: + case .messagePack, .protobuf, .igbinary, .avro: // A NULL keeps the hex editor, the way bytes get into it. if type.isBlobType || isBinaryValue, originalValue != nil, let format = override.binaryValueFormat { return .binaryValue(format) diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift new file mode 100644 index 0000000000..d9dcb6848f --- /dev/null +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift @@ -0,0 +1,178 @@ +// +// AvroDecoderTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +/// Fixtures written by fastavro 1.13.1, `schemaless_writer(buffer, parse_schema(schema), record)`. +struct AvroDecoderTests { + private static let userSchema = #"{"type":"record","name":"User","namespace":"com.example","fields":["# + + #"{"name":"id","type":"long"},{"name":"name","type":"string"},{"name":"email","type":["null","string"]},"# + + #"{"name":"active","type":"boolean"},{"name":"score","type":"double"},{"name":"ratio","type":"float"},"# + + #"{"name":"tags","type":{"type":"array","items":"string"}},{"name":"attrs","type":{"type":"map","values":"int"}},"# + + #"{"name":"kind","type":{"type":"enum","name":"Kind","symbols":["A","B","C"]}},"# + + #"{"name":"hash","type":{"type":"fixed","name":"Hash","size":4}},{"name":"raw","type":"bytes"},"# + + #"{"name":"nothing","type":"null"}]}"# + + private static let logicalSchema = #"{"type":"record","name":"Logical","fields":["# + + #"{"name":"id","type":{"type":"string","logicalType":"uuid"}},"# + + #"{"name":"day","type":{"type":"int","logicalType":"date"}},"# + + #"{"name":"tms","type":{"type":"int","logicalType":"time-millis"}},"# + + #"{"name":"tus","type":{"type":"long","logicalType":"time-micros"}},"# + + #"{"name":"tsms","type":{"type":"long","logicalType":"timestamp-millis"}},"# + + #"{"name":"tsus","type":{"type":"long","logicalType":"timestamp-micros"}},"# + + #"{"name":"local","type":{"type":"long","logicalType":"local-timestamp-millis"}},"# + + #"{"name":"price","type":{"type":"bytes","logicalType":"decimal","precision":10,"scale":2}},"# + + #"{"name":"weight","type":{"type":"fixed","name":"Weight","size":8,"logicalType":"decimal","precision":12,"scale":3}}]}"# + + private static func json(_ hex: String, schema: String) throws -> String { + let message = try AvroDecoder.decode(BinaryFixture.data(hex), schema: try AvroSchema.parse(schema)) + return MessagePackTextWriter.json(message.value, pretty: false) + } + + @Test("A record with every primitive and complex type") + func record() throws { + let hex = "9593d89fee470e416e6e204c6565021e616e6e406578616d706c652e636f6d01000000000000f83f000080be0402610662c3a9" + + "000402780202790300040102feff040010" + let expected = #"{"id":-1234567890123,"name":"Ann Lee","email":"ann@example.com","active":true,"score":1.5,"# + + #""ratio":-0.25,"tags":["a","bé"],"attrs":{"x":1,"y":-2},"kind":"C","hash":"0x0102FEFF","raw":"0x0010","# + + #""nothing":null}"# + #expect(try Self.json(hex, schema: Self.userSchema) == expected) + } + + @Test("A datum that starts with zero bytes reads as it is, not as a Kafka frame") + func leadingZeroDatum() throws { + let message = try AvroDecoder.decode( + BinaryFixture.data(String(repeating: "00", count: 24)), + schema: try AvroSchema.parse(Self.userSchema) + ) + #expect(message.schemaID == nil) + let expected = #"{"id":0,"name":"","email":null,"active":false,"score":0.0,"ratio":0.0,"tags":[],"attrs":{},"# + + #""kind":"A","hash":"0x00000000","raw":"0x","nothing":null}"# + #expect(MessagePackTextWriter.json(message.value, pretty: false) == expected) + } + + @Test("A Kafka value keeps its schema ID and decodes past the prefix") + func confluentFrame() throws { + let hex = "000000002a0e024b000100000000000000400000803f0202740002026b06000261626364027f" + let message = try AvroDecoder.decode(BinaryFixture.data(hex), schema: try AvroSchema.parse(Self.userSchema)) + #expect(message.schemaID == 42) + let expected = #"{"id":7,"name":"K","email":null,"active":true,"score":2.0,"ratio":1.0,"tags":["t"],"attrs":{"k":3},"# + + #""kind":"B","hash":"0x61626364","raw":"0x7F","nothing":null}"# + #expect(MessagePackTextWriter.json(message.value, pretty: false) == expected) + } + + @Test("Logical types render as the values they stand for") + func logicalTypes() throws { + let hex = "4836636364373830632d626162612d313032362d393536342d35623863363536303234646296b502aab2992bcaa1c4de1b" + + "aa82e1e9c163018ef6e0e9c16306fe1dc00000000000003039" + let expected = #"{"id":"6ccd780c-baba-1026-9564-5b8c656024db","day":"2024-03-05","tms":"12:34:56.789","# + + #""tus":"01:02:03.004005","tsms":"2024-03-05T12:34:56.789Z","tsus":"1969-12-31T23:59:59.999999Z","# + + #""local":"2024-03-05T12:34:56.007","price":"-1234.56","weight":"12.345"}"# + #expect(try Self.json(hex, schema: Self.logicalSchema) == expected) + } + + @Test("A record that refers to itself by name") + func recursiveRecord() throws { + let schema = #"{"type":"record","name":"Node","namespace":"com.example.list","fields":["# + + #"{"name":"value","type":"int"},{"name":"next","type":["null","Node"]}]}"# + #expect(try Self.json("020204020600", schema: schema) + == #"{"value":1,"next":{"value":2,"next":{"value":3,"next":null}}}"#) + } + + @Test("A bare long at its minimum, and a union of several branches") + func primitivesAndUnions() throws { + #expect(try Self.json("ffffffffffffffffff01", schema: #""long""#) == "-9223372036854775808") + let union = #"{"type":"array","items":["int","string",{"type":"array","items":"long"}]}"# + #expect(try Self.json("06000a020273040402010000", schema: union) == #"[5,"s",[1,-1]]"#) + } + + @Test("An array block with a negative count carries its byte size") + func negativeBlockCount() throws { + #expect(try Self.json("0304020400", schema: #"{"type":"array","items":"int"}"#) == "[1,2]") + } + + @Test("An array of nulls holds more items than bytes, up to a limit") + func zeroByteItems() throws { + let nulls = #"{"type":"array","items":"null"}"# + #expect(try Self.json("0400", schema: nulls) == "[null,null]") + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("feffffffffffffffff0100"), schema: try AvroSchema.parse(nulls)) + } + #expect(error?.kind == .tooLarge) + } + + @Test("A decimal without a usable precision and scale reads as its bytes") + func invalidDecimalFallsBack() throws { + for annotation in [#""scale":2"#, #""precision":2,"scale":3"#, #""precision":100000000,"scale":100000000"#] { + let schema = #"{"type":"bytes","logicalType":"decimal","# + annotation + "}" + #expect(try Self.json("0230", schema: schema) == #""0x30""#, "\(annotation)") + } + } + + @Test("Decimals of any width keep their sign and scale") + func decimalText() { + #expect(AvroLogicalRendering.decimalText(unscaled: [0x00], scale: 2) == "0.00") + #expect(AvroLogicalRendering.decimalText(unscaled: [0xFF], scale: 3) == "-0.001") + #expect(AvroLogicalRendering.decimalText(unscaled: [0x30, 0x39], scale: 0) == "12345") + #expect(AvroLogicalRendering.decimalText(unscaled: [0x80] + Array(repeating: 0, count: 15), scale: 0) + == "-170141183460469231731687303715884105728") + } + + @Test("A value that does not fit the schema names what is wrong and where") + func brokenValues() throws { + let schema = try AvroSchema.parse(Self.userSchema) + let cases: [(hex: String, kind: BinaryValueDecodeError.Kind)] = [ + ("02", .unexpectedEnd), + ("020e416e6e", .unexpectedEnd), + ("02000600", .indexOutOfRange(3)), + ("0201", .invalidLength), + ("02000002", .invalidByte(0x02)), + ] + for testCase in cases { + let error = #expect(throws: BinaryValueDecodeError.self, "\(testCase.hex)") { + try AvroDecoder.decode(BinaryFixture.data(testCase.hex), schema: schema) + } + #expect(error?.kind == testCase.kind, "\(testCase.hex)") + } + let int = try AvroSchema.parse(#""int""#) + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("8080808020"), schema: int) + } + #expect(error?.kind == .integerOutOfRange) + let trailing = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("0202"), schema: int) + } + #expect(trailing?.kind == .trailingBytes(1)) + } + + @Test("A schema that is not one says why") + func brokenSchemas() { + for text in [ + "not json", + #"{"type":"record","name":"R"}"#, + #"{"type":"record","name":"R","fields":[{"name":"x","type":"Missing"}]}"#, + #"{"type":"enum","name":"E"}"#, + #"{"type":"fixed","name":"F"}"#, + #"{"type":"array"}"#, + "42", + ] { + #expect(throws: AvroSchemaError.self, "\(text)") { try AvroSchema.parse(text) } + } + } + + @Test("A named type resolves inside its namespace and by full name") + func namespaces() throws { + let schema = try AvroSchema.parse( + #"{"type":"record","name":"Outer","namespace":"a.b","fields":["# + + #"{"name":"e","type":{"type":"enum","name":"Color","symbols":["RED"]}},"# + + #"{"name":"f","type":"Color"},{"name":"g","type":"a.b.Color"}]}"# + ) + #expect(schema.namedTypes["a.b.Color"] == .enumeration(name: "a.b.Color", symbols: ["RED"])) + let value = try AvroDecoder.decode(BinaryFixture.data("000000"), schema: schema).value + #expect(MessagePackTextWriter.json(value, pretty: false) == #"{"e":"RED","f":"RED","g":"RED"}"#) + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift index 298f4fa66a..bd6f048caf 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift @@ -131,4 +131,51 @@ struct BinaryDisplayFormatTests { #expect(text == "1: 150\n") #expect(note?.contains("42") == true) } + + @Test("igbinary and Avro render in the cell, and a broken value falls back to hex") + func igbinaryAndAvroRender() throws { + let blob = ColumnType.blob(rawType: "BLOB") + let igbinary = BinaryFixture.data("000000021705506f696e7414011101780601") + #expect(CellDisplayFormatter.format(.bytes(igbinary), columnType: blob, displayFormat: .igbinary) + == #"O:5:"Point":1:{s:1:"x";i:1;}"#) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("0000000211")), columnType: blob, displayFormat: .igbinary) + == "0x0000000211") + + let schemaID = try AvroSchemaLibrary.shared.register( + text: #"{"type":"record","name":"R","fields":[{"name":"id","type":"long"}]}"#, + name: "r.avsc" + ) + let avro = ValueDisplayFormat.avro(schemaID: schemaID) + #expect(avro.displayName.contains("r.avsc")) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("54")), columnType: blob, displayFormat: avro) == #"{"id":42}"#) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("80")), columnType: blob, displayFormat: avro) == "0x80") + } + + @Test("The viewer shows igbinary in the PHP tree and Avro with its frame's schema ID") + func igbinaryAndAvroDocuments() throws { + let igbinary = BinaryValueDecoding.document(BinaryFixture.data("000000020607"), format: .igbinary) + guard case .decoded(.php, let text, nil) = igbinary else { + Issue.record("expected a decoded PHP tree") + return + } + #expect(text == "i:7;") + + let schemaID = try AvroSchemaLibrary.shared.register(text: #"{"type":"array","items":"int"}"#, name: "ints.avsc") + let framed = BinaryValueDecoding.document(BinaryFixture.data("000000002a020200"), format: .avro(schemaID: schemaID)) + guard case .decoded(.json, _, let note) = framed else { + Issue.record("expected a decoded JSON tree") + return + } + #expect(note?.contains("42") == true) + } + + @Test("An Avro column whose schema is gone says so") + func avroSchemaMissing() { + let document = BinaryValueDecoding.document(Data([0x02]), format: .avro(schemaID: "0000000000000000")) + guard case .failed(let error) = document else { + Issue.record("expected a failure") + return + } + #expect(error.kind == .schemaMissing) + } } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift new file mode 100644 index 0000000000..4b9fed3b2e --- /dev/null +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift @@ -0,0 +1,182 @@ +// +// IgbinaryDecoderTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +/// Fixtures written by the igbinary 3.2.17 extension on PHP 8.3, `igbinary_serialize($value)`, each beside +/// `serialize($value)` of the same value. +struct IgbinaryDecoderTests { + private static let pointProperties = #"s:1:"x";i:1;s:4:"\#u{0}*\#u{0}y";i:2;s:8:"\#u{0}Point\#u{0}z";i:3;"# + + private static let fixtures: [(name: String, hex: String, serialized: String)] = [ + ("null", "0000000200", "N;"), + ("true", "0000000205", "b:1;"), + ("false", "0000000204", "b:0;"), + ("small int", "000000020607", "i:7;"), + ("negative int", "0000000209012c", "i:-300;"), + ("32-bit int", "000000020a00011170", "i:70000;"), + ("64-bit int", "0000000220000000012a05f200", "i:5000000000;"), + ("PHP_INT_MAX", "00000002207fffffffffffffff", "i:9223372036854775807;"), + ("PHP_INT_MIN", "00000002218000000000000000", "i:-9223372036854775808;"), + ("float", "000000020c400a000000000000", "d:3.25;"), + ("empty string", "000000020d", #"s:0:"";"#), + ("UTF-8 string", "00000002110d68c3a96c6c6f2077c3b6726c64", #"s:13:"héllo wörld";"#), + ( + "list of scalars", + "000000021406060006010601060206021105746872656506030c4012000000000000060400060504", + #"a:6:{i:0;i:1;i:1;i:2;i:2;s:5:"three";i:3;d:4.5;i:4;N;i:5;b:0;}"# + ), + ( + "nested map with int keys", + "00000002140511046e616d651103416e6e11047461677314020600110161060111016211046d65746114020e00" + + "1105696e6e657211016e0701060a110374656e070511036e6567", + #"a:5:{s:4:"name";s:3:"Ann";s:4:"tags";a:2:{i:0;s:1:"a";i:1;s:1:"b";}s:4:"meta";"# + + #"a:2:{s:4:"name";s:5:"inner";s:1:"n";i:-1;}i:10;s:3:"ten";i:-5;s:3:"neg";}"# + ), + ( + "repeated strings by ID", + "00000002140506001105616c706861060111046265746106020e0006030e0106040e00", + #"a:5:{i:0;s:5:"alpha";i:1;s:4:"beta";i:2;s:5:"alpha";i:3;s:4:"beta";i:4;s:5:"alpha";}"# + ), + ( + "object with every visibility", + "000000021705506f696e74140311017806011104002a00790602110800506f696e74007a0603", + #"O:5:"Point":3:{"# + pointProperties + "}" + ), + ( + "stdClass", + "000000021708737464436c617373140211016106011101621401060005", + #"O:8:"stdClass":2:{s:1:"a";i:1;s:1:"b";a:1:{i:0;b:1;}}"# + ), + ("enum case", "00000002170453756974271106486561727473", #"E:11:"Suit:Hearts";"#), + ] + + @Test("Each fixture decodes to the value PHP's serialize() wrote, and writes it back the same") + func fixturesMatchSerialize() throws { + for fixture in Self.fixtures { + let decoded = try IgbinaryDecoder.decode(BinaryFixture.data(fixture.hex)) + #expect(decoded == PhpSerializeParser.parse(fixture.serialized), "\(fixture.name)") + #expect(PhpSerializeWriter.text(decoded) == fixture.serialized, "\(fixture.name)") + } + } + + @Test("A string of 300 bytes uses the 16-bit length") + func longString() throws { + let hex = "0000000212012c" + String(repeating: "61", count: 300) + #expect(try IgbinaryDecoder.decode(BinaryFixture.data(hex)) == .string(String(repeating: "a", count: 300))) + } + + @Test("A repeated object refers back with the number serialize() gives it") + func repeatedObject() throws { + let hex = "00000002140306001705506f696e74140311017806011104002a00790602110800506f696e74007a0603" + + "0601220106021a0014030e0106010e0206020e030603" + let point = #"O:5:"Point":3:{"# + Self.pointProperties + "}" + let serialized = "a:3:{i:0;\(point)i:1;r:2;i:2;\(point)}" + #expect(try IgbinaryDecoder.decode(BinaryFixture.data(hex)) == PhpSerializeParser.parse(serialized)) + } + + @Test("A repeated enum case refers back to the first one") + func repeatedEnumCase() throws { + let hex = "000000021403060017045375697427110648656172747306011a0027110653706164657306022201" + let serialized = #"a:3:{i:0;E:11:"Suit:Hearts";i:1;E:11:"Suit:Spades";i:2;r:2;}"# + #expect(try IgbinaryDecoder.decode(BinaryFixture.data(hex)) == PhpSerializeParser.parse(serialized)) + } + + @Test("A PHP reference counts once, as R: numbers it") + func phpReference() throws { + let hex = "0000000214020600251401060011067368617265640601250101" + let serialized = #"a:2:{i:0;a:1:{i:0;s:6:"shared";}i:1;R:2;}"# + #expect(try IgbinaryDecoder.decode(BinaryFixture.data(hex)) == PhpSerializeParser.parse(serialized)) + #expect(PhpSerializeWriter.text(try IgbinaryDecoder.decode(BinaryFixture.data(hex))) == serialized) + } + + @Test("An array repeated by value is written out again, as serialize() writes it, and later numbers follow") + func arraysByValue() throws { + let cases: [(hex: String, serialized: String)] = [ + ( + "0000000214020600140206000601060111017806010101", + #"a:2:{i:0;a:2:{i:0;i:1;i:1;s:1:"x";}i:1;a:2:{i:0;i:1;i:1;s:1:"x";}}"# + ), + ( + "0000000214030600140206001705506f696e7414011101780601060106020601010106022202", + #"a:3:{i:0;a:2:{i:0;O:5:"Point":1:{s:1:"x";i:1;}i:1;i:2;}i:1;a:2:{i:0;r:3;i:1;i:2;}i:2;r:3;}"# + ), + ( + "0000000214030600140111016b140206000601060106020601010206020101", + #"a:3:{i:0;a:1:{s:1:"k";a:2:{i:0;i:1;i:1;i:2;}}i:1;a:2:{i:0;i:1;i:1;i:2;}i:2;a:1:{s:1:"k";a:2:{i:0;i:1;i:1;i:2;}}}"# + ), + ( + "000000021404060014030600060106010602060206030601010106021705506f696e741401110178060106032202", + #"a:4:{i:0;a:3:{i:0;i:1;i:1;i:2;i:2;i:3;}i:1;a:3:{i:0;i:1;i:1;i:2;i:2;i:3;}i:2;O:5:"Point":1:{s:1:"x";i:1;}i:3;r:10;}"# + ), + ("000000021708737464436c6173731401110473656c662200", #"O:8:"stdClass":1:{s:4:"self";r:1;}"#), + ("00000002140306002506050601250101060211017a", #"a:3:{i:0;i:5;i:1;R:2;i:2;s:1:"z";}"#), + ] + for testCase in cases { + #expect(try IgbinaryDecoder.decode(BinaryFixture.data(testCase.hex)) == PhpSerializeParser.parse(testCase.serialized)) + } + } + + @Test("Arrays nested by value forty deep are refused rather than expanded") + func exponentialCopiesAreRefused() { + func level(_ depth: Int, of total: Int) -> String { + guard depth > 0 else { return "140206000601060106010601" } + return "14020600" + level(depth - 1, of: total) + "060101" + String(format: "%02x", total - depth + 1) + } + let error = #expect(throws: BinaryValueDecodeError.self) { + try IgbinaryDecoder.decode(BinaryFixture.data("00000002" + level(40, of: 40))) + } + #expect(error?.kind == .tooLarge) + #expect(throws: Never.self) { try IgbinaryDecoder.decode(BinaryFixture.data("00000002" + level(3, of: 3))) } + } + + @Test("A long string repeated by ID past the limit is refused") + func repeatedLongStringIsRefused() { + var hex = "0000000215" + String(format: "%04x", 6_000) + "080000" + "1300001000" + String(repeating: "61", count: 4_096) + for index in 1..<6_000 { + hex += "08" + String(format: "%04x", index) + "0e00" + } + let error = #expect(throws: BinaryValueDecodeError.self) { try IgbinaryDecoder.decode(BinaryFixture.data(hex)) } + #expect(error?.kind == .tooLarge) + } + + @Test("The version 1 header reads too") + func versionOne() throws { + #expect(try IgbinaryDecoder.decode(BinaryFixture.data("000000010607")) == .int(7)) + } + + @Test("A value that is not igbinary names what is wrong and where") + func brokenValues() { + let cases: [(hex: String, kind: BinaryValueDecodeError.Kind)] = [ + ("613a313a7b", .unknownVersion(0x613A_313A)), + ("00000002", .unexpectedEnd), + ("00000002110568656c", .unexpectedEnd), + ("00000002ff", .invalidByte(0xFF)), + ("000000020607aa", .trailingBytes(1)), + ("00000002140106000e05", .unknownReference(5)), + ("0000000214010600220a", .unknownReference(10)), + ("0000000214011400", .invalidByte(0x14)), + ("0000000220ffffffffffffffff", .integerOutOfRange), + ("0000000214ff", .unexpectedEnd), + ] + for testCase in cases { + let error = #expect(throws: BinaryValueDecodeError.self, "\(testCase.hex)") { + try IgbinaryDecoder.decode(BinaryFixture.data(testCase.hex)) + } + #expect(error?.kind == testCase.kind, "\(testCase.hex)") + } + } + + @Test("Floats follow serialize_precision -1") + func floatText() { + #expect(PhpSerializeWriter.floatText(1) == "1") + #expect(PhpSerializeWriter.floatText(0.1) == "0.1") + #expect(PhpSerializeWriter.floatText(1e25) == "1.0E+25") + #expect(PhpSerializeWriter.floatText(1.5e-7) == "1.5E-7") + #expect(PhpSerializeWriter.floatText(-.infinity) == "-INF") + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift index a49ee6578c..771fa87b66 100644 --- a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift @@ -21,6 +21,21 @@ struct ValueDisplayFormatTests { #expect(ValueDisplayFormat.uuidSwappedTime.rawValue == "uuidSwappedTime") #expect(ValueDisplayFormat.messagePack.rawValue == "messagePack") #expect(ValueDisplayFormat.protobuf.rawValue == "protobuf") + #expect(ValueDisplayFormat.igbinary.rawValue == "igbinary") + #expect(ValueDisplayFormat.avro(schemaID: "ab12").rawValue == "avro:ab12") + } + + @Test("Every format reads back from its stored form, an Avro one with its schema ID") + func codableRoundTrip() throws { + let formats: [String: ValueDisplayFormat] = [ + "a": .messagePack, "b": .igbinary, "c": .avro(schemaID: "ab12"), "d": .raw, + ] + let data = try JSONEncoder().encode(formats) + #expect(try JSONDecoder().decode([String: ValueDisplayFormat].self, from: data) == formats) + #expect(try JSONDecoder().decode([String: ValueDisplayFormat].self, from: Data(#"{"x":"protobuf"}"#.utf8)) + == ["x": .protobuf]) + #expect(ValueDisplayFormat(rawValue: "avro:") == nil) + #expect(ValueDisplayFormat(rawValue: "nope") == nil) } @Test("only the formats that produce characters count as rendering binary as text") @@ -30,6 +45,8 @@ struct ValueDisplayFormatTests { #expect(ValueDisplayFormat.uuidSwappedTime.rendersBinaryAsText) #expect(ValueDisplayFormat.messagePack.rendersBinaryAsText) #expect(ValueDisplayFormat.protobuf.rendersBinaryAsText) + #expect(ValueDisplayFormat.igbinary.rendersBinaryAsText) + #expect(ValueDisplayFormat.avro(schemaID: "ab12").rendersBinaryAsText) #expect(!ValueDisplayFormat.raw.rendersBinaryAsText) #expect(!ValueDisplayFormat.unixTimestamp.rendersBinaryAsText) #expect(!ValueDisplayFormat.json.rendersBinaryAsText) @@ -60,7 +77,7 @@ struct ValueDisplayFormatTests { let formats = ValueDisplayFormat.applicableFormats(for: .blob(rawType: "BLOB")) #expect(!formats.contains(.json)) #expect(!formats.contains(.phpSerialized)) - #expect(formats == [.raw, .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf]) + #expect(formats == [.raw, .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf, .igbinary]) } @Test("The binary decoders and the swapped UUID stay off text and integer columns") @@ -70,6 +87,8 @@ struct ValueDisplayFormatTests { #expect(!formats.contains(.uuidSwappedTime)) #expect(!formats.contains(.messagePack)) #expect(!formats.contains(.protobuf)) + #expect(!formats.contains(.igbinary)) + #expect(!ValueDisplayFormat.avro(schemaID: "ab12").isApplicable(to: columnType)) } } @@ -81,7 +100,7 @@ struct ValueDisplayFormatTests { ) #expect(!formats.contains(.uuid)) - #expect(formats == [.raw, .text, .messagePack, .protobuf]) + #expect(formats == [.raw, .text, .messagePack, .protobuf, .igbinary]) } @Test("nil column type returns only raw") From 74112986786fe501b0dbe3f41106e0410013c604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:28:50 +0700 Subject: [PATCH 02/22] docs(changelog): add the igbinary and Avro entry --- apple/changelog.d/3962.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 apple/changelog.d/3962.added.md diff --git a/apple/changelog.d/3962.added.md b/apple/changelog.d/3962.added.md new file mode 100644 index 0000000000..95ab228dca --- /dev/null +++ b/apple/changelog.d/3962.added.md @@ -0,0 +1 @@ +Display As decodes igbinary and Avro values in binary columns, Avro with a schema file chosen for the column. From 93c3982e0d00f3e373d588b87a069340b11c98c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:39:53 +0700 Subject: [PATCH 03/22] test(display-format): build the nested igbinary fixture with the right leaf --- .../Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift index 4b9fed3b2e..63f9ac5abf 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift @@ -124,7 +124,7 @@ struct IgbinaryDecoderTests { @Test("Arrays nested by value forty deep are refused rather than expanded") func exponentialCopiesAreRefused() { func level(_ depth: Int, of total: Int) -> String { - guard depth > 0 else { return "140206000601060106010601" } + guard depth > 0 else { return "14020600060106010601" } return "14020600" + level(depth - 1, of: total) + "060101" + String(format: "%02x", total - depth + 1) } let error = #expect(throws: BinaryValueDecodeError.self) { From 9cd5287acd109ceb80a6bdc8fa858bdec98ac7ca Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 06:17:50 +0700 Subject: [PATCH 04/22] refactor(display-format): match serialize() and Avro's readers, keep formats readable by 0.83 Fixes the review of #3962: igbinary r:/R: numbering, copies, caps and float text; Avro framing, decimal width, named logical types and zero-byte caps; a second storage key for formats 0.83 lacks; schema files instead of defaults. --- .../Core/DataGrid/DataGridModel.swift | 9 + .../Formatting/BinaryValues/AvroDecoder.swift | 335 +++++++++++----- .../Formatting/BinaryValues/AvroSchema.swift | 123 +++++- .../BinaryValues/BinaryValueDecoding.swift | 22 +- .../BinaryValues/BinaryValueReader.swift | 9 +- .../BinaryValues/IgbinaryDecoder.swift | 360 +++++++++++------- .../BinaryValues/PhpSerializeWriter.swift | 61 ++- .../Formatting/PhpSerializeParser.swift | 28 +- .../Formatting/ValueDisplayFormat.swift | 51 +-- .../ValueDisplayFormatService.swift | 16 +- .../Formatting/ValueDisplayFormatter.swift | 16 +- .../Infrastructure/AvroSchemaFilePanel.swift | 71 ++++ .../Core/Storage/AvroSchemaLibrary.swift | 61 --- .../Core/Storage/AvroSchemaStore.swift | 87 +++++ .../Storage/ValueDisplayFormatStorage.swift | 106 ++++-- apple/TablePro/Models/UI/PhpTreeNode.swift | 26 +- .../TablePro/Resources/Localizable.xcstrings | 26 +- .../DataGridViewController+HeaderMenu.swift | 62 ++- .../Views/Results/BinaryValueViewerView.swift | 9 + .../FieldEditors/FieldEditorResolver.swift | 70 ++-- .../BinaryValues/AvroDecoderTests.swift | 146 ++++++- .../BinaryDisplayFormatTests.swift | 21 +- .../BinaryValues/IgbinaryDecoderTests.swift | 252 ++++++++---- .../Formatting/PhpSerializeParserTests.swift | 17 +- .../Formatting/ValueDisplayFormatTests.swift | 4 +- .../Core/Storage/AvroSchemaStoreTests.swift | 67 ++++ .../ValueDisplayFormatStorageTests.swift | 79 ++++ .../DataGrid/DataGridDisplayCacheTests.swift | 17 + .../CellInteractionResolverTests.swift | 2 + .../Shared/FieldEditorResolverTests.swift | 89 +++-- 30 files changed, 1590 insertions(+), 652 deletions(-) create mode 100644 apple/TablePro/Core/Services/Infrastructure/AvroSchemaFilePanel.swift delete mode 100644 apple/TablePro/Core/Storage/AvroSchemaLibrary.swift create mode 100644 apple/TablePro/Core/Storage/AvroSchemaStore.swift create mode 100644 apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index 5e665060df..e79c9008fc 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -270,6 +270,15 @@ final class DataGridModel: ObservableObject { store.valueFilter = remapped } store.displayFormats = formats + redrawFormatted(column, in: current) + } + + // The format is unchanged but its decoder now reads what it could not, as when a lost Avro schema is chosen again. + func redisplay(_ column: DataGridColumn) { + redrawFormatted(column, in: rows) + } + + private func redrawFormatted(_ column: DataGridColumn, in current: TableRows) { display.cache.removeAll() if store.valueFilter.isActive { order = resolveOrder(in: current) diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift index 3a69bff31f..656d1636e7 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift @@ -5,48 +5,75 @@ import Foundation +internal enum AvroFraming: Equatable, Sendable { + // Confluent Schema Registry: magic byte 0, then the schema ID, big-endian. + case confluent(schemaID: UInt32) + // Avro single-object encoding: C3 01, then the CRC-64-AVRO fingerprint of the writer schema, little-endian. + case singleObject(fingerprint: UInt64) +} + internal struct AvroMessage: Equatable, Sendable { let value: MessagePackValue - let schemaID: UInt32? + let framing: AvroFraming? } // Values land in MessagePack's model, which already has the tree, the JSON writer and the timestamp badge. // A union shows the branch it holds, as Avro's own readers return it. internal enum AvroDecoder { static let maxDepth = 256 + // Items that take no bytes (null, an empty record) are bounded by this, not by the bytes left. static let maxZeroByteItems: Int64 = 1_000_000 - // Long division is quadratic in the width; past this a decimal shows as its bytes. - static let maxDecimalBytes = 512 - // A Kafka value carries the Confluent prefix: magic byte 0 and a four-byte schema ID. Zero also starts - // plenty of plain datums, so the prefix is only tried once the value fails to read without it. + // A single-object header that names this schema's fingerprint is certain, so it goes first. A Confluent + // prefix proves nothing (a plain datum can start with zero), so it is only tried once the plain read fails. static func decode(_ data: Data, schema: AvroSchema, maxValues: Int = .max) throws -> AvroMessage { + let singleObject = singleObjectFingerprint(data) + if let singleObject, singleObject == schema.fingerprint { + let value = try datum(data.dropFirst(10), schema: schema, maxValues: maxValues) + return AvroMessage(value: value, framing: .singleObject(fingerprint: singleObject)) + } do { - return AvroMessage(value: try datum(data, schema: schema, maxValues: maxValues), schemaID: nil) + return AvroMessage(value: try datum(data, schema: schema, maxValues: maxValues), framing: nil) } catch let plainError as BinaryValueDecodeError { - guard data.count >= 5, data.first == 0 else { throw plainError } + // Avro's own reader refuses a value written with a schema it does not have. + if let singleObject { throw BinaryValueDecodeError(kind: .otherSchema(fingerprint: singleObject), offset: 2) } + guard data.count >= 5, data.first == 0x00 else { throw plainError } var header = BinaryValueReader(data) _ = try header.readByte() - let schemaID = try header.readUInt32() - guard let value = try? datum(data.dropFirst(5), schema: schema, maxValues: maxValues) else { throw plainError } - return AvroMessage(value: value, schemaID: schemaID) + let framing = AvroFraming.confluent(schemaID: try header.readUInt32()) + let body = data.dropFirst(5) + // Confluent's serializer writes a bytes value as it is, with no length in front. + if case .bytes = schema.resolve(schema.root) { + return AvroMessage(value: .binary(Data(body)), framing: framing) + } + guard let value = try? datum(body, schema: schema, maxValues: maxValues) else { throw plainError } + return AvroMessage(value: value, framing: framing) } } + private static func singleObjectFingerprint(_ data: Data) -> UInt64? { + guard data.count >= 10, data.starts(with: [0xC3, 0x01]) else { return nil } + var header = BinaryValueReader(data) + try? header.skip(2) + return try? header.readUInt64LittleEndian() + } + private static func datum(_ data: Data, schema: AvroSchema, maxValues: Int) throws -> MessagePackValue { var parser = Parser(reader: BinaryValueReader(Data(data)), schema: schema, budget: maxValues) let value = try parser.value(schema.root, depth: 0) - if !parser.isTruncated, !parser.reader.isAtEnd { + if !parser.reader.isAtEnd { throw parser.reader.error(.trailingBytes(parser.reader.remaining)) } return value } + // Past the budget the parser keeps reading and stops building, so the cell judges the frame and the + // trailing bytes on the whole value, the same as the viewer does. private struct Parser { var reader: BinaryValueReader let schema: AvroSchema - var budget: Int - var isTruncated = false + private var budget: Int + private var isTruncated = false private var zeroByteItems: Int64 = 0 init(reader: BinaryValueReader, schema: AvroSchema, budget: Int) { @@ -55,66 +82,114 @@ internal enum AvroDecoder { self.budget = budget } + // Only a dispatch: every kind of type is its own function, so the frames a deep value stacks up stay small. mutating func value(_ type: AvroType, depth: Int) throws -> MessagePackValue { guard depth <= AvroDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } - guard budget > 0 else { - isTruncated = true - return .truncated + if !isTruncated { + budget -= 1 + isTruncated = budget < 0 } - budget -= 1 - switch schema.resolve(type) { - case .null: - return .null + let builds = !isTruncated + let resolved = schema.resolve(type) + switch resolved { + case .record(_, let fields): + return try record(fields, depth: depth, builds: builds) + case .array(let items): + return try array(items, depth: depth, builds: builds) + case .map(let valueType): + return try map(valueType, depth: depth, builds: builds) + case .union(let branches): + // The branch is the value itself, so it takes no share of the budget of its own. + budget += builds ? 1 : 0 + return try value(branches[try index(below: branches.count)], depth: depth) + case .logical(let logical, let base): + budget += builds ? 1 : 0 + return try annotated(logical, base, depth: depth, builds: builds) + case .named: + throw reader.error(.invalidField) + default: + return try primitive(resolved, builds: builds) + } + } + + private mutating func primitive(_ type: AvroType, builds: Bool) throws -> MessagePackValue { + switch type { case .boolean: let start = reader.offset let byte = try reader.readByte() guard byte <= 1 else { throw BinaryValueDecodeError(kind: .invalidByte(byte), offset: start) } - return .bool(byte == 1) + return builds ? .bool(byte == 1) : .truncated case .int: - return .int(try int()) + let number = try int() + return builds ? .int(number) : .truncated case .long: - return .int(try long()) + let number = try long() + return builds ? .int(number) : .truncated case .float: - return .float(Float(bitPattern: try reader.readUInt32LittleEndian())) + let bits = try reader.readUInt32LittleEndian() + return builds ? .float(Float(bitPattern: bits)) : .truncated case .double: - return .double(Double(bitPattern: try reader.readUInt64LittleEndian())) - case .bytes: - return .binary(try bytes()) - case .string: - return try string() - case .record(_, let fields): - var pairs: [MessagePackPair] = [] - pairs.reserveCapacity(fields.count) - for field in fields { - let item = try value(field.type, depth: depth + 1) - pairs.append(MessagePackPair(key: .string(field.name), value: item)) - if isTruncated { break } + let bits = try reader.readUInt64LittleEndian() + return builds ? .double(Double(bitPattern: bits)) : .truncated + case .bytes, .string, .fixed: + let count: Int + if case .fixed(_, let size) = type { count = size } else { count = try byteCount() } + guard builds else { + try reader.skip(count) + return .truncated } - return .map(pairs) + let data = try reader.readBytes(count) + guard case .string = type else { return .binary(data) } + // Writers that ignore the encoding put any bytes in a string, so those show as bytes. + return String(data: data, encoding: .utf8).map(MessagePackValue.string) ?? .binary(data) case .enumeration(_, let symbols): - return .string(symbols[try index(below: symbols.count)]) - case .array(let items): - var values: [MessagePackValue] = [] - try blocks(takesNoBytes: schema.takesNoBytes(items)) { parser in - values.append(try parser.value(items, depth: depth + 1)) - } - return .array(values) - case .map(let valueType): - var pairs: [MessagePackPair] = [] - try blocks(takesNoBytes: false) { parser in - let key = try parser.string() - pairs.append(MessagePackPair(key: key, value: try parser.value(valueType, depth: depth + 1))) - } - return .map(pairs) - case .union(let branches): - return try value(branches[try index(below: branches.count)], depth: depth) - case .fixed(_, let size): - return .binary(try reader.readBytes(size)) - case .named: - throw reader.error(.invalidField) - case .logical(let logical, let base): - return AvroLogicalRendering.render(try value(base, depth: depth), as: logical) + let symbol = symbols[try index(below: symbols.count)] + return builds ? .string(symbol) : .truncated + default: + return builds ? .null : .truncated + } + } + + private mutating func record(_ fields: [AvroField], depth: Int, builds: Bool) throws -> MessagePackValue { + var pairs: [MessagePackPair] = [] + pairs.reserveCapacity(builds ? fields.count : 0) + for field in fields { + let wasTruncated = isTruncated + let item = try value(field.type, depth: depth + 1) + if !wasTruncated { pairs.append(MessagePackPair(key: .string(field.name), value: item)) } + } + return builds ? .map(pairs) : .truncated + } + + private mutating func array(_ items: AvroType, depth: Int, builds: Bool) throws -> MessagePackValue { + var values: [MessagePackValue] = [] + try blocks(takesNoBytes: schema.takesNoBytes(items)) { parser in + let wasTruncated = parser.isTruncated + let item = try parser.value(items, depth: depth + 1) + if !wasTruncated { values.append(item) } } + return builds ? .array(values) : .truncated + } + + private mutating func map(_ valueType: AvroType, depth: Int, builds: Bool) throws -> MessagePackValue { + var pairs: [MessagePackPair] = [] + try blocks(takesNoBytes: false) { parser in + let wasTruncated = parser.isTruncated + let key = try parser.value(.string, depth: depth + 1) + let item = try parser.value(valueType, depth: depth + 1) + if !wasTruncated { pairs.append(MessagePackPair(key: key, value: item)) } + } + return builds ? .map(pairs) : .truncated + } + + private mutating func annotated(_ logical: AvroLogicalType, _ base: AvroType, depth: Int, builds: Bool) throws -> MessagePackValue { + let raw = try value(base, depth: depth) + guard builds, raw != .truncated else { return raw } + if case .decimal = logical, case .binary(let bytes) = raw { + // Long division costs the square of the width, so a wide decimal draws on the budget by its width. + budget -= AvroDecimal.significantBytes([UInt8](bytes)).count + } + return AvroLogicalRendering.render(raw, as: logical) } // Each block is a count, then that many items; a negative count is followed by the block's byte size. @@ -128,17 +203,17 @@ internal enum AvroDecoder { count = -count _ = try long() } - // A count past the bytes left is forged, unless the items take no bytes; then the count alone bounds the loop. - if count > Int64(reader.remaining) { - guard takesNoBytes else { throw BinaryValueDecodeError(kind: .unexpectedEnd, offset: reader.end) } - zeroByteItems += count - guard zeroByteItems <= AvroDecoder.maxZeroByteItems else { + // Every item takes a byte at least, unless its type takes none; then only the cap bounds the loop. + if takesNoBytes { + guard count <= AvroDecoder.maxZeroByteItems - zeroByteItems else { throw BinaryValueDecodeError(kind: .tooLarge, offset: start) } + zeroByteItems += count + } else if count > Int64(reader.remaining) { + throw BinaryValueDecodeError(kind: .unexpectedEnd, offset: reader.end) } for _ in 0.. Data { + private mutating func byteCount() throws -> Int { let start = reader.offset let count = try long() - guard count >= 0, count <= Int64(reader.remaining) else { - throw BinaryValueDecodeError(kind: count < 0 ? .invalidLength : .unexpectedEnd, offset: start) - } - return try reader.readBytes(Int(count)) - } - - // Writers that ignore the encoding put any bytes in a string, so those show as bytes. - mutating func string() throws -> MessagePackValue { - let data = try bytes() - guard let text = String(data: data, encoding: .utf8) else { return .binary(data) } - return .string(text) + guard count >= 0 else { throw BinaryValueDecodeError(kind: .invalidLength, offset: start) } + guard count <= Int64(reader.remaining) else { throw reader.error(.unexpectedEnd) } + return Int(count) } private mutating func int() throws -> Int64 { @@ -187,8 +254,10 @@ internal enum AvroDecoder { internal enum AvroLogicalRendering { static func render(_ value: MessagePackValue, as logical: AvroLogicalType) -> MessagePackValue { switch (logical, value) { - case (.decimal(let scale), .binary(let data)) where data.count <= AvroDecoder.maxDecimalBytes: - return .string(decimalText(unscaled: [UInt8](data), scale: scale)) + case (.decimal(let precision, let scale), .binary(let data)): + let unscaled = AvroDecimal.significantBytes([UInt8](data)) + guard unscaled.count <= AvroDecimal.maxBytes(precision: precision) else { return value } + return .string(AvroDecimal.text(unscaled: unscaled, scale: scale)) case (.uuid, .string): return value case (.uuid, .binary(let data)) where data.count == 16: @@ -203,10 +272,16 @@ internal enum AvroLogicalRendering { return timestamp(units: millis, perSecond: 1_000) case (.timestampMicros, .int(let micros)): return timestamp(units: micros, perSecond: 1_000_000) + case (.timestampNanos, .int(let nanos)): + return timestamp(units: nanos, perSecond: 1_000_000_000) case (.localTimestampMillis, .int(let millis)): - return .string(localText(timestamp(units: millis, perSecond: 1_000))) + return localText(timestamp(units: millis, perSecond: 1_000)).map(MessagePackValue.string) ?? value case (.localTimestampMicros, .int(let micros)): - return .string(localText(timestamp(units: micros, perSecond: 1_000_000))) + return localText(timestamp(units: micros, perSecond: 1_000_000)).map(MessagePackValue.string) ?? value + case (.localTimestampNanos, .int(let nanos)): + return localText(timestamp(units: nanos, perSecond: 1_000_000_000)).map(MessagePackValue.string) ?? value + case (.duration, .binary(let data)) where data.count == 12: + return .string(durationText([UInt8](data))) default: return value } @@ -217,9 +292,11 @@ internal enum AvroLogicalRendering { return .timestamp(seconds: seconds, nanoseconds: UInt32(remainder * (1_000_000_000 / perSecond))) } - private static func localText(_ timestamp: MessagePackValue) -> String { - guard case .timestamp(let seconds, let nanoseconds) = timestamp else { return "" } - return String(RFC3339Timestamp.format(seconds: seconds, nanoseconds: nanoseconds).dropLast()) + // A local timestamp names no zone, so it reads as the wall-clock text without the trailing Z. + private static func localText(_ timestamp: MessagePackValue) -> String? { + guard case .timestamp(let seconds, let nanoseconds) = timestamp else { return nil } + let text = RFC3339Timestamp.format(seconds: seconds, nanoseconds: nanoseconds) + return text.hasSuffix("Z") ? String(text.dropLast()) : nil } private static func dateText(days: Int64) -> String { @@ -241,17 +318,63 @@ internal enum AvroLogicalRendering { return text } + // Three little-endian unsigned ints: months, days, milliseconds, written as an ISO 8601 duration. + private static func durationText(_ bytes: [UInt8]) -> String { + func part(_ offset: Int) -> UInt64 { + (0..<4).reduce(0) { $0 | UInt64(bytes[offset + $1]) << (8 * UInt64($1)) } + } + let months = part(0), days = part(4), millis = part(8) + var text = "P" + if months > 0 { text += "\(months)M" } + if days > 0 { text += "\(days)D" } + if millis > 0 { + text += "T\(millis / 1_000)" + if millis % 1_000 > 0 { text += String(format: ".%03llu", millis % 1_000) } + text += "S" + } + return text == "P" ? "PT0S" : text + } + private static func floorDivide(_ value: Int64, _ divisor: Int64) -> (Int64, Int64) { let quotient = value / divisor let remainder = value % divisor return remainder < 0 ? (quotient - 1, remainder + divisor) : (quotient, remainder) } - // Big-endian two's complement of any width, converted to base 10 by long division. - static func decimalText(unscaled bytes: [UInt8], scale: Int) -> String { - guard !bytes.isEmpty else { return "0" } - let isNegative = bytes[0] & 0x80 != 0 - var magnitude = bytes + private static func uuidTuple(_ bytes: [UInt8]) -> uuid_t { + (bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]) + } +} + +// Big-endian two's complement of any width, as Avro's decimal stores the unscaled value. +internal enum AvroDecimal { + // The most digits a two's complement of this many bytes holds: floor(log10(2^(8n-1) - 1)). + static func maxPrecision(bytes: Int) -> Int { + guard bytes > 0 else { return 0 } + return Int((Double(8 * min(bytes, 100_000) - 1) * log10(2.0)).rounded(.down)) + } + + // The fewest bytes that hold any value of this many digits, sign bit included. + static func maxBytes(precision: Int) -> Int { + Int(((Double(precision) * log2(10.0) + 1) / 8).rounded(.up)) + } + + // Leading bytes that only repeat the sign; a fixed pads with them. + static func significantBytes(_ bytes: [UInt8]) -> ArraySlice { + var start = bytes.startIndex + while bytes.endIndex - start > 1 { + let lead = bytes[start], next = bytes[start + 1] + guard (lead == 0x00 && next < 0x80) || (lead == 0xFF && next >= 0x80) else { break } + start += 1 + } + return bytes[start...] + } + + static func text(unscaled bytes: ArraySlice, scale: Int) -> String { + guard let first = bytes.first else { return "0" } + let isNegative = first & 0x80 != 0 + var magnitude = Array(bytes) if isNegative { var carry = true for index in magnitude.indices.reversed() { @@ -263,18 +386,24 @@ internal enum AvroLogicalRendering { } } } - var digits: [UInt8] = [] - while magnitude.contains(where: { $0 != 0 }) { - var remainder = 0 - for index in magnitude.indices { - let current = remainder << 8 | Int(magnitude[index]) - magnitude[index] = UInt8(current / 10) - remainder = current % 10 + var words = Self.words(magnitude) + // Base 10^9 groups, least significant first: one pass of division yields nine digits. + var groups: [UInt32] = [] + while let lead = words.firstIndex(where: { $0 != 0 }) { + words.removeFirst(lead) + var remainder: UInt64 = 0 + for index in words.indices { + let current = remainder << 32 | UInt64(words[index]) + words[index] = UInt32(current / 1_000_000_000) + remainder = current % 1_000_000_000 } - digits.append(UInt8(remainder)) + groups.append(UInt32(remainder)) + } + var text = groups.last.map { String($0) } ?? "0" + for group in groups.dropLast().reversed() { + let digits = String(group) + text += String(repeating: "0", count: 9 - digits.count) + digits } - var text = String(digits.reversed().map { Character(String($0)) }) - if text.isEmpty { text = "0" } if scale > 0 { if text.count <= scale { text = String(repeating: "0", count: scale - text.count + 1) + text @@ -284,8 +413,10 @@ internal enum AvroLogicalRendering { return isNegative ? "-" + text : text } - private static func uuidTuple(_ bytes: [UInt8]) -> uuid_t { - (bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], - bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]) + private static func words(_ bytes: [UInt8]) -> [UInt32] { + let padded = [UInt8](repeating: 0, count: (4 - bytes.count % 4) % 4) + bytes + return stride(from: 0, to: padded.count, by: 4).map { start in + padded[start..<(start + 4)].reduce(0) { $0 << 8 | UInt32($1) } + } } } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift index 9178e646e0..409343006b 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift @@ -6,15 +6,18 @@ import Foundation internal enum AvroLogicalType: Equatable, Sendable { - case decimal(scale: Int) + case decimal(precision: Int, scale: Int) case uuid case date case timeMillis case timeMicros case timestampMillis case timestampMicros + case timestampNanos case localTimestampMillis case localTimestampMicros + case localTimestampNanos + case duration } internal struct AvroField: Equatable, Sendable { @@ -50,6 +53,8 @@ internal struct AvroSchemaError: Error, Equatable, Sendable { internal struct AvroSchema: Equatable, Sendable { let root: AvroType let namedTypes: [String: AvroType] + // CRC-64-AVRO of the Parsing Canonical Form, what a single-object encoded value names its writer schema by. + let fingerprint: UInt64 static func parse(_ text: String) throws -> AvroSchema { let json: Any @@ -60,12 +65,75 @@ internal struct AvroSchema: Equatable, Sendable { } var parser = Parser() let root = try parser.type(json, namespace: nil) - return AvroSchema(root: root, namedTypes: parser.namedTypes) + return AvroSchema(root: root, namedTypes: parser.namedTypes, fingerprint: fingerprint(of: canonicalForm(root))) + } + + // https://avro.apache.org/docs/1.12.0/specification/#parsing-canonical-form-for-schemas + // Full names, only the attributes that shape the bytes, in the specified order, no whitespace. A named + // type is written out where the schema defines it and by name everywhere else, as the parsed tree holds it. + static func canonicalForm(_ type: AvroType) -> String { + switch type { + case .null: return #""null""# + case .boolean: return #""boolean""# + case .int: return #""int""# + case .long: return #""long""# + case .float: return #""float""# + case .double: return #""double""# + case .bytes: return #""bytes""# + case .string: return #""string""# + case .named(let name): return quoted(name) + case .logical(_, let base): return canonicalForm(base) + case .union(let branches): return "[" + branches.map(canonicalForm).joined(separator: ",") + "]" + case .array(let items): return #"{"type":"array","items":"# + canonicalForm(items) + "}" + case .map(let values): return #"{"type":"map","values":"# + canonicalForm(values) + "}" + case .fixed(let name, let size): return #"{"name":"# + quoted(name) + #","type":"fixed","size":"# + "\(size)}" + case .enumeration(let name, let symbols): + return #"{"name":"# + quoted(name) + #","type":"enum","symbols":["# + symbols.map(quoted).joined(separator: ",") + "]}" + case .record(let name, let fields): + let members = fields.map { #"{"name":"# + quoted($0.name) + #","type":"# + canonicalForm($0.type) + "}" } + return #"{"name":"# + quoted(name) + #","type":"record","fields":["# + members.joined(separator: ",") + "]}" + } + } + + static func fingerprint(of canonicalForm: String) -> UInt64 { + canonicalForm.utf8.reduce(emptyFingerprint) { ($0 >> 8) ^ fingerprintTable[Int(($0 ^ UInt64($1)) & 0xFF)] } + } + + private static let emptyFingerprint: UInt64 = 0xC15D_213A_A4D7_A795 + + private static let fingerprintTable: [UInt64] = (0..<256).map { index in + var value = UInt64(index) + for _ in 0..<8 { + value = (value >> 1) ^ (emptyFingerprint & (0 &- (value & 1))) + } + return value + } + + private static func quoted(_ text: String) -> String { + "\"" + text.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "\"", with: "\\\"") + "\"" } // Far past any engine's decimal; a larger one would only let a schema file pad a cell with zeros. static let maxDecimalPrecision = 1_000 + // The root type's own name, without its namespace, for menus. + var displayName: String? { + Self.name(of: root) + } + + private static func name(of type: AvroType) -> String? { + switch type { + case .record(let name, _), .enumeration(let name, _), .fixed(let name, _), .named(let name): + return name.split(separator: ".").last.map(String.init) + case .logical(_, let base): + return Self.name(of: base) + case .union(let branches): + return branches.lazy.compactMap { Self.name(of: $0) }.first + case .array, .map, .null, .boolean, .int, .long, .float, .double, .bytes, .string: + return nil + } + } + func resolve(_ type: AvroType) -> AvroType { if case .named(let name) = type, let defined = namedTypes[name] { return defined } return type @@ -100,7 +168,10 @@ internal struct AvroSchema: Equatable, Sendable { let base = try complex(object, namespace: namespace) guard let logicalName = object["logicalType"] as? String, let logical = Self.logical(logicalName, object: object, base: base) else { return base } - return .logical(logical, base) + let annotated = AvroType.logical(logical, base) + // The logical type belongs to the named type, so a later reference by name keeps it. + if case .fixed(let name, _) = base { namedTypes[name] = annotated } + return annotated } private mutating func complex(_ object: [String: Any], namespace: String?) throws -> AvroType { @@ -110,7 +181,7 @@ internal struct AvroSchema: Equatable, Sendable { guard let kind = typeName as? String else { return try type(typeName, namespace: namespace) } switch kind { case "record", "error": - let name = try fullName(object, namespace: namespace) + let name = try newName(object, namespace: namespace) namedTypes[name] = .record(name: name, fields: []) let inner = Self.namespace(of: name) guard let rawFields = object["fields"] as? [[String: Any]] else { @@ -126,7 +197,7 @@ internal struct AvroSchema: Equatable, Sendable { namedTypes[name] = record return record case "enum": - let name = try fullName(object, namespace: namespace) + let name = try newName(object, namespace: namespace) guard let symbols = object["symbols"] as? [String] else { throw AvroSchemaError(reason: String(format: String(localized: "Enum %@ has no symbols."), name)) } @@ -134,8 +205,8 @@ internal struct AvroSchema: Equatable, Sendable { namedTypes[name] = enumeration return enumeration case "fixed": - let name = try fullName(object, namespace: namespace) - guard let size = object["size"] as? Int, size >= 0 else { + let name = try newName(object, namespace: namespace) + guard let size = Self.integer(object["size"]), size >= 0 else { throw AvroSchemaError(reason: String(format: String(localized: "Fixed type %@ has no size."), name)) } let fixed = AvroType.fixed(name: name, size: size) @@ -177,14 +248,18 @@ internal struct AvroSchema: Equatable, Sendable { return .named(name) } - private func fullName(_ object: [String: Any], namespace: String?) throws -> String { + private func newName(_ object: [String: Any], namespace: String?) throws -> String { guard let name = object["name"] as? String, !name.isEmpty else { throw AvroSchemaError(reason: String(localized: "A record, enum or fixed type has no name.")) } - if name.contains(".") { return name } - let space = object["namespace"] as? String ?? namespace - guard let space, !space.isEmpty else { return name } - return "\(space).\(name)" + var fullName = name + if !name.contains("."), let space = object["namespace"] as? String ?? namespace, !space.isEmpty { + fullName = "\(space).\(name)" + } + guard namedTypes[fullName] == nil else { + throw AvroSchemaError(reason: String(format: String(localized: "The schema defines type %@ twice."), fullName)) + } + return fullName } private static func namespace(of fullName: String) -> String? { @@ -195,20 +270,36 @@ internal struct AvroSchema: Equatable, Sendable { // An unknown or misapplied logical type falls back to its underlying type, as the specification says. private static func logical(_ name: String, object: [String: Any], base: AvroType) -> AvroLogicalType? { switch (name, base) { - case ("decimal", .bytes), ("decimal", .fixed): - guard let precision = object["precision"] as? Int, (1...AvroSchema.maxDecimalPrecision).contains(precision) else { return nil } - let scale = object["scale"] as? Int ?? 0 - return (0...precision).contains(scale) ? .decimal(scale: scale) : nil + case ("decimal", .bytes): + return decimal(object, maxPrecision: AvroSchema.maxDecimalPrecision) + case ("decimal", .fixed(_, let size)): + return decimal(object, maxPrecision: min(AvroSchema.maxDecimalPrecision, AvroDecimal.maxPrecision(bytes: size))) case ("uuid", .string), ("uuid", .fixed(_, 16)): return .uuid case ("date", .int): return .date case ("time-millis", .int): return .timeMillis case ("time-micros", .long): return .timeMicros case ("timestamp-millis", .long): return .timestampMillis case ("timestamp-micros", .long): return .timestampMicros + case ("timestamp-nanos", .long): return .timestampNanos case ("local-timestamp-millis", .long): return .localTimestampMillis case ("local-timestamp-micros", .long): return .localTimestampMicros + case ("local-timestamp-nanos", .long): return .localTimestampNanos + case ("duration", .fixed(_, 12)): return .duration default: return nil } } + + private static func decimal(_ object: [String: Any], maxPrecision: Int) -> AvroLogicalType? { + guard let precision = integer(object["precision"]), precision >= 1, precision <= maxPrecision else { return nil } + let scale = object["scale"] == nil ? 0 : integer(object["scale"]) ?? -1 + return (0...precision).contains(scale) ? .decimal(precision: precision, scale: scale) : nil + } + + // `as? Int` also takes `true` and `4.0`, which Avro's own parser refuses as a size, precision or scale. + private static func integer(_ value: Any?) -> Int? { + guard let number = value as? NSNumber, CFGetTypeID(number) != CFBooleanGetTypeID(), + !CFNumberIsFloatType(number) else { return nil } + return Int(exactly: number.int64Value) + } } } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift index ba2ee33002..2763e9262f 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift @@ -30,6 +30,8 @@ internal enum BinaryValueTree: Sendable { internal enum BinaryValueDocument: Sendable { case decoded(tree: BinaryValueTree, text: String, note: String?) case failed(BinaryValueDecodeError) + // Nothing to judge the bytes by: the format names a schema this Mac no longer has. + case missingSchema } // Pure: the grid calls it per cell on the main actor, the viewer off it. @@ -56,7 +58,7 @@ internal enum BinaryValueDecoding { case .igbinary: text = PhpSerializeWriter.text(try IgbinaryDecoder.decode(data, maxValues: budget), limit: limit) case .avro(let schemaID): - guard let schema = AvroSchemaLibrary.shared.schema(id: schemaID) else { return nil } + guard let schema = AvroSchemaStore.shared.schema(id: schemaID) else { return nil } let value = try AvroDecoder.decode(data, schema: schema, maxValues: budget).value text = MessagePackTextWriter.json(value, pretty: false, limit: limit) } @@ -114,9 +116,7 @@ internal enum BinaryValueDecoding { note: nil ) case .avro(let schemaID): - guard let schema = AvroSchemaLibrary.shared.schema(id: schemaID) else { - return .failed(BinaryValueDecodeError(kind: .schemaMissing, offset: 0)) - } + guard let schema = AvroSchemaStore.shared.schema(id: schemaID) else { return .missingSchema } let message = try AvroDecoder.decode(data, schema: schema) return .decoded( tree: .json(MessagePackTreeBuilder.build(from: message.value)), @@ -124,7 +124,7 @@ internal enum BinaryValueDecoding { MessagePackTextWriter.json(message.value, pretty: true, limit: writerLimit(forCap: maxViewerTextLength)), at: maxViewerTextLength ), - note: message.schemaID.map(schemaIDNote) + note: message.framing.map(framingNote) ) } } catch let error as BinaryValueDecodeError { @@ -134,8 +134,16 @@ internal enum BinaryValueDecoding { } } - private static func schemaIDNote(_ schemaID: UInt32) -> String { - String(format: String(localized: "Schema Registry framing: schema ID %lld"), Int(schemaID)) + private static func framingNote(_ framing: AvroFraming) -> String { + switch framing { + case .confluent(let schemaID): + return String(format: String(localized: "Schema Registry framing: schema ID %lld"), Int(schemaID)) + case .singleObject(let fingerprint): + return String( + format: String(localized: "Avro single-object encoding: schema fingerprint %@"), + String(format: "%016llX", fingerprint) + ) + } } private static func framingNote(_ framing: ProtobufFraming) -> String { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift index 9a0a87a7f1..edcb95f1c1 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift @@ -19,7 +19,7 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { case unknownReference(Int) case indexOutOfRange(Int) case invalidLength - case schemaMissing + case otherSchema(fingerprint: UInt64) } let kind: Kind @@ -59,8 +59,11 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { return String(format: String(localized: "Byte %lld picks entry %lld, which the schema does not have."), offset, index) case .invalidLength: return String(format: String(localized: "The length at byte %lld is negative."), offset) - case .schemaMissing: - return String(localized: "The Avro schema for this column is missing. Choose the schema file again.") + case .otherSchema(let fingerprint): + return String( + format: String(localized: "The value was written with the schema whose fingerprint is %@, not this column's."), + String(format: "%016llX", fingerprint) + ) } } } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift index fb7749de2d..1c39235faa 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift @@ -6,13 +6,13 @@ import Foundation // https://github.com/igbinary/igbinary/blob/master/src/php7/igbinary.c -// References carry the number PHP's serialize() would give them, so `r:`/`R:` read the same in both viewers. +// Values are numbered the way PHP's serialize() numbers them (php_add_var_hash in ext/standard/var.c), so the +// text reads the same `r:` and `R:` serialize() writes. internal enum IgbinaryDecoder { static let maxDepth = PhpSerializeParser.depthCap - // Arrays repeated by value share storage, so a few bytes can stand for billions of values; past this it is refused. - static let maxSlots = 10_000_000 - // A two-byte string ID can repeat a long string; the tree and Find would each walk every copy. - static let maxRepeatedStringBytes = 20_000_000 + // A repeated array or string ID costs a few bytes and expands in the tree, its text and Find. + static let maxExpandedValues = 10_000_000 + static let maxExpandedStringBytes = 20_000_000 static func decode(_ data: Data, maxValues: Int = .max) throws -> PhpValue { var reader = BinaryValueReader(data) @@ -21,7 +21,7 @@ internal enum IgbinaryDecoder { throw BinaryValueDecodeError(kind: .unknownVersion(version), offset: 0) } var parser = Parser(reader: reader, budget: maxValues) - let value = try parser.value(depth: 0) + let value = try parser.value(depth: 0).value if !parser.isTruncated, !parser.reader.isAtEnd { throw parser.reader.error(.trailingBytes(parser.reader.remaining)) } @@ -48,10 +48,18 @@ internal enum IgbinaryDecoder { case enumCase } - // What serialize() writes where igbinary refers back by value: arrays in full, objects as `r:`. + // What serialize() writes when this value turns up again inside an array igbinary repeats by value: + // a PHP reference as `R:`, an object as `r:`, anything else written out again. private struct Copy { let value: PhpValue let slots: Int + let stringBytes: Int + // Levels of nesting below the copy, so one placed deeper than the original still fits the depth cap. + let height: Int + + static func of(_ value: PhpValue, slots: Int, stringBytes: Int = 0) -> Copy { + Copy(value: value, slots: slots, stringBytes: stringBytes, height: 0) + } } private struct Decoded { @@ -59,174 +67,250 @@ internal enum IgbinaryDecoder { let copy: Copy } + private struct Entry { + let key: PhpValue + let keyBytes: Int + let item: Decoded + } + + private struct Reference { + let slot: Int + // Nil while an array is still being read, so a repeat of it from inside is refused. + var copy: Copy? + } + private struct Parser { var reader: BinaryValueReader var budget: Int var isTruncated = false - private var strings: [String] = [] - // Indexed like igbinary's reference table: each array, object and referenced scalar, as serialize() numbers it. - private var references: [(slot: Int, copy: Copy?)] = [] + private var strings: [PhpValue] = [] + // igbinary's table: every array, every object, and every PHP reference to anything else. + private var references: [Reference] = [] + // The number serialize() gives the value being read. private var slot = 0 - private var repeatedStringBytes = 0 + private var stringBytes = 0 init(reader: BinaryValueReader, budget: Int) { self.reader = reader self.budget = budget } - mutating func value(depth: Int) throws -> PhpValue { - try decoded(depth: depth).value - } - - private mutating func decoded(depth: Int) throws -> Decoded { + mutating func value(depth: Int) throws -> Decoded { guard depth <= IgbinaryDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } guard budget > 0 else { isTruncated = true - return Decoded(value: .tooLarge, copy: Copy(value: .tooLarge, slots: 1)) + return Decoded(value: .tooLarge, copy: .of(.tooLarge, slots: 1)) } budget -= 1 - slot += 1 - return try decoded(tag: try readTag(), depth: depth, isReference: false) + try expand(slots: 1, stringBytes: 0) + return try value(tag: try readTag(), depth: depth, isReference: false) } - private mutating func decoded(tag: Tag, depth: Int, isReference: Bool) throws -> Decoded { - let ownSlot = slot + // Only a dispatch: every case is its own function, so the frames a deep value stacks up stay small. + private mutating func value(tag: Tag, depth: Int, isReference: Bool) throws -> Decoded { switch tag { case .reference: - guard !isReference else { throw invalid(tag, at: reader.offset - 1) } - let result = try decoded(tag: try readTag(), depth: depth, isReference: true) - switch result.value { - case .null, .bool, .int, .float, .string: references.append((ownSlot, result.copy)) - default: break - } - return result - case .ref8, .ref16, .ref32: - let target = try reference(width: tag) - if isReference { - // A PHP reference to a value already written takes no number of its own, as `R:` in serialize(). - slot -= 1 - let value = PhpValue.reference(id: target.slot) - return Decoded(value: value, copy: Copy(value: value, slots: 0)) - } - // Not wrapped as a reference: the same array again by value, which serialize() writes out in full. - guard let copy = target.copy else { - throw BinaryValueDecodeError(kind: .unknownReference(target.id), offset: target.offset) - } - slot += copy.slots - 1 - guard slot <= IgbinaryDecoder.maxSlots else { throw reader.error(.tooLarge) } - return Decoded(value: copy.value, copy: copy) - case .objref8, .objref16, .objref32: - let value = PhpValue.reference(id: try reference(width: tag).slot) - return Decoded(value: value, copy: Copy(value: value, slots: 1)) - case .null: - return Self.scalar(.null) - case .boolFalse: - return Self.scalar(.bool(false)) - case .boolTrue: - return Self.scalar(.bool(true)) - case .long8p, .long8n, .long16p, .long16n, .long32p, .long32n, .long64p, .long64n: - return Self.scalar(.int(try integer(tag))) - case .double: - return Self.scalar(.float(Double(bitPattern: try reader.readUInt64()))) - case .stringEmpty: - return Self.scalar(.string("")) - case .stringID8, .stringID16, .stringID32: - return Self.scalar(.string(try storedString(width: tag))) - case .string8, .string16, .string32, .string64: - return Self.scalar(.string(try newString(tag))) + guard !isReference else { throw invalid(tag) } + return try phpReference(depth: depth) + case .ref8, .ref16, .ref32, .objref8, .objref16, .objref32: + return try repeated(tag, depth: depth, isReference: isReference) case .array8, .array16, .array32: - let count = try length(tag) - let index = references.count - references.append((ownSlot, nil)) - let children = try entries(count: count, depth: depth) - let copy = Copy( - value: .array(children.map { PhpKeyValue(key: $0.key, value: $0.copy.value) }), - slots: children.reduce(1) { $0 + $1.copy.slots } - ) - references[index].copy = copy - return Decoded(value: .array(children.map { PhpKeyValue(key: $0.key, value: $0.value) }), copy: copy) - case .object8, .object16, .object32: - return try object(className: try newString(tag), depth: depth, slot: ownSlot) - case .objectID8, .objectID16, .objectID32: - return try object(className: try storedString(width: tag), depth: depth, slot: ownSlot) + return try array(tag, depth: depth) + case .object8, .object16, .object32, .objectID8, .objectID16, .objectID32: + return try object(tag, depth: depth) case .objectSer8, .objectSer16, .objectSer32, .enumCase: - throw invalid(tag, at: reader.offset - 1) + throw invalid(tag) + default: + return try scalar(tag) + } + } + + private mutating func scalar(_ tag: Tag) throws -> Decoded { + let value: PhpValue + var bytes = 0 + switch tag { + case .null: value = .null + case .boolFalse: value = .bool(false) + case .boolTrue: value = .bool(true) + case .double: value = .float(Double(bitPattern: try reader.readUInt64())) + case .stringEmpty, .stringID8, .stringID16, .stringID32, .string8, .string16, .string32, .string64: + let text = try string(tag) + value = text.value + bytes = text.bytes + default: value = .int(try integer(tag)) + } + return Decoded(value: value, copy: .of(value, slots: 1, stringBytes: bytes)) + } + + private mutating func phpReference(depth: Int) throws -> Decoded { + let own = slot + let target = try value(tag: try readTag(), depth: depth, isReference: true) + if case .reference = target.value { return target } + let copy = Copy.of(.reference(id: own), slots: 0) + switch target.value { + case .array, .object, .serializable, .enumCase: break + default: references.append(Reference(slot: own, copy: copy)) + } + return Decoded(value: target.value, copy: copy) + } + + private mutating func repeated(_ tag: Tag, depth: Int, isReference: Bool) throws -> Decoded { + let start = reader.offset + let target = try reference(width: tag) + if isReference { + // A PHP reference to a value already numbered takes no number of its own. + slot -= 1 + let value = PhpValue.reference(id: target.slot) + return Decoded(value: value, copy: .of(value, slots: 0)) + } + if tag == .objref8 || tag == .objref16 || tag == .objref32 { + let value = PhpValue.objectReference(id: target.slot) + return Decoded(value: value, copy: .of(value, slots: 1)) } + // The same array again by value, which serialize() writes out in full. + guard let copy = target.copy else { + throw BinaryValueDecodeError(kind: .unknownReference(target.index), offset: start) + } + guard depth + copy.height <= IgbinaryDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } + try expand(slots: copy.slots - 1, stringBytes: copy.stringBytes) + return Decoded(value: copy.value, copy: copy) + } + + private mutating func array(_ tag: Tag, depth: Int) throws -> Decoded { + let own = slot + let count = try length(tag) + let index = references.count + references.append(Reference(slot: own, copy: nil)) + let entries = try entries(count: count, depth: depth) + let copy = Self.copy(of: entries) + references[index].copy = copy + return Decoded(value: .array(entries.map { PhpKeyValue(key: $0.key, value: $0.item.value) }), copy: copy) } - private static func scalar(_ value: PhpValue) -> Decoded { - Decoded(value: value, copy: Copy(value: value, slots: 1)) + private static func copy(of entries: [Entry]) -> Copy { + Copy( + value: .array(entries.map { PhpKeyValue(key: $0.key, value: $0.item.copy.value) }), + slots: entries.reduce(1) { $0 + $1.item.copy.slots }, + stringBytes: entries.reduce(0) { $0 + $1.keyBytes + $1.item.copy.stringBytes }, + height: entries.map { $0.item.copy.height + 1 }.max() ?? 0 + ) } - private mutating func object(className: String, depth: Int, slot ownSlot: Int) throws -> Decoded { - let objectCopy = Copy(value: .reference(id: ownSlot), slots: 1) - references.append((ownSlot, objectCopy)) + // An object takes its number before its properties are read, so one that holds itself refers back to it. + private mutating func object(_ nameTag: Tag, depth: Int) throws -> Decoded { + let own = slot + let className = Self.text(try string(nameTag).value) + let copy = Copy.of(.objectReference(id: own), slots: 1) + references.append(Reference(slot: own, copy: copy)) let tag = try readTag() let value: PhpValue switch tag { case .array8, .array16, .array32: - let count = try length(tag) - let properties = try entries(count: count, depth: depth).map { entry -> PhpProperty in - let property = decodePhpPropertyKey(Self.keyText(entry.key)) - return PhpProperty(name: property.name, visibility: property.visibility, value: entry.value) + let properties = try entries(count: try length(tag), depth: depth).map { entry -> PhpProperty in + if case .int(let index) = entry.key { + return PhpProperty(name: String(index), visibility: .publicVisibility, value: entry.item.value, index: index) + } + let property = decodePhpPropertyKey(Self.text(entry.key)) + return PhpProperty(name: property.name, visibility: property.visibility, value: entry.item.value) } value = .object(className: className, properties: properties) case .objectSer8, .objectSer16, .objectSer32: let payload = try reader.readBytes(try length(tag)) - value = .serializable(className: className, rawPayload: Self.text(payload)) + value = .serializable(className: className, rawPayload: Self.text(Self.phpString(payload))) case .enumCase: let caseTag = try readTag() - let caseName: String switch caseTag { - case .string8, .string16, .string32, .string64: caseName = try newString(caseTag) - case .stringID8, .stringID16, .stringID32: caseName = try storedString(width: caseTag) - default: throw invalid(caseTag, at: reader.offset - 1) + case .string8, .string16, .string32, .string64, .stringID8, .stringID16, .stringID32: + value = .enumCase(className: className, caseName: Self.text(try string(caseTag).value)) + default: + throw invalid(caseTag) } - value = .enumCase(className: className, caseName: caseName) default: - throw invalid(tag, at: reader.offset - 1) + throw invalid(tag) } - return Decoded(value: value, copy: objectCopy) + return Decoded(value: value, copy: copy) } // A count past the bytes left is forged; checking first keeps it from reserving gigabytes. - private mutating func entries(count: Int, depth: Int) throws -> [(key: PhpValue, value: PhpValue, copy: Copy)] { + private mutating func entries(count: Int, depth: Int) throws -> [Entry] { guard count <= reader.remaining else { throw reader.error(.unexpectedEnd) } - var entries: [(key: PhpValue, value: PhpValue, copy: Copy)] = [] + var entries: [Entry] = [] entries.reserveCapacity(min(count, 1_024)) for _ in 0.. String { - String(bytes: bytes, encoding: .utf8) ?? String(bytes: bytes, encoding: .isoLatin1) ?? "" + private mutating func key() throws -> (value: PhpValue, bytes: Int)? { + let tag = try readTag() + switch tag { + case .null: + return nil + case .long8p, .long8n, .long16p, .long16n, .long32p, .long32n, .long64p, .long64n: + return (.int(try integer(tag)), 0) + case .stringEmpty, .stringID8, .stringID16, .stringID32, .string8, .string16, .string32, .string64: + return try string(tag) + default: + throw invalid(tag) + } + } + + // Every string igbinary writes in full joins the table its string IDs index: keys, values, class names. + private mutating func string(_ tag: Tag) throws -> (value: PhpValue, bytes: Int) { + switch tag { + case .stringEmpty: + return (.string(""), 0) + case .stringID8, .stringID16, .stringID32, .objectID8, .objectID16, .objectID32: + let start = reader.offset + let id = try index(width: tag) + guard id < strings.count else { + throw BinaryValueDecodeError(kind: .unknownReference(id), offset: start) + } + let value = strings[id] + let bytes = Self.byteCount(value) + try expand(slots: 0, stringBytes: bytes) + return (value, bytes) + default: + let data = try reader.readBytes(try length(tag)) + let value = Self.phpString(data) + strings.append(value) + try expand(slots: 0, stringBytes: data.count) + return (value, data.count) + } + } + + private mutating func expand(slots: Int, stringBytes added: Int) throws { + slot += slots + stringBytes += added + guard slot <= IgbinaryDecoder.maxExpandedValues, stringBytes <= IgbinaryDecoder.maxExpandedStringBytes else { + throw reader.error(.tooLarge) + } + } + + private static func phpString(_ bytes: Data) -> PhpValue { + String(data: bytes, encoding: .utf8).map(PhpValue.string) ?? .binaryString(bytes) + } + + private static func byteCount(_ value: PhpValue) -> Int { + switch value { + case .string(let text): return text.utf8.count + case .binaryString(let bytes): return bytes.count + default: return 0 + } } - private static func keyText(_ key: PhpValue) -> String { - if case .int(let number) = key { return String(number) } - if case .string(let text) = key { return text } - return "" + // Names and keys PHP keeps as text; bytes that are not UTF-8 show one character each. + private static func text(_ value: PhpValue) -> String { + switch value { + case .int(let number): return String(number) + case .string(let text): return text + case .binaryString(let bytes): return String(data: bytes, encoding: .isoLatin1) ?? "" + default: return "" + } } private mutating func integer(_ tag: Tag) throws -> Int { @@ -288,33 +372,13 @@ internal enum IgbinaryDecoder { } } - // Every string igbinary writes in full, keys and class names included, joins the table it refers back to. - private mutating func newString(_ tag: Tag) throws -> String { - let text = Self.text(try reader.readBytes(try length(tag))) - strings.append(text) - return text - } - - private mutating func storedString(width tag: Tag) throws -> String { - let start = reader.offset - let id = try index(width: tag) - guard id < strings.count else { - throw BinaryValueDecodeError(kind: .unknownReference(id), offset: start) - } - repeatedStringBytes += strings[id].utf8.count - guard repeatedStringBytes <= IgbinaryDecoder.maxRepeatedStringBytes else { - throw BinaryValueDecodeError(kind: .tooLarge, offset: start) - } - return strings[id] - } - - private mutating func reference(width tag: Tag) throws -> (id: Int, offset: Int, slot: Int, copy: Copy?) { + private mutating func reference(width tag: Tag) throws -> (index: Int, slot: Int, copy: Copy?) { let start = reader.offset - let id = try index(width: tag) - guard id < references.count else { - throw BinaryValueDecodeError(kind: .unknownReference(id), offset: start) + let index = try index(width: tag) + guard index < references.count else { + throw BinaryValueDecodeError(kind: .unknownReference(index), offset: start) } - return (id, start, references[id].slot, references[id].copy) + return (index, references[index].slot, references[index].copy) } private mutating func readTag() throws -> Tag { @@ -326,8 +390,8 @@ internal enum IgbinaryDecoder { return tag } - private func invalid(_ tag: Tag, at offset: Int) -> BinaryValueDecodeError { - BinaryValueDecodeError(kind: .invalidByte(tag.rawValue), offset: offset) + private func invalid(_ tag: Tag) -> BinaryValueDecodeError { + BinaryValueDecodeError(kind: .invalidByte(tag.rawValue), offset: reader.offset - 1) } } } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift index 993bbaa171..214af58e58 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift @@ -26,6 +26,9 @@ internal enum PhpSerializeWriter { output += "d:\(floatText(number));" case .string(let text): output += quoted("s", text) + ";" + case .binaryString(let bytes): + // The length is the byte count serialize() writes; the bytes show one character each. + output += "s:\(bytes.count):\"\(String(data: bytes, encoding: .isoLatin1) ?? "")\";" case .array(let entries): output += "a:\(entries.count):{" for entry in entries { @@ -38,7 +41,11 @@ internal enum PhpSerializeWriter { output += quoted("O", className) + ":\(properties.count):{" for property in properties { guard output.utf8.count < limit else { return } - output += quoted("s", mangled(property)) + ";" + if let index = property.index { + output += "i:\(index);" + } else { + output += quoted("s", mangled(property)) + ";" + } write(property.value, into: &output, limit: limit) } output += "}" @@ -48,6 +55,8 @@ internal enum PhpSerializeWriter { output += quoted("E", "\(className):\(caseName)") + ";" case .reference(let id): output += "R:\(id);" + case .objectReference(let id): + output += "r:\(id);" case .unsupported(let token): output += token case .depthExceeded, .tooLarge: @@ -67,16 +76,48 @@ internal enum PhpSerializeWriter { } } - // serialize_precision -1: the shortest text that reads back, `1.0E+25` past 15 digits. + // php_gcvt with serialize_precision -1: the shortest digits that read back, in E notation only when the + // decimal point falls more than 17 digits right or 4 digits left of the first digit. static func floatText(_ value: Double) -> String { if value.isNaN { return "NAN" } - if value.isInfinite { return value > 0 ? "INF" : "-INF" } - if value == value.rounded(), abs(value) < 1e15 { return String(Int64(value)) } - let text = "\(value)" - guard let exponentIndex = text.firstIndex(of: "e") else { return text } - var mantissa = String(text[.. 17 { + let mantissa = digits.count == 1 ? digits + ".0" : "\(digits.prefix(1)).\(digits.dropFirst())" + let exponent = point - 1 + return sign + mantissa + "E" + (exponent < 0 ? "-" : "+") + String(abs(exponent)) + } + if point <= 0 { + return sign + "0." + String(repeating: "0", count: -point) + digits + } + if digits.count <= point { + return sign + digits + String(repeating: "0", count: point - digits.count) + } + return sign + digits.prefix(point) + "." + digits.dropFirst(point) + } + + // Swift's description is the shortest round-trip text too; this reads the digits and the decimal point's + // position off it, as dtoa mode 0 returns them. + private static func shortestDigits(_ value: Double) -> (digits: String, point: Int) { + var text = "\(value)" + var exponent = 0 + if let marker = text.firstIndex(of: "e") { + exponent = Int(text[text.index(after: marker)...]) ?? 0 + text = String(text[.. 1 ? parts[1] : "")) + var point = parts[0].count + exponent + while digits.first == "0" { + digits = digits.dropFirst() + point -= 1 + } + while digits.last == "0" { + digits = digits.dropLast() + } + return (String(digits), point) } } diff --git a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift index 6cf51a3fe2..ba89bf7a82 100644 --- a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift +++ b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift @@ -11,11 +11,16 @@ internal indirect enum PhpValue: Equatable { case int(Int) case float(Double) case string(String) + // A PHP string is bytes; igbinary hands over ones that are not UTF-8 as they are. + case binaryString(Data) case array([PhpKeyValue]) case object(className: String, properties: [PhpProperty]) case serializable(className: String, rawPayload: String) case enumCase(className: String, caseName: String) + // `R:`, a PHP reference to the value numbered `id`. case reference(id: Int) + // `r:`, the object numbered `id` again. + case objectReference(id: Int) case unsupported(token: String) case depthExceeded case tooLarge @@ -29,6 +34,7 @@ internal indirect enum PhpValue: Equatable { if a.isNaN && b.isNaN { return true } return a == b case (.string(let a), .string(let b)): return a == b + case (.binaryString(let a), .binaryString(let b)): return a == b case (.array(let a), .array(let b)): return a == b case (.object(let aName, let aProps), .object(let bName, let bProps)): return aName == bName && aProps == bProps @@ -37,6 +43,7 @@ internal indirect enum PhpValue: Equatable { case (.enumCase(let aName, let aCase), .enumCase(let bName, let bCase)): return aName == bName && aCase == bCase case (.reference(let a), .reference(let b)): return a == b + case (.objectReference(let a), .objectReference(let b)): return a == b case (.unsupported(let a), .unsupported(let b)): return a == b case (.depthExceeded, .depthExceeded): return true case (.tooLarge, .tooLarge): return true @@ -54,6 +61,8 @@ internal struct PhpProperty: Equatable { let name: String let visibility: PhpVisibility let value: PhpValue + // An `i:` member, which __serialize data such as ArrayObject's holds; `name` spells the same number. + var index: Int? } internal enum PhpVisibility: Equatable { @@ -111,7 +120,7 @@ private struct Cursor { case UInt8(ascii: "E"): return parseEnumCase() case UInt8(ascii: "o"): return .unsupported(token: "o") - case UInt8(ascii: "r"), UInt8(ascii: "R"): return parseReference() + case UInt8(ascii: "r"), UInt8(ascii: "R"): return parseReference(token: token) default: return nil } } @@ -213,10 +222,17 @@ private struct Cursor { var properties: [PhpProperty] = [] properties.reserveCapacity(count) for _ in 0.. PhpValue? { + private mutating func parseReference(token: UInt8) -> PhpValue? { index += 1 guard expect(UInt8(ascii: ":")) else { return nil } guard let raw = readUntil(UInt8(ascii: ";")) else { return nil } guard let value = Int(raw) else { return nil } guard expect(UInt8(ascii: ";")) else { return nil } - return .reference(id: value) + return token == UInt8(ascii: "R") ? .reference(id: value) : .objectReference(id: value) } private mutating func readUntil(_ terminator: UInt8) -> String? { diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift index bfdc5b83b8..9ded656ca1 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift @@ -2,14 +2,10 @@ // ValueDisplayFormat.swift // TablePro // -// Semantic display formats for raw database values. -// Enables auto-detection and per-column overrides for values like -// UUIDs stored in BINARY(16) or Unix timestamps in INT columns. -// import Foundation -enum ValueDisplayFormat: Hashable, Codable, CaseIterable, Identifiable, RawRepresentable { +enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case raw case text case uuid @@ -21,25 +17,36 @@ enum ValueDisplayFormat: Hashable, Codable, CaseIterable, Identifiable, RawRepre case messagePack case protobuf case igbinary - // The schema lives in AvroSchemaLibrary; the format carries its ID so every per-column path can decode. + // The schema is kept by AvroSchemaStore under this ID, so every per-column path can decode with it. case avro(schemaID: String) - private static let avroPrefix = "avro:" - - static let allCases: [ValueDisplayFormat] = [ + // Every format but Avro, which needs a schema chosen first. + static let presetFormats: [ValueDisplayFormat] = [ .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized, - .messagePack, .protobuf, .igbinary + .messagePack, .protobuf, .igbinary, ] + private static let avroPrefix = "avro:" + init?(rawValue: String) { - if rawValue.hasPrefix(Self.avroPrefix) { + switch rawValue { + case "raw": self = .raw + case "text": self = .text + case "uuid": self = .uuid + case "uuidSwappedTime": self = .uuidSwappedTime + case "unixTimestamp": self = .unixTimestamp + case "unixTimestampMillis": self = .unixTimestampMillis + case "json": self = .json + case "phpSerialized": self = .phpSerialized + case "messagePack": self = .messagePack + case "protobuf": self = .protobuf + case "igbinary": self = .igbinary + default: + guard rawValue.hasPrefix(Self.avroPrefix) else { return nil } let schemaID = String(rawValue.dropFirst(Self.avroPrefix.count)) - guard !schemaID.isEmpty else { return nil } + guard AvroSchemaStore.isSchemaID(schemaID) else { return nil } self = .avro(schemaID: schemaID) - return } - guard let format = Self.allCases.first(where: { $0.rawValue == rawValue }) else { return nil } - self = format } var rawValue: String { @@ -87,7 +94,7 @@ enum ValueDisplayFormat: Hashable, Codable, CaseIterable, Identifiable, RawRepre case .phpSerialized: return String(localized: "PHP Serialized") case .messagePack, .protobuf, .igbinary: return binaryValueFormat?.displayName ?? rawValue case .avro(let schemaID): - guard let name = AvroSchemaLibrary.shared.name(id: schemaID) else { return "Avro" } + guard let name = AvroSchemaStore.shared.schema(id: schemaID)?.displayName else { return "Avro" } return String(format: String(localized: "Avro (%@)"), name) } } @@ -103,7 +110,6 @@ enum ValueDisplayFormat: Hashable, Codable, CaseIterable, Identifiable, RawRepre } } - /// Column types this format can apply to. var applicableColumnTypes: Set { switch self { case .raw: @@ -137,8 +143,7 @@ enum ValueDisplayFormat: Hashable, Codable, CaseIterable, Identifiable, RawRepre return applicableColumnTypes.contains(typeKey) } - /// Whether this format turns stored bytes into readable characters. A binary column is worth - /// searching exactly when one of these is on it; hex is not what anyone types into Find. + // A binary column is worth searching exactly when one of these is on it; hex is not what anyone types into Find. var rendersBinaryAsText: Bool { switch self { case .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro: @@ -148,13 +153,15 @@ enum ValueDisplayFormat: Hashable, Codable, CaseIterable, Identifiable, RawRepre } } - /// Returns applicable formats for a given column type. - /// Always includes `.raw` as the first option. Avro is not listed: it needs a schema chosen first. static func applicableFormats( for columnType: ColumnType?, databaseType: DatabaseType? = nil ) -> [ValueDisplayFormat] { - allCases.filter { $0.isApplicable(to: columnType, databaseType: databaseType) } + presetFormats.filter { $0.isApplicable(to: columnType, databaseType: databaseType) } + } + + static func offersAvro(for columnType: ColumnType?, databaseType: DatabaseType? = nil) -> Bool { + avro(schemaID: "").isApplicable(to: columnType, databaseType: databaseType) } } diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatService.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatService.swift index 6417114bf4..4248baf06d 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatService.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatService.swift @@ -8,11 +8,11 @@ import Foundation import os -/// Owns which format a column is shown in, and where a user's choice is persisted. -/// -/// Rendering a value under a format is a separate job that must not reach the preference layer; -/// it lives in `ValueDisplayFormatter`. Keeping the two apart is what lets the data grid's per -/// cell path stay free of storage, and it is why this type no longer exposes `applyFormat`. +// Owns which format a column is shown in, and where a user's choice is persisted. +// +// Rendering a value under a format is a separate job that must not reach the preference layer; +// it lives in `ValueDisplayFormatter`. Keeping the two apart is what lets the data grid's per +// cell path stay free of storage, and it is why this type no longer exposes `applyFormat`. @MainActor final class ValueDisplayFormatService { static let shared = ValueDisplayFormatService() @@ -83,11 +83,7 @@ final class ValueDisplayFormatService { overrides.removeValue(forKey: columnKey) } - if overrides.isEmpty { - storage.clear(for: scope) - } else { - storage.save(overrides, for: scope) - } + storage.save(overrides, for: scope) overridesVersion &+= 1 } diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift index 23780cd542..83e8941087 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift @@ -6,16 +6,16 @@ import Foundation import TableProUtilities -/// Renders one raw cell value under one display format. -/// -/// Nothing here reads a setting or a store, which is the point: this is what the data grid calls -/// per cell, and it has to stay reachable without dragging the preference layer behind it. -/// Resolving *which* format a column carries is a separate job with separate dependencies, and it -/// lives in `ValueDisplayFormatService`. +// Renders one raw cell value under one display format. +// +// Nothing here reads a setting or a store, which is the point: this is what the data grid calls +// per cell, and it has to stay reachable without dragging the preference layer behind it. +// Resolving *which* format a column carries is a separate job with separate dependencies, and it +// lives in `ValueDisplayFormatService`. @MainActor enum ValueDisplayFormatter { - /// nil where the format cannot render this value, so the caller falls back to how the column - /// would have read without it rather than showing a value the format did not actually produce. + // nil where the format cannot render this value, so the caller falls back to how the column + // would have read without it rather than showing a value the format did not actually produce. static func apply( _ rawValue: String, format: ValueDisplayFormat, diff --git a/apple/TablePro/Core/Services/Infrastructure/AvroSchemaFilePanel.swift b/apple/TablePro/Core/Services/Infrastructure/AvroSchemaFilePanel.swift new file mode 100644 index 0000000000..5ef39b8a7a --- /dev/null +++ b/apple/TablePro/Core/Services/Infrastructure/AvroSchemaFilePanel.swift @@ -0,0 +1,71 @@ +// +// AvroSchemaFilePanel.swift +// TablePro +// + +import AppKit + +// Every file is enabled, since schemas come as .avsc, .json or no extension at all. The panel's validate hook +// is the gate: a file that is not a schema shows why and leaves the panel open on it. +@MainActor +internal enum AvroSchemaFilePanel { + static func chooseSchemaText(in window: NSWindow) async -> String? { + let panel = NSOpenPanel() + panel.allowedContentTypes = [] + panel.canChooseFiles = true + panel.canChooseDirectories = false + panel.allowsMultipleSelection = false + panel.prompt = String(localized: "Choose Schema") + panel.message = String(localized: "Choose the Avro schema the values in this column were written with.") + let gate = AvroSchemaFileGate() + panel.delegate = gate + let response = await panel.presentAsSheet(for: window) + return withExtendedLifetime(gate) { response == .OK ? gate.text : nil } + } +} + +@MainActor +private final class AvroSchemaFileGate: NSObject, NSOpenSavePanelDelegate { + private(set) var text: String? + + func panel(_ sender: Any, validate url: URL) throws { + let name = url.lastPathComponent + let size = (try? url.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0 + guard size <= AvroSchemaStore.maxFileBytes else { throw AvroSchemaFileError(fileName: name, reason: .tooLarge) } + guard let text = try? String(contentsOf: url, encoding: .utf8) else { + throw AvroSchemaFileError(fileName: name, reason: .notText) + } + do { + _ = try AvroSchema.parse(text) + } catch let error as AvroSchemaError { + throw AvroSchemaFileError(fileName: name, reason: .invalid(error.reason)) + } + self.text = text + } +} + +private struct AvroSchemaFileError: LocalizedError { + enum Reason { + case tooLarge + case notText + case invalid(String) + } + + let fileName: String + let reason: Reason + + var errorDescription: String? { + String(format: String(localized: "“%@” is not a usable Avro schema."), fileName) + } + + var recoverySuggestion: String? { + switch reason { + case .tooLarge: + return String(localized: "The file is larger than any Avro schema.") + case .notText: + return String(localized: "The file is not UTF-8 text.") + case .invalid(let reason): + return reason + } + } +} diff --git a/apple/TablePro/Core/Storage/AvroSchemaLibrary.swift b/apple/TablePro/Core/Storage/AvroSchemaLibrary.swift deleted file mode 100644 index cf45750665..0000000000 --- a/apple/TablePro/Core/Storage/AvroSchemaLibrary.swift +++ /dev/null @@ -1,61 +0,0 @@ -// -// AvroSchemaLibrary.swift -// TablePro -// - -import CryptoKit -import Foundation - -// A column's Avro format names its schema by a hash of the file's text, so the same file chosen twice is -// stored once. Read from the grid on the main actor and from the viewer off it. -internal final class AvroSchemaLibrary: @unchecked Sendable { - static let shared = AvroSchemaLibrary() - - static let keyPrefix = "com.TablePro.avroSchemas." - - private struct Entry: Codable { - let name: String - let text: String - } - - private let store: KeyValueStore - private let lock = NSLock() - private var parsed: [String: AvroSchema] = [:] - - init(store: KeyValueStore = AppStorageEnvironment.shared.defaults) { - self.store = store - } - - func register(text: String, name: String) throws -> String { - let schema = try AvroSchema.parse(text) - let id = SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() - guard let data = try? JSONEncoder().encode(Entry(name: name, text: text)) else { - throw AvroSchemaError(reason: String(localized: "The schema could not be saved.")) - } - lock.lock() - defer { lock.unlock() } - store.setDataValue(data, forKey: Self.keyPrefix + id) - parsed[id] = schema - return id - } - - func schema(id: String) -> AvroSchema? { - lock.lock() - defer { lock.unlock() } - if let schema = parsed[id] { return schema } - guard let entry = entry(id: id), let schema = try? AvroSchema.parse(entry.text) else { return nil } - parsed[id] = schema - return schema - } - - func name(id: String) -> String? { - lock.lock() - defer { lock.unlock() } - return entry(id: id)?.name - } - - private func entry(id: String) -> Entry? { - guard let data = store.dataValue(forKey: Self.keyPrefix + id) else { return nil } - return try? JSONDecoder().decode(Entry.self, from: data) - } -} diff --git a/apple/TablePro/Core/Storage/AvroSchemaStore.swift b/apple/TablePro/Core/Storage/AvroSchemaStore.swift new file mode 100644 index 0000000000..40a3806092 --- /dev/null +++ b/apple/TablePro/Core/Storage/AvroSchemaStore.swift @@ -0,0 +1,87 @@ +// +// AvroSchemaStore.swift +// TablePro +// + +import CryptoKit +import Foundation +import os + +// The schema files chosen for Avro columns, one file each, named by a hash of the text, so a column still +// decodes after the original file moves and the same file chosen twice is kept once. Read from the grid on +// the main actor and from the viewer off it. +internal final class AvroSchemaStore: Sendable { + static let shared = AvroSchemaStore( + directory: AppStorageEnvironment.shared.supportDirectory.appendingPathComponent("AvroSchemas", isDirectory: true) + ) + + // Far past any real schema; a file this large is the wrong file. + static let maxFileBytes = 4_000_000 + + private static let fileExtension = "avsc" + + private struct State { + var parsed: [String: AvroSchema] = [:] + // Kept from a sweep: a schema can be added before the column's format is saved. + var addedThisLaunch: Set = [] + } + + private let directory: URL + private let state = OSAllocatedUnfairLock(initialState: State()) + + init(directory: URL) { + self.directory = directory + } + + static func isSchemaID(_ text: String) -> Bool { + text.utf8.count == 32 && text.utf8.allSatisfy { (0x30...0x39).contains($0) || (0x61...0x66).contains($0) } + } + + // Parses first, so nothing is kept for a file that is not a schema. + func add(_ text: String) throws -> String { + let schema = try AvroSchema.parse(text) + let id = SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() + let url = fileURL(id) + if !FileManager.default.fileExists(atPath: url.path) { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try Data(text.utf8).write(to: url, options: .atomic) + } + state.withLock { + $0.parsed[id] = schema + $0.addedThisLaunch.insert(id) + } + return id + } + + func schema(id: String) -> AvroSchema? { + if let cached = state.withLock({ $0.parsed[id] }) { return cached } + guard Self.isSchemaID(id), + let data = try? Data(contentsOf: fileURL(id)), + let text = String(data: data, encoding: .utf8), + let schema = try? AvroSchema.parse(text) else { return nil } + state.withLock { $0.parsed[id] = schema } + return schema + } + + var storedIDs: Set { + let names = (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] + return Set(names.compactMap { name in + let url = URL(fileURLWithPath: name) + guard url.pathExtension == Self.fileExtension else { return nil } + let id = url.deletingPathExtension().lastPathComponent + return Self.isSchemaID(id) ? id : nil + }) + } + + func removeAll(except kept: Set) { + let addedThisLaunch = state.withLock { $0.addedThisLaunch } + for id in storedIDs where !kept.contains(id) && !addedThisLaunch.contains(id) { + try? FileManager.default.removeItem(at: fileURL(id)) + state.withLock { _ = $0.parsed.removeValue(forKey: id) } + } + } + + private func fileURL(_ id: String) -> URL { + directory.appendingPathComponent(id, isDirectory: false).appendingPathExtension(Self.fileExtension) + } +} diff --git a/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift b/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift index 287d1b59b2..fbc2b90b7b 100644 --- a/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift +++ b/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift @@ -5,45 +5,67 @@ import Foundation +// 0.83 decodes a table's formats as one dictionary and drops all of them on a name it does not know, so a +// format added since lives in a second key under the same prefix. Each key is read entry by entry, the first +// wins for a column in both (0.83 writes only the first), and a name this build does not know is kept. @MainActor internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { static let shared = ValueDisplayFormatStorage() + private static let namesReadBy083: Set = [ + "raw", "text", "uuid", "uuidSwappedTime", "unixTimestamp", "unixTimestampMillis", "json", "phpSerialized", + "messagePack", "protobuf", + ] + private let store: KeyValueStore + private let schemas: AvroSchemaStore - init(defaults: KeyValueStore = AppStorageEnvironment.shared.defaults) { + init(defaults: KeyValueStore = AppStorageEnvironment.shared.defaults, schemas: AvroSchemaStore = .shared) { store = defaults + self.schemas = schemas } func save(_ formats: [String: ValueDisplayFormat], for scope: TableScope) { - guard !formats.isEmpty else { - clear(for: scope) - return + let key = Self.key(for: scope) + let addedKey = Self.addedKey(for: scope) + var first: [String: String] = [:] + var added = (Self.names(store.dataValue(forKey: addedKey)) ?? [:]) + .filter { ValueDisplayFormat(rawValue: $0.value) == nil && formats[$0.key] == nil } + for (column, format) in formats { + if Self.namesReadBy083.contains(format.rawValue) { + first[column] = format.rawValue + } else { + added[column] = format.rawValue + } } - guard let data = try? JSONEncoder().encode(formats) else { return } - store.setDataValue(data, forKey: PreferenceKeys.columnDisplayFormats(scope).name) + write(first, forKey: key) + write(added, forKey: addedKey) removeLegacy(for: scope) + sweepSchemas() } func load(for scope: TableScope) -> [String: ValueDisplayFormat]? { - if let data = store.dataValue(forKey: PreferenceKeys.columnDisplayFormats(scope).name), - let formats = try? JSONDecoder().decode([String: ValueDisplayFormat].self, from: data) { - return formats - } - return migrateLegacy(for: scope) + let first = Self.names(store.dataValue(forKey: Self.key(for: scope))) + let added = Self.names(store.dataValue(forKey: Self.addedKey(for: scope))) + guard first != nil || added != nil else { return migrateLegacy(for: scope) } + let names = (added ?? [:]).merging(first ?? [:]) { _, firstName in firstName } + return names.compactMapValues(ValueDisplayFormat.init(rawValue:)) } func clear(for scope: TableScope) { - store.setDataValue(nil, forKey: PreferenceKeys.columnDisplayFormats(scope).name) + store.setDataValue(nil, forKey: Self.key(for: scope)) + store.setDataValue(nil, forKey: Self.addedKey(for: scope)) removeLegacy(for: scope) + sweepSchemas() } func renameTable(from oldScope: TableScope, to newScope: TableScope) { - let oldKey = PreferenceKeys.columnDisplayFormats(oldScope).name + let oldKey = Self.key(for: oldScope) if store.dataValue(forKey: oldKey) == nil { migrateLegacy(for: oldScope) } - store.moveValue(fromKey: oldKey, toKey: PreferenceKeys.columnDisplayFormats(newScope).name) + store.moveValue(fromKey: oldKey, toKey: Self.key(for: newScope)) + store.moveValue(fromKey: Self.addedKey(for: oldScope), toKey: Self.addedKey(for: newScope)) } func renameContainer( @@ -65,15 +87,15 @@ internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { clear(for: scope) } - /// The legacy key is deliberately left alone. It is `prefix + connectionId + table` and names no - /// database or schema, so nothing here can tell which container it belonged to, and sweeping the - /// connection's would take the formats of tables in databases that still exist. `dropTable` - /// clears it, because there the table is named. + // The legacy key names no database or schema, so nothing here can tell which container it belonged to, + // and sweeping the connection's would take the formats of tables in databases that still exist. + // `dropTable` clears it, because there the table is named. func dropContainer(connectionId: UUID, database: String, schema: String?) { store.removeValues( withPrefix: Self.keyPrefix + TableScope.storagePrefix(connectionId: connectionId, database: database, schema: schema) ) + sweepSchemas() } func purgeConnections(_ connectionIds: Set, leavesTombstones: Bool) { @@ -81,29 +103,59 @@ internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { store.removeValues(withPrefix: Self.keyPrefix + TableScope.storagePrefix(connectionId: connectionId)) store.removeValues(withPrefix: Self.legacyKeyPrefix(for: connectionId)) } + sweepSchemas() + } + + // A schema file lives as long as a saved format names it. + private func sweepSchemas() { + guard !schemas.storedIDs.isEmpty else { return } + var named: Set = [] + for key in store.keys(withPrefix: Self.keyPrefix) where key.hasSuffix(Self.addedSuffix) { + for name in (Self.names(store.dataValue(forKey: key)) ?? [:]).values { + if case .avro(let schemaID)? = ValueDisplayFormat(rawValue: name) { named.insert(schemaID) } + } + } + schemas.removeAll(except: named) } @discardableResult private func migrateLegacy(for scope: TableScope) -> [String: ValueDisplayFormat]? { let legacyKey = Self.legacyKey(for: scope) - guard let data = store.dataValue(forKey: legacyKey), - let formats = try? JSONDecoder().decode([String: ValueDisplayFormat].self, from: data), - !formats.isEmpty else { - return nil - } - if let encoded = try? JSONEncoder().encode(formats) { - store.setDataValue(encoded, forKey: PreferenceKeys.columnDisplayFormats(scope).name) - } + guard let names = Self.names(store.dataValue(forKey: legacyKey)), !names.isEmpty else { return nil } + write(names, forKey: Self.key(for: scope)) store.setDataValue(nil, forKey: legacyKey) - return formats + return names.compactMapValues(ValueDisplayFormat.init(rawValue:)) + } + + private func write(_ names: [String: String], forKey key: String) { + guard !names.isEmpty, let data = try? JSONEncoder().encode(names) else { + store.setDataValue(nil, forKey: key) + return + } + store.setDataValue(data, forKey: key) } private func removeLegacy(for scope: TableScope) { store.setDataValue(nil, forKey: Self.legacyKey(for: scope)) } + private static func names(_ data: Data?) -> [String: String]? { + data.flatMap { try? JSONDecoder().decode([String: String].self, from: $0) } + } + private static let keyPrefix = PreferenceKeys.columnDisplayFormatsPrefix + // A table key is four dot-joined, percent-encoded parts, so a fifth part never names another table. + private static let addedSuffix = ".added" + + private static func key(for scope: TableScope) -> String { + PreferenceKeys.columnDisplayFormats(scope).name + } + + private static func addedKey(for scope: TableScope) -> String { + key(for: scope) + addedSuffix + } + private static func legacyKey(for scope: TableScope) -> String { legacyKeyPrefix(for: scope.connectionId) + scope.table } diff --git a/apple/TablePro/Models/UI/PhpTreeNode.swift b/apple/TablePro/Models/UI/PhpTreeNode.swift index 87b0e71692..1c11c256ef 100644 --- a/apple/TablePro/Models/UI/PhpTreeNode.swift +++ b/apple/TablePro/Models/UI/PhpTreeNode.swift @@ -94,12 +94,12 @@ internal struct PhpTreeNode: Identifiable { } extension PhpTreeNode: FilterableTreeNode { - /// An integer key is stored in the value, unlike a JSON array index, so it stays searchable. + // An integer key is stored in the value, unlike a JSON array index, so it stays searchable. internal var searchableKey: String? { key } - /// A class name is data. A count and the cut marker are text the tree made up. + // A class name is data. A count and the cut marker are text the tree made up. internal var searchableText: String { switch nodeType { case .array, .truncated: return "" @@ -121,7 +121,7 @@ extension PhpTreeNode: FilterableTreeNode { rawValue ?? displayValue } - /// An array or an object copies its summary: there is no serializer to write its members back out. + // An array or an object copies its summary: there is no serializer to write its members back out. internal var copyableValueIncludesDescendants: Bool { false } @@ -201,6 +201,15 @@ internal enum PhpTreeBuilder { rawValue: stringValue, visibilityBadge: badge ) + case .binaryString(let bytes): + let hex = "0x" + bytes.uppercaseHex() + let display = TreeDisplayText.cut(hex, limit: maxDisplayLength) + return PhpTreeNode( + key: key, keyPath: keyPath, path: path, nodeType: .string, + displayValue: display.text, isDisplayCut: display.isCut, + rawValue: hex, visibilityBadge: badge + ) + case .array(let entries): var children: [PhpTreeNode] = [] children.reserveCapacity(entries.count) @@ -238,9 +247,10 @@ internal enum PhpTreeBuilder { } children.append( makeNode( - key: property.name, - keyPath: "\(keyPath).\(TreeKeyPathSyntax.member(property.name))", - path: path.appending(siblings.key(property.name)), + key: property.index.map { "[\($0)]" } ?? property.name, + keyPath: property.index.map { "\(keyPath)[\($0)]" } + ?? "\(keyPath).\(TreeKeyPathSyntax.member(property.name))", + path: path.appending(property.index.map { siblings.integerKey($0) } ?? siblings.key(property.name)), value: property.value, formats: formats, nodeCount: &nodeCount, @@ -267,7 +277,7 @@ internal enum PhpTreeBuilder { displayValue: "\(className)::\(caseName)", className: className, visibilityBadge: badge ) - case .reference(let identifier): + case .reference(let identifier), .objectReference(let identifier): return PhpTreeNode( key: key, keyPath: keyPath, path: path, nodeType: .reference, displayValue: "→ #\(identifier)", visibilityBadge: badge @@ -307,6 +317,7 @@ internal enum PhpTreeBuilder { switch key { case .int(let intValue): return "[\(intValue)]" case .string(let stringValue): return stringValue + case .binaryString(let bytes): return "0x" + bytes.uppercaseHex() default: return "?" } } @@ -315,6 +326,7 @@ internal enum PhpTreeBuilder { switch key { case .int(let intValue): return "\(parent)[\(intValue)]" case .string(let stringValue): return "\(parent).\(TreeKeyPathSyntax.member(stringValue))" + case .binaryString: return "\(parent)[\(arrayKeyDisplay(key))]" default: return parent } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 9c55e9ec34..117f59b770 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -939,6 +939,9 @@ } } } + }, + "“%@” is not a usable Avro schema." : { + }, "\"%@\" is not a valid %@. Example: %@" : { "extractionState" : "manual", @@ -4011,9 +4014,6 @@ } } } - }, - "%@ is not a usable Avro schema" : { - }, "%@ is not a Vault address. Use a URL such as https://vault.example.com:8200." : { @@ -29720,6 +29720,9 @@ }, "Avro Schema…" : { + }, + "Avro single-object encoding: schema fingerprint %@" : { + }, "AWS answered without the Redshift user and password." : { @@ -183356,9 +183359,6 @@ } } } - }, - "The Avro schema for this column is missing. Choose the schema file again." : { - }, "The AWS credentials for this profile have expired. Sign in again and retry." : { "localizations" : { @@ -186452,9 +186452,15 @@ }, "The file is larger than %@, too large for a rules file." : { + }, + "The file is larger than any Avro schema." : { + }, "The file is not a JSON object." : { + }, + "The file is not UTF-8 text." : { + }, "The file is not valid JSON." : { "extractionState" : "manual", @@ -189981,7 +189987,7 @@ "The schema changes staged in this diagram will be lost." : { }, - "The schema could not be saved." : { + "The schema defines type %@ twice." : { }, "The schema has a type that is not a name, an object or a union." : { @@ -193223,6 +193229,9 @@ } } } + }, + "The value was written with the schema whose fingerprint is %@, not this column's." : { + }, "The values this save replaced are no longer being kept." : { "localizations" : { @@ -197837,6 +197846,9 @@ } } } + }, + "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…" : { + }, "This Mac no longer holds a seat." : { "localizations" : { diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift index d10e5b55fd..77e0b6d010 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift @@ -4,7 +4,6 @@ // import AppKit -import UniformTypeIdentifiers struct DataGridDisplayFormatChoice { let columnID: DataGridColumnID @@ -101,8 +100,7 @@ extension DataGridViewController { item.state = format == current ? .on : .off submenu.addItem(item) } - let databaseType = model.configuration.traits.databaseType - if ValueDisplayFormat.avro(schemaID: "").isApplicable(to: column.type, databaseType: databaseType) { + if ValueDisplayFormat.offersAvro(for: column.type, databaseType: model.configuration.traits.databaseType) { if case .avro = current { let item = menuItem( current.displayName, @@ -239,11 +237,15 @@ extension DataGridViewController { // Only a value filter can renumber rows on a format change, so only then does the owner confirm it. @objc func setDisplayFormatFromMenu(_ sender: Any?) { - guard let choice = (sender as? NSMenuItem)?.representedObject as? DataGridDisplayFormatChoice, - model.columns.column(id: choice.columnID) != nil else { return } + guard let choice = (sender as? NSMenuItem)?.representedObject as? DataGridDisplayFormatChoice else { return } + setDisplayFormat(choice.format, columnID: choice.columnID) + } + + private func setDisplayFormat(_ format: ValueDisplayFormat, columnID: DataGridColumnID) { + guard model.columns.column(id: columnID) != nil else { return } let apply: @MainActor () -> Void = { [weak self] in - guard let self, let column = self.model.columns.column(id: choice.columnID) else { return } - self.model.setDisplayFormat(choice.format, for: column) + guard let self, let column = self.model.columns.column(id: columnID) else { return } + self.model.setDisplayFormat(format, for: column) } guard model.store.valueFilter.isActive, let actions = model.actions else { apply() @@ -252,39 +254,25 @@ extension DataGridViewController { actions.gridConfirmDisplayOrderChange(then: apply) } - // The schema is read once and kept, so the column still decodes after the file moves. + // The schema is copied in, so the column still decodes after the file moves. @objc func chooseAvroSchemaFromMenu(_ sender: Any?) { guard let columnID = menuColumn(sender)?.id, let window = view.window else { return } - let panel = NSOpenPanel() - panel.canChooseFiles = true - panel.canChooseDirectories = false - panel.allowsMultipleSelection = false - panel.allowedContentTypes = [UTType(filenameExtension: "avsc") ?? .json, .json] - panel.prompt = String(localized: "Choose Schema") - panel.message = String(localized: "Choose the Avro schema the values in this column were written with.") - Task { [weak self] in - guard await panel.presentAsSheet(for: window) == .OK, let url = panel.url else { return } - self?.applyAvroSchema(at: url, columnID: columnID, window: window) - } - } - - private func applyAvroSchema(at url: URL, columnID: DataGridColumnID, window: NSWindow) { - let schemaID: String - do { - let text = try String(contentsOf: url, encoding: .utf8) - schemaID = try AvroSchemaLibrary.shared.register(text: text, name: url.lastPathComponent) - } catch { - let reason = (error as? AvroSchemaError)?.reason ?? error.localizedDescription - let alert = NSAlert() - alert.alertStyle = .warning - alert.messageText = String(format: String(localized: "%@ is not a usable Avro schema"), url.lastPathComponent) - alert.informativeText = reason - alert.beginSheetModal(for: window) - return + Task { @MainActor [weak self] in + guard let text = await AvroSchemaFilePanel.chooseSchemaText(in: window) else { return } + let format: ValueDisplayFormat + do { + format = .avro(schemaID: try AvroSchemaStore.shared.add(text)) + } catch { + NSAlert(error: error).beginSheetModal(for: window, completionHandler: nil) + return + } + guard let self else { return } + if let column = model.columns.column(id: columnID), model.displayFormat(for: column) == format { + model.redisplay(column) + } else { + setDisplayFormat(format, columnID: columnID) + } } - let item = NSMenuItem() - item.representedObject = DataGridDisplayFormatChoice(columnID: columnID, format: .avro(schemaID: schemaID)) - setDisplayFormatFromMenu(item) } @objc func sizeColumnToFitFromMenu(_ sender: Any?) { diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index 75fdb52f26..54ecbf4d2a 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -100,6 +100,15 @@ internal struct BinaryValueViewerView: View { Divider() hexBody } + case .missingSchema?: + VStack(spacing: 0) { + banner( + String(localized: "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…"), + systemImage: "questionmark.folder" + ) + Divider() + hexBody + } case .decoded(let tree, let text, let note)?: VStack(spacing: 0) { if let note { diff --git a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift index 0ebe813d53..13bed1dadc 100644 --- a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift +++ b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift @@ -9,8 +9,8 @@ import TableProTextEngine @MainActor internal enum FieldEditorResolver { - /// Answers from the field's own resolution when it has one. Only a field built before - /// `MultiRowEditState` filled it in pays the detectors here. + // Answers from the field's own resolution when it has one. Only a field built before + // `MultiRowEditState` filled it in pays the detectors here. static func resolve(field: FieldEditState) -> FieldEditorKind { if let resolved = field.resolvedEditor { return resolved } return resolve( @@ -105,19 +105,19 @@ internal enum FieldEditorResolver { return .singleLine } - /// The element editor an array column's value opens on, or nil where the list cannot represent - /// it and the plain text editor stays. - /// - /// The value is parsed, not the type alone: `jsonb[]` and `jsonb[][]` are one type in - /// PostgreSQL's catalog, and a dimension-prefixed literal such as `[0:2]={a,b,c}` is legal in - /// any array column, so the declared type cannot rule either out on a given row. This is the - /// same gate the grid applies before opening the popover. - /// - /// It is also what scopes the editor to the engines whose arrays are written this way. The - /// classifier's `[]` rule takes no engine, and a document store's `object[]` classifies the - /// same, so a field with no literal to read keeps the plain editor rather than being offered a - /// list that commits PostgreSQL `{…}` syntax. That leaves a stored NULL and a multi-row - /// selection on the plain editor, which is where they already were. + // The element editor an array column's value opens on, or nil where the list cannot represent + // it and the plain text editor stays. + // + // The value is parsed, not the type alone: `jsonb[]` and `jsonb[][]` are one type in + // PostgreSQL's catalog, and a dimension-prefixed literal such as `[0:2]={a,b,c}` is legal in + // any array column, so the declared type cannot rule either out on a given row. This is the + // same gate the grid applies before opening the popover. + // + // It is also what scopes the editor to the engines whose arrays are written this way. The + // classifier's `[]` rule takes no engine, and a document store's `object[]` classifies the + // same, so a field with no literal to read keeps the plain editor rather than being offered a + // list that commits PostgreSQL `{…}` syntax. That leaves a stored NULL and a multi-row + // selection on the plain editor, which is where they already were. private static func arrayElementEditor(for type: ColumnType, originalValue: String?) -> ArrayElementEditor? { guard let elementEditor = type.arrayElementEditor, let originalValue, @@ -129,11 +129,11 @@ internal enum FieldEditorResolver { return elementEditor } - /// Runs ahead of the JSON rule, which would claim GeoJSON in a spatial column. The readers are - /// the ones `GeometryFieldPreview` draws with, so the kind and the Map segment cannot disagree. - /// - /// This runs on every selection change, so a value too long for the preview to read on the main - /// actor is classified by how it opens instead. + // Runs ahead of the JSON rule, which would claim GeoJSON in a spatial column. The readers are + // the ones `GeometryFieldPreview` draws with, so the kind and the Map segment cannot disagree. + // + // This runs on every selection change, so a value too long for the preview to read on the main + // actor is classified by how it opens instead. private static func geometryEditor( for type: ColumnType, originalValue: String?, @@ -150,8 +150,8 @@ internal enum FieldEditorResolver { } } - /// GeoJSON only: the sniffing reader takes any pair of numbers, and any document with `lat` and - /// `lon`, for a point. The parse goes first because on a large document it is cheaper. + // GeoJSON only: the sniffing reader takes any pair of numbers, and any document with `lat` and + // `lon`, for a point. The parse goes first because on a large document it is cheaper. private static func isGeoJSON(_ value: String) -> Bool { guard GeometryFieldPreview.readsSynchronously(value) else { return GeometryValueSniffer.opensGeoJSON(value) @@ -160,9 +160,9 @@ internal enum FieldEditorResolver { return GeoJSONGeometryReader.looksLikeGeoJSON(value) } - /// Bytes never fall through to a text editor: they arrive as one character per byte, and one - /// keystroke in a text field would stage that string over the blob. A type the map names but - /// cannot draw is still a geometry field, as it is for text. + // Bytes never fall through to a text editor: they arrive as one character per byte, and one + // keystroke in a text field would stage that string over the blob. A type the map names but + // cannot draw is still a geometry field, as it is for text. private static func binaryGeometryEditor(_ originalValue: String?) -> FieldEditorKind { guard let originalValue else { return .blobHex } let geometry = FieldEditorKind.geometry(GeometryFieldDescriptor(textEditor: .hex, source: .binary)) @@ -177,8 +177,8 @@ internal enum FieldEditorResolver { } } - /// A value with nothing to read yet is still a geometry field, and so is a type the map names - /// but cannot draw. Text no reader takes, as SQL Server, Oracle and Teradata send, is not. + // A value with nothing to read yet is still a geometry field, and so is a type the map names + // but cannot draw. Text no reader takes, as SQL Server, Oracle and Teradata send, is not. private static func spatialTextEditor(_ originalValue: String?) -> FieldEditorKind? { guard let originalValue, !originalValue.isEmpty else { return .geometry(GeometryFieldDescriptor(textEditor: .multiLine, source: .spatialColumn)) @@ -199,11 +199,11 @@ internal enum FieldEditorResolver { } } - /// `isLongText` only matches six exact type names, so a large value in `VARCHAR(MAX)`, - /// `NCLOB` or ClickHouse's `Nullable(String)` never reached the multi-line editor. Whether a - /// value belongs on one line is a property of the value, so ask the value as well. - /// A one-line field cannot mark an invisible character, so a value holding one takes the text - /// view that can, while the marks are on. + // `isLongText` only matches six exact type names, so a large value in `VARCHAR(MAX)`, + // `NCLOB` or ClickHouse's `Nullable(String)` never reached the multi-line editor. Whether a + // value belongs on one line is a property of the value, so ask the value as well. + // A one-line field cannot mark an invisible character, so a value holding one takes the text + // view that can, while the marks are on. static func needsMultiLineEditor(_ value: String?, revealsInvisibleCharacters: Bool = false) -> Bool { guard let value, !value.isEmpty else { return false } let text = value as NSString @@ -216,8 +216,8 @@ internal enum FieldEditorResolver { (0 ..< text.length).contains { SpecialCharacter.classify(in: text, at: $0) != nil } } - /// Two lines' worth at the inspector's minimum width. The value font is user-configurable, so how - /// many characters actually fit a line moves with it; the threshold stays a fixed count on purpose, - /// because which editor a value gets must not change under the user when they resize the font. + // Two lines' worth at the inspector's minimum width. The value font is user-configurable, so how + // many characters actually fit a line moves with it; the threshold stays a fixed count on purpose, + // because which editor a value gets must not change under the user when they resize the font. static let multiLineValueThreshold = 80 } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift index d9dcb6848f..09dd7d8d9a 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift @@ -7,7 +7,8 @@ import Foundation @testable import TablePro import Testing -/// Fixtures written by fastavro 1.13.1, `schemaless_writer(buffer, parse_schema(schema), record)`. +// Fixtures written by fastavro, `schemaless_writer(buffer, parse_schema(schema), record)`: 1.13.1, and 1.12.2 for +// the named decimal, the padded decimal and the nanosecond timestamps. struct AvroDecoderTests { private static let userSchema = #"{"type":"record","name":"User","namespace":"com.example","fields":["# + #"{"name":"id","type":"long"},{"name":"name","type":"string"},{"name":"email","type":["null","string"]},"# @@ -49,7 +50,7 @@ struct AvroDecoderTests { BinaryFixture.data(String(repeating: "00", count: 24)), schema: try AvroSchema.parse(Self.userSchema) ) - #expect(message.schemaID == nil) + #expect(message.framing == nil) let expected = #"{"id":0,"name":"","email":null,"active":false,"score":0.0,"ratio":0.0,"tags":[],"attrs":{},"# + #""kind":"A","hash":"0x00000000","raw":"0x","nothing":null}"# #expect(MessagePackTextWriter.json(message.value, pretty: false) == expected) @@ -59,7 +60,7 @@ struct AvroDecoderTests { func confluentFrame() throws { let hex = "000000002a0e024b000100000000000000400000803f0202740002026b06000261626364027f" let message = try AvroDecoder.decode(BinaryFixture.data(hex), schema: try AvroSchema.parse(Self.userSchema)) - #expect(message.schemaID == 42) + #expect(message.framing == .confluent(schemaID: 42)) let expected = #"{"id":7,"name":"K","email":null,"active":true,"score":2.0,"ratio":1.0,"tags":["t"],"attrs":{"k":3},"# + #""kind":"B","hash":"0x61626364","raw":"0x7F","nothing":null}"# #expect(MessagePackTextWriter.json(message.value, pretty: false) == expected) @@ -105,6 +106,95 @@ struct AvroDecoderTests { #expect(error?.kind == .tooLarge) } + @Test("Items that take no bytes count toward the limit across every block, nested ones included") + func zeroByteItemsAcrossBlocks() throws { + let nested = try AvroSchema.parse(#"{"type":"array","items":{"type":"array","items":"null"}}"#) + let fourThousandNulls = "c03e00" + #expect(throws: Never.self) { + try AvroDecoder.decode(BinaryFixture.data("04" + String(repeating: fourThousandNulls, count: 2) + "00"), schema: nested) + } + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("d804" + String(repeating: fourThousandNulls, count: 300) + "00"), schema: nested) + } + #expect(error?.kind == .tooLarge) + } + + @Test("A cell's budget stops building, not reading, so it finds the Kafka frame the viewer finds") + func budgetKeepsTheFramingDecision() throws { + let schema = try AvroSchema.parse( + #"{"type":"record","name":"Six","fields":[{"name":"a","type":"long"},{"name":"b","type":"long"},"# + + #"{"name":"c","type":"long"},{"name":"d","type":"long"},{"name":"e","type":"long"},{"name":"f","type":"long"}]}"# + ) + let framed = BinaryFixture.data("0000000005" + "020406080a0c") + let full = try AvroDecoder.decode(framed, schema: schema) + #expect(full.framing == .confluent(schemaID: 5)) + #expect(MessagePackTextWriter.json(full.value, pretty: false) == #"{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6}"#) + let budgeted = try AvroDecoder.decode(framed, schema: schema, maxValues: 3) + #expect(budgeted.framing == .confluent(schemaID: 5)) + #expect(MessagePackTextWriter.json(budgeted.value, pretty: false) == #"{"a":1,"b":2,"c":…}"#) + } + + @Test("Fingerprints match Avro Java's CRC-64-AVRO of the Parsing Canonical Form") + func fingerprints() throws { + #expect(try AvroSchema.parse(#""int""#).fingerprint == 0x7275_D51A_3F39_5C8F) + #expect(try AvroSchema.parse(#"{"type":"string"}"#).fingerprint == 0x8F01_4872_6345_03C7) + #expect(try AvroSchema.parse(#""bytes""#).fingerprint == 0x4FC0_16DA_C320_1965) + let record = #"{"type":"record","name":"R","doc":"ignored","fields":[{"name":"a","type":"long"},"# + + #"{"name":"b","type":{"type":"string","logicalType":"uuid"}}]}"# + #expect(try AvroSchema.parse(record).fingerprint == 0xD99C_4645_DD9B_2051) + let payment = #"{"type":"record","name":"Payment","namespace":"com.example","fields":[{"name":"id","type":"string"},"# + + #"{"name":"amount","type":{"type":"fixed","name":"Money","size":8,"logicalType":"decimal","precision":12,"scale":2}},"# + + #"{"name":"fee","type":["null","Money"]},{"name":"kind","type":{"type":"enum","name":"Kind","symbols":["A","B"]}},"# + + #"{"name":"tags","type":{"type":"map","values":{"type":"array","items":"int"}}}]}"# + let parsed = try AvroSchema.parse(payment) + #expect(AvroSchema.canonicalForm(parsed.root) == #"{"name":"com.example.Payment","type":"record","fields":["# + + #"{"name":"id","type":"string"},{"name":"amount","type":{"name":"com.example.Money","type":"fixed","size":8}},"# + + #"{"name":"fee","type":["null","com.example.Money"]},"# + + #"{"name":"kind","type":{"name":"com.example.Kind","type":"enum","symbols":["A","B"]}},"# + + #"{"name":"tags","type":{"type":"map","values":{"type":"array","items":"int"}}}]}"#) + #expect(parsed.fingerprint == 0x51D2_8FF2_5F71_BF64) + } + + @Test("A single-object encoded value is read past its header when the fingerprint is the schema's, and refused when not") + func singleObjectEncoding() throws { + let schema = try AvroSchema.parse( + #"{"type":"record","name":"R","fields":[{"name":"a","type":"long"},{"name":"b","type":"string"}]}"# + ) + let data = BinaryFixture.data("c301" + "51209bdd45469cd9" + "020278") + let message = try AvroDecoder.decode(data, schema: schema) + #expect(message.framing == .singleObject(fingerprint: 0xD99C_4645_DD9B_2051)) + #expect(MessagePackTextWriter.json(message.value, pretty: false) == #"{"a":1,"b":"x"}"#) + + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(data, schema: try AvroSchema.parse(#""long""#)) + } + #expect(error?.kind == .otherSchema(fingerprint: 0xD99C_4645_DD9B_2051)) + } + + @Test("A Kafka value of a bytes schema is the bytes themselves, with no length in front") + func confluentBytes() throws { + let message = try AvroDecoder.decode(BinaryFixture.data("0000000007dead"), schema: try AvroSchema.parse(#""bytes""#)) + #expect(message.framing == .confluent(schemaID: 7)) + #expect(message.value == .binary(Data([0xDE, 0xAD]))) + } + + @Test("A huge zero-byte block after a small one is refused, not added up until it overflows") + func zeroByteCountOverflow() throws { + let nulls = try AvroSchema.parse(#"{"type":"array","items":"null"}"#) + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("c801feffffffffffffffff0100"), schema: nulls) + } + #expect(error?.kind == .tooLarge) + } + + @Test("A size, precision or scale that is not a whole number is refused, as Avro's parser refuses it") + func strictIntegerAttributes() throws { + #expect(throws: AvroSchemaError.self) { try AvroSchema.parse(#"{"type":"fixed","name":"F","size":true}"#) } + #expect(throws: AvroSchemaError.self) { try AvroSchema.parse(#"{"type":"fixed","name":"F","size":4.5}"#) } + #expect(try Self.json("0230", schema: #"{"type":"bytes","logicalType":"decimal","precision":4.0}"#) == #""0x30""#) + #expect(try Self.json("0230", schema: #"{"type":"bytes","logicalType":"decimal","precision":4,"scale":true}"#) == #""0x30""#) + } + @Test("A decimal without a usable precision and scale reads as its bytes") func invalidDecimalFallsBack() throws { for annotation in [#""scale":2"#, #""precision":2,"scale":3"#, #""precision":100000000,"scale":100000000"#] { @@ -115,11 +205,51 @@ struct AvroDecoderTests { @Test("Decimals of any width keep their sign and scale") func decimalText() { - #expect(AvroLogicalRendering.decimalText(unscaled: [0x00], scale: 2) == "0.00") - #expect(AvroLogicalRendering.decimalText(unscaled: [0xFF], scale: 3) == "-0.001") - #expect(AvroLogicalRendering.decimalText(unscaled: [0x30, 0x39], scale: 0) == "12345") - #expect(AvroLogicalRendering.decimalText(unscaled: [0x80] + Array(repeating: 0, count: 15), scale: 0) + #expect(AvroDecimal.text(unscaled: [0x00], scale: 2) == "0.00") + #expect(AvroDecimal.text(unscaled: [0xFF], scale: 3) == "-0.001") + #expect(AvroDecimal.text(unscaled: [0x30, 0x39], scale: 0) == "12345") + #expect(AvroDecimal.text(unscaled: [0x80] + Array(repeating: 0, count: 15), scale: 0) == "-170141183460469231731687303715884105728") + #expect(AvroDecimal.text(unscaled: [0x0D, 0xE0, 0xB6, 0xB3, 0xA7, 0x64, 0x00, 0x00], scale: 18) == "1.000000000000000000") + #expect(AvroDecimal.text(unscaled: [0x3B, 0x9A, 0xCA, 0x00], scale: 0) == "1000000000") + } + + @Test("A decimal reads as its bytes once its significant width passes what its precision holds") + func decimalWidthFollowsPrecision() throws { + let schema = #"{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}"# + #expect(try Self.json("0404d2", schema: schema) == #""12.34""#) + #expect(try Self.json("08000004d2", schema: schema) == #""12.34""#) + #expect(try Self.json("067fffff", schema: schema) == #""0x7FFFFF""#) + let padded = #"{"type":"fixed","name":"Wide","size":16,"logicalType":"decimal","precision":4,"scale":2}"# + #expect(try Self.json("000000000000000000000000000004d2", schema: padded) == #""12.34""#) + #expect(AvroDecimal.maxBytes(precision: 38) == 16) + #expect(AvroDecimal.maxPrecision(bytes: 16) == 38) + } + + @Test("A fixed's decimal precision past what its size holds is ignored, as Avro's reader does") + func fixedDecimalPrecisionFitsSize() throws { + let schema = #"{"type":"fixed","name":"Small","size":2,"logicalType":"decimal","precision":5,"scale":0}"# + #expect(try Self.json("04d2", schema: schema) == #""0x04D2""#) + } + + @Test("A named fixed keeps its logical type where it is referred to by name") + func namedLogicalType() throws { + let schema = #"{"type":"record","name":"Prices","fields":["# + + #"{"name":"a","type":{"type":"fixed","name":"Money","size":8,"logicalType":"decimal","precision":12,"scale":2}},"# + + #"{"name":"b","type":"Money"}]}"# + #expect(try Self.json("00000000000004d2fffffffffffffffb", schema: schema) == #"{"a":"12.34","b":"-0.05"}"#) + } + + @Test("Nanosecond timestamps and durations render as values") + func nanosAndDuration() throws { + let nanos = #"{"type":"record","name":"N","fields":["# + + #"{"name":"ts","type":{"type":"long","logicalType":"timestamp-nanos"}},"# + + #"{"name":"lts","type":{"type":"long","logicalType":"local-timestamp-nanos"}}]}"# + #expect(try Self.json("aab4aed8c7bfce972faab4aed8c7bfce972f", schema: nanos) + == #"{"ts":"2023-11-14T22:13:20.123456789Z","lts":"2023-11-14T22:13:20.123456789"}"#) + let duration = #"{"type":"fixed","name":"D","size":12,"logicalType":"duration"}"# + #expect(try Self.json("0100000002000000bc0b0000", schema: duration) == #""P1M2DT3.004S""#) + #expect(try Self.json("000000000000000000000000", schema: duration) == #""PT0S""#) } @Test("A value that does not fit the schema names what is wrong and where") @@ -159,6 +289,7 @@ struct AvroDecoderTests { #"{"type":"fixed","name":"F"}"#, #"{"type":"array"}"#, "42", + #"{"type":"record","name":"R","fields":[{"name":"a","type":{"type":"enum","name":"R","symbols":["X"]}}]}"#, ] { #expect(throws: AvroSchemaError.self, "\(text)") { try AvroSchema.parse(text) } } @@ -172,6 +303,7 @@ struct AvroDecoderTests { + #"{"name":"f","type":"Color"},{"name":"g","type":"a.b.Color"}]}"# ) #expect(schema.namedTypes["a.b.Color"] == .enumeration(name: "a.b.Color", symbols: ["RED"])) + #expect(schema.displayName == "Outer") let value = try AvroDecoder.decode(BinaryFixture.data("000000"), schema: schema).value #expect(MessagePackTextWriter.json(value, pretty: false) == #"{"e":"RED","f":"RED","g":"RED"}"#) } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift index bd6f048caf..44057ed83b 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift @@ -10,7 +10,7 @@ import Testing @MainActor struct BinaryDisplayFormatTests { - /// `HEX(UUID_TO_BIN('6ccd780c-baba-1026-9564-5b8c656024db', 1))` on MySQL 8.4. + // `HEX(UUID_TO_BIN('6ccd780c-baba-1026-9564-5b8c656024db', 1))` on MySQL 8.4. private static let swappedStored = BinaryFixture.data("1026BABA6CCD780C95645B8C656024DB") private static let blob = ColumnType.blob(rawType: "BINARY(16)") @@ -141,12 +141,9 @@ struct BinaryDisplayFormatTests { #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("0000000211")), columnType: blob, displayFormat: .igbinary) == "0x0000000211") - let schemaID = try AvroSchemaLibrary.shared.register( - text: #"{"type":"record","name":"R","fields":[{"name":"id","type":"long"}]}"#, - name: "r.avsc" - ) + let schemaID = try AvroSchemaStore.shared.add(#"{"type":"record","name":"Reading","fields":[{"name":"id","type":"long"}]}"#) let avro = ValueDisplayFormat.avro(schemaID: schemaID) - #expect(avro.displayName.contains("r.avsc")) + #expect(avro.displayName.contains("Reading")) #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("54")), columnType: blob, displayFormat: avro) == #"{"id":42}"#) #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("80")), columnType: blob, displayFormat: avro) == "0x80") } @@ -160,7 +157,7 @@ struct BinaryDisplayFormatTests { } #expect(text == "i:7;") - let schemaID = try AvroSchemaLibrary.shared.register(text: #"{"type":"array","items":"int"}"#, name: "ints.avsc") + let schemaID = try AvroSchemaStore.shared.add(#"{"type":"array","items":"int"}"#) let framed = BinaryValueDecoding.document(BinaryFixture.data("000000002a020200"), format: .avro(schemaID: schemaID)) guard case .decoded(.json, _, let note) = framed else { Issue.record("expected a decoded JSON tree") @@ -169,13 +166,13 @@ struct BinaryDisplayFormatTests { #expect(note?.contains("42") == true) } - @Test("An Avro column whose schema is gone says so") + @Test("An Avro column whose schema is gone says so, rather than calling the value invalid, and the cell stays hex") func avroSchemaMissing() { - let document = BinaryValueDecoding.document(Data([0x02]), format: .avro(schemaID: "0000000000000000")) - guard case .failed(let error) = document else { - Issue.record("expected a failure") + let format = BinaryValueFormat.avro(schemaID: String(repeating: "0", count: 32)) + guard case .missingSchema = BinaryValueDecoding.document(Data([0x02]), format: format) else { + Issue.record("expected a missing schema") return } - #expect(error.kind == .schemaMissing) + #expect(BinaryValueDecoding.text(Data([0x02]), format: format) == nil) } } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift index 4b9fed3b2e..d78b69fbc8 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift @@ -7,22 +7,18 @@ import Foundation @testable import TablePro import Testing -/// Fixtures written by the igbinary 3.2.17 extension on PHP 8.3, `igbinary_serialize($value)`, each beside -/// `serialize($value)` of the same value. +// Fixtures from igbinary 3.2.17RC1 on PHP 8.3: igbinary_serialize($value) beside serialize($value). struct IgbinaryDecoderTests { - private static let pointProperties = #"s:1:"x";i:1;s:4:"\#u{0}*\#u{0}y";i:2;s:8:"\#u{0}Point\#u{0}z";i:3;"# - private static let fixtures: [(name: String, hex: String, serialized: String)] = [ - ("null", "0000000200", "N;"), - ("true", "0000000205", "b:1;"), - ("false", "0000000204", "b:0;"), - ("small int", "000000020607", "i:7;"), - ("negative int", "0000000209012c", "i:-300;"), - ("32-bit int", "000000020a00011170", "i:70000;"), - ("64-bit int", "0000000220000000012a05f200", "i:5000000000;"), - ("PHP_INT_MAX", "00000002207fffffffffffffff", "i:9223372036854775807;"), - ("PHP_INT_MIN", "00000002218000000000000000", "i:-9223372036854775808;"), - ("float", "000000020c400a000000000000", "d:3.25;"), + ("null", "0000000200", #"N;"#), + ("true", "0000000205", #"b:1;"#), + ("false", "0000000204", #"b:0;"#), + ("small int", "000000020607", #"i:7;"#), + ("negative int", "0000000209012c", #"i:-300;"#), + ("32-bit int", "000000020a00011170", #"i:70000;"#), + ("64-bit int", "0000000220000000012a05f200", #"i:5000000000;"#), + ("PHP_INT_MAX", "00000002207fffffffffffffff", #"i:9223372036854775807;"#), + ("PHP_INT_MIN", "00000002218000000000000000", #"i:-9223372036854775808;"#), ("empty string", "000000020d", #"s:0:"";"#), ("UTF-8 string", "00000002110d68c3a96c6c6f2077c3b6726c64", #"s:13:"héllo wörld";"#), ( @@ -32,99 +28,147 @@ struct IgbinaryDecoderTests { ), ( "nested map with int keys", - "00000002140511046e616d651103416e6e11047461677314020600110161060111016211046d65746114020e00" - + "1105696e6e657211016e0701060a110374656e070511036e6567", - #"a:5:{s:4:"name";s:3:"Ann";s:4:"tags";a:2:{i:0;s:1:"a";i:1;s:1:"b";}s:4:"meta";"# - + #"a:2:{s:4:"name";s:5:"inner";s:1:"n";i:-1;}i:10;s:3:"ten";i:-5;s:3:"neg";}"# + "00000002140511046e616d651103416e6e11047461677314020600110161060111016211046d65746114020e00110569" + + "6e6e657211016e0701060a110374656e070511036e6567", + #"a:5:{s:4:"name";s:3:"Ann";s:4:"tags";a:2:{i:0;s:1:"a";i:1;s:1:"b";}s:4:"meta";a:2:{s:4:"na"# + + #"me";s:5:"inner";s:1:"n";i:-1;}i:10;s:3:"ten";i:-5;s:3:"neg";}"# ), ( "repeated strings by ID", "00000002140506001105616c706861060111046265746106020e0006030e0106040e00", #"a:5:{i:0;s:5:"alpha";i:1;s:4:"beta";i:2;s:5:"alpha";i:3;s:4:"beta";i:4;s:5:"alpha";}"# ), + ( + "keys and values share string IDs", + "00000002140311016b1101760e010e0011017a14010e000e01", + #"a:3:{s:1:"k";s:1:"v";s:1:"v";s:1:"k";s:1:"z";a:1:{s:1:"k";s:1:"v";}}"# + ), ( "object with every visibility", "000000021705506f696e74140311017806011104002a00790602110800506f696e74007a0603", - #"O:5:"Point":3:{"# + pointProperties + "}" + #"O:5:"Point":3:{s:1:"x";i:1;s:4:"\#u{0}*\#u{0}y";i:2;s:8:"\#u{0}Point\#u{0}z";i:3;}"# + ), + ("stdClass", "000000021708737464436c617373140211016106011101621401060005", #"O:8:"stdClass":2:{s:1:"a";i:1;s:1:"b";a:1:{i:0;b:1;}}"#), + ("empty stdClass", "000000021708737464436c6173731400", #"O:8:"stdClass":0:{}"#), + ("enum case", "00000002170453756974271106486561727473", #"E:11:"Suit:Hearts";"#), + ( + "repeated enum case", + "00000002140406001704537569742711064865617274730601220106021a0027110653706164657306030601", + #"a:4:{i:0;E:11:"Suit:Hearts";i:1;r:2;i:2;E:11:"Suit:Spades";i:3;i:1;}"# ), ( - "stdClass", - "000000021708737464436c617373140211016106011101621401060005", - #"O:8:"stdClass":2:{s:1:"a";i:1;s:1:"b";a:1:{i:0;b:1;}}"# + "PHP reference to an enum case", + "000000021403060025170453756974271106486561727473060125220106020602", + #"a:3:{i:0;E:11:"Suit:Hearts";i:1;R:2;i:2;i:2;}"# ), - ("enum case", "00000002170453756974271106486561727473", #"E:11:"Suit:Hearts";"#), + ( + "repeated object", + "00000002140306001705506f696e74140311017806011104002a00790602110800506f696e74007a0603060122010602" + + "2201", + #"a:3:{i:0;O:5:"Point":3:{s:1:"x";i:1;s:4:"\#u{0}*\#u{0}y";i:2;s:8:"\#u{0}Point\#u{0}z";i:3;}i:1;r:2;i:2;r:2;}"# + ), + ( + "__serialize object repeated", + "00000002140306001703426f78140111017606050601220106020603", + #"a:3:{i:0;O:3:"Box":1:{s:1:"v";i:5;}i:1;r:2;i:2;i:3;}"# + ), + ("Serializable object repeated", "000000021403060017034f6c641d036162630601220106020604", #"a:3:{i:0;C:3:"Old":3:{abc}i:1;r:2;i:2;i:4;}"#), + ("object holding itself", "000000021708737464436c6173731401110473656c662200", #"O:8:"stdClass":1:{s:4:"self";r:1;}"#), + ("PHP reference inside an array repeated by value", "00000002140206001401060025060106010101", #"a:2:{i:0;a:1:{i:0;i:1;}i:1;a:1:{i:0;R:3;}}"#), + ( + "PHP reference to an object, then objects", + "0000000214040600251708737464436c6173731400060125220106021a00140006032202", + #"a:4:{i:0;O:8:"stdClass":0:{}i:1;R:2;i:2;O:8:"stdClass":0:{}i:3;r:3;}"# + ), + ( + "PHP reference to an array", + "00000002140306002514010600060106012501010602140106000601", + #"a:3:{i:0;a:1:{i:0;i:1;}i:1;R:2;i:2;a:1:{i:0;i:1;}}"# + ), + ( + "PHP reference inside an object", + "00000002140206001708737464436c617373140211016125110373747211016225010206012201", + #"a:2:{i:0;O:8:"stdClass":2:{s:1:"a";s:3:"str";s:1:"b";R:3;}i:1;r:2;}"# + ), + ( + "object and its reference in an array repeated by value", + "0000000214020600140206001708737464436c6173731400060125220206010101", + #"a:2:{i:0;a:2:{i:0;O:8:"stdClass":0:{}i:1;R:3;}i:1;a:2:{i:0;r:3;i:1;R:3;}}"# + ), + ("scalar reference, then its value", "0000000214030600251102616106010e000602250101", #"a:3:{i:0;s:2:"aa";i:1;s:2:"aa";i:2;R:2;}"#), + ("PHP reference to null", "000000021402060025000601250101", #"a:2:{i:0;N;i:1;R:2;}"#), + ( + "array repeated by value", + "0000000214030600140111016b1101760601010106020101", + #"a:3:{i:0;a:1:{s:1:"k";s:1:"v";}i:1;a:1:{s:1:"k";s:1:"v";}i:2;a:1:{s:1:"k";s:1:"v";}}"# + ), + ( + "arrays repeated by value inside one another", + "000000021402060014020600140111017806010601010206010101", + #"a:2:{i:0;a:2:{i:0;a:1:{s:1:"x";i:1;}i:1;a:1:{s:1:"x";i:1;}}i:1;a:2:{i:0;a:1:{s:1:"x";i:1;}"# + + #"i:1;a:1:{s:1:"x";i:1;}}}"# + ), + ( + "object inside an array repeated by value", + "0000000214030600140206001705506f696e74140311017806011104002a00790602110800506f696e74007a06030601" + + "06020601010106022202", + #"a:3:{i:0;a:2:{i:0;O:5:"Point":3:{s:1:"x";i:1;s:4:"\#u{0}*\#u{0}y";i:2;s:8:"\#u{0}Point\#u{0}z";i:3;}i:1;i:2;}i"# + + #":1;a:2:{i:0;r:3;i:1;i:2;}i:2;r:3;}"# + ), + ( + "floats", + "00000002140e06000c800000000000000006010c430c6bf52634000006020c4341c37937e0800006030c4376345785d8" + + "a00006040c3fb999999999999a06050c3ee4f8b588e368f106060c3f1a36e2eb1c432d06070c3e8421f5f40d83760608" + + "0c40fe240c9fbe76c906090c0000000000000001060a0c7fefffffffffffff060b0c4345ee2a2eb5a5c4060c0c7ff000" + + "0000000000060d0cfff0000000000000", + #"a:14:{i:0;d:-0;i:1;d:1000000000000000;i:2;d:10000000000000000;i:3;d:1.0E+17;i:4;d:0.1;i:5;"# + + #"d:1.0E-5;i:6;d:0.0001;i:7;d:1.5E-7;i:8;d:123456.789;i:9;d:5.0E-324;i:10;d:1.79769313486231"# + + #"57E+308;i:11;d:12345678901234568;i:12;d:INF;i:13;d:-INF;}"# + ), + ( + "ArrayObject, whose members have integer keys", + "00000002170b41727261794f626a65637414040600060006011402060006010601060206021400060300", + #"O:11:"ArrayObject":4:{i:0;i:0;i:1;a:2:{i:0;i:1;i:1;i:2;}i:2;a:0:{}i:3;N;}"# + ), + ("__serialize list", "000000021401060017074c697374426f7814020600060a0601110178", #"a:1:{i:0;O:7:"ListBox":2:{i:0;i:10;i:1;s:1:"x";}}"#), ] - @Test("Each fixture decodes to the value PHP's serialize() wrote, and writes it back the same") + @Test("Each fixture decodes to what serialize() wrote, numbering, r and R included") func fixturesMatchSerialize() throws { for fixture in Self.fixtures { let decoded = try IgbinaryDecoder.decode(BinaryFixture.data(fixture.hex)) - #expect(decoded == PhpSerializeParser.parse(fixture.serialized), "\(fixture.name)") #expect(PhpSerializeWriter.text(decoded) == fixture.serialized, "\(fixture.name)") + #expect(decoded == PhpSerializeParser.parse(fixture.serialized), "\(fixture.name)") } } + @Test("A string that is not UTF-8 keeps its bytes, and its length is the byte count serialize() writes") + func binaryStrings() throws { + let value = try IgbinaryDecoder.decode(BinaryFixture.data("00000002110600ff80616263")) + #expect(value == .binaryString(Data([0x00, 0xFF, 0x80, 0x61, 0x62, 0x63]))) + #expect(PhpSerializeWriter.text(value) == Self.latin1("733a363a2200ff80616263223b")) + + let keyed = try IgbinaryDecoder.decode(BinaryFixture.data("0000000214021102fffe06011103fffe781101ff")) + #expect(keyed == .array([ + PhpKeyValue(key: .binaryString(Data([0xFF, 0xFE])), value: .int(1)), + PhpKeyValue(key: .binaryString(Data([0xFF, 0xFE, 0x78])), value: .binaryString(Data([0xFF]))), + ])) + #expect( + PhpSerializeWriter.text(keyed) + == Self.latin1("613a323a7b733a323a22fffe223b693a313b733a333a22fffe78223b733a313a22ff223b7d") + ) + } + @Test("A string of 300 bytes uses the 16-bit length") func longString() throws { let hex = "0000000212012c" + String(repeating: "61", count: 300) #expect(try IgbinaryDecoder.decode(BinaryFixture.data(hex)) == .string(String(repeating: "a", count: 300))) } - @Test("A repeated object refers back with the number serialize() gives it") - func repeatedObject() throws { - let hex = "00000002140306001705506f696e74140311017806011104002a00790602110800506f696e74007a0603" - + "0601220106021a0014030e0106010e0206020e030603" - let point = #"O:5:"Point":3:{"# + Self.pointProperties + "}" - let serialized = "a:3:{i:0;\(point)i:1;r:2;i:2;\(point)}" - #expect(try IgbinaryDecoder.decode(BinaryFixture.data(hex)) == PhpSerializeParser.parse(serialized)) - } - - @Test("A repeated enum case refers back to the first one") - func repeatedEnumCase() throws { - let hex = "000000021403060017045375697427110648656172747306011a0027110653706164657306022201" - let serialized = #"a:3:{i:0;E:11:"Suit:Hearts";i:1;E:11:"Suit:Spades";i:2;r:2;}"# - #expect(try IgbinaryDecoder.decode(BinaryFixture.data(hex)) == PhpSerializeParser.parse(serialized)) - } - - @Test("A PHP reference counts once, as R: numbers it") - func phpReference() throws { - let hex = "0000000214020600251401060011067368617265640601250101" - let serialized = #"a:2:{i:0;a:1:{i:0;s:6:"shared";}i:1;R:2;}"# - #expect(try IgbinaryDecoder.decode(BinaryFixture.data(hex)) == PhpSerializeParser.parse(serialized)) - #expect(PhpSerializeWriter.text(try IgbinaryDecoder.decode(BinaryFixture.data(hex))) == serialized) - } - - @Test("An array repeated by value is written out again, as serialize() writes it, and later numbers follow") - func arraysByValue() throws { - let cases: [(hex: String, serialized: String)] = [ - ( - "0000000214020600140206000601060111017806010101", - #"a:2:{i:0;a:2:{i:0;i:1;i:1;s:1:"x";}i:1;a:2:{i:0;i:1;i:1;s:1:"x";}}"# - ), - ( - "0000000214030600140206001705506f696e7414011101780601060106020601010106022202", - #"a:3:{i:0;a:2:{i:0;O:5:"Point":1:{s:1:"x";i:1;}i:1;i:2;}i:1;a:2:{i:0;r:3;i:1;i:2;}i:2;r:3;}"# - ), - ( - "0000000214030600140111016b140206000601060106020601010206020101", - #"a:3:{i:0;a:1:{s:1:"k";a:2:{i:0;i:1;i:1;i:2;}}i:1;a:2:{i:0;i:1;i:1;i:2;}i:2;a:1:{s:1:"k";a:2:{i:0;i:1;i:1;i:2;}}}"# - ), - ( - "000000021404060014030600060106010602060206030601010106021705506f696e741401110178060106032202", - #"a:4:{i:0;a:3:{i:0;i:1;i:1;i:2;i:2;i:3;}i:1;a:3:{i:0;i:1;i:1;i:2;i:2;i:3;}i:2;O:5:"Point":1:{s:1:"x";i:1;}i:3;r:10;}"# - ), - ("000000021708737464436c6173731401110473656c662200", #"O:8:"stdClass":1:{s:4:"self";r:1;}"#), - ("00000002140306002506050601250101060211017a", #"a:3:{i:0;i:5;i:1;R:2;i:2;s:1:"z";}"#), - ] - for testCase in cases { - #expect(try IgbinaryDecoder.decode(BinaryFixture.data(testCase.hex)) == PhpSerializeParser.parse(testCase.serialized)) - } - } - @Test("Arrays nested by value forty deep are refused rather than expanded") func exponentialCopiesAreRefused() { func level(_ depth: Int, of total: Int) -> String { - guard depth > 0 else { return "140206000601060106010601" } + guard depth > 0 else { return "14020600060106010601" } return "14020600" + level(depth - 1, of: total) + "060101" + String(format: "%02x", total - depth + 1) } let error = #expect(throws: BinaryValueDecodeError.self) { @@ -144,6 +188,40 @@ struct IgbinaryDecoderTests { #expect(error?.kind == .tooLarge) } + @Test("Strings inside an array repeated by value count toward the string limit") + func repeatedArrayStringsAreRefused() { + let text = Data(repeating: 0x61, count: 1_000_000) + func value(copies: Int) -> Data { + var data = Data([0, 0, 0, 2, 0x15]) + Self.bigEndian(UInt16(copies + 1)) + data += Data([0x06, 0x00, 0x14, 0x01, 0x06, 0x00, 0x13]) + Self.bigEndian(UInt32(text.count)) + text + for index in 1...copies { + data += Data([0x08]) + Self.bigEndian(UInt16(index)) + Data([0x01, 0x01]) + } + return data + } + #expect(throws: Never.self) { try IgbinaryDecoder.decode(value(copies: 10)) } + let error = #expect(throws: BinaryValueDecodeError.self) { try IgbinaryDecoder.decode(value(copies: 30)) } + #expect(error?.kind == .tooLarge) + } + + @Test("A copy placed deeper than the array it repeats still fits the depth cap") + func copiesKeepTheDepthCap() { + func value(levels: Int) -> Data { + let chain = 80 + var data = Data([0, 0, 0, 2, 0x15]) + Self.bigEndian(UInt16(levels + 1)) + Data([0x06, 0x00]) + data += Data(Array(repeating: [0x14, 0x01, 0x06, 0x00] as [UInt8], count: chain).joined()) + Data([0x00]) + for level in 1...levels { + let target = level == 1 ? 1 : chain + level - 1 + data += Data([0x08]) + Self.bigEndian(UInt16(level)) + data += Data([0x14, 0x01, 0x06, 0x00, 0x02]) + Self.bigEndian(UInt16(target)) + } + return data + } + #expect(throws: Never.self) { try IgbinaryDecoder.decode(value(levels: 100)) } + let error = #expect(throws: BinaryValueDecodeError.self) { try IgbinaryDecoder.decode(value(levels: 200)) } + #expect(error?.kind == .nestingTooDeep) + } + @Test("The version 1 header reads too") func versionOne() throws { #expect(try IgbinaryDecoder.decode(BinaryFixture.data("000000010607")) == .int(7)) @@ -159,9 +237,11 @@ struct IgbinaryDecoderTests { ("000000020607aa", .trailingBytes(1)), ("00000002140106000e05", .unknownReference(5)), ("0000000214010600220a", .unknownReference(10)), + ("000000021401060001" + "00", .unknownReference(0)), ("0000000214011400", .invalidByte(0x14)), ("0000000220ffffffffffffffff", .integerOutOfRange), ("0000000214ff", .unexpectedEnd), + ("000000022525" + "00", .invalidByte(0x25)), ] for testCase in cases { let error = #expect(throws: BinaryValueDecodeError.self, "\(testCase.hex)") { @@ -171,12 +251,20 @@ struct IgbinaryDecoderTests { } } - @Test("Floats follow serialize_precision -1") + @Test("Floats PHP writes with serialize_precision -1, past what the fixtures hold") func floatText() { - #expect(PhpSerializeWriter.floatText(1) == "1") - #expect(PhpSerializeWriter.floatText(0.1) == "0.1") + #expect(PhpSerializeWriter.floatText(.nan) == "NAN") + #expect(PhpSerializeWriter.floatText(0.5) == "0.5") + #expect(PhpSerializeWriter.floatText(-1.5e-7) == "-1.5E-7") #expect(PhpSerializeWriter.floatText(1e25) == "1.0E+25") - #expect(PhpSerializeWriter.floatText(1.5e-7) == "1.5E-7") - #expect(PhpSerializeWriter.floatText(-.infinity) == "-INF") + #expect(PhpSerializeWriter.floatText(99_999_999_999_999_984) == "99999999999999980") + } + + private static func latin1(_ hex: String) -> String { + String(data: BinaryFixture.data(hex), encoding: .isoLatin1) ?? "" + } + + private static func bigEndian(_ value: T) -> Data { + withUnsafeBytes(of: value.bigEndian) { Data($0) } } } diff --git a/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift b/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift index ab31cb0d41..22e2ab9b59 100644 --- a/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift @@ -211,7 +211,7 @@ struct PhpSerializeParserSpecialTests { #expect(payload == "xyzab") } - /// PHP 8.1 writes an enum case as `E:`. Before, any value holding one failed to parse at all. + // PHP 8.1 writes an enum case as `E:`. Before, any value holding one failed to parse at all. @Test("E token returns .enumCase, inside an array too") func enumCaseToken() { #expect(PhpSerializeParser.parse(#"E:11:"Suit:Hearts";"#) == .enumCase(className: "Suit", caseName: "Hearts")) @@ -228,9 +228,20 @@ struct PhpSerializeParserSpecialTests { #expect(tree.badgeLabel == "enum") } - @Test("r token returns .reference") + @Test("An object member with an integer key, as ArrayObject and __serialize data have, parses and writes back") + func integerObjectKeys() { + let text = #"O:11:"ArrayObject":4:{i:0;i:0;i:1;a:2:{i:0;i:1;i:1;i:2;}i:2;a:0:{}i:3;N;}"# + guard case .object("ArrayObject", let properties)? = PhpSerializeParser.parse(text) else { + Issue.record("expected an object") + return + } + #expect(properties.map(\.index) == [0, 1, 2, 3]) + #expect(PhpSerializeWriter.text(.object(className: "ArrayObject", properties: properties)) == text) + } + + @Test("r token returns .objectReference, so it is written back as r and not as R") func referenceLowerR() { - #expect(PhpSerializeParser.parse("r:7;") == .reference(id: 7)) + #expect(PhpSerializeParser.parse("r:7;") == .objectReference(id: 7)) } @Test("R token returns .reference") diff --git a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift index 771fa87b66..72d719c5e2 100644 --- a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift @@ -28,13 +28,15 @@ struct ValueDisplayFormatTests { @Test("Every format reads back from its stored form, an Avro one with its schema ID") func codableRoundTrip() throws { let formats: [String: ValueDisplayFormat] = [ - "a": .messagePack, "b": .igbinary, "c": .avro(schemaID: "ab12"), "d": .raw, + "a": .messagePack, "b": .igbinary, "c": .avro(schemaID: String(repeating: "ab12", count: 8)), "d": .raw, ] let data = try JSONEncoder().encode(formats) #expect(try JSONDecoder().decode([String: ValueDisplayFormat].self, from: data) == formats) #expect(try JSONDecoder().decode([String: ValueDisplayFormat].self, from: Data(#"{"x":"protobuf"}"#.utf8)) == ["x": .protobuf]) #expect(ValueDisplayFormat(rawValue: "avro:") == nil) + #expect(ValueDisplayFormat(rawValue: "avro:ab12") == nil) + #expect(ValueDisplayFormat(rawValue: "avro:" + String(repeating: "AB12", count: 8)) == nil) #expect(ValueDisplayFormat(rawValue: "nope") == nil) } diff --git a/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift new file mode 100644 index 0000000000..e37cc292bc --- /dev/null +++ b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift @@ -0,0 +1,67 @@ +// +// AvroSchemaStoreTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct AvroSchemaStoreTests { + private static let schema = #"{"type":"record","name":"Reading","namespace":"com.example","fields":[{"name":"id","type":"long"}]}"# + + private func withDirectory(_ body: (URL) throws -> Void) rethrows { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + try body(directory) + } + + @Test("The same text kept twice is one file, read back by a later launch") + func sameTextIsOneFile() throws { + try withDirectory { directory in + let store = AvroSchemaStore(directory: directory) + let id = try store.add(Self.schema) + #expect(try store.add(Self.schema) == id) + #expect(AvroSchemaStore.isSchemaID(id)) + #expect(store.storedIDs == [id]) + #expect(AvroSchemaStore(directory: directory).schema(id: id)?.displayName == "Reading") + } + } + + @Test("Text that is not a schema is refused and leaves nothing behind") + func invalidTextIsRefused() throws { + try withDirectory { directory in + let store = AvroSchemaStore(directory: directory) + #expect(throws: AvroSchemaError.self) { try store.add(#"{"type":"record","name":"R"}"#) } + #expect(store.storedIDs.isEmpty) + } + } + + @Test("An ID that is not one of the store's never reaches the file system") + func foreignIDsAreRefused() throws { + try withDirectory { directory in + let store = AvroSchemaStore(directory: directory) + #expect(store.schema(id: "../../etc/hosts") == nil) + #expect(ValueDisplayFormat(rawValue: "avro:../../etc/hosts") == nil) + #expect(ValueDisplayFormat(rawValue: "avro:") == nil) + let id = try store.add(Self.schema) + #expect(ValueDisplayFormat(rawValue: "avro:" + id) == .avro(schemaID: id)) + } + } + + @Test("A sweep keeps what is named and what this launch added") + func sweepKeepsNamedAndNew() throws { + try withDirectory { directory in + let earlier = AvroSchemaStore(directory: directory) + let named = try earlier.add(Self.schema) + let orphan = try earlier.add(#""long""#) + let store = AvroSchemaStore(directory: directory) + let fresh = try store.add(#""string""#) + + store.removeAll(except: [named]) + + #expect(store.storedIDs == [named, fresh]) + #expect(store.schema(id: orphan) == nil) + } + } +} diff --git a/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift b/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift index 8b3d551c94..d55432ed9e 100644 --- a/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift +++ b/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift @@ -235,4 +235,83 @@ struct ValueDisplayFormatStorageTests { #expect(defaults.data(forKey: legacyKey) == nil) #expect(storage.load(for: scope("public", connectionId: other)) == ["id": .json]) } + + @Test("A format 0.83 does not have goes in a second key, so the key 0.83 decodes whole still decodes") + func newFormatsStayOutOfThe083Key() throws { + let (storage, defaults) = try makeStorage() + let target = scope("public", connectionId: UUID()) + let avro = ValueDisplayFormat.avro(schemaID: String(repeating: "a", count: 32)) + storage.save(["a": .json, "b": .igbinary, "c": avro], for: target) + + let data = try #require(defaults.data(forKey: PreferenceKeys.columnDisplayFormats(target).name)) + #expect(try JSONDecoder().decode([String: FormatsIn083].self, from: data) == ["a": .json]) + #expect(storage.load(for: target) == ["a": .json, "b": .igbinary, "c": avro]) + } + + @Test("A format 0.83 sets later wins over the one it could not read") + func formatSetBy083Wins() throws { + let (storage, defaults) = try makeStorage() + let target = scope("public", connectionId: UUID()) + storage.save(["b": .igbinary], for: target) + + defaults.set(try JSONEncoder().encode(["b": FormatsIn083.json]), forKey: PreferenceKeys.columnDisplayFormats(target).name) + + #expect(storage.load(for: target) == ["b": .json]) + } + + @Test("A format name this build does not know is kept through other columns' changes") + func unknownFormatsAreKept() throws { + let (storage, defaults) = try makeStorage() + let service = ValueDisplayFormatService(storage: storage) + let target = scope("public", connectionId: UUID()) + let addedKey = PreferenceKeys.columnDisplayFormats(target).name + ".added" + defaults.set(try JSONEncoder().encode(["x": "formatFromALaterBuild"]), forKey: addedKey) + + service.setOverride(.igbinary, columnKey: "a", scope: target) + service.setOverride(nil, columnKey: "a", scope: target) + + #expect(storage.load(for: target) == [:]) + let kept = try #require(defaults.data(forKey: addedKey)) + #expect(try JSONDecoder().decode([String: String].self, from: kept) == ["x": "formatFromALaterBuild"]) + } + + @Test("A table rename moves the formats in both keys") + func renameTableMovesBothKeys() throws { + let (storage, _) = try makeStorage() + let conn = UUID() + storage.save(["id": .uuid, "payload": .igbinary], for: scope("public", connectionId: conn)) + + storage.renameTable( + from: scope("public", connectionId: conn), + to: scope("public", connectionId: conn, table: "purchases") + ) + + #expect(storage.load(for: scope("public", connectionId: conn)) == nil) + #expect(storage.load(for: scope("public", connectionId: conn, table: "purchases")) == ["id": .uuid, "payload": .igbinary]) + } + + @Test("A schema file goes once no saved format names it") + func schemaFilesFollowTheFormatsThatNameThem() throws { + let defaults = try pool.make().defaults + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + let earlier = AvroSchemaStore(directory: directory) + let kept = try earlier.add(#"{"type":"array","items":"int"}"#) + let orphan = try earlier.add(#"{"type":"array","items":"long"}"#) + let schemas = AvroSchemaStore(directory: directory) + let storage = ValueDisplayFormatStorage(defaults: defaults, schemas: schemas) + let target = scope("public", connectionId: UUID()) + + storage.save(["payload": .avro(schemaID: kept)], for: target) + #expect(schemas.storedIDs == [kept]) + + storage.dropTable(target) + #expect(schemas.storedIDs.isEmpty) + #expect(orphan != kept) + } +} + +// What 0.83.1 decodes a table's formats as: a String enum of the names it shipped. +private enum FormatsIn083: String, Codable { + case raw, text, uuid, uuidSwappedTime, unixTimestamp, unixTimestampMillis, json, phpSerialized, messagePack, protobuf } diff --git a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift index 41e006c2fd..32433b8c54 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift @@ -4,6 +4,7 @@ // import AppKit +import CryptoKit import Foundation @testable import TablePro import TableProPluginKit @@ -108,6 +109,22 @@ struct DataGridDisplayCacheTests { #expect(grid.text(row: 0) == Self.rawBinaryText) } + @Test("A schema chosen again after this Mac lost it redraws the cells it had left as hex") + func redisplayAfterTheSchemaReturns() throws { + let grid = DisplayGrid(Self.binaryRows) + let column = try #require(grid.model.columns.column(at: 0)) + let text = #"{"type":"fixed","name":"F"# + UUID().uuidString.filter(\.isLetter) + #"","size":16}"# + let schemaID = SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() + grid.model.setDisplayFormat(.avro(schemaID: schemaID), for: column) + #expect(grid.text(row: 0) == Self.rawBinaryText) + + #expect(try AvroSchemaStore.shared.add(text) == schemaID) + #expect(grid.text(row: 0) == Self.rawBinaryText) + + grid.model.redisplay(column) + #expect(grid.text(row: 0) == "\"" + Self.rawBinaryText + "\"") + } + @Test("A display format is kept in the grid's store, where its owner reads it") func displayFormatIsWrittenToTheStore() throws { let grid = DisplayGrid(Self.binaryRows) diff --git a/apple/TableProTests/Views/Results/CellInteractionResolverTests.swift b/apple/TableProTests/Views/Results/CellInteractionResolverTests.swift index 2629294ba0..2475fe52f5 100644 --- a/apple/TableProTests/Views/Results/CellInteractionResolverTests.swift +++ b/apple/TableProTests/Views/Results/CellInteractionResolverTests.swift @@ -238,6 +238,8 @@ struct CellInteractionResolverBinaryTests { arguments: [ (ValueDisplayFormat.messagePack, BinaryValueFormat.messagePack), (.protobuf, .protobuf), + (.igbinary, .igbinary), + (.avro(schemaID: String(repeating: "c", count: 32)), .avro(schemaID: String(repeating: "c", count: 32))), ] ) func decoderFormatOpensDecodedViewer(format: ValueDisplayFormat, expected: BinaryValueFormat) { diff --git a/apple/TableProTests/Views/Shared/FieldEditorResolverTests.swift b/apple/TableProTests/Views/Shared/FieldEditorResolverTests.swift index 2bce527453..e42d85d1df 100644 --- a/apple/TableProTests/Views/Shared/FieldEditorResolverTests.swift +++ b/apple/TableProTests/Views/Shared/FieldEditorResolverTests.swift @@ -8,7 +8,7 @@ import Foundation @testable import TablePro import Testing -/// What a binary cell is handed to the inspector as: one character per stored byte. +// What a binary cell is handed to the inspector as: one character per stored byte. private func binaryText(hex: String) -> String { var bytes: [UInt8] = [] var rest = Substring(hex) @@ -23,9 +23,9 @@ private enum GeometrySample { static let wkbPoint = "0101000000000000000000F03F0000000000000040" static let ewkbPoint = "0101000020E6100000000000000000F03F0000000000000040" static let geoJsonPoint = #"{"type":"Point","coordinates":[-122.4194,37.7749]}"# - /// A GeoPackage blob: the `GP` header and SRS id ahead of the same WKB point. + // A GeoPackage blob: the `GP` header and SRS id ahead of the same WKB point. static let geoPackagePoint = "47500001E6100000" + wkbPoint - /// A SpatiaLite blob: start byte, byte order, SRID, bounding box, then the point and an end byte. + // A SpatiaLite blob: start byte, byte order, SRID, bounding box, then the point and an end byte. static let spatiaLitePoint = "0001E6100000" + "000000000000F03F0000000000000040000000000000F03F0000000000000040" + "7C01000000000000000000F03F0000000000000040FE" @@ -65,14 +65,21 @@ struct FieldEditorResolverTests { @Test("A binary decoder format on a binary column opens the decoded viewer") func binaryDecoderFormat() { - let kind = FieldEditorResolver.resolve( - for: .blob(rawType: "BLOB"), - isLongText: false, - originalValue: "\u{82}", - displayFormatOverride: .protobuf, - isBinaryValue: true - ) - #expect(kind == .binaryValue(.protobuf)) + let schemaID = String(repeating: "b", count: 32) + let formats: [(ValueDisplayFormat, BinaryValueFormat)] = [ + (.protobuf, .protobuf), (.messagePack, .messagePack), (.igbinary, .igbinary), + (.avro(schemaID: schemaID), .avro(schemaID: schemaID)), + ] + for (format, expected) in formats { + let kind = FieldEditorResolver.resolve( + for: .blob(rawType: "BLOB"), + isLongText: false, + originalValue: "\u{82}", + displayFormatOverride: format, + isBinaryValue: true + ) + #expect(kind == .binaryValue(expected), "\(format)") + } let null = FieldEditorResolver.resolve( for: .blob(rawType: "BLOB"), isLongText: false, @@ -295,15 +302,15 @@ struct FieldEditorResolverTests { } } - /// PostgreSQL hands the raw EWKB hex over as text when its `ST_AsEWKT` rewrite fails. + // PostgreSQL hands the raw EWKB hex over as text when its `ST_AsEWKT` rewrite fails. @Test("EWKB hex text in a spatial column resolves to the geometry field") func spatialEwkbHexTextResolvesToGeometry() { #expect(resolveSpatial(GeometrySample.ewkbPoint) == geometry(.multiLine, .spatialColumn)) #expect(resolveSpatial(GeometrySample.wkbPoint) == geometry(.multiLine, .spatialColumn)) } - /// The JSON rule used to answer first, so a spatial column holding GeoJSON was a JSON field - /// and nothing downstream knew it was a geometry. + // The JSON rule used to answer first, so a spatial column holding GeoJSON was a JSON field + // and nothing downstream knew it was a geometry. @Test("JSON-shaped geometry in a spatial column is a geometry field over the JSON editor") func spatialJsonShapedValueKeepsTheJsonEditor() { let values = [ @@ -329,7 +336,7 @@ struct FieldEditorResolverTests { } } - /// SQL Server, Oracle and Teradata send text no reader takes, and MySQL falls back to `0x` hex. + // SQL Server, Oracle and Teradata send text no reader takes, and MySQL falls back to `0x` hex. @Test("Spatial text no reader takes keeps the editor it had") func unreadableSpatialTextKeepsItsEditor() { #expect(resolveSpatial("0xE61000000101000000000000000000F03F0000000000000040") == .singleLine) @@ -350,8 +357,8 @@ struct FieldEditorResolverTests { } } - /// WKB type 8 is CIRCULARSTRING and 10 is CURVEPOLYGON. The reader names the type from the - /// header, so the Map segment can say which one it cannot draw, as it does for text. + // WKB type 8 is CIRCULARSTRING and 10 is CURVEPOLYGON. The reader names the type from the + // header, so the Map segment can say which one it cannot draw, as it does for text. @Test("WKB of a type the map cannot draw is still a geometry field over the hex editor") func unsupportedBinaryTypeIsStillGeometry() { let zero = "0000000000000000" @@ -370,8 +377,8 @@ struct FieldEditorResolverTests { } } - /// The field was a text field over the Latin-1 spelling of the bytes, where one keystroke - /// staged that string over the blob. + // The field was a text field over the Latin-1 spelling of the bytes, where one keystroke + // staged that string over the blob. @Test("Bytes in a spatial column that are not WKB get the hex editor, never a text field") func unreadableBinarySpatialIsHex() { let blobs = [ @@ -388,8 +395,8 @@ struct FieldEditorResolverTests { #expect(resolveSpatial(nil, isBinaryValue: true) == .blobHex) } - /// The text readers never see a blob: its bytes can spell WKT, and the hex of a short one is a - /// valid geohash. + // The text readers never see a blob: its bytes can spell WKT, and the hex of a short one is a + // valid geohash. @Test("A binary cell is read as bytes, not as the text its bytes spell") func binarySpatialIsNotReadAsText() { #expect(resolveSpatial("POINT(1 2)", isBinaryValue: true) == .blobHex) @@ -421,8 +428,8 @@ struct FieldEditorResolverTests { } } - /// The sniffing reader takes each of these for a point. A JSON column is held to GeoJSON, or - /// every pair of numbers and every document with `lat` and `lon` would open on a map. + // The sniffing reader takes each of these for a point. A JSON column is held to GeoJSON, or + // every pair of numbers and every document with `lat` and `lon` would open on a map. @Test("JSON that is not GeoJSON stays a JSON field") func jsonThatIsNotGeoJsonStaysJson() { let values = [ @@ -487,8 +494,8 @@ struct FieldEditorResolverTests { // MARK: - Large geometry values - /// No reader accepts this after any opening, so a large value that still resolves to a - /// geometry field was classified by how it opens, not read in full. + // No reader accepts this after any opening, so a large value that still resolves to a + // geometry field was classified by how it opens, not read in full. private static let unreadableTail = String(repeating: "x", count: 2_000_000) @Test("A large spatial value is classified by how it opens") @@ -598,8 +605,8 @@ struct FieldEditorResolverImageTests { #expect(kind == .image(.svg)) } - /// The image branch runs before the blob branch, or a PNG in a BLOB column would only ever be - /// a hex dump in the inspector while the grid popover drew it. + // The image branch runs before the blob branch, or a PNG in a BLOB column would only ever be + // a hex dump in the inspector while the grid popover drew it. @Test("PNG bytes in a blob column resolve to the image editor") func pngBlobResolvesToImage() throws { let value = try #require(String(data: encodedPng(), encoding: .isoLatin1)) @@ -670,11 +677,11 @@ struct FieldEditorResolverImageTests { #expect(kind == .arrayElements(element: .scalar, values: [])) } - /// `jsonb[]` and `jsonb[][]` are one type in PostgreSQL's catalog and any array column may - /// carry a dimension prefix, so the declared type cannot rule either out on a given row. The - /// text editor over the raw literal is the lossless fallback, as it is in the grid, and which - /// text editor is the value's own length talking: both literals here sit under - /// `multiLineValueThreshold`. + // `jsonb[]` and `jsonb[][]` are one type in PostgreSQL's catalog and any array column may + // carry a dimension prefix, so the declared type cannot rule either out on a given row. The + // text editor over the raw literal is the lossless fallback, as it is in the grid, and which + // text editor is the value's own length talking: both literals here sit under + // `multiLineValueThreshold`. @Test("A literal the element list cannot represent falls back to the text editor") func unrepresentableArrayFallsBackToText() { #expect( @@ -693,8 +700,8 @@ struct FieldEditorResolverImageTests { ) } - /// An engine whose list literal is not PostgreSQL's reaches the same gate and fails it, so the - /// classifier's engine-blind `[]` rule cannot hand another driver's value to this editor. + // An engine whose list literal is not PostgreSQL's reaches the same gate and fails it, so the + // classifier's engine-blind `[]` rule cannot hand another driver's value to this editor. @Test("A list literal from another engine does not reach the element editor") func foreignListLiteralFallsBackToText() { let kind = FieldEditorResolver.resolve( @@ -705,9 +712,9 @@ struct FieldEditorResolverImageTests { #expect(kind == .singleLine) } - /// The parse is also what scopes the editor to the engines whose arrays are written this way: - /// the classifier's `[]` rule takes no engine, and a document store's `object[]` classifies the - /// same, so a field with no literal to read must not be offered a list that commits `{…}`. + // The parse is also what scopes the editor to the engines whose arrays are written this way: + // the classifier's `[]` rule takes no engine, and a document store's `object[]` classifies the + // same, so a field with no literal to read must not be offered a list that commits `{…}`. @Test("An array column with no value keeps the plain editor") func absentArrayValueKeepsPlainEditor() { #expect( @@ -720,8 +727,8 @@ struct FieldEditorResolverImageTests { ) } - /// The labels arrive separately in `TableRows.columnEnumValues` and are injected into the type - /// before the inspector resolves it, so they have to reach the element inside the array. + // The labels arrive separately in `TableRows.columnEnumValues` and are injected into the type + // before the inspector resolves it, so they have to reach the element inside the array. @Test("An enum array carries its declared labels into the element editor") func enumArrayCarriesItsLabels() { let labels = ["sad", "ok", "happy"] @@ -734,8 +741,8 @@ struct FieldEditorResolverImageTests { ) } - /// An empty array literal is itself valid JSON, so resolving the array before the JSON branch - /// is what keeps `{}` out of the JSON editor. + // An empty array literal is itself valid JSON, so resolving the array before the JSON branch + // is what keeps `{}` out of the JSON editor. @Test("An empty array opens the element editor rather than the JSON editor") func emptyArrayResolvesToElements() { let kind = FieldEditorResolver.resolve( From b8188d63ba3ee79fc379c1ff2c81a4f130dd7f8c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 06:30:11 +0700 Subject: [PATCH 05/22] fix(display-format): bound Avro reads by values, and refresh a column whose schema comes back --- .../Core/DataGrid/DataGridModel.swift | 9 +++- .../Formatting/BinaryValues/AvroDecoder.swift | 6 +++ .../Formatting/BinaryValues/AvroSchema.swift | 43 ++++++++++++++++--- .../Core/Storage/AvroSchemaStore.swift | 10 ++++- .../DataGridViewController+HeaderMenu.swift | 12 +++--- .../Views/Results/BinaryValueViewerView.swift | 9 +++- .../BinaryValues/AvroDecoderTests.swift | 18 ++++++++ .../DataGrid/DataGridDisplayCacheTests.swift | 6 +++ 8 files changed, 96 insertions(+), 17 deletions(-) diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index e79c9008fc..b9aa905e77 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -273,9 +273,14 @@ final class DataGridModel: ObservableObject { redrawFormatted(column, in: current) } - // The format is unchanged but its decoder now reads what it could not, as when a lost Avro schema is chosen again. + // The format is unchanged but now decodes values it showed as stored bytes, as when a lost Avro schema is chosen + // again. A value filter picked from those bytes follows them to the decoded text. func redisplay(_ column: DataGridColumn) { - redrawFormatted(column, in: rows) + let current = rows + if let remapped = remappedValueFilter(column: column, from: nil, to: displayFormat(for: column), in: current) { + store.valueFilter = remapped + } + redrawFormatted(column, in: current) } private func redrawFormatted(_ column: DataGridColumn, in current: TableRows) { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift index 656d1636e7..eb0d8d0921 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift @@ -23,6 +23,9 @@ internal enum AvroDecoder { static let maxDepth = 256 // Items that take no bytes (null, an empty record) are bounded by this, not by the bytes left. static let maxZeroByteItems: Int64 = 1_000_000 + // Records of records that take no bytes describe exponentially many values in a short schema, so every value + // read counts, built or not. + static let maxValuesRead = 10_000_000 // A single-object header that names this schema's fingerprint is certain, so it goes first. A Confluent // prefix proves nothing (a plain datum can start with zero), so it is only tried once the plain read fails. @@ -75,6 +78,7 @@ internal enum AvroDecoder { private var budget: Int private var isTruncated = false private var zeroByteItems: Int64 = 0 + private var valuesRead = 0 init(reader: BinaryValueReader, schema: AvroSchema, budget: Int) { self.reader = reader @@ -85,6 +89,8 @@ internal enum AvroDecoder { // Only a dispatch: every kind of type is its own function, so the frames a deep value stacks up stay small. mutating func value(_ type: AvroType, depth: Int) throws -> MessagePackValue { guard depth <= AvroDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } + valuesRead += 1 + guard valuesRead <= AvroDecoder.maxValuesRead else { throw reader.error(.tooLarge) } if !isTruncated { budget -= 1 isTruncated = budget < 0 diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift index 409343006b..8a8736e77d 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift @@ -55,6 +55,8 @@ internal struct AvroSchema: Equatable, Sendable { let namedTypes: [String: AvroType] // CRC-64-AVRO of the Parsing Canonical Form, what a single-object encoded value names its writer schema by. let fingerprint: UInt64 + // Named types whose values take no bytes, worked out once: walking a shared type on every check is exponential. + let zeroByteNames: Set static func parse(_ text: String) throws -> AvroSchema { let json: Any @@ -65,7 +67,36 @@ internal struct AvroSchema: Equatable, Sendable { } var parser = Parser() let root = try parser.type(json, namespace: nil) - return AvroSchema(root: root, namedTypes: parser.namedTypes, fingerprint: fingerprint(of: canonicalForm(root))) + return AvroSchema( + root: root, + namedTypes: parser.namedTypes, + fingerprint: fingerprint(of: canonicalForm(root)), + zeroByteNames: zeroByteNames(parser.namedTypes) + ) + } + + private static func zeroByteNames(_ namedTypes: [String: AvroType]) -> Set { + var known: [String: Bool] = [:] + func takesNoBytes(_ type: AvroType, visiting: Set) -> Bool { + switch type { + case .null: return true + case .fixed(_, let size): return size == 0 + case .logical(_, let base): return takesNoBytes(base, visiting: visiting) + case .named(let name): + if let answer = known[name] { return answer } + guard !visiting.contains(name), let defined = namedTypes[name] else { return false } + let answer = takesNoBytes(defined, visiting: visiting.union([name])) + known[name] = answer + return answer + case .record(let name, let fields): + if let answer = known[name] { return answer } + let answer = fields.allSatisfy { takesNoBytes($0.type, visiting: visiting.union([name])) } + known[name] = answer + return answer + default: return false + } + } + return Set(namedTypes.keys.filter { takesNoBytes(.named($0), visiting: []) }) } // https://avro.apache.org/docs/1.12.0/specification/#parsing-canonical-form-for-schemas @@ -140,14 +171,12 @@ internal struct AvroSchema: Equatable, Sendable { } // Items of these types take no bytes, so an array of them can hold more items than bytes remain. - func takesNoBytes(_ type: AvroType, visiting: Set = []) -> Bool { - switch resolve(type) { + func takesNoBytes(_ type: AvroType) -> Bool { + switch type { case .null: return true case .fixed(_, let size): return size == 0 - case .logical(_, let base): return takesNoBytes(base, visiting: visiting) - case .record(let name, let fields): - guard !visiting.contains(name) else { return false } - return fields.allSatisfy { takesNoBytes($0.type, visiting: visiting.union([name])) } + case .logical(_, let base): return takesNoBytes(base) + case .named(let name), .record(let name, _): return zeroByteNames.contains(name) default: return false } } diff --git a/apple/TablePro/Core/Storage/AvroSchemaStore.swift b/apple/TablePro/Core/Storage/AvroSchemaStore.swift index 40a3806092..52a324255c 100644 --- a/apple/TablePro/Core/Storage/AvroSchemaStore.swift +++ b/apple/TablePro/Core/Storage/AvroSchemaStore.swift @@ -18,6 +18,9 @@ internal final class AvroSchemaStore: Sendable { // Far past any real schema; a file this large is the wrong file. static let maxFileBytes = 4_000_000 + // Posted after a schema is kept, so a viewer that found it missing decodes again. + static let didAddSchema = Notification.Name("com.TablePro.avroSchemaStore.didAddSchema") + private static let fileExtension = "avsc" private struct State { @@ -37,10 +40,14 @@ internal final class AvroSchemaStore: Sendable { text.utf8.count == 32 && text.utf8.allSatisfy { (0x30...0x39).contains($0) || (0x61...0x66).contains($0) } } + static func schemaID(of text: String) -> String { + SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() + } + // Parses first, so nothing is kept for a file that is not a schema. func add(_ text: String) throws -> String { let schema = try AvroSchema.parse(text) - let id = SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() + let id = Self.schemaID(of: text) let url = fileURL(id) if !FileManager.default.fileExists(atPath: url.path) { try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) @@ -50,6 +57,7 @@ internal final class AvroSchemaStore: Sendable { $0.parsed[id] = schema $0.addedThisLaunch.insert(id) } + NotificationCenter.default.post(name: Self.didAddSchema, object: self) return id } diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift index 77e0b6d010..ab52ce1999 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift @@ -259,19 +259,21 @@ extension DataGridViewController { guard let columnID = menuColumn(sender)?.id, let window = view.window else { return } Task { @MainActor [weak self] in guard let text = await AvroSchemaFilePanel.chooseSchemaText(in: window) else { return } - let format: ValueDisplayFormat + let schemaID = AvroSchemaStore.schemaID(of: text) + let format = ValueDisplayFormat.avro(schemaID: schemaID) + let wasMissing = AvroSchemaStore.shared.schema(id: schemaID) == nil do { - format = .avro(schemaID: try AvroSchemaStore.shared.add(text)) + _ = try AvroSchemaStore.shared.add(text) } catch { NSAlert(error: error).beginSheetModal(for: window, completionHandler: nil) return } guard let self else { return } - if let column = model.columns.column(id: columnID), model.displayFormat(for: column) == format { - model.redisplay(column) - } else { + guard let column = model.columns.column(id: columnID), model.displayFormat(for: column) == format else { setDisplayFormat(format, columnID: columnID) + return } + if wasMissing { model.redisplay(column) } } } diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index 54ecbf4d2a..0ad20e797a 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -23,6 +23,7 @@ internal struct BinaryValueViewerView: View { @State private var hexDump = "" @State private var decodedKey: TaskKey? @State private var searchText = "" + @State private var schemaRevision = 0 init(data: Data, format: BinaryValueFormat, onPopOut: (() -> Void)? = nil) { self.data = data @@ -37,14 +38,18 @@ internal struct BinaryValueViewerView: View { content .frame(maxWidth: .infinity, maxHeight: .infinity) } - .task(id: TaskKey(data: data, format: format)) { + .task(id: TaskKey(data: data, format: format, schemaRevision: schemaRevision)) { await decode() } + .onReceive(NotificationCenter.default.publisher(for: AvroSchemaStore.didAddSchema)) { _ in + if case .missingSchema? = document { schemaRevision += 1 } + } } private struct TaskKey: Equatable { let data: Data let format: BinaryValueFormat + let schemaRevision: Int } // MARK: - Toolbar @@ -189,7 +194,7 @@ internal struct BinaryValueViewerView: View { // A pane added back restarts the task with the same key. private func decode() async { - let key = TaskKey(data: data, format: format) + let key = TaskKey(data: data, format: format, schemaRevision: schemaRevision) guard decodedKey != key else { return } let data = data let format = format diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift index 09dd7d8d9a..f3c4d267c7 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift @@ -178,6 +178,24 @@ struct AvroDecoderTests { #expect(message.value == .binary(Data([0xDE, 0xAD]))) } + @Test("Records that take no bytes, nested two to a level, are refused rather than walked exponentially") + func zeroByteRecordsDoNotExpand() throws { + var definitions = [#"{"type":"record","name":"R0","fields":[]}"#] + for level in 1...30 { + definitions.append(#"{"type":"record","name":"R"# + "\(level)" + #"","fields":[{"name":"a","type":"R"# + + "\(level - 1)" + #""},{"name":"b","type":"R"# + "\(level - 1)" + #""}]}"#) + } + let fields = definitions.enumerated().map { #"{"name":"f"# + "\($0.offset)" + #"","type":"# + $0.element + "}" } + let holder = #"{"type":"record","name":"Holder","fields":["# + fields.dropLast().joined(separator: ",") + "]}" + let schema = try AvroSchema.parse(#"{"type":"array","items":{"type":"record","name":"Wrap","fields":[{"name":"h","type":"# + + holder + #"},{"name":"r","type":"# + definitions[30] + "}]}}") + #expect(try AvroDecoder.decode(Data([0x00]), schema: schema).value == .array([])) + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(Data([0x02, 0x00]), schema: schema) + } + #expect(error?.kind == .tooLarge) + } + @Test("A huge zero-byte block after a small one is refused, not added up until it overflows") func zeroByteCountOverflow() throws { let nulls = try AvroSchema.parse(#"{"type":"array","items":"null"}"#) diff --git a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift index 32433b8c54..f54d02bbb8 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift @@ -121,8 +121,14 @@ struct DataGridDisplayCacheTests { #expect(try AvroSchemaStore.shared.add(text) == schemaID) #expect(grid.text(row: 0) == Self.rawBinaryText) + var filter = GridValueFilterState() + filter.set(ColumnValueFilter(selectedValues: [Self.rawBinaryText], includesNull: false), columnName: "id", forColumn: 0) + grid.model.setValueFilter(filter) + grid.model.redisplay(column) #expect(grid.text(row: 0) == "\"" + Self.rawBinaryText + "\"") + #expect(grid.model.store.valueFilter.filter(forColumn: 0)?.selectedValues == ["\"" + Self.rawBinaryText + "\""]) + #expect(grid.model.rowCount == 1) } @Test("A display format is kept in the grid's store, where its owner reads it") From 8f955220b577f2ce9b6da04b833e4f306bdcbd92 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 06:42:35 +0700 Subject: [PATCH 06/22] fix(display-format): restore a lost Avro schema in every grid, keep binary Serializable payloads --- .../Core/DataGrid/DataGridModel.swift | 43 +++++++++++++------ .../Formatting/BinaryValues/AvroDecoder.swift | 18 ++++++-- .../BinaryValues/IgbinaryDecoder.swift | 5 ++- .../BinaryValues/PhpSerializeWriter.swift | 2 + .../Formatting/PhpSerializeParser.swift | 4 ++ .../Core/Storage/AvroSchemaStore.swift | 23 +++++++--- apple/TablePro/Models/UI/PhpTreeNode.swift | 8 ++++ .../DataGridViewController+HeaderMenu.swift | 13 +----- .../Views/Results/BinaryValueViewerView.swift | 2 +- .../BinaryValues/AvroDecoderTests.swift | 2 + .../BinaryValues/IgbinaryDecoderTests.swift | 7 +++ .../DataGrid/DataGridDisplayCacheTests.swift | 20 ++++----- 12 files changed, 99 insertions(+), 48 deletions(-) diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index b9aa905e77..c474e53dd6 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -56,6 +56,7 @@ final class DataGridModel: ObservableObject { private(set) weak var mountedView: (any DataGridMountedView)? private var orderRevision = 0 + private var schemaReturned: AnyCancellable? init( store: any DataGridStateStore = InMemoryDataGridStateStore(), @@ -67,6 +68,11 @@ final class DataGridModel: ObservableObject { self.configuration = configuration self.themeScope = themeScope self.selectionSummary = selectionSummary + // The model owns the cached text, so a grid that is not on screen catches up too. + schemaReturned = NotificationCenter.default.publisher(for: AvroSchemaStore.didRestoreSchema) + .compactMap { $0.userInfo?[AvroSchemaStore.schemaIDKey] as? String } + .receive(on: RunLoop.main) + .sink { [weak self] schemaID in self?.schemaDidReturn(schemaID) } } // MARK: - Rows @@ -270,20 +276,32 @@ final class DataGridModel: ObservableObject { store.valueFilter = remapped } store.displayFormats = formats - redrawFormatted(column, in: current) - } - - // The format is unchanged but now decodes values it showed as stored bytes, as when a lost Avro schema is chosen - // again. A value filter picked from those bytes follows them to the decoded text. - func redisplay(_ column: DataGridColumn) { - let current = rows - if let remapped = remappedValueFilter(column: column, from: nil, to: displayFormat(for: column), in: current) { - store.valueFilter = remapped + redrawFormatted([column], in: current) + } + + // Columns that showed the stored bytes for want of this schema now decode, the way a format change does: a value + // filter picked from those bytes follows them, behind the same confirmation. + private func schemaDidReturn(_ schemaID: String) { + let affected = columns.columns.filter { displayFormat(for: $0) == .avro(schemaID: schemaID) } + guard !affected.isEmpty else { return } + let apply: @MainActor () -> Void = { [weak self] in + guard let self else { return } + let current = rows + for column in affected { + if let remapped = remappedValueFilter(column: column, from: nil, to: displayFormat(for: column), in: current) { + store.valueFilter = remapped + } + } + redrawFormatted(affected, in: current) + } + guard store.valueFilter.isActive, let actions else { + apply() + return } - redrawFormatted(column, in: current) + actions.gridConfirmDisplayOrderChange(then: apply) } - private func redrawFormatted(_ column: DataGridColumn, in current: TableRows) { + private func redrawFormatted(_ changed: [DataGridColumn], in current: TableRows) { display.cache.removeAll() if store.valueFilter.isActive { order = resolveOrder(in: current) @@ -291,7 +309,8 @@ final class DataGridModel: ObservableObject { displayRevision &+= 1 mountedView?.apply(.reload) } else { - mountedView?.apply(.rowsUpdated(IndexSet(0.. MessagePackValue? { + switch schema.resolve(schema.root) { + case .bytes: + return .binary(body) + case .logical(let logical, let base): + guard case .bytes = schema.resolve(base) else { return nil } + return AvroLogicalRendering.render(.binary(body), as: logical) + default: + return nil + } + } + private static func singleObjectFingerprint(_ data: Data) -> UInt64? { guard data.count >= 10, data.starts(with: [0xC3, 0x01]) else { return nil } var header = BinaryValueReader(data) diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift index 1c39235faa..ab56f8858a 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift @@ -148,7 +148,7 @@ internal enum IgbinaryDecoder { if case .reference = target.value { return target } let copy = Copy.of(.reference(id: own), slots: 0) switch target.value { - case .array, .object, .serializable, .enumCase: break + case .array, .object, .serializable, .binarySerializable, .enumCase: break default: references.append(Reference(slot: own, copy: copy)) } return Decoded(value: target.value, copy: copy) @@ -216,7 +216,8 @@ internal enum IgbinaryDecoder { value = .object(className: className, properties: properties) case .objectSer8, .objectSer16, .objectSer32: let payload = try reader.readBytes(try length(tag)) - value = .serializable(className: className, rawPayload: Self.text(Self.phpString(payload))) + value = String(data: payload, encoding: .utf8).map { .serializable(className: className, rawPayload: $0) } + ?? .binarySerializable(className: className, payload: payload) case .enumCase: let caseTag = try readTag() switch caseTag { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift index 214af58e58..0ae14d1fb0 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift @@ -51,6 +51,8 @@ internal enum PhpSerializeWriter { output += "}" case .serializable(let className, let rawPayload): output += quoted("C", className) + ":\(rawPayload.utf8.count):{\(rawPayload)}" + case .binarySerializable(let className, let payload): + output += quoted("C", className) + ":\(payload.count):{\(String(data: payload, encoding: .isoLatin1) ?? "")}" case .enumCase(let className, let caseName): output += quoted("E", "\(className):\(caseName)") + ";" case .reference(let id): diff --git a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift index ba89bf7a82..0e4166a99d 100644 --- a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift +++ b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift @@ -16,6 +16,8 @@ internal indirect enum PhpValue: Equatable { case array([PhpKeyValue]) case object(className: String, properties: [PhpProperty]) case serializable(className: String, rawPayload: String) + // A Serializable payload that is not UTF-8, which igbinary hands over as it is. + case binarySerializable(className: String, payload: Data) case enumCase(className: String, caseName: String) // `R:`, a PHP reference to the value numbered `id`. case reference(id: Int) @@ -40,6 +42,8 @@ internal indirect enum PhpValue: Equatable { return aName == bName && aProps == bProps case (.serializable(let aName, let aPayload), .serializable(let bName, let bPayload)): return aName == bName && aPayload == bPayload + case (.binarySerializable(let aName, let aPayload), .binarySerializable(let bName, let bPayload)): + return aName == bName && aPayload == bPayload case (.enumCase(let aName, let aCase), .enumCase(let bName, let bCase)): return aName == bName && aCase == bCase case (.reference(let a), .reference(let b)): return a == b diff --git a/apple/TablePro/Core/Storage/AvroSchemaStore.swift b/apple/TablePro/Core/Storage/AvroSchemaStore.swift index 52a324255c..85c2abd4e3 100644 --- a/apple/TablePro/Core/Storage/AvroSchemaStore.swift +++ b/apple/TablePro/Core/Storage/AvroSchemaStore.swift @@ -18,8 +18,9 @@ internal final class AvroSchemaStore: Sendable { // Far past any real schema; a file this large is the wrong file. static let maxFileBytes = 4_000_000 - // Posted after a schema is kept, so a viewer that found it missing decodes again. - static let didAddSchema = Notification.Name("com.TablePro.avroSchemaStore.didAddSchema") + // Posted when a schema this Mac did not have is kept, so grids and viewers that found it missing decode again. + static let didRestoreSchema = Notification.Name("com.TablePro.avroSchemaStore.didRestoreSchema") + static let schemaIDKey = "schemaID" private static let fileExtension = "avsc" @@ -27,6 +28,8 @@ internal final class AvroSchemaStore: Sendable { var parsed: [String: AvroSchema] = [:] // Kept from a sweep: a schema can be added before the column's format is saved. var addedThisLaunch: Set = [] + // Asked for and not found, so adding one later is a restore that grids and viewers have to hear about. + var missing: Set = [] } private let directory: URL @@ -53,11 +56,14 @@ internal final class AvroSchemaStore: Sendable { try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) try Data(text.utf8).write(to: url, options: .atomic) } - state.withLock { - $0.parsed[id] = schema - $0.addedThisLaunch.insert(id) + let wasMissing = state.withLock { state -> Bool in + state.parsed[id] = schema + state.addedThisLaunch.insert(id) + return state.missing.remove(id) != nil + } + if wasMissing { + NotificationCenter.default.post(name: Self.didRestoreSchema, object: self, userInfo: [Self.schemaIDKey: id]) } - NotificationCenter.default.post(name: Self.didAddSchema, object: self) return id } @@ -66,7 +72,10 @@ internal final class AvroSchemaStore: Sendable { guard Self.isSchemaID(id), let data = try? Data(contentsOf: fileURL(id)), let text = String(data: data, encoding: .utf8), - let schema = try? AvroSchema.parse(text) else { return nil } + let schema = try? AvroSchema.parse(text) else { + state.withLock { _ = $0.missing.insert(id) } + return nil + } state.withLock { $0.parsed[id] = schema } return schema } diff --git a/apple/TablePro/Models/UI/PhpTreeNode.swift b/apple/TablePro/Models/UI/PhpTreeNode.swift index 1c11c256ef..14e168c3c5 100644 --- a/apple/TablePro/Models/UI/PhpTreeNode.swift +++ b/apple/TablePro/Models/UI/PhpTreeNode.swift @@ -264,6 +264,14 @@ internal enum PhpTreeBuilder { className: className, visibilityBadge: badge, children: children ) + case .binarySerializable(let className, let payload): + let hex = "0x" + payload.uppercaseHex() + return PhpTreeNode( + key: key, keyPath: keyPath, path: path, nodeType: .serializable, + displayValue: serializableDisplay(className: className, payload: hex), + rawValue: hex, className: className, visibilityBadge: badge + ) + case .serializable(let className, let rawPayload): return PhpTreeNode( key: key, keyPath: keyPath, path: path, nodeType: .serializable, diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift index ab52ce1999..09786a3302 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift @@ -259,21 +259,12 @@ extension DataGridViewController { guard let columnID = menuColumn(sender)?.id, let window = view.window else { return } Task { @MainActor [weak self] in guard let text = await AvroSchemaFilePanel.chooseSchemaText(in: window) else { return } - let schemaID = AvroSchemaStore.schemaID(of: text) - let format = ValueDisplayFormat.avro(schemaID: schemaID) - let wasMissing = AvroSchemaStore.shared.schema(id: schemaID) == nil + // A schema this Mac had lost redraws every grid that names it through AvroSchemaStore.didRestoreSchema. do { - _ = try AvroSchemaStore.shared.add(text) + self?.setDisplayFormat(.avro(schemaID: try AvroSchemaStore.shared.add(text)), columnID: columnID) } catch { NSAlert(error: error).beginSheetModal(for: window, completionHandler: nil) - return } - guard let self else { return } - guard let column = model.columns.column(id: columnID), model.displayFormat(for: column) == format else { - setDisplayFormat(format, columnID: columnID) - return - } - if wasMissing { model.redisplay(column) } } } diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index 0ad20e797a..8f680f533d 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -41,7 +41,7 @@ internal struct BinaryValueViewerView: View { .task(id: TaskKey(data: data, format: format, schemaRevision: schemaRevision)) { await decode() } - .onReceive(NotificationCenter.default.publisher(for: AvroSchemaStore.didAddSchema)) { _ in + .onReceive(NotificationCenter.default.publisher(for: AvroSchemaStore.didRestoreSchema)) { _ in if case .missingSchema? = document { schemaRevision += 1 } } } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift index f3c4d267c7..b150766fa1 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift @@ -176,6 +176,8 @@ struct AvroDecoderTests { let message = try AvroDecoder.decode(BinaryFixture.data("0000000007dead"), schema: try AvroSchema.parse(#""bytes""#)) #expect(message.framing == .confluent(schemaID: 7)) #expect(message.value == .binary(Data([0xDE, 0xAD]))) + let decimal = try AvroSchema.parse(#"{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}"#) + #expect(try AvroDecoder.decode(BinaryFixture.data("000000000704d2"), schema: decimal).value == .string("12.34")) } @Test("Records that take no bytes, nested two to a level, are refused rather than walked exponentially") diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift index d78b69fbc8..4fcd244143 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift @@ -159,6 +159,13 @@ struct IgbinaryDecoderTests { ) } + @Test("A Serializable payload that is not UTF-8 keeps its bytes and its byte count") + func binarySerializablePayload() throws { + let value = try IgbinaryDecoder.decode(BinaryFixture.data("0000000217034f6c641d01ff")) + #expect(value == .binarySerializable(className: "Old", payload: Data([0xFF]))) + #expect(PhpSerializeWriter.text(value) == #"C:3:"Old":1:{"# + "\u{FF}}") + } + @Test("A string of 300 bytes uses the 16-bit length") func longString() throws { let hex = "0000000212012c" + String(repeating: "61", count: 300) diff --git a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift index f54d02bbb8..3d928da283 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift @@ -4,7 +4,6 @@ // import AppKit -import CryptoKit import Foundation @testable import TablePro import TableProPluginKit @@ -109,25 +108,22 @@ struct DataGridDisplayCacheTests { #expect(grid.text(row: 0) == Self.rawBinaryText) } - @Test("A schema chosen again after this Mac lost it redraws the cells it had left as hex") - func redisplayAfterTheSchemaReturns() throws { + @Test("A schema this Mac lost, chosen again, redraws the cells it left as hex and carries their value filter over") + func schemaReturnsToTheGrid() async throws { let grid = DisplayGrid(Self.binaryRows) let column = try #require(grid.model.columns.column(at: 0)) let text = #"{"type":"fixed","name":"F"# + UUID().uuidString.filter(\.isLetter) + #"","size":16}"# - let schemaID = SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() - grid.model.setDisplayFormat(.avro(schemaID: schemaID), for: column) + grid.model.setDisplayFormat(.avro(schemaID: AvroSchemaStore.schemaID(of: text)), for: column) #expect(grid.text(row: 0) == Self.rawBinaryText) - - #expect(try AvroSchemaStore.shared.add(text) == schemaID) - #expect(grid.text(row: 0) == Self.rawBinaryText) - var filter = GridValueFilterState() filter.set(ColumnValueFilter(selectedValues: [Self.rawBinaryText], includesNull: false), columnName: "id", forColumn: 0) grid.model.setValueFilter(filter) - grid.model.redisplay(column) - #expect(grid.text(row: 0) == "\"" + Self.rawBinaryText + "\"") - #expect(grid.model.store.valueFilter.filter(forColumn: 0)?.selectedValues == ["\"" + Self.rawBinaryText + "\""]) + _ = try AvroSchemaStore.shared.add(text) + + let decoded = "\"" + Self.rawBinaryText + "\"" + try await waitUntil { grid.text(row: 0) == decoded } + #expect(grid.model.store.valueFilter.filter(forColumn: 0)?.selectedValues == [decoded]) #expect(grid.model.rowCount == 1) } From 5a15346dd3afc3b68044df020db50c22016f5473 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 06:54:16 +0700 Subject: [PATCH 07/22] fix(display-format): close the races between a schema lookup, its restore and an open viewer --- .../Core/Storage/AvroSchemaStore.swift | 8 ++++-- .../Views/Results/BinaryValueViewerView.swift | 8 ++++-- .../Core/Storage/AvroSchemaStoreTests.swift | 26 +++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/apple/TablePro/Core/Storage/AvroSchemaStore.swift b/apple/TablePro/Core/Storage/AvroSchemaStore.swift index 85c2abd4e3..3cee030650 100644 --- a/apple/TablePro/Core/Storage/AvroSchemaStore.swift +++ b/apple/TablePro/Core/Storage/AvroSchemaStore.swift @@ -73,8 +73,12 @@ internal final class AvroSchemaStore: Sendable { let data = try? Data(contentsOf: fileURL(id)), let text = String(data: data, encoding: .utf8), let schema = try? AvroSchema.parse(text) else { - state.withLock { _ = $0.missing.insert(id) } - return nil + // An add() may have landed since the read failed; recorded as missing only if it did not. + return state.withLock { state in + if let added = state.parsed[id] { return added } + state.missing.insert(id) + return nil + } } state.withLock { $0.parsed[id] = schema } return schema diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index 8f680f533d..e0e6df95ae 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -3,6 +3,7 @@ // TablePro // +import Combine import SwiftUI import TableProTheme @@ -41,8 +42,11 @@ internal struct BinaryValueViewerView: View { .task(id: TaskKey(data: data, format: format, schemaRevision: schemaRevision)) { await decode() } - .onReceive(NotificationCenter.default.publisher(for: AvroSchemaStore.didRestoreSchema)) { _ in - if case .missingSchema? = document { schemaRevision += 1 } + // Matched on the format, not the document: a decode still running may be about to report it missing. + .onReceive(NotificationCenter.default.publisher(for: AvroSchemaStore.didRestoreSchema).receive(on: RunLoop.main)) { notification in + guard case .avro(let schemaID) = format, + notification.userInfo?[AvroSchemaStore.schemaIDKey] as? String == schemaID else { return } + schemaRevision += 1 } } diff --git a/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift index e37cc292bc..65749dc6e9 100644 --- a/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift +++ b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift @@ -4,6 +4,7 @@ // import Foundation +import os @testable import TablePro import Testing @@ -49,6 +50,31 @@ struct AvroSchemaStoreTests { } } + @Test("Adding a schema that was looked up and missing announces it, and only then") + func restoreIsAnnounced() throws { + try withDirectory { directory in + let store = AvroSchemaStore(directory: directory) + let announced = OSAllocatedUnfairLock<[String]>(initialState: []) + let observer = NotificationCenter.default.addObserver( + forName: AvroSchemaStore.didRestoreSchema, object: store, queue: nil + ) { notification in + let schemaID = notification.userInfo?[AvroSchemaStore.schemaIDKey] as? String ?? "" + announced.withLock { $0.append(schemaID) } + } + defer { NotificationCenter.default.removeObserver(observer) } + + _ = try store.add(#""int""#) + #expect(announced.withLock { $0 }.isEmpty) + + let lost = AvroSchemaStore.schemaID(of: Self.schema) + #expect(store.schema(id: lost) == nil) + #expect(try store.add(Self.schema) == lost) + #expect(announced.withLock { $0 } == [lost]) + _ = try store.add(Self.schema) + #expect(announced.withLock { $0 } == [lost]) + } + } + @Test("A sweep keeps what is named and what this launch added") func sweepKeepsNamedAndNew() throws { try withDirectory { directory in From bd05db2c2a5f6fa771aa0ebd2687d29e4375aff4 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 07:03:04 +0700 Subject: [PATCH 08/22] perf(display-format): cut payloads to the preview and cap what an Avro cell reads --- .../Formatting/BinaryValues/AvroDecoder.swift | 27 ++++++++++++------- .../BinaryValues/BinaryValueDecoding.swift | 4 ++- .../BinaryValues/PhpSerializeWriter.swift | 27 ++++++++++++------- .../BinaryValues/AvroDecoderTests.swift | 11 ++++++++ .../BinaryValues/IgbinaryDecoderTests.swift | 10 +++++++ 5 files changed, 59 insertions(+), 20 deletions(-) diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift index e844a94772..fee14447d7 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift @@ -24,19 +24,26 @@ internal enum AvroDecoder { // Items that take no bytes (null, an empty record) are bounded by this, not by the bytes left. static let maxZeroByteItems: Int64 = 1_000_000 // Records of records that take no bytes describe exponentially many values in a short schema, so every value - // read counts, built or not. + // read counts, built or not. A grid cell reads synchronously and stops far sooner, showing hex past that. static let maxValuesRead = 10_000_000 + static let maxValuesReadInCell = 1_000_000 // A single-object header that names this schema's fingerprint is certain, so it goes first. A Confluent // prefix proves nothing (a plain datum can start with zero), so it is only tried once the plain read fails. - static func decode(_ data: Data, schema: AvroSchema, maxValues: Int = .max) throws -> AvroMessage { + static func decode( + _ data: Data, + schema: AvroSchema, + maxValues: Int = .max, + maxValuesRead: Int = AvroDecoder.maxValuesRead + ) throws -> AvroMessage { + let limits = (build: maxValues, read: maxValuesRead) let singleObject = singleObjectFingerprint(data) if let singleObject, singleObject == schema.fingerprint { - let value = try datum(data.dropFirst(10), schema: schema, maxValues: maxValues) + let value = try datum(data.dropFirst(10), schema: schema, limits: limits) return AvroMessage(value: value, framing: .singleObject(fingerprint: singleObject)) } do { - return AvroMessage(value: try datum(data, schema: schema, maxValues: maxValues), framing: nil) + return AvroMessage(value: try datum(data, schema: schema, limits: limits), framing: nil) } catch let plainError as BinaryValueDecodeError { // Avro's own reader refuses a value written with a schema it does not have. if let singleObject { throw BinaryValueDecodeError(kind: .otherSchema(fingerprint: singleObject), offset: 2) } @@ -49,7 +56,7 @@ internal enum AvroDecoder { if let raw = rawBytes(Data(body), schema: schema) { return AvroMessage(value: raw, framing: framing) } - guard let value = try? datum(body, schema: schema, maxValues: maxValues) else { throw plainError } + guard let value = try? datum(body, schema: schema, limits: limits) else { throw plainError } return AvroMessage(value: value, framing: framing) } } @@ -73,8 +80,8 @@ internal enum AvroDecoder { return try? header.readUInt64LittleEndian() } - private static func datum(_ data: Data, schema: AvroSchema, maxValues: Int) throws -> MessagePackValue { - var parser = Parser(reader: BinaryValueReader(Data(data)), schema: schema, budget: maxValues) + private static func datum(_ data: Data, schema: AvroSchema, limits: (build: Int, read: Int)) throws -> MessagePackValue { + var parser = Parser(reader: BinaryValueReader(Data(data)), schema: schema, budget: limits.build, readLimit: limits.read) let value = try parser.value(schema.root, depth: 0) if !parser.reader.isAtEnd { throw parser.reader.error(.trailingBytes(parser.reader.remaining)) @@ -91,18 +98,20 @@ internal enum AvroDecoder { private var isTruncated = false private var zeroByteItems: Int64 = 0 private var valuesRead = 0 + private let readLimit: Int - init(reader: BinaryValueReader, schema: AvroSchema, budget: Int) { + init(reader: BinaryValueReader, schema: AvroSchema, budget: Int, readLimit: Int) { self.reader = reader self.schema = schema self.budget = budget + self.readLimit = readLimit } // Only a dispatch: every kind of type is its own function, so the frames a deep value stacks up stay small. mutating func value(_ type: AvroType, depth: Int) throws -> MessagePackValue { guard depth <= AvroDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } valuesRead += 1 - guard valuesRead <= AvroDecoder.maxValuesRead else { throw reader.error(.tooLarge) } + guard valuesRead <= readLimit else { throw reader.error(.tooLarge) } if !isTruncated { budget -= 1 isTruncated = budget < 0 diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift index 2763e9262f..4bd1fcbc58 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift @@ -59,7 +59,9 @@ internal enum BinaryValueDecoding { text = PhpSerializeWriter.text(try IgbinaryDecoder.decode(data, maxValues: budget), limit: limit) case .avro(let schemaID): guard let schema = AvroSchemaStore.shared.schema(id: schemaID) else { return nil } - let value = try AvroDecoder.decode(data, schema: schema, maxValues: budget).value + let value = try AvroDecoder.decode( + data, schema: schema, maxValues: budget, maxValuesRead: AvroDecoder.maxValuesReadInCell + ).value text = MessagePackTextWriter.json(value, pretty: false, limit: limit) } } catch { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift index 0ae14d1fb0..03bd2f6a0d 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift @@ -13,8 +13,11 @@ internal enum PhpSerializeWriter { return output } + // Text past `limit` is cut before it is copied, so a large string costs what the preview shows of it. The lengths + // stay the full ones, and the caller marks the cut. private static func write(_ value: PhpValue, into output: inout String, limit: Int) { - guard output.utf8.count < limit else { return } + let room = limit - output.utf8.count + guard room > 0 else { return } switch value { case .null: output += "N;" @@ -25,10 +28,10 @@ internal enum PhpSerializeWriter { case .float(let number): output += "d:\(floatText(number));" case .string(let text): - output += quoted("s", text) + ";" + output += quoted("s", text, room: room) + ";" case .binaryString(let bytes): // The length is the byte count serialize() writes; the bytes show one character each. - output += "s:\(bytes.count):\"\(String(data: bytes, encoding: .isoLatin1) ?? "")\";" + output += "s:\(bytes.count):\"\(latin1(bytes.prefix(room)))\";" case .array(let entries): output += "a:\(entries.count):{" for entry in entries { @@ -38,23 +41,23 @@ internal enum PhpSerializeWriter { } output += "}" case .object(let className, let properties): - output += quoted("O", className) + ":\(properties.count):{" + output += quoted("O", className, room: room) + ":\(properties.count):{" for property in properties { guard output.utf8.count < limit else { return } if let index = property.index { output += "i:\(index);" } else { - output += quoted("s", mangled(property)) + ";" + output += quoted("s", mangled(property), room: limit - output.utf8.count) + ";" } write(property.value, into: &output, limit: limit) } output += "}" case .serializable(let className, let rawPayload): - output += quoted("C", className) + ":\(rawPayload.utf8.count):{\(rawPayload)}" + output += quoted("C", className, room: room) + ":\(rawPayload.utf8.count):{\(rawPayload.scalarPrefix(room))}" case .binarySerializable(let className, let payload): - output += quoted("C", className) + ":\(payload.count):{\(String(data: payload, encoding: .isoLatin1) ?? "")}" + output += quoted("C", className, room: room) + ":\(payload.count):{\(latin1(payload.prefix(room)))}" case .enumCase(let className, let caseName): - output += quoted("E", "\(className):\(caseName)") + ";" + output += quoted("E", "\(className):\(caseName)", room: room) + ";" case .reference(let id): output += "R:\(id);" case .objectReference(let id): @@ -66,8 +69,12 @@ internal enum PhpSerializeWriter { } } - private static func quoted(_ token: String, _ text: String) -> String { - "\(token):\(text.utf8.count):\"\(text)\"" + private static func quoted(_ token: String, _ text: String, room: Int) -> String { + "\(token):\(text.utf8.count):\"\(text.scalarPrefix(room))\"" + } + + private static func latin1(_ bytes: Data) -> String { + String(data: bytes, encoding: .isoLatin1) ?? "" } private static func mangled(_ property: PhpProperty) -> String { diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift index b150766fa1..6cf3d956e4 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift @@ -198,6 +198,17 @@ struct AvroDecoderTests { #expect(error?.kind == .tooLarge) } + @Test("A grid cell reads a value only so far, then leaves it to the viewer") + func cellReadLimit() throws { + let booleans = try AvroSchema.parse(#"{"type":"array","items":"boolean"}"#) + let data = Data([0x90, 0x03]) + Data(repeating: 0x01, count: 200) + Data([0x00]) + #expect(throws: Never.self) { try AvroDecoder.decode(data, schema: booleans, maxValues: 10) } + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(data, schema: booleans, maxValues: 10, maxValuesRead: 100) + } + #expect(error?.kind == .tooLarge) + } + @Test("A huge zero-byte block after a small one is refused, not added up until it overflows") func zeroByteCountOverflow() throws { let nulls = try AvroSchema.parse(#"{"type":"array","items":"null"}"#) diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift index 4fcd244143..f053be5993 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift @@ -166,6 +166,16 @@ struct IgbinaryDecoderTests { #expect(PhpSerializeWriter.text(value) == #"C:3:"Old":1:{"# + "\u{FF}}") } + @Test("Text past the limit is cut before it is copied, and its length stays the full one") + func writerCutsLargePayloads() { + let text = PhpSerializeWriter.text(.string(String(repeating: "a", count: 1_000_000)), limit: 100) + #expect(text.hasPrefix(#"s:1000000:"aaa"#)) + #expect(text.utf8.count < 200) + let bytes = PhpSerializeWriter.text(.binarySerializable(className: "Old", payload: Data(count: 1_000_000)), limit: 100) + #expect(bytes.hasPrefix(#"C:3:"Old":1000000:{"#)) + #expect(bytes.utf8.count < 200) + } + @Test("A string of 300 bytes uses the 16-bit length") func longString() throws { let hex = "0000000212012c" + String(repeating: "61", count: 300) From 70ff77d96ca7b3ab12771750e23a7e957e99b1e1 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 07:16:14 +0700 Subject: [PATCH 09/22] fix(display-format): restore a lost schema in the grid it is chosen in, keep byte property names --- .../Core/DataGrid/DataGridModel.swift | 32 ++++----------- .../Formatting/BinaryValues/AvroSchema.swift | 39 +++++++++++-------- .../BinaryValues/IgbinaryDecoder.swift | 6 ++- .../BinaryValues/PhpSerializeWriter.swift | 2 + .../Formatting/PhpSerializeParser.swift | 2 + .../Core/Storage/AvroSchemaStore.swift | 8 ++-- .../DataGridViewController+HeaderMenu.swift | 23 ++++++++++- .../BinaryValues/AvroDecoderTests.swift | 11 ++++++ .../BinaryValues/IgbinaryDecoderTests.swift | 6 +++ .../DataGrid/DataGridDisplayCacheTests.swift | 6 ++- 10 files changed, 86 insertions(+), 49 deletions(-) diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index c474e53dd6..c6004d67f5 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -56,7 +56,6 @@ final class DataGridModel: ObservableObject { private(set) weak var mountedView: (any DataGridMountedView)? private var orderRevision = 0 - private var schemaReturned: AnyCancellable? init( store: any DataGridStateStore = InMemoryDataGridStateStore(), @@ -68,11 +67,6 @@ final class DataGridModel: ObservableObject { self.configuration = configuration self.themeScope = themeScope self.selectionSummary = selectionSummary - // The model owns the cached text, so a grid that is not on screen catches up too. - schemaReturned = NotificationCenter.default.publisher(for: AvroSchemaStore.didRestoreSchema) - .compactMap { $0.userInfo?[AvroSchemaStore.schemaIDKey] as? String } - .receive(on: RunLoop.main) - .sink { [weak self] schemaID in self?.schemaDidReturn(schemaID) } } // MARK: - Rows @@ -279,26 +273,16 @@ final class DataGridModel: ObservableObject { redrawFormatted([column], in: current) } - // Columns that showed the stored bytes for want of this schema now decode, the way a format change does: a value - // filter picked from those bytes follows them, behind the same confirmation. - private func schemaDidReturn(_ schemaID: String) { - let affected = columns.columns.filter { displayFormat(for: $0) == .avro(schemaID: schemaID) } - guard !affected.isEmpty else { return } - let apply: @MainActor () -> Void = { [weak self] in - guard let self else { return } - let current = rows - for column in affected { - if let remapped = remappedValueFilter(column: column, from: nil, to: displayFormat(for: column), in: current) { - store.valueFilter = remapped - } + // Columns whose format is unchanged now decode values they showed as stored bytes, as when a lost Avro schema is + // chosen again: a value filter picked from those bytes follows them, the way a format change carries it. + func redisplay(_ changed: [DataGridColumn]) { + let current = rows + for column in changed { + if let remapped = remappedValueFilter(column: column, from: nil, to: displayFormat(for: column), in: current) { + store.valueFilter = remapped } - redrawFormatted(affected, in: current) - } - guard store.valueFilter.isActive, let actions else { - apply() - return } - actions.gridConfirmDisplayOrderChange(then: apply) + redrawFormatted(changed, in: current) } private func redrawFormatted(_ changed: [DataGridColumn], in current: TableRows) { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift index 8a8736e77d..583dc7fc02 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift @@ -71,32 +71,33 @@ internal struct AvroSchema: Equatable, Sendable { root: root, namedTypes: parser.namedTypes, fingerprint: fingerprint(of: canonicalForm(root)), - zeroByteNames: zeroByteNames(parser.namedTypes) + zeroByteNames: zeroByteNames(parser.completed) ) } - private static func zeroByteNames(_ namedTypes: [String: AvroType]) -> Set { - var known: [String: Bool] = [:] - func takesNoBytes(_ type: AvroType, visiting: Set) -> Bool { + // In the order definitions finish, so every name a type refers to is settled before it, except its own and its + // enclosing types', which a value cannot hold without bytes anyway. No recursion: a schema can chain thousands. + private static func zeroByteNames(_ completed: [(name: String, type: AvroType)]) -> Set { + var names: Set = [] + func takesNoBytes(_ type: AvroType) -> Bool { switch type { case .null: return true case .fixed(_, let size): return size == 0 - case .logical(_, let base): return takesNoBytes(base, visiting: visiting) - case .named(let name): - if let answer = known[name] { return answer } - guard !visiting.contains(name), let defined = namedTypes[name] else { return false } - let answer = takesNoBytes(defined, visiting: visiting.union([name])) - known[name] = answer - return answer - case .record(let name, let fields): - if let answer = known[name] { return answer } - let answer = fields.allSatisfy { takesNoBytes($0.type, visiting: visiting.union([name])) } - known[name] = answer - return answer + case .named(let name), .record(let name, _), .enumeration(let name, _): return names.contains(name) + case .logical(_, let base): return takesNoBytes(base) default: return false } } - return Set(namedTypes.keys.filter { takesNoBytes(.named($0), visiting: []) }) + for (name, type) in completed { + let base: AvroType + if case .logical(_, let inner) = type { base = inner } else { base = type } + switch base { + case .record(_, let fields) where fields.allSatisfy({ takesNoBytes($0.type) }): names.insert(name) + case .fixed(_, 0): names.insert(name) + default: break + } + } + return names } // https://avro.apache.org/docs/1.12.0/specification/#parsing-canonical-form-for-schemas @@ -183,6 +184,7 @@ internal struct AvroSchema: Equatable, Sendable { private struct Parser { var namedTypes: [String: AvroType] = [:] + private(set) var completed: [(name: String, type: AvroType)] = [] mutating func type(_ json: Any, namespace: String?) throws -> AvroType { if let name = json as? String { @@ -224,6 +226,7 @@ internal struct AvroSchema: Equatable, Sendable { } let record = AvroType.record(name: name, fields: fields) namedTypes[name] = record + completed.append((name, record)) return record case "enum": let name = try newName(object, namespace: namespace) @@ -232,6 +235,7 @@ internal struct AvroSchema: Equatable, Sendable { } let enumeration = AvroType.enumeration(name: name, symbols: symbols) namedTypes[name] = enumeration + completed.append((name, enumeration)) return enumeration case "fixed": let name = try newName(object, namespace: namespace) @@ -240,6 +244,7 @@ internal struct AvroSchema: Equatable, Sendable { } let fixed = AvroType.fixed(name: name, size: size) namedTypes[name] = fixed + completed.append((name, fixed)) return fixed case "array": guard let items = object["items"] else { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift index ab56f8858a..d69f1b506e 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift @@ -211,7 +211,11 @@ internal enum IgbinaryDecoder { return PhpProperty(name: String(index), visibility: .publicVisibility, value: entry.item.value, index: index) } let property = decodePhpPropertyKey(Self.text(entry.key)) - return PhpProperty(name: property.name, visibility: property.visibility, value: entry.item.value) + var binaryKey: Data? + if case .binaryString(let bytes) = entry.key { binaryKey = bytes } + return PhpProperty( + name: property.name, visibility: property.visibility, value: entry.item.value, binaryKey: binaryKey + ) } value = .object(className: className, properties: properties) case .objectSer8, .objectSer16, .objectSer32: diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift index 03bd2f6a0d..4d0159e00a 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift @@ -46,6 +46,8 @@ internal enum PhpSerializeWriter { guard output.utf8.count < limit else { return } if let index = property.index { output += "i:\(index);" + } else if let binaryKey = property.binaryKey { + output += "s:\(binaryKey.count):\"\(latin1(binaryKey.prefix(limit - output.utf8.count)))\";" } else { output += quoted("s", mangled(property), room: limit - output.utf8.count) + ";" } diff --git a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift index 0e4166a99d..dd19369787 100644 --- a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift +++ b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift @@ -67,6 +67,8 @@ internal struct PhpProperty: Equatable { let value: PhpValue // An `i:` member, which __serialize data such as ArrayObject's holds; `name` spells the same number. var index: Int? + // The key as igbinary stored it when it is not UTF-8, visibility prefix included; `name` shows it a byte a character. + var binaryKey: Data? } internal enum PhpVisibility: Equatable { diff --git a/apple/TablePro/Core/Storage/AvroSchemaStore.swift b/apple/TablePro/Core/Storage/AvroSchemaStore.swift index 3cee030650..3b3bdb9a1a 100644 --- a/apple/TablePro/Core/Storage/AvroSchemaStore.swift +++ b/apple/TablePro/Core/Storage/AvroSchemaStore.swift @@ -18,7 +18,7 @@ internal final class AvroSchemaStore: Sendable { // Far past any real schema; a file this large is the wrong file. static let maxFileBytes = 4_000_000 - // Posted when a schema this Mac did not have is kept, so grids and viewers that found it missing decode again. + // Posted when a schema this Mac did not have is kept, so a viewer that found it missing decodes again. static let didRestoreSchema = Notification.Name("com.TablePro.avroSchemaStore.didRestoreSchema") static let schemaIDKey = "schemaID" @@ -52,9 +52,11 @@ internal final class AvroSchemaStore: Sendable { let schema = try AvroSchema.parse(text) let id = Self.schemaID(of: text) let url = fileURL(id) - if !FileManager.default.fileExists(atPath: url.path) { + // A copy that went bad on disk is written over, or the next launch would find it missing again. + let contents = Data(text.utf8) + if (try? Data(contentsOf: url)) != contents { try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - try Data(text.utf8).write(to: url, options: .atomic) + try contents.write(to: url, options: .atomic) } let wasMissing = state.withLock { state -> Bool in state.parsed[id] = schema diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift index 09786a3302..8d2c634087 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift @@ -259,15 +259,34 @@ extension DataGridViewController { guard let columnID = menuColumn(sender)?.id, let window = view.window else { return } Task { @MainActor [weak self] in guard let text = await AvroSchemaFilePanel.chooseSchemaText(in: window) else { return } - // A schema this Mac had lost redraws every grid that names it through AvroSchemaStore.didRestoreSchema. + let schemaID = AvroSchemaStore.schemaID(of: text) + let wasMissing = AvroSchemaStore.shared.schema(id: schemaID) == nil do { - self?.setDisplayFormat(.avro(schemaID: try AvroSchemaStore.shared.add(text)), columnID: columnID) + _ = try AvroSchemaStore.shared.add(text) } catch { NSAlert(error: error).beginSheetModal(for: window, completionHandler: nil) + return } + self?.schemaChosen(.avro(schemaID: schemaID), columnID: columnID, wasMissing: wasMissing) } } + // The same file chosen again after this Mac lost it: the format stands, so this grid's columns that name it are + // redrawn instead, behind the confirmation a format change takes. + private func schemaChosen(_ format: ValueDisplayFormat, columnID: DataGridColumnID, wasMissing: Bool) { + let named = model.columns.columns.filter { model.displayFormat(for: $0) == format } + guard wasMissing, named.contains(where: { $0.id == columnID }) else { + setDisplayFormat(format, columnID: columnID) + return + } + let apply: @MainActor () -> Void = { [weak self] in self?.model.redisplay(named) } + guard model.store.valueFilter.isActive, let actions = model.actions else { + apply() + return + } + actions.gridConfirmDisplayOrderChange(then: apply) + } + @objc func sizeColumnToFitFromMenu(_ sender: Any?) { guard let column = menuColumn(sender) else { return } sizeColumnToFit(column) diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift index 6cf3d956e4..e84d7d6e3d 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift @@ -209,6 +209,17 @@ struct AvroDecoderTests { #expect(error?.kind == .tooLarge) } + @Test("A schema that chains thousands of named records parses without walking the chain") + func longNamedChain() throws { + var records = [#"{"type":"record","name":"N0","fields":[]}"#] + for index in 1...20_000 { + records.append(#"{"type":"record","name":"N"# + "\(index)" + #"","fields":[{"name":"p","type":"N"# + "\(index - 1)" + #""}]}"#) + } + let schema = try AvroSchema.parse("[" + records.joined(separator: ",") + "]") + #expect(schema.takesNoBytes(.named("N20000"))) + #expect(!schema.takesNoBytes(.union([.null]))) + } + @Test("A huge zero-byte block after a small one is refused, not added up until it overflows") func zeroByteCountOverflow() throws { let nulls = try AvroSchema.parse(#"{"type":"array","items":"null"}"#) diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift index f053be5993..2b39e3908a 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift @@ -176,6 +176,12 @@ struct IgbinaryDecoderTests { #expect(bytes.utf8.count < 200) } + @Test("An object property name that is not UTF-8 is written with its own bytes") + func binaryPropertyName() throws { + let value = try IgbinaryDecoder.decode(BinaryFixture.data("000000021708737464436c617373140111" + "01ff" + "0601")) + #expect(PhpSerializeWriter.text(value) == #"O:8:"stdClass":1:{s:1:""# + "\u{FF}" + #"";i:1;}"#) + } + @Test("A string of 300 bytes uses the 16-bit length") func longString() throws { let hex = "0000000212012c" + String(repeating: "61", count: 300) diff --git a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift index 3d928da283..4eb332f0fe 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift @@ -109,7 +109,7 @@ struct DataGridDisplayCacheTests { } @Test("A schema this Mac lost, chosen again, redraws the cells it left as hex and carries their value filter over") - func schemaReturnsToTheGrid() async throws { + func schemaReturnsToTheGrid() throws { let grid = DisplayGrid(Self.binaryRows) let column = try #require(grid.model.columns.column(at: 0)) let text = #"{"type":"fixed","name":"F"# + UUID().uuidString.filter(\.isLetter) + #"","size":16}"# @@ -120,9 +120,11 @@ struct DataGridDisplayCacheTests { grid.model.setValueFilter(filter) _ = try AvroSchemaStore.shared.add(text) + #expect(grid.text(row: 0) == Self.rawBinaryText) + grid.model.redisplay([column]) let decoded = "\"" + Self.rawBinaryText + "\"" - try await waitUntil { grid.text(row: 0) == decoded } + #expect(grid.text(row: 0) == decoded) #expect(grid.model.store.valueFilter.filter(forColumn: 0)?.selectedValues == [decoded]) #expect(grid.model.rowCount == 1) } From ed9d02e4757651d6bb319e8754a249b4b599f3c1 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 07:23:50 +0700 Subject: [PATCH 10/22] refactor(display-format): drop the in-grid schema restore, a lost schema shows from the next load --- .../Core/DataGrid/DataGridModel.swift | 19 +--------------- .../DataGridViewController+HeaderMenu.swift | 22 +------------------ .../DataGrid/DataGridDisplayCacheTests.swift | 21 ------------------ 3 files changed, 2 insertions(+), 60 deletions(-) diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index c6004d67f5..5e665060df 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -270,22 +270,6 @@ final class DataGridModel: ObservableObject { store.valueFilter = remapped } store.displayFormats = formats - redrawFormatted([column], in: current) - } - - // Columns whose format is unchanged now decode values they showed as stored bytes, as when a lost Avro schema is - // chosen again: a value filter picked from those bytes follows them, the way a format change carries it. - func redisplay(_ changed: [DataGridColumn]) { - let current = rows - for column in changed { - if let remapped = remappedValueFilter(column: column, from: nil, to: displayFormat(for: column), in: current) { - store.valueFilter = remapped - } - } - redrawFormatted(changed, in: current) - } - - private func redrawFormatted(_ changed: [DataGridColumn], in current: TableRows) { display.cache.removeAll() if store.valueFilter.isActive { order = resolveOrder(in: current) @@ -293,8 +277,7 @@ final class DataGridModel: ObservableObject { displayRevision &+= 1 mountedView?.apply(.reload) } else { - let positions = IndexSet(changed.compactMap { columns.displayPosition(of: $0.id) }) - mountedView?.apply(.rowsUpdated(IndexSet(0.. Void = { [weak self] in self?.model.redisplay(named) } - guard model.store.valueFilter.isActive, let actions = model.actions else { - apply() - return - } - actions.gridConfirmDisplayOrderChange(then: apply) - } - @objc func sizeColumnToFitFromMenu(_ sender: Any?) { guard let column = menuColumn(sender) else { return } sizeColumnToFit(column) diff --git a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift index 4eb332f0fe..41e006c2fd 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridDisplayCacheTests.swift @@ -108,27 +108,6 @@ struct DataGridDisplayCacheTests { #expect(grid.text(row: 0) == Self.rawBinaryText) } - @Test("A schema this Mac lost, chosen again, redraws the cells it left as hex and carries their value filter over") - func schemaReturnsToTheGrid() throws { - let grid = DisplayGrid(Self.binaryRows) - let column = try #require(grid.model.columns.column(at: 0)) - let text = #"{"type":"fixed","name":"F"# + UUID().uuidString.filter(\.isLetter) + #"","size":16}"# - grid.model.setDisplayFormat(.avro(schemaID: AvroSchemaStore.schemaID(of: text)), for: column) - #expect(grid.text(row: 0) == Self.rawBinaryText) - var filter = GridValueFilterState() - filter.set(ColumnValueFilter(selectedValues: [Self.rawBinaryText], includesNull: false), columnName: "id", forColumn: 0) - grid.model.setValueFilter(filter) - - _ = try AvroSchemaStore.shared.add(text) - #expect(grid.text(row: 0) == Self.rawBinaryText) - - grid.model.redisplay([column]) - let decoded = "\"" + Self.rawBinaryText + "\"" - #expect(grid.text(row: 0) == decoded) - #expect(grid.model.store.valueFilter.filter(forColumn: 0)?.selectedValues == [decoded]) - #expect(grid.model.rowCount == 1) - } - @Test("A display format is kept in the grid's store, where its owner reads it") func displayFormatIsWrittenToTheStore() throws { let grid = DisplayGrid(Self.binaryRows) From 6410ed73b73c503de2f2ac5c7da23c2ae26236ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:36:20 +0700 Subject: [PATCH 11/22] feat(display-format): decode Protobuf with a schema file and fetch schemas from a Schema Registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Display As > Protobuf Schema… reads a .proto file or descriptor set and shows field names and declared types. Display As > Schema Registry fetches the Avro, Protobuf or JSON schema each Kafka value names by its wire-format prefix, from the registry set on the connection. --- .../KafkaConnectionField.swift | 24 + .../KafkaMessageFlattener.swift | 3 +- .../Core/DataGrid/DataGridModel.swift | 21 +- ...PluginMetadataRegistry+KafkaDefaults.swift | 20 + .../Formatting/BinaryValues/AvroDecoder.swift | 11 + .../Formatting/BinaryValues/AvroSchema.swift | 22 +- .../BinaryValues/BinaryValueDecoding.swift | 109 ++++- .../BinaryValues/BinaryValueReader.swift | 3 + .../BinaryValues/ProtoFileParser.swift | 388 ++++++++++++++++ .../BinaryValues/ProtobufRendering.swift | 4 +- .../BinaryValues/ProtobufSchema.swift | 373 ++++++++++++++++ .../BinaryValues/ProtobufSchemaDecoder.swift | 421 ++++++++++++++++++ .../BinaryValues/ProtobufWireDecoder.swift | 13 +- .../Formatting/ValueDisplayFormat.swift | 43 +- .../Formatting/ValueDisplayFormatter.swift | 8 +- .../ProtobufSchemaFilePanel.swift | 81 ++++ .../SchemaRegistry/SchemaRegistryCache.swift | 134 ++++++ .../SchemaRegistry/SchemaRegistryClient.swift | 207 +++++++++ .../Core/Storage/AvroSchemaStore.swift | 78 +--- .../Core/Storage/ProtobufSchemaStore.swift | 45 ++ .../Core/Storage/SchemaFileCache.swift | 96 ++++ .../Storage/ValueDisplayFormatStorage.swift | 22 +- .../TablePro/Models/UI/ProtobufTreeNode.swift | 16 +- .../TablePro/Resources/Localizable.xcstrings | 122 +++++ .../DataGridViewController+HeaderMenu.swift | 53 ++- .../DataGrid/DataGridViewController.swift | 4 + .../Views/Results/BinaryValueViewerView.swift | 31 +- .../Results/CellInteractionResolver.swift | 2 +- .../FieldEditors/FieldEditorResolver.swift | 2 +- .../BinaryDisplayFormatTests.swift | 11 + .../ProtobufSchemaDecoderTests.swift | 220 +++++++++ .../Formatting/ValueDisplayFormatTests.swift | 16 + .../SchemaRegistry/SchemaRegistryTests.swift | 171 +++++++ .../ValueDisplayFormatStorageTests.swift | 22 + 34 files changed, 2689 insertions(+), 107 deletions(-) create mode 100644 apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift create mode 100644 apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift create mode 100644 apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift create mode 100644 apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift create mode 100644 apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift create mode 100644 apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift create mode 100644 apple/TablePro/Core/Storage/ProtobufSchemaStore.swift create mode 100644 apple/TablePro/Core/Storage/SchemaFileCache.swift create mode 100644 apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift create mode 100644 apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift b/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift index cadacdd281..7bc3e9709e 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift @@ -88,6 +88,10 @@ enum KafkaConnectionField { static let saslMechanism = "kafkaSaslMechanism" static let brokerRouting = "kafkaBrokerRouting" static let connectTimeout = "kafkaConnectTimeout" + // Read by the app, which fetches the schemas a value names; the driver never uses them. + static let schemaRegistryUrl = "schemaRegistryUrl" + static let schemaRegistryUsername = "schemaRegistryUsername" + static let schemaRegistryPassword = "schemaRegistryPassword" enum SecurityProtocol: String, CaseIterable { case plaintext = "PLAINTEXT" @@ -202,6 +206,26 @@ enum KafkaConnectionField { ) ]), section: .advanced + ), + ConnectionField( + id: schemaRegistryUrl, + label: String(localized: "Schema Registry URL"), + placeholder: "http://localhost:8081", + required: false, + section: .advanced + ), + ConnectionField( + id: schemaRegistryUsername, + label: String(localized: "Schema Registry User"), + required: false, + section: .advanced + ), + ConnectionField( + id: schemaRegistryPassword, + label: String(localized: "Schema Registry Password"), + required: false, + fieldType: .secure, + section: .advanced ) ] } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift b/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift index 370504fe12..c3ad2a57ac 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift @@ -99,7 +99,8 @@ enum KafkaMessageFlattener { /// binary value is enough to make the whole column binary, because rendering a binary /// payload through a text column is the lossier of the two mistakes. private static func allDecodeAsText(_ payloads: S) -> Bool where S.Element == Data? { - for case let payload? in payloads where String(data: payload, encoding: .utf8) == nil { + // A Schema Registry value starts with a zero byte, and its payload can still be valid UTF-8. + for case let payload? in payloads where payload.first == 0 || String(data: payload, encoding: .utf8) == nil { return false } return true diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index 5e665060df..3ccb1ea6ee 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -270,6 +270,24 @@ final class DataGridModel: ObservableObject { store.valueFilter = remapped } store.displayFormats = formats + redrawFormatted([column], in: current) + } + + // A Schema Registry column shows stored bytes until its schemas arrive. The format is unchanged, so a value + // filter picked from those bytes follows the decoded text the way a format change carries it. + func schemaRegistryDidLoad(connectionID: UUID) { + let current = rows + let changed = columns.columns.filter { displayFormat(for: $0) == .schemaRegistry(connectionID: connectionID) } + guard !changed.isEmpty else { return } + for column in changed { + if let remapped = remappedValueFilter(column: column, from: nil, to: displayFormat(for: column), in: current) { + store.valueFilter = remapped + } + } + redrawFormatted(changed, in: current) + } + + private func redrawFormatted(_ changed: [DataGridColumn], in current: TableRows) { display.cache.removeAll() if store.valueFilter.isActive { order = resolveOrder(in: current) @@ -277,7 +295,8 @@ final class DataGridModel: ObservableObject { displayRevision &+= 1 mountedView?.apply(.reload) } else { - mountedView?.apply(.rowsUpdated(IndexSet(0.. [ConnectionField] { .init(value: "bootstrapOnly", label: String(localized: "Only use the bootstrap address")) ]), section: .advanced + ), + ConnectionField( + id: SchemaRegistryConnectionField.url, + label: String(localized: "Schema Registry URL"), + placeholder: "http://localhost:8081", + required: false, + section: .advanced + ), + ConnectionField( + id: SchemaRegistryConnectionField.username, + label: String(localized: "Schema Registry User"), + required: false, + section: .advanced + ), + ConnectionField( + id: SchemaRegistryConnectionField.password, + label: String(localized: "Schema Registry Password"), + required: false, + fieldType: .secure, + section: .advanced ) ] } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift index fee14447d7..4be954eed8 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift @@ -61,6 +61,17 @@ internal enum AvroDecoder { } } + // The payload behind a Schema Registry prefix the caller has read, so a datum that starts with zero stays a datum. + static func decodeRegistryPayload( + _ body: Data, + schema: AvroSchema, + maxValues: Int = .max, + maxValuesRead: Int = AvroDecoder.maxValuesRead + ) throws -> MessagePackValue { + if let raw = rawBytes(Data(body), schema: schema) { return raw } + return try datum(body, schema: schema, limits: (maxValues, maxValuesRead)) + } + private static func rawBytes(_ body: Data, schema: AvroSchema) -> MessagePackValue? { switch schema.resolve(schema.root) { case .bytes: diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift index 583dc7fc02..5c3b15ddb6 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift @@ -58,15 +58,13 @@ internal struct AvroSchema: Equatable, Sendable { // Named types whose values take no bytes, worked out once: walking a shared type on every check is exponential. let zeroByteNames: Set - static func parse(_ text: String) throws -> AvroSchema { - let json: Any - do { - json = try JSONSerialization.jsonObject(with: Data(text.utf8), options: [.fragmentsAllowed]) - } catch { - throw AvroSchemaError(reason: String(localized: "The schema is not valid JSON.")) - } + // A Schema Registry schema can name types another subject defines; `references` are those, each before its users. + static func parse(_ text: String, references: [String] = []) throws -> AvroSchema { var parser = Parser() - let root = try parser.type(json, namespace: nil) + for reference in references { + _ = try parser.type(try json(reference), namespace: nil) + } + let root = try parser.type(try json(text), namespace: nil) return AvroSchema( root: root, namedTypes: parser.namedTypes, @@ -75,6 +73,14 @@ internal struct AvroSchema: Equatable, Sendable { ) } + private static func json(_ text: String) throws -> Any { + do { + return try JSONSerialization.jsonObject(with: Data(text.utf8), options: [.fragmentsAllowed]) + } catch { + throw AvroSchemaError(reason: String(localized: "The schema is not valid JSON.")) + } + } + // In the order definitions finish, so every name a type refers to is settled before it, except its own and its // enclosing types', which a value cannot hold without bytes anyway. No recursion: a schema can chain thousands. private static func zeroByteNames(_ completed: [(name: String, type: AvroType)]) -> Set { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift index 4bd1fcbc58..5d54c7c558 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift @@ -10,13 +10,16 @@ internal enum BinaryValueFormat: Hashable, Sendable { case protobuf case igbinary case avro(schemaID: String) + case protobufSchema(schemaID: String, message: String) + case schemaRegistry(connectionID: UUID) var displayName: String { switch self { case .messagePack: return "MessagePack" - case .protobuf: return "Protobuf" + case .protobuf, .protobufSchema: return "Protobuf" case .igbinary: return "igbinary" case .avro: return "Avro" + case .schemaRegistry: return "Schema Registry" } } } @@ -32,9 +35,31 @@ internal enum BinaryValueDocument: Sendable { case failed(BinaryValueDecodeError) // Nothing to judge the bytes by: the format names a schema this Mac no longer has. case missingSchema + case registryUnavailable(schemaID: UInt32, reason: String) } -// Pure: the grid calls it per cell on the main actor, the viewer off it. +// The Confluent wire format: magic byte 0, then the schema ID, big-endian. +internal enum SchemaRegistryFrame { + static let headerLength = 5 + + static func schemaID(_ data: Data) throws -> UInt32 { + guard data.count >= headerLength, data.first == 0 else { + throw BinaryValueDecodeError(kind: .notFramed, offset: 0) + } + var reader = BinaryValueReader(data) + try reader.skip(1) + return try reader.readUInt32() + } + + static func jsonText(_ data: Data) throws -> String { + guard let text = String(data: data.dropFirst(headerLength), encoding: .utf8) else { + throw BinaryValueDecodeError(kind: .invalidField, offset: headerLength) + } + return text + } +} + +// Pure but for the schema caches: the grid calls it per cell on the main actor, the viewer off it. internal enum BinaryValueDecoding { static let maxTextLength = BinaryTextDecoder.maxDisplayBytes // The grid stops here too, so a cell never shows a value the viewer then refuses. @@ -63,6 +88,16 @@ internal enum BinaryValueDecoding { data, schema: schema, maxValues: budget, maxValuesRead: AvroDecoder.maxValuesReadInCell ).value text = MessagePackTextWriter.json(value, pretty: false, limit: limit) + case .protobufSchema(let schemaID, let message): + guard let schema = ProtobufSchemaStore.shared.schema(id: schemaID) else { return nil } + let decoded = try ProtobufSchemaDecoder.decode(data, schema: schema, message: message, maxValues: budget) + text = ProtobufSchemaTextWriter.text(decoded.fields, multiLine: false, limit: limit) + case .schemaRegistry(let connectionID): + let schemaID = try SchemaRegistryFrame.schemaID(data) + guard case .success(let schema)? = SchemaRegistryCache.shared.cachedSchema( + connectionID: connectionID, schemaID: schemaID + ) else { return nil } + text = try registryText(data, schema: schema, maxValues: budget, limit: limit) } } catch { return nil @@ -128,6 +163,21 @@ internal enum BinaryValueDecoding { ), note: message.framing.map(framingNote) ) + case .protobufSchema(let schemaID, let message): + guard let schema = ProtobufSchemaStore.shared.schema(id: schemaID) else { return .missingSchema } + return protobufDocument(try ProtobufSchemaDecoder.decode(data, schema: schema, message: message)) + case .schemaRegistry(let connectionID): + let schemaID = try SchemaRegistryFrame.schemaID(data) + switch SchemaRegistryCache.shared.cachedSchema(connectionID: connectionID, schemaID: schemaID) { + case .success(let schema)?: + return try registryDocument(data, schema: schema) + case .failure(let error)?: + return .registryUnavailable(schemaID: schemaID, reason: error.reason) + case nil: + return .registryUnavailable( + schemaID: schemaID, reason: String(localized: "The schema is still being fetched.") + ) + } } } catch let error as BinaryValueDecodeError { return .failed(error) @@ -136,6 +186,61 @@ internal enum BinaryValueDecoding { } } + static func registryText(_ data: Data, schema: RegistrySchema, maxValues: Int, limit: Int) throws -> String { + switch schema { + case .avro(let avro): + let value = try AvroDecoder.decodeRegistryPayload( + data.dropFirst(SchemaRegistryFrame.headerLength), schema: avro, + maxValues: maxValues, maxValuesRead: AvroDecoder.maxValuesReadInCell + ) + return MessagePackTextWriter.json(value, pretty: false, limit: limit) + case .protobuf(let protobuf): + let message = try ProtobufSchemaDecoder.decode(data, schema: protobuf, message: nil, maxValues: maxValues) + return ProtobufSchemaTextWriter.text(message.fields, multiLine: false, limit: limit) + case .json: + return try SchemaRegistryFrame.jsonText(data) + } + } + + static func registryDocument(_ data: Data, schema: RegistrySchema) throws -> BinaryValueDocument { + switch schema { + case .avro(let avro): + let schemaID = try SchemaRegistryFrame.schemaID(data) + let value = try AvroDecoder.decodeRegistryPayload(data.dropFirst(SchemaRegistryFrame.headerLength), schema: avro) + return .decoded( + tree: .json(MessagePackTreeBuilder.build(from: value)), + text: capped( + MessagePackTextWriter.json(value, pretty: true, limit: writerLimit(forCap: maxViewerTextLength)), + at: maxViewerTextLength + ), + note: framingNote(.confluent(schemaID: schemaID)) + ) + case .protobuf(let protobuf): + return protobufDocument(try ProtobufSchemaDecoder.decode(data, schema: protobuf, message: nil)) + case .json: + let text = try SchemaRegistryFrame.jsonText(data) + guard case .success(let tree) = JSONTreeParser.parse(text) else { + throw BinaryValueDecodeError(kind: .invalidField, offset: SchemaRegistryFrame.headerLength) + } + return .decoded( + tree: .json(tree), + text: capped(text, at: maxViewerTextLength), + note: framingNote(.confluent(schemaID: try SchemaRegistryFrame.schemaID(data))) + ) + } + } + + private static func protobufDocument(_ message: ProtobufTypedMessage) -> BinaryValueDocument { + .decoded( + tree: .protobuf(ProtobufSchemaTreeBuilder.build(from: message)), + text: capped( + ProtobufSchemaTextWriter.text(message.fields, multiLine: true, limit: writerLimit(forCap: maxViewerTextLength)), + at: maxViewerTextLength + ), + note: message.framing.map(framingNote) + ) + } + private static func framingNote(_ framing: AvroFraming) -> String { switch framing { case .confluent(let schemaID): diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift index edcb95f1c1..2961c7e0c5 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift @@ -20,6 +20,7 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { case indexOutOfRange(Int) case invalidLength case otherSchema(fingerprint: UInt64) + case notFramed } let kind: Kind @@ -59,6 +60,8 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { return String(format: String(localized: "Byte %lld picks entry %lld, which the schema does not have."), offset, index) case .invalidLength: return String(format: String(localized: "The length at byte %lld is negative."), offset) + case .notFramed: + return String(localized: "The value does not start with the Schema Registry prefix: a zero byte and a schema ID.") case .otherSchema(let fingerprint): return String( format: String(localized: "The value was written with the schema whose fingerprint is %@, not this column's."), diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift new file mode 100644 index 0000000000..90a0b5c30f --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift @@ -0,0 +1,388 @@ +// +// ProtoFileParser.swift +// TablePro +// + +import Foundation + +// https://protobuf.dev/reference/protobuf/proto3-spec/ and the proto2 and editions grammars: what a value needs is +// read, and options, services, extensions and reserved ranges are skipped. +internal enum ProtoFileParser { + static let maxTokens = 2_000_000 + static let maxDepth = 100 + + static func parse(_ text: String) throws -> [ProtobufFileDefinition] { + var parser = Parser(tokens: try Tokenizer.tokens(text)) + return [try parser.file()] + } + + fileprivate enum Token: Equatable { + case identifier(String) + case number(String) + case string(String) + case symbol(Character) + } + + private enum Tokenizer { + static func tokens(_ text: String) throws -> [Token] { + let scalars = Array(text.unicodeScalars) + var tokens: [Token] = [] + var index = 0 + while index < scalars.count { + guard tokens.count < ProtoFileParser.maxTokens else { throw error(String(localized: "The file is too large.")) } + let scalar = scalars[index] + if CharacterSet.whitespacesAndNewlines.contains(scalar) { + index += 1 + } else if scalar == "/", index + 1 < scalars.count, scalars[index + 1] == "/" { + while index < scalars.count, scalars[index] != "\n" { index += 1 } + } else if scalar == "/", index + 1 < scalars.count, scalars[index + 1] == "*" { + index += 2 + while index + 1 < scalars.count, !(scalars[index] == "*" && scalars[index + 1] == "/") { index += 1 } + guard index + 1 < scalars.count else { throw error(String(localized: "A comment is never closed.")) } + index += 2 + } else if isIdentifierStart(scalar) + || (scalar == "." && index + 1 < scalars.count && isIdentifierStart(scalars[index + 1])) { + let start = index + while index < scalars.count, isIdentifierPart(scalars[index]) || scalars[index] == "." { index += 1 } + tokens.append(.identifier(String(String.UnicodeScalarView(scalars[start..=;,:-+".unicodeScalars.contains(scalar) { + tokens.append(.symbol(Character(scalar))) + index += 1 + } else { + throw error(String(format: String(localized: "Unexpected character “%@”."), String(Character(scalar)))) + } + } + return tokens + } + + private static func isIdentifierStart(_ scalar: Unicode.Scalar) -> Bool { + scalar == "_" || (scalar.isASCII && CharacterSet.letters.contains(scalar)) + } + + private static func isIdentifierPart(_ scalar: Unicode.Scalar) -> Bool { + isIdentifierStart(scalar) || isDigit(scalar) + } + + private static func isDigit(_ scalar: Unicode.Scalar) -> Bool { + ("0"..."9").contains(scalar) + } + } + + fileprivate static func error(_ reason: String) -> ProtobufSchemaError { + ProtobufSchemaError(reason: reason) + } + + private struct Parser { + let tokens: [Token] + var position = 0 + var messages: [ProtobufMessageDefinition] = [] + var enums: [ProtobufEnumDescriptor] = [] + + init(tokens: [Token]) { + self.tokens = tokens + } + + mutating func file() throws -> ProtobufFileDefinition { + var package = "" + var topMessages: [ProtobufMessageDefinition] = [] + var topEnums: [ProtobufEnumDescriptor] = [] + while let token = peek() { + switch token { + case .identifier("syntax"), .identifier("edition"): + try skipStatement() + case .identifier("package"): + advance() + package = try identifier() + try expect(";") + case .identifier("import"), .identifier("option"): + try skipStatement() + case .identifier("message"): + advance() + topMessages.append(try message(scope: package, depth: 0)) + case .identifier("enum"): + advance() + topEnums.append(try enumeration(scope: package)) + case .identifier("service"), .identifier("extend"): + try skipStatement() + case .symbol(";"): + advance() + default: + throw unexpected() + } + } + return ProtobufFileDefinition(messages: topMessages, enums: topEnums, allMessages: messages, allEnums: enums) + } + + private mutating func message(scope: String, depth: Int) throws -> ProtobufMessageDefinition { + let name = try identifier() + let fullName = scope.isEmpty ? name : "\(scope).\(name)" + try expect("{") + let slot = messages.count + messages.append(ProtobufMessageDefinition(fullName: fullName, fields: [], nestedMessages: [], isMapEntry: false)) + var definition = messages[slot] + try body(of: &definition, depth: depth) + messages[slot] = definition + return definition + } + + private mutating func body(of definition: inout ProtobufMessageDefinition, depth: Int) throws { + guard depth < ProtoFileParser.maxDepth else { throw ProtoFileParser.error(String(localized: "Messages are nested too deeply.")) } + while true { + guard let token = peek() else { throw ProtoFileParser.error(String(localized: "A message is never closed.")) } + switch token { + case .symbol("}"): + advance() + return + case .symbol(";"): + advance() + case .identifier("message"): + advance() + definition.nestedMessages.append(try message(scope: definition.fullName, depth: depth + 1).fullName) + case .identifier("enum"): + advance() + _ = try enumeration(scope: definition.fullName) + case .identifier("oneof"): + advance() + _ = try identifier() + try expect("{") + try body(of: &definition, depth: depth + 1) + // A Schema Registry stores a map field as the entry message protoc synthesizes for it. + case .identifier("option") where peek(1) == .identifier("map_entry") && peek(3) == .identifier("true"): + definition.isMapEntry = true + try skipStatement() + case .identifier("option"), .identifier("reserved"), .identifier("extensions"), .identifier("extend"): + try skipStatement() + case .identifier("map") where peek(1) == .symbol("<"): + advance() + try mapField(into: &definition) + default: + try field(into: &definition, depth: depth) + } + } + } + + private mutating func field(into definition: inout ProtobufMessageDefinition, depth: Int) throws { + var isRepeated = false + if case .identifier(let label)? = peek(), ["optional", "required", "repeated"].contains(label), + peek(1).map(Self.isTypeToken) == true, peek(2).map(Self.isTypeToken) == true || peek(2) == .symbol("=") { + isRepeated = label == "repeated" + advance() + } + let typeName = try identifier() + if typeName == "group" { + let groupName = try identifier() + try expect("=") + let number = try fieldNumber() + try skipOptions() + try expect("{") + let fullName = "\(definition.fullName).\(groupName)" + let slot = messages.count + messages.append(ProtobufMessageDefinition(fullName: fullName, fields: [], nestedMessages: [], isMapEntry: false)) + var group = messages[slot] + try body(of: &group, depth: depth + 1) + messages[slot] = group + definition.nestedMessages.append(fullName) + definition.fields.append(ProtobufFieldDefinition( + name: groupName.lowercased(), number: number, type: .group(fullName), isRepeated: isRepeated + )) + return + } + let name = try identifier() + try expect("=") + let number = try fieldNumber() + try skipOptions() + try expect(";") + let type: ProtobufFieldDefinition.TypeReference = ProtobufScalarType(rawValue: typeName).map { .scalar($0) } + ?? .named(typeName) + definition.fields.append(ProtobufFieldDefinition(name: name, number: number, type: type, isRepeated: isRepeated)) + } + + // protoc's synthesized entry: `Entry { key = 1; value = 2; }`, nested where the field is declared. + private mutating func mapField(into definition: inout ProtobufMessageDefinition) throws { + try expect("<") + let keyType = try identifier() + try expect(",") + let valueType = try identifier() + try expect(">") + let name = try identifier() + try expect("=") + let number = try fieldNumber() + try skipOptions() + try expect(";") + let entryName = "\(definition.fullName).\(Self.camelCased(name))Entry" + func reference(_ type: String) -> ProtobufFieldDefinition.TypeReference { + ProtobufScalarType(rawValue: type).map { .scalar($0) } ?? .named(type) + } + messages.append(ProtobufMessageDefinition( + fullName: entryName, + fields: [ + ProtobufFieldDefinition(name: "key", number: 1, type: reference(keyType), isRepeated: false), + ProtobufFieldDefinition(name: "value", number: 2, type: reference(valueType), isRepeated: false), + ], + nestedMessages: [], + isMapEntry: true + )) + definition.nestedMessages.append(entryName) + // The entry sits in the message's scope, so its value type resolves from there. + definition.fields.append(ProtobufFieldDefinition(name: name, number: number, type: .named(".\(entryName)"), isRepeated: true)) + } + + private mutating func enumeration(scope: String) throws -> ProtobufEnumDescriptor { + let name = try identifier() + try expect("{") + var values: [Int32: String] = [:] + while true { + guard let token = peek() else { throw ProtoFileParser.error(String(localized: "An enum is never closed.")) } + switch token { + case .symbol("}"): + advance() + let enumeration = ProtobufEnumDescriptor(fullName: scope.isEmpty ? name : "\(scope).\(name)", values: values) + enums.append(enumeration) + return enumeration + case .symbol(";"): + advance() + case .identifier("option"), .identifier("reserved"): + try skipStatement() + default: + let valueName = try identifier() + try expect("=") + var negative = false + if peek() == .symbol("-") { + negative = true + advance() + } + guard case .number(let digits)? = peek(), let magnitude = Self.integer(digits), + let number = Int32(exactly: negative ? -magnitude : magnitude) else { throw unexpected() } + advance() + try skipOptions() + try expect(";") + if values[number] == nil { values[number] = valueName } + } + } + } + + private static func isTypeToken(_ token: Token) -> Bool { + if case .identifier = token { return true } + return false + } + + private static func camelCased(_ name: String) -> String { + var result = "" + var upper = true + for character in name { + if character == "_" { + upper = true + } else { + result += upper ? character.uppercased() : String(character) + upper = false + } + } + return result + } + + private static func integer(_ digits: String) -> Int64? { + let lower = digits.lowercased() + if lower.hasPrefix("0x") { return Int64(lower.dropFirst(2), radix: 16) } + if lower.count > 1, lower.hasPrefix("0") { return Int64(lower.dropFirst(), radix: 8) } + return Int64(lower) + } + + private mutating func fieldNumber() throws -> Int { + guard case .number(let digits)? = peek(), let value = Self.integer(digits), + value > 0, value <= Int64(ProtobufWireDecoder.maxFieldNumber) else { throw unexpected() } + advance() + return Int(value) + } + + // `[packed = true, (custom).option = { ... }]` + private mutating func skipOptions() throws { + guard peek() == .symbol("[") else { return } + try skipBalanced(open: "[", close: "]") + } + + // To the `;` that ends it, or past the `{ ... }` block that does. + private mutating func skipStatement() throws { + while let token = peek() { + switch token { + case .symbol(";"): + advance() + return + case .symbol("{"): + try skipBalanced(open: "{", close: "}") + if peek() == .symbol(";") { advance() } + return + case .symbol("["): + try skipBalanced(open: "[", close: "]") + default: + advance() + } + } + throw ProtoFileParser.error(String(localized: "The file ends in the middle of a statement.")) + } + + private mutating func skipBalanced(open: Character, close: Character) throws { + var depth = 0 + while let token = peek() { + advance() + if token == .symbol(open) { depth += 1 } + if token == .symbol(close) { + depth -= 1 + if depth == 0 { return } + } + } + throw ProtoFileParser.error(String(localized: "The file ends in the middle of a statement.")) + } + + private mutating func identifier() throws -> String { + guard case .identifier(let name)? = peek() else { throw unexpected() } + advance() + return name + } + + private mutating func expect(_ symbol: Character) throws { + guard peek() == .symbol(symbol) else { throw unexpected() } + advance() + } + + private func peek(_ offset: Int = 0) -> Token? { + position + offset < tokens.count ? tokens[position + offset] : nil + } + + private mutating func advance() { + position += 1 + } + + private func unexpected() -> ProtobufSchemaError { + guard let token = peek() else { + return ProtoFileParser.error(String(localized: "The file ends in the middle of a statement.")) + } + let text: String + switch token { + case .identifier(let value), .number(let value): text = value + case .string(let value): text = "\"\(value)\"" + case .symbol(let value): text = String(value) + } + return ProtoFileParser.error(String(format: String(localized: "Unexpected “%@” in the .proto file."), text)) + } + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift index 622a3c6420..299d3c1a7b 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift @@ -24,7 +24,7 @@ internal enum ProtobufTextWriter { } } - private static func write( + static func write( _ fields: [ProtobufField], into output: inout String, multiLine: Bool, @@ -126,7 +126,7 @@ internal enum ProtobufTreeBuilder { return children } - private static func node( + static func node( key: String, keyPath: String, path: TreeNodePath, diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift new file mode 100644 index 0000000000..cb87bd3fa3 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift @@ -0,0 +1,373 @@ +// +// ProtobufSchema.swift +// TablePro +// + +import Foundation + +internal enum ProtobufScalarType: String, Equatable, Sendable { + case double, float, int64, uint64, int32, fixed64, fixed32, bool, string, bytes, uint32 + case sfixed32, sfixed64, sint32, sint64 +} + +internal enum ProtobufFieldType: Equatable, Sendable { + case scalar(ProtobufScalarType) + case message(String) + case enumeration(String) + case group(String) + // Named by a file the schema does not include, so its values read as they would with no schema. + case unresolved(String) +} + +internal struct ProtobufFieldDescriptor: Equatable, Sendable { + let name: String + let number: Int + let type: ProtobufFieldType + let isRepeated: Bool +} + +internal struct ProtobufMessageDescriptor: Equatable, Sendable { + let fullName: String + let fields: [ProtobufFieldDescriptor] + // In declaration order, map entries included: a Schema Registry message index counts them. + let nestedMessages: [String] + let isMapEntry: Bool + private let fieldIndexes: [Int: Int] + + init(fullName: String, fields: [ProtobufFieldDescriptor], nestedMessages: [String], isMapEntry: Bool) { + self.fullName = fullName + self.fields = fields + self.nestedMessages = nestedMessages + self.isMapEntry = isMapEntry + fieldIndexes = Dictionary(fields.enumerated().map { ($0.element.number, $0.offset) }) { first, _ in first } + } + + var shortName: String { + fullName.split(separator: ".").last.map(String.init) ?? fullName + } + + func field(number: Int) -> ProtobufFieldDescriptor? { + fieldIndexes[number].map { fields[$0] } + } +} + +internal struct ProtobufEnumDescriptor: Equatable, Sendable { + let fullName: String + let values: [Int32: String] + + var shortName: String { + fullName.split(separator: ".").last.map(String.init) ?? fullName + } +} + +internal struct ProtobufSchemaError: Error, Equatable, Sendable { + let reason: String +} + +// Messages and enums by full name, without the leading dot. The first file is the one chosen or fetched; +// a Schema Registry message index walks its top-level messages. +internal struct ProtobufSchema: Equatable, Sendable { + let messages: [String: ProtobufMessageDescriptor] + let enums: [String: ProtobufEnumDescriptor] + let rootMessages: [String] + // Every message a value can be read as, in file order, map entries left out. + let messageNames: [String] + + static let maxMessages = 20_000 + + // Both forms protoc reads: `.proto` text, and the FileDescriptorSet `--descriptor_set_out` writes. + static func parse(_ data: Data) throws -> ProtobufSchema { + if let text = String(data: data, encoding: .utf8), let files = try? ProtoFileParser.parse(text), !files.isEmpty { + return try build([files]) + } + do { + return try build([ProtobufDescriptorSetParser.parse(data)]) + } catch let error as ProtobufSchemaError { + if let text = String(data: data, encoding: .utf8) { + // Text that is not a descriptor set: the parser's own reason says what is wrong with it. + _ = try ProtoFileParser.parse(text) + throw ProtobufSchemaError(reason: String(localized: "The file defines no message.")) + } + throw error + } + } + + // A fetched schema and the files it imports, each `.proto` text, the root first. + static func parse(protoFiles texts: [String]) throws -> ProtobufSchema { + try build(texts.map { try ProtoFileParser.parse($0) }) + } + + func messageIndexed(by indexes: [Int]) -> ProtobufMessageDescriptor? { + guard let first = indexes.first, rootMessages.indices.contains(first), + var message = messages[rootMessages[first]] else { return nil } + for index in indexes.dropFirst() { + guard message.nestedMessages.indices.contains(index), + let nested = messages[message.nestedMessages[index]] else { return nil } + message = nested + } + return message + } + + private static func build(_ fileGroups: [[ProtobufFileDefinition]]) throws -> ProtobufSchema { + let files = fileGroups.flatMap { $0 } + var messages: [String: ProtobufMessageDefinition] = [:] + var enums: [String: ProtobufEnumDescriptor] = [:] + var order: [String] = [] + for file in files { + for message in file.allMessages { + if messages[message.fullName] == nil { order.append(message.fullName) } + messages[message.fullName] = message + } + for enumeration in file.allEnums { + enums[enumeration.fullName] = enumeration + } + } + guard messages.count <= maxMessages else { + throw ProtobufSchemaError(reason: String(localized: "The schema defines more messages than TablePro reads.")) + } + guard !messages.isEmpty else { + throw ProtobufSchemaError(reason: String(localized: "The file defines no message.")) + } + let resolver = ProtobufTypeResolver(messageNames: Set(messages.keys), enumNames: Set(enums.keys)) + var resolved: [String: ProtobufMessageDescriptor] = [:] + for (name, message) in messages { + resolved[name] = ProtobufMessageDescriptor( + fullName: name, + fields: message.fields.map { resolver.resolve($0, scope: name) }, + nestedMessages: message.nestedMessages, + isMapEntry: message.isMapEntry + ) + } + return ProtobufSchema( + messages: resolved, + enums: enums, + rootMessages: files.first?.messages.map(\.fullName) ?? [], + messageNames: order.filter { resolved[$0]?.isMapEntry == false } + ) + } +} + +// What a parser read, before type names are resolved against every file. +internal struct ProtobufFileDefinition: Sendable { + let messages: [ProtobufMessageDefinition] + let enums: [ProtobufEnumDescriptor] + // Nested types too, so build() sees every name the file declares. + let allMessages: [ProtobufMessageDefinition] + let allEnums: [ProtobufEnumDescriptor] +} + +internal struct ProtobufMessageDefinition: Sendable { + let fullName: String + var fields: [ProtobufFieldDefinition] + var nestedMessages: [String] + var isMapEntry: Bool +} + +internal struct ProtobufFieldDefinition: Sendable { + enum TypeReference: Sendable { + case scalar(ProtobufScalarType) + // As written: relative, dotted, or absolute with a leading dot. + case named(String) + case group(String) + } + + let name: String + let number: Int + let type: TypeReference + let isRepeated: Bool +} + +// C++ scoping: a name is looked up from the innermost scope outward, by its first component. +private struct ProtobufTypeResolver { + let messageNames: Set + let enumNames: Set + + func resolve(_ field: ProtobufFieldDefinition, scope: String) -> ProtobufFieldDescriptor { + let type: ProtobufFieldType + switch field.type { + case .scalar(let scalar): + type = .scalar(scalar) + case .group(let fullName): + type = .group(fullName) + case .named(let reference): + type = resolve(reference, scope: scope) + } + return ProtobufFieldDescriptor(name: field.name, number: field.number, type: type, isRepeated: field.isRepeated) + } + + private func resolve(_ reference: String, scope: String) -> ProtobufFieldType { + if reference.hasPrefix(".") { + return lookup(String(reference.dropFirst())) ?? .unresolved(reference) + } + var components = scope.split(separator: ".").map(String.init) + while true { + let candidate = (components + [reference]).joined(separator: ".") + if let found = lookup(candidate) { return found } + guard !components.isEmpty else { return .unresolved(reference) } + components.removeLast() + } + } + + private func lookup(_ name: String) -> ProtobufFieldType? { + if messageNames.contains(name) { return .message(name) } + if enumNames.contains(name) { return .enumeration(name) } + return nil + } +} + +// descriptor.proto, read with the wire reader: FileDescriptorSet.file = 1. +internal enum ProtobufDescriptorSetParser { + static func parse(_ data: Data) throws -> [ProtobufFileDefinition] { + let invalid = ProtobufSchemaError(reason: String(localized: "The file is neither a .proto file nor a descriptor set.")) + guard let set = try? fields(data) else { throw invalid } + var files: [ProtobufFileDefinition] = [] + for (number, value) in set { + guard number == 1, case .bytes(let bytes) = value else { throw invalid } + files.append(try file(bytes)) + } + guard files.contains(where: { !$0.allMessages.isEmpty }) else { throw invalid } + return files + } + + private enum Value { + case varint(UInt64) + case bytes(Data) + } + + private static func fields(_ data: Data) throws -> [(Int, Value)] { + var reader = BinaryValueReader(data) + var result: [(Int, Value)] = [] + while !reader.isAtEnd { + let tag = try reader.readVarint() + let number = Int(truncatingIfNeeded: tag >> 3) + switch tag & 7 { + case 0: + result.append((number, .varint(try reader.readVarint()))) + case 1: + try reader.skip(8) + case 2: + let length = try reader.readVarint() + guard length <= UInt64(reader.remaining) else { throw reader.error(.unexpectedEnd) } + result.append((number, .bytes(try reader.readBytes(Int(length))))) + case 5: + try reader.skip(4) + default: + throw reader.error(.invalidField) + } + } + return result + } + + private static func string(_ value: Value) -> String? { + guard case .bytes(let data) = value else { return nil } + return String(data: data, encoding: .utf8) + } + + private static func file(_ data: Data) throws -> ProtobufFileDefinition { + var package = "" + var messageData: [Data] = [] + var enumData: [Data] = [] + for (number, value) in try fields(data) { + switch (number, value) { + case (2, _): package = string(value) ?? "" + case (4, .bytes(let bytes)): messageData.append(bytes) + case (5, .bytes(let bytes)): enumData.append(bytes) + default: break + } + } + var collector = Collector() + let messages = try messageData.map { try collector.message($0, scope: package, depth: 0) } + let enums = try enumData.map { try collector.enumeration($0, scope: package) } + return ProtobufFileDefinition( + messages: messages, enums: enums, allMessages: collector.messages, allEnums: collector.enums + ) + } + + private struct Collector { + var messages: [ProtobufMessageDefinition] = [] + var enums: [ProtobufEnumDescriptor] = [] + + mutating func message(_ data: Data, scope: String, depth: Int) throws -> ProtobufMessageDefinition { + guard depth < ProtoFileParser.maxDepth else { + throw ProtobufSchemaError(reason: String(localized: "Messages are nested too deeply.")) + } + let entries = try ProtobufDescriptorSetParser.fields(data) + let name = entries.first { $0.0 == 1 }.flatMap { string($0.1) } ?? "" + let fullName = scope.isEmpty ? name : "\(scope).\(name)" + let slot = messages.count + messages.append(ProtobufMessageDefinition(fullName: fullName, fields: [], nestedMessages: [], isMapEntry: false)) + var definition = messages[slot] + for (number, value) in entries { + guard case .bytes(let bytes) = value else { continue } + switch number { + case 2: definition.fields.append(try field(bytes)) + case 3: definition.nestedMessages.append(try message(bytes, scope: fullName, depth: depth + 1).fullName) + case 4: enums.append(try enumeration(bytes, scope: fullName)) + case 7: definition.isMapEntry = try ProtobufDescriptorSetParser.fields(bytes).contains { + if $0.0 == 7, case .varint(let flag) = $0.1 { return flag != 0 } + return false + } + default: break + } + } + messages[slot] = definition + return definition + } + + mutating func enumeration(_ data: Data, scope: String) throws -> ProtobufEnumDescriptor { + var name = "" + var values: [Int32: String] = [:] + for (number, value) in try ProtobufDescriptorSetParser.fields(data) { + if number == 1 { name = string(value) ?? "" } + guard number == 2, case .bytes(let bytes) = value else { continue } + var valueName = "" + var valueNumber: Int32 = 0 + for (member, memberValue) in try ProtobufDescriptorSetParser.fields(bytes) { + if member == 1 { valueName = string(memberValue) ?? "" } + if member == 2, case .varint(let raw) = memberValue { valueNumber = Int32(truncatingIfNeeded: raw) } + } + // The first name wins for an aliased number, as protoc prints it. + if values[valueNumber] == nil { values[valueNumber] = valueName } + } + let enumeration = ProtobufEnumDescriptor(fullName: scope.isEmpty ? name : "\(scope).\(name)", values: values) + enums.append(enumeration) + return enumeration + } + + private func field(_ data: Data) throws -> ProtobufFieldDefinition { + var name = "" + var number = 0 + var label: UInt64 = 1 + var typeCode: UInt64 = 0 + var typeName = "" + for (member, value) in try ProtobufDescriptorSetParser.fields(data) { + switch (member, value) { + case (1, _): name = string(value) ?? "" + case (3, .varint(let raw)): number = Int(truncatingIfNeeded: raw) + case (4, .varint(let raw)): label = raw + case (5, .varint(let raw)): typeCode = raw + case (6, _): typeName = string(value) ?? "" + default: break + } + } + let type: ProtobufFieldDefinition.TypeReference + switch typeCode { + case 10: type = .group(String(typeName.drop { $0 == "." })) + case 11, 14, 0: type = .named(typeName) + default: + guard let scalar = Self.scalarTypes[typeCode] else { + throw ProtobufSchemaError(reason: String( + format: String(localized: "The descriptor set names an unknown field type %llu."), typeCode + )) + } + type = .scalar(scalar) + } + return ProtobufFieldDefinition(name: name, number: number, type: type, isRepeated: label == 3) + } + + private static let scalarTypes: [UInt64: ProtobufScalarType] = [ + 1: .double, 2: .float, 3: .int64, 4: .uint64, 5: .int32, 6: .fixed64, 7: .fixed32, 8: .bool, + 9: .string, 12: .bytes, 13: .uint32, 15: .sfixed32, 16: .sfixed64, 17: .sint32, 18: .sint64, + ] + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift new file mode 100644 index 0000000000..e073b41a9a --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -0,0 +1,421 @@ +// +// ProtobufSchemaDecoder.swift +// TablePro +// + +import Foundation + +internal struct ProtobufTypedField: Equatable, Sendable { + // The field's name, or its number when the schema does not name it. + let name: String + // The declared type, which the viewer shows as the badge; nil for a field the schema does not name. + let typeName: String? + let value: ProtobufTypedValue +} + +internal indirect enum ProtobufTypedValue: Equatable, Sendable { + // Numbers, bools and enum values, already in `protoc --decode` spelling. + case scalar(String) + case string(String) + case bytes(Data) + case message([ProtobufTypedField]) + case unknown(ProtobufFieldValue) + case truncated +} + +internal struct ProtobufTypedMessage: Equatable, Sendable { + let typeName: String + let fields: [ProtobufTypedField] + let framing: ProtobufFraming? +} + +// Reads a value as `protoc --decode=` does: known fields in number order, then the ones the schema does not +// name, by number, as `--decode_raw` shows them. A field whose wire type does not match its declaration is one of those. +internal enum ProtobufSchemaDecoder { + // A nil message takes the one a Schema Registry frame's message indexes name. + static func decode( + _ data: Data, + schema: ProtobufSchema, + message messageName: String?, + maxValues: Int = .max + ) throws -> ProtobufTypedMessage { + var reader = BinaryValueReader(data) + var framing: ProtobufFraming? + if reader.peekByte() == 0 { + framing = try ProtobufWireDecoder.readFraming(&reader) + } + let message: ProtobufMessageDescriptor? + if let messageName { + message = schema.messages[messageName] + } else if let framing { + message = schema.messageIndexed(by: framing.messageIndexes) + if message == nil { + throw BinaryValueDecodeError(kind: .indexOutOfRange(framing.messageIndexes.last ?? 0), offset: 1) + } + } else { + message = schema.rootMessages.first.flatMap { schema.messages[$0] } + } + guard let message else { throw BinaryValueDecodeError(kind: .invalidField, offset: 0) } + var walker = Walker(schema: schema, budget: maxValues) + let fields = try walker.fields(&reader, message: message, depth: 0, group: nil) + return ProtobufTypedMessage(typeName: message.fullName, fields: fields, framing: framing) + } + + private struct Walker { + let schema: ProtobufSchema + var budget: Int + var isTruncated = false + + mutating func fields( + _ reader: inout BinaryValueReader, + message: ProtobufMessageDescriptor?, + depth: Int, + group: Int? + ) throws -> [ProtobufTypedField] { + guard depth < ProtobufWireDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } + var known: [Int: [ProtobufTypedField]] = [:] + var unknown: [ProtobufTypedField] = [] + // A singular message seen twice is merged, which is what parsing its occurrences' bytes back to back does. + var singularMessages: [Int: Data] = [:] + func add(_ field: ProtobufTypedField, descriptor: ProtobufFieldDescriptor?) { + guard let descriptor else { + unknown.append(field) + return + } + // A repeated field keeps every value; any other keeps the last, as a parser does. + if descriptor.isRepeated { + known[descriptor.number, default: []].append(field) + } else { + known[descriptor.number] = [field] + } + } + var ended = group == nil + while !reader.isAtEnd { + guard budget > 0 else { + isTruncated = true + break + } + let start = reader.offset + let tag = try reader.readVarint() + let number = tag >> 3 + guard tag <= UInt64(UInt32.max), number > 0, number <= UInt64(ProtobufWireDecoder.maxFieldNumber) else { + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + let fieldNumber = Int(number) + let wireType = tag & 7 + if wireType == 4 { + guard group == fieldNumber else { throw BinaryValueDecodeError(kind: .unmatchedGroupEnd, offset: start) } + ended = true + break + } + budget -= 1 + let descriptor = message?.field(number: fieldNumber) + switch wireType { + case 0: + let bits = try reader.readVarint() + if let descriptor, let text = varintText(bits, type: descriptor.type) { + add(typed(descriptor, .scalar(text)), descriptor: descriptor) + } else { + add(raw(fieldNumber, .varint(bits)), descriptor: nil) + } + case 1: + let bits = try reader.readUInt64LittleEndian() + if let descriptor, let text = fixed64Text(bits, type: descriptor.type) { + add(typed(descriptor, .scalar(text)), descriptor: descriptor) + } else { + add(raw(fieldNumber, .fixed64(bits)), descriptor: nil) + } + case 5: + let bits = try reader.readUInt32LittleEndian() + if let descriptor, let text = fixed32Text(bits, type: descriptor.type) { + add(typed(descriptor, .scalar(text)), descriptor: descriptor) + } else { + add(raw(fieldNumber, .fixed32(bits)), descriptor: nil) + } + case 2: + let length = try reader.readVarint() + guard length <= UInt64(reader.remaining) else { throw reader.error(.unexpectedEnd) } + let bytes = try reader.readBytes(Int(length)) + if let descriptor, case .message = descriptor.type, !descriptor.isRepeated { + singularMessages[fieldNumber, default: Data()].append(bytes) + } else if let descriptor, let values = lengthDelimited(bytes, descriptor: descriptor, depth: depth) { + for value in values { add(typed(descriptor, value), descriptor: descriptor) } + } else { + add(raw(fieldNumber, ProtobufWireDecoder.lengthDelimited(bytes)), descriptor: nil) + } + case 3: + if let descriptor, case .group(let name) = descriptor.type { + let members = try fields(&reader, message: schema.messages[name], depth: depth + 1, group: fieldNumber) + let groupName = schema.messages[name]?.shortName ?? descriptor.name + add( + ProtobufTypedField(name: groupName, typeName: "group", value: .message(members)), + descriptor: descriptor + ) + } else { + let members = try fields(&reader, message: nil, depth: depth + 1, group: fieldNumber) + add(ProtobufTypedField(name: String(fieldNumber), typeName: nil, value: .message(members)), descriptor: nil) + } + default: + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + if isTruncated { break } + } + if !ended, !isTruncated { throw reader.error(.unexpectedEnd) } + for (number, bytes) in singularMessages { + guard let descriptor = message?.field(number: number) else { continue } + if let values = lengthDelimited(bytes, descriptor: descriptor, depth: depth) { + known[number] = values.map { typed(descriptor, $0) } + } else { + unknown.append(raw(number, ProtobufWireDecoder.lengthDelimited(bytes))) + } + } + var result = known.keys.sorted().flatMap { known[$0] ?? [] } + unknown + if isTruncated, depth == 0 { result.append(ProtobufTypedField(name: "", typeName: nil, value: .truncated)) } + return result + } + + private func typed(_ descriptor: ProtobufFieldDescriptor, _ value: ProtobufTypedValue) -> ProtobufTypedField { + ProtobufTypedField(name: descriptor.name, typeName: typeName(descriptor.type), value: value) + } + + private func raw(_ number: Int, _ value: ProtobufFieldValue) -> ProtobufTypedField { + ProtobufTypedField(name: String(number), typeName: nil, value: .unknown(value)) + } + + private func typeName(_ type: ProtobufFieldType) -> String { + switch type { + case .scalar(let scalar): return scalar.rawValue + case .message(let name): return schema.messages[name]?.shortName ?? name + case .enumeration(let name): return schema.enums[name]?.shortName ?? name + case .group: return "group" + case .unresolved(let name): return name + } + } + + // nil leaves the value to the fields the schema does not name. + private mutating func lengthDelimited( + _ bytes: Data, + descriptor: ProtobufFieldDescriptor, + depth: Int + ) -> [ProtobufTypedValue]? { + switch descriptor.type { + case .scalar(.string): + guard let text = String(data: bytes, encoding: .utf8) else { return nil } + return [.string(text)] + case .scalar(.bytes): + return [.bytes(bytes)] + case .message(let name): + var nested = BinaryValueReader(bytes) + var attempt = self + guard let members = try? attempt.fields(&nested, message: schema.messages[name], depth: depth + 1, group: nil) else { + return nil + } + self = attempt + return [.message(members)] + case .scalar, .enumeration: + guard descriptor.isRepeated else { return nil } + return packed(bytes, type: descriptor.type) + case .group, .unresolved: + return nil + } + } + + private mutating func packed(_ bytes: Data, type: ProtobufFieldType) -> [ProtobufTypedValue]? { + var reader = BinaryValueReader(bytes) + var values: [ProtobufTypedValue] = [] + while !reader.isAtEnd { + guard budget > 0 else { + isTruncated = true + break + } + let text: String? + switch type { + case .scalar(.double), .scalar(.fixed64), .scalar(.sfixed64): + text = (try? reader.readUInt64LittleEndian()).flatMap { fixed64Text($0, type: type) } + case .scalar(.float), .scalar(.fixed32), .scalar(.sfixed32): + text = (try? reader.readUInt32LittleEndian()).flatMap { fixed32Text($0, type: type) } + default: + text = (try? reader.readVarint()).flatMap { varintText($0, type: type) } + } + guard let text else { return nil } + values.append(.scalar(text)) + budget -= 1 + } + return values + } + + private func varintText(_ raw: UInt64, type: ProtobufFieldType) -> String? { + switch type { + case .scalar(.int32): return String(Int32(truncatingIfNeeded: raw)) + case .scalar(.int64): return String(Int64(bitPattern: raw)) + case .scalar(.uint32): return String(UInt32(truncatingIfNeeded: raw)) + case .scalar(.uint64): return String(raw) + case .scalar(.sint32): return String(Int32(truncatingIfNeeded: ProtobufWireDecoder.zigzag(raw))) + case .scalar(.sint64): return String(ProtobufWireDecoder.zigzag(raw)) + case .scalar(.bool): return raw != 0 ? "true" : "false" + case .enumeration(let name): + let number = Int32(truncatingIfNeeded: raw) + return schema.enums[name]?.values[number] ?? String(number) + default: return nil + } + } + + private func fixed64Text(_ raw: UInt64, type: ProtobufFieldType) -> String? { + switch type { + case .scalar(.double): return ProtobufNumberText.double(Double(bitPattern: raw)) + case .scalar(.fixed64): return String(raw) + case .scalar(.sfixed64): return String(Int64(bitPattern: raw)) + default: return nil + } + } + + private func fixed32Text(_ raw: UInt32, type: ProtobufFieldType) -> String? { + switch type { + case .scalar(.float): return ProtobufNumberText.float(Float(bitPattern: raw)) + case .scalar(.fixed32): return String(raw) + case .scalar(.sfixed32): return String(Int32(bitPattern: raw)) + default: return nil + } + } + } +} + +// protobuf's SimpleDtoa and SimpleFtoa: the shortest of DBL_DIG or FLT_DIG digits that reads back, else the full count. +internal enum ProtobufNumberText { + static func double(_ value: Double) -> String { + if let special = special(value) { return special } + let short = String(format: "%.15g", value) + return Double(short) == value ? short : String(format: "%.17g", value) + } + + static func float(_ value: Float) -> String { + if let special = special(Double(value)) { return special } + let short = String(format: "%.6g", Double(value)) + return Float(short) == value ? short : String(format: "%.9g", Double(value)) + } + + private static func special(_ value: Double) -> String? { + if value.isNaN { return "nan" } + if value.isInfinite { return value < 0 ? "-inf" : "inf" } + return nil + } +} + +internal enum ProtobufSchemaTextWriter { + static func text(_ fields: [ProtobufTypedField], multiLine: Bool, limit: Int = .max) -> String { + var output = "" + write(fields, into: &output, multiLine: multiLine, depth: 0, limit: limit) + return multiLine ? output : output.trimmingCharacters(in: .whitespaces) + } + + static func scalarText(_ value: ProtobufTypedValue, limit: Int = .max) -> String? { + switch value { + case .scalar(let text): return text + case .string(let text): return ProtobufTextWriter.scalarText(.string(text), limit: limit) + case .bytes(let data): return ProtobufTextWriter.scalarText(.bytes(data), limit: limit) + case .unknown(let raw): return ProtobufTextWriter.scalarText(raw, limit: limit) + case .message, .truncated: return nil + } + } + + private static func write( + _ fields: [ProtobufTypedField], + into output: inout String, + multiLine: Bool, + depth: Int, + limit: Int + ) { + let indent = multiLine ? String(repeating: " ", count: depth) : "" + let separator = multiLine ? "\n" : " " + for field in fields { + let room = limit - output.utf8.count + guard room > 0 else { return } + switch field.value { + case .message(let members): + output += "\(indent)\(field.name) {\(separator)" + write(members, into: &output, multiLine: multiLine, depth: depth + 1, limit: limit) + output += "\(indent)}\(separator)" + case .unknown(let raw): + ProtobufTextWriter.write( + [ProtobufField(number: Int(field.name) ?? 0, value: raw)], + into: &output, multiLine: multiLine, depth: depth, limit: limit + ) + case .truncated: + output += "\(indent)…\(separator)" + case .scalar, .string, .bytes: + output += "\(indent)\(field.name): \(scalarText(field.value, limit: room) ?? "")\(separator)" + } + } + } +} + +internal enum ProtobufSchemaTreeBuilder { + private static let maxDisplayLength = 300 + + static func build(from message: ProtobufTypedMessage, formats: TreeSummaryFormats = .localized) -> ProtobufTreeNode { + var nodeCount = 0 + let fields = message.fields.filter { $0.value != .truncated } + return ProtobufTreeNode( + key: nil, keyPath: "", path: .root, kind: .message, + displayValue: "{\(formats.fields.text(fields.count))}", + typeBadge: message.typeName.split(separator: ".").last.map(String.init), + typedMembers: message.fields, + children: makeChildren(fields, keyPath: nil, path: .root, formats: formats, nodeCount: &nodeCount) + ) + } + + private static func makeChildren( + _ fields: [ProtobufTypedField], + keyPath: String?, + path: TreeNodePath, + formats: TreeSummaryFormats, + nodeCount: inout Int + ) -> [ProtobufTreeNode] { + var children: [ProtobufTreeNode] = [] + var siblings = TreeSiblingCounter() + for field in fields where field.value != .truncated { + guard nodeCount < TreeNodeLimits.maxNodes else { + children.append(ProtobufTreeNode( + key: nil, keyPath: "", path: path.appending(.truncationMarker), kind: .truncated, + displayValue: "… (\(fields.count - children.count) more)" + )) + break + } + nodeCount += 1 + let fieldKeyPath = keyPath.map { "\($0).\(field.name)" } ?? field.name + let fieldPath = path.appending(siblings.key(field.name)) + switch field.value { + case .message(let members): + let shown = members.filter { $0.value != .truncated } + children.append(ProtobufTreeNode( + key: field.name, keyPath: fieldKeyPath, path: fieldPath, kind: .message, + displayValue: "{\(formats.fields.text(shown.count))}", + typeBadge: field.typeName, + typedMembers: members, + children: makeChildren(shown, keyPath: fieldKeyPath, path: fieldPath, formats: formats, nodeCount: &nodeCount) + )) + case .unknown(let raw): + children.append(ProtobufTreeBuilder.node( + key: field.name, keyPath: fieldKeyPath, path: fieldPath, value: raw, formats: formats, nodeCount: &nodeCount + )) + case .string(let text): + let display = TreeDisplayText.quoted(text, limit: maxDisplayLength) + children.append(ProtobufTreeNode( + key: field.name, keyPath: fieldKeyPath, path: fieldPath, kind: .string, + displayValue: display.text, isDisplayCut: display.isCut, rawValue: text, typeBadge: field.typeName + )) + case .scalar, .bytes, .truncated: + let text = ProtobufSchemaTextWriter.scalarText(field.value) ?? "" + let display = TreeDisplayText.cut(text, limit: maxDisplayLength) + let kind: ProtobufTreeNode.Kind + if case .bytes = field.value { kind = .bytes } else { kind = .varint } + children.append(ProtobufTreeNode( + key: field.name, keyPath: fieldKeyPath, path: fieldPath, kind: kind, + displayValue: display.isCut ? display.text + "…" : display.text, + isDisplayCut: display.isCut, rawValue: text, typeBadge: field.typeName + )) + } + } + return children + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift index 980b986578..4d024265e7 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift @@ -49,7 +49,7 @@ internal enum ProtobufWireDecoder { return ProtobufMessage(fields: fields, framing: framing) } - private static func readFraming(_ reader: inout BinaryValueReader) throws -> ProtobufFraming { + static func readFraming(_ reader: inout BinaryValueReader) throws -> ProtobufFraming { _ = try reader.readByte() let schemaID = try reader.readUInt32() let start = reader.offset @@ -65,7 +65,14 @@ internal enum ProtobufWireDecoder { return ProtobufFraming(schemaID: schemaID, messageIndexes: indexes) } - private static func zigzag(_ value: UInt64) -> Int64 { + // A length-delimited value no schema names, read the way decode() reads one. + static func lengthDelimited(_ data: Data, maxValues: Int = .max) -> ProtobufFieldValue { + var parser = Parser(budget: maxValues) + var window = BinaryValueReader(data) + return parser.lengthDelimited(&window, depth: 0) + } + + static func zigzag(_ value: UInt64) -> Int64 { Int64(bitPattern: value >> 1) ^ -Int64(bitPattern: value & 1) } @@ -128,7 +135,7 @@ internal enum ProtobufWireDecoder { return result } - private mutating func lengthDelimited(_ window: inout BinaryValueReader, depth: Int) -> ProtobufFieldValue { + mutating func lengthDelimited(_ window: inout BinaryValueReader, depth: Int) -> ProtobufFieldValue { if window.remaining > 0, depth < ProtobufWireDecoder.maxDepth, budget > 0 { var attempt = self var nested = window diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift index 9ded656ca1..04c7aa4632 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift @@ -19,6 +19,10 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case igbinary // The schema is kept by AvroSchemaStore under this ID, so every per-column path can decode with it. case avro(schemaID: String) + // Kept by ProtobufSchemaStore, with the message type the values are read as. + case protobufSchema(schemaID: String, message: String) + // Each value names its schema by ID; the connection says which Schema Registry to fetch it from. + case schemaRegistry(connectionID: UUID) // Every format but Avro, which needs a schema chosen first. static let presetFormats: [ValueDisplayFormat] = [ @@ -27,6 +31,8 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { ] private static let avroPrefix = "avro:" + private static let protobufSchemaPrefix = "protobuf:" + private static let schemaRegistryPrefix = "schemaRegistry:" init?(rawValue: String) { switch rawValue { @@ -42,10 +48,20 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case "protobuf": self = .protobuf case "igbinary": self = .igbinary default: - guard rawValue.hasPrefix(Self.avroPrefix) else { return nil } - let schemaID = String(rawValue.dropFirst(Self.avroPrefix.count)) - guard AvroSchemaStore.isSchemaID(schemaID) else { return nil } - self = .avro(schemaID: schemaID) + if rawValue.hasPrefix(Self.avroPrefix) { + let schemaID = String(rawValue.dropFirst(Self.avroPrefix.count)) + guard AvroSchemaStore.isSchemaID(schemaID) else { return nil } + self = .avro(schemaID: schemaID) + } else if rawValue.hasPrefix(Self.protobufSchemaPrefix) { + let parts = rawValue.dropFirst(Self.protobufSchemaPrefix.count).split(separator: ":", maxSplits: 1) + guard parts.count == 2, AvroSchemaStore.isSchemaID(String(parts[0])), !parts[1].isEmpty else { return nil } + self = .protobufSchema(schemaID: String(parts[0]), message: String(parts[1])) + } else if rawValue.hasPrefix(Self.schemaRegistryPrefix), + let connectionID = UUID(uuidString: String(rawValue.dropFirst(Self.schemaRegistryPrefix.count))) { + self = .schemaRegistry(connectionID: connectionID) + } else { + return nil + } } } @@ -63,6 +79,8 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case .protobuf: return "protobuf" case .igbinary: return "igbinary" case .avro(let schemaID): return Self.avroPrefix + schemaID + case .protobufSchema(let schemaID, let message): return Self.protobufSchemaPrefix + schemaID + ":" + message + case .schemaRegistry(let connectionID): return Self.schemaRegistryPrefix + connectionID.uuidString } } @@ -96,6 +114,10 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case .avro(let schemaID): guard let name = AvroSchemaStore.shared.schema(id: schemaID)?.displayName else { return "Avro" } return String(format: String(localized: "Avro (%@)"), name) + case .protobufSchema(_, let message): + return String(format: String(localized: "Protobuf (%@)"), message.split(separator: ".").last.map(String.init) ?? message) + case .schemaRegistry: + return String(localized: "Schema Registry") } } @@ -105,6 +127,8 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case .protobuf: return .protobuf case .igbinary: return .igbinary case .avro(let schemaID): return .avro(schemaID: schemaID) + case .protobufSchema(let schemaID, let message): return .protobufSchema(schemaID: schemaID, message: message) + case .schemaRegistry(let connectionID): return .schemaRegistry(connectionID: connectionID) case .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: return nil } @@ -114,8 +138,11 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { switch self { case .raw: return [] - case .text, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro: + case .text, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro, .protobufSchema: return ["blob"] + // A framed value whose payload happens to be UTF-8 lands in a text column. + case .schemaRegistry: + return ["blob", "text"] case .uuid: return ["blob", "text"] case .unixTimestamp, .unixTimestampMillis: @@ -146,7 +173,7 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { // A binary column is worth searching exactly when one of these is on it; hex is not what anyone types into Find. var rendersBinaryAsText: Bool { switch self { - case .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro: + case .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: return true case .raw, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: return false @@ -163,6 +190,10 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { static func offersAvro(for columnType: ColumnType?, databaseType: DatabaseType? = nil) -> Bool { avro(schemaID: "").isApplicable(to: columnType, databaseType: databaseType) } + + static func offersProtobufSchema(for columnType: ColumnType?, databaseType: DatabaseType? = nil) -> Bool { + protobufSchema(schemaID: "", message: "").isApplicable(to: columnType, databaseType: databaseType) + } } enum ValueDisplayFormatColumnKey { diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift index 83e8941087..680082e70e 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift @@ -36,8 +36,10 @@ enum ValueDisplayFormatter { return formatAsTimestamp(rawValue, divideBy: 1_000) case .json, .phpSerialized: return rawValue - case .messagePack, .protobuf, .igbinary, .avro: - guard let data = rawValue.data(using: .isoLatin1), let decoder = format.binaryValueFormat else { return nil } + case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: + // A text column holds the bytes as UTF-8; a binary one as one Latin-1 character per byte. + let encoding: String.Encoding = columnType?.isBlobType == false ? .utf8 : .isoLatin1 + guard let data = rawValue.data(using: encoding), let decoder = format.binaryValueFormat else { return nil } return BinaryValueDecoding.text(data, format: decoder) } } @@ -54,7 +56,7 @@ enum ValueDisplayFormatter { guard rawValue.count == 16 else { return nil } let order: UUIDByteOrder = format == .uuidSwappedTime ? .swappedTime : .stored return insertUuidHyphens(order.uuidBytes(fromStored: [UInt8](rawValue)).hexEncoded) - case .messagePack, .protobuf, .igbinary, .avro: + case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: guard let decoder = format.binaryValueFormat else { return nil } return BinaryValueDecoding.text(rawValue, format: decoder) case .raw, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: diff --git a/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift b/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift new file mode 100644 index 0000000000..e0e70ccf8a --- /dev/null +++ b/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift @@ -0,0 +1,81 @@ +// +// ProtobufSchemaFilePanel.swift +// TablePro +// + +import AppKit + +internal struct ProtobufSchemaChoice { + let contents: Data + let message: String +} + +// A `.proto` file or a descriptor set, any extension. The panel's validate hook refuses anything else with the reason, +// and a file with several messages then asks which one the column's values are. +@MainActor +internal enum ProtobufSchemaFilePanel { + static func chooseSchema(in window: NSWindow) async -> ProtobufSchemaChoice? { + let panel = NSOpenPanel() + panel.allowedContentTypes = [] + panel.canChooseFiles = true + panel.canChooseDirectories = false + panel.allowsMultipleSelection = false + panel.prompt = String(localized: "Choose Schema") + panel.message = String(localized: "Choose the .proto file or descriptor set the values in this column were written with.") + let gate = ProtobufSchemaFileGate() + panel.delegate = gate + let response = await panel.presentAsSheet(for: window) + guard response == .OK, let contents = gate.contents, let schema = gate.schema else { return nil } + guard let message = await chooseMessage(in: schema, window: window) else { return nil } + return ProtobufSchemaChoice(contents: contents, message: message) + } + + private static func chooseMessage(in schema: ProtobufSchema, window: NSWindow) async -> String? { + let names = schema.messageNames + guard names.count > 1 else { return names.first } + let alert = NSAlert() + alert.messageText = String(localized: "Which message are the values?") + alert.informativeText = String(localized: "The file defines more than one message. Choose the one this column holds.") + let popUp = NSPopUpButton(frame: NSRect(x: 0, y: 0, width: 320, height: 26), pullsDown: false) + popUp.addItems(withTitles: names) + if let first = schema.rootMessages.first { popUp.selectItem(withTitle: first) } + alert.accessoryView = popUp + alert.addButton(withTitle: String(localized: "Choose")) + alert.addButton(withTitle: String(localized: "Cancel")) + let response = await alert.beginSheetModal(for: window) + return response == .alertFirstButtonReturn ? popUp.titleOfSelectedItem : nil + } +} + +@MainActor +private final class ProtobufSchemaFileGate: NSObject, NSOpenSavePanelDelegate { + private(set) var contents: Data? + private(set) var schema: ProtobufSchema? + + func panel(_ sender: Any, validate url: URL) throws { + let name = url.lastPathComponent + let size = (try? url.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0 + guard size <= ProtobufSchemaStore.maxFileBytes, let data = try? Data(contentsOf: url) else { + throw ProtobufSchemaFileError(fileName: name, reason: String(localized: "The file is larger than any schema.")) + } + do { + schema = try ProtobufSchema.parse(data) + } catch let error as ProtobufSchemaError { + throw ProtobufSchemaFileError(fileName: name, reason: error.reason) + } + contents = data + } +} + +private struct ProtobufSchemaFileError: LocalizedError { + let fileName: String + let reason: String + + var errorDescription: String? { + String(format: String(localized: "“%@” is not a usable Protobuf schema."), fileName) + } + + var recoverySuggestion: String? { + reason + } +} diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift new file mode 100644 index 0000000000..9f03c6d9fd --- /dev/null +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift @@ -0,0 +1,134 @@ +// +// SchemaRegistryCache.swift +// TablePro +// + +import Foundation +import os + +// Schemas fetched from a connection's Schema Registry, by connection and schema ID, for the session. A registry +// never changes the schema behind an ID, so a fetched one is kept; a failure is kept briefly, so a grid of values +// naming the same ID asks once and a registry that comes back is asked again. +// +// The grid reads synchronously: a miss starts the fetch and the grid redraws on `didLoadSchema`. The viewer awaits it. +internal final class SchemaRegistryCache: Sendable { + typealias EndpointProvider = @Sendable (UUID) async -> SchemaRegistryEndpoint? + + static let shared = SchemaRegistryCache() + + static let didLoadSchema = Notification.Name("com.TablePro.schemaRegistryCache.didLoadSchema") + static let connectionIDKey = "connectionID" + + static let failureLifetime: Duration = .seconds(30) + + private struct Key: Hashable { + let connectionID: UUID + let schemaID: UInt32 + } + + private enum Entry { + case loaded(RegistrySchema) + case failed(SchemaRegistryError, at: ContinuousClock.Instant) + } + + private struct State { + var entries: [Key: Entry] = [:] + var inFlight: [Key: Task, Never>] = [:] + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + private let endpoint: EndpointProvider + private let transport: SchemaRegistryClient.Transport + + init( + endpoint: @escaping EndpointProvider = SchemaRegistryCache.savedEndpoint, + transport: @escaping SchemaRegistryClient.Transport = SchemaRegistryClient.urlSessionTransport + ) { + self.endpoint = endpoint + self.transport = transport + } + + // The fields a connect resolves, secure ones and a linked credential profile's included. + static let savedEndpoint: EndpointProvider = { connectionID in + await MainActor.run { + guard let connection = ConnectionStorage.shared.loadConnections().first(where: { $0.id == connectionID }) else { + return nil + } + let fields = DatabaseDriverFactory.resolvedAdditionalFields(for: connection) ?? connection.additionalFields + return SchemaRegistryEndpoint(fields: fields) + } + } + + // nil while the schema is being fetched, which a miss starts. + func cachedSchema(connectionID: UUID, schemaID: UInt32) -> Result? { + let key = Key(connectionID: connectionID, schemaID: schemaID) + if let result = state.withLock({ current(key, in: &$0) }) { return result } + _ = start(key) + return nil + } + + func schema(connectionID: UUID, schemaID: UInt32) async -> Result { + let key = Key(connectionID: connectionID, schemaID: schemaID) + if let result = state.withLock({ current(key, in: &$0) }) { return result } + return await start(key).value + } + + // What the grid can use now; an expired failure is dropped so the next read fetches again. + private func current(_ key: Key, in state: inout State) -> Result? { + switch state.entries[key] { + case .loaded(let schema)?: + return .success(schema) + case .failed(let error, let at)?: + guard ContinuousClock.now - at >= Self.failureLifetime else { return .failure(error) } + state.entries[key] = nil + return nil + case nil: + return nil + } + } + + private func start(_ key: Key) -> Task, Never> { + state.withLock { state in + if let running = state.inFlight[key] { return running } + let task = Task { [endpoint, transport] in + let result = await Self.fetch(key, endpoint: endpoint, transport: transport) + self.finish(key, with: result) + return result + } + state.inFlight[key] = task + return task + } + } + + private func finish(_ key: Key, with result: Result) { + state.withLock { state in + state.inFlight[key] = nil + switch result { + case .success(let schema): state.entries[key] = .loaded(schema) + case .failure(let error): state.entries[key] = .failed(error, at: .now) + } + } + NotificationCenter.default.post( + name: Self.didLoadSchema, object: self, userInfo: [Self.connectionIDKey: key.connectionID] + ) + } + + private static func fetch( + _ key: Key, + endpoint: EndpointProvider, + transport: @escaping SchemaRegistryClient.Transport + ) async -> Result { + guard let resolved = await endpoint(key.connectionID) else { + return .failure(SchemaRegistryError( + reason: String(localized: "The connection names no Schema Registry URL. Add one in the connection's settings.") + )) + } + do { + return .success(try await SchemaRegistryClient(endpoint: resolved, transport: transport).schema(id: key.schemaID)) + } catch let error as SchemaRegistryError { + return .failure(error) + } catch { + return .failure(SchemaRegistryError(reason: error.localizedDescription)) + } + } +} diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift new file mode 100644 index 0000000000..de1150070d --- /dev/null +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift @@ -0,0 +1,207 @@ +// +// SchemaRegistryClient.swift +// TablePro +// + +import Foundation + +// The connection fields a driver declares to name its Schema Registry. Engine neutral: any plugin that declares +// them gets the Schema Registry display format. +internal enum SchemaRegistryConnectionField { + static let url = "schemaRegistryUrl" + static let username = "schemaRegistryUsername" + static let password = "schemaRegistryPassword" +} + +internal struct SchemaRegistryEndpoint: Equatable, Sendable { + let baseURL: URL + let username: String? + let password: String? + + // Only http and https: the URL is fetched from, so a file or custom scheme is refused here. + init?(fields: [String: String]) { + let text = (fields[SchemaRegistryConnectionField.url] ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), ["http", "https"].contains(scheme), + url.host?.isEmpty == false else { return nil } + baseURL = url + let username = fields[SchemaRegistryConnectionField.username] ?? "" + self.username = username.isEmpty ? nil : username + password = fields[SchemaRegistryConnectionField.password] + } +} + +internal enum RegistrySchema: Sendable { + case avro(AvroSchema) + case protobuf(ProtobufSchema) + // A JSON Schema value is JSON text behind the prefix; the schema adds nothing to reading it. + case json +} + +internal struct SchemaRegistryError: Error, Equatable, Sendable { + let reason: String +} + +// https://docs.confluent.io/platform/current/schema-registry/develop/api.html +internal struct SchemaRegistryClient: Sendable { + typealias Transport = @Sendable (URLRequest) async throws -> (Data, URLResponse) + + static let maxResponseBytes = 4_000_000 + static let maxReferences = 200 + static let timeout: TimeInterval = 15 + + let endpoint: SchemaRegistryEndpoint + let transport: Transport + + init(endpoint: SchemaRegistryEndpoint, transport: @escaping Transport = Self.urlSessionTransport) { + self.endpoint = endpoint + self.transport = transport + } + + private static let session: URLSession = { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForResource = 60 + return URLSession(configuration: configuration) + }() + + // Read up to the cap, so a registry that never stops sending cannot fill memory. + static let urlSessionTransport: Transport = { request in + let (bytes, response) = try await session.bytes(for: request, delegate: SameOriginRedirects()) + var data = Data() + for try await byte in bytes { + data.append(byte) + if data.count > maxResponseBytes { break } + } + return (data, response) + } + + func schema(id: UInt32) async throws -> RegistrySchema { + let root: SchemaResponse = try await get(["schemas", "ids", String(id)]) + switch root.schemaType?.uppercased() ?? "AVRO" { + case "AVRO": + var references: [String] = [] + var seen: Set = [] + try await collectReferences(root.references ?? [], into: &references, seen: &seen) + do { + return .avro(try AvroSchema.parse(root.schema, references: references)) + } catch let error as AvroSchemaError { + throw SchemaRegistryError(reason: error.reason) + } + case "PROTOBUF": + var references: [String] = [] + var seen: Set = [] + try await collectReferences(root.references ?? [], into: &references, seen: &seen) + do { + return .protobuf(try ProtobufSchema.parse(protoFiles: [root.schema] + references)) + } catch let error as ProtobufSchemaError { + throw SchemaRegistryError(reason: error.reason) + } + case "JSON": + return .json + case let other: + throw SchemaRegistryError(reason: String( + format: String(localized: "The Schema Registry names a schema type TablePro does not read: %@."), other + )) + } + } + + // Depth first, each subject version once and after the ones it references, so a type is defined before its users. + private func collectReferences( + _ references: [SchemaReference], + into texts: inout [String], + seen: inout Set + ) async throws { + for reference in references { + let key = "\(reference.subject)@\(reference.version)" + guard seen.insert(key).inserted else { continue } + guard seen.count <= Self.maxReferences else { + throw SchemaRegistryError(reason: String(localized: "The schema imports more files than TablePro reads.")) + } + let version: SchemaResponse = try await get(["subjects", reference.subject, "versions", String(reference.version)]) + try await collectReferences(version.references ?? [], into: &texts, seen: &seen) + texts.append(version.schema) + } + } + + private func get(_ components: [String]) async throws -> Response { + let request = request(components) + let data: Data + let response: URLResponse + do { + (data, response) = try await transport(request) + } catch { + throw SchemaRegistryError(reason: String( + format: String(localized: "TablePro could not reach the Schema Registry: %@"), error.localizedDescription + )) + } + guard data.count <= Self.maxResponseBytes else { + throw SchemaRegistryError(reason: String(localized: "The Schema Registry's answer is larger than any schema.")) + } + let status = (response as? HTTPURLResponse)?.statusCode ?? 200 + guard (200..<300).contains(status) else { + let message = (try? JSONDecoder().decode(ErrorResponse.self, from: data))?.message + ?? HTTPURLResponse.localizedString(forStatusCode: status) + throw SchemaRegistryError(reason: String( + format: String(localized: "The Schema Registry answered %lld: %@"), status, message + )) + } + do { + return try JSONDecoder().decode(Response.self, from: data) + } catch { + throw SchemaRegistryError(reason: String(localized: "The Schema Registry's answer is not a schema.")) + } + } + + // Each component is encoded on its own: a subject such as `demo/common.proto` holds a slash. + func request(_ components: [String]) -> URLRequest { + var allowed = CharacterSet.urlPathAllowed + allowed.remove("/") + let base = endpoint.baseURL.absoluteString.hasSuffix("/") + ? String(endpoint.baseURL.absoluteString.dropLast()) + : endpoint.baseURL.absoluteString + let path = components + .map { $0.addingPercentEncoding(withAllowedCharacters: allowed) ?? $0 } + .joined(separator: "/") + var request = URLRequest(url: URL(string: "\(base)/\(path)") ?? endpoint.baseURL, timeoutInterval: Self.timeout) + request.setValue("application/vnd.schemaregistry.v1+json, application/json", forHTTPHeaderField: "Accept") + if let username = endpoint.username { + let credentials = Data("\(username):\(endpoint.password ?? "")".utf8).base64EncodedString() + request.setValue("Basic \(credentials)", forHTTPHeaderField: "Authorization") + } + return request + } + + private struct SchemaResponse: Decodable { + let schema: String + let schemaType: String? + let references: [SchemaReference]? + } + + private struct SchemaReference: Decodable { + let subject: String + let version: Int + } + + private struct ErrorResponse: Decodable { + let message: String + } +} + +// The credentials go to the registry the connection names, so a redirect elsewhere, or down to http, is not followed. +private final class SameOriginRedirects: NSObject, URLSessionTaskDelegate { + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping @Sendable (URLRequest?) -> Void + ) { + guard let from = task.originalRequest?.url, let to = request.url, + from.scheme?.lowercased() == to.scheme?.lowercased(), + from.host?.lowercased() == to.host?.lowercased(), + from.port == to.port else { + completionHandler(nil) + return + } + completionHandler(request) + } +} diff --git a/apple/TablePro/Core/Storage/AvroSchemaStore.swift b/apple/TablePro/Core/Storage/AvroSchemaStore.swift index 3b3bdb9a1a..2527c70379 100644 --- a/apple/TablePro/Core/Storage/AvroSchemaStore.swift +++ b/apple/TablePro/Core/Storage/AvroSchemaStore.swift @@ -3,13 +3,8 @@ // TablePro // -import CryptoKit import Foundation -import os -// The schema files chosen for Avro columns, one file each, named by a hash of the text, so a column still -// decodes after the original file moves and the same file chosen twice is kept once. Read from the grid on -// the main actor and from the viewer off it. internal final class AvroSchemaStore: Sendable { static let shared = AvroSchemaStore( directory: AppStorageEnvironment.shared.supportDirectory.appendingPathComponent("AvroSchemas", isDirectory: true) @@ -22,47 +17,27 @@ internal final class AvroSchemaStore: Sendable { static let didRestoreSchema = Notification.Name("com.TablePro.avroSchemaStore.didRestoreSchema") static let schemaIDKey = "schemaID" - private static let fileExtension = "avsc" - - private struct State { - var parsed: [String: AvroSchema] = [:] - // Kept from a sweep: a schema can be added before the column's format is saved. - var addedThisLaunch: Set = [] - // Asked for and not found, so adding one later is a restore that grids and viewers have to hear about. - var missing: Set = [] - } - - private let directory: URL - private let state = OSAllocatedUnfairLock(initialState: State()) + private let files: SchemaFileCache init(directory: URL) { - self.directory = directory + files = SchemaFileCache(directory: directory, fileExtension: "avsc") { data in + guard let text = String(data: data, encoding: .utf8) else { + throw AvroSchemaError(reason: String(localized: "The file is not UTF-8 text.")) + } + return try AvroSchema.parse(text) + } } static func isSchemaID(_ text: String) -> Bool { - text.utf8.count == 32 && text.utf8.allSatisfy { (0x30...0x39).contains($0) || (0x61...0x66).contains($0) } + SchemaFileCache.isSchemaID(text) } static func schemaID(of text: String) -> String { - SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() + SchemaFileCache.schemaID(of: Data(text.utf8)) } - // Parses first, so nothing is kept for a file that is not a schema. func add(_ text: String) throws -> String { - let schema = try AvroSchema.parse(text) - let id = Self.schemaID(of: text) - let url = fileURL(id) - // A copy that went bad on disk is written over, or the next launch would find it missing again. - let contents = Data(text.utf8) - if (try? Data(contentsOf: url)) != contents { - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - try contents.write(to: url, options: .atomic) - } - let wasMissing = state.withLock { state -> Bool in - state.parsed[id] = schema - state.addedThisLaunch.insert(id) - return state.missing.remove(id) != nil - } + let (id, wasMissing) = try files.add(Data(text.utf8)) if wasMissing { NotificationCenter.default.post(name: Self.didRestoreSchema, object: self, userInfo: [Self.schemaIDKey: id]) } @@ -70,41 +45,14 @@ internal final class AvroSchemaStore: Sendable { } func schema(id: String) -> AvroSchema? { - if let cached = state.withLock({ $0.parsed[id] }) { return cached } - guard Self.isSchemaID(id), - let data = try? Data(contentsOf: fileURL(id)), - let text = String(data: data, encoding: .utf8), - let schema = try? AvroSchema.parse(text) else { - // An add() may have landed since the read failed; recorded as missing only if it did not. - return state.withLock { state in - if let added = state.parsed[id] { return added } - state.missing.insert(id) - return nil - } - } - state.withLock { $0.parsed[id] = schema } - return schema + files.schema(id: id) } var storedIDs: Set { - let names = (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] - return Set(names.compactMap { name in - let url = URL(fileURLWithPath: name) - guard url.pathExtension == Self.fileExtension else { return nil } - let id = url.deletingPathExtension().lastPathComponent - return Self.isSchemaID(id) ? id : nil - }) + files.storedIDs } func removeAll(except kept: Set) { - let addedThisLaunch = state.withLock { $0.addedThisLaunch } - for id in storedIDs where !kept.contains(id) && !addedThisLaunch.contains(id) { - try? FileManager.default.removeItem(at: fileURL(id)) - state.withLock { _ = $0.parsed.removeValue(forKey: id) } - } - } - - private func fileURL(_ id: String) -> URL { - directory.appendingPathComponent(id, isDirectory: false).appendingPathExtension(Self.fileExtension) + files.removeAll(except: kept) } } diff --git a/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift b/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift new file mode 100644 index 0000000000..bcd0d2a0ce --- /dev/null +++ b/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift @@ -0,0 +1,45 @@ +// +// ProtobufSchemaStore.swift +// TablePro +// + +import Foundation + +// A `.proto` file or a descriptor set, kept as chosen; ProtobufSchema.parse tells the two apart. +internal final class ProtobufSchemaStore: Sendable { + static let shared = ProtobufSchemaStore( + directory: AppStorageEnvironment.shared.supportDirectory.appendingPathComponent("ProtobufSchemas", isDirectory: true) + ) + + static let maxFileBytes = 4_000_000 + + static let didRestoreSchema = Notification.Name("com.TablePro.protobufSchemaStore.didRestoreSchema") + + private let files: SchemaFileCache + + init(directory: URL) { + files = SchemaFileCache(directory: directory, fileExtension: "pbschema") { try ProtobufSchema.parse($0) } + } + + func add(_ contents: Data) throws -> String { + let (id, wasMissing) = try files.add(contents) + if wasMissing { + NotificationCenter.default.post( + name: Self.didRestoreSchema, object: self, userInfo: [AvroSchemaStore.schemaIDKey: id] + ) + } + return id + } + + func schema(id: String) -> ProtobufSchema? { + files.schema(id: id) + } + + var storedIDs: Set { + files.storedIDs + } + + func removeAll(except kept: Set) { + files.removeAll(except: kept) + } +} diff --git a/apple/TablePro/Core/Storage/SchemaFileCache.swift b/apple/TablePro/Core/Storage/SchemaFileCache.swift new file mode 100644 index 0000000000..d36c48edce --- /dev/null +++ b/apple/TablePro/Core/Storage/SchemaFileCache.swift @@ -0,0 +1,96 @@ +// +// SchemaFileCache.swift +// TablePro +// + +import CryptoKit +import Foundation +import os + +// The schema files chosen for columns, one file each, named by a hash of the contents, so a column still decodes +// after the original file moves and the same file chosen twice is kept once. Read from the grid on the main actor +// and from the viewer off it. +internal final class SchemaFileCache: Sendable { + private struct State { + var parsed: [String: Schema] = [:] + // Kept from a sweep: a schema can be added before the column's format is saved. + var addedThisLaunch: Set = [] + // Asked for and not found, so adding one later is a restore that grids and viewers have to hear about. + var missing: Set = [] + } + + private let directory: URL + private let fileExtension: String + private let parse: @Sendable (Data) throws -> Schema + private let state = OSAllocatedUnfairLock(initialState: State()) + + init(directory: URL, fileExtension: String, parse: @escaping @Sendable (Data) throws -> Schema) { + self.directory = directory + self.fileExtension = fileExtension + self.parse = parse + } + + static func isSchemaID(_ text: String) -> Bool { + text.utf8.count == 32 && text.utf8.allSatisfy { (0x30...0x39).contains($0) || (0x61...0x66).contains($0) } + } + + static func schemaID(of contents: Data) -> String { + SHA256.hash(data: contents).prefix(16).map { String(format: "%02x", $0) }.joined() + } + + // Parses first, so nothing is kept for a file that is not a schema. `wasMissing` says a reader asked for it before. + func add(_ contents: Data) throws -> (id: String, wasMissing: Bool) { + let schema = try parse(contents) + let id = Self.schemaID(of: contents) + let url = fileURL(id) + // A copy that went bad on disk is written over, or the next launch would find it missing again. + if (try? Data(contentsOf: url)) != contents { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try contents.write(to: url, options: .atomic) + } + let wasMissing = state.withLock { state -> Bool in + state.parsed[id] = schema + state.addedThisLaunch.insert(id) + return state.missing.remove(id) != nil + } + return (id, wasMissing) + } + + func schema(id: String) -> Schema? { + if let cached = state.withLock({ $0.parsed[id] }) { return cached } + guard Self.isSchemaID(id), + let data = try? Data(contentsOf: fileURL(id)), + let schema = try? parse(data) else { + // An add() may have landed since the read failed; recorded as missing only if it did not. + return state.withLock { state in + if let added = state.parsed[id] { return added } + state.missing.insert(id) + return nil + } + } + state.withLock { $0.parsed[id] = schema } + return schema + } + + var storedIDs: Set { + let names = (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] + return Set(names.compactMap { name in + let url = URL(fileURLWithPath: name) + guard url.pathExtension == fileExtension else { return nil } + let id = url.deletingPathExtension().lastPathComponent + return Self.isSchemaID(id) ? id : nil + }) + } + + func removeAll(except kept: Set) { + let addedThisLaunch = state.withLock { $0.addedThisLaunch } + for id in storedIDs where !kept.contains(id) && !addedThisLaunch.contains(id) { + try? FileManager.default.removeItem(at: fileURL(id)) + state.withLock { _ = $0.parsed.removeValue(forKey: id) } + } + } + + private func fileURL(_ id: String) -> URL { + directory.appendingPathComponent(id, isDirectory: false).appendingPathExtension(fileExtension) + } +} diff --git a/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift b/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift index fbc2b90b7b..9921cf8941 100644 --- a/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift +++ b/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift @@ -19,10 +19,16 @@ internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { private let store: KeyValueStore private let schemas: AvroSchemaStore + private let protobufSchemas: ProtobufSchemaStore - init(defaults: KeyValueStore = AppStorageEnvironment.shared.defaults, schemas: AvroSchemaStore = .shared) { + init( + defaults: KeyValueStore = AppStorageEnvironment.shared.defaults, + schemas: AvroSchemaStore = .shared, + protobufSchemas: ProtobufSchemaStore = .shared + ) { store = defaults self.schemas = schemas + self.protobufSchemas = protobufSchemas } func save(_ formats: [String: ValueDisplayFormat], for scope: TableScope) { @@ -108,14 +114,20 @@ internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { // A schema file lives as long as a saved format names it. private func sweepSchemas() { - guard !schemas.storedIDs.isEmpty else { return } - var named: Set = [] + guard !schemas.storedIDs.isEmpty || !protobufSchemas.storedIDs.isEmpty else { return } + var avro: Set = [] + var protobuf: Set = [] for key in store.keys(withPrefix: Self.keyPrefix) where key.hasSuffix(Self.addedSuffix) { for name in (Self.names(store.dataValue(forKey: key)) ?? [:]).values { - if case .avro(let schemaID)? = ValueDisplayFormat(rawValue: name) { named.insert(schemaID) } + switch ValueDisplayFormat(rawValue: name) { + case .avro(let schemaID)?: avro.insert(schemaID) + case .protobufSchema(let schemaID, _)?: protobuf.insert(schemaID) + default: break + } } } - schemas.removeAll(except: named) + schemas.removeAll(except: avro) + protobufSchemas.removeAll(except: protobuf) } @discardableResult diff --git a/apple/TablePro/Models/UI/ProtobufTreeNode.swift b/apple/TablePro/Models/UI/ProtobufTreeNode.swift index 737364b0d8..629b96c603 100644 --- a/apple/TablePro/Models/UI/ProtobufTreeNode.swift +++ b/apple/TablePro/Models/UI/ProtobufTreeNode.swift @@ -5,7 +5,8 @@ import Foundation -// Without a schema the bytes give only a field number, the key, and a wire type, the badge. +// Without a schema the bytes give only a field number, the key, and a wire type, the badge. A schema gives the +// field's name and declared type instead. internal struct ProtobufTreeNode: Identifiable { internal enum Kind: Equatable { case message @@ -61,6 +62,8 @@ internal struct ProtobufTreeNode: Identifiable { let isDisplayCut: Bool let rawValue: String? let members: [ProtobufField] + let typeBadge: String? + let typedMembers: [ProtobufTypedField] let children: [ProtobufTreeNode] init( @@ -73,6 +76,8 @@ internal struct ProtobufTreeNode: Identifiable { isDisplayCut: Bool = false, rawValue: String? = nil, members: [ProtobufField] = [], + typeBadge: String? = nil, + typedMembers: [ProtobufTypedField] = [], children: [ProtobufTreeNode] = [] ) { self.id = id @@ -84,6 +89,8 @@ internal struct ProtobufTreeNode: Identifiable { self.isDisplayCut = isDisplayCut self.rawValue = rawValue self.members = members + self.typeBadge = typeBadge + self.typedMembers = typedMembers self.children = children } } @@ -101,7 +108,7 @@ extension ProtobufTreeNode: FilterableTreeNode { } internal var badgeLabel: String { - kind.badgeLabel + typeBadge ?? kind.badgeLabel } internal var isTruncationMarker: Bool { @@ -111,6 +118,7 @@ extension ProtobufTreeNode: FilterableTreeNode { internal var copyableValue: String { switch kind { case .message, .group: + guard typedMembers.isEmpty else { return ProtobufSchemaTextWriter.text(typedMembers, multiLine: false) } return ProtobufTextWriter.text(members, multiLine: false) case .varint, .fixed64, .fixed32, .string, .bytes, .truncated: return rawValue ?? displayValue @@ -131,7 +139,7 @@ extension ProtobufTreeNode: FilterableTreeNode { string: kind == .string ? rawValue : nil, isCut: isDisplayCut, tone: kind.tone, - typeBadge: kind.badgeLabel, + typeBadge: badgeLabel, visibilityBadge: nil ) } @@ -140,7 +148,7 @@ extension ProtobufTreeNode: FilterableTreeNode { ProtobufTreeNode( id: id, key: key, keyPath: keyPath, path: path, kind: kind, displayValue: displayValue, isDisplayCut: isDisplayCut, rawValue: rawValue, members: members, - children: children + typeBadge: typeBadge, typedMembers: typedMembers, children: children ) } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 117f59b770..6615fe86e2 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -942,6 +942,9 @@ }, "“%@” is not a usable Avro schema." : { + }, + "“%@” is not a usable Protobuf schema." : { + }, "\"%@\" is not a valid %@. Example: %@" : { "extractionState" : "manual", @@ -15105,6 +15108,9 @@ } } } + }, + "A comment is never closed." : { + }, "A comparison is already running." : { "localizations" : { @@ -15858,6 +15864,9 @@ } } } + }, + "A message is never closed." : { + }, "A MongoDB field has no type, default or nullability to change. Only its name can change." : { "extractionState" : "manual", @@ -16832,6 +16841,9 @@ } } } + }, + "A string is never closed." : { + }, "A string literal is missing its closing quote." : { "extractionState" : "manual", @@ -24053,6 +24065,9 @@ "An auto-increment column takes its values from a sequence, so it cannot also have a default." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "An enum is never closed." : { + }, "An enum's labels in declaration order" : { "localizations" : { @@ -38409,6 +38424,9 @@ }, "Choose Schema" : { + }, + "Choose the .proto file or descriptor set the values in this column were written with." : { + }, "Choose the Avro schema the values in this column were written with." : { @@ -108858,6 +108876,9 @@ } } } + }, + "Messages are nested too deeply." : { + }, "Metadata" : { "localizations" : { @@ -138154,6 +138175,12 @@ } } } + }, + "Protobuf (%@)" : { + + }, + "Protobuf Schema…" : { + }, "Protocol" : { "extractionState" : "manual", @@ -156694,6 +156721,16 @@ } } }, + "Schema %lld could not be read from the Schema Registry. %@ Showing the stored bytes." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Schema %1$lld could not be read from the Schema Registry. %2$@ Showing the stored bytes." + } + } + } + }, "Schema and table privileges in this database are only visible when the connection is using it." : { "localizations" : { "fr" : { @@ -157191,6 +157228,9 @@ } } } + }, + "Schema Registry" : { + }, "Schema Registry framing: schema ID %lld" : { @@ -157205,6 +157245,18 @@ } } }, + "Schema Registry Password" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Schema Registry URL" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Schema Registry User" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Schema response was empty." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -179242,6 +179294,9 @@ } } } + }, + "TablePro could not reach the Schema Registry: %@" : { + }, "TablePro could not read the database's own name from the connection." : { "localizations" : { @@ -184781,6 +184836,9 @@ } } } + }, + "The connection names no Schema Registry URL. Add one in the connection's settings." : { + }, "The connection stopped responding." : { "localizations" : { @@ -185733,6 +185791,9 @@ } } } + }, + "The descriptor set names an unknown field type %llu." : { + }, "The destination has no table named %@" : { "localizations" : { @@ -186323,10 +186384,19 @@ } } } + }, + "The file defines more than one message. Choose the one this column holds." : { + + }, + "The file defines no message." : { + }, "The file does not declare a theme format version." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The file ends in the middle of a statement." : { + }, "The file goes back to its last committed version. You can't undo this action." : { "localizations" : { @@ -186455,6 +186525,12 @@ }, "The file is larger than any Avro schema." : { + }, + "The file is larger than any schema." : { + + }, + "The file is neither a .proto file nor a descriptor set." : { + }, "The file is not a JSON object." : { @@ -186475,6 +186551,9 @@ } } } + }, + "The file is too large." : { + }, "The file is too large. A theme file can be at most %@." : { "extractionState" : "manual", @@ -189986,18 +190065,46 @@ }, "The schema changes staged in this diagram will be lost." : { + }, + "The schema defines more messages than TablePro reads." : { + }, "The schema defines type %@ twice." : { }, "The schema has a type that is not a name, an object or a union." : { + }, + "The schema imports more files than TablePro reads." : { + }, "The schema is not valid JSON." : { + }, + "The schema is still being fetched." : { + }, "The schema names type %@ before defining it." : { + }, + "The Schema Registry answered %lld: %@" : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "The Schema Registry answered %1$lld: %2$@" + } + } + } + }, + "The Schema Registry names a schema type TablePro does not read: %@." : { + + }, + "The Schema Registry's answer is larger than any schema." : { + + }, + "The Schema Registry's answer is not a schema." : { + }, "The script already ran. Compare again to see where the target stands." : { "localizations" : { @@ -193179,6 +193286,9 @@ } } } + }, + "The value does not start with the Schema Registry prefix: a zero byte and a schema ID." : { + }, "The value ends early, at byte %lld." : { @@ -197849,6 +197959,9 @@ }, "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…" : { + }, + "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Protobuf Schema…" : { + }, "This Mac no longer holds a seat." : { "localizations" : { @@ -204665,6 +204778,9 @@ } } } + }, + "Unexpected “%@” in the .proto file." : { + }, "Unexpected %@ in a CONSUME statement." : { "extractionState" : "manual", @@ -204689,6 +204805,9 @@ } } } + }, + "Unexpected character “%@”." : { + }, "Unexpected response from the SSO portal for profile \"%@\"." : { "extractionState" : "manual", @@ -213691,6 +213810,9 @@ } } } + }, + "Which message are the values?" : { + }, "Whitespace in text" : { "localizations" : { diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift index 04815f481b..6718a45cce 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift @@ -87,8 +87,11 @@ extension DataGridViewController { } private func displayAsItem(for column: DataGridColumn) -> NSMenuItem? { - let formats = ValueDisplayFormat.applicableFormats(for: column.type, databaseType: model.configuration.traits.databaseType) - guard formats.count > 1 else { return nil } + let databaseType = model.configuration.traits.databaseType + let formats = ValueDisplayFormat.applicableFormats(for: column.type, databaseType: databaseType) + guard formats.count > 1 || schemaRegistryFormat?.isApplicable(to: column.type, databaseType: databaseType) == true else { + return nil + } let current = model.displayFormat(for: column) ?? .raw let submenu = NSMenu() for format in formats { @@ -100,7 +103,7 @@ extension DataGridViewController { item.state = format == current ? .on : .off submenu.addItem(item) } - if ValueDisplayFormat.offersAvro(for: column.type, databaseType: model.configuration.traits.databaseType) { + if ValueDisplayFormat.offersAvro(for: column.type, databaseType: databaseType) { if case .avro = current { let item = menuItem( current.displayName, @@ -112,11 +115,40 @@ extension DataGridViewController { } submenu.addItem(menuItem(String(localized: "Avro Schema…"), #selector(chooseAvroSchemaFromMenu(_:)), column.id)) } + if ValueDisplayFormat.offersProtobufSchema(for: column.type, databaseType: databaseType) { + if case .protobufSchema = current { + let item = menuItem( + current.displayName, + #selector(setDisplayFormatFromMenu(_:)), + DataGridDisplayFormatChoice(columnID: column.id, format: current) + ) + item.state = .on + submenu.addItem(item) + } + submenu.addItem(menuItem(String(localized: "Protobuf Schema…"), #selector(chooseProtobufSchemaFromMenu(_:)), column.id)) + } + if let registry = schemaRegistryFormat, registry.isApplicable(to: column.type, databaseType: databaseType) { + let item = menuItem( + registry.displayName, + #selector(setDisplayFormatFromMenu(_:)), + DataGridDisplayFormatChoice(columnID: column.id, format: registry) + ) + item.state = registry == current ? .on : .off + submenu.addItem(item) + } let item = NSMenuItem(title: String(localized: "Display As"), action: nil, keyEquivalent: "") item.submenu = submenu return item } + // Offered only where the connection names a Schema Registry to fetch the schemas from. + private var schemaRegistryFormat: ValueDisplayFormat? { + guard let connectionID = model.configuration.traits.connectionScope?.connectionId, + let fields = DatabaseManager.shared.session(for: connectionID)?.connection.additionalFields, + SchemaRegistryEndpoint(fields: fields) != nil else { return nil } + return .schemaRegistry(connectionID: connectionID) + } + // Refused, with the reason, when the frozen run would leave too little of the grid to scroll. private func freezeItems(position: Int, columnID: DataGridColumnID) -> [NSMenuItem] { var items: [NSMenuItem] = [] @@ -155,6 +187,7 @@ extension DataGridViewController { #selector(DataGridViewController.clearAllValueFiltersFromMenu(_:)), #selector(DataGridViewController.setDisplayFormatFromMenu(_:)), #selector(DataGridViewController.chooseAvroSchemaFromMenu(_:)), + #selector(DataGridViewController.chooseProtobufSchemaFromMenu(_:)), #selector(DataGridViewController.sizeColumnToFitFromMenu(_:)), #selector(DataGridViewController.sizeAllColumnsToFitFromMenu(_:)), #selector(DataGridViewController.hideColumnFromMenu(_:)), @@ -267,6 +300,20 @@ extension DataGridViewController { } } + // A file with several messages asks which one the values are; the schema is copied in like Avro's. + @objc func chooseProtobufSchemaFromMenu(_ sender: Any?) { + guard let columnID = menuColumn(sender)?.id, let window = view.window else { return } + Task { @MainActor [weak self] in + guard let choice = await ProtobufSchemaFilePanel.chooseSchema(in: window) else { return } + do { + let schemaID = try ProtobufSchemaStore.shared.add(choice.contents) + self?.setDisplayFormat(.protobufSchema(schemaID: schemaID, message: choice.message), columnID: columnID) + } catch { + NSAlert(error: error).beginSheetModal(for: window, completionHandler: nil) + } + } + } + @objc func sizeColumnToFitFromMenu(_ sender: Any?) { guard let column = menuColumn(sender) else { return } sizeColumnToFit(column) diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController.swift b/apple/TablePro/Views/DataGrid/DataGridViewController.swift index dd7176f7af..f243d5d69b 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController.swift @@ -219,6 +219,10 @@ final class DataGridViewController: NSViewController, DataGridMountedView { observers.append(center.addObserver(forName: DataGridAccessibility.didActivateNotification, object: nil, queue: nil) { [weak self] _ in Task { @MainActor [weak self] in self?.remountAccessibilityCells() } }) + observers.append(center.addObserver(forName: SchemaRegistryCache.didLoadSchema, object: nil, queue: .main) { [weak self] note in + guard let connectionID = note.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID else { return } + MainActor.assumeIsolated { self?.model.schemaRegistryDidLoad(connectionID: connectionID) } + }) let scrollView = main.scrollView observers.append(center.addObserver(forName: NSScrollView.willStartLiveScrollNotification, object: scrollView, queue: .main) { [weak self] _ in MainActor.assumeIsolated { self?.pausePrewarm() } diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index e0e6df95ae..7bf51cb15d 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -48,6 +48,11 @@ internal struct BinaryValueViewerView: View { notification.userInfo?[AvroSchemaStore.schemaIDKey] as? String == schemaID else { return } schemaRevision += 1 } + .onReceive(NotificationCenter.default.publisher(for: ProtobufSchemaStore.didRestoreSchema).receive(on: RunLoop.main)) { notification in + guard case .protobufSchema(let schemaID, _) = format, + notification.userInfo?[AvroSchemaStore.schemaIDKey] as? String == schemaID else { return } + schemaRevision += 1 + } } private struct TaskKey: Equatable { @@ -110,10 +115,20 @@ internal struct BinaryValueViewerView: View { hexBody } case .missingSchema?: + VStack(spacing: 0) { + banner(missingSchemaMessage, systemImage: "questionmark.folder") + Divider() + hexBody + } + case .registryUnavailable(let schemaID, let reason)?: VStack(spacing: 0) { banner( - String(localized: "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…"), - systemImage: "questionmark.folder" + String( + format: String(localized: "Schema %lld could not be read from the Schema Registry. %@ Showing the stored bytes."), + Int(schemaID), + reason + ), + systemImage: "exclamationmark.triangle" ) Divider() hexBody @@ -173,6 +188,13 @@ internal struct BinaryValueViewerView: View { HexDumpDisplayView(text: hexDump, font: themeEngine.valueFont) } + private var missingSchemaMessage: String { + if case .protobufSchema = format { + return String(localized: "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Protobuf Schema…") + } + return String(localized: "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…") + } + private func failureBanner(_ error: BinaryValueDecodeError) -> some View { banner( String(format: String(localized: "Not valid %@. %@ Showing the stored bytes."), format.displayName, error.reason), @@ -202,6 +224,11 @@ internal struct BinaryValueViewerView: View { guard decodedKey != key else { return } let data = data let format = format + // The grid fetches a schema without waiting for it; the viewer waits, so it never shows a fetch in progress. + if case .schemaRegistry(let connectionID) = format, let schemaID = try? SchemaRegistryFrame.schemaID(data) { + _ = await SchemaRegistryCache.shared.schema(connectionID: connectionID, schemaID: schemaID) + guard !Task.isCancelled else { return } + } let (decoded, dump) = await Task.detached(priority: .userInitiated) { ( BinaryValueDecoding.document(data, format: format), diff --git a/apple/TablePro/Views/Results/CellInteractionResolver.swift b/apple/TablePro/Views/Results/CellInteractionResolver.swift index 5c9be4a713..8dcec9664e 100644 --- a/apple/TablePro/Views/Results/CellInteractionResolver.swift +++ b/apple/TablePro/Views/Results/CellInteractionResolver.swift @@ -99,7 +99,7 @@ internal struct CellInteractionResolver { case .phpSerialized: return .viewPhpSerialized case .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .messagePack, .protobuf, - .igbinary, .avro, .none: + .igbinary, .avro, .protobufSchema, .schemaRegistry, .none: return plainText(for: context, isReadOnly: isReadOnly) } } diff --git a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift index 13bed1dadc..57ab627a1e 100644 --- a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift +++ b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift @@ -51,7 +51,7 @@ internal enum FieldEditorResolver { return .phpSerialized case .json: return .json - case .messagePack, .protobuf, .igbinary, .avro: + case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: // A NULL keeps the hex editor, the way bytes get into it. if type.isBlobType || isBinaryValue, originalValue != nil, let format = override.binaryValueFormat { return .binaryValue(format) diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift index 44057ed83b..dce228b129 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift @@ -175,4 +175,15 @@ struct BinaryDisplayFormatTests { } #expect(BinaryValueDecoding.text(Data([0x02]), format: format) == nil) } + + @Test("A Protobuf schema column renders field names in the cell and falls back to hex on a broken value") + func protobufSchemaRenders() throws { + let blob = ColumnType.blob(rawType: "BLOB") + let schemaID = try ProtobufSchemaStore.shared.add(Data("syntax = \"proto3\"; message Point { sint32 x = 1; string label = 2; }".utf8)) + let format = ValueDisplayFormat.protobufSchema(schemaID: schemaID, message: "Point") + #expect(format.displayName.contains("Point")) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("08051202686b")), columnType: blob, displayFormat: format) + == #"x: -3 label: "hk""#) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("0f")), columnType: blob, displayFormat: format) == "0x0F") + } } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift new file mode 100644 index 0000000000..ed0043f61d --- /dev/null +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -0,0 +1,220 @@ +// +// ProtobufSchemaDecoderTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +// Fixtures written by `protoc --encode` (libprotoc 36.2); every expected text is `protoc --decode` output for them. +struct ProtobufSchemaDecoderTests { + private static let shopProto = """ + syntax = "proto3"; + package shop; + + // Every scalar type, an enum, packed and unpacked repeats, a map, a oneof and nesting. + message Item { + int32 i32 = 1; + int64 i64 = 2; + uint32 u32 = 3; + uint64 u64 = 4; + sint32 s32 = 5; + sint64 s64 = 6; + fixed32 f32 = 7; + fixed64 f64 = 8; + sfixed32 sf32 = 9; + sfixed64 sf64 = 10; + float fl = 11; + double db = 12; + bool flag = 13; + string name = 14; + bytes raw = 15; + Color color = 16; + repeated int32 packed = 17; + repeated string labels = 18; + map inners = 19; + Inner inner = 20; + oneof choice { + string text = 21; + int32 number = 22; + } + message Inner { + string note = 1; + repeated Color colors = 2; + } + } + + enum Color { + RED = 0; + GREEN = 1; + BLUE = 2; + } + + message Other { + string x = 1; + } + """ + + // protoc --descriptor_set_out=shop.desc shop.proto + private static let shopDescriptorSet = "0ae7050a0a73686f702e70726f746f120473686f70228c050a044974656d12100a03693332180120012805520369333212100a036936341802200128" + + "03520369363412100a0375333218032001280d520375333212100a03753634180420012804520375363412100a037333321805200128115203733332" + + "12100a03733634180620012812520373363412100a03663332180720012807520366333212100a03663634180820012806520366363412120a047366" + + "333218092001280f52047366333212120a0473663634180a20012810520473663634120e0a02666c180b200128025202666c120e0a026462180c2001" + + "28015202646212120a04666c6167180d200128085204666c616712120a046e616d65180e2001280952046e616d6512100a03726177180f2001280c52" + + "0372617712210a05636f6c6f7218102001280e320b2e73686f702e436f6c6f725205636f6c6f7212160a067061636b65641811200328055206706163" + + "6b656412160a066c6162656c7318122003280952066c6162656c73122e0a06696e6e65727318132003280b32162e73686f702e4974656d2e496e6e65" + + "7273456e7472795206696e6e65727312260a05696e6e657218142001280b32102e73686f702e4974656d2e496e6e65725205696e6e657212140a0474" + + "657874181520012809480052047465787412180a066e756d626572181620012805480052066e756d6265721a4b0a0b496e6e657273456e7472791210" + + "0a036b657918012001280952036b657912260a0576616c756518022001280b32102e73686f702e4974656d2e496e6e6572520576616c75653a023801" + + "1a400a05496e6e657212120a046e6f746518012001280952046e6f746512230a06636f6c6f727318022003280e320b2e73686f702e436f6c6f725206" + + "636f6c6f727342080a0663686f69636522150a054f74686572120c0a01781801200128095201782a250a05436f6c6f7212070a03524544100012090a" + + "05475245454e100112080a04424c55451002620670726f746f33" + + private static let item = "08fbffffffffffffffff011080ccbbbcdeffffffff011880d0acf30e208080a0a89c94b6e6f901280d30ffef99c589043d7b00000041c801000000" + + "0000004df8ffffff51f7ffffffffffffff5d0000c03f619a9999999999b93f680172075461622250726f7a030102ff8001028a01040102ac0292010161" + + "920101629a010d0a026b3112070a016e12020102a201060a0464656570b0012a" + + private static let itemText = """ + i32: -5 + i64: -9000000000 + u32: 4000000000 + u64: 18000000000000000000 + s32: -7 + s64: -70000000000 + f32: 123 + f64: 456 + sf32: -8 + sf64: -9 + fl: 1.5 + db: 0.1 + flag: true + name: "Tab\\"Pro" + raw: "\\001\\002\\377" + color: BLUE + packed: 1 + packed: 2 + packed: 300 + labels: "a" + labels: "b" + inners { + key: "k1" + value { + note: "n" + colors: GREEN + colors: BLUE + } + } + inner { + note: "deep" + } + number: 42 + + """ + + private static func text(_ hex: String, schema: ProtobufSchema, message: String?, multiLine: Bool = true) throws -> String { + let decoded = try ProtobufSchemaDecoder.decode(BinaryFixture.data(hex), schema: schema, message: message) + return ProtobufSchemaTextWriter.text(decoded.fields, multiLine: multiLine) + } + + @Test("A .proto file reads every type the way protoc --decode prints it") + func protoFile() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + #expect(schema.messageNames == ["shop.Item", "shop.Item.Inner", "shop.Other"]) + #expect(try Self.text(Self.item, schema: schema, message: "shop.Item") == Self.itemText) + } + + @Test("A descriptor set from protoc reads the same as the .proto it was built from") + func descriptorSet() throws { + let schema = try ProtobufSchema.parse(BinaryFixture.data(Self.shopDescriptorSet)) + #expect(schema == (try ProtobufSchema.parse(Data(Self.shopProto.utf8)))) + #expect(try Self.text(Self.item, schema: schema, message: "shop.Item") == Self.itemText) + } + + @Test("Fields the message does not declare, or declares with another wire type, show by number") + func unknownFields() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + let text = try Self.text(Self.item, schema: schema, message: "shop.Other") + let lines = text.split(separator: "\n").map(String.init) + #expect(lines.prefix(3) == ["1: 18446744073709551611", "2: 18446744064709551616", "3: 4000000000"]) + #expect(lines.contains("7: 0x0000007b")) + #expect(lines.contains(#"17: "\001\002\254\002""#)) + #expect(lines.last == "22: 42") + } + + @Test("A proto2 group and packed and unpacked repeats") + func proto2() throws { + let proto = """ + syntax = "proto2"; + package legacy; + + message Msg { + optional int32 id = 1; + repeated group Entry = 2 { + optional string k = 3; + } + repeated int32 nums = 4 [packed = true]; + repeated int32 loose = 5; + optional float ratio = 6; + } + """ + let schema = try ProtobufSchema.parse(Data(proto.utf8)) + let expected = "id: 1\nEntry {\n k: \"a\"\n}\nEntry {\n k: \"b\"\n}\nnums: 5\nnums: 6\nloose: 7\nloose: 8\nratio: 0.1\n" + #expect(try Self.text("0801131a016114131a016214220205062807280835cdcccc3d", schema: schema, message: "legacy.Msg") == expected) + } + + @Test("The tree names each field and badges it with its declared type") + func tree() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(Self.item), schema: schema, message: "shop.Item") + let root = ProtobufSchemaTreeBuilder.build(from: message, formats: BinaryFixture.english) + #expect(root.badgeLabel == "Item") + let color = try #require(root.children.first { $0.key == "color" }) + #expect(color.badgeLabel == "Color") + #expect(color.displayValue == "BLUE") + let inner = try #require(root.children.first { $0.key == "inner" }) + #expect(inner.badgeLabel == "Inner") + #expect(inner.copyableValue == #"note: "deep""#) + #expect(root.children.first { $0.key == "s64" }?.badgeLabel == "sint64") + } + + @Test("The one-line text a cell shows") + func singleLine() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + #expect(try Self.text("a201060a0464656570", schema: schema, message: "shop.Item", multiLine: false) == #"inner { note: "deep" }"#) + } + + @Test("A singular message sent twice is merged, as a parser does") + func mergesSingularMessages() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + #expect(try Self.text("a201030a0161a20103120101", schema: schema, message: "shop.Item") + == "inner {\n note: \"a\"\n colors: GREEN\n}\n") + } + + @Test("A cell stops reading a packed field at its budget") + func packedFieldBudget() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + let packed = "8a01" + "c801" + String(repeating: "01", count: 200) + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(packed), schema: schema, message: "shop.Item", maxValues: 10) + #expect(message.fields.count < 20) + #expect(message.fields.last?.value == .truncated) + } + + @Test("A broken value is refused, so the cell falls back to hex") + func brokenValue() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + #expect(throws: BinaryValueDecodeError.self) { + try ProtobufSchemaDecoder.decode(BinaryFixture.data("0a05ab"), schema: schema, message: "shop.Item") + } + #expect(throws: BinaryValueDecodeError.self) { + try ProtobufSchemaDecoder.decode(BinaryFixture.data("0f"), schema: schema, message: "shop.Item") + } + } + + @Test("A file that is not a schema says why") + func invalidSchema() { + #expect(throws: ProtobufSchemaError.self) { try ProtobufSchema.parse(Data("message Item { int32 id = ; }".utf8)) } + #expect(throws: ProtobufSchemaError.self) { try ProtobufSchema.parse(Data("enum Only { A = 0; }".utf8)) } + #expect(throws: ProtobufSchemaError.self) { try ProtobufSchema.parse(Data([0xFF, 0x00, 0x13])) } + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift index 72d719c5e2..ff1d8b1d10 100644 --- a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift @@ -40,6 +40,22 @@ struct ValueDisplayFormatTests { #expect(ValueDisplayFormat(rawValue: "nope") == nil) } + @Test("A Protobuf schema format keeps its schema and message, a Schema Registry one its connection") + func schemaFormatsRoundTrip() throws { + let schemaID = String(repeating: "ab12", count: 8) + let connection = UUID() + let protobuf = ValueDisplayFormat.protobufSchema(schemaID: schemaID, message: "shop.Item") + #expect(protobuf.rawValue == "protobuf:\(schemaID):shop.Item") + #expect(ValueDisplayFormat(rawValue: protobuf.rawValue) == protobuf) + #expect(ValueDisplayFormat(rawValue: "schemaRegistry:\(connection.uuidString)") == .schemaRegistry(connectionID: connection)) + #expect(ValueDisplayFormat(rawValue: "protobuf:\(schemaID):") == nil) + #expect(ValueDisplayFormat(rawValue: "protobuf:ab12:shop.Item") == nil) + #expect(ValueDisplayFormat(rawValue: "schemaRegistry:nope") == nil) + #expect(protobuf.isApplicable(to: .blob(rawType: "BLOB"))) + #expect(!protobuf.isApplicable(to: .text(rawType: "TEXT"))) + #expect(ValueDisplayFormat.schemaRegistry(connectionID: connection).isApplicable(to: .text(rawType: "TEXT"))) + } + @Test("only the formats that produce characters count as rendering binary as text") func rendersBinaryAsText() { #expect(ValueDisplayFormat.text.rendersBinaryAsText) diff --git a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift new file mode 100644 index 0000000000..8e02bafb35 --- /dev/null +++ b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift @@ -0,0 +1,171 @@ +// +// SchemaRegistryTests.swift +// TableProTests +// + +import Foundation +import os +@testable import TablePro +import Testing + +// Recorded from confluentinc/cp-schema-registry 7.7.1. The values were written by confluent-kafka 2.6.1's +// AvroSerializer and ProtobufSerializer against it, and the expected texts are what those libraries decode. +struct SchemaRegistryTests { + private static let responses: [String: (Int, String)] = [ + "/schemas/ids/1": (200, #"{"schema":"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"com.example\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"email\",\"type\":[\"null\",\"string\"]},{\"name\":\"kind\",\"type\":{\"type\":\"enum\",\"name\":\"Kind\",\"symbols\":[\"A\",\"B\"]}}]}"}"#), + "/schemas/ids/3": (200, #"{"schemaType":"PROTOBUF","schema":"syntax = \"proto3\";\npackage demo;\n\nimport \"demo/common.proto\";\n\nmessage Order {\n string id = 1;\n .demo.Money total = 2;\n repeated string tags = 3;\n .demo.Status status = 4;\n repeated .demo.Order.Line lines = 5;\n repeated .demo.Order.CountsEntry counts = 6;\n\n message Line {\n string sku = 1;\n sint32 qty = 2;\n }\n message CountsEntry {\n option map_entry = true;\n \n string key = 1;\n int32 value = 2;\n }\n}\nmessage Refund {\n string order_id = 1;\n double amount = 2;\n}\nenum Status {\n STATUS_UNKNOWN = 0;\n STATUS_PAID = 1;\n}\n","references":[{"name":"demo/common.proto","subject":"demo/common.proto","version":1}]}"#), + "/subjects/demo%2Fcommon.proto/versions/1": (200, #"{"subject":"demo/common.proto","version":1,"id":2,"schemaType":"PROTOBUF","schema":"syntax = \"proto3\";\npackage demo;\n\nmessage Money {\n string currency = 1;\n int64 units = 2;\n}\n"}"#), + "/schemas/ids/99999": (404, #"{"error_code":40403,"message":"Schema 99999 not found"}"#), + ] + + private static let avroUser = "00000000015406416e6e021e616e6e406578616d706c652e636f6d02" + private static let order = "0000000003000a036f2d3112080a0345555210e2091a01611a016220012a050a0178100532050a016b1007" + private static let refund = "000000000302020a036f2d31110000000000002940" + + private static let endpoint = SchemaRegistryEndpoint(fields: [ + SchemaRegistryConnectionField.url: "https://registry.example.com/base/", + SchemaRegistryConnectionField.username: "key", + SchemaRegistryConnectionField.password: "secret", + ]) + + private final class RecordedTransport: Sendable { + let requests = OSAllocatedUnfairLock<[URLRequest]>(initialState: []) + + var transport: SchemaRegistryClient.Transport { + { [self] request in + requests.withLock { $0.append(request) } + let path = String((request.url?.absoluteString ?? "").dropFirst("https://registry.example.com/base".count)) + let (status, body) = SchemaRegistryTests.responses[path] ?? (404, #"{"message":"no such path"}"#) + let response = HTTPURLResponse(url: request.url!, statusCode: status, httpVersion: nil, headerFields: nil)! + return (Data(body.utf8), response) + } + } + } + + private func fetch(_ id: UInt32, transport: RecordedTransport = RecordedTransport()) async throws -> RegistrySchema { + try await SchemaRegistryClient(endpoint: try #require(Self.endpoint), transport: transport.transport).schema(id: id) + } + + @Test("An Avro value decodes with the schema its prefix names") + func avro() async throws { + let schema = try await fetch(1) + let text = try BinaryValueDecoding.registryText(BinaryFixture.data(Self.avroUser), schema: schema, maxValues: .max, limit: .max) + #expect(text == #"{"id":42,"name":"Ann","email":"ann@example.com","kind":"B"}"#) + guard case .decoded(.json, _, let note) = try BinaryValueDecoding.registryDocument(BinaryFixture.data(Self.avroUser), schema: schema) else { + Issue.record("expected a decoded JSON tree") + return + } + #expect(note?.contains("1") == true) + } + + @Test("A Protobuf schema is fetched with the files it imports, and the message index picks the message") + func protobuf() async throws { + let transport = RecordedTransport() + let schema = try await fetch(3, transport: transport) + let paths = transport.requests.withLock { $0.compactMap { $0.url?.path(percentEncoded: true) } } + #expect(paths == ["/base/schemas/ids/3", "/base/subjects/demo%2Fcommon.proto/versions/1"]) + + let order = try BinaryValueDecoding.registryText(BinaryFixture.data(Self.order), schema: schema, maxValues: .max, limit: .max) + #expect(order == #"id: "o-1" total { currency: "EUR" units: 1250 } tags: "a" tags: "b" status: STATUS_PAID "# + + #"lines { sku: "x" qty: -3 } counts { key: "k" value: 7 }"#) + + guard case .decoded(.protobuf, let text, let note) = try BinaryValueDecoding.registryDocument( + BinaryFixture.data(Self.refund), schema: schema + ) else { + Issue.record("expected a decoded Protobuf tree") + return + } + #expect(text == "order_id: \"o-1\"\namount: 12.5\n") + #expect(note?.contains("3") == true) + } + + @Test("An Avro schema can name a type another subject defines") + func avroReferences() throws { + let money = #"{"type":"record","name":"Money","namespace":"demo","fields":[{"name":"units","type":"long"}]}"# + let order = #"{"type":"record","name":"Order","namespace":"demo","fields":[{"name":"total","type":"demo.Money"}]}"# + #expect(throws: AvroSchemaError.self) { try AvroSchema.parse(order) } + let schema = try AvroSchema.parse(order, references: [money]) + let text = try BinaryValueDecoding.registryText(BinaryFixture.data("000000000754"), schema: .avro(schema), maxValues: .max, limit: .max) + #expect(text == #"{"total":{"units":42}}"#) + } + + @Test("Every request carries the connection's credentials") + func basicAuth() async throws { + let transport = RecordedTransport() + _ = try await fetch(1, transport: transport) + let header = transport.requests.withLock { $0.first?.value(forHTTPHeaderField: "Authorization") } + #expect(header == "Basic " + Data("key:secret".utf8).base64EncodedString()) + } + + @Test("The registry's own error message is the reason a schema is missing") + func notFound() async { + await #expect(throws: SchemaRegistryError(reason: "The Schema Registry answered 404: Schema 99999 not found")) { + try await fetch(99_999) + } + } + + @Test("Only an http or https URL names a registry") + func endpointSchemes() { + #expect(SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: "file:///etc/passwd"]) == nil) + #expect(SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: ""]) == nil) + #expect(SchemaRegistryEndpoint(fields: [:]) == nil) + #expect(SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: " http://localhost:8081 "])?.username == nil) + } + + @Test("A value without the prefix, or one its schema cannot read, is refused, so the cell stays hex") + func brokenValues() async throws { + let avro = try await fetch(1) + #expect(throws: BinaryValueDecodeError(kind: .notFramed, offset: 0)) { + try SchemaRegistryFrame.schemaID(BinaryFixture.data("5406416e6e")) + } + #expect(throws: BinaryValueDecodeError.self) { + try BinaryValueDecoding.registryText(BinaryFixture.data("000000000154"), schema: avro, maxValues: .max, limit: .max) + } + let protobuf = try await fetch(3) + #expect(throws: BinaryValueDecodeError.self) { + try BinaryValueDecoding.registryText(BinaryFixture.data("000000000302080a"), schema: protobuf, maxValues: .max, limit: .max) + } + } + + @Test("The cache asks once for many cells, tells the grid when the schema lands, and keeps a failure only briefly") + func cache() async throws { + let transport = RecordedTransport() + let connection = UUID() + let cache = SchemaRegistryCache(endpoint: { _ in Self.endpoint }, transport: transport.transport) + let (landed, continuation) = AsyncStream.makeStream() + let observer = NotificationCenter.default.addObserver(forName: SchemaRegistryCache.didLoadSchema, object: cache, queue: nil) { + if let id = $0.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID { continuation.yield(id) } + } + defer { NotificationCenter.default.removeObserver(observer) } + + #expect(cache.cachedSchema(connectionID: connection, schemaID: 1) == nil) + #expect(cache.cachedSchema(connectionID: connection, schemaID: 1) == nil) + var iterator = landed.makeAsyncIterator() + #expect(await iterator.next() == connection) + guard case .success(.avro)? = cache.cachedSchema(connectionID: connection, schemaID: 1) else { + Issue.record("expected the fetched Avro schema") + return + } + #expect(transport.requests.withLock { $0.count } == 1) + + guard case .failure = await cache.schema(connectionID: connection, schemaID: 99_999) else { + Issue.record("expected a failure") + return + } + guard case .failure? = cache.cachedSchema(connectionID: connection, schemaID: 99_999) else { + Issue.record("expected the failure to be kept") + return + } + #expect(transport.requests.withLock { $0.count } == 2) + } + + @Test("A connection with no registry URL says so instead of fetching") + func noEndpoint() async { + let cache = SchemaRegistryCache(endpoint: { _ in nil }, transport: { _ in throw URLError(.badURL) }) + guard case .failure(let error) = await cache.schema(connectionID: UUID(), schemaID: 1) else { + Issue.record("expected a failure") + return + } + #expect(error.reason.contains("Schema Registry URL")) + } +} diff --git a/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift b/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift index d55432ed9e..47ba0cff3e 100644 --- a/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift +++ b/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift @@ -309,6 +309,28 @@ struct ValueDisplayFormatStorageTests { #expect(schemas.storedIDs.isEmpty) #expect(orphan != kept) } + + @Test("A Protobuf schema file goes once no saved format names it, beside the Avro ones") + func protobufSchemaFilesFollowTheFormatsThatNameThem() throws { + let defaults = try pool.make().defaults + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + let protobuf = ProtobufSchemaStore(directory: directory.appendingPathComponent("proto", isDirectory: true)) + let kept = try protobuf.add(Data("message A { int32 id = 1; }".utf8)) + let storage = ValueDisplayFormatStorage( + defaults: defaults, + schemas: AvroSchemaStore(directory: directory.appendingPathComponent("avro", isDirectory: true)), + protobufSchemas: ProtobufSchemaStore(directory: directory.appendingPathComponent("proto", isDirectory: true)) + ) + let target = scope("public", connectionId: UUID()) + + storage.save(["payload": .protobufSchema(schemaID: kept, message: "A")], for: target) + #expect(storage.load(for: target) == ["payload": .protobufSchema(schemaID: kept, message: "A")]) + #expect(protobuf.storedIDs == [kept]) + + storage.dropTable(target) + #expect(protobuf.storedIDs.isEmpty) + } } // What 0.83.1 decodes a table's formats as: a String enum of the names it shipped. From 08af6cee273493ac4afcd169e3b3860f49775895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:36:40 +0700 Subject: [PATCH 12/22] docs(changelog): add the Protobuf schema and Schema Registry entry --- apple/changelog.d/3979.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 apple/changelog.d/3979.added.md diff --git a/apple/changelog.d/3979.added.md b/apple/changelog.d/3979.added.md new file mode 100644 index 0000000000..f923100243 --- /dev/null +++ b/apple/changelog.d/3979.added.md @@ -0,0 +1 @@ +Display As decodes Protobuf with a .proto file chosen for the column, and Kafka values with a Schema Registry. From 7695906f72bfecacf58bb8aade89a8cd68569587 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 21:33:16 +0700 Subject: [PATCH 13/22] refactor(display-format): grid-owned registry fetches and protoc-exact Protobuf decoding --- .../KafkaConnectionField.swift | 2 +- .../KafkaMessageFlattener.swift | 5 +- .../Core/DataGrid/DataGridModel.swift | 145 ++++- .../Core/DataGrid/RowDisplayBox.swift | 14 + .../Outside/MCPRemoteServerTransport.swift | 41 +- .../Network/SameOriginRedirectGuard.swift | 56 ++ .../BinaryValues/BinaryValueBytes.swift | 17 + .../BinaryValues/BinaryValueDecoding.swift | 20 +- .../BinaryValues/ProtoFileParser.swift | 177 +++--- .../BinaryValues/ProtobufRendering.swift | 3 +- .../BinaryValues/ProtobufSchema.swift | 337 ++++++++--- .../BinaryValues/ProtobufSchemaDecoder.swift | 526 +++++++++++++----- .../BinaryValues/ProtobufWireDecoder.swift | 56 +- .../Formatting/ValueDisplayFormat.swift | 7 +- .../Formatting/ValueDisplayFormatter.swift | 5 +- .../ProtobufSchemaFilePanel.swift | 7 +- .../SchemaRegistry/SchemaRegistryCache.swift | 187 ++++--- .../SchemaRegistry/SchemaRegistryClient.swift | 86 ++- .../Core/Storage/ProtobufSchemaStore.swift | 4 +- .../Core/Storage/SchemaFileCache.swift | 13 +- .../TablePro/Models/UI/ProtobufTreeNode.swift | 3 +- .../DataGridViewController+HeaderMenu.swift | 5 +- .../DataGrid/DataGridViewController.swift | 4 - .../DataGridViewController+TypedEditors.swift | 4 +- .../Views/Results/BinaryValueViewerView.swift | 13 +- .../Results/CellInteractionResolver.swift | 5 +- .../FieldEditors/BinaryValueFieldView.swift | 8 +- .../FieldEditors/FieldEditorContext.swift | 5 +- .../RowInspector/InspectorFieldListView.swift | 3 +- .../Views/RowInspector/RowInspectorView.swift | 7 +- .../FieldEditors/FieldEditorResolver.swift | 3 +- .../SameOriginRedirectGuardTests.swift | 31 ++ .../ProtobufSchemaDecoderTests.swift | 320 ++++++++++- .../ProtobufWireDecoderTests.swift | 23 +- .../SchemaRegistry/SchemaRegistryTests.swift | 130 ++++- .../Core/Storage/AvroSchemaStoreTests.swift | 17 + .../DataGridSchemaRegistryTests.swift | 154 +++++ .../CellInteractionResolverTests.swift | 10 + .../Shared/FieldEditorResolverTests.swift | 17 + 39 files changed, 1927 insertions(+), 543 deletions(-) create mode 100644 apple/TablePro/Core/Network/SameOriginRedirectGuard.swift create mode 100644 apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueBytes.swift create mode 100644 apple/TableProTests/Core/Network/SameOriginRedirectGuardTests.swift create mode 100644 apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift b/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift index 7bc3e9709e..d4265ada9c 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift @@ -88,7 +88,7 @@ enum KafkaConnectionField { static let saslMechanism = "kafkaSaslMechanism" static let brokerRouting = "kafkaBrokerRouting" static let connectTimeout = "kafkaConnectTimeout" - // Read by the app, which fetches the schemas a value names; the driver never uses them. + // The app reads these to fetch schemas; the driver never does. static let schemaRegistryUrl = "schemaRegistryUrl" static let schemaRegistryUsername = "schemaRegistryUsername" static let schemaRegistryPassword = "schemaRegistryPassword" diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift b/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift index c3ad2a57ac..b3e27c33ff 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift @@ -95,11 +95,8 @@ enum KafkaMessageFlattener { return .text("{\(pairs.joined(separator: ","))}") } - /// A column is text only if every non-null payload on the page decodes as UTF-8. One - /// binary value is enough to make the whole column binary, because rendering a binary - /// payload through a text column is the lossier of the two mistakes. + // Bytes shown as text is the lossier mistake, and a Schema Registry value's leading zero can sit in valid UTF-8. private static func allDecodeAsText(_ payloads: S) -> Bool where S.Element == Data? { - // A Schema Registry value starts with a zero byte, and its payload can still be valid UTF-8. for case let payload? in payloads where payload.first == 0 || String(data: payload, encoding: .utf8) == nil { return false } diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index 3ccb1ea6ee..4037eb492b 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -57,16 +57,23 @@ final class DataGridModel: ObservableObject { private(set) weak var mountedView: (any DataGridMountedView)? private var orderRevision = 0 + private let schemaRegistry: SchemaRegistryCache + private var schemaObservers: Set = [] + private var staleSchemaColumns: Set = [] + init( store: any DataGridStateStore = InMemoryDataGridStateStore(), configuration: DataGridConfiguration = DataGridConfiguration(), themeScope: ThemeScope = .live, - selectionSummary: SelectionSummaryState? = nil + selectionSummary: SelectionSummaryState? = nil, + schemaRegistry: SchemaRegistryCache = .shared ) { self.store = store self.configuration = configuration self.themeScope = themeScope self.selectionSummary = selectionSummary + self.schemaRegistry = schemaRegistry + observeSchemas() } // MARK: - Rows @@ -101,6 +108,7 @@ final class DataGridModel: ObservableObject { displayRevision &+= 1 mountedView?.apply(.reload) refreshFind() + prefetchSchemas(in: current, storageRows: nil, retryingFailures: true) } func apply(_ delta: Delta) { @@ -123,8 +131,21 @@ final class DataGridModel: ObservableObject { displayRevision &+= 1 mountedView?.apply(change) refreshFind() + if case .rowsInserted(let inserted) = delta { + prefetchSchemas(in: current, storageRows: inserted, retryingFailures: false) + } else if delta == .columnsReplaced { + prefetchSchemas(in: current, storageRows: nil, retryingFailures: false) + } return } + switch delta { + case .cellChanged(let row, _): + prefetchSchemas(in: current, storageRows: IndexSet(integer: row), retryingFailures: false) + case .cellsChanged(let positions): + prefetchSchemas(in: current, storageRows: IndexSet(positions.map(\.row)), retryingFailures: false) + default: + break + } let change = DataGridChange.from(delta, order: order, rows: current, columns: columns) guard !change.isNone else { return } if case .rowsUpdated(let displayRows, _) = change { @@ -271,14 +292,80 @@ final class DataGridModel: ObservableObject { } store.displayFormats = formats redrawFormatted([column], in: current) + prefetchSchemas(in: current, storageRows: nil, retryingFailures: true) } - // A Schema Registry column shows stored bytes until its schemas arrive. The format is unchanged, so a value - // filter picked from those bytes follows the decoded text the way a format change carries it. - func schemaRegistryDidLoad(connectionID: UUID) { - let current = rows - let changed = columns.columns.filter { displayFormat(for: $0) == .schemaRegistry(connectionID: connectionID) } + private func redrawFormatted(_ changed: [DataGridColumn], in current: TableRows) { + display.cache.clearValues(forDataIndexes: Set(changed.map(\.dataIndex))) + if changed.contains(where: { store.valueFilter.filter(forColumn: $0.dataIndex) != nil }) { + order = resolveOrder(in: current) + restoreSelection(in: current) + displayRevision &+= 1 + mountedView?.apply(.reload) + } else { + let positions = IndexSet(changed.compactMap { columns.displayPosition(of: $0.id) }) + if !positions.isEmpty { + mountedView?.apply(.rowsUpdated(IndexSet(0.. Bool) { + let wasEmpty = staleSchemaColumns.isEmpty + for (dataIndex, format) in store.displayFormats.enumerated() { + if let format, matches(format) { staleSchemaColumns.insert(dataIndex) } + } + guard wasEmpty, !staleSchemaColumns.isEmpty else { return } + DispatchQueue.main.async { [weak self] in + MainActor.assumeIsolated { self?.redrawStaleSchemaColumns() } + } + } + + private func redrawStaleSchemaColumns() { + let stale = staleSchemaColumns + staleSchemaColumns = [] + let changed = columns.columns.filter { stale.contains($0.dataIndex) } guard !changed.isEmpty else { return } + let current = rows + // The format is the same, but the values move from the stored bytes to decoded text, and a filter has to follow. for column in changed { if let remapped = remappedValueFilter(column: column, from: nil, to: displayFormat(for: column), in: current) { store.valueFilter = remapped @@ -287,18 +374,42 @@ final class DataGridModel: ObservableObject { redrawFormatted(changed, in: current) } - private func redrawFormatted(_ changed: [DataGridColumn], in current: TableRows) { - display.cache.removeAll() - if store.valueFilter.isActive { - order = resolveOrder(in: current) - restoreSelection(in: current) - displayRevision &+= 1 - mountedView?.apply(.reload) - } else { - let positions = IndexSet(changed.compactMap { columns.displayPosition(of: $0.id) }) - mountedView?.apply(.rowsUpdated(IndexSet(0..] = [:] + for (dataIndex, format) in store.displayFormats.enumerated() { + guard case .schemaRegistry(let connectionID)? = format else { continue } + let type = dataIndex < current.columnTypes.count ? current.columnTypes[dataIndex] : nil + guard format?.isApplicable(to: type, databaseType: databaseType) == true else { continue } + let holdsBytes = BinaryValueBytes.holdsBytes(columnType: type) + var ids = wanted[connectionID, default: []] + func collect(_ row: Row) { + guard dataIndex < row.values.count, + let id = Self.schemaID(of: row.values[dataIndex], holdsBytes: holdsBytes) else { return } + ids.insert(id) + } + if let storageRows { + for index in storageRows where index < current.rows.count { collect(current.rows[index]) } + } else { + for row in current.rows { collect(row) } + } + wanted[connectionID] = ids + } + for (connectionID, ids) in wanted { + schemaRegistry.prefetch(connectionID: connectionID, schemaIDs: ids, retryingFailures: retryingFailures) + } + } + + private static func schemaID(of value: PluginCellValue, holdsBytes: Bool) -> UInt32? { + switch value { + case .null: + return nil + case .bytes(let data): + return SchemaRegistryFrame.schemaID(prefix: data) + case .text(let text): + let head = String(String.UnicodeScalarView(text.unicodeScalars.prefix(SchemaRegistryFrame.headerLength))) + return BinaryValueBytes.data(of: head, holdsBytes: holdsBytes).flatMap { SchemaRegistryFrame.schemaID(prefix: $0) } } - refreshFind() } private func remappedValueFilter( diff --git a/apple/TablePro/Core/DataGrid/RowDisplayBox.swift b/apple/TablePro/Core/DataGrid/RowDisplayBox.swift index 99f39e1bd2..209b20d4f5 100644 --- a/apple/TablePro/Core/DataGrid/RowDisplayBox.swift +++ b/apple/TablePro/Core/DataGrid/RowDisplayBox.swift @@ -94,6 +94,20 @@ final class RowDisplayCache { storage[id] = Entry(box: existing.box, cost: 0) } + // For a display format change: every other column's text is still right. + func clearValues(forDataIndexes dataIndexes: Set) { + guard !dataIndexes.isEmpty else { return } + highlights.removeAll(keepingCapacity: true) + storage = storage.mapValues { entry in + for index in dataIndexes where index < entry.box.values.count { + entry.box.values[index] = nil + } + let cost = Self.rowCost(entry.box.values) + totalCost += cost - entry.cost + return Entry(box: entry.box, cost: cost) + } + } + private func evictIfNeeded() { while storage.count > countLimit || totalCost > costLimit { guard insertionHead < insertionOrder.count else { break } diff --git a/apple/TablePro/Core/MCP/Outside/MCPRemoteServerTransport.swift b/apple/TablePro/Core/MCP/Outside/MCPRemoteServerTransport.swift index e180d8d471..ccfe17c629 100644 --- a/apple/TablePro/Core/MCP/Outside/MCPRemoteServerTransport.swift +++ b/apple/TablePro/Core/MCP/Outside/MCPRemoteServerTransport.swift @@ -30,7 +30,7 @@ internal actor MCPRemoteServerTransport { private let endpoint: URL private let bearerToken: String private let urlSession: URLSession - private let redirectGuard: MCPRedirectGuard + private let redirectGuard: SameOriginRedirectGuard private let timeout: Duration private var negotiatedSessionId: String? @@ -48,7 +48,7 @@ internal actor MCPRemoteServerTransport { self.endpoint = endpoint self.bearerToken = bearerToken self.timeout = timeout - self.redirectGuard = MCPRedirectGuard(origin: endpoint) + self.redirectGuard = SameOriginRedirectGuard(origin: endpoint) if let urlSession { self.urlSession = urlSession } else { @@ -276,40 +276,3 @@ internal actor MCPRemoteServerTransport { String(localized: "The connection to the server was closed.") } } - -/// Refuses to follow a redirect off the endpoint's own origin. -/// -/// `URLSession` follows redirects on its own and carries the request's headers to wherever it lands. -/// The header here is `Authorization: Bearer `, so a server that answers `302` pointing at -/// another host hands that host the credential, and the user never sees a request they did not make. -/// A same-origin redirect is ordinary routing and is followed; anything else is answered with the -/// redirect response itself, which surfaces as an HTTP failure naming the status. -private final class MCPRedirectGuard: NSObject, URLSessionTaskDelegate, @unchecked Sendable { - private let scheme: String? - private let host: String? - private let port: Int? - - init(origin: URL) { - scheme = origin.scheme?.lowercased() - host = origin.host?.lowercased() - port = origin.port - } - - func urlSession( - _ session: URLSession, - task: URLSessionTask, - willPerformHTTPRedirection response: HTTPURLResponse, - newRequest request: URLRequest, - completionHandler: @escaping (URLRequest?) -> Void - ) { - guard let target = request.url, - target.scheme?.lowercased() == scheme, - target.host?.lowercased() == host, - target.port == port - else { - completionHandler(nil) - return - } - completionHandler(request) - } -} diff --git a/apple/TablePro/Core/Network/SameOriginRedirectGuard.swift b/apple/TablePro/Core/Network/SameOriginRedirectGuard.swift new file mode 100644 index 0000000000..3eeadf2f31 --- /dev/null +++ b/apple/TablePro/Core/Network/SameOriginRedirectGuard.swift @@ -0,0 +1,56 @@ +// +// SameOriginRedirectGuard.swift +// TablePro +// + +import Foundation + +// URLSession carries a request's Authorization header to wherever a redirect points, so only same-origin redirects are followed. +internal final class SameOriginRedirectGuard: NSObject, URLSessionTaskDelegate, Sendable { + private let origin: Origin? + + init(origin: URL) { + self.origin = Origin(origin) + } + + func allows(_ target: URL) -> Bool { + guard let origin, let destination = Origin(target) else { return false } + return origin == destination + } + + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping @Sendable (URLRequest?) -> Void + ) { + guard let target = request.url, allows(target) else { + completionHandler(nil) + return + } + completionHandler(request) + } + + private struct Origin: Equatable, Sendable { + let scheme: String + let host: String + let port: Int + + init?(_ url: URL) { + guard let scheme = url.scheme?.lowercased(), let host = url.host?.lowercased(), !host.isEmpty else { return nil } + guard let port = url.port ?? Self.defaultPort(scheme) else { return nil } + self.scheme = scheme + self.host = host + self.port = port + } + + private static func defaultPort(_ scheme: String) -> Int? { + switch scheme { + case "http": return 80 + case "https": return 443 + default: return nil + } + } + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueBytes.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueBytes.swift new file mode 100644 index 0000000000..f5b11b3d03 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueBytes.swift @@ -0,0 +1,17 @@ +// +// BinaryValueBytes.swift +// TablePro +// + +import Foundation + +internal enum BinaryValueBytes { + // Bytes reach a String one Latin-1 character per byte; a text column's value is its own UTF-8. + static func data(of text: String, holdsBytes: Bool) -> Data? { + holdsBytes ? text.data(using: .isoLatin1) : Data(text.utf8) + } + + static func holdsBytes(columnType: ColumnType?, isBinaryValue: Bool = false) -> Bool { + isBinaryValue || columnType?.isBlobType != false + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift index 5d54c7c558..e675154fde 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift @@ -38,17 +38,23 @@ internal enum BinaryValueDocument: Sendable { case registryUnavailable(schemaID: UInt32, reason: String) } -// The Confluent wire format: magic byte 0, then the schema ID, big-endian. internal enum SchemaRegistryFrame { static let headerLength = 5 static func schemaID(_ data: Data) throws -> UInt32 { - guard data.count >= headerLength, data.first == 0 else { - throw BinaryValueDecodeError(kind: .notFramed, offset: 0) + guard let id = schemaID(prefix: data) else { throw BinaryValueDecodeError(kind: .notFramed, offset: 0) } + return id + } + + static func schemaID>(prefix bytes: Bytes) -> UInt32? { + var iterator = bytes.makeIterator() + guard iterator.next() == 0 else { return nil } + var id: UInt32 = 0 + for _ in 0..<4 { + guard let byte = iterator.next() else { return nil } + id = id << 8 | UInt32(byte) } - var reader = BinaryValueReader(data) - try reader.skip(1) - return try reader.readUInt32() + return id } static func jsonText(_ data: Data) throws -> String { @@ -59,7 +65,7 @@ internal enum SchemaRegistryFrame { } } -// Pure but for the schema caches: the grid calls it per cell on the main actor, the viewer off it. +// Pure: the grid calls it per cell on the main actor, the viewer off it. internal enum BinaryValueDecoding { static let maxTextLength = BinaryTextDecoder.maxDisplayBytes // The grid stops here too, so a cell never shows a value the viewer then refuses. diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift index 90a0b5c30f..f4a8cf0f1d 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift @@ -5,8 +5,7 @@ import Foundation -// https://protobuf.dev/reference/protobuf/proto3-spec/ and the proto2 and editions grammars: what a value needs is -// read, and options, services, extensions and reserved ranges are skipped. +// https://protobuf.dev/reference/protobuf/proto3-spec/; only options that change how a value reads are kept. internal enum ProtoFileParser { static let maxTokens = 2_000_000 static let maxDepth = 100 @@ -96,32 +95,38 @@ internal enum ProtoFileParser { let tokens: [Token] var position = 0 var messages: [ProtobufMessageDefinition] = [] - var enums: [ProtobufEnumDescriptor] = [] + var enums: [ProtobufEnumDefinition] = [] init(tokens: [Token]) { self.tokens = tokens } mutating func file() throws -> ProtobufFileDefinition { + var file = ProtobufFileDefinition() var package = "" - var topMessages: [ProtobufMessageDefinition] = [] - var topEnums: [ProtobufEnumDescriptor] = [] while let token = peek() { switch token { - case .identifier("syntax"), .identifier("edition"): + case .identifier("syntax"): + if peek(1) == .symbol("="), peek(2) == .string("proto3") { file.syntax = .proto3 } + try skipStatement() + case .identifier("edition"): + file.syntax = .editions try skipStatement() case .identifier("package"): advance() package = try identifier() try expect(";") - case .identifier("import"), .identifier("option"): + case .identifier("import"): + try skipStatement() + case .identifier("option"): + feature(at: position + 1, into: &file.features) try skipStatement() case .identifier("message"): advance() - topMessages.append(try message(scope: package, depth: 0)) + file.messages.append(try message(scope: package, depth: 0)) case .identifier("enum"): advance() - topEnums.append(try enumeration(scope: package)) + file.enums.append(try enumeration(scope: package)) case .identifier("service"), .identifier("extend"): try skipStatement() case .symbol(";"): @@ -130,22 +135,28 @@ internal enum ProtoFileParser { throw unexpected() } } - return ProtobufFileDefinition(messages: topMessages, enums: topEnums, allMessages: messages, allEnums: enums) + file.allMessages = messages + file.allEnums = enums + return file } - private mutating func message(scope: String, depth: Int) throws -> ProtobufMessageDefinition { + private mutating func message(scope: String, depth: Int) throws -> String { let name = try identifier() let fullName = scope.isEmpty ? name : "\(scope).\(name)" try expect("{") + try messageBody(fullName: fullName, depth: depth) + return fullName + } + + private mutating func messageBody(fullName: String, depth: Int) throws { let slot = messages.count - messages.append(ProtobufMessageDefinition(fullName: fullName, fields: [], nestedMessages: [], isMapEntry: false)) + messages.append(ProtobufMessageDefinition(fullName: fullName)) var definition = messages[slot] - try body(of: &definition, depth: depth) + try body(of: &definition, depth: depth, oneof: nil) messages[slot] = definition - return definition } - private mutating func body(of definition: inout ProtobufMessageDefinition, depth: Int) throws { + private mutating func body(of definition: inout ProtobufMessageDefinition, depth: Int, oneof: Int?) throws { guard depth < ProtoFileParser.maxDepth else { throw ProtoFileParser.error(String(localized: "Messages are nested too deeply.")) } while true { guard let token = peek() else { throw ProtoFileParser.error(String(localized: "A message is never closed.")) } @@ -157,64 +168,68 @@ internal enum ProtoFileParser { advance() case .identifier("message"): advance() - definition.nestedMessages.append(try message(scope: definition.fullName, depth: depth + 1).fullName) + definition.nestedMessages.append(try message(scope: definition.fullName, depth: depth + 1)) case .identifier("enum"): advance() - _ = try enumeration(scope: definition.fullName) + definition.nestedEnums.append(try enumeration(scope: definition.fullName)) case .identifier("oneof"): advance() _ = try identifier() try expect("{") - try body(of: &definition, depth: depth + 1) + let index = (definition.fields.compactMap(\.oneofIndex).max() ?? -1) + 1 + try body(of: &definition, depth: depth + 1, oneof: index) // A Schema Registry stores a map field as the entry message protoc synthesizes for it. case .identifier("option") where peek(1) == .identifier("map_entry") && peek(3) == .identifier("true"): definition.isMapEntry = true try skipStatement() - case .identifier("option"), .identifier("reserved"), .identifier("extensions"), .identifier("extend"): + case .identifier("option"): + if oneof == nil { feature(at: position + 1, into: &definition.features) } + try skipStatement() + case .identifier("reserved"), .identifier("extensions"), .identifier("extend"): try skipStatement() case .identifier("map") where peek(1) == .symbol("<"): advance() try mapField(into: &definition) default: - try field(into: &definition, depth: depth) + try field(into: &definition, depth: depth, oneof: oneof) } } } - private mutating func field(into definition: inout ProtobufMessageDefinition, depth: Int) throws { - var isRepeated = false - if case .identifier(let label)? = peek(), ["optional", "required", "repeated"].contains(label), - peek(1).map(Self.isTypeToken) == true, peek(2).map(Self.isTypeToken) == true || peek(2) == .symbol("=") { - isRepeated = label == "repeated" + private mutating func field(into definition: inout ProtobufMessageDefinition, depth: Int, oneof: Int?) throws { + var label: String? + // `optional foo = 1;` is a field of a message named `optional`: a label needs a type and a name after it. + if case .identifier(let word)? = peek(), ["optional", "required", "repeated"].contains(word), + peek(1).map(Self.isIdentifier) == true, peek(2).map(Self.isIdentifier) == true, peek(3) == .symbol("=") { + label = word advance() } let typeName = try identifier() + let name = try identifier() + try expect("=") + let number = try fieldNumber() + let features = try fieldOptions() + let type: ProtobufFieldDefinition.TypeReference if typeName == "group" { - let groupName = try identifier() - try expect("=") - let number = try fieldNumber() - try skipOptions() try expect("{") - let fullName = "\(definition.fullName).\(groupName)" - let slot = messages.count - messages.append(ProtobufMessageDefinition(fullName: fullName, fields: [], nestedMessages: [], isMapEntry: false)) - var group = messages[slot] - try body(of: &group, depth: depth + 1) - messages[slot] = group + let fullName = "\(definition.fullName).\(name)" + try messageBody(fullName: fullName, depth: depth + 1) definition.nestedMessages.append(fullName) - definition.fields.append(ProtobufFieldDefinition( - name: groupName.lowercased(), number: number, type: .group(fullName), isRepeated: isRepeated - )) - return + type = .group(fullName) + } else { + try expect(";") + type = ProtobufScalarType(rawValue: typeName).map { .scalar($0) } ?? .named(typeName) } - let name = try identifier() - try expect("=") - let number = try fieldNumber() - try skipOptions() - try expect(";") - let type: ProtobufFieldDefinition.TypeReference = ProtobufScalarType(rawValue: typeName).map { .scalar($0) } - ?? .named(typeName) - definition.fields.append(ProtobufFieldDefinition(name: name, number: number, type: type, isRepeated: isRepeated)) + var field = ProtobufFieldDefinition( + name: typeName == "group" ? name.lowercased() : name, + number: number, + type: type, + isRepeated: label == "repeated" + ) + field.isOptional = label == "optional" + field.oneofIndex = oneof + field.features = features + definition.fields.append(field) } // protoc's synthesized entry: `Entry { key = 1; value = 2; }`, nested where the field is declared. @@ -227,41 +242,42 @@ internal enum ProtoFileParser { let name = try identifier() try expect("=") let number = try fieldNumber() - try skipOptions() + _ = try fieldOptions() try expect(";") let entryName = "\(definition.fullName).\(Self.camelCased(name))Entry" func reference(_ type: String) -> ProtobufFieldDefinition.TypeReference { ProtobufScalarType(rawValue: type).map { .scalar($0) } ?? .named(type) } - messages.append(ProtobufMessageDefinition( - fullName: entryName, - fields: [ - ProtobufFieldDefinition(name: "key", number: 1, type: reference(keyType), isRepeated: false), - ProtobufFieldDefinition(name: "value", number: 2, type: reference(valueType), isRepeated: false), - ], - nestedMessages: [], - isMapEntry: true - )) + var entry = ProtobufMessageDefinition(fullName: entryName) + entry.fields = [ + ProtobufFieldDefinition(name: "key", number: 1, type: reference(keyType), isRepeated: false), + ProtobufFieldDefinition(name: "value", number: 2, type: reference(valueType), isRepeated: false), + ] + entry.isMapEntry = true + messages.append(entry) definition.nestedMessages.append(entryName) // The entry sits in the message's scope, so its value type resolves from there. definition.fields.append(ProtobufFieldDefinition(name: name, number: number, type: .named(".\(entryName)"), isRepeated: true)) } - private mutating func enumeration(scope: String) throws -> ProtobufEnumDescriptor { + private mutating func enumeration(scope: String) throws -> String { let name = try identifier() + let fullName = scope.isEmpty ? name : "\(scope).\(name)" try expect("{") - var values: [Int32: String] = [:] + var enumeration = ProtobufEnumDefinition(fullName: fullName) while true { guard let token = peek() else { throw ProtoFileParser.error(String(localized: "An enum is never closed.")) } switch token { case .symbol("}"): advance() - let enumeration = ProtobufEnumDescriptor(fullName: scope.isEmpty ? name : "\(scope).\(name)", values: values) enums.append(enumeration) - return enumeration + return fullName case .symbol(";"): advance() - case .identifier("option"), .identifier("reserved"): + case .identifier("option"): + feature(at: position + 1, into: &enumeration.features) + try skipStatement() + case .identifier("reserved"): try skipStatement() default: let valueName = try identifier() @@ -276,16 +292,47 @@ internal enum ProtoFileParser { advance() try skipOptions() try expect(";") - if values[number] == nil { values[number] = valueName } + if enumeration.values[number] == nil { enumeration.values[number] = valueName } } } } - private static func isTypeToken(_ token: Token) -> Bool { + private static func isIdentifier(_ token: Token) -> Bool { if case .identifier = token { return true } return false } + // `features.field_presence = IMPLICIT` and `features.enum_type = CLOSED`, as a statement or a field option. + private func feature(at index: Int, into overrides: inout ProtobufFeatureOverrides) { + guard index + 2 < tokens.count, case .identifier(let name) = tokens[index], tokens[index + 1] == .symbol("="), + case .identifier(let value) = tokens[index + 2] else { return } + switch (name, value) { + case ("features.field_presence", "IMPLICIT"): overrides.implicitPresence = true + case ("features.field_presence", "EXPLICIT"), ("features.field_presence", "LEGACY_REQUIRED"): + overrides.implicitPresence = false + case ("features.enum_type", "CLOSED"): overrides.closedEnum = true + case ("features.enum_type", "OPEN"): overrides.closedEnum = false + default: break + } + } + + private mutating func fieldOptions() throws -> ProtobufFeatureOverrides { + var overrides = ProtobufFeatureOverrides() + let start = position + try skipOptions() + var nesting = 0 + for index in start.. String { var result = "" var upper = true @@ -314,13 +361,11 @@ internal enum ProtoFileParser { return Int(value) } - // `[packed = true, (custom).option = { ... }]` private mutating func skipOptions() throws { guard peek() == .symbol("[") else { return } try skipBalanced(open: "[", close: "]") } - // To the `;` that ends it, or past the `{ ... }` block that does. private mutating func skipStatement() throws { while let token = peek() { switch token { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift index 299d3c1a7b..0cc87cddcf 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift @@ -113,6 +113,7 @@ internal enum ProtobufTreeBuilder { )) break } + nodeCount += 1 let name = String(field.number) children.append(node( key: name, @@ -126,6 +127,7 @@ internal enum ProtobufTreeBuilder { return children } + // The caller counts the node itself. static func node( key: String, keyPath: String, @@ -134,7 +136,6 @@ internal enum ProtobufTreeBuilder { formats: TreeSummaryFormats, nodeCount: inout Int ) -> ProtobufTreeNode { - nodeCount += 1 switch value { case .message(let members), .group(let members): let children = makeChildren(members, keyPath: keyPath, path: path, formats: formats, nodeCount: &nodeCount) diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift index cb87bd3fa3..e8bd68a41a 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift @@ -24,12 +24,16 @@ internal struct ProtobufFieldDescriptor: Equatable, Sendable { let number: Int let type: ProtobufFieldType let isRepeated: Bool + // A real oneof only: proto3 `optional` puts a field in a oneof of its own. + let oneofIndex: Int? + // Without presence, a field holding its default is not printed. + let hasPresence: Bool } internal struct ProtobufMessageDescriptor: Equatable, Sendable { let fullName: String let fields: [ProtobufFieldDescriptor] - // In declaration order, map entries included: a Schema Registry message index counts them. + // Map entries included: a Schema Registry message index counts them. let nestedMessages: [String] let isMapEntry: Bool private let fieldIndexes: [Int: Int] @@ -54,6 +58,8 @@ internal struct ProtobufMessageDescriptor: Equatable, Sendable { internal struct ProtobufEnumDescriptor: Equatable, Sendable { let fullName: String let values: [Int32: String] + // A closed enum keeps a number it does not declare out of the field, with the unknown fields. + let isClosed: Bool var shortName: String { fullName.split(separator: ".").last.map(String.init) ?? fullName @@ -62,39 +68,45 @@ internal struct ProtobufEnumDescriptor: Equatable, Sendable { internal struct ProtobufSchemaError: Error, Equatable, Sendable { let reason: String + + static var notASchema: ProtobufSchemaError { + ProtobufSchemaError(reason: String(localized: "The file is neither a .proto file nor a descriptor set.")) + } } -// Messages and enums by full name, without the leading dot. The first file is the one chosen or fetched; -// a Schema Registry message index walks its top-level messages. +// The first file is the one chosen, the primary file of a descriptor set, or the subject a registry returned. internal struct ProtobufSchema: Equatable, Sendable { let messages: [String: ProtobufMessageDescriptor] let enums: [String: ProtobufEnumDescriptor] + // A Schema Registry message index walks these. let rootMessages: [String] - // Every message a value can be read as, in file order, map entries left out. + // The first file's messages first, map entries left out. let messageNames: [String] static let maxMessages = 20_000 - // Both forms protoc reads: `.proto` text, and the FileDescriptorSet `--descriptor_set_out` writes. static func parse(_ data: Data) throws -> ProtobufSchema { - if let text = String(data: data, encoding: .utf8), let files = try? ProtoFileParser.parse(text), !files.isEmpty { - return try build([files]) + var textError: ProtobufSchemaError? + if let text = String(data: data, encoding: .utf8) { + let files: [ProtobufFileDefinition]? + do { + files = try ProtoFileParser.parse(text) + } catch { + files = nil + textError = error as? ProtobufSchemaError + } + if let files { return try build(files) } } do { - return try build([ProtobufDescriptorSetParser.parse(data)]) - } catch let error as ProtobufSchemaError { - if let text = String(data: data, encoding: .utf8) { - // Text that is not a descriptor set: the parser's own reason says what is wrong with it. - _ = try ProtoFileParser.parse(text) - throw ProtobufSchemaError(reason: String(localized: "The file defines no message.")) - } - throw error + return try build(ProtobufDescriptorSetParser.parse(data)) + } catch { + if let textError, looksLikeText(data) { throw textError } + throw (error as? ProtobufSchemaError) ?? .notASchema } } - // A fetched schema and the files it imports, each `.proto` text, the root first. static func parse(protoFiles texts: [String]) throws -> ProtobufSchema { - try build(texts.map { try ProtoFileParser.parse($0) }) + try build(texts.flatMap { try ProtoFileParser.parse($0) }) } func messageIndexed(by indexes: [Int]) -> ProtobufMessageDescriptor? { @@ -108,59 +120,91 @@ internal struct ProtobufSchema: Equatable, Sendable { return message } - private static func build(_ fileGroups: [[ProtobufFileDefinition]]) throws -> ProtobufSchema { - let files = fileGroups.flatMap { $0 } - var messages: [String: ProtobufMessageDefinition] = [:] - var enums: [String: ProtobufEnumDescriptor] = [:] + // A descriptor set always holds control bytes; a .proto file holds none but whitespace. + private static func looksLikeText(_ data: Data) -> Bool { + !data.contains { $0 < 0x20 && !(0x09...0x0D).contains($0) } + } + + private static func build(_ files: [ProtobufFileDefinition]) throws -> ProtobufSchema { + var definitions: [String: ProtobufMessageDefinition] = [:] + var enumDefinitions: [String: ProtobufEnumDefinition] = [:] var order: [String] = [] for file in files { for message in file.allMessages { - if messages[message.fullName] == nil { order.append(message.fullName) } - messages[message.fullName] = message + if definitions[message.fullName] == nil { order.append(message.fullName) } + definitions[message.fullName] = message } for enumeration in file.allEnums { - enums[enumeration.fullName] = enumeration + enumDefinitions[enumeration.fullName] = enumeration } } - guard messages.count <= maxMessages else { + guard definitions.count <= maxMessages else { throw ProtobufSchemaError(reason: String(localized: "The schema defines more messages than TablePro reads.")) } - guard !messages.isEmpty else { + guard !definitions.isEmpty else { throw ProtobufSchemaError(reason: String(localized: "The file defines no message.")) } - let resolver = ProtobufTypeResolver(messageNames: Set(messages.keys), enumNames: Set(enums.keys)) + let scopes = ProtobufFeatureScopes(files: files, messages: definitions, enums: enumDefinitions) + let resolver = ProtobufTypeResolver(messageNames: Set(definitions.keys), enumNames: Set(enumDefinitions.keys)) var resolved: [String: ProtobufMessageDescriptor] = [:] - for (name, message) in messages { + for (name, message) in definitions { + let features = scopes.messages[name] ?? ProtobufFeatures(syntax: .proto2) resolved[name] = ProtobufMessageDescriptor( fullName: name, - fields: message.fields.map { resolver.resolve($0, scope: name) }, + fields: message.fields.map { resolver.resolve($0, scope: name, features: features) }, nestedMessages: message.nestedMessages, isMapEntry: message.isMapEntry ) } + var enums: [String: ProtobufEnumDescriptor] = [:] + for (name, enumeration) in enumDefinitions { + enums[name] = ProtobufEnumDescriptor( + fullName: name, values: enumeration.values, isClosed: scopes.closedEnums[name] ?? false + ) + } return ProtobufSchema( messages: resolved, enums: enums, - rootMessages: files.first?.messages.map(\.fullName) ?? [], + rootMessages: files.first?.messages ?? [], messageNames: order.filter { resolved[$0]?.isMapEntry == false } ) } } -// What a parser read, before type names are resolved against every file. +internal enum ProtobufSyntax: Sendable { + case proto2, proto3, editions +} + +// The `features` a scope sets; only an editions file can set them. +internal struct ProtobufFeatureOverrides: Equatable, Sendable { + var implicitPresence: Bool? + var closedEnum: Bool? +} + internal struct ProtobufFileDefinition: Sendable { - let messages: [ProtobufMessageDefinition] - let enums: [ProtobufEnumDescriptor] - // Nested types too, so build() sees every name the file declares. - let allMessages: [ProtobufMessageDefinition] - let allEnums: [ProtobufEnumDescriptor] + var name = "" + var dependencies: [String] = [] + var syntax = ProtobufSyntax.proto2 + var features = ProtobufFeatureOverrides() + var messages: [String] = [] + var enums: [String] = [] + var allMessages: [ProtobufMessageDefinition] = [] + var allEnums: [ProtobufEnumDefinition] = [] } internal struct ProtobufMessageDefinition: Sendable { let fullName: String - var fields: [ProtobufFieldDefinition] - var nestedMessages: [String] - var isMapEntry: Bool + var fields: [ProtobufFieldDefinition] = [] + var nestedMessages: [String] = [] + var nestedEnums: [String] = [] + var isMapEntry = false + var features = ProtobufFeatureOverrides() +} + +internal struct ProtobufEnumDefinition: Sendable { + let fullName: String + var values: [Int32: String] = [:] + var features = ProtobufFeatureOverrides() } internal struct ProtobufFieldDefinition: Sendable { @@ -175,6 +219,64 @@ internal struct ProtobufFieldDefinition: Sendable { let number: Int let type: TypeReference let isRepeated: Bool + var isOptional = false + var oneofIndex: Int? + var features = ProtobufFeatureOverrides() +} + +private struct ProtobufFeatures { + let syntax: ProtobufSyntax + var implicitPresence: Bool + var closedEnum: Bool + + init(syntax: ProtobufSyntax) { + self.syntax = syntax + implicitPresence = syntax == .proto3 + closedEnum = syntax == .proto2 + } + + func applying(_ overrides: ProtobufFeatureOverrides?) -> ProtobufFeatures { + guard syntax == .editions, let overrides else { return self } + var result = self + if let implicitPresence = overrides.implicitPresence { result.implicitPresence = implicitPresence } + if let closedEnum = overrides.closedEnum { result.closedEnum = closedEnum } + return result + } +} + +// Features inherit from the file through each enclosing message, wherever in the scope the option is written. +private struct ProtobufFeatureScopes { + private(set) var messages: [String: ProtobufFeatures] = [:] + private(set) var closedEnums: [String: Bool] = [:] + + init( + files: [ProtobufFileDefinition], + messages definitions: [String: ProtobufMessageDefinition], + enums enumDefinitions: [String: ProtobufEnumDefinition] + ) { + for file in files { + let base = ProtobufFeatures(syntax: file.syntax).applying(file.features) + var pending = file.messages.map { ($0, base) } + for name in file.enums { + closedEnums[name] = base.applying(enumDefinitions[name]?.features).closedEnum + } + while let (name, inherited) = pending.popLast() { + guard messages[name] == nil, let definition = definitions[name] else { continue } + let features = inherited.applying(definition.features) + messages[name] = features + pending += definition.nestedMessages.map { ($0, features) } + for nested in definition.nestedEnums { + closedEnums[nested] = features.applying(enumDefinitions[nested]?.features).closedEnum + } + } + for message in file.allMessages where messages[message.fullName] == nil { + messages[message.fullName] = base.applying(message.features) + } + for enumeration in file.allEnums where closedEnums[enumeration.fullName] == nil { + closedEnums[enumeration.fullName] = base.applying(enumeration.features).closedEnum + } + } + } } // C++ scoping: a name is looked up from the innermost scope outward, by its first component. @@ -182,7 +284,7 @@ private struct ProtobufTypeResolver { let messageNames: Set let enumNames: Set - func resolve(_ field: ProtobufFieldDefinition, scope: String) -> ProtobufFieldDescriptor { + func resolve(_ field: ProtobufFieldDefinition, scope: String, features: ProtobufFeatures) -> ProtobufFieldDescriptor { let type: ProtobufFieldType switch field.type { case .scalar(let scalar): @@ -192,7 +294,20 @@ private struct ProtobufTypeResolver { case .named(let reference): type = resolve(reference, scope: scope) } - return ProtobufFieldDescriptor(name: field.name, number: field.number, type: type, isRepeated: field.isRepeated) + let isImplicit = features.applying(field.features).implicitPresence && !field.isOptional + let isMessage: Bool + switch type { + case .message, .group: isMessage = true + case .scalar, .enumeration, .unresolved: isMessage = false + } + return ProtobufFieldDescriptor( + name: field.name, + number: field.number, + type: type, + isRepeated: field.isRepeated, + oneofIndex: field.oneofIndex, + hasPresence: !field.isRepeated && (field.oneofIndex != nil || isMessage || !isImplicit) + ) } private func resolve(_ reference: String, scope: String) -> ProtobufFieldType { @@ -215,18 +330,31 @@ private struct ProtobufTypeResolver { } } -// descriptor.proto, read with the wire reader: FileDescriptorSet.file = 1. +// descriptor.proto read with the wire reader. Field numbers are descriptor.proto's. internal enum ProtobufDescriptorSetParser { static func parse(_ data: Data) throws -> [ProtobufFileDefinition] { - let invalid = ProtobufSchemaError(reason: String(localized: "The file is neither a .proto file nor a descriptor set.")) - guard let set = try? fields(data) else { throw invalid } var files: [ProtobufFileDefinition] = [] - for (number, value) in set { - guard number == 1, case .bytes(let bytes) = value else { throw invalid } - files.append(try file(bytes)) + do { + for (number, value) in try fields(data) { + guard number == 1, case .bytes(let bytes) = value else { throw ProtobufSchemaError.notASchema } + files.append(try file(bytes)) + } + } catch let error as ProtobufSchemaError { + throw error + } catch { + throw ProtobufSchemaError.notASchema } - guard files.contains(where: { !$0.allMessages.isEmpty }) else { throw invalid } - return files + guard files.contains(where: { !$0.allMessages.isEmpty }) else { throw ProtobufSchemaError.notASchema } + return primaryFirst(files) + } + + // `--include_imports` writes each dependency before the files that import it. + private static func primaryFirst(_ files: [ProtobufFileDefinition]) -> [ProtobufFileDefinition] { + let imported = Set(files.flatMap(\.dependencies)) + guard let primary = files.lastIndex(where: { !imported.contains($0.name) }) else { return files } + var ordered = files + let file = ordered.remove(at: primary) + return [file] + ordered } private enum Value { @@ -263,31 +391,67 @@ internal enum ProtobufDescriptorSetParser { return String(data: data, encoding: .utf8) } + private static func varint(_ entries: [(Int, Value)], _ number: Int) -> UInt64? { + for (member, value) in entries.reversed() where member == number { + if case .varint(let raw) = value { return raw } + } + return nil + } + + private static func bytes(_ entries: [(Int, Value)], _ number: Int) -> Data? { + for (member, value) in entries.reversed() where member == number { + if case .bytes(let data) = value { return data } + } + return nil + } + + // options.features: FeatureSet.field_presence = 1 (2 is IMPLICIT), enum_type = 2 (2 is CLOSED). + private static func features(_ entries: [(Int, Value)], options: Int, features: Int) throws -> ProtobufFeatureOverrides { + guard let optionBytes = bytes(entries, options), + let featureBytes = bytes(try fields(optionBytes), features) else { return ProtobufFeatureOverrides() } + let set = try fields(featureBytes) + var overrides = ProtobufFeatureOverrides() + if let presence = varint(set, 1), (1...3).contains(presence) { overrides.implicitPresence = presence == 2 } + if let enumType = varint(set, 2), (1...2).contains(enumType) { overrides.closedEnum = enumType == 2 } + return overrides + } + private static func file(_ data: Data) throws -> ProtobufFileDefinition { + let entries = try fields(data) + var file = ProtobufFileDefinition() var package = "" var messageData: [Data] = [] var enumData: [Data] = [] - for (number, value) in try fields(data) { + for (number, value) in entries { switch (number, value) { + case (1, _): file.name = string(value) ?? "" case (2, _): package = string(value) ?? "" + case (3, _): file.dependencies.append(string(value) ?? "") case (4, .bytes(let bytes)): messageData.append(bytes) case (5, .bytes(let bytes)): enumData.append(bytes) default: break } } + let syntax = bytes(entries, 12).flatMap { String(data: $0, encoding: .utf8) } + switch syntax { + case "proto3": file.syntax = .proto3 + case "editions": file.syntax = .editions + default: file.syntax = (varint(entries, 14) ?? 0) >= 1_000 ? .editions : .proto2 + } + file.features = try features(entries, options: 8, features: 50) var collector = Collector() - let messages = try messageData.map { try collector.message($0, scope: package, depth: 0) } - let enums = try enumData.map { try collector.enumeration($0, scope: package) } - return ProtobufFileDefinition( - messages: messages, enums: enums, allMessages: collector.messages, allEnums: collector.enums - ) + file.messages = try messageData.map { try collector.message($0, scope: package, depth: 0) } + file.enums = try enumData.map { try collector.enumeration($0, scope: package) } + file.allMessages = collector.messages + file.allEnums = collector.enums + return file } private struct Collector { var messages: [ProtobufMessageDefinition] = [] - var enums: [ProtobufEnumDescriptor] = [] + var enums: [ProtobufEnumDefinition] = [] - mutating func message(_ data: Data, scope: String, depth: Int) throws -> ProtobufMessageDefinition { + mutating func message(_ data: Data, scope: String, depth: Int) throws -> String { guard depth < ProtoFileParser.maxDepth else { throw ProtobufSchemaError(reason: String(localized: "Messages are nested too deeply.")) } @@ -295,29 +459,28 @@ internal enum ProtobufDescriptorSetParser { let name = entries.first { $0.0 == 1 }.flatMap { string($0.1) } ?? "" let fullName = scope.isEmpty ? name : "\(scope).\(name)" let slot = messages.count - messages.append(ProtobufMessageDefinition(fullName: fullName, fields: [], nestedMessages: [], isMapEntry: false)) + messages.append(ProtobufMessageDefinition(fullName: fullName)) var definition = messages[slot] + definition.features = try features(entries, options: 7, features: 12) for (number, value) in entries { guard case .bytes(let bytes) = value else { continue } switch number { case 2: definition.fields.append(try field(bytes)) - case 3: definition.nestedMessages.append(try message(bytes, scope: fullName, depth: depth + 1).fullName) - case 4: enums.append(try enumeration(bytes, scope: fullName)) - case 7: definition.isMapEntry = try ProtobufDescriptorSetParser.fields(bytes).contains { - if $0.0 == 7, case .varint(let flag) = $0.1 { return flag != 0 } - return false - } + case 3: definition.nestedMessages.append(try message(bytes, scope: fullName, depth: depth + 1)) + case 4: definition.nestedEnums.append(try enumeration(bytes, scope: fullName)) + case 7: definition.isMapEntry = varint(try fields(bytes), 7).map { $0 != 0 } ?? false default: break } } messages[slot] = definition - return definition + return fullName } - mutating func enumeration(_ data: Data, scope: String) throws -> ProtobufEnumDescriptor { + mutating func enumeration(_ data: Data, scope: String) throws -> String { + let entries = try ProtobufDescriptorSetParser.fields(data) var name = "" var values: [Int32: String] = [:] - for (number, value) in try ProtobufDescriptorSetParser.fields(data) { + for (number, value) in entries { if number == 1 { name = string(value) ?? "" } guard number == 2, case .bytes(let bytes) = value else { continue } var valueName = "" @@ -326,30 +489,20 @@ internal enum ProtobufDescriptorSetParser { if member == 1 { valueName = string(memberValue) ?? "" } if member == 2, case .varint(let raw) = memberValue { valueNumber = Int32(truncatingIfNeeded: raw) } } - // The first name wins for an aliased number, as protoc prints it. + // protoc prints an aliased number by its first name. if values[valueNumber] == nil { values[valueNumber] = valueName } } - let enumeration = ProtobufEnumDescriptor(fullName: scope.isEmpty ? name : "\(scope).\(name)", values: values) - enums.append(enumeration) - return enumeration + let fullName = scope.isEmpty ? name : "\(scope).\(name)" + enums.append(ProtobufEnumDefinition( + fullName: fullName, values: values, features: try features(entries, options: 3, features: 7) + )) + return fullName } private func field(_ data: Data) throws -> ProtobufFieldDefinition { - var name = "" - var number = 0 - var label: UInt64 = 1 - var typeCode: UInt64 = 0 - var typeName = "" - for (member, value) in try ProtobufDescriptorSetParser.fields(data) { - switch (member, value) { - case (1, _): name = string(value) ?? "" - case (3, .varint(let raw)): number = Int(truncatingIfNeeded: raw) - case (4, .varint(let raw)): label = raw - case (5, .varint(let raw)): typeCode = raw - case (6, _): typeName = string(value) ?? "" - default: break - } - } + let entries = try ProtobufDescriptorSetParser.fields(data) + let typeCode = varint(entries, 5) ?? 0 + let typeName = bytes(entries, 6).flatMap { String(data: $0, encoding: .utf8) } ?? "" let type: ProtobufFieldDefinition.TypeReference switch typeCode { case 10: type = .group(String(typeName.drop { $0 == "." })) @@ -362,7 +515,17 @@ internal enum ProtobufDescriptorSetParser { } type = .scalar(scalar) } - return ProtobufFieldDefinition(name: name, number: number, type: type, isRepeated: label == 3) + let isProto3Optional = (varint(entries, 17) ?? 0) != 0 + var definition = ProtobufFieldDefinition( + name: bytes(entries, 1).flatMap { String(data: $0, encoding: .utf8) } ?? "", + number: Int(truncatingIfNeeded: varint(entries, 3) ?? 0), + type: type, + isRepeated: varint(entries, 4) == 3 + ) + definition.isOptional = isProto3Optional + if !isProto3Optional, let index = varint(entries, 9) { definition.oneofIndex = Int(truncatingIfNeeded: index) } + definition.features = try features(entries, options: 8, features: 21) + return definition } private static let scalarTypes: [UInt64: ProtobufScalarType] = [ diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift index e073b41a9a..d448b2d138 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -29,8 +29,7 @@ internal struct ProtobufTypedMessage: Equatable, Sendable { let framing: ProtobufFraming? } -// Reads a value as `protoc --decode=` does: known fields in number order, then the ones the schema does not -// name, by number, as `--decode_raw` shows them. A field whose wire type does not match its declaration is one of those. +// Matches `protoc --decode=`: known fields by number, then the rest in wire order as `--decode_raw` prints them. internal enum ProtobufSchemaDecoder { // A nil message takes the one a Schema Registry frame's message indexes name. static func decode( @@ -57,129 +56,392 @@ internal enum ProtobufSchemaDecoder { } guard let message else { throw BinaryValueDecodeError(kind: .invalidField, offset: 0) } var walker = Walker(schema: schema, budget: maxValues) - let fields = try walker.fields(&reader, message: message, depth: 0, group: nil) + let payload = try reader.window(reader.remaining) + let fields = try walker.members(of: [payload], message: message, depth: 0, asMapEntry: false) return ProtobufTypedMessage(typeName: message.fullName, fields: fields, framing: framing) } + private struct Occurrence { + enum Payload { + case varint(UInt64) + case fixed64(UInt64) + case fixed32(UInt32) + case lengthDelimited(BinaryValueReader) + case group(BinaryValueReader) + } + + let number: Int + let payload: Payload + let position: Int + } + + private enum Match { + case accepted + case mismatched + case undeclaredEnumValue + } + + // protoc moves a map entry whose closed enum value is undeclared to the unknown fields whole. + private struct MapEntryRejected: Error {} + + private struct Level { + var fields: [ProtobufTypedField] = [] + var unknown: [Occurrence] = [] + var isCut = false + } + + private enum MapKey: Comparable { + case signed(Int64) + case unsigned(UInt64) + case bytes([UInt8]) + + static func < (lhs: MapKey, rhs: MapKey) -> Bool { + switch (lhs, rhs) { + case (.signed(let left), .signed(let right)): return left < right + case (.unsigned(let left), .unsigned(let right)): return left < right + case (.bytes(let left), .bytes(let right)): return left.lexicographicallyPrecedes(right) + default: return false + } + } + } + + // An occurrence and each packed value cost one, so a cell reads no more of a large value than it can show. private struct Walker { let schema: ProtobufSchema var budget: Int - var isTruncated = false - mutating func fields( - _ reader: inout BinaryValueReader, - message: ProtobufMessageDescriptor?, + mutating func members( + of windows: [BinaryValueReader], + message: ProtobufMessageDescriptor, depth: Int, - group: Int? + asMapEntry: Bool ) throws -> [ProtobufTypedField] { - guard depth < ProtobufWireDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } - var known: [Int: [ProtobufTypedField]] = [:] - var unknown: [ProtobufTypedField] = [] - // A singular message seen twice is merged, which is what parsing its occurrences' bytes back to back does. - var singularMessages: [Int: Data] = [:] - func add(_ field: ProtobufTypedField, descriptor: ProtobufFieldDescriptor?) { - guard let descriptor else { - unknown.append(field) + guard depth < ProtobufWireDecoder.maxDepth else { + throw BinaryValueDecodeError(kind: .nestingTooDeep, offset: windows.first?.offset ?? 0) + } + var level = Level() + var occurrences: [Occurrence] = [] + scan: for window in windows { + var reader = window + while !reader.isAtEnd { + guard budget > 0 else { + level.isCut = true + break scan + } + occurrences.append(try Self.occurrence(&reader, depth: depth)) + budget -= 1 + } + } + var accepted: [Int: [Occurrence]] = [:] + for occurrence in occurrences { + let field = message.field(number: occurrence.number) + switch field.map({ match(occurrence.payload, field: $0) }) ?? .mismatched { + case .accepted: + accepted[occurrence.number, default: []].append(occurrence) + case .undeclaredEnumValue where asMapEntry: + throw MapEntryRejected() + case .mismatched, .undeclaredEnumValue: + // A map entry drops what it does not declare. + if !asMapEntry { level.unknown.append(occurrence) } + } + } + keepLastOneofMember(&accepted, message: message) + var numbers = Set(accepted.keys) + // protoc prints an entry's key and value even when the bytes leave them out. + if asMapEntry, !level.isCut { numbers.formUnion(message.fields.map(\.number)) } + for number in numbers.sorted() { + guard let field = message.field(number: number) else { continue } + if let found = accepted[number] { + try append(field, found, to: &level, depth: depth, asMapEntry: asMapEntry) + } else if let value = defaultValue(field.type) { + level.fields.append(typed(field, value)) + } + } + // An entry that cannot hold a value goes to the unknown fields whole, as protoc's parse would fail. + if asMapEntry, !level.unknown.isEmpty { throw MapEntryRejected() } + var fields = level.fields + let unknown = level.unknown.enumerated().sorted { + ($0.element.position, $0.offset) < ($1.element.position, $1.offset) + } + for (_, occurrence) in unknown { + fields.append(try raw(occurrence, depth: depth)) + } + if level.isCut { fields.append(ProtobufTypedField(name: "", typeName: nil, value: .truncated)) } + return fields + } + + private static func occurrence(_ reader: inout BinaryValueReader, depth: Int) throws -> Occurrence { + let start = reader.offset + let tag = try reader.readVarint() + let number = tag >> 3 + guard tag <= UInt64(UInt32.max), number > 0, number <= UInt64(ProtobufWireDecoder.maxFieldNumber) else { + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + let payload: Occurrence.Payload + switch tag & 7 { + case 0: + payload = .varint(try reader.readVarint()) + case 1: + payload = .fixed64(try reader.readUInt64LittleEndian()) + case 2: + let length = try reader.readVarint() + guard length <= UInt64(reader.remaining) else { throw reader.error(.unexpectedEnd) } + payload = .lengthDelimited(try reader.window(Int(length))) + try reader.skip(Int(length)) + case 3: + payload = .group(try ProtobufWireDecoder.groupContents(&reader, number: Int(number), depth: depth)) + case 4: + throw BinaryValueDecodeError(kind: .unmatchedGroupEnd, offset: start) + case 5: + payload = .fixed32(try reader.readUInt32LittleEndian()) + default: + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + return Occurrence(number: Int(number), payload: payload, position: start) + } + + private func match(_ payload: Occurrence.Payload, field: ProtobufFieldDescriptor) -> Match { + switch (field.type, payload) { + case (.scalar(.string), .lengthDelimited), (.scalar(.bytes), .lengthDelimited), + (.message, .lengthDelimited), (.group, .group): + return .accepted + case (.scalar(.string), _), (.scalar(.bytes), _): + return .mismatched + case (.scalar, .lengthDelimited), (.enumeration, .lengthDelimited): + return field.isRepeated ? .accepted : .mismatched + case (.scalar(let scalar), .varint): + return scalar.wireType == 0 ? .accepted : .mismatched + case (.scalar(let scalar), .fixed64): + return scalar.wireType == 1 ? .accepted : .mismatched + case (.scalar(let scalar), .fixed32): + return scalar.wireType == 5 ? .accepted : .mismatched + case (.enumeration(let name), .varint(let raw)): + return isDeclared(raw, enumeration: name) ? .accepted : .undeclaredEnumValue + default: + return .mismatched + } + } + + private func isDeclared(_ raw: UInt64, enumeration name: String) -> Bool { + guard let enumeration = schema.enums[name], enumeration.isClosed else { return true } + return enumeration.values[Int32(truncatingIfNeeded: raw)] != nil + } + + // The member set last wins, and only what follows the last switch to it counts. + private func keepLastOneofMember(_ accepted: inout [Int: [Occurrence]], message: ProtobufMessageDescriptor) { + var oneofs: [Int: [Int]] = [:] + for field in message.fields where accepted[field.number] != nil { + if let index = field.oneofIndex { oneofs[index, default: []].append(field.number) } + } + for members in oneofs.values where members.count > 1 { + let lastPositions = members.map { (number: $0, position: accepted[$0]?.last?.position ?? -1) } + guard let winner = lastPositions.max(by: { $0.position < $1.position })?.number else { continue } + let cutoff = lastPositions.filter { $0.number != winner }.map { $0.position }.max() ?? -1 + for number in members where number != winner { accepted[number] = nil } + accepted[winner] = accepted[winner]?.filter { $0.position > cutoff } + } + } + + private mutating func append( + _ field: ProtobufFieldDescriptor, + _ found: [Occurrence], + to level: inout Level, + depth: Int, + asMapEntry: Bool + ) throws { + switch field.type { + case .message(let name), .group(let name): + guard let nested = schema.messages[name] else { + level.unknown += found return } - // A repeated field keeps every value; any other keeps the last, as a parser does. - if descriptor.isRepeated { - known[descriptor.number, default: []].append(field) + if !field.isRepeated { + appendMessage(field, nested, found, to: &level, depth: depth) + } else if nested.isMapEntry { + appendMapEntries(field, nested, found, to: &level, depth: depth) } else { - known[descriptor.number] = [field] + for occurrence in found { appendMessage(field, nested, [occurrence], to: &level, depth: depth) } } + case .scalar, .enumeration: + if field.isRepeated { + for occurrence in found { appendRepeated(field, occurrence, to: &level) } + } else if let last = found.last { + // Last wins first: `count: 5` then `count: 0` leaves an implicit field at its default. + if !asMapEntry, !field.hasPresence, holdsDefault(last.payload, type: field.type) { return } + level.fields.append(typed(field, value(last.payload, type: field.type))) + } + case .unresolved: + level.unknown += found } - var ended = group == nil - while !reader.isAtEnd { - guard budget > 0 else { - isTruncated = true - break + } + + // A singular message seen again merges, which reads the same as its occurrences back to back. + private mutating func appendMessage( + _ field: ProtobufFieldDescriptor, + _ nested: ProtobufMessageDescriptor, + _ found: [Occurrence], + to level: inout Level, + depth: Int + ) { + let windows = found.compactMap { occurrence -> BinaryValueReader? in + switch occurrence.payload { + case .lengthDelimited(let window), .group(let window): return window + case .varint, .fixed64, .fixed32: return nil } - let start = reader.offset - let tag = try reader.readVarint() - let number = tag >> 3 - guard tag <= UInt64(UInt32.max), number > 0, number <= UInt64(ProtobufWireDecoder.maxFieldNumber) else { - throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + let start = budget + do { + let members = try self.members(of: windows, message: nested, depth: depth + 1, asMapEntry: false) + if case .group = field.type { + level.fields.append(ProtobufTypedField(name: nested.shortName, typeName: "group", value: .message(members))) + } else { + level.fields.append(typed(field, .message(members))) + } + } catch { + budget = start + level.unknown += found + } + } + + // protoc sorts a map by key and keeps every entry, a repeated key included. + private mutating func appendMapEntries( + _ field: ProtobufFieldDescriptor, + _ entry: ProtobufMessageDescriptor, + _ found: [Occurrence], + to level: inout Level, + depth: Int + ) { + var entries: [(key: MapKey, field: ProtobufTypedField)] = [] + for occurrence in found { + guard case .lengthDelimited(let window) = occurrence.payload else { continue } + let start = budget + do { + let members = try self.members(of: [window], message: entry, depth: depth + 1, asMapEntry: true) + entries.append((mapKey(members, entry: entry), typed(field, .message(members)))) + } catch { + budget = start + level.unknown.append(occurrence) } - let fieldNumber = Int(number) - let wireType = tag & 7 - if wireType == 4 { - guard group == fieldNumber else { throw BinaryValueDecodeError(kind: .unmatchedGroupEnd, offset: start) } - ended = true + } + let sorted = entries.enumerated().sorted { lhs, rhs in + lhs.element.key == rhs.element.key ? lhs.offset < rhs.offset : lhs.element.key < rhs.element.key + } + level.fields += sorted.map { $0.element.field } + } + + private mutating func appendRepeated(_ field: ProtobufFieldDescriptor, _ occurrence: Occurrence, to level: inout Level) { + guard case .lengthDelimited(let window) = occurrence.payload, + field.type != .scalar(.string), field.type != .scalar(.bytes) else { + level.fields.append(typed(field, value(occurrence.payload, type: field.type))) + return + } + var reader = window + var values: [ProtobufTypedField] = [] + var undeclared: [Occurrence] = [] + var remaining = budget + var isCut = false + while !reader.isAtEnd { + guard remaining > 0 else { + isCut = true break } - budget -= 1 - let descriptor = message?.field(number: fieldNumber) - switch wireType { - case 0: - let bits = try reader.readVarint() - if let descriptor, let text = varintText(bits, type: descriptor.type) { - add(typed(descriptor, .scalar(text)), descriptor: descriptor) - } else { - add(raw(fieldNumber, .varint(bits)), descriptor: nil) - } - case 1: - let bits = try reader.readUInt64LittleEndian() - if let descriptor, let text = fixed64Text(bits, type: descriptor.type) { - add(typed(descriptor, .scalar(text)), descriptor: descriptor) - } else { - add(raw(fieldNumber, .fixed64(bits)), descriptor: nil) - } - case 5: - let bits = try reader.readUInt32LittleEndian() - if let descriptor, let text = fixed32Text(bits, type: descriptor.type) { - add(typed(descriptor, .scalar(text)), descriptor: descriptor) - } else { - add(raw(fieldNumber, .fixed32(bits)), descriptor: nil) - } - case 2: - let length = try reader.readVarint() - guard length <= UInt64(reader.remaining) else { throw reader.error(.unexpectedEnd) } - let bytes = try reader.readBytes(Int(length)) - if let descriptor, case .message = descriptor.type, !descriptor.isRepeated { - singularMessages[fieldNumber, default: Data()].append(bytes) - } else if let descriptor, let values = lengthDelimited(bytes, descriptor: descriptor, depth: depth) { - for value in values { add(typed(descriptor, value), descriptor: descriptor) } - } else { - add(raw(fieldNumber, ProtobufWireDecoder.lengthDelimited(bytes)), descriptor: nil) - } - case 3: - if let descriptor, case .group(let name) = descriptor.type { - let members = try fields(&reader, message: schema.messages[name], depth: depth + 1, group: fieldNumber) - let groupName = schema.messages[name]?.shortName ?? descriptor.name - add( - ProtobufTypedField(name: groupName, typeName: "group", value: .message(members)), - descriptor: descriptor - ) - } else { - let members = try fields(&reader, message: nil, depth: depth + 1, group: fieldNumber) - add(ProtobufTypedField(name: String(fieldNumber), typeName: nil, value: .message(members)), descriptor: nil) - } - default: - throw BinaryValueDecodeError(kind: .invalidField, offset: start) + guard let item = try? Self.packedItem(&reader, type: field.type) else { + level.unknown.append(occurrence) + return } - if isTruncated { break } - } - if !ended, !isTruncated { throw reader.error(.unexpectedEnd) } - for (number, bytes) in singularMessages { - guard let descriptor = message?.field(number: number) else { continue } - if let values = lengthDelimited(bytes, descriptor: descriptor, depth: depth) { - known[number] = values.map { typed(descriptor, $0) } + remaining -= 1 + if case .varint(let raw) = item, case .enumeration(let name) = field.type, !isDeclared(raw, enumeration: name) { + undeclared.append(Occurrence(number: field.number, payload: item, position: occurrence.position)) } else { - unknown.append(raw(number, ProtobufWireDecoder.lengthDelimited(bytes))) + values.append(typed(field, .scalar(scalarText(item, type: field.type)))) } } - var result = known.keys.sorted().flatMap { known[$0] ?? [] } + unknown - if isTruncated, depth == 0 { result.append(ProtobufTypedField(name: "", typeName: nil, value: .truncated)) } - return result + budget = remaining + level.fields += values + level.unknown += undeclared + if isCut { level.isCut = true } } - private func typed(_ descriptor: ProtobufFieldDescriptor, _ value: ProtobufTypedValue) -> ProtobufTypedField { - ProtobufTypedField(name: descriptor.name, typeName: typeName(descriptor.type), value: value) + private static func packedItem(_ reader: inout BinaryValueReader, type: ProtobufFieldType) throws -> Occurrence.Payload { + guard case .scalar(let scalar) = type else { return .varint(try reader.readVarint()) } + switch scalar.wireType { + case 1: return .fixed64(try reader.readUInt64LittleEndian()) + case 5: return .fixed32(try reader.readUInt32LittleEndian()) + default: return .varint(try reader.readVarint()) + } } - private func raw(_ number: Int, _ value: ProtobufFieldValue) -> ProtobufTypedField { - ProtobufTypedField(name: String(number), typeName: nil, value: .unknown(value)) + private mutating func raw(_ occurrence: Occurrence, depth: Int) throws -> ProtobufTypedField { + let value: ProtobufFieldValue + switch occurrence.payload { + case .varint(let raw): value = .varint(raw) + case .fixed64(let raw): value = .fixed64(raw) + case .fixed32(let raw): value = .fixed32(raw) + case .lengthDelimited(let window): + value = ProtobufWireDecoder.lengthDelimited(window, depth: depth, budget: &budget) + case .group(let window): + value = .group(try ProtobufWireDecoder.fields(in: window, depth: depth + 1, budget: &budget)) + } + return ProtobufTypedField(name: String(occurrence.number), typeName: nil, value: .unknown(value)) + } + + private func mapKey(_ members: [ProtobufTypedField], entry: ProtobufMessageDescriptor) -> MapKey { + guard let key = entry.field(number: 1) else { return .signed(0) } + let value = members.first { $0.name == key.name }?.value + switch key.type { + case .scalar(.string): + if case .string(let text)? = value { return .bytes(Array(text.utf8)) } + if case .bytes(let data)? = value { return .bytes(Array(data)) } + return .bytes([]) + case .scalar(.bool): + return .unsigned(value == .scalar("true") ? 1 : 0) + case .scalar(.uint32), .scalar(.uint64), .scalar(.fixed32), .scalar(.fixed64): + guard case .scalar(let text)? = value else { return .unsigned(0) } + return .unsigned(UInt64(text) ?? 0) + default: + guard case .scalar(let text)? = value else { return .signed(0) } + return .signed(Int64(text) ?? 0) + } + } + + // Floats compare by bit pattern, so -0 is not the default. + private func holdsDefault(_ payload: Occurrence.Payload, type: ProtobufFieldType) -> Bool { + switch payload { + case .varint(let raw): + switch type { + case .scalar(.int64), .scalar(.uint64), .scalar(.sint64), .scalar(.bool): return raw == 0 + default: return UInt32(truncatingIfNeeded: raw) == 0 + } + case .fixed64(let raw): return raw == 0 + case .fixed32(let raw): return raw == 0 + case .lengthDelimited(let window): return window.remaining == 0 + case .group: return false + } + } + + private func defaultValue(_ type: ProtobufFieldType) -> ProtobufTypedValue? { + switch type { + case .scalar(.string): return .string("") + case .scalar(.bytes): return .bytes(Data()) + case .scalar(.bool): return .scalar("false") + case .scalar: return .scalar("0") + case .enumeration(let name): return .scalar(schema.enums[name]?.values[0] ?? "0") + case .message, .group: return .message([]) + case .unresolved: return nil + } + } + + // A string that is not UTF-8 prints escaped, as protoc prints a proto2 string. + private func value(_ payload: Occurrence.Payload, type: ProtobufFieldType) -> ProtobufTypedValue { + guard case .lengthDelimited(let window) = payload else { return .scalar(scalarText(payload, type: type)) } + var reader = window + let data = (try? reader.readBytes(reader.remaining)) ?? Data() + if type == .scalar(.string), let text = String(data: data, encoding: .utf8) { return .string(text) } + return .bytes(data) + } + + private func typed(_ descriptor: ProtobufFieldDescriptor, _ value: ProtobufTypedValue) -> ProtobufTypedField { + ProtobufTypedField(name: descriptor.name, typeName: typeName(descriptor.type), value: value) } private func typeName(_ type: ProtobufFieldType) -> String { @@ -192,56 +454,13 @@ internal enum ProtobufSchemaDecoder { } } - // nil leaves the value to the fields the schema does not name. - private mutating func lengthDelimited( - _ bytes: Data, - descriptor: ProtobufFieldDescriptor, - depth: Int - ) -> [ProtobufTypedValue]? { - switch descriptor.type { - case .scalar(.string): - guard let text = String(data: bytes, encoding: .utf8) else { return nil } - return [.string(text)] - case .scalar(.bytes): - return [.bytes(bytes)] - case .message(let name): - var nested = BinaryValueReader(bytes) - var attempt = self - guard let members = try? attempt.fields(&nested, message: schema.messages[name], depth: depth + 1, group: nil) else { - return nil - } - self = attempt - return [.message(members)] - case .scalar, .enumeration: - guard descriptor.isRepeated else { return nil } - return packed(bytes, type: descriptor.type) - case .group, .unresolved: - return nil - } - } - - private mutating func packed(_ bytes: Data, type: ProtobufFieldType) -> [ProtobufTypedValue]? { - var reader = BinaryValueReader(bytes) - var values: [ProtobufTypedValue] = [] - while !reader.isAtEnd { - guard budget > 0 else { - isTruncated = true - break - } - let text: String? - switch type { - case .scalar(.double), .scalar(.fixed64), .scalar(.sfixed64): - text = (try? reader.readUInt64LittleEndian()).flatMap { fixed64Text($0, type: type) } - case .scalar(.float), .scalar(.fixed32), .scalar(.sfixed32): - text = (try? reader.readUInt32LittleEndian()).flatMap { fixed32Text($0, type: type) } - default: - text = (try? reader.readVarint()).flatMap { varintText($0, type: type) } - } - guard let text else { return nil } - values.append(.scalar(text)) - budget -= 1 + private func scalarText(_ payload: Occurrence.Payload, type: ProtobufFieldType) -> String { + switch payload { + case .varint(let raw): return varintText(raw, type: type) ?? String(raw) + case .fixed64(let raw): return fixed64Text(raw, type: type) ?? String(raw) + case .fixed32(let raw): return fixed32Text(raw, type: type) ?? String(raw) + case .lengthDelimited, .group: return "" } - return values } private func varintText(_ raw: UInt64, type: ProtobufFieldType) -> String? { @@ -280,6 +499,17 @@ internal enum ProtobufSchemaDecoder { } } +private extension ProtobufScalarType { + var wireType: UInt64 { + switch self { + case .double, .fixed64, .sfixed64: return 1 + case .float, .fixed32, .sfixed32: return 5 + case .string, .bytes: return 2 + case .int32, .int64, .uint32, .uint64, .sint32, .sint64, .bool: return 0 + } + } +} + // protobuf's SimpleDtoa and SimpleFtoa: the shortest of DBL_DIG or FLT_DIG digits that reads back, else the full count. internal enum ProtobufNumberText { static func double(_ value: Double) -> String { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift index 4d024265e7..11ebbd1b37 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift @@ -65,11 +65,57 @@ internal enum ProtobufWireDecoder { return ProtobufFraming(schemaID: schemaID, messageIndexes: indexes) } - // A length-delimited value no schema names, read the way decode() reads one. - static func lengthDelimited(_ data: Data, maxValues: Int = .max) -> ProtobufFieldValue { - var parser = Parser(budget: maxValues) - var window = BinaryValueReader(data) - return parser.lengthDelimited(&window, depth: 0) + // Spends the caller's budget, so a cell stops inside a large field the schema does not name. + static func lengthDelimited(_ window: BinaryValueReader, depth: Int, budget: inout Int) -> ProtobufFieldValue { + var parser = Parser(budget: budget) + var window = window + let value = parser.lengthDelimited(&window, depth: depth) + budget = parser.budget + return value + } + + static func fields(in window: BinaryValueReader, depth: Int, budget: inout Int) throws -> [ProtobufField] { + var parser = Parser(budget: budget) + var window = window + let fields = try parser.fields(&window, depth: depth, group: nil) + budget = parser.budget + return fields + } + + // The bytes between a group's start tag, already read, and its end tag, which the reader is left past. + static func groupContents(_ reader: inout BinaryValueReader, number: Int, depth: Int) throws -> BinaryValueReader { + let contents = reader + var openGroups = [number] + while true { + let start = reader.offset + guard !reader.isAtEnd else { throw reader.error(.unexpectedEnd) } + let tag = try reader.readVarint() + let fieldNumber = tag >> 3 + guard tag <= UInt64(UInt32.max), fieldNumber > 0, fieldNumber <= UInt64(maxFieldNumber) else { + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + switch tag & 7 { + case 0: + _ = try reader.readVarint() + case 1: + try reader.skip(8) + case 2: + let length = try reader.readVarint() + guard length <= UInt64(reader.remaining) else { throw reader.error(.unexpectedEnd) } + try reader.skip(Int(length)) + case 3: + guard depth + openGroups.count < maxDepth else { throw BinaryValueDecodeError(kind: .nestingTooDeep, offset: start) } + openGroups.append(Int(fieldNumber)) + case 4: + guard openGroups.last == Int(fieldNumber) else { throw BinaryValueDecodeError(kind: .unmatchedGroupEnd, offset: start) } + openGroups.removeLast() + if openGroups.isEmpty { return try contents.window(start - contents.offset) } + case 5: + try reader.skip(4) + default: + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + } } static func zigzag(_ value: UInt64) -> Int64 { diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift index 04c7aa4632..f3833ba647 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift @@ -17,14 +17,11 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case messagePack case protobuf case igbinary - // The schema is kept by AvroSchemaStore under this ID, so every per-column path can decode with it. case avro(schemaID: String) - // Kept by ProtobufSchemaStore, with the message type the values are read as. case protobufSchema(schemaID: String, message: String) - // Each value names its schema by ID; the connection says which Schema Registry to fetch it from. case schemaRegistry(connectionID: UUID) - // Every format but Avro, which needs a schema chosen first. + // The schema formats are left out: each needs a file or a registry chosen first. static let presetFormats: [ValueDisplayFormat] = [ .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized, .messagePack, .protobuf, .igbinary, @@ -140,7 +137,7 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { return [] case .text, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro, .protobufSchema: return ["blob"] - // A framed value whose payload happens to be UTF-8 lands in a text column. + // An older Kafka plugin puts a framed value whose bytes are valid UTF-8 in a text column. case .schemaRegistry: return ["blob", "text"] case .uuid: diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift index 680082e70e..3732e7d5b0 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift @@ -37,9 +37,8 @@ enum ValueDisplayFormatter { case .json, .phpSerialized: return rawValue case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: - // A text column holds the bytes as UTF-8; a binary one as one Latin-1 character per byte. - let encoding: String.Encoding = columnType?.isBlobType == false ? .utf8 : .isoLatin1 - guard let data = rawValue.data(using: encoding), let decoder = format.binaryValueFormat else { return nil } + guard let data = BinaryValueBytes.data(of: rawValue, holdsBytes: BinaryValueBytes.holdsBytes(columnType: columnType)), + let decoder = format.binaryValueFormat else { return nil } return BinaryValueDecoding.text(data, format: decoder) } } diff --git a/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift b/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift index e0e70ccf8a..768385daf1 100644 --- a/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift +++ b/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift @@ -10,8 +10,7 @@ internal struct ProtobufSchemaChoice { let message: String } -// A `.proto` file or a descriptor set, any extension. The panel's validate hook refuses anything else with the reason, -// and a file with several messages then asks which one the column's values are. +// Any extension: a descriptor set has none of its own, so the validate hook refuses a file that is not a schema. @MainActor internal enum ProtobufSchemaFilePanel { static func chooseSchema(in window: NSWindow) async -> ProtobufSchemaChoice? { @@ -60,8 +59,8 @@ private final class ProtobufSchemaFileGate: NSObject, NSOpenSavePanelDelegate { } do { schema = try ProtobufSchema.parse(data) - } catch let error as ProtobufSchemaError { - throw ProtobufSchemaFileError(fileName: name, reason: error.reason) + } catch { + throw ProtobufSchemaFileError(fileName: name, reason: ((error as? ProtobufSchemaError) ?? .notASchema).reason) } contents = data } diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift index 9f03c6d9fd..737e40c80e 100644 --- a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift @@ -6,23 +6,19 @@ import Foundation import os -// Schemas fetched from a connection's Schema Registry, by connection and schema ID, for the session. A registry -// never changes the schema behind an ID, so a fetched one is kept; a failure is kept briefly, so a grid of values -// naming the same ID asks once and a registry that comes back is asked again. -// -// The grid reads synchronously: a miss starts the fetch and the grid redraws on `didLoadSchema`. The viewer awaits it. +// A registry never changes the schema behind an ID, so a loaded one is kept for the session, keyed by the endpoint it came from. internal final class SchemaRegistryCache: Sendable { - typealias EndpointProvider = @Sendable (UUID) async -> SchemaRegistryEndpoint? + typealias Lookup = Result + typealias EndpointResolver = @MainActor @Sendable (UUID) -> SchemaRegistryEndpoint? static let shared = SchemaRegistryCache() - static let didLoadSchema = Notification.Name("com.TablePro.schemaRegistryCache.didLoadSchema") + // Posted once per batch that loaded a schema, and when a connection's endpoint changed. + static let didChange = Notification.Name("com.TablePro.schemaRegistryCache.didChange") static let connectionIDKey = "connectionID" - static let failureLifetime: Duration = .seconds(30) - private struct Key: Hashable { - let connectionID: UUID + let endpoint: SchemaRegistryEndpoint let schemaID: UInt32 } @@ -31,76 +27,153 @@ internal final class SchemaRegistryCache: Sendable { case failed(SchemaRegistryError, at: ContinuousClock.Instant) } + private enum Resolved { + case configured(SchemaRegistryEndpoint) + case unconfigured + } + private struct State { + var endpoints: [UUID: Resolved] = [:] var entries: [Key: Entry] = [:] - var inFlight: [Key: Task, Never>] = [:] + var inFlight: [Key: Task] = [:] } private let state = OSAllocatedUnfairLock(initialState: State()) - private let endpoint: EndpointProvider + private let resolveEndpoint: EndpointResolver private let transport: SchemaRegistryClient.Transport + private let failureLifetime: Duration init( - endpoint: @escaping EndpointProvider = SchemaRegistryCache.savedEndpoint, + failureLifetime: Duration = .seconds(30), + resolveEndpoint: @escaping EndpointResolver = { SchemaRegistryEndpoint.resolve(for: $0) }, transport: @escaping SchemaRegistryClient.Transport = SchemaRegistryClient.urlSessionTransport ) { - self.endpoint = endpoint + self.failureLifetime = failureLifetime + self.resolveEndpoint = resolveEndpoint self.transport = transport } - // The fields a connect resolves, secure ones and a linked credential profile's included. - static let savedEndpoint: EndpointProvider = { connectionID in - await MainActor.run { - guard let connection = ConnectionStorage.shared.loadConnections().first(where: { $0.id == connectionID }) else { + // Never fetches: nil until a prefetch or the viewer has asked for the schema. + func cachedSchema(connectionID: UUID, schemaID: UInt32) -> Lookup? { + state.withLock { state in + switch state.endpoints[connectionID] { + case nil: return nil + case .unconfigured?: + return .failure(Self.noEndpoint) + case .configured(let endpoint)?: + switch state.entries[Key(endpoint: endpoint, schemaID: schemaID)] { + case .loaded(let schema)?: return .success(schema) + case .failed(let error, _)?: return .failure(error) + case nil: return nil + } } - let fields = DatabaseDriverFactory.resolvedAdditionalFields(for: connection) ?? connection.additionalFields - return SchemaRegistryEndpoint(fields: fields) } } - // nil while the schema is being fetched, which a miss starts. - func cachedSchema(connectionID: UUID, schemaID: UInt32) -> Result? { - let key = Key(connectionID: connectionID, schemaID: schemaID) - if let result = state.withLock({ current(key, in: &$0) }) { return result } - _ = start(key) - return nil + // A full reload retries failures at once; anything else waits out the failure lifetime. + @MainActor + func prefetch(connectionID: UUID, schemaIDs: Set, retryingFailures: Bool) { + guard !schemaIDs.isEmpty else { return } + let allLoaded = state.withLock { state -> Bool in + guard case .configured(let endpoint)? = state.endpoints[connectionID] else { return false } + return schemaIDs.allSatisfy { + if case .loaded? = state.entries[Key(endpoint: endpoint, schemaID: $0)] { return true } + return false + } + } + // Resolving reads the Keychain, so a batch that needs nothing new skips it unless a reload asked to look again. + if allLoaded, !retryingFailures { return } + let endpoint = resolveEndpoint(connectionID) + let (changed, pending) = state.withLock { state -> (Bool, [Task]) in + let changed = adopt(endpoint, for: connectionID, in: &state) + guard let endpoint else { return (changed, []) } + let pending = schemaIDs.compactMap { schemaID -> Task? in + let key = Key(endpoint: endpoint, schemaID: schemaID) + if let running = state.inFlight[key] { return running } + guard usable(state.entries[key], retryingFailures: retryingFailures) == nil else { return nil } + return start(key, in: &state) + } + return (changed, pending) + } + if changed { Self.post(connectionID) } + guard !pending.isEmpty else { return } + Task { + var loaded = false + for task in pending { + if case .success = await task.value { loaded = true } + } + if loaded { Self.post(connectionID) } + } } - func schema(connectionID: UUID, schemaID: UInt32) async -> Result { - let key = Key(connectionID: connectionID, schemaID: schemaID) - if let result = state.withLock({ current(key, in: &$0) }) { return result } - return await start(key).value + @MainActor + func schema(connectionID: UUID, schemaID: UInt32) async -> Lookup { + let endpoint = resolveEndpoint(connectionID) + let (changed, pending, cached) = state.withLock { state -> (Bool, Task?, Lookup?) in + let changed = adopt(endpoint, for: connectionID, in: &state) + guard let endpoint else { return (changed, nil, .failure(Self.noEndpoint)) } + let key = Key(endpoint: endpoint, schemaID: schemaID) + if let running = state.inFlight[key] { return (changed, running, nil) } + if let cached = usable(state.entries[key], retryingFailures: false) { return (changed, nil, cached) } + return (changed, start(key, in: &state), nil) + } + if changed { Self.post(connectionID) } + guard let pending else { return cached ?? .failure(Self.noEndpoint) } + let result = await pending.value + if case .success = result { Self.post(connectionID) } + return result } - // What the grid can use now; an expired failure is dropped so the next read fetches again. - private func current(_ key: Key, in state: inout State) -> Result? { - switch state.entries[key] { + private static var noEndpoint: SchemaRegistryError { + SchemaRegistryError( + reason: String(localized: "The connection names no Schema Registry URL. Add one in the connection's settings.") + ) + } + + private func adopt(_ endpoint: SchemaRegistryEndpoint?, for connectionID: UUID, in state: inout State) -> Bool { + let resolved: Resolved = endpoint.map { .configured($0) } ?? .unconfigured + let previous = state.endpoints.updateValue(resolved, forKey: connectionID) + switch (previous, resolved) { + case (nil, _), (.unconfigured?, .unconfigured): + return false + case (.configured(let old)?, .configured(let new)): + return old != new + default: + return true + } + } + + private func usable(_ entry: Entry?, retryingFailures: Bool) -> Lookup? { + switch entry { case .loaded(let schema)?: return .success(schema) case .failed(let error, let at)?: - guard ContinuousClock.now - at >= Self.failureLifetime else { return .failure(error) } - state.entries[key] = nil - return nil + return retryingFailures || ContinuousClock.now - at >= failureLifetime ? nil : .failure(error) case nil: return nil } } - private func start(_ key: Key) -> Task, Never> { - state.withLock { state in - if let running = state.inFlight[key] { return running } - let task = Task { [endpoint, transport] in - let result = await Self.fetch(key, endpoint: endpoint, transport: transport) - self.finish(key, with: result) - return result + private func start(_ key: Key, in state: inout State) -> Task { + let client = SchemaRegistryClient(endpoint: key.endpoint, transport: transport) + let task = Task { + let result: Lookup + do { + result = .success(try await client.schema(id: key.schemaID)) + } catch let error as SchemaRegistryError { + result = .failure(error) + } catch { + result = .failure(SchemaRegistryError(reason: error.localizedDescription)) } - state.inFlight[key] = task - return task + self.finish(key, with: result) + return result } + state.inFlight[key] = task + return task } - private func finish(_ key: Key, with result: Result) { + private func finish(_ key: Key, with result: Lookup) { state.withLock { state in state.inFlight[key] = nil switch result { @@ -108,27 +181,9 @@ internal final class SchemaRegistryCache: Sendable { case .failure(let error): state.entries[key] = .failed(error, at: .now) } } - NotificationCenter.default.post( - name: Self.didLoadSchema, object: self, userInfo: [Self.connectionIDKey: key.connectionID] - ) } - private static func fetch( - _ key: Key, - endpoint: EndpointProvider, - transport: @escaping SchemaRegistryClient.Transport - ) async -> Result { - guard let resolved = await endpoint(key.connectionID) else { - return .failure(SchemaRegistryError( - reason: String(localized: "The connection names no Schema Registry URL. Add one in the connection's settings.") - )) - } - do { - return .success(try await SchemaRegistryClient(endpoint: resolved, transport: transport).schema(id: key.schemaID)) - } catch let error as SchemaRegistryError { - return .failure(error) - } catch { - return .failure(SchemaRegistryError(reason: error.localizedDescription)) - } + private static func post(_ connectionID: UUID) { + NotificationCenter.default.post(name: didChange, object: nil, userInfo: [connectionIDKey: connectionID]) } } diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift index de1150070d..4981ce79d7 100644 --- a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift @@ -5,20 +5,18 @@ import Foundation -// The connection fields a driver declares to name its Schema Registry. Engine neutral: any plugin that declares -// them gets the Schema Registry display format. +// Engine neutral: any driver that declares these connection fields gets the Schema Registry display format. internal enum SchemaRegistryConnectionField { static let url = "schemaRegistryUrl" static let username = "schemaRegistryUsername" static let password = "schemaRegistryPassword" } -internal struct SchemaRegistryEndpoint: Equatable, Sendable { +internal struct SchemaRegistryEndpoint: Hashable, Sendable { let baseURL: URL let username: String? let password: String? - // Only http and https: the URL is fetched from, so a file or custom scheme is refused here. init?(fields: [String: String]) { let text = (fields[SchemaRegistryConnectionField.url] ?? "").trimmingCharacters(in: .whitespacesAndNewlines) guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), ["http", "https"].contains(scheme), @@ -28,12 +26,31 @@ internal struct SchemaRegistryEndpoint: Equatable, Sendable { self.username = username.isEmpty ? nil : username password = fields[SchemaRegistryConnectionField.password] } + + // The session's record, so the endpoint changes when the connection does: on the next connect, like every other field. + @MainActor + static func connection(for connectionID: UUID) -> DatabaseConnection? { + DatabaseManager.shared.session(for: connectionID)?.connection + ?? ConnectionStorage.shared.loadConnection(id: connectionID) + } + + @MainActor + static func isConfigured(for connectionID: UUID) -> Bool { + guard let connection = connection(for: connectionID) else { return false } + return SchemaRegistryEndpoint(fields: connection.additionalFields) != nil + } + + @MainActor + static func resolve(for connectionID: UUID) -> SchemaRegistryEndpoint? { + guard let connection = connection(for: connectionID) else { return nil } + let fields = DatabaseDriverFactory.resolvedAdditionalFields(for: connection) ?? connection.additionalFields + return SchemaRegistryEndpoint(fields: fields) + } } internal enum RegistrySchema: Sendable { case avro(AvroSchema) case protobuf(ProtobufSchema) - // A JSON Schema value is JSON text behind the prefix; the schema adds nothing to reading it. case json } @@ -60,15 +77,17 @@ internal struct SchemaRegistryClient: Sendable { private static let session: URLSession = { let configuration = URLSessionConfiguration.ephemeral configuration.timeoutIntervalForResource = 60 + configuration.httpShouldSetCookies = false return URLSession(configuration: configuration) }() - // Read up to the cap, so a registry that never stops sending cannot fill memory. static let urlSessionTransport: Transport = { request in - let (bytes, response) = try await session.bytes(for: request, delegate: SameOriginRedirects()) + guard let url = request.url else { throw URLError(.badURL) } + let (bytes, response) = try await session.bytes(for: request, delegate: SameOriginRedirectGuard(origin: url)) var data = Data() for try await byte in bytes { data.append(byte) + // Stops a registry that never ends its answer from filling memory. if data.count > maxResponseBytes { break } } return (data, response) @@ -78,18 +97,14 @@ internal struct SchemaRegistryClient: Sendable { let root: SchemaResponse = try await get(["schemas", "ids", String(id)]) switch root.schemaType?.uppercased() ?? "AVRO" { case "AVRO": - var references: [String] = [] - var seen: Set = [] - try await collectReferences(root.references ?? [], into: &references, seen: &seen) + let references = try await referencedSchemas(root.references ?? []) do { return .avro(try AvroSchema.parse(root.schema, references: references)) } catch let error as AvroSchemaError { throw SchemaRegistryError(reason: error.reason) } case "PROTOBUF": - var references: [String] = [] - var seen: Set = [] - try await collectReferences(root.references ?? [], into: &references, seen: &seen) + let references = try await referencedSchemas(root.references ?? []) do { return .protobuf(try ProtobufSchema.parse(protoFiles: [root.schema] + references)) } catch let error as ProtobufSchemaError { @@ -104,30 +119,31 @@ internal struct SchemaRegistryClient: Sendable { } } - // Depth first, each subject version once and after the ones it references, so a type is defined before its users. - private func collectReferences( - _ references: [SchemaReference], - into texts: inout [String], - seen: inout Set - ) async throws { + // Each subject version once, after the ones it references, so a type is defined before its users. + private func referencedSchemas(_ references: [SchemaReference]) async throws -> [String] { + var texts: [String] = [] + var seen: Set = [] + try await collect(references, into: &texts, seen: &seen) + return texts + } + + private func collect(_ references: [SchemaReference], into texts: inout [String], seen: inout Set) async throws { for reference in references { - let key = "\(reference.subject)@\(reference.version)" - guard seen.insert(key).inserted else { continue } + guard seen.insert("\(reference.subject)@\(reference.version)").inserted else { continue } guard seen.count <= Self.maxReferences else { throw SchemaRegistryError(reason: String(localized: "The schema imports more files than TablePro reads.")) } let version: SchemaResponse = try await get(["subjects", reference.subject, "versions", String(reference.version)]) - try await collectReferences(version.references ?? [], into: &texts, seen: &seen) + try await collect(version.references ?? [], into: &texts, seen: &seen) texts.append(version.schema) } } private func get(_ components: [String]) async throws -> Response { - let request = request(components) let data: Data let response: URLResponse do { - (data, response) = try await transport(request) + (data, response) = try await transport(request(components)) } catch { throw SchemaRegistryError(reason: String( format: String(localized: "TablePro could not reach the Schema Registry: %@"), error.localizedDescription @@ -151,7 +167,7 @@ internal struct SchemaRegistryClient: Sendable { } } - // Each component is encoded on its own: a subject such as `demo/common.proto` holds a slash. + // Each component is encoded alone: a subject such as `demo/common.proto` holds a slash. func request(_ components: [String]) -> URLRequest { var allowed = CharacterSet.urlPathAllowed allowed.remove("/") @@ -185,23 +201,3 @@ internal struct SchemaRegistryClient: Sendable { let message: String } } - -// The credentials go to the registry the connection names, so a redirect elsewhere, or down to http, is not followed. -private final class SameOriginRedirects: NSObject, URLSessionTaskDelegate { - func urlSession( - _ session: URLSession, - task: URLSessionTask, - willPerformHTTPRedirection response: HTTPURLResponse, - newRequest request: URLRequest, - completionHandler: @escaping @Sendable (URLRequest?) -> Void - ) { - guard let from = task.originalRequest?.url, let to = request.url, - from.scheme?.lowercased() == to.scheme?.lowercased(), - from.host?.lowercased() == to.host?.lowercased(), - from.port == to.port else { - completionHandler(nil) - return - } - completionHandler(request) - } -} diff --git a/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift b/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift index bcd0d2a0ce..5646f2d8a8 100644 --- a/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift +++ b/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift @@ -5,7 +5,6 @@ import Foundation -// A `.proto` file or a descriptor set, kept as chosen; ProtobufSchema.parse tells the two apart. internal final class ProtobufSchemaStore: Sendable { static let shared = ProtobufSchemaStore( directory: AppStorageEnvironment.shared.supportDirectory.appendingPathComponent("ProtobufSchemas", isDirectory: true) @@ -14,6 +13,7 @@ internal final class ProtobufSchemaStore: Sendable { static let maxFileBytes = 4_000_000 static let didRestoreSchema = Notification.Name("com.TablePro.protobufSchemaStore.didRestoreSchema") + static let schemaIDKey = "schemaID" private let files: SchemaFileCache @@ -25,7 +25,7 @@ internal final class ProtobufSchemaStore: Sendable { let (id, wasMissing) = try files.add(contents) if wasMissing { NotificationCenter.default.post( - name: Self.didRestoreSchema, object: self, userInfo: [AvroSchemaStore.schemaIDKey: id] + name: Self.didRestoreSchema, object: self, userInfo: [Self.schemaIDKey: id] ) } return id diff --git a/apple/TablePro/Core/Storage/SchemaFileCache.swift b/apple/TablePro/Core/Storage/SchemaFileCache.swift index d36c48edce..2a54eded99 100644 --- a/apple/TablePro/Core/Storage/SchemaFileCache.swift +++ b/apple/TablePro/Core/Storage/SchemaFileCache.swift @@ -7,15 +7,13 @@ import CryptoKit import Foundation import os -// The schema files chosen for columns, one file each, named by a hash of the contents, so a column still decodes -// after the original file moves and the same file chosen twice is kept once. Read from the grid on the main actor -// and from the viewer off it. +// Named by a hash of the contents, so a column still decodes after the original file moves. internal final class SchemaFileCache: Sendable { private struct State { var parsed: [String: Schema] = [:] // Kept from a sweep: a schema can be added before the column's format is saved. var addedThisLaunch: Set = [] - // Asked for and not found, so adding one later is a restore that grids and viewers have to hear about. + // Also answers later reads without the disk, since the grid asks once per cell. var missing: Set = [] } @@ -38,7 +36,6 @@ internal final class SchemaFileCache: Sendable { SHA256.hash(data: contents).prefix(16).map { String(format: "%02x", $0) }.joined() } - // Parses first, so nothing is kept for a file that is not a schema. `wasMissing` says a reader asked for it before. func add(_ contents: Data) throws -> (id: String, wasMissing: Bool) { let schema = try parse(contents) let id = Self.schemaID(of: contents) @@ -57,7 +54,11 @@ internal final class SchemaFileCache: Sendable { } func schema(id: String) -> Schema? { - if let cached = state.withLock({ $0.parsed[id] }) { return cached } + let known = state.withLock { state -> (schema: Schema?, isMissing: Bool) in + (state.parsed[id], state.missing.contains(id)) + } + if let cached = known.schema { return cached } + if known.isMissing { return nil } guard Self.isSchemaID(id), let data = try? Data(contentsOf: fileURL(id)), let schema = try? parse(data) else { diff --git a/apple/TablePro/Models/UI/ProtobufTreeNode.swift b/apple/TablePro/Models/UI/ProtobufTreeNode.swift index 629b96c603..c3f67de671 100644 --- a/apple/TablePro/Models/UI/ProtobufTreeNode.swift +++ b/apple/TablePro/Models/UI/ProtobufTreeNode.swift @@ -5,8 +5,7 @@ import Foundation -// Without a schema the bytes give only a field number, the key, and a wire type, the badge. A schema gives the -// field's name and declared type instead. +// Without a schema the key is the field number and the badge the wire type; a schema gives name and declared type. internal struct ProtobufTreeNode: Identifiable { internal enum Kind: Equatable { case message diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift index 6718a45cce..84d08cd831 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift @@ -141,11 +141,9 @@ extension DataGridViewController { return item } - // Offered only where the connection names a Schema Registry to fetch the schemas from. private var schemaRegistryFormat: ValueDisplayFormat? { guard let connectionID = model.configuration.traits.connectionScope?.connectionId, - let fields = DatabaseManager.shared.session(for: connectionID)?.connection.additionalFields, - SchemaRegistryEndpoint(fields: fields) != nil else { return nil } + SchemaRegistryEndpoint.isConfigured(for: connectionID) else { return nil } return .schemaRegistry(connectionID: connectionID) } @@ -300,7 +298,6 @@ extension DataGridViewController { } } - // A file with several messages asks which one the values are; the schema is copied in like Avro's. @objc func chooseProtobufSchemaFromMenu(_ sender: Any?) { guard let columnID = menuColumn(sender)?.id, let window = view.window else { return } Task { @MainActor [weak self] in diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController.swift b/apple/TablePro/Views/DataGrid/DataGridViewController.swift index f243d5d69b..dd7176f7af 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController.swift @@ -219,10 +219,6 @@ final class DataGridViewController: NSViewController, DataGridMountedView { observers.append(center.addObserver(forName: DataGridAccessibility.didActivateNotification, object: nil, queue: nil) { [weak self] _ in Task { @MainActor [weak self] in self?.remountAccessibilityCells() } }) - observers.append(center.addObserver(forName: SchemaRegistryCache.didLoadSchema, object: nil, queue: .main) { [weak self] note in - guard let connectionID = note.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID else { return } - MainActor.assumeIsolated { self?.model.schemaRegistryDidLoad(connectionID: connectionID) } - }) let scrollView = main.scrollView observers.append(center.addObserver(forName: NSScrollView.willStartLiveScrollNotification, object: scrollView, queue: .main) { [weak self] _ in MainActor.assumeIsolated { self?.pausePrewarm() } diff --git a/apple/TablePro/Views/DataGrid/Editing/DataGridViewController+TypedEditors.swift b/apple/TablePro/Views/DataGrid/Editing/DataGridViewController+TypedEditors.swift index a0a4227f07..b7cb10390b 100644 --- a/apple/TablePro/Views/DataGrid/Editing/DataGridViewController+TypedEditors.swift +++ b/apple/TablePro/Views/DataGrid/Editing/DataGridViewController+TypedEditors.swift @@ -373,7 +373,9 @@ extension DataGridViewController { switch cell.value { case .null: return case .bytes(let bytes): data = bytes - case .text(let text): data = text.storedBytes + case .text(let text): + let holdsBytes = BinaryValueBytes.holdsBytes(columnType: cell.column.type) + data = BinaryValueBytes.data(of: text, holdsBytes: holdsBytes) ?? text.storedBytes } let name = cell.column.name presentTypedPopover(for: cell, edits: false, contentSize: NSSize(width: 560, height: 360)) { dismiss in diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index 7bf51cb15d..1e0c45af07 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -50,7 +50,12 @@ internal struct BinaryValueViewerView: View { } .onReceive(NotificationCenter.default.publisher(for: ProtobufSchemaStore.didRestoreSchema).receive(on: RunLoop.main)) { notification in guard case .protobufSchema(let schemaID, _) = format, - notification.userInfo?[AvroSchemaStore.schemaIDKey] as? String == schemaID else { return } + notification.userInfo?[ProtobufSchemaStore.schemaIDKey] as? String == schemaID else { return } + schemaRevision += 1 + } + .onReceive(NotificationCenter.default.publisher(for: SchemaRegistryCache.didChange).receive(on: RunLoop.main)) { notification in + guard case .schemaRegistry(let connectionID) = format, + notification.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID == connectionID else { return } schemaRevision += 1 } } @@ -224,7 +229,11 @@ internal struct BinaryValueViewerView: View { guard decodedKey != key else { return } let data = data let format = format - // The grid fetches a schema without waiting for it; the viewer waits, so it never shows a fetch in progress. + // A new value never shows the last one's document while its schema is fetched. + if decodedKey?.data != data || decodedKey?.format != format { + document = nil + hexDump = "" + } if case .schemaRegistry(let connectionID) = format, let schemaID = try? SchemaRegistryFrame.schemaID(data) { _ = await SchemaRegistryCache.shared.schema(connectionID: connectionID, schemaID: schemaID) guard !Task.isCancelled else { return } diff --git a/apple/TablePro/Views/Results/CellInteractionResolver.swift b/apple/TablePro/Views/Results/CellInteractionResolver.swift index 8dcec9664e..83255a105f 100644 --- a/apple/TablePro/Views/Results/CellInteractionResolver.swift +++ b/apple/TablePro/Views/Results/CellInteractionResolver.swift @@ -75,8 +75,9 @@ internal struct CellInteractionResolver { let holdsBytes = context.columnType?.isBlobType == true || context.isBinaryValue // Read-only: nothing writes a decoded tree back. A NULL keeps the hex editor, the way bytes get into it. - if holdsBytes, context.isBinaryValue || context.value != nil, - let format = context.displayFormatOverride?.binaryValueFormat { + if let override = context.displayFormatOverride, let format = override.binaryValueFormat, + context.isBinaryValue || context.value != nil, + holdsBytes || override.isApplicable(to: context.columnType) { return .viewBinaryValue(format) } diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/BinaryValueFieldView.swift b/apple/TablePro/Views/RowInspector/FieldEditors/BinaryValueFieldView.swift index da317d0891..46212c9190 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/BinaryValueFieldView.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/BinaryValueFieldView.swift @@ -23,7 +23,7 @@ internal struct BinaryValueFieldView: View { .padding(.vertical, 3) } else { BinaryValueViewerView( - data: context.value.wrappedValue.storedBytes, + data: bytes, format: format, onPopOut: onPopOut.map { popOut in { popOut(context.value.wrappedValue) } } ) @@ -33,4 +33,10 @@ internal struct BinaryValueFieldView: View { } .valueWell(clipsContent: true) } + + private var bytes: Data { + let text = context.value.wrappedValue + let holdsBytes = BinaryValueBytes.holdsBytes(columnType: context.columnType, isBinaryValue: context.isBinaryValue) + return BinaryValueBytes.data(of: text, holdsBytes: holdsBytes) ?? text.storedBytes + } } diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift b/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift index 46174a22b3..a8e66e7b6b 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift @@ -25,6 +25,7 @@ internal struct FieldEditorContext { // False where the store has no NULL or DEFAULT to write, as in a data file. internal let allowsNullAndDefault: Bool + internal let isBinaryValue: Bool internal init( columnName: String, @@ -36,7 +37,8 @@ internal struct FieldEditorContext { isReadOnly: Bool, commitBytes: ((Data) -> Void)? = nil, editor: FieldEditorKind? = nil, - allowsNullAndDefault: Bool = true + allowsNullAndDefault: Bool = true, + isBinaryValue: Bool = false ) { self.columnName = columnName self.columnType = columnType @@ -48,6 +50,7 @@ internal struct FieldEditorContext { self.commitBytes = commitBytes self.editor = editor self.allowsNullAndDefault = allowsNullAndDefault + self.isBinaryValue = isBinaryValue } internal var hasMultipleValues: Bool { valueState == .multipleValues } diff --git a/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift b/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift index c3ec4ce0c4..6f8a1f812c 100644 --- a/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift +++ b/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift @@ -179,7 +179,8 @@ internal struct InspectorFieldListView: View { ? { editState.setFieldToBytes(at: field.columnIndex, data: $0) } : nil, editor: kind, - allowsNullAndDefault: offersDatabaseValues + allowsNullAndDefault: offersDatabaseValues, + isBinaryValue: field.isBinaryValue ) } diff --git a/apple/TablePro/Views/RowInspector/RowInspectorView.swift b/apple/TablePro/Views/RowInspector/RowInspectorView.swift index 03a8555d7f..112c74b661 100644 --- a/apple/TablePro/Views/RowInspector/RowInspectorView.swift +++ b/apple/TablePro/Views/RowInspector/RowInspectorView.swift @@ -158,7 +158,12 @@ internal struct RowInspectorView: View { case .php: PhpViewerWindowController.open(text: text, columnName: field.columnName) case .binary(let format): - BinaryValueViewerWindowController.open(data: text.storedBytes, format: format, columnName: field.columnName) + let holdsBytes = BinaryValueBytes.holdsBytes(columnType: field.columnTypeEnum, isBinaryValue: field.isBinaryValue) + BinaryValueViewerWindowController.open( + data: BinaryValueBytes.data(of: text, holdsBytes: holdsBytes) ?? text.storedBytes, + format: format, + columnName: field.columnName + ) case .text: TextViewerWindowController.open( text: text, diff --git a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift index 57ab627a1e..c98ff27917 100644 --- a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift +++ b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift @@ -53,7 +53,8 @@ internal enum FieldEditorResolver { return .json case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: // A NULL keeps the hex editor, the way bytes get into it. - if type.isBlobType || isBinaryValue, originalValue != nil, let format = override.binaryValueFormat { + if type.isBlobType || isBinaryValue || override.isApplicable(to: type), originalValue != nil, + let format = override.binaryValueFormat { return .binaryValue(format) } structuredAllowed = true diff --git a/apple/TableProTests/Core/Network/SameOriginRedirectGuardTests.swift b/apple/TableProTests/Core/Network/SameOriginRedirectGuardTests.swift new file mode 100644 index 0000000000..14b1ad02ea --- /dev/null +++ b/apple/TableProTests/Core/Network/SameOriginRedirectGuardTests.swift @@ -0,0 +1,31 @@ +// +// SameOriginRedirectGuardTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct SameOriginRedirectGuardTests { + private static func allows(_ origin: String, _ target: String) throws -> Bool { + SameOriginRedirectGuard(origin: try #require(URL(string: origin))).allows(try #require(URL(string: target))) + } + + @Test("A redirect on the same scheme, host and port is followed, the default port written or not") + func sameOrigin() throws { + #expect(try Self.allows("https://registry.example.com/base", "https://REGISTRY.example.com/other")) + #expect(try Self.allows("http://registry.example.com", "http://registry.example.com:80/schemas")) + #expect(try Self.allows("https://registry.example.com:443/", "https://registry.example.com/schemas")) + #expect(try Self.allows("http://localhost:8081", "http://localhost:8081/subjects")) + } + + @Test("A redirect to another host, port or scheme is refused, a downgrade to http included") + func otherOrigin() throws { + #expect(try !Self.allows("https://registry.example.com", "https://evil.example.com")) + #expect(try !Self.allows("https://registry.example.com", "http://registry.example.com")) + #expect(try !Self.allows("http://localhost:8081", "http://localhost:8082")) + #expect(try !Self.allows("https://registry.example.com", "https://registry.example.com:8443")) + #expect(try !Self.allows("https://registry.example.com", "file:///etc/passwd")) + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift index ed0043f61d..2d668d58c8 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -7,13 +7,12 @@ import Foundation @testable import TablePro import Testing -// Fixtures written by `protoc --encode` (libprotoc 36.2); every expected text is `protoc --decode` output for them. +// libprotoc 36.2. Each fixture says how it was written; every expected text is `protoc --decode` output for it. struct ProtobufSchemaDecoderTests { private static let shopProto = """ syntax = "proto3"; package shop; - // Every scalar type, an enum, packed and unpacked repeats, a map, a oneof and nesting. message Item { int32 i32 = 1; int64 i64 = 2; @@ -71,6 +70,7 @@ struct ProtobufSchemaDecoderTests { + "636f6c6f727342080a0663686f69636522150a054f74686572120c0a01781801200128095201782a250a05436f6c6f7212070a03524544100012090a" + "05475245454e100112080a04424c55451002620670726f746f33" + // protoc --encode=shop.Item shop.proto private static let item = "08fbffffffffffffffff011080ccbbbcdeffffffff011880d0acf30e208080a0a89c94b6e6f901280d30ffef99c589043d7b00000041c801000000" + "0000004df8ffffff51f7ffffffffffffff5d0000c03f619a9999999999b93f680172075461622250726f7a030102ff8001028a01040102ac0292010161" + "920101629a010d0a026b3112070a016e12020102a201060a0464656570b0012a" @@ -217,4 +217,320 @@ struct ProtobufSchemaDecoderTests { #expect(throws: ProtobufSchemaError.self) { try ProtobufSchema.parse(Data("enum Only { A = 0; }".utf8)) } #expect(throws: ProtobufSchemaError.self) { try ProtobufSchema.parse(Data([0xFF, 0x00, 0x13])) } } + + private static let demoProto = """ + syntax = "proto3"; + package demo; + + message M { + map m = 1; + map im = 2; + oneof choice { + string text = 3; + int32 number = 4; + Inner pick = 5; + } + int32 count = 6; + optional int32 opt = 7; + string name = 8; + bytes data = 9; + E e = 10; + double ratio = 11; + Inner inner = 12; + } + + enum E { + E0 = 0; + E1 = 1; + } + + message Inner { + int32 x = 1; + int32 y = 2; + } + """ + + // protoc --descriptor_set_out=demo.desc demo.proto + private static let demoDescriptorSet = "0a90040a0a64656d6f2e70726f746f120464656d6f22b9030a014d121c0a016d18012003280b320e2e64656d6f2e4d2e4d456e74727952016d121f0a" + + "02696d18022003280b320f2e64656d6f2e4d2e496d456e7472795202696d12140a0474657874180320012809480052047465787412180a066e756d62" + + "6572180420012805480052066e756d62657212210a047069636b18052001280b320b2e64656d6f2e496e6e6572480052047069636b12140a05636f75" + + "6e741806200128055205636f756e7412150a036f7074180720012805480152036f707488010112120a046e616d6518082001280952046e616d651212" + + "0a046461746118092001280c52046461746112150a0165180a2001280e32072e64656d6f2e4552016512140a05726174696f180b2001280152057261" + + "74696f12210a05696e6e6572180c2001280b320b2e64656d6f2e496e6e65725205696e6e65721a340a064d456e74727912100a036b65791801200128" + + "0952036b657912140a0576616c7565180220012805520576616c75653a0238011a350a07496d456e74727912100a036b657918012001280552036b65" + + "7912140a0576616c7565180220012809520576616c75653a02380142080a0663686f69636542060a045f6f707422230a05496e6e6572120c0a017818" + + "0120012805520178120c0a01791802200128055201792a130a014512060a024530100012060a0245311001620670726f746f33" + + // protoc --encode=demo.M: m { key: "b" value: 1 } m { key: "a" value: 2 } m { key: "a" value: 3 } + // im { key: 5 value: "x" } im { key: -1 value: "y" } im { key: 2 value: "z" } + private static let maps = "0a050a016210010a050a016110020a050a0161100312050805120178120e08ffffffffffffffffff011201791205080212017a" + + private static let mapsText = """ + m { + key: "a" + value: 2 + } + m { + key: "a" + value: 3 + } + m { + key: "b" + value: 1 + } + im { + key: -1 + value: "y" + } + im { + key: 2 + value: "z" + } + im { + key: 5 + value: "x" + } + + """ + + // Hand-written: count, name, data, e and ratio at their defaults, then protoc --encode=demo.M of `opt: 0 inner {}`. + private static let defaults = "300042004a00500059000000000000000038006200" + + private static let editionsProto = """ + edition = "2023"; + package ed; + option features.field_presence = IMPLICIT; + + enum Open { + O0 = 0; + O1 = 1; + } + + enum Closed { + option features.enum_type = CLOSED; + C0 = 0; + C1 = 1; + } + + message M { + int32 a = 1; + int32 b = 2 [features.field_presence = EXPLICIT]; + int32 c = 3 [features.field_presence = LEGACY_REQUIRED]; + Sub sub = 4; + Open open = 5; + Closed closed = 6 [features.field_presence = EXPLICIT]; + string s = 7; + + message Sub { + int32 x = 1; + int32 y = 2 [features.field_presence = EXPLICIT]; + } + } + """ + + // protoc --descriptor_set_out=ed.desc ed.proto + private static let editionsDescriptorSet = "0ab6020a0865642e70726f746f1202656422d9010a014d120c0a016118012001280552016112130a01621802200128054205aa010208015201621213" + + "0a01631803200128054205aa01020803520163121b0a0373756218042001280b32092e65642e4d2e5375625203737562121c0a046f70656e18052001" + + "280e32082e65642e4f70656e52046f70656e12290a06636c6f73656418062001280e320a2e65642e436c6f7365644205aa010208015206636c6f7365" + + "64120c0a01731807200128095201731a280a03537562120c0a017818012001280552017812130a01791802200128054205aa010208015201792a160a" + + "044f70656e12060a024f30100012060a024f3110012a1e0a06436c6f73656412060a024330100012060a02433110011a043a02100242059203020802" + + "620865646974696f6e7370e807" + + // protoc --include_imports --descriptor_set_out=order.desc order.proto, where order.proto imports money.proto. + private static let orderDescriptorSet = "0a570a0b6d6f6e65792e70726f746f12056d6f6e657922390a054d6f6e657912140a05756e6974731801200128035205756e697473121a0a08637572" + + "72656e6379180220012809520863757272656e6379620670726f746f330acb010a0b6f726465722e70726f746f12056f726465721a0b6d6f6e65792e" + + "70726f746f2285010a054f72646572120e0a0269641801200128095202696412220a05746f74616c18022001280b320c2e6d6f6e65792e4d6f6e6579" + + "5205746f74616c122b0a0673746174757318032001280e32132e6f726465722e4f726465722e5374617475735206737461747573221b0a0653746174" + + "757312070a034e4557100012080a04444f4e45100122180a044c696e6512100a03736b751801200128095203736b75620670726f746f33" + + private static let budgetProto = """ + syntax = "proto2"; + package budget; + + message Box { + optional Part a = 1; + optional Part b = 2; + optional Part c = 3; + } + + message Part { + repeated int32 v = 1; + } + """ + + @Test("Map entries print sorted by key, and a repeated key keeps every entry in wire order") + func mapsSortByKey() throws { + let schema = try ProtobufSchema.parse(Data(Self.demoProto.utf8)) + #expect(try Self.text(Self.maps, schema: schema, message: "demo.M") == Self.mapsText) + // Hand-written: an empty entry, then one with a field 3 the entry does not declare. + #expect(try Self.text("0a000a050a017a1809", schema: schema, message: "demo.M") + == "m {\n key: \"\"\n value: 0\n}\nm {\n key: \"z\"\n value: 0\n}\n") + } + + @Test("The oneof member set last wins, and a message member keeps only what follows the last switch to it") + func oneofLastMemberWins() throws { + let schema = try ProtobufSchema.parse(Data(Self.demoProto.utf8)) + // protoc --encode=demo.M of `text: "hello"` and of `number: 7`, joined in both orders. + #expect(try Self.text("1a0568656c6c6f2007", schema: schema, message: "demo.M") == "number: 7\n") + #expect(try Self.text("20071a0568656c6c6f", schema: schema, message: "demo.M") == "text: \"hello\"\n") + // protoc --encode=demo.M of `pick { x: 1 }`, `number: 7` and `pick { y: 3 }`, joined. + #expect(try Self.text("2a02080120072a021003", schema: schema, message: "demo.M") == "pick {\n y: 3\n}\n") + } + + @Test("A proto3 field at its default is left out, after the last value wins; optional and message fields stay") + func implicitPresence() throws { + let schema = try ProtobufSchema.parse(Data(Self.demoProto.utf8)) + // protoc --encode=demo.M of `count: 5`, then a hand-written `count: 0`. + #expect(try Self.text("30053000", schema: schema, message: "demo.M").isEmpty) + #expect(try Self.text(Self.defaults, schema: schema, message: "demo.M") == "opt: 0\ninner {\n}\n") + // protoc --encode=demo.M of `ratio: -0` + #expect(try Self.text("590000000000000080", schema: schema, message: "demo.M") == "ratio: -0\n") + } + + @Test("A proto2 field prints whatever value the bytes carry") + func proto2KeepsDefaults() throws { + let proto = """ + syntax = "proto2"; + package legacy; + message Msg { + optional int32 n = 1; + optional string s = 2; + } + """ + let schema = try ProtobufSchema.parse(Data(proto.utf8)) + // protoc --encode=legacy.Msg of `n: 0 s: ""` + #expect(try Self.text("08001200", schema: schema, message: "legacy.Msg") == "n: 0\ns: \"\"\n") + } + + @Test("A number a closed enum does not declare goes to the unknown fields, a map entry with it whole") + func closedEnumUnknownNumbers() throws { + let proto = """ + syntax = "proto2"; + package legacy; + + enum Level { + LOW = 0; + HIGH = 1; + } + + message Msg { + optional int32 n = 1; + optional string s = 2; + optional Level level = 3; + repeated Level levels = 4 [packed = true]; + map named = 5; + } + """ + let schema = try ProtobufSchema.parse(Data(proto.utf8)) + // protoc --encode=legacy.Msg of `level: HIGH`, then hand-written: level 7, 99: 1, levels packed [1, 9, 0], + // named { 4: 8 } and named { 2: HIGH }. + let fixture = "1801180798060122030109002a04080410082a0408021001" + let expected = "level: HIGH\nlevels: HIGH\nlevels: LOW\nnamed {\n key: 2\n value: HIGH\n}\n3: 7\n99: 1\n4: 9\n" + + "5 {\n 1: 4\n 2: 8\n}\n" + #expect(try Self.text(fixture, schema: schema, message: "legacy.Msg") == expected) + } + + @Test("Editions presence follows features.field_presence on the file and the field") + func editionsPresence() throws { + let schema = try ProtobufSchema.parse(Data(Self.editionsProto.utf8)) + let descriptorSet = try ProtobufSchema.parse(BinaryFixture.data(Self.editionsDescriptorSet)) + #expect(descriptorSet == schema) + // Hand-written: every field of ed.M at zero, sub with x and y at zero. + let zeros = "080010001800220408001000280030003a00" + let expected = "b: 0\nc: 0\nsub {\n y: 0\n}\nclosed: C0\n" + #expect(try Self.text(zeros, schema: schema, message: "ed.M") == expected) + #expect(try Self.text(zeros, schema: descriptorSet, message: "ed.M") == expected) + // Hand-written: a 1, closed 7, open 7. + #expect(try Self.text("080130072807", schema: schema, message: "ed.M") == "a: 1\nopen: 7\n6: 7\n") + } + + @Test("A descriptor set reads the same as the .proto protoc built it from") + func descriptorSetMatchesProto() throws { + let schema = try ProtobufSchema.parse(Data(Self.demoProto.utf8)) + let descriptorSet = try ProtobufSchema.parse(BinaryFixture.data(Self.demoDescriptorSet)) + #expect(descriptorSet == schema) + #expect(try Self.text(Self.maps, schema: descriptorSet, message: "demo.M") == Self.mapsText) + #expect(try Self.text(Self.defaults, schema: descriptorSet, message: "demo.M") == "opt: 0\ninner {\n}\n") + } + + @Test("A descriptor set that does not read says it is not a schema") + func brokenDescriptorSet() { + // order.desc's money.proto with its last three bytes cut, framed as a set again. + let truncated = "0a540a0b6d6f6e65792e70726f746f12056d6f6e657922390a054d6f6e657912140a05756e6974731801200128035205756e697473121a0a0863" + + "757272656e6379180220012809520863757272656e6379620670726f" + #expect(throws: ProtobufSchemaError.notASchema) { try ProtobufSchema.parse(BinaryFixture.data(truncated)) } + // Hand-written: a file whose message names more bytes than it holds. + #expect(throws: ProtobufSchemaError.notASchema) { try ProtobufSchema.parse(BinaryFixture.data("0a0422020a05")) } + } + + @Test("A set built with --include_imports reads its last file as the one chosen") + func includeImports() throws { + let schema = try ProtobufSchema.parse(BinaryFixture.data(Self.orderDescriptorSet)) + #expect(schema.rootMessages == ["order.Order", "order.Line"]) + #expect(schema.messageNames == ["order.Order", "order.Line", "money.Money"]) + // protoc --encode=order.Order of `id: "A1" total { units: 12 currency: "EUR" } status: DONE` + #expect(try Self.text("0a0241311207080c12034555521801", schema: schema, message: nil) + == "id: \"A1\"\ntotal {\n units: 12\n currency: \"EUR\"\n}\nstatus: DONE\n") + } + + @Test("A nested enum in a descriptor set is read once") + func nestedEnumOnce() throws { + let files = try ProtobufDescriptorSetParser.parse(BinaryFixture.data(Self.orderDescriptorSet)) + let names = files.flatMap { $0.allEnums.map { $0.fullName } } + #expect(names == ["order.Order.Status"]) + } + + @Test("A field whose type is a message named optional is not read as a label") + func optionalAsTypeName() throws { + let proto = """ + syntax = "proto2"; + package opt; + + message optional { + optional int32 v = 1; + } + + message M { + optional foo = 1; + repeated optional bar = 2; + } + """ + let schema = try ProtobufSchema.parse(Data(proto.utf8)) + // protoc --encode=opt.M of `foo { v: 1 } bar { v: 2 } bar { v: 3 }`, with `foo` declared `optional optional foo = 1;` + #expect(try Self.text("0a0208011202080212020803", schema: schema, message: "opt.M") + == "foo {\n v: 1\n}\nbar {\n v: 2\n}\nbar {\n v: 3\n}\n") + } + + @Test("A cell spends its budget inside a field the schema does not name") + func unknownFieldBudget() throws { + let schema = try ProtobufSchema.parse(Data(Self.budgetProto.utf8)) + // Hand-written: field 9 holding fifty `1: 1`. + let fixture = "4a64" + String(repeating: "0801", count: 50) + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "budget.Box", maxValues: 5) + #expect(ProtobufSchemaTextWriter.text(message.fields, multiLine: false) == "9 { 1: 1 1: 1 1: 1 1: 1 … }") + } + + @Test("Messages read in field order whatever their wire order, and the budget runs out where it runs out") + func messagesInFieldOrder() throws { + let schema = try ProtobufSchema.parse(Data(Self.budgetProto.utf8)) + // protoc --encode=budget.Box of `c { v: 7 v: 8 v: 9 }`, `b { v: 4 v: 5 v: 6 }` and `a { v: 1 v: 2 v: 3 }`, joined. + let fixture = "1a0608070808080912060804080508060a06080108020803" + #expect(try Self.text(fixture, schema: schema, message: "budget.Box") + == "a {\n v: 1\n v: 2\n v: 3\n}\nb {\n v: 4\n v: 5\n v: 6\n}\nc {\n v: 7\n v: 8\n v: 9\n}\n") + for _ in 0..<5 { + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "budget.Box", maxValues: 7) + #expect(ProtobufSchemaTextWriter.text(message.fields, multiLine: false) == "a { v: 1 v: 2 v: 3 } b { v: 4 … } c { … }") + } + } + + @Test("A field the schema does not name counts once toward the tree's node limit") + func unknownFieldsCountOnce() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + // Hand-written: 3,000 copies of `2: 1`, which shop.Other does not declare. + let fixture = Data((0..<3_000).flatMap { _ in [UInt8(0x10), 0x01] }) + let message = try ProtobufSchemaDecoder.decode(fixture, schema: schema, message: "shop.Other") + let root = ProtobufSchemaTreeBuilder.build(from: message, formats: BinaryFixture.english) + #expect(root.children.count == 3_000) + #expect(root.children.last?.kind == .varint) + } } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufWireDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufWireDecoderTests.swift index 47c95c8fb5..5e9f5094da 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufWireDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufWireDecoderTests.swift @@ -8,9 +8,10 @@ import Testing @testable import TablePro -/// Fixtures written by `protoc --encode` (libprotoc 36.2). Each expectation is what -/// `protoc --decode_raw` printed for the same bytes, except that UTF-8 text keeps its characters. +// Each expectation is what `protoc --decode_raw` (libprotoc 36.2) printed for the same bytes, except that UTF-8 text +// keeps its characters. struct ProtobufWireDecoderTests { + // protoc --encode of a message setting every wire type, a nested and a repeated message, and field 1000. private static let order = "089601120774657374696e671a0908011205636166c3a925070000002909000000000000003206038e029ea7053a" + "030001ff400348ffffffffffffffffff01516e861bf0f92109405d0000c03f6202080262050803120178c03eb5b8f0fe2d" @@ -59,6 +60,7 @@ struct ProtobufWireDecoderTests { 5: "end" """ + // protoc --encode of a proto2 message with a group between two fields. #expect(try Self.text("0801131a08687474703a2f2f782002142a03656e64") == expected) let message = try ProtobufWireDecoder.decode(BinaryFixture.data("0801131a08687474703a2f2f782002142a03656e64")) #expect(message.fields[1].value == .group([ @@ -104,6 +106,23 @@ struct ProtobufWireDecoderTests { #expect(ProtobufTextWriter.text(message.fields, multiLine: false) == #"1: 150 2: "testing" …"#) } + @Test("A length-delimited value spends the caller's budget and leaves the rest of it") + func lengthDelimitedBudget() { + // Hand-written: `1: 1`, `1: 2`, `1: 3`. + let window = BinaryValueReader(BinaryFixture.data("080108020803")) + var budget = 2 + let value = ProtobufWireDecoder.lengthDelimited(window, depth: 0, budget: &budget) + #expect(value == .message([ + ProtobufField(number: 1, value: .varint(1)), + ProtobufField(number: 1, value: .varint(2)), + ProtobufField(number: 0, value: .truncated), + ])) + #expect(budget == 0) + var roomy = 10 + _ = ProtobufWireDecoder.lengthDelimited(window, depth: 0, budget: &roomy) + #expect(roomy == 7) + } + @Test("The tree keys fields by number and badges their wire type") func tree() throws { let message = try ProtobufWireDecoder.decode(BinaryFixture.data(Self.order)) diff --git a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift index 8e02bafb35..5758714714 100644 --- a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift +++ b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift @@ -5,6 +5,7 @@ import Foundation import os +import TableProTestSupport @testable import TablePro import Testing @@ -127,27 +128,74 @@ struct SchemaRegistryTests { } } - @Test("The cache asks once for many cells, tells the grid when the schema lands, and keeps a failure only briefly") - func cache() async throws { + private final class Resolver: Sendable { + let calls = OSAllocatedUnfairLock(initialState: 0) + let current: OSAllocatedUnfairLock + + init(_ endpoint: SchemaRegistryEndpoint?) { + current = OSAllocatedUnfairLock(initialState: endpoint) + } + + var resolve: SchemaRegistryCache.EndpointResolver { + { [self] _ in + calls.withLock { $0 += 1 } + return current.withLock { $0 } + } + } + } + + private static func changes(for connection: UUID) -> (AsyncStream, NSObjectProtocol) { + let (stream, continuation) = AsyncStream.makeStream() + let observer = NotificationCenter.default.addObserver(forName: SchemaRegistryCache.didChange, object: nil, queue: nil) { + if $0.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID == connection { continuation.yield() } + } + return (stream, observer) + } + + @Test("Reading the cache never fetches; a prefetch asks once per schema and announces the batch") + @MainActor + func prefetch() async throws { let transport = RecordedTransport() let connection = UUID() - let cache = SchemaRegistryCache(endpoint: { _ in Self.endpoint }, transport: transport.transport) - let (landed, continuation) = AsyncStream.makeStream() - let observer = NotificationCenter.default.addObserver(forName: SchemaRegistryCache.didLoadSchema, object: cache, queue: nil) { - if let id = $0.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID { continuation.yield(id) } - } + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: transport.transport) + let (changes, observer) = Self.changes(for: connection) defer { NotificationCenter.default.removeObserver(observer) } #expect(cache.cachedSchema(connectionID: connection, schemaID: 1) == nil) - #expect(cache.cachedSchema(connectionID: connection, schemaID: 1) == nil) - var iterator = landed.makeAsyncIterator() - #expect(await iterator.next() == connection) + #expect(transport.requests.withLock { $0.isEmpty }) + + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: false) + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: false) + var iterator = changes.makeAsyncIterator() + _ = await iterator.next() guard case .success(.avro)? = cache.cachedSchema(connectionID: connection, schemaID: 1) else { Issue.record("expected the fetched Avro schema") return } #expect(transport.requests.withLock { $0.count } == 1) + } + + @Test("A loaded batch skips the endpoint lookup, which reads the Keychain") + @MainActor + func loadedBatchSkipsResolve() async throws { + let resolver = Resolver(Self.endpoint) + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: resolver.resolve, transport: RecordedTransport().transport) + _ = await cache.schema(connectionID: connection, schemaID: 1) + let calls = resolver.calls.withLock { $0 } + + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: false) + #expect(resolver.calls.withLock { $0 } == calls) + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true) + #expect(resolver.calls.withLock { $0 } == calls + 1) + } + @Test("A failure is kept for its lifetime, and a full reload asks again at once") + @MainActor + func failureRetry() async throws { + let transport = RecordedTransport() + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: transport.transport) guard case .failure = await cache.schema(connectionID: connection, schemaID: 99_999) else { Issue.record("expected a failure") return @@ -156,16 +204,72 @@ struct SchemaRegistryTests { Issue.record("expected the failure to be kept") return } - #expect(transport.requests.withLock { $0.count } == 2) + cache.prefetch(connectionID: connection, schemaIDs: [99_999], retryingFailures: false) + #expect(transport.requests.withLock { $0.count } == 1) + + cache.prefetch(connectionID: connection, schemaIDs: [99_999], retryingFailures: true) + try await waitUntil { transport.requests.withLock { $0.count } == 2 } + } + + @Test("A failure past its lifetime is fetched again by the next prefetch") + @MainActor + func failureExpires() async throws { + let transport = RecordedTransport() + let connection = UUID() + let cache = SchemaRegistryCache( + failureLifetime: .zero, resolveEndpoint: Resolver(Self.endpoint).resolve, transport: transport.transport + ) + _ = await cache.schema(connectionID: connection, schemaID: 99_999) + cache.prefetch(connectionID: connection, schemaIDs: [99_999], retryingFailures: false) + try await waitUntil { transport.requests.withLock { $0.count } == 2 } + } + + @Test("A connection whose registry changed reads schemas from the new one, and the change is announced") + @MainActor + func endpointChange() async throws { + let transport = RecordedTransport() + let resolver = Resolver(Self.endpoint) + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: resolver.resolve, transport: transport.transport) + guard case .success = await cache.schema(connectionID: connection, schemaID: 1) else { + Issue.record("expected the schema from the first registry") + return + } + let (changes, observer) = Self.changes(for: connection) + defer { NotificationCenter.default.removeObserver(observer) } + + let other = SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: "https://other.example.com"]) + resolver.current.withLock { $0 = other } + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true) + var iterator = changes.makeAsyncIterator() + _ = await iterator.next() + + try await waitUntil { transport.requests.withLock { $0.count } == 2 } + let host = transport.requests.withLock { $0.last?.url?.host } + #expect(host == "other.example.com") } @Test("A connection with no registry URL says so instead of fetching") + @MainActor func noEndpoint() async { - let cache = SchemaRegistryCache(endpoint: { _ in nil }, transport: { _ in throw URLError(.badURL) }) - guard case .failure(let error) = await cache.schema(connectionID: UUID(), schemaID: 1) else { + let cache = SchemaRegistryCache(resolveEndpoint: { _ in nil }, transport: { _ in throw URLError(.badURL) }) + let connection = UUID() + guard case .failure(let error) = await cache.schema(connectionID: connection, schemaID: 1) else { Issue.record("expected a failure") return } #expect(error.reason.contains("Schema Registry URL")) + guard case .failure? = cache.cachedSchema(connectionID: connection, schemaID: 1) else { + Issue.record("expected the grid to read the same failure") + return + } + } + + @Test("The frame's schema ID reads from any byte sequence, and anything shorter or unframed has none") + func framePrefix() { + #expect(SchemaRegistryFrame.schemaID(prefix: Data([0, 0, 0, 1, 2, 9])) == 258) + #expect(SchemaRegistryFrame.schemaID(prefix: Data([0, 0, 0, 1])) == nil) + #expect(SchemaRegistryFrame.schemaID(prefix: Data([1, 0, 0, 1, 2])) == nil) + #expect(SchemaRegistryFrame.schemaID(prefix: "\u{0}\u{0}\u{0}\u{0}\u{7}{}".utf8) == 7) } } diff --git a/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift index 65749dc6e9..0e59862593 100644 --- a/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift +++ b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift @@ -29,6 +29,23 @@ struct AvroSchemaStoreTests { } } + @Test("A schema file that cannot be read is not read again on every cell, until it is added back") + func unreadableFileIsRememberedUntilAdded() throws { + try withDirectory { directory in + let id = AvroSchemaStore.schemaID(of: Self.schema) + let file = directory.appendingPathComponent(id).appendingPathExtension("avsc") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try Data("not json".utf8).write(to: file) + let store = AvroSchemaStore(directory: directory) + #expect(store.schema(id: id) == nil) + + try Data(Self.schema.utf8).write(to: file) + #expect(store.schema(id: id) == nil) + #expect(try store.add(Self.schema) == id) + #expect(store.schema(id: id)?.displayName == "Reading") + } + } + @Test("Text that is not a schema is refused and leaves nothing behind") func invalidTextIsRefused() throws { try withDirectory { directory in diff --git a/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift b/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift new file mode 100644 index 0000000000..e59abb34ad --- /dev/null +++ b/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift @@ -0,0 +1,154 @@ +// +// DataGridSchemaRegistryTests.swift +// TableProTests +// + +import Foundation +import os +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct DataGridSchemaRegistryTests { + private final class Requests: Sendable { + let ids = OSAllocatedUnfairLock<[String]>(initialState: []) + + var transport: SchemaRegistryClient.Transport { + { [self] request in + ids.withLock { $0.append(request.url?.lastPathComponent ?? "") } + let response = HTTPURLResponse(url: request.url!, statusCode: 404, httpVersion: nil, headerFields: nil)! + return (Data(#"{"message":"not here"}"#.utf8), response) + } + } + } + + private struct Grid { + let owner: GridTestOwner + let model: DataGridModel + let presenter: DataGridCellPresenter + let requests: Requests + } + + private static let connection = UUID() + + private static func grid(_ rows: TableRows) -> Grid { + let requests = Requests() + let endpoint = SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: "http://localhost:8081"]) + let cache = SchemaRegistryCache(resolveEndpoint: { _ in endpoint }, transport: requests.transport) + let owner = GridTestOwner(rows) + let model = DataGridModel(schemaRegistry: cache) + model.dataSource = owner + model.actions = owner + owner.model = model + model.reloadRows() + return Grid(owner: owner, model: model, presenter: DataGridCellPresenter(model: model), requests: requests) + } + + private static func framed(_ id: UInt8, _ payload: [UInt8] = [0x02]) -> PluginCellValue { + .bytes(Data([0, 0, 0, 0, id] + payload)) + } + + private static func cachedText(_ grid: Grid, row: Int, column: Int) -> String? { + guard let id = grid.model.row(at: row)?.id, let box = grid.model.display.cache.box(forID: id), + column < box.values.count else { return nil } + return box.values[column] + } + + private static func fill(_ grid: Grid, rows: Int, columns: Int) { + for row in 0.. Date: Sun, 1 Nov 2026 21:49:21 +0700 Subject: [PATCH 14/22] fix(display-format): refuse invalid UTF-8 in verified Protobuf strings, read a late package and a BOM --- .../BinaryValues/BinaryValueReader.swift | 3 ++ .../BinaryValues/ProtoFileParser.swift | 25 ++++++++--- .../BinaryValues/ProtobufSchema.swift | 15 +++++-- .../BinaryValues/ProtobufSchemaDecoder.swift | 36 ++++++++++----- .../TablePro/Resources/Localizable.xcstrings | 3 ++ .../ProtobufSchemaDecoderTests.swift | 44 +++++++++++++++++++ 6 files changed, 107 insertions(+), 19 deletions(-) diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift index 2961c7e0c5..8337d63b40 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift @@ -21,6 +21,7 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { case invalidLength case otherSchema(fingerprint: UInt64) case notFramed + case invalidUTF8 } let kind: Kind @@ -62,6 +63,8 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { return String(format: String(localized: "The length at byte %lld is negative."), offset) case .notFramed: return String(localized: "The value does not start with the Schema Registry prefix: a zero byte and a schema ID.") + case .invalidUTF8: + return String(format: String(localized: "The text at byte %lld is not valid UTF-8."), offset) case .otherSchema(let fingerprint): return String( format: String(localized: "The value was written with the schema whose fingerprint is %@, not this column's."), diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift index f4a8cf0f1d..db01eea8bb 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift @@ -26,7 +26,7 @@ internal enum ProtoFileParser { static func tokens(_ text: String) throws -> [Token] { let scalars = Array(text.unicodeScalars) var tokens: [Token] = [] - var index = 0 + var index = scalars.first == "\u{FEFF}" ? 1 : 0 while index < scalars.count { guard tokens.count < ProtoFileParser.maxTokens else { throw error(String(localized: "The file is too large.")) } let scalar = scalars[index] @@ -103,7 +103,7 @@ internal enum ProtoFileParser { mutating func file() throws -> ProtobufFileDefinition { var file = ProtobufFileDefinition() - var package = "" + let package = packageName() while let token = peek() { switch token { case .identifier("syntax"): @@ -113,9 +113,7 @@ internal enum ProtoFileParser { file.syntax = .editions try skipStatement() case .identifier("package"): - advance() - package = try identifier() - try expect(";") + try skipStatement() case .identifier("import"): try skipStatement() case .identifier("option"): @@ -140,6 +138,21 @@ internal enum ProtoFileParser { return file } + // It qualifies every definition in the file, those written before it too. + private func packageName() -> String { + var depth = 0 + for index in tokens.indices { + switch tokens[index] { + case .symbol("{"): depth += 1 + case .symbol("}"): depth -= 1 + case .identifier("package") where depth == 0 && index + 1 < tokens.count: + if case .identifier(let name) = tokens[index + 1] { return name } + default: break + } + } + return "" + } + private mutating func message(scope: String, depth: Int) throws -> String { let name = try identifier() let fullName = scope.isEmpty ? name : "\(scope).\(name)" @@ -312,6 +325,8 @@ internal enum ProtoFileParser { overrides.implicitPresence = false case ("features.enum_type", "CLOSED"): overrides.closedEnum = true case ("features.enum_type", "OPEN"): overrides.closedEnum = false + case ("features.utf8_validation", "VERIFY"): overrides.validatesUTF8 = true + case ("features.utf8_validation", "NONE"): overrides.validatesUTF8 = false default: break } } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift index e8bd68a41a..766f94b82f 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift @@ -28,6 +28,8 @@ internal struct ProtobufFieldDescriptor: Equatable, Sendable { let oneofIndex: Int? // Without presence, a field holding its default is not printed. let hasPresence: Bool + // protoc refuses the whole value when such a string is not UTF-8. + let validatesUTF8: Bool } internal struct ProtobufMessageDescriptor: Equatable, Sendable { @@ -179,6 +181,7 @@ internal enum ProtobufSyntax: Sendable { internal struct ProtobufFeatureOverrides: Equatable, Sendable { var implicitPresence: Bool? var closedEnum: Bool? + var validatesUTF8: Bool? } internal struct ProtobufFileDefinition: Sendable { @@ -228,11 +231,13 @@ private struct ProtobufFeatures { let syntax: ProtobufSyntax var implicitPresence: Bool var closedEnum: Bool + var validatesUTF8: Bool init(syntax: ProtobufSyntax) { self.syntax = syntax implicitPresence = syntax == .proto3 closedEnum = syntax == .proto2 + validatesUTF8 = syntax != .proto2 } func applying(_ overrides: ProtobufFeatureOverrides?) -> ProtobufFeatures { @@ -240,6 +245,7 @@ private struct ProtobufFeatures { var result = self if let implicitPresence = overrides.implicitPresence { result.implicitPresence = implicitPresence } if let closedEnum = overrides.closedEnum { result.closedEnum = closedEnum } + if let validatesUTF8 = overrides.validatesUTF8 { result.validatesUTF8 = validatesUTF8 } return result } } @@ -294,7 +300,8 @@ private struct ProtobufTypeResolver { case .named(let reference): type = resolve(reference, scope: scope) } - let isImplicit = features.applying(field.features).implicitPresence && !field.isOptional + let fieldFeatures = features.applying(field.features) + let isImplicit = fieldFeatures.implicitPresence && !field.isOptional let isMessage: Bool switch type { case .message, .group: isMessage = true @@ -306,7 +313,8 @@ private struct ProtobufTypeResolver { type: type, isRepeated: field.isRepeated, oneofIndex: field.oneofIndex, - hasPresence: !field.isRepeated && (field.oneofIndex != nil || isMessage || !isImplicit) + hasPresence: !field.isRepeated && (field.oneofIndex != nil || isMessage || !isImplicit), + validatesUTF8: fieldFeatures.validatesUTF8 ) } @@ -405,7 +413,7 @@ internal enum ProtobufDescriptorSetParser { return nil } - // options.features: FeatureSet.field_presence = 1 (2 is IMPLICIT), enum_type = 2 (2 is CLOSED). + // FeatureSet: field_presence = 1 (2 IMPLICIT), enum_type = 2 (2 CLOSED), utf8_validation = 4 (3 NONE). private static func features(_ entries: [(Int, Value)], options: Int, features: Int) throws -> ProtobufFeatureOverrides { guard let optionBytes = bytes(entries, options), let featureBytes = bytes(try fields(optionBytes), features) else { return ProtobufFeatureOverrides() } @@ -413,6 +421,7 @@ internal enum ProtobufDescriptorSetParser { var overrides = ProtobufFeatureOverrides() if let presence = varint(set, 1), (1...3).contains(presence) { overrides.implicitPresence = presence == 2 } if let enumType = varint(set, 2), (1...2).contains(enumType) { overrides.closedEnum = enumType == 2 } + if let validation = varint(set, 4), (2...3).contains(validation) { overrides.validatesUTF8 = validation == 2 } return overrides } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift index d448b2d138..3c2f64b98f 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -137,6 +137,7 @@ internal enum ProtobufSchemaDecoder { let field = message.field(number: occurrence.number) switch field.map({ match(occurrence.payload, field: $0) }) ?? .mismatched { case .accepted: + if let field { try verifyUTF8(occurrence.payload, field: field) } accepted[occurrence.number, default: []].append(occurrence) case .undeclaredEnumValue where asMapEntry: throw MapEntryRejected() @@ -256,11 +257,11 @@ internal enum ProtobufSchemaDecoder { return } if !field.isRepeated { - appendMessage(field, nested, found, to: &level, depth: depth) + try appendMessage(field, nested, found, to: &level, depth: depth) } else if nested.isMapEntry { - appendMapEntries(field, nested, found, to: &level, depth: depth) + try appendMapEntries(field, nested, found, to: &level, depth: depth) } else { - for occurrence in found { appendMessage(field, nested, [occurrence], to: &level, depth: depth) } + for occurrence in found { try appendMessage(field, nested, [occurrence], to: &level, depth: depth) } } case .scalar, .enumeration: if field.isRepeated { @@ -268,7 +269,7 @@ internal enum ProtobufSchemaDecoder { } else if let last = found.last { // Last wins first: `count: 5` then `count: 0` leaves an implicit field at its default. if !asMapEntry, !field.hasPresence, holdsDefault(last.payload, type: field.type) { return } - level.fields.append(typed(field, value(last.payload, type: field.type))) + level.fields.append(typed(field, value(last.payload, field: field))) } case .unresolved: level.unknown += found @@ -282,7 +283,7 @@ internal enum ProtobufSchemaDecoder { _ found: [Occurrence], to level: inout Level, depth: Int - ) { + ) throws { let windows = found.compactMap { occurrence -> BinaryValueReader? in switch occurrence.payload { case .lengthDelimited(let window), .group(let window): return window @@ -297,6 +298,8 @@ internal enum ProtobufSchemaDecoder { } else { level.fields.append(typed(field, .message(members))) } + } catch let error as BinaryValueDecodeError where error.kind == .invalidUTF8 { + throw error } catch { budget = start level.unknown += found @@ -310,7 +313,7 @@ internal enum ProtobufSchemaDecoder { _ found: [Occurrence], to level: inout Level, depth: Int - ) { + ) throws { var entries: [(key: MapKey, field: ProtobufTypedField)] = [] for occurrence in found { guard case .lengthDelimited(let window) = occurrence.payload else { continue } @@ -318,6 +321,8 @@ internal enum ProtobufSchemaDecoder { do { let members = try self.members(of: [window], message: entry, depth: depth + 1, asMapEntry: true) entries.append((mapKey(members, entry: entry), typed(field, .message(members)))) + } catch let error as BinaryValueDecodeError where error.kind == .invalidUTF8 { + throw error } catch { budget = start level.unknown.append(occurrence) @@ -332,7 +337,7 @@ internal enum ProtobufSchemaDecoder { private mutating func appendRepeated(_ field: ProtobufFieldDescriptor, _ occurrence: Occurrence, to level: inout Level) { guard case .lengthDelimited(let window) = occurrence.payload, field.type != .scalar(.string), field.type != .scalar(.bytes) else { - level.fields.append(typed(field, value(occurrence.payload, type: field.type))) + level.fields.append(typed(field, value(occurrence.payload, field: field))) return } var reader = window @@ -431,12 +436,21 @@ internal enum ProtobufSchemaDecoder { } } - // A string that is not UTF-8 prints escaped, as protoc prints a proto2 string. - private func value(_ payload: Occurrence.Payload, type: ProtobufFieldType) -> ProtobufTypedValue { - guard case .lengthDelimited(let window) = payload else { return .scalar(scalarText(payload, type: type)) } + // Every occurrence, as protoc checks each one it parses, the ones a later value replaces included. + private func verifyUTF8(_ payload: Occurrence.Payload, field: ProtobufFieldDescriptor) throws { + guard field.validatesUTF8, field.type == .scalar(.string), case .lengthDelimited(let window) = payload else { return } + var reader = window + let data = (try? reader.readBytes(reader.remaining)) ?? Data() + guard String(data: data, encoding: .utf8) == nil else { return } + throw BinaryValueDecodeError(kind: .invalidUTF8, offset: window.offset) + } + + // An unverified string that is not UTF-8 prints escaped, as protoc prints a proto2 string. + private func value(_ payload: Occurrence.Payload, field: ProtobufFieldDescriptor) -> ProtobufTypedValue { + guard case .lengthDelimited(let window) = payload else { return .scalar(scalarText(payload, type: field.type)) } var reader = window let data = (try? reader.readBytes(reader.remaining)) ?? Data() - if type == .scalar(.string), let text = String(data: data, encoding: .utf8) { return .string(text) } + if field.type == .scalar(.string), let text = String(data: data, encoding: .utf8) { return .string(text) } return .bytes(data) } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 5f14165857..9e9558d9c5 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -193043,6 +193043,9 @@ } } } + }, + "The text at byte %lld is not valid UTF-8." : { + }, "The text could not be parsed as JSON." : { "localizations" : { diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift index 2d668d58c8..b909d1536c 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -501,6 +501,50 @@ struct ProtobufSchemaDecoderTests { == "foo {\n v: 1\n}\nbar {\n v: 2\n}\nbar {\n v: 3\n}\n") } + @Test("A verified string that is not UTF-8 refuses the whole value, as protoc does, and an unverified one prints escaped") + func stringValidation() throws { + let proto3 = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package u; + message Inner { string s = 1; } + message M { string s = 1; Inner inner = 2; repeated string r = 3; map m = 4; } + """.utf8)) + // protoc --decode=u.M fails "contains invalid UTF-8 data" on each of these. + for fixture in ["0a01ff", "12030a01ff", "1a01ff", "22050a01ff1001", "0a01ff0a0141"] { + #expect(throws: BinaryValueDecodeError.self, "\(fixture)") { + try Self.text(fixture, schema: proto3, message: "u.M") + } + } + let proto2 = try ProtobufSchema.parse(Data("syntax = \"proto2\"; message M { optional string s = 1; }".utf8)) + #expect(try Self.text("0a01ff", schema: proto2, message: "M") == "s: \"\\377\"\n") + let editions = try ProtobufSchema.parse(Data(""" + edition = "2023"; + package ue; + message M { string s = 1 [features.utf8_validation = NONE]; string v = 2; } + """.utf8)) + #expect(try Self.text("0a01ff", schema: editions, message: "ue.M") == "s: \"\\377\"\n") + #expect(throws: BinaryValueDecodeError.self) { try Self.text("1201ff", schema: editions, message: "ue.M") } + } + + @Test("The package qualifies the definitions written before it too") + func packageAfterMessage() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto2"; + message A { optional B b = 1; } + package x; + message B { optional int32 v = 1; } + """.utf8)) + #expect(schema.messageNames == ["x.A", "x.B"]) + // protoc --decode=x.A of 0a020807 + #expect(try Self.text("0a020807", schema: schema, message: "x.A") == "b {\n v: 7\n}\n") + } + + @Test("A .proto file saved with a UTF-8 byte order mark reads as protoc reads it") + func byteOrderMark() throws { + let schema = try ProtobufSchema.parse(Data([0xEF, 0xBB, 0xBF]) + Data("syntax = \"proto3\"; message Q { int32 a = 1; }".utf8)) + #expect(try Self.text("0805", schema: schema, message: "Q") == "a: 5\n") + } + @Test("A cell spends its budget inside a field the schema does not name") func unknownFieldBudget() throws { let schema = try ProtobufSchema.parse(Data(Self.budgetProto.utf8)) From 687e4ebfe33ccd1e1ded6182de73bc27287e5ec8 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 22:05:28 +0700 Subject: [PATCH 15/22] fix(display-format): fail malformed declared Protobuf fields and bound registry fetches --- .../BinaryValues/ProtobufSchemaDecoder.swift | 32 +++------ .../SchemaRegistry/SchemaRegistryCache.swift | 66 +++++++++++++++++-- .../ProtobufSchemaDecoderTests.swift | 19 ++++++ .../SchemaRegistry/SchemaRegistryTests.swift | 34 ++++++++++ 4 files changed, 125 insertions(+), 26 deletions(-) diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift index 3c2f64b98f..065dbd7479 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -265,7 +265,7 @@ internal enum ProtobufSchemaDecoder { } case .scalar, .enumeration: if field.isRepeated { - for occurrence in found { appendRepeated(field, occurrence, to: &level) } + for occurrence in found { try appendRepeated(field, occurrence, to: &level) } } else if let last = found.last { // Last wins first: `count: 5` then `count: 0` leaves an implicit field at its default. if !asMapEntry, !field.hasPresence, holdsDefault(last.payload, type: field.type) { return } @@ -290,19 +290,12 @@ internal enum ProtobufSchemaDecoder { case .varint, .fixed64, .fixed32: return nil } } - let start = budget - do { - let members = try self.members(of: windows, message: nested, depth: depth + 1, asMapEntry: false) - if case .group = field.type { - level.fields.append(ProtobufTypedField(name: nested.shortName, typeName: "group", value: .message(members))) - } else { - level.fields.append(typed(field, .message(members))) - } - } catch let error as BinaryValueDecodeError where error.kind == .invalidUTF8 { - throw error - } catch { - budget = start - level.unknown += found + // A declared message that does not parse fails the value, as it fails protoc's parse. + let members = try self.members(of: windows, message: nested, depth: depth + 1, asMapEntry: false) + if case .group = field.type { + level.fields.append(ProtobufTypedField(name: nested.shortName, typeName: "group", value: .message(members))) + } else { + level.fields.append(typed(field, .message(members))) } } @@ -321,9 +314,7 @@ internal enum ProtobufSchemaDecoder { do { let members = try self.members(of: [window], message: entry, depth: depth + 1, asMapEntry: true) entries.append((mapKey(members, entry: entry), typed(field, .message(members)))) - } catch let error as BinaryValueDecodeError where error.kind == .invalidUTF8 { - throw error - } catch { + } catch is MapEntryRejected { budget = start level.unknown.append(occurrence) } @@ -334,7 +325,7 @@ internal enum ProtobufSchemaDecoder { level.fields += sorted.map { $0.element.field } } - private mutating func appendRepeated(_ field: ProtobufFieldDescriptor, _ occurrence: Occurrence, to level: inout Level) { + private mutating func appendRepeated(_ field: ProtobufFieldDescriptor, _ occurrence: Occurrence, to level: inout Level) throws { guard case .lengthDelimited(let window) = occurrence.payload, field.type != .scalar(.string), field.type != .scalar(.bytes) else { level.fields.append(typed(field, value(occurrence.payload, field: field))) @@ -350,10 +341,7 @@ internal enum ProtobufSchemaDecoder { isCut = true break } - guard let item = try? Self.packedItem(&reader, type: field.type) else { - level.unknown.append(occurrence) - return - } + let item = try Self.packedItem(&reader, type: field.type) remaining -= 1 if case .varint(let raw) = item, case .enumeration(let name) = field.type, !isDeclared(raw, enumeration: name) { undeclared.append(Occurrence(number: field.number, payload: item, position: occurrence.position)) diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift index 737e40c80e..a5fa02dbf5 100644 --- a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift @@ -17,6 +17,11 @@ internal final class SchemaRegistryCache: Sendable { static let didChange = Notification.Name("com.TablePro.schemaRegistryCache.didChange") static let connectionIDKey = "connectionID" + // A page of values can name thousands of IDs that are only bytes starting with zero, so fetches are bounded. + static let maxConcurrentFetches = 4 + static let maxPendingFetches = 256 + static let maxFailures = 1_024 + private struct Key: Hashable { let endpoint: SchemaRegistryEndpoint let schemaID: UInt32 @@ -42,6 +47,7 @@ internal final class SchemaRegistryCache: Sendable { private let resolveEndpoint: EndpointResolver private let transport: SchemaRegistryClient.Transport private let failureLifetime: Duration + private let limiter = FetchLimiter(limit: SchemaRegistryCache.maxConcurrentFetches) init( failureLifetime: Duration = .seconds(30), @@ -88,11 +94,16 @@ internal final class SchemaRegistryCache: Sendable { let (changed, pending) = state.withLock { state -> (Bool, [Task]) in let changed = adopt(endpoint, for: connectionID, in: &state) guard let endpoint else { return (changed, []) } - let pending = schemaIDs.compactMap { schemaID -> Task? in + pruneFailures(in: &state) + var pending: [Task] = [] + for schemaID in schemaIDs.sorted() { let key = Key(endpoint: endpoint, schemaID: schemaID) - if let running = state.inFlight[key] { return running } - guard usable(state.entries[key], retryingFailures: retryingFailures) == nil else { return nil } - return start(key, in: &state) + if let running = state.inFlight[key] { + pending.append(running) + } else if usable(state.entries[key], retryingFailures: retryingFailures) == nil, + state.inFlight.count < Self.maxPendingFetches { + pending.append(start(key, in: &state)) + } } return (changed, pending) } @@ -157,7 +168,9 @@ internal final class SchemaRegistryCache: Sendable { private func start(_ key: Key, in state: inout State) -> Task { let client = SchemaRegistryClient(endpoint: key.endpoint, transport: transport) + let limiter = limiter let task = Task { + await limiter.acquire() let result: Lookup do { result = .success(try await client.schema(id: key.schemaID)) @@ -166,6 +179,7 @@ internal final class SchemaRegistryCache: Sendable { } catch { result = .failure(SchemaRegistryError(reason: error.localizedDescription)) } + await limiter.release() self.finish(key, with: result) return result } @@ -183,7 +197,51 @@ internal final class SchemaRegistryCache: Sendable { } } + // Garbage prefixes would otherwise leave one failure each for the session. + private func pruneFailures(in state: inout State) { + let now = ContinuousClock.now + var failures: [(key: Key, at: ContinuousClock.Instant)] = [] + for (key, entry) in state.entries { + guard case .failed(_, let at) = entry else { continue } + if now - at >= failureLifetime { + state.entries[key] = nil + } else { + failures.append((key, at)) + } + } + guard failures.count > Self.maxFailures else { return } + for failure in failures.sorted(by: { $0.at < $1.at }).prefix(failures.count - Self.maxFailures) { + state.entries[failure.key] = nil + } + } + private static func post(_ connectionID: UUID) { NotificationCenter.default.post(name: didChange, object: nil, userInfo: [connectionIDKey: connectionID]) } } + +private actor FetchLimiter { + private let limit: Int + private var running = 0 + private var waiting: [CheckedContinuation] = [] + + init(limit: Int) { + self.limit = limit + } + + func acquire() async { + guard running >= limit else { + running += 1 + return + } + await withCheckedContinuation { waiting.append($0) } + } + + func release() { + guard !waiting.isEmpty else { + running -= 1 + return + } + waiting.removeFirst().resume() + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift index b909d1536c..13a647e10b 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -526,6 +526,25 @@ struct ProtobufSchemaDecoderTests { #expect(throws: BinaryValueDecodeError.self) { try Self.text("1201ff", schema: editions, message: "ue.M") } } + @Test("A declared message or packed field whose bytes do not parse refuses the value, as protoc does") + func malformedDeclaredFields() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package n; + message Inner { int32 a = 1; } + message Inner2 { string s = 1; } + message M { Inner inner = 1; repeated int32 p = 2; Inner2 g = 3; } + """.utf8)) + // protoc --decode=n.M fails "Failed to parse input." on each of these. + for fixture in ["0a0108", "1203ffffff", "0a03ff0101"] { + #expect(throws: BinaryValueDecodeError.self, "\(fixture)") { + try Self.text(fixture, schema: schema, message: "n.M") + } + } + // protoc --decode=n.M of 1a020801: a wire type the nested field does not declare is only unknown. + #expect(try Self.text("1a020801", schema: schema, message: "n.M") == "g {\n 1: 1\n}\n") + } + @Test("The package qualifies the definitions written before it too") func packageAfterMessage() throws { let schema = try ProtobufSchema.parse(Data(""" diff --git a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift index 5758714714..17e68792e5 100644 --- a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift +++ b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift @@ -249,6 +249,40 @@ struct SchemaRegistryTests { #expect(host == "other.example.com") } + private final class HeldTransport: Sendable { + let gate = AsyncGate() + let calls = OSAllocatedUnfairLock(initialState: 0) + let active = OSAllocatedUnfairLock(initialState: (now: 0, most: 0)) + + var transport: SchemaRegistryClient.Transport { + { [self] request in + calls.withLock { $0 += 1 } + active.withLock { $0 = (now: $0.now + 1, most: max($0.most, $0.now + 1)) } + await gate.wait() + active.withLock { $0.now -= 1 } + let response = HTTPURLResponse(url: request.url!, statusCode: 404, httpVersion: nil, headerFields: nil)! + return (Data(#"{"message":"not here"}"#.utf8), response) + } + } + } + + @Test("A page naming thousands of IDs queues a bounded number of fetches and runs a few at a time") + @MainActor + func prefetchIsBounded() async throws { + let held = HeldTransport() + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: held.transport) + cache.prefetch(connectionID: connection, schemaIDs: Set(UInt32(1)...5_000), retryingFailures: false) + + await held.gate.arrived(SchemaRegistryCache.maxConcurrentFetches) + #expect(held.calls.withLock { $0 } == SchemaRegistryCache.maxConcurrentFetches) + await held.gate.open() + try await waitUntil { held.calls.withLock { $0 } == SchemaRegistryCache.maxPendingFetches } + try await waitUntil { held.active.withLock { $0.now } == 0 } + #expect(held.active.withLock { $0.most } == SchemaRegistryCache.maxConcurrentFetches) + #expect(held.calls.withLock { $0 } == SchemaRegistryCache.maxPendingFetches) + } + @Test("A connection with no registry URL says so instead of fetching") @MainActor func noEndpoint() async { From 4ed9ba5d5fc0e69335ef0afeb0e71f39a6b0d159 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 22:34:44 +0700 Subject: [PATCH 16/22] fix(display-format): parse replaced oneof members, read DELIMITED fields, resolve the registry before filtering --- .../Core/DataGrid/DataGridModel.swift | 33 +++++++------ .../BinaryValues/ProtoFileParser.swift | 2 + .../BinaryValues/ProtobufSchema.swift | 24 ++++++++-- .../BinaryValues/ProtobufSchemaDecoder.swift | 40 +++++++++++++--- .../ProtobufSchemaDecoderTests.swift | 46 +++++++++++++++++++ .../SchemaRegistry/SchemaRegistryTests.swift | 19 ++++++++ 6 files changed, 138 insertions(+), 26 deletions(-) diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index 4037eb492b..b5c78f56cf 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -96,6 +96,7 @@ final class DataGridModel: ObservableObject { func reloadRows() { let current = rows + prefetchSchemas(in: current, storageRows: nil, retryingFailures: true) store.valueFilter = store.valueFilter.remapped(toColumns: current.columns) let rebuilt = DataGridColumnSet(tableRows: current, layout: store.columnLayout) if rebuilt != columns { @@ -108,7 +109,6 @@ final class DataGridModel: ObservableObject { displayRevision &+= 1 mountedView?.apply(.reload) refreshFind() - prefetchSchemas(in: current, storageRows: nil, retryingFailures: true) } func apply(_ delta: Delta) { @@ -117,6 +117,18 @@ final class DataGridModel: ObservableObject { return } let current = rows + switch delta { + case .rowsInserted(let inserted): + prefetchSchemas(in: current, storageRows: inserted, retryingFailures: false) + case .columnsReplaced: + prefetchSchemas(in: current, storageRows: nil, retryingFailures: false) + case .cellChanged(let row, _): + prefetchSchemas(in: current, storageRows: IndexSet(integer: row), retryingFailures: false) + case .cellsChanged(let positions): + prefetchSchemas(in: current, storageRows: IndexSet(positions.map(\.row)), retryingFailures: false) + case .rowsRemoved, .fullReplace: + break + } if delta.changesRowSet || delta == .columnsReplaced { if delta == .columnsReplaced { columns = DataGridColumnSet(tableRows: current, layout: store.columnLayout) @@ -131,21 +143,8 @@ final class DataGridModel: ObservableObject { displayRevision &+= 1 mountedView?.apply(change) refreshFind() - if case .rowsInserted(let inserted) = delta { - prefetchSchemas(in: current, storageRows: inserted, retryingFailures: false) - } else if delta == .columnsReplaced { - prefetchSchemas(in: current, storageRows: nil, retryingFailures: false) - } return } - switch delta { - case .cellChanged(let row, _): - prefetchSchemas(in: current, storageRows: IndexSet(integer: row), retryingFailures: false) - case .cellsChanged(let positions): - prefetchSchemas(in: current, storageRows: IndexSet(positions.map(\.row)), retryingFailures: false) - default: - break - } let change = DataGridChange.from(delta, order: order, rows: current, columns: columns) guard !change.isNone else { return } if case .rowsUpdated(let displayRows, _) = change { @@ -283,16 +282,16 @@ final class DataGridModel: ObservableObject { let current = rows let previous = displayFormat(for: column) guard previous != format else { return } - let formats = DisplayFormatArray.setting( + store.displayFormats = DisplayFormatArray.setting( format, at: column.dataIndex, in: store.displayFormats, columnCount: current.columns.count ) + // Before the remap reads it: another connection may have loaded the schemas already. + prefetchSchemas(in: current, storageRows: nil, retryingFailures: true) // A filter stores the shown text, so the values the user picked follow the new format. if let remapped = remappedValueFilter(column: column, from: previous, to: format, in: current) { store.valueFilter = remapped } - store.displayFormats = formats redrawFormatted([column], in: current) - prefetchSchemas(in: current, storageRows: nil, retryingFailures: true) } private func redrawFormatted(_ changed: [DataGridColumn], in current: TableRows) { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift index db01eea8bb..71023d8a49 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift @@ -327,6 +327,8 @@ internal enum ProtoFileParser { case ("features.enum_type", "OPEN"): overrides.closedEnum = false case ("features.utf8_validation", "VERIFY"): overrides.validatesUTF8 = true case ("features.utf8_validation", "NONE"): overrides.validatesUTF8 = false + case ("features.message_encoding", "DELIMITED"): overrides.delimitedMessages = true + case ("features.message_encoding", "LENGTH_PREFIXED"): overrides.delimitedMessages = false default: break } } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift index 766f94b82f..bbccf26936 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift @@ -30,6 +30,8 @@ internal struct ProtobufFieldDescriptor: Equatable, Sendable { let hasPresence: Bool // protoc refuses the whole value when such a string is not UTF-8. let validatesUTF8: Bool + // An editions message field sent with group framing. + let isDelimited: Bool } internal struct ProtobufMessageDescriptor: Equatable, Sendable { @@ -147,7 +149,11 @@ internal struct ProtobufSchema: Equatable, Sendable { throw ProtobufSchemaError(reason: String(localized: "The file defines no message.")) } let scopes = ProtobufFeatureScopes(files: files, messages: definitions, enums: enumDefinitions) - let resolver = ProtobufTypeResolver(messageNames: Set(definitions.keys), enumNames: Set(enumDefinitions.keys)) + let resolver = ProtobufTypeResolver( + messageNames: Set(definitions.keys), + enumNames: Set(enumDefinitions.keys), + mapEntryNames: Set(definitions.values.filter(\.isMapEntry).map(\.fullName)) + ) var resolved: [String: ProtobufMessageDescriptor] = [:] for (name, message) in definitions { let features = scopes.messages[name] ?? ProtobufFeatures(syntax: .proto2) @@ -182,6 +188,7 @@ internal struct ProtobufFeatureOverrides: Equatable, Sendable { var implicitPresence: Bool? var closedEnum: Bool? var validatesUTF8: Bool? + var delimitedMessages: Bool? } internal struct ProtobufFileDefinition: Sendable { @@ -232,6 +239,7 @@ private struct ProtobufFeatures { var implicitPresence: Bool var closedEnum: Bool var validatesUTF8: Bool + var delimitedMessages = false init(syntax: ProtobufSyntax) { self.syntax = syntax @@ -246,6 +254,7 @@ private struct ProtobufFeatures { if let implicitPresence = overrides.implicitPresence { result.implicitPresence = implicitPresence } if let closedEnum = overrides.closedEnum { result.closedEnum = closedEnum } if let validatesUTF8 = overrides.validatesUTF8 { result.validatesUTF8 = validatesUTF8 } + if let delimitedMessages = overrides.delimitedMessages { result.delimitedMessages = delimitedMessages } return result } } @@ -289,6 +298,7 @@ private struct ProtobufFeatureScopes { private struct ProtobufTypeResolver { let messageNames: Set let enumNames: Set + let mapEntryNames: Set func resolve(_ field: ProtobufFieldDefinition, scope: String, features: ProtobufFeatures) -> ProtobufFieldDescriptor { let type: ProtobufFieldType @@ -314,10 +324,17 @@ private struct ProtobufTypeResolver { isRepeated: field.isRepeated, oneofIndex: field.oneofIndex, hasPresence: !field.isRepeated && (field.oneofIndex != nil || isMessage || !isImplicit), - validatesUTF8: fieldFeatures.validatesUTF8 + validatesUTF8: fieldFeatures.validatesUTF8, + isDelimited: isDelimited(type, features: fieldFeatures) ) } + // A map's entries stay length-prefixed whatever the file sets. + private func isDelimited(_ type: ProtobufFieldType, features: ProtobufFeatures) -> Bool { + guard features.delimitedMessages, case .message(let name) = type else { return false } + return !mapEntryNames.contains(name) + } + private func resolve(_ reference: String, scope: String) -> ProtobufFieldType { if reference.hasPrefix(".") { return lookup(String(reference.dropFirst())) ?? .unresolved(reference) @@ -413,7 +430,7 @@ internal enum ProtobufDescriptorSetParser { return nil } - // FeatureSet: field_presence = 1 (2 IMPLICIT), enum_type = 2 (2 CLOSED), utf8_validation = 4 (3 NONE). + // FeatureSet: field_presence 1 (2 IMPLICIT), enum_type 2 (2 CLOSED), utf8_validation 4 (3 NONE), message_encoding 5. private static func features(_ entries: [(Int, Value)], options: Int, features: Int) throws -> ProtobufFeatureOverrides { guard let optionBytes = bytes(entries, options), let featureBytes = bytes(try fields(optionBytes), features) else { return ProtobufFeatureOverrides() } @@ -422,6 +439,7 @@ internal enum ProtobufDescriptorSetParser { if let presence = varint(set, 1), (1...3).contains(presence) { overrides.implicitPresence = presence == 2 } if let enumType = varint(set, 2), (1...2).contains(enumType) { overrides.closedEnum = enumType == 2 } if let validation = varint(set, 4), (2...3).contains(validation) { overrides.validatesUTF8 = validation == 2 } + if let encoding = varint(set, 5), (1...2).contains(encoding) { overrides.delimitedMessages = encoding == 2 } return overrides } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift index 065dbd7479..4170a0c643 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -146,7 +146,7 @@ internal enum ProtobufSchemaDecoder { if !asMapEntry { level.unknown.append(occurrence) } } } - keepLastOneofMember(&accepted, message: message) + try parseDiscarded(keepLastOneofMember(&accepted, message: message), of: message, depth: depth) var numbers = Set(accepted.keys) // protoc prints an entry's key and value even when the bytes leave them out. if asMapEntry, !level.isCut { numbers.formUnion(message.fields.map(\.number)) } @@ -203,9 +203,12 @@ internal enum ProtobufSchemaDecoder { private func match(_ payload: Occurrence.Payload, field: ProtobufFieldDescriptor) -> Match { switch (field.type, payload) { - case (.scalar(.string), .lengthDelimited), (.scalar(.bytes), .lengthDelimited), - (.message, .lengthDelimited), (.group, .group): + case (.scalar(.string), .lengthDelimited), (.scalar(.bytes), .lengthDelimited), (.group, .group): return .accepted + case (.message, .lengthDelimited): + return field.isDelimited ? .mismatched : .accepted + case (.message, .group): + return field.isDelimited ? .accepted : .mismatched case (.scalar(.string), _), (.scalar(.bytes), _): return .mismatched case (.scalar, .lengthDelimited), (.enumeration, .lengthDelimited): @@ -228,19 +231,44 @@ internal enum ProtobufSchemaDecoder { return enumeration.values[Int32(truncatingIfNeeded: raw)] != nil } - // The member set last wins, and only what follows the last switch to it counts. - private func keepLastOneofMember(_ accepted: inout [Int: [Occurrence]], message: ProtobufMessageDescriptor) { + // The member set last wins, and only what follows the last switch to it counts. Returns what it drops. + private func keepLastOneofMember( + _ accepted: inout [Int: [Occurrence]], + message: ProtobufMessageDescriptor + ) -> [Occurrence] { var oneofs: [Int: [Int]] = [:] for field in message.fields where accepted[field.number] != nil { if let index = field.oneofIndex { oneofs[index, default: []].append(field.number) } } + var dropped: [Occurrence] = [] for members in oneofs.values where members.count > 1 { let lastPositions = members.map { (number: $0, position: accepted[$0]?.last?.position ?? -1) } guard let winner = lastPositions.max(by: { $0.position < $1.position })?.number else { continue } let cutoff = lastPositions.filter { $0.number != winner }.map { $0.position }.max() ?? -1 - for number in members where number != winner { accepted[number] = nil } + for number in members where number != winner { + dropped += accepted[number] ?? [] + accepted[number] = nil + } + dropped += accepted[winner]?.filter { $0.position <= cutoff } ?? [] accepted[winner] = accepted[winner]?.filter { $0.position > cutoff } } + return dropped + } + + // protoc parses a member before a later one replaces it, so a malformed one still fails the value. + private func parseDiscarded(_ dropped: [Occurrence], of message: ProtobufMessageDescriptor, depth: Int) throws { + for occurrence in dropped { + guard let field = message.field(number: occurrence.number) else { continue } + switch (field.type, occurrence.payload) { + case (.message(let name), .lengthDelimited(let window)), (.message(let name), .group(let window)), + (.group(let name), .group(let window)): + guard let nested = schema.messages[name] else { continue } + var probe = self + _ = try probe.members(of: [window], message: nested, depth: depth + 1, asMapEntry: false) + default: + continue + } + } } private mutating func append( diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift index 13a647e10b..7d4ae11cc7 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -545,6 +545,52 @@ struct ProtobufSchemaDecoderTests { #expect(try Self.text("1a020801", schema: schema, message: "n.M") == "g {\n 1: 1\n}\n") } + @Test("A oneof message member that a later member replaces must still parse, as protoc parses it first") + func discardedOneofMemberIsParsed() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package o2; + message Inner { int32 n = 1; } + message M { oneof c { Inner a = 1; int32 b = 2; } } + """.utf8)) + // protoc --decode=o2.M fails on both: `a` holds a truncated varint. + for fixture in ["0a01081007", "10070a0108"] { + #expect(throws: BinaryValueDecodeError.self, "\(fixture)") { + try Self.text(fixture, schema: schema, message: "o2.M") + } + } + #expect(try Self.text("0a0208011007", schema: schema, message: "o2.M") == "b: 7\n") + } + + @Test("An editions message field set DELIMITED reads its group framing, from a .proto and a descriptor set") + func delimitedMessageEncoding() throws { + let proto = """ + edition = "2023"; + package d; + message Inner { int32 n = 1; } + message M { Inner inner = 1 [features.message_encoding = DELIMITED]; Inner plain = 2; } + """ + // protoc --descriptor_set_out=d.desc of the file above. + let descriptorSet = "0a7c0a07642e70726f746f12016422150a05496e6e6572120c0a016e18012001280552016e224a0a014d12250a05696e6e6572" + + "18012001280b32082e642e496e6e65724205aa010228025205696e6e6572121e0a05706c61696e18022001280b32082e642e496e6e" + + "65725205706c61696e620865646974696f6e7370e807" + for schema in [try ProtobufSchema.parse(Data(proto.utf8)), try ProtobufSchema.parse(BinaryFixture.data(descriptorSet))] { + // protoc --decode=d.M of 0b08070c, then of 0a020807 sent length-prefixed. + #expect(try Self.text("0b08070c", schema: schema, message: "d.M") == "inner {\n n: 7\n}\n") + #expect(try Self.text("0a020807", schema: schema, message: "d.M") == "1 {\n 1: 7\n}\n") + } + let fileLevel = try ProtobufSchema.parse(Data(""" + edition = "2023"; + package d2; + option features.message_encoding = DELIMITED; + message Inner { int32 n = 1; } + message M { Inner inner = 1; repeated Inner many = 2; } + """.utf8)) + // protoc --decode=d2.M of 0b08070c1308011413080214 + #expect(try Self.text("0b08070c1308011413080214", schema: fileLevel, message: "d2.M") + == "inner {\n n: 7\n}\nmany {\n n: 1\n}\nmany {\n n: 2\n}\n") + } + @Test("The package qualifies the definitions written before it too") func packageAfterMessage() throws { let schema = try ProtobufSchema.parse(Data(""" diff --git a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift index 17e68792e5..eb8fa47d09 100644 --- a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift +++ b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift @@ -283,6 +283,25 @@ struct SchemaRegistryTests { #expect(held.calls.withLock { $0 } == SchemaRegistryCache.maxPendingFetches) } + @Test("A connection on a registry another one already read gets its schemas as soon as it prefetches") + @MainActor + func sharedEndpointAdoptsLoadedSchemas() async throws { + let transport = RecordedTransport() + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: transport.transport) + guard case .success = await cache.schema(connectionID: UUID(), schemaID: 1) else { + Issue.record("expected the schema for the first connection") + return + } + let second = UUID() + #expect(cache.cachedSchema(connectionID: second, schemaID: 1) == nil) + cache.prefetch(connectionID: second, schemaIDs: [1], retryingFailures: false) + guard case .success? = cache.cachedSchema(connectionID: second, schemaID: 1) else { + Issue.record("expected the second connection to read the loaded schema at once") + return + } + #expect(transport.requests.withLock { $0.count } == 1) + } + @Test("A connection with no registry URL says so instead of fetching") @MainActor func noEndpoint() async { From 9efbae48f840f9fb7388100b211369ae6146935c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 22:52:13 +0700 Subject: [PATCH 17/22] fix(display-format): refresh hidden registry columns and budget Protobuf group scans --- .../Core/DataGrid/DataGridModel.swift | 46 +++++++++++-------- .../BinaryValues/ProtobufSchemaDecoder.swift | 20 ++++++-- .../BinaryValues/ProtobufWireDecoder.swift | 17 +++++-- .../ProtobufSchemaDecoderTests.swift | 18 ++++++++ .../DataGridSchemaRegistryTests.swift | 24 ++++++++++ 5 files changed, 100 insertions(+), 25 deletions(-) diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index b5c78f56cf..a34a459cb8 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -288,21 +288,26 @@ final class DataGridModel: ObservableObject { // Before the remap reads it: another connection may have loaded the schemas already. prefetchSchemas(in: current, storageRows: nil, retryingFailures: true) // A filter stores the shown text, so the values the user picked follow the new format. - if let remapped = remappedValueFilter(column: column, from: previous, to: format, in: current) { + if let remapped = remappedValueFilter( + dataIndex: column.dataIndex, name: column.name, type: column.type, from: previous, to: format, in: current + ) { store.valueFilter = remapped } - redrawFormatted([column], in: current) + redrawFormatted([column.dataIndex], in: current) } - private func redrawFormatted(_ changed: [DataGridColumn], in current: TableRows) { - display.cache.clearValues(forDataIndexes: Set(changed.map(\.dataIndex))) - if changed.contains(where: { store.valueFilter.filter(forColumn: $0.dataIndex) != nil }) { + // Hidden columns too: their cached text and filters outlive the hiding. + private func redrawFormatted(_ dataIndexes: Set, in current: TableRows) { + display.cache.clearValues(forDataIndexes: dataIndexes) + if dataIndexes.contains(where: { store.valueFilter.filter(forColumn: $0) != nil }) { order = resolveOrder(in: current) restoreSelection(in: current) displayRevision &+= 1 mountedView?.apply(.reload) } else { - let positions = IndexSet(changed.compactMap { columns.displayPosition(of: $0.id) }) + let positions = IndexSet(columns.columns.filter { dataIndexes.contains($0.dataIndex) }.compactMap { + columns.displayPosition(of: $0.id) + }) if !positions.isEmpty { mountedView?.apply(.rowsUpdated(IndexSet(0.. GridValueFilterState? { - let dataIndex = column.dataIndex guard let existing = store.valueFilter.filter(forColumn: dataIndex) else { return nil } let databaseType = configuration.traits.databaseType var removed = Set() @@ -426,11 +436,11 @@ final class DataGridModel: ObservableObject { let raw = row.values[dataIndex] guard raw != .null else { continue } let old = GridDisplayOrderResolver.displayValue( - raw, columnType: column.type, displayFormat: oldFormat, databaseType: databaseType + raw, columnType: type, displayFormat: oldFormat, databaseType: databaseType ) guard existing.selectedValues.contains(old) else { continue } let new = GridDisplayOrderResolver.displayValue( - raw, columnType: column.type, displayFormat: newFormat, databaseType: databaseType + raw, columnType: type, displayFormat: newFormat, databaseType: databaseType ) if old != new { removed.insert(old) @@ -444,7 +454,7 @@ final class DataGridModel: ObservableObject { var filter = store.valueFilter filter.set( ColumnValueFilter(selectedValues: selected, includesNull: existing.includesNull), - columnName: column.name, + columnName: name, forColumn: dataIndex ) return filter diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift index 4170a0c643..f2db70b87d 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -128,8 +128,13 @@ internal enum ProtobufSchemaDecoder { level.isCut = true break scan } - occurrences.append(try Self.occurrence(&reader, depth: depth)) + let (occurrence, isCut) = try Self.occurrence(&reader, depth: depth, limit: budget) + occurrences.append(occurrence) budget -= 1 + if isCut { + level.isCut = true + break scan + } } } var accepted: [Int: [Occurrence]] = [:] @@ -171,13 +176,18 @@ internal enum ProtobufSchemaDecoder { return fields } - private static func occurrence(_ reader: inout BinaryValueReader, depth: Int) throws -> Occurrence { + private static func occurrence( + _ reader: inout BinaryValueReader, + depth: Int, + limit: Int + ) throws -> (Occurrence, isCut: Bool) { let start = reader.offset let tag = try reader.readVarint() let number = tag >> 3 guard tag <= UInt64(UInt32.max), number > 0, number <= UInt64(ProtobufWireDecoder.maxFieldNumber) else { throw BinaryValueDecodeError(kind: .invalidField, offset: start) } + var isCut = false let payload: Occurrence.Payload switch tag & 7 { case 0: @@ -190,7 +200,9 @@ internal enum ProtobufSchemaDecoder { payload = .lengthDelimited(try reader.window(Int(length))) try reader.skip(Int(length)) case 3: - payload = .group(try ProtobufWireDecoder.groupContents(&reader, number: Int(number), depth: depth)) + let group = try ProtobufWireDecoder.groupContents(&reader, number: Int(number), depth: depth, limit: limit) + payload = .group(group.contents) + isCut = group.isCut case 4: throw BinaryValueDecodeError(kind: .unmatchedGroupEnd, offset: start) case 5: @@ -198,7 +210,7 @@ internal enum ProtobufSchemaDecoder { default: throw BinaryValueDecodeError(kind: .invalidField, offset: start) } - return Occurrence(number: Int(number), payload: payload, position: start) + return (Occurrence(number: Int(number), payload: payload, position: start), isCut) } private func match(_ payload: Occurrence.Payload, field: ProtobufFieldDescriptor) -> Match { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift index 11ebbd1b37..4e72c33fb8 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift @@ -82,14 +82,25 @@ internal enum ProtobufWireDecoder { return fields } - // The bytes between a group's start tag, already read, and its end tag, which the reader is left past. - static func groupContents(_ reader: inout BinaryValueReader, number: Int, depth: Int) throws -> BinaryValueReader { + // The bytes between a group's start tag, already read, and its end tag. Past `limit` tags it stops at the last whole + // field and reports the cut, so a cell never reads more of a group than its budget. + static func groupContents( + _ reader: inout BinaryValueReader, + number: Int, + depth: Int, + limit: Int + ) throws -> (contents: BinaryValueReader, isCut: Bool) { let contents = reader var openGroups = [number] + var boundary = reader.offset + var scanned = 0 while true { let start = reader.offset + if openGroups.count == 1 { boundary = start } + guard scanned < limit else { return (try contents.window(boundary - contents.offset), true) } guard !reader.isAtEnd else { throw reader.error(.unexpectedEnd) } let tag = try reader.readVarint() + scanned += 1 let fieldNumber = tag >> 3 guard tag <= UInt64(UInt32.max), fieldNumber > 0, fieldNumber <= UInt64(maxFieldNumber) else { throw BinaryValueDecodeError(kind: .invalidField, offset: start) @@ -109,7 +120,7 @@ internal enum ProtobufWireDecoder { case 4: guard openGroups.last == Int(fieldNumber) else { throw BinaryValueDecodeError(kind: .unmatchedGroupEnd, offset: start) } openGroups.removeLast() - if openGroups.isEmpty { return try contents.window(start - contents.offset) } + if openGroups.isEmpty { return (try contents.window(start - contents.offset), false) } case 5: try reader.skip(4) default: diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift index 7d4ae11cc7..9a6604d6d7 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -591,6 +591,24 @@ struct ProtobufSchemaDecoderTests { == "inner {\n n: 7\n}\nmany {\n n: 1\n}\nmany {\n n: 2\n}\n") } + @Test("A cell reads no further into a group than its budget, so bytes past it, even broken ones, are never read") + func groupScanIsBudgeted() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto2"; + package gb; + message M { optional group Item = 1 { repeated int32 v = 2; } } + """.utf8)) + // Hand-written: a group of fifty `v: 1`, then a byte that starts no field and no end tag. + let fixture = "0b" + String(repeating: "1001", count: 50) + "ff" + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "gb.M", maxValues: 5) + let text = ProtobufSchemaTextWriter.text(message.fields, multiLine: false) + #expect(text.hasPrefix("Item { v: 1")) + #expect(text.hasSuffix("…")) + #expect(throws: BinaryValueDecodeError.self) { + try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "gb.M") + } + } + @Test("The package qualifies the definitions written before it too") func packageAfterMessage() throws { let schema = try ProtobufSchema.parse(Data(""" diff --git a/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift b/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift index e59abb34ad..133cfb9680 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift @@ -132,6 +132,30 @@ struct DataGridSchemaRegistryTests { #expect(Self.cachedText(grid, row: 0, column: 1) == "kept") } + @Test("A registry change reaches a hidden column's cached text and value filter, so showing it again is current") + func changeReachesHiddenColumns() async throws { + let grid = Self.grid(TableRows( + rows: [Row(id: .existing(0), values: [Self.framed(1), .text("kept")])], + columns: ["value", "key"], + columnTypes: [.blob(rawType: "BLOB"), .text(rawType: nil)] + )) + let column = try #require(grid.model.columns.column(at: 0)) + grid.model.setDisplayFormat(.schemaRegistry(connectionID: Self.connection), for: column) + let row = try #require(grid.model.row(at: 0)) + let pass = (formats: grid.model.store.displayFormats, columnCount: grid.model.rows.columns.count) + _ = grid.presenter.displayText( + of: row, dataIndex: 0, type: .blob(rawType: "BLOB"), formats: pass.formats, databaseType: nil, columnCount: pass.columnCount + ) + #expect(grid.model.display.cache.box(forID: row.id)?.values[0] != nil) + grid.model.updateColumnLayout { $0.hiddenColumns.insert("value") } + #expect(grid.model.columns.columns.allSatisfy { $0.dataIndex != 0 }) + + NotificationCenter.default.post( + name: SchemaRegistryCache.didChange, object: nil, userInfo: [SchemaRegistryCache.connectionIDKey: Self.connection] + ) + try await waitUntil { grid.model.display.cache.box(forID: row.id)?.values[0] == nil } + } + @Test("Adding back a schema file this Mac had lost redraws the columns that name it") func restoredSchemaRedraws() async throws { let grid = Self.grid(TableRows( From 358bf80648d1d6af2e89aedf258b660d8e16f006 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 23:05:45 +0700 Subject: [PATCH 18/22] fix(display-format): keep filtered rows across a registry switch, read aggregate features, cut sint32 to 32 bits --- .../Core/DataGrid/DataGridModel.swift | 55 ++++++++++++++++++- .../BinaryValues/ProtoFileParser.swift | 23 +++++++- .../BinaryValues/ProtobufSchemaDecoder.swift | 5 +- .../SchemaRegistry/SchemaRegistryCache.swift | 31 +++++++---- .../ProtobufSchemaDecoderTests.swift | 26 +++++++++ .../SchemaRegistry/SchemaRegistryTests.swift | 25 +++++++++ 6 files changed, 150 insertions(+), 15 deletions(-) diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index a34a459cb8..918f56cbb2 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -404,10 +404,63 @@ final class DataGridModel: ObservableObject { wanted[connectionID] = ids } for (connectionID, ids) in wanted { - schemaRegistry.prefetch(connectionID: connectionID, schemaIDs: ids, retryingFailures: retryingFailures) + var shown: [Int: [Int: String]] = [:] + let switched = schemaRegistry.prefetch(connectionID: connectionID, schemaIDs: ids, retryingFailures: retryingFailures) { + shown = filteredRegistryText(of: connectionID, in: current) + } + if switched { followRegistrySwitch(shown, in: current) } } } + // What each row passing a filter on this connection's registry columns shows now, read before a switch changes it. + private func filteredRegistryText(of connectionID: UUID, in current: TableRows) -> [Int: [Int: String]] { + let databaseType = configuration.traits.databaseType + var shown: [Int: [Int: String]] = [:] + for (dataIndex, format) in store.displayFormats.enumerated() where format == .schemaRegistry(connectionID: connectionID) { + guard let filter = store.valueFilter.filter(forColumn: dataIndex) else { continue } + let type = dataIndex < current.columnTypes.count ? current.columnTypes[dataIndex] : nil + for (rowIndex, row) in current.rows.enumerated() where dataIndex < row.values.count && row.values[dataIndex] != .null { + let text = GridDisplayOrderResolver.displayValue( + row.values[dataIndex], columnType: type, displayFormat: format, databaseType: databaseType + ) + if filter.selectedValues.contains(text) { shown[dataIndex, default: [:]][rowIndex] = text } + } + } + return shown + } + + // A switched registry reads the same bytes differently, so the rows a filter kept stay kept under their new text. + private func followRegistrySwitch(_ shown: [Int: [Int: String]], in current: TableRows) { + let databaseType = configuration.traits.databaseType + for (dataIndex, rows) in shown { + guard let existing = store.valueFilter.filter(forColumn: dataIndex), dataIndex < current.columns.count else { continue } + let type = dataIndex < current.columnTypes.count ? current.columnTypes[dataIndex] : nil + let format = displayFormatAt(dataIndex) + var selected = existing.selectedValues + for (rowIndex, old) in rows where rowIndex < current.rows.count { + let new = GridDisplayOrderResolver.displayValue( + current.rows[rowIndex].values[dataIndex], columnType: type, displayFormat: format, databaseType: databaseType + ) + guard new != old else { continue } + selected.remove(old) + selected.insert(new) + } + guard selected != existing.selectedValues else { continue } + var filter = store.valueFilter + filter.set( + ColumnValueFilter(selectedValues: selected, includesNull: existing.includesNull), + columnName: current.columns[dataIndex], + forColumn: dataIndex + ) + store.valueFilter = filter + } + } + + private func displayFormatAt(_ dataIndex: Int) -> ValueDisplayFormat? { + let formats = store.displayFormats + return dataIndex < formats.count ? formats[dataIndex] : nil + } + private static func schemaID(of value: PluginCellValue, holdsBytes: Bool) -> UInt32? { switch value { case .null: diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift index 71023d8a49..8ed18de9a8 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift @@ -315,10 +315,27 @@ internal enum ProtoFileParser { return false } - // `features.field_presence = IMPLICIT` and `features.enum_type = CLOSED`, as a statement or a field option. + // `features.field_presence = IMPLICIT` or `features = { field_presence: IMPLICIT }`, as a statement or a field option. private func feature(at index: Int, into overrides: inout ProtobufFeatureOverrides) { - guard index + 2 < tokens.count, case .identifier(let name) = tokens[index], tokens[index + 1] == .symbol("="), - case .identifier(let value) = tokens[index + 2] else { return } + guard index + 2 < tokens.count, case .identifier(let name) = tokens[index], tokens[index + 1] == .symbol("=") else { return } + if name == "features", tokens[index + 2] == .symbol("{") { + var cursor = index + 3 + while cursor + 2 < tokens.count, tokens[cursor] != .symbol("}") { + if case .identifier(let key) = tokens[cursor], tokens[cursor + 1] == .symbol(":"), + case .identifier(let value) = tokens[cursor + 2] { + Self.apply("features.\(key)", value, into: &overrides) + cursor += 3 + } else { + cursor += 1 + } + } + return + } + guard case .identifier(let value) = tokens[index + 2] else { return } + Self.apply(name, value, into: &overrides) + } + + private static func apply(_ name: String, _ value: String, into overrides: inout ProtobufFeatureOverrides) { switch (name, value) { case ("features.field_presence", "IMPLICIT"): overrides.implicitPresence = true case ("features.field_presence", "EXPLICIT"), ("features.field_presence", "LEGACY_REQUIRED"): diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift index f2db70b87d..af8cc93a10 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -511,7 +511,10 @@ internal enum ProtobufSchemaDecoder { case .scalar(.int64): return String(Int64(bitPattern: raw)) case .scalar(.uint32): return String(UInt32(truncatingIfNeeded: raw)) case .scalar(.uint64): return String(raw) - case .scalar(.sint32): return String(Int32(truncatingIfNeeded: ProtobufWireDecoder.zigzag(raw))) + case .scalar(.sint32): + // Truncated before the ZigZag step, as protoc reads a value a sint64 writer sent. + let bits = UInt32(truncatingIfNeeded: raw) + return String(Int32(bitPattern: (bits >> 1) ^ (0 &- (bits & 1)))) case .scalar(.sint64): return String(ProtobufWireDecoder.zigzag(raw)) case .scalar(.bool): return raw != 0 ? "true" : "false" case .enumeration(let name): diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift index a5fa02dbf5..cfc7df68c7 100644 --- a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift @@ -77,10 +77,16 @@ internal final class SchemaRegistryCache: Sendable { } } - // A full reload retries failures at once; anything else waits out the failure lifetime. + // A reload retries failures at once. `beforeSwitch` runs while the old endpoint still answers, before it is replaced. @MainActor - func prefetch(connectionID: UUID, schemaIDs: Set, retryingFailures: Bool) { - guard !schemaIDs.isEmpty else { return } + @discardableResult + func prefetch( + connectionID: UUID, + schemaIDs: Set, + retryingFailures: Bool, + beforeSwitch: () -> Void = {} + ) -> Bool { + guard !schemaIDs.isEmpty else { return false } let allLoaded = state.withLock { state -> Bool in guard case .configured(let endpoint)? = state.endpoints[connectionID] else { return false } return schemaIDs.allSatisfy { @@ -89,8 +95,9 @@ internal final class SchemaRegistryCache: Sendable { } } // Resolving reads the Keychain, so a batch that needs nothing new skips it unless a reload asked to look again. - if allLoaded, !retryingFailures { return } + if allLoaded, !retryingFailures { return false } let endpoint = resolveEndpoint(connectionID) + if state.withLock({ Self.isSwitch(from: $0.endpoints[connectionID], to: endpoint) }) { beforeSwitch() } let (changed, pending) = state.withLock { state -> (Bool, [Task]) in let changed = adopt(endpoint, for: connectionID, in: &state) guard let endpoint else { return (changed, []) } @@ -108,7 +115,7 @@ internal final class SchemaRegistryCache: Sendable { return (changed, pending) } if changed { Self.post(connectionID) } - guard !pending.isEmpty else { return } + guard !pending.isEmpty else { return changed } Task { var loaded = false for task in pending { @@ -116,6 +123,7 @@ internal final class SchemaRegistryCache: Sendable { } if loaded { Self.post(connectionID) } } + return changed } @MainActor @@ -143,12 +151,15 @@ internal final class SchemaRegistryCache: Sendable { } private func adopt(_ endpoint: SchemaRegistryEndpoint?, for connectionID: UUID, in state: inout State) -> Bool { - let resolved: Resolved = endpoint.map { .configured($0) } ?? .unconfigured - let previous = state.endpoints.updateValue(resolved, forKey: connectionID) - switch (previous, resolved) { - case (nil, _), (.unconfigured?, .unconfigured): + let previous = state.endpoints.updateValue(endpoint.map { .configured($0) } ?? .unconfigured, forKey: connectionID) + return Self.isSwitch(from: previous, to: endpoint) + } + + private static func isSwitch(from previous: Resolved?, to endpoint: SchemaRegistryEndpoint?) -> Bool { + switch (previous, endpoint) { + case (nil, _), (.unconfigured?, nil): return false - case (.configured(let old)?, .configured(let new)): + case (.configured(let old)?, let new?): return old != new default: return true diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift index 9a6604d6d7..31f32044eb 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -609,6 +609,32 @@ struct ProtobufSchemaDecoderTests { } } + @Test("A sint32 wider than 32 bits is cut to 32 before ZigZag, as protoc reads one a sint64 writer sent") + func wideSint32() throws { + let schema = try ProtobufSchema.parse(Data("syntax = \"proto3\"; message M { sint32 n = 1; }".utf8)) + // protoc --decode=M of 088280808010 + #expect(try Self.text("088280808010", schema: schema, message: "M") == "n: 1\n") + } + + @Test("Features written as one aggregate option apply like dotted ones") + func aggregateFeatures() throws { + let schema = try ProtobufSchema.parse(Data(""" + edition = "2023"; + package ag; + option features = { message_encoding: DELIMITED, field_presence: IMPLICIT }; + message Inner { int32 n = 1; } + message M { + Inner inner = 1; + int32 z = 2; + string t = 3 [features = { utf8_validation: NONE }]; + int32 e = 4 [features = { field_presence: EXPLICIT }]; + } + """.utf8)) + // protoc --decode=ag.M of 0b08070c10001a01ff2000 + #expect(try Self.text("0b08070c10001a01ff2000", schema: schema, message: "ag.M") + == "inner {\n n: 7\n}\nt: \"\\377\"\ne: 0\n") + } + @Test("The package qualifies the definitions written before it too") func packageAfterMessage() throws { let schema = try ProtobufSchema.parse(Data(""" diff --git a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift index eb8fa47d09..18510ef7a7 100644 --- a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift +++ b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift @@ -302,6 +302,31 @@ struct SchemaRegistryTests { #expect(transport.requests.withLock { $0.count } == 1) } + @Test("A grid hears of a registry switch before it happens, while the old schemas still answer") + @MainActor + func beforeSwitch() async throws { + let resolver = Resolver(Self.endpoint) + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: resolver.resolve, transport: RecordedTransport().transport) + _ = await cache.schema(connectionID: connection, schemaID: 1) + var calls = 0 + #expect(!cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true) { calls += 1 }) + #expect(calls == 0) + + resolver.current.withLock { $0 = SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: "https://other.example.com"]) } + var oldStillAnswered = false + let switched = cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true) { + calls += 1 + if case .success? = cache.cachedSchema(connectionID: connection, schemaID: 1) { oldStillAnswered = true } + } + #expect(switched) + #expect(calls == 1) + #expect(oldStillAnswered) + if case .success? = cache.cachedSchema(connectionID: connection, schemaID: 1) { + Issue.record("expected the new registry's schema to be unread") + } + } + @Test("A connection with no registry URL says so instead of fetching") @MainActor func noEndpoint() async { From 9682b7f200b866ca2a82eaa5ec83d5cb4a5b03a4 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 23:20:57 +0700 Subject: [PATCH 19/22] fix(display-format): carry every tab's filter across a registry switch and keep a map entry's last value --- .../Core/DataGrid/DataGridModel.swift | 54 ++++++++++++------- .../BinaryValues/ProtobufSchemaDecoder.swift | 54 +++++++++++++------ .../SchemaRegistry/SchemaRegistryCache.swift | 20 ++++--- .../ProtobufSchemaDecoderTests.swift | 13 +++++ .../SchemaRegistry/SchemaRegistryTests.swift | 29 +++++----- 5 files changed, 114 insertions(+), 56 deletions(-) diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index 918f56cbb2..608496a96f 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -60,6 +60,8 @@ final class DataGridModel: ObservableObject { private let schemaRegistry: SchemaRegistryCache private var schemaObservers: Set = [] private var staleSchemaColumns: Set = [] + // Per connection, per data index: what each row a filter kept showed before its registry switched. + private var registrySwitchSnapshots: [UUID: [Int: [RowID: String]]] = [:] init( store: any DataGridStateStore = InMemoryDataGridStateStore(), @@ -328,6 +330,13 @@ final class DataGridModel: ObservableObject { } } .store(in: &schemaObservers) + // Delivered where it is posted, the main actor, before the switch it announces. + center.publisher(for: SchemaRegistryCache.willSwitch) + .sink { [weak self] notification in + guard let connectionID = notification.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID else { return } + MainActor.assumeIsolated { self?.registryWillSwitch(connectionID) } + } + .store(in: &schemaObservers) center.publisher(for: AvroSchemaStore.didRestoreSchema) .receive(on: DispatchQueue.main) .sink { [weak self] notification in @@ -365,6 +374,7 @@ final class DataGridModel: ObservableObject { private func redrawStaleSchemaColumns() { let current = rows + followRegistrySwitches(in: current) let stale = staleSchemaColumns.filter { $0 < current.columns.count } staleSchemaColumns = [] guard !stale.isEmpty else { return } @@ -404,47 +414,55 @@ final class DataGridModel: ObservableObject { wanted[connectionID] = ids } for (connectionID, ids) in wanted { - var shown: [Int: [Int: String]] = [:] - let switched = schemaRegistry.prefetch(connectionID: connectionID, schemaIDs: ids, retryingFailures: retryingFailures) { - shown = filteredRegistryText(of: connectionID, in: current) + if schemaRegistry.prefetch(connectionID: connectionID, schemaIDs: ids, retryingFailures: retryingFailures) { + followRegistrySwitches(in: current) } - if switched { followRegistrySwitch(shown, in: current) } } } - // What each row passing a filter on this connection's registry columns shows now, read before a switch changes it. - private func filteredRegistryText(of connectionID: UUID, in current: TableRows) -> [Int: [Int: String]] { + private func registryWillSwitch(_ connectionID: UUID) { + let current = rows let databaseType = configuration.traits.databaseType - var shown: [Int: [Int: String]] = [:] for (dataIndex, format) in store.displayFormats.enumerated() where format == .schemaRegistry(connectionID: connectionID) { guard let filter = store.valueFilter.filter(forColumn: dataIndex) else { continue } let type = dataIndex < current.columnTypes.count ? current.columnTypes[dataIndex] : nil - for (rowIndex, row) in current.rows.enumerated() where dataIndex < row.values.count && row.values[dataIndex] != .null { + var shown: [RowID: String] = [:] + for row in current.rows where dataIndex < row.values.count && row.values[dataIndex] != .null { let text = GridDisplayOrderResolver.displayValue( row.values[dataIndex], columnType: type, displayFormat: format, databaseType: databaseType ) - if filter.selectedValues.contains(text) { shown[dataIndex, default: [:]][rowIndex] = text } + if filter.selectedValues.contains(text) { shown[row.id] = text } } + if !shown.isEmpty { registrySwitchSnapshots[connectionID, default: [:]][dataIndex] = shown } } - return shown } // A switched registry reads the same bytes differently, so the rows a filter kept stay kept under their new text. - private func followRegistrySwitch(_ shown: [Int: [Int: String]], in current: TableRows) { + private func followRegistrySwitches(in current: TableRows) { + guard !registrySwitchSnapshots.isEmpty else { return } + let snapshots = registrySwitchSnapshots + registrySwitchSnapshots = [:] let databaseType = configuration.traits.databaseType - for (dataIndex, rows) in shown { + let rowsByID = Dictionary(current.rows.map { ($0.id, $0) }) { first, _ in first } + for (dataIndex, shown) in snapshots.values.flatMap({ $0 }) { guard let existing = store.valueFilter.filter(forColumn: dataIndex), dataIndex < current.columns.count else { continue } let type = dataIndex < current.columnTypes.count ? current.columnTypes[dataIndex] : nil let format = displayFormatAt(dataIndex) - var selected = existing.selectedValues - for (rowIndex, old) in rows where rowIndex < current.rows.count { + var removed = Set() + var added = Set() + for (rowID, old) in shown { + guard let row = rowsByID[rowID], dataIndex < row.values.count else { continue } let new = GridDisplayOrderResolver.displayValue( - current.rows[rowIndex].values[dataIndex], columnType: type, displayFormat: format, databaseType: databaseType + row.values[dataIndex], columnType: type, displayFormat: format, databaseType: databaseType ) - guard new != old else { continue } - selected.remove(old) - selected.insert(new) + if new != old { + removed.insert(old) + added.insert(new) + } } + var selected = existing.selectedValues + selected.subtract(removed) + selected.formUnion(added) guard selected != existing.selectedValues else { continue } var filter = store.valueFilter filter.set( diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift index af8cc93a10..e024eeb39e 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -68,6 +68,8 @@ internal enum ProtobufSchemaDecoder { case fixed32(UInt32) case lengthDelimited(BinaryValueReader) case group(BinaryValueReader) + // A rejected map entry, kept as protoc writes it back: its last key and value. + case entry([Occurrence]) } let number: Int @@ -82,7 +84,10 @@ internal enum ProtobufSchemaDecoder { } // protoc moves a map entry whose closed enum value is undeclared to the unknown fields whole. - private struct MapEntryRejected: Error {} + // `kept` is what protoc writes back for the entry; nil keeps its bytes as they came. + private struct MapEntryRejected: Error { + let kept: [Occurrence]? + } private struct Level { var fields: [ProtobufTypedField] = [] @@ -138,19 +143,26 @@ internal enum ProtobufSchemaDecoder { } } var accepted: [Int: [Occurrence]] = [:] + var lastInEntry: [Int: (occurrence: Occurrence, match: Match)] = [:] for occurrence in occurrences { let field = message.field(number: occurrence.number) - switch field.map({ match(occurrence.payload, field: $0) }) ?? .mismatched { + let result = field.map { match(occurrence.payload, field: $0) } ?? .mismatched + if asMapEntry, result != .mismatched { lastInEntry[occurrence.number] = (occurrence, result) } + switch result { case .accepted: if let field { try verifyUTF8(occurrence.payload, field: field) } accepted[occurrence.number, default: []].append(occurrence) case .undeclaredEnumValue where asMapEntry: - throw MapEntryRejected() + break case .mismatched, .undeclaredEnumValue: // A map entry drops what it does not declare. if !asMapEntry { level.unknown.append(occurrence) } } } + // Last wins in an entry too: only an undeclared closed enum number left standing rejects it. + if lastInEntry.values.contains(where: { $0.match == .undeclaredEnumValue }) { + throw MapEntryRejected(kept: lastInEntry.keys.sorted().compactMap { lastInEntry[$0]?.occurrence }) + } try parseDiscarded(keepLastOneofMember(&accepted, message: message), of: message, depth: depth) var numbers = Set(accepted.keys) // protoc prints an entry's key and value even when the bytes leave them out. @@ -164,7 +176,7 @@ internal enum ProtobufSchemaDecoder { } } // An entry that cannot hold a value goes to the unknown fields whole, as protoc's parse would fail. - if asMapEntry, !level.unknown.isEmpty { throw MapEntryRejected() } + if asMapEntry, !level.unknown.isEmpty { throw MapEntryRejected(kept: nil) } var fields = level.fields let unknown = level.unknown.enumerated().sorted { ($0.element.position, $0.offset) < ($1.element.position, $1.offset) @@ -327,7 +339,7 @@ internal enum ProtobufSchemaDecoder { let windows = found.compactMap { occurrence -> BinaryValueReader? in switch occurrence.payload { case .lengthDelimited(let window), .group(let window): return window - case .varint, .fixed64, .fixed32: return nil + case .varint, .fixed64, .fixed32, .entry: return nil } } // A declared message that does not parse fails the value, as it fails protoc's parse. @@ -354,9 +366,11 @@ internal enum ProtobufSchemaDecoder { do { let members = try self.members(of: [window], message: entry, depth: depth + 1, asMapEntry: true) entries.append((mapKey(members, entry: entry), typed(field, .message(members)))) - } catch is MapEntryRejected { + } catch let rejected as MapEntryRejected { budget = start - level.unknown.append(occurrence) + level.unknown.append(rejected.kept.map { + Occurrence(number: occurrence.number, payload: .entry($0), position: occurrence.position) + } ?? occurrence) } } let sorted = entries.enumerated().sorted { lhs, rhs in @@ -405,17 +419,23 @@ internal enum ProtobufSchemaDecoder { } private mutating func raw(_ occurrence: Occurrence, depth: Int) throws -> ProtobufTypedField { - let value: ProtobufFieldValue - switch occurrence.payload { - case .varint(let raw): value = .varint(raw) - case .fixed64(let raw): value = .fixed64(raw) - case .fixed32(let raw): value = .fixed32(raw) + ProtobufTypedField( + name: String(occurrence.number), typeName: nil, value: .unknown(try rawValue(occurrence.payload, depth: depth)) + ) + } + + private mutating func rawValue(_ payload: Occurrence.Payload, depth: Int) throws -> ProtobufFieldValue { + switch payload { + case .varint(let raw): return .varint(raw) + case .fixed64(let raw): return .fixed64(raw) + case .fixed32(let raw): return .fixed32(raw) case .lengthDelimited(let window): - value = ProtobufWireDecoder.lengthDelimited(window, depth: depth, budget: &budget) + return ProtobufWireDecoder.lengthDelimited(window, depth: depth, budget: &budget) case .group(let window): - value = .group(try ProtobufWireDecoder.fields(in: window, depth: depth + 1, budget: &budget)) + return .group(try ProtobufWireDecoder.fields(in: window, depth: depth + 1, budget: &budget)) + case .entry(let kept): + return .message(try kept.map { ProtobufField(number: $0.number, value: try rawValue($0.payload, depth: depth + 1)) }) } - return ProtobufTypedField(name: String(occurrence.number), typeName: nil, value: .unknown(value)) } private func mapKey(_ members: [ProtobufTypedField], entry: ProtobufMessageDescriptor) -> MapKey { @@ -448,7 +468,7 @@ internal enum ProtobufSchemaDecoder { case .fixed64(let raw): return raw == 0 case .fixed32(let raw): return raw == 0 case .lengthDelimited(let window): return window.remaining == 0 - case .group: return false + case .group, .entry: return false } } @@ -501,7 +521,7 @@ internal enum ProtobufSchemaDecoder { case .varint(let raw): return varintText(raw, type: type) ?? String(raw) case .fixed64(let raw): return fixed64Text(raw, type: type) ?? String(raw) case .fixed32(let raw): return fixed32Text(raw, type: type) ?? String(raw) - case .lengthDelimited, .group: return "" + case .lengthDelimited, .group, .entry: return "" } } diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift index cfc7df68c7..39b10d14c8 100644 --- a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift @@ -15,6 +15,8 @@ internal final class SchemaRegistryCache: Sendable { // Posted once per batch that loaded a schema, and when a connection's endpoint changed. static let didChange = Notification.Name("com.TablePro.schemaRegistryCache.didChange") + // Posted on the main actor while the old endpoint still answers, so every grid can note what its filters show. + static let willSwitch = Notification.Name("com.TablePro.schemaRegistryCache.willSwitch") static let connectionIDKey = "connectionID" // A page of values can name thousands of IDs that are only bytes starting with zero, so fetches are bounded. @@ -77,15 +79,10 @@ internal final class SchemaRegistryCache: Sendable { } } - // A reload retries failures at once. `beforeSwitch` runs while the old endpoint still answers, before it is replaced. + // A reload retries failures at once; anything else waits out the failure lifetime. True when the endpoint switched. @MainActor @discardableResult - func prefetch( - connectionID: UUID, - schemaIDs: Set, - retryingFailures: Bool, - beforeSwitch: () -> Void = {} - ) -> Bool { + func prefetch(connectionID: UUID, schemaIDs: Set, retryingFailures: Bool) -> Bool { guard !schemaIDs.isEmpty else { return false } let allLoaded = state.withLock { state -> Bool in guard case .configured(let endpoint)? = state.endpoints[connectionID] else { return false } @@ -97,7 +94,7 @@ internal final class SchemaRegistryCache: Sendable { // Resolving reads the Keychain, so a batch that needs nothing new skips it unless a reload asked to look again. if allLoaded, !retryingFailures { return false } let endpoint = resolveEndpoint(connectionID) - if state.withLock({ Self.isSwitch(from: $0.endpoints[connectionID], to: endpoint) }) { beforeSwitch() } + announceSwitch(to: endpoint, for: connectionID) let (changed, pending) = state.withLock { state -> (Bool, [Task]) in let changed = adopt(endpoint, for: connectionID, in: &state) guard let endpoint else { return (changed, []) } @@ -129,6 +126,7 @@ internal final class SchemaRegistryCache: Sendable { @MainActor func schema(connectionID: UUID, schemaID: UInt32) async -> Lookup { let endpoint = resolveEndpoint(connectionID) + announceSwitch(to: endpoint, for: connectionID) let (changed, pending, cached) = state.withLock { state -> (Bool, Task?, Lookup?) in let changed = adopt(endpoint, for: connectionID, in: &state) guard let endpoint else { return (changed, nil, .failure(Self.noEndpoint)) } @@ -155,6 +153,12 @@ internal final class SchemaRegistryCache: Sendable { return Self.isSwitch(from: previous, to: endpoint) } + @MainActor + private func announceSwitch(to endpoint: SchemaRegistryEndpoint?, for connectionID: UUID) { + guard state.withLock({ Self.isSwitch(from: $0.endpoints[connectionID], to: endpoint) }) else { return } + NotificationCenter.default.post(name: Self.willSwitch, object: nil, userInfo: [Self.connectionIDKey: connectionID]) + } + private static func isSwitch(from previous: Resolved?, to endpoint: SchemaRegistryEndpoint?) -> Bool { switch (previous, endpoint) { case (nil, _), (.unconfigured?, nil): diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift index 31f32044eb..63ad1773d9 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -635,6 +635,19 @@ struct ProtobufSchemaDecoderTests { == "inner {\n n: 7\n}\nt: \"\\377\"\ne: 0\n") } + @Test("A map entry keeps its last value: a closed enum number it does not declare rejects it only when left standing") + func mapEntryLastValueWins() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto2"; + package me; + enum E { A = 0; B = 1; } + message M { map m = 1; } + """.utf8)) + // protoc --decode=me.M of 0a06080110091001, then of 0a06080110011009. + #expect(try Self.text("0a06080110091001", schema: schema, message: "me.M") == "m {\n key: 1\n value: B\n}\n") + #expect(try Self.text("0a06080110011009", schema: schema, message: "me.M") == "1 {\n 1: 1\n 2: 9\n}\n") + } + @Test("The package qualifies the definitions written before it too") func packageAfterMessage() throws { let schema = try ProtobufSchema.parse(Data(""" diff --git a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift index 18510ef7a7..ba3310ade0 100644 --- a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift +++ b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift @@ -302,26 +302,29 @@ struct SchemaRegistryTests { #expect(transport.requests.withLock { $0.count } == 1) } - @Test("A grid hears of a registry switch before it happens, while the old schemas still answer") + @Test("Every grid hears of a registry switch before it happens, while the old schemas still answer") @MainActor - func beforeSwitch() async throws { + func willSwitch() async throws { let resolver = Resolver(Self.endpoint) let connection = UUID() let cache = SchemaRegistryCache(resolveEndpoint: resolver.resolve, transport: RecordedTransport().transport) _ = await cache.schema(connectionID: connection, schemaID: 1) - var calls = 0 - #expect(!cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true) { calls += 1 }) - #expect(calls == 0) + let heard = OSAllocatedUnfairLock(initialState: (calls: 0, oldStillAnswered: false)) + let observer = NotificationCenter.default.addObserver(forName: SchemaRegistryCache.willSwitch, object: nil, queue: nil) { + guard $0.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID == connection else { return } + let answered: Bool + if case .success? = cache.cachedSchema(connectionID: connection, schemaID: 1) { answered = true } else { answered = false } + heard.withLock { $0 = (calls: $0.calls + 1, oldStillAnswered: answered) } + } + defer { NotificationCenter.default.removeObserver(observer) } + + #expect(!cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true)) + #expect(heard.withLock { $0.calls } == 0) resolver.current.withLock { $0 = SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: "https://other.example.com"]) } - var oldStillAnswered = false - let switched = cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true) { - calls += 1 - if case .success? = cache.cachedSchema(connectionID: connection, schemaID: 1) { oldStillAnswered = true } - } - #expect(switched) - #expect(calls == 1) - #expect(oldStillAnswered) + #expect(cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true)) + #expect(heard.withLock { $0.calls } == 1) + #expect(heard.withLock { $0.oldStillAnswered }) if case .success? = cache.cachedSchema(connectionID: connection, schemaID: 1) { Issue.record("expected the new registry's schema to be unread") } From a45cee2fd6cdef06fb8de7c8d22b393875801b7d Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 23:34:11 +0700 Subject: [PATCH 20/22] fix(display-format): announce each loaded schema, keep map values length-prefixed, share the oneof budget --- .../BinaryValues/ProtobufSchema.swift | 15 +++++++--- .../BinaryValues/ProtobufSchemaDecoder.swift | 9 +++--- .../SchemaRegistry/SchemaRegistryCache.swift | 9 +++--- .../ProtobufSchemaDecoderTests.swift | 28 +++++++++++++++++++ .../SchemaRegistry/SchemaRegistryTests.swift | 25 +++++++++++++++++ 5 files changed, 72 insertions(+), 14 deletions(-) diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift index bbccf26936..7b493dd14d 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift @@ -159,7 +159,9 @@ internal struct ProtobufSchema: Equatable, Sendable { let features = scopes.messages[name] ?? ProtobufFeatures(syntax: .proto2) resolved[name] = ProtobufMessageDescriptor( fullName: name, - fields: message.fields.map { resolver.resolve($0, scope: name, features: features) }, + fields: message.fields.map { + resolver.resolve($0, scope: name, features: features, inMapEntry: message.isMapEntry) + }, nestedMessages: message.nestedMessages, isMapEntry: message.isMapEntry ) @@ -300,7 +302,12 @@ private struct ProtobufTypeResolver { let enumNames: Set let mapEntryNames: Set - func resolve(_ field: ProtobufFieldDefinition, scope: String, features: ProtobufFeatures) -> ProtobufFieldDescriptor { + func resolve( + _ field: ProtobufFieldDefinition, + scope: String, + features: ProtobufFeatures, + inMapEntry: Bool + ) -> ProtobufFieldDescriptor { let type: ProtobufFieldType switch field.type { case .scalar(let scalar): @@ -325,11 +332,11 @@ private struct ProtobufTypeResolver { oneofIndex: field.oneofIndex, hasPresence: !field.isRepeated && (field.oneofIndex != nil || isMessage || !isImplicit), validatesUTF8: fieldFeatures.validatesUTF8, - isDelimited: isDelimited(type, features: fieldFeatures) + isDelimited: !inMapEntry && isDelimited(type, features: fieldFeatures) ) } - // A map's entries stay length-prefixed whatever the file sets. + // A map's entries, and the messages they hold, stay length-prefixed whatever the file sets. private func isDelimited(_ type: ProtobufFieldType, features: ProtobufFeatures) -> Bool { guard features.delimitedMessages, case .message(let name) = type else { return false } return !mapEntryNames.contains(name) diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift index e024eeb39e..e941e44bd7 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -279,16 +279,15 @@ internal enum ProtobufSchemaDecoder { return dropped } - // protoc parses a member before a later one replaces it, so a malformed one still fails the value. - private func parseDiscarded(_ dropped: [Occurrence], of message: ProtobufMessageDescriptor, depth: Int) throws { + // protoc parses a member a later one replaces, so a broken one fails the value; this spends the shared budget. + private mutating func parseDiscarded(_ dropped: [Occurrence], of message: ProtobufMessageDescriptor, depth: Int) throws { for occurrence in dropped { guard let field = message.field(number: occurrence.number) else { continue } switch (field.type, occurrence.payload) { case (.message(let name), .lengthDelimited(let window)), (.message(let name), .group(let window)), (.group(let name), .group(let window)): - guard let nested = schema.messages[name] else { continue } - var probe = self - _ = try probe.members(of: [window], message: nested, depth: depth + 1, asMapEntry: false) + guard let nested = schema.messages[name], budget > 0 else { continue } + _ = try members(of: [window], message: nested, depth: depth + 1, asMapEntry: false) default: continue } diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift index 39b10d14c8..73942e991e 100644 --- a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift @@ -113,12 +113,11 @@ internal final class SchemaRegistryCache: Sendable { } if changed { Self.post(connectionID) } guard !pending.isEmpty else { return changed } - Task { - var loaded = false - for task in pending { - if case .success = await task.value { loaded = true } + // One notice per schema as it lands, so a slow lookup holds back no other; the grid coalesces them. + for task in pending { + Task { + if case .success = await task.value { Self.post(connectionID) } } - if loaded { Self.post(connectionID) } } return changed } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift index 63ad1773d9..49543875b0 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -648,6 +648,34 @@ struct ProtobufSchemaDecoderTests { #expect(try Self.text("0a06080110011009", schema: schema, message: "me.M") == "1 {\n 1: 1\n 2: 9\n}\n") } + @Test("A map's message values stay length-prefixed in a file that sets DELIMITED") + func delimitedFileKeepsMapValuesLengthPrefixed() throws { + let schema = try ProtobufSchema.parse(Data(""" + edition = "2023"; + package dm; + option features.message_encoding = DELIMITED; + message Child { int32 n = 1; } + message M { map m = 1; } + """.utf8)) + // protoc --decode=dm.M of 0a070a016112020805 + #expect(try Self.text("0a070a016112020805", schema: schema, message: "dm.M") + == "m {\n key: \"a\"\n value {\n n: 5\n }\n}\n") + } + + @Test("Parsing a replaced oneof member spends the cell's budget instead of a fresh one") + func discardedOneofSpendsBudget() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package ob; + message Inner { repeated int32 v = 1; } + message M { oneof c { Inner a = 1; int32 b = 2; } } + """.utf8)) + // Hand-written: `a` holding a hundred unpacked `v: 1`, then `b: 7`. + let fixture = "0ac801" + String(repeating: "0801", count: 100) + "1007" + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "ob.M", maxValues: 5) + #expect(ProtobufSchemaTextWriter.text(message.fields, multiLine: false) == "b: 7") + } + @Test("The package qualifies the definitions written before it too") func packageAfterMessage() throws { let schema = try ProtobufSchema.parse(Data(""" diff --git a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift index ba3310ade0..82405f5599 100644 --- a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift +++ b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift @@ -330,6 +330,31 @@ struct SchemaRegistryTests { } } + @Test("A loaded schema is announced at once, not after a slower one in the same batch") + @MainActor + func loadsAreAnnouncedOneByOne() async throws { + let gate = AsyncGate() + let recorded = RecordedTransport() + let slow: SchemaRegistryClient.Transport = { request in + if request.url?.lastPathComponent == "99999" { await gate.wait() } + return try await recorded.transport(request) + } + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: slow) + let (changes, observer) = Self.changes(for: connection) + defer { NotificationCenter.default.removeObserver(observer) } + + cache.prefetch(connectionID: connection, schemaIDs: [1, 99_999], retryingFailures: false) + var iterator = changes.makeAsyncIterator() + _ = await iterator.next() + guard case .success? = cache.cachedSchema(connectionID: connection, schemaID: 1) else { + Issue.record("expected schema 1 while 99999 is still being fetched") + return + } + #expect(cache.cachedSchema(connectionID: connection, schemaID: 99_999) == nil) + await gate.open() + } + @Test("A connection with no registry URL says so instead of fetching") @MainActor func noEndpoint() async { From b27d3583e5cce7010ad57e0fc3220f9203fefa35 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 1 Nov 2026 23:53:54 +0700 Subject: [PATCH 21/22] fix(display-format): keep registry fields over an older Kafka plugin, announce loads to every reader, parse dropped members in order --- .../Core/Plugins/PluginMetadataRegistry.swift | 9 +++- .../BinaryValues/ProtobufSchemaDecoder.swift | 3 +- .../SchemaRegistry/SchemaRegistryCache.swift | 44 +++++++++---------- .../SchemaRegistry/SchemaRegistryClient.swift | 1 + .../ProtobufSchemaDecoderTests.swift | 17 +++++++ .../SchemaRegistry/SchemaRegistryTests.swift | 44 +++++++++++++++++++ 6 files changed, 92 insertions(+), 26 deletions(-) diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift index bb72c1db7e..99d29cb84a 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -689,7 +689,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { let parameterStyle = driverType.parameterStyle let schemes = driverType.urlSchemes let primaryScheme = schemes.first ?? driverType.databaseTypeId.lowercased() - let additionalConnectionFields = driverType.additionalConnectionFields.filter { + let pluginFields = driverType.additionalConnectionFields.filter { // Kafka releases before the shared timeout setting exposed a second control under // this id. An installed old bundle still registers its metadata over the curated // snapshot, so retire the field here too while its saved value migrates in the model. @@ -702,6 +702,13 @@ final class PluginMetadataRegistry: @unchecked Sendable { // the property crash with EXC_BAD_INSTRUCTION (missing witness table entry). let existingSnapshot = snapshot(forRegisteredTypeId: driverType.databaseTypeId) + // The app reads the Schema Registry fields itself, so an older bundle keeps the curated ones. + let declared = Set(pluginFields.map(\.id)) + let additionalConnectionFields = pluginFields + + (existingSnapshot?.connection.additionalConnectionFields ?? []).filter { + SchemaRegistryConnectionField.all.contains($0.id) && !declared.contains($0.id) + } + return PluginMetadataSnapshot( displayName: driverType.databaseDisplayName, iconName: driverType.iconName, diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift index e941e44bd7..5c0fb341f0 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -276,7 +276,8 @@ internal enum ProtobufSchemaDecoder { dropped += accepted[winner]?.filter { $0.position <= cutoff } ?? [] accepted[winner] = accepted[winner]?.filter { $0.position > cutoff } } - return dropped + // Wire order, so the shared budget meets the same members first on every launch. + return dropped.sorted { $0.position < $1.position } } // protoc parses a member a later one replaces, so a broken one fails the value; this spends the shared budget. diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift index 73942e991e..9ff0ce2c8a 100644 --- a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift @@ -13,7 +13,7 @@ internal final class SchemaRegistryCache: Sendable { static let shared = SchemaRegistryCache() - // Posted once per batch that loaded a schema, and when a connection's endpoint changed. + // Posted per loaded schema to every connection on that endpoint, and when a connection's endpoint changed. static let didChange = Notification.Name("com.TablePro.schemaRegistryCache.didChange") // Posted on the main actor while the old endpoint still answers, so every grid can note what its filters show. static let willSwitch = Notification.Name("com.TablePro.schemaRegistryCache.willSwitch") @@ -95,30 +95,19 @@ internal final class SchemaRegistryCache: Sendable { if allLoaded, !retryingFailures { return false } let endpoint = resolveEndpoint(connectionID) announceSwitch(to: endpoint, for: connectionID) - let (changed, pending) = state.withLock { state -> (Bool, [Task]) in + let changed = state.withLock { state -> Bool in let changed = adopt(endpoint, for: connectionID, in: &state) - guard let endpoint else { return (changed, []) } + guard let endpoint else { return changed } pruneFailures(in: &state) - var pending: [Task] = [] for schemaID in schemaIDs.sorted() { let key = Key(endpoint: endpoint, schemaID: schemaID) - if let running = state.inFlight[key] { - pending.append(running) - } else if usable(state.entries[key], retryingFailures: retryingFailures) == nil, - state.inFlight.count < Self.maxPendingFetches { - pending.append(start(key, in: &state)) - } + guard state.inFlight[key] == nil, usable(state.entries[key], retryingFailures: retryingFailures) == nil, + state.inFlight.count < Self.maxPendingFetches else { continue } + _ = start(key, in: &state) } - return (changed, pending) + return changed } if changed { Self.post(connectionID) } - guard !pending.isEmpty else { return changed } - // One notice per schema as it lands, so a slow lookup holds back no other; the grid coalesces them. - for task in pending { - Task { - if case .success = await task.value { Self.post(connectionID) } - } - } return changed } @@ -136,9 +125,7 @@ internal final class SchemaRegistryCache: Sendable { } if changed { Self.post(connectionID) } guard let pending else { return cached ?? .failure(Self.noEndpoint) } - let result = await pending.value - if case .success = result { Self.post(connectionID) } - return result + return await pending.value } private static var noEndpoint: SchemaRegistryError { @@ -201,14 +188,23 @@ internal final class SchemaRegistryCache: Sendable { return task } + // A load is announced as it lands, to every connection reading this endpoint; the grid coalesces the notices. private func finish(_ key: Key, with result: Lookup) { - state.withLock { state in + let readers = state.withLock { state -> [UUID] in state.inFlight[key] = nil switch result { - case .success(let schema): state.entries[key] = .loaded(schema) - case .failure(let error): state.entries[key] = .failed(error, at: .now) + case .success(let schema): + state.entries[key] = .loaded(schema) + return state.endpoints.compactMap { id, resolved in + if case .configured(let endpoint) = resolved, endpoint == key.endpoint { return id } + return nil + } + case .failure(let error): + state.entries[key] = .failed(error, at: .now) + return [] } } + for connectionID in readers { Self.post(connectionID) } } // Garbage prefixes would otherwise leave one failure each for the session. diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift index 4981ce79d7..1f1c9bcadf 100644 --- a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift @@ -10,6 +10,7 @@ internal enum SchemaRegistryConnectionField { static let url = "schemaRegistryUrl" static let username = "schemaRegistryUsername" static let password = "schemaRegistryPassword" + static let all: Set = [url, username, password] } internal struct SchemaRegistryEndpoint: Hashable, Sendable { diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift index 49543875b0..4054bfb2f2 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -676,6 +676,23 @@ struct ProtobufSchemaDecoderTests { #expect(ProtobufSchemaTextWriter.text(message.fields, multiLine: false) == "b: 7") } + @Test("Replaced oneof members parse in wire order, so a broken one fails the value whatever the hash order") + func discardedMembersParseInWireOrder() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package wo; + message Inner { repeated int32 v = 1; } + message M { oneof x { Inner a = 1; int32 b = 2; } oneof y { Inner c = 3; int32 d = 4; } } + """.utf8)) + // Hand-written: `c` holding a truncated varint, `a` holding a hundred `v: 1`, then `b: 1` and `d: 1`. + let fixture = "1a0108" + "0ac801" + String(repeating: "0801", count: 100) + "1001" + "2001" + for _ in 0..<20 { + #expect(throws: BinaryValueDecodeError.self) { + try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "wo.M", maxValues: 6) + } + } + } + @Test("The package qualifies the definitions written before it too") func packageAfterMessage() throws { let schema = try ProtobufSchema.parse(Data(""" diff --git a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift index 82405f5599..2e50b423fe 100644 --- a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift +++ b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift @@ -7,6 +7,7 @@ import Foundation import os import TableProTestSupport @testable import TablePro +import TableProPluginKit import Testing // Recorded from confluentinc/cp-schema-registry 7.7.1. The values were written by confluent-kafka 2.6.1's @@ -355,6 +356,49 @@ struct SchemaRegistryTests { await gate.open() } + @Test("A schema one connection loads is announced to every connection on that registry") + @MainActor + func loadReachesEveryConnectionOnTheEndpoint() async throws { + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: RecordedTransport().transport) + let first = UUID() + let second = UUID() + _ = await cache.schema(connectionID: second, schemaID: 99_999) + let (changes, observer) = Self.changes(for: second) + defer { NotificationCenter.default.removeObserver(observer) } + + _ = await cache.schema(connectionID: first, schemaID: 1) + var iterator = changes.makeAsyncIterator() + _ = await iterator.next() + guard case .success? = cache.cachedSchema(connectionID: second, schemaID: 1) else { + Issue.record("expected the second connection to read the schema the first loaded") + return + } + } + + private final class OlderKafkaPlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Kafka" + static let pluginVersion = "1.0.0" + static let pluginDescription = "A Kafka bundle from before the Schema Registry fields" + static let capabilities: [PluginCapability] = [.databaseDriver] + static let databaseTypeId = "Kafka" + static let databaseDisplayName = "Kafka" + static let iconName = "kafka-icon" + static let defaultPort = 9_092 + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + fatalError("Not used in tests") + } + } + + @Test("A Kafka bundle from before the registry fields keeps them, the password among the secure fields") + @MainActor + func olderPluginKeepsRegistryFields() { + let built = PluginMetadataRegistry.shared.buildMetadataSnapshot(from: OlderKafkaPlugin.self) + let fields = built.connection.additionalConnectionFields + #expect(Set(fields.map(\.id)).isSuperset(of: SchemaRegistryConnectionField.all)) + #expect(fields.first { $0.id == SchemaRegistryConnectionField.password }?.isSecure == true) + } + @Test("A connection with no registry URL says so instead of fetching") @MainActor func noEndpoint() async { From c52d32ff4020801f27ba7531db2b6660027b5baf Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Mon, 2 Nov 2026 01:11:28 +0700 Subject: [PATCH 22/22] fix(plugin-kafka): keep the Schema Registry fields under Authentication so the password reads back from the Keychain --- apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift | 6 +++--- .../Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift b/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift index d4265ada9c..5fb0d05fee 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift @@ -212,20 +212,20 @@ enum KafkaConnectionField { label: String(localized: "Schema Registry URL"), placeholder: "http://localhost:8081", required: false, - section: .advanced + section: .authentication ), ConnectionField( id: schemaRegistryUsername, label: String(localized: "Schema Registry User"), required: false, - section: .advanced + section: .authentication ), ConnectionField( id: schemaRegistryPassword, label: String(localized: "Schema Registry Password"), required: false, fieldType: .secure, - section: .advanced + section: .authentication ) ] } diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift index 1d5bfe0927..3ed9a9ab7c 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift @@ -147,20 +147,20 @@ private func kafkaConnectionFields() -> [ConnectionField] { label: String(localized: "Schema Registry URL"), placeholder: "http://localhost:8081", required: false, - section: .advanced + section: .authentication ), ConnectionField( id: SchemaRegistryConnectionField.username, label: String(localized: "Schema Registry User"), required: false, - section: .advanced + section: .authentication ), ConnectionField( id: SchemaRegistryConnectionField.password, label: String(localized: "Schema Registry Password"), required: false, fieldType: .secure, - section: .advanced + section: .authentication ) ] }