diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index dbda3557e7..13d36f3be0 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -70,6 +70,9 @@ "ParquetExport": { "name": "Parquet Export" }, + "QdrantDriverPlugin": { + "name": "Qdrant Driver" + }, "SnowflakeDriverPlugin": { "name": "Snowflake Driver" }, diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 53d96abf4e..4369056b17 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -37,6 +37,7 @@ let package = Package( .library(name: "TableProGoogleCloud", targets: ["TableProGoogleCloud"]), .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]), .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), + .library(name: "TableProQdrantCore", targets: ["TableProQdrantCore"]), .library(name: "TableProInfluxDBCore", targets: ["TableProInfluxDBCore"]), .library(name: "TableProNeo4jCore", targets: ["TableProNeo4jCore"]), .library(name: "TableProFirebaseCore", targets: ["TableProFirebaseCore"]), @@ -156,7 +157,7 @@ let package = Package( ), .target( name: "TableProTrinoCore", - dependencies: ["TableProSQLGrammar"], + dependencies: ["TableProSQLGrammar", "TableProTLSClientIdentity"], path: "Sources/TableProTrinoCore" ), .target( @@ -195,6 +196,12 @@ let package = Package( dependencies: [], path: "Sources/TableProWeaviateCore" ), + .target( + name: "TableProQdrantCore", + dependencies: [], + path: "Sources/TableProQdrantCore", + swiftSettings: pluginCoreSettings + ), .target( name: "TableProInfluxDBCore", dependencies: [], @@ -319,6 +326,12 @@ let package = Package( dependencies: ["TableProTeradataCore"], path: "Tests/TableProTeradataCoreTests" ), + .testTarget( + name: "TableProQdrantCoreTests", + dependencies: ["TableProQdrantCore"], + path: "Tests/TableProQdrantCoreTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProInfluxDBCoreTests", dependencies: ["TableProInfluxDBCore"], diff --git a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift index e6f427c686..2e44a14d5b 100644 --- a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift +++ b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift @@ -36,6 +36,7 @@ public enum SQLDDLFallbackPolicy { "Elasticsearch", "Typesense", "Weaviate", + "Qdrant", "Memcached", "Kafka", "etcd", diff --git a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift index b9b9da2621..d3cf5e8255 100644 --- a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift +++ b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift @@ -43,6 +43,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let kafka = DatabaseType(rawValue: "Kafka") public static let cloudflareR2SQL = DatabaseType(rawValue: "Cloudflare R2 SQL") public static let weaviate = DatabaseType(rawValue: "Weaviate") + public static let qdrant = DatabaseType(rawValue: "Qdrant") public static let memcached = DatabaseType(rawValue: "Memcached") public static let firestore = DatabaseType(rawValue: "Firestore") public static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") @@ -52,7 +53,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { .mysql, .mariadb, .tidb, .databend, .oceanbase, .starrocks, .postgresql, .sqlite, .redis, .mongodb, .clickhouse, .mssql, .oracle, .dameng, .duckdb, .cassandra, .redshift, .etcd, .cloudflareD1, .dynamodb, .bigquery, .spanner, .snowflake, .libsql, .beancount, - .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .memcached, .firestore, .firebaseRealtimeDatabase, .bigtable + .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .qdrant, .memcached, .firestore, .firebaseRealtimeDatabase, .bigtable ] /// Icon name for this database type — asset catalog name (e.g. "mysql-icon") or SF Symbol fallback @@ -90,6 +91,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .kafka: return "kafka-icon" case .cloudflareR2SQL: return "cloudflare-r2-sql-icon" case .weaviate: return "weaviate-icon" + case .qdrant: return "qdrant-icon" case .memcached: return "memorychip" case .firestore, .firebaseRealtimeDatabase: return "firebase-icon" case .bigtable: return "bigtable-icon" diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift new file mode 100644 index 0000000000..c2c5ddefab --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift @@ -0,0 +1,115 @@ +import Foundation + +public struct QdrantSortSpec: Sendable, Equatable { + public let column: String + public let ascending: Bool + + public init(column: String, ascending: Bool) { + self.column = column + self.ascending = ascending + } +} + +public struct QdrantBrowseRequest: Sendable, Equatable { + public let collection: String + public let offset: Int + public let limit: Int + public let sorts: [QdrantSortSpec] + public let filters: [QdrantFilterSpec] + public let logicMode: String + + public init( + collection: String, + offset: Int, + limit: Int, + sorts: [QdrantSortSpec], + filters: [QdrantFilterSpec], + logicMode: String + ) { + self.collection = collection + self.offset = max(offset, 0) + self.limit = max(limit, 1) + self.sorts = sorts + self.filters = filters + self.logicMode = logicMode + } +} + +// The page is built before the driver knows the field types the filter JSON depends on, so it travels +// from `buildFilteredQuery` to `execute` as tagged text. +public enum QdrantBrowseQuery { + public static let tag = "QDRANT_BROWSE:" + + public static func encode(_ request: QdrantBrowseRequest) -> String { + let payload: [String: Any] = [ + "collection": request.collection, + "offset": request.offset, + "limit": request.limit, + "logicMode": request.logicMode, + "sorts": request.sorts.map { ["column": $0.column, "ascending": $0.ascending] }, + "filters": request.filters.map { filter -> [String: Any] in + var encoded: [String: Any] = ["column": filter.column, "op": filter.op, "value": filter.value] + if let secondValue = filter.secondValue { + encoded["secondValue"] = secondValue + } + return encoded + } + ] + let body = (try? QdrantJSON.data(payload)) ?? Data() + return tag + body.base64EncodedString() + } + + public static func parse(_ query: String) -> QdrantBrowseRequest? { + guard query.hasPrefix(tag), + let data = Data(base64Encoded: String(query.dropFirst(tag.count))), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let collection = json["collection"] as? String + else { return nil } + let sorts = (json["sorts"] as? [[String: Any]] ?? []).compactMap { item -> QdrantSortSpec? in + guard let column = item["column"] as? String else { return nil } + return QdrantSortSpec(column: column, ascending: item["ascending"] as? Bool ?? true) + } + let filters = (json["filters"] as? [[String: Any]] ?? []).compactMap { item -> QdrantFilterSpec? in + guard let column = item["column"] as? String, let op = item["op"] as? String else { return nil } + return QdrantFilterSpec( + column: column, + op: op, + value: item["value"] as? String ?? "", + secondValue: item["secondValue"] as? String + ) + } + return QdrantBrowseRequest( + collection: collection, + offset: json["offset"] as? Int ?? 0, + limit: json["limit"] as? Int ?? 100, + sorts: sorts, + filters: filters, + logicMode: json["logicMode"] as? String ?? "AND" + ) + } +} + +public enum QdrantSortPlan: Sendable, Equatable { + case byID + case byPayload(column: String, key: String, ascending: Bool) + + public static func resolve(_ sorts: [QdrantSortSpec]) throws -> QdrantSortPlan { + guard let sort = sorts.first else { return .byID } + guard sorts.count == 1 else { + throw QdrantError.unsupported(String(localized: "Qdrant sorts by one field at a time. Remove the sort from the other columns.")) + } + if sort.column == QdrantSchema.idColumn { + guard sort.ascending else { + throw QdrantError.unsupported(String(localized: "Qdrant lists points by id in ascending order only.")) + } + return .byID + } + guard !QdrantSchema.isVectorColumn(sort.column), sort.column != QdrantSchema.scoreColumn else { + throw QdrantError.unsupported(String(format: String(localized: "Qdrant cannot sort by %@."), sort.column)) + } + guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: sort.column)) else { + throw QdrantFilterError.keyNotAddressable(column: sort.column) + } + return .byPayload(column: sort.column, key: key, ascending: sort.ascending) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift new file mode 100644 index 0000000000..63e3d523e6 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift @@ -0,0 +1,53 @@ +import Foundation + +public enum QdrantFieldID { + public static let apiKey = "qdApiKey" +} + +public struct QdrantConnectionSettings: Sendable, Equatable { + public static let defaultPort = 6_333 + + public let host: String + public let port: Int + public let usesTLS: Bool + public let apiKey: String + + public init(host: String, port: Int, usesTLS: Bool, apiKey: String) { + self.host = host.trimmingCharacters(in: .whitespacesAndNewlines) + self.port = port + self.usesTLS = usesTLS + self.apiKey = apiKey.trimmingCharacters(in: .whitespacesAndNewlines) + } + + public static func parse( + host: String, + port: Int, + usesTLS: Bool, + fields: [String: String] + ) throws -> QdrantConnectionSettings { + let resolvedHost = host.trimmingCharacters(in: .whitespacesAndNewlines) + let settings = QdrantConnectionSettings( + host: resolvedHost.isEmpty ? "localhost" : resolvedHost, + port: port > 0 ? port : defaultPort, + usesTLS: usesTLS, + apiKey: fields[QdrantFieldID.apiKey] ?? "" + ) + _ = try settings.baseURL() + return settings + } + + public var headers: [String: String] { + apiKey.isEmpty ? [:] : ["api-key": apiKey] + } + + public func baseURL() throws -> URL { + var components = URLComponents() + components.scheme = usesTLS ? "https" : "http" + components.host = host + components.port = port + guard let url = components.url else { + throw QdrantError.configuration(String(localized: "The host is not valid in a URL.")) + } + return url + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift new file mode 100644 index 0000000000..01945bf62a --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift @@ -0,0 +1,176 @@ +import Foundation + +public struct QdrantConsoleRequest: Sendable, Equatable { + public let method: String + public let path: String + public let body: String? + + public init(method: String, path: String, body: String?) { + self.method = method + self.path = path + self.body = body + } + + // What the review sheet shows, the classifier tiers and the driver runs are this one string. + public var text: String { + guard let body else { return "\(method) \(path)" } + return "\(method) \(path)\n\(body)" + } + + public var kind: QdrantRequestKind { QdrantRequestKind.of(method: method, path: path) } +} + +// `METHOD /path`, then an optional JSON body that may start on the request line and go on below it. +public enum QdrantConsoleParser { + static let methods: Set = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"] + + public static func parse(_ input: String) -> QdrantConsoleRequest? { + let trimmed = input.trimmingCharacters(in: .whitespacesAndNewlines) + let newline = trimmed.firstIndex(where: \.isNewline) + let header = newline.map { trimmed[..<$0] } ?? trimmed[...] + let following = newline.map { trimmed[trimmed.index(after: $0)...] } ?? "" + + let parts = header.split(maxSplits: 2, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) + guard parts.count >= 2 else { return nil } + let method = String(parts[0]).uppercased() + guard methods.contains(method) else { return nil } + let path = String(parts[1]) + guard path.hasPrefix("/"), !path.hasPrefix("//") else { return nil } + let inline = parts.count > 2 ? parts[2] : "" + let body = [inline, following] + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .joined(separator: "\n") + return QdrantConsoleRequest(method: method, path: path, body: body.isEmpty ? nil : body) + } +} + +public struct QdrantRequestKind: Sendable, Equatable { + public enum Tier: Sendable, Equatable { + case read + case write + case remove + } + + public let tier: Tier + // A snapshot is written to the server's disk, and recovering one fetches a URL from the server. + public let touchesServerFiles: Bool + + public init(tier: Tier, touchesServerFiles: Bool) { + self.tier = tier + self.touchesServerFiles = touchesServerFiles + } + + // The POSTs under `/collections//` that only read. Retrieving points by id is a POST to `points`. + static let readRoutes: Set<[String]> = [ + ["points"], ["points", "scroll"], ["points", "search"], ["points", "query"], ["points", "count"], + ["points", "recommend"], ["points", "discover"], ["facet"], + ["points", "search", "batch"], ["points", "query", "batch"], ["points", "recommend", "batch"], + ["points", "discover", "batch"], ["points", "search", "groups"], ["points", "query", "groups"], + ["points", "recommend", "groups"], ["points", "search", "matrix", "pairs"], + ["points", "search", "matrix", "offsets"] + ] + + // `/snapshots`, `/collections/{name}/snapshots` and `/collections/{name}/shards/{id}/snapshots`, by + // position, so a collection that is itself named `snapshots` is not one. + static func touchesSnapshots(_ route: [String]) -> Bool { + if route.first == "snapshots" { return true } + guard route.count >= 3, route[0] == "collections" else { return false } + return route[2] == "snapshots" || (route.count >= 5 && route[2] == "shards" && route[4] == "snapshots") + } + + public static func of(method: String, path: String) -> QdrantRequestKind { + let verb = method.uppercased() + guard let route = QdrantPathEncoding.segments(of: path) else { + return QdrantRequestKind(tier: .write, touchesServerFiles: true) + } + let touchesServerFiles = touchesSnapshots(route) + if verb == "GET" || verb == "HEAD" { + return QdrantRequestKind(tier: .read, touchesServerFiles: touchesServerFiles) + } + let subroute = route.count >= 3 && route[0] == "collections" ? Array(route.dropFirst(2)) : [] + if verb == "POST", readRoutes.contains(subroute) { + return QdrantRequestKind(tier: .read, touchesServerFiles: touchesServerFiles) + } + // A batch can carry deletes, and a POST ending in `delete` or `clear` removes points or payload. Some + // routes delete through the body: a PUT of the payload replaces it whole, the cluster route drops + // shard keys and replicas, and the aliases route deletes aliases. + let removes = verb == "DELETE" || route.last == "delete" || route.last == "clear" + || subroute == ["points", "batch"] + || (verb == "PUT" && subroute == ["points", "payload"]) + || subroute.first == "cluster" + || route == ["collections", "aliases"] + return QdrantRequestKind(tier: removes ? .remove : .write, touchesServerFiles: touchesServerFiles) + } +} + +public enum QdrantPathEncoding { + private static let allowed: CharacterSet = { + var set = CharacterSet.alphanumerics + set.insert(charactersIn: "-_.") + return set + }() + + public static func segment(_ value: String) -> String { + value.addingPercentEncoding(withAllowedCharacters: allowed) ?? value + } + + // Decoded and lowercased; nil for a dot segment, which would move the request to a route other than + // the one it reads as. + public static func segments(of path: String) -> [String]? { + guard let requested = URLComponents(string: path) else { return nil } + let segments = requested.percentEncodedPath.split(separator: "/").map { String($0) } + var decoded: [String] = [] + for segment in segments { + guard let text = segment.removingPercentEncoding, text != ".", text != "..", !text.contains("/") else { + return nil + } + decoded.append(text.lowercased()) + } + return decoded + } + + // Assigning the path whole to `URLComponents.path` would encode the `?` of `?wait=true`. + public static func resolve(_ path: String, against base: URL) -> URL? { + guard path.hasPrefix("/"), !path.hasPrefix("//"), !path.contains("://"), + segments(of: path) != nil, + var components = URLComponents(url: base, resolvingAgainstBaseURL: false), + let requested = URLComponents(string: path) + else { return nil } + components.percentEncodedPath = requested.percentEncodedPath + components.percentEncodedQuery = requested.percentEncodedQuery + return components.url + } +} + +public enum QdrantOperations { + public static let exportTag = "QDRANT_EXPORT:" + + public static func deleteCollection(named name: String, objectType: String) -> String? { + guard objectType.uppercased() == "TABLE", !name.isEmpty else { return nil } + return "DELETE /collections/\(QdrantPathEncoding.segment(name))" + } + + // An empty filter matches every point and keeps the collection with its config and indexes. + public static func deleteAllPoints(collection: String) -> String? { + guard !collection.isEmpty else { return nil } + return "POST /collections/\(QdrantPathEncoding.segment(collection))/points/delete?wait=true\n{\"filter\":{}}" + } + + public static func encodeExport(collection: String) -> String { + exportTag + Data(collection.utf8).base64EncodedString() + } + + public static func decodeExport(_ query: String) -> String? { + guard query.hasPrefix(exportTag), + let data = Data(base64Encoded: String(query.dropFirst(exportTag.count))), + let collection = String(data: data, encoding: .utf8), + !collection.isEmpty + else { return nil } + return collection + } + + public static func isTagged(_ query: String) -> Bool { + query.hasPrefix(QdrantBrowseQuery.tag) || query.hasPrefix(exportTag) || query.hasPrefix(QdrantInsertCheck.tag) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift new file mode 100644 index 0000000000..a38693615b --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift @@ -0,0 +1,54 @@ +import Foundation + +public enum QdrantError: Error, LocalizedError, Equatable, Sendable { + case configuration(String) + case notConnected + case transport(String) + case authentication(String) + case api(status: Int, message: String) + case malformedResponse(String) + case unsupported(String) + case cancelled + + public var errorDescription: String? { + switch self { + case .configuration(let detail), .transport(let detail), .malformedResponse(let detail), + .authentication(let detail), .unsupported(let detail): + return detail + case .notConnected: + return String(localized: "Not connected to Qdrant.") + case .api(_, let message): + return message + case .cancelled: + return String(localized: "The request was cancelled.") + } + } + + public static func from(status: Int, body: Data) -> QdrantError { + let message = apiMessage(from: body) + ?? String(format: String(localized: "Qdrant returned HTTP %d."), status) + if status == 401 || status == 403 { + return .authentication(message) + } + return .api(status: status, message: message) + } + + // A proxy in front of Qdrant may answer with plain text instead of `status.error`. + public static func apiMessage(from body: Data) -> String? { + guard let json = try? JSONSerialization.jsonObject(with: body) else { + let text = String(data: body, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) + return (text?.isEmpty ?? true) ? nil : text + } + guard let object = json as? [String: Any] else { return nil } + if let status = object["status"] as? [String: Any], + let error = status["error"] as? String, !error.isEmpty { + return error + } + for key in ["error", "message"] { + if let text = object[key] as? String, !text.isEmpty { + return text + } + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift new file mode 100644 index 0000000000..8b20a24cde --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift @@ -0,0 +1,250 @@ +import Foundation + +public struct QdrantFilterSpec: Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + + public init(column: String, op: String, value: String, secondValue: String? = nil) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + } +} + +public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { + case unsupportedOperator(String) + case missingUpperBound(column: String) + case notAPointID(value: String) + case emptyList(column: String) + case vectorNotFilterable(column: String) + case keyNotAddressable(column: String) + case boundNotExact(column: String, value: String) + case rawFilterNotAnObject + + public var errorDescription: String? { + switch self { + case .unsupportedOperator(let op): + return String(format: String(localized: "Qdrant cannot filter with %@."), op) + case .missingUpperBound(let column): + return String(format: String(localized: "BETWEEN on %@ needs an upper bound."), column) + case .notAPointID(let value): + return String( + format: String(localized: "%@ is not a point id. Qdrant ids are unsigned integers or UUIDs."), + value + ) + case .emptyList(let column): + return String(format: String(localized: "IN on %@ needs at least one value."), column) + case .vectorNotFilterable(let column): + return String(format: String(localized: "Qdrant cannot filter on %@."), column) + case .keyNotAddressable(let column): + return String( + format: String(localized: "Qdrant cannot filter on %@, because a key holding a double quote has no path it accepts."), + column + ) + case .boundNotExact(let column, let value): + return String( + format: String(localized: "Qdrant compares %@ as a 64-bit float, which cannot hold %@ exactly."), + column, value + ) + case .rawFilterNotAnObject: + return String(localized: "A raw filter for Qdrant is a JSON condition, such as {\"key\": \"city\", \"match\": {\"value\": \"Berlin\"}}.") + } + } +} + +// AND rows go under `must` and OR rows under `should`. A negated row is a nested filter with `must_not`, +// which, like `$ne` in a document store, also matches a point the key is missing from. +// +// A payload index does not constrain what points store, and a sample cannot prove it does, so a value +// matches every type its text can spell: `42` finds "42", 42, 42.0 and a list holding either. The +// index still serves the branch of its own type. +public enum QdrantFilterBuilder { + public static let rawFilterColumn = "__RAW__" + + public static func filter(_ filters: [QdrantFilterSpec], logicMode: String) throws -> [String: Any]? { + guard !filters.isEmpty else { return nil } + let conditions = try filters.map(condition(for:)) + return [logicMode.uppercased() == "OR" ? "should" : "must": conditions] + } + + public static func condition(for filter: QdrantFilterSpec) throws -> [String: Any] { + if filter.column == rawFilterColumn { + return try raw(filter.value) + } + let op = filter.op.uppercased() + guard !QdrantSchema.isVectorColumn(filter.column) else { + // Search All Rows asks every column; a vector holds no text, so it finds nothing there. + if op == "CONTAINS" { return ["has_id": [Any]()] } + if op == "NOT CONTAINS" { return negated([["has_id": [Any]()]]) } + throw QdrantFilterError.vectorNotFilterable(column: filter.column) + } + if filter.column == QdrantSchema.idColumn { + if op == "CONTAINS" || op == "NOT CONTAINS" { + return idContains(op: op, value: filter.value) + } + return try idCondition(op: op, value: filter.value) + } + guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: filter.column)) else { + throw QdrantFilterError.keyNotAddressable(column: filter.column) + } + let field = Field(column: filter.column, key: key) + + switch op { + case "=": + return field.equality(filter.value) + case "!=", "<>": + return negated([field.equality(filter.value)]) + case ">", ">=", "<", "<=": + return try field.range([rangeKey(op): filter.value]) + case "BETWEEN": + guard let upper = filter.secondValue, !upper.trimmingCharacters(in: .whitespaces).isEmpty else { + throw QdrantFilterError.missingUpperBound(column: filter.column) + } + return try field.range(["gte": lowerBound(of: filter.value, upperBound: upper), "lte": upper]) + case "IN": + return field.anyOf(try list(filter.value, column: filter.column)) + case "NOT IN": + return negated(try list(filter.value, column: filter.column).map(field.equality)) + case "CONTAINS": + return field.textMatch(filter.value) + case "NOT CONTAINS": + return negated([field.textMatch(filter.value)]) + case "IS NULL": + return field.isEmpty + case "IS NOT NULL": + return negated([field.isEmpty]) + case "IS EMPTY": + return field.isBlank + case "IS NOT EMPTY": + return negated([field.isBlank]) + default: + throw QdrantFilterError.unsupportedOperator(op) + } + } + + private struct Field { + let column: String + let key: String + + func equality(_ value: String) -> [String: Any] { + let trimmed = value.trimmingCharacters(in: .whitespaces) + var options = [match(value)] + // A range also finds 42.0, but a double rounds an integer past 2^53, so a larger one matches exactly. + if let integer = QdrantValue.integer(trimmed), integer.magnitude > 1 << 53 { + options.append(match(NSNumber(value: integer))) + } else if let number = QdrantValue.number(trimmed) { + options.append(["key": key, "range": ["gte": NSNumber(value: number), "lte": NSNumber(value: number)]]) + } + if let flag = QdrantValue.bool(trimmed) { + options.append(match(flag)) + } + return options.count == 1 ? options[0] : ["should": options] + } + + // Qdrant's `match.any` takes strings alone, so it collapses only a list of plain text. + func anyOf(_ values: [String]) -> [String: Any] { + let conditions = values.map(equality) + let texts = conditions.compactMap { ($0["match"] as? [String: Any])?["value"] as? String } + if texts.count == conditions.count { + return ["key": key, "match": ["any": texts]] + } + return ["should": conditions] + } + + // A number bounds numbers; any other text is a date, which Qdrant parses. Qdrant reads a bound as + // a 64-bit float, so an integer past 2^53 would be rounded to a different cutoff. + func range(_ bounds: [String: String]) throws -> [String: Any] { + var range: [String: Any] = [:] + for (bound, text) in bounds { + let trimmed = text.trimmingCharacters(in: .whitespaces) + if let integer = QdrantValue.integer(trimmed), integer.magnitude > 1 << 53 { + throw QdrantFilterError.boundNotExact(column: column, value: trimmed) + } + range[bound] = QdrantValue.number(trimmed).map { NSNumber(value: $0) as Any } ?? trimmed + } + return ["key": key, "range": range] + } + + // Without a full-text index Qdrant still matches the words, lowercased, and a value that is not + // text matches nothing rather than failing, so Search All Rows can ask every column. + func textMatch(_ value: String) -> [String: Any] { + ["key": key, "match": ["text": value]] + } + + // `is_empty` covers a missing key, null and []; `is_null` would miss the missing key the grid shows as NULL. + var isEmpty: [String: Any] { + ["is_empty": ["key": key]] + } + + var isBlank: [String: Any] { + ["should": [isEmpty, match("")]] + } + + private func match(_ value: Any) -> [String: Any] { + ["key": key, "match": ["value": value]] + } + } + + private static func raw(_ text: String) throws -> [String: Any] { + guard let object = QdrantJSON.parse(text) as? [String: Any] else { + throw QdrantFilterError.rawFilterNotAnObject + } + return object + } + + private static func idCondition(op: String, value: String) throws -> [String: Any] { + let parts = value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: QdrantSchema.idColumn) } + let ids = try parts.map { part -> Any in + guard let id = QdrantPointID.json(from: part) else { throw QdrantFilterError.notAPointID(value: part) } + return id + } + switch op { + case "=", "IN": + return ["has_id": ids] + case "!=", "<>", "NOT IN": + return negated([["has_id": ids]]) + default: + throw QdrantFilterError.unsupportedOperator(op) + } + } + + // Ids have no substring search, so only a whole id is found; Search All Rows still runs on the other columns. + private static func idContains(op: String, value: String) -> [String: Any] { + let found: [Any] = QdrantPointID.json(from: value).map { [$0] } ?? [] + return op == "NOT CONTAINS" ? negated([["has_id": found]]) : ["has_id": found] + } + + private static func list(_ value: String, column: String) throws -> [String] { + let parts = value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: column) } + return parts + } + + private static func rangeKey(_ op: String) -> String { + switch op { + case ">": return "gt" + case ">=": return "gte" + case "<": return "lt" + default: return "lte" + } + } + + private static func lowerBound(of joinedValue: String, upperBound: String) -> String { + let suffix = "," + upperBound + guard joinedValue.hasSuffix(suffix) else { return joinedValue } + return String(joinedValue.dropLast(suffix.count)) + } + + static func negated(_ conditions: [[String: Any]]) -> [String: Any] { + ["must_not": conditions] + } + + static func allOf(_ filter: [String: Any]?, _ condition: [String: Any]) -> [String: Any] { + guard let filter else { return ["must": [condition]] } + return ["must": [filter, condition]] + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift new file mode 100644 index 0000000000..57192f0dec --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift @@ -0,0 +1,47 @@ +import Foundation + +public enum QdrantJSON { + public static func data(_ object: Any, pretty: Bool = false) throws -> Data { + guard JSONSerialization.isValidJSONObject(object) else { + throw QdrantError.malformedResponse(String(localized: "Request body is not valid JSON.")) + } + var options: JSONSerialization.WritingOptions = [.sortedKeys, .withoutEscapingSlashes] + if pretty { + options.insert(.prettyPrinted) + } + return try JSONSerialization.data(withJSONObject: object, options: options) + } + + public static func text(_ object: Any, pretty: Bool = false) throws -> String { + String(data: try data(object, pretty: pretty), encoding: .utf8) ?? "{}" + } + + public static func parse(_ text: String) -> Any? { + guard let data = text.trimmingCharacters(in: .whitespacesAndNewlines).data(using: .utf8) else { return nil } + return try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) + } + + public static func isBoolean(_ number: NSNumber) -> Bool { + CFGetTypeID(number) == CFBooleanGetTypeID() + } + + public static func displayText(_ value: Any?) -> String? { + switch value { + case nil, is NSNull: + return nil + case let text as String: + return text + case let number as NSNumber: + if isBoolean(number) { + return number.boolValue ? "true" : "false" + } + return number.stringValue + case let object as [String: Any]: + return try? text(object) + case let array as [Any]: + return try? text(array) + default: + return String(describing: value as Any) + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift new file mode 100644 index 0000000000..b725bd810c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift @@ -0,0 +1,119 @@ +import Foundation + +public enum QdrantPointID { + // Qdrant refuses an integer id spelled as a string and the other way round. + public static func json(from text: String) -> Any? { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + if trimmed.allSatisfy(\.isASCII), trimmed.allSatisfy(\.isNumber), let number = UInt64(trimmed) { + return NSNumber(value: number) + } + guard UUID(uuidString: trimmed) != nil else { return nil } + return trimmed.lowercased() + } +} + +public struct QdrantPoint: @unchecked Sendable { + public let id: Any? + public let payload: [String: Any] + // Nil when the request left vectors out, an array for an unnamed vector alone, a map by name otherwise. + public let vector: Any? + public let score: Double? + + public init(id: Any?, payload: [String: Any], vector: Any?, score: Double?) { + self.id = id + self.payload = payload + self.vector = vector + self.score = score + } + + public var idText: String? { QdrantJSON.displayText(id) } + + public var payloadKeys: [String] { payload.keys.sorted() } + + public static func parse(_ json: Any) -> QdrantPoint? { + guard let object = json as? [String: Any], object.keys.contains("id") else { return nil } + let vector = object["vector"] ?? object["vectors"] + return QdrantPoint( + id: object["id"], + payload: object["payload"] as? [String: Any] ?? [:], + vector: vector is NSNull ? nil : vector, + score: (object["score"] as? NSNumber)?.doubleValue + ) + } + + // `scroll` and `query` wrap the list in `points`; retrieve and `search` answer the array itself. + public static func list(fromResponse json: Any?) -> [QdrantPoint]? { + guard let result = (json as? [String: Any])?["result"] else { return nil } + let items: [Any] + if let wrapped = (result as? [String: Any])?["points"] as? [Any] { + items = wrapped + } else if let array = result as? [Any] { + items = array + } else { + return nil + } + let points = items.compactMap(parse) + return points.count == items.count ? points : nil + } + + public func vectorValue(forColumn column: String) -> Any? { + guard let vector, let name = QdrantSchema.vectorName(ofColumn: column) else { return nil } + if name.isEmpty, vector is [Any] { return vector } + return (vector as? [String: Any])?[name] + } + + // A column holding `"42"` in one point and `42` in another is typed `json`, so its strings keep their + // quotes: without them both cells read 42 and a save would turn the string into a number. + public func cell(for column: String, quotesText: Bool) -> String? { + guard quotesText, !QdrantSchema.isPointColumn(column), + let text = payload[QdrantSchema.payloadKey(forColumn: column)] as? String + else { + return cell(for: column) + } + return (try? JSONSerialization.data(withJSONObject: text, options: [.fragmentsAllowed, .withoutEscapingSlashes])) + .flatMap { String(data: $0, encoding: .utf8) } + } + + public func cell(for column: String) -> String? { + if column == QdrantSchema.idColumn { + return idText + } + if QdrantSchema.isVectorColumn(column) { + return QdrantJSON.displayText(vectorValue(forColumn: column)) + } + if column == QdrantSchema.scoreColumn { + return score.map { String($0) } + } + return QdrantJSON.displayText(payload[QdrantSchema.payloadKey(forColumn: column)]) + } + + public func row(columns: [String]) -> [String?] { + columns.map(cell(for:)) + } + + public func row(columns: [String], quotingTextIn jsonColumns: Set) -> [String?] { + columns.map { cell(for: $0, quotesText: jsonColumns.contains($0)) } + } + + public static func columns(of points: [QdrantPoint], collection: QdrantCollection?) -> [String] { + var columns = [QdrantSchema.idColumn] + if points.contains(where: { $0.score != nil }) { + columns.append(QdrantSchema.scoreColumn) + } + let returned = Set(points.flatMap(\.payloadKeys).map(QdrantSchema.column(forPayloadKey:))) + let declared = collection?.payloadFields.map { QdrantSchema.column(forPayloadKey: $0.name) } ?? [] + columns += declared.filter { returned.contains($0) && !columns.contains($0) } + columns += returned.subtracting(declared).subtracting(columns).sorted() + + var vectorColumns: Set = [] + for point in points { + if point.vector is [Any] { + vectorColumns.insert(QdrantSchema.vectorColumn) + } else if let named = point.vector as? [String: Any] { + vectorColumns.formUnion(named.keys.map { QdrantVectorSpec(name: $0, kind: .dense(size: nil)).column }) + } + } + return columns + vectorColumns.subtracting(columns).sorted() + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift new file mode 100644 index 0000000000..ce87b6ac90 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift @@ -0,0 +1,237 @@ +import Foundation + +public struct QdrantCell: Sendable, Equatable { + public let column: String + public let text: String? + + public init(column: String, text: String?) { + self.column = column + self.text = text + } +} + +public struct QdrantRowChange: Sendable, Equatable { + public enum Kind: Sendable, Equatable { + case insert + case update + case delete + } + + public let kind: Kind + public let rowIndex: Int + public let id: String? + // An insert's every column, or an update's changed cells. + public let cells: [QdrantCell] + + public init(kind: Kind, rowIndex: Int, id: String?, cells: [QdrantCell] = []) { + self.kind = kind + self.rowIndex = rowIndex + self.id = id + self.cells = cells + } +} + +public struct QdrantRowWrite: Sendable, Equatable { + public let request: QdrantConsoleRequest + public let rowIndex: Int + // A read that must find no point before the write runs: an insert that leaves a taken id alone + // still answers success, so without it the save would report a row it never added. + public let mustFindNothing: QdrantConsoleRequest? + // Run right after the write: a row back means the point under the id is not the one written, which + // happens when another client takes the id between the lookup and the write. + public let landedCheck: String? +} + +public enum QdrantInsertCheck { + public static let tag = "QDRANT_INSERT_CHECK:" + + public static func encode(collection: String, id: Any, payload: [String: Any], vectors: [String: Any]) -> String? { + let spec: [String: Any] = ["collection": collection, "id": id, "payload": payload, "vectors": vectors] + guard let data = try? QdrantJSON.data(spec) else { return nil } + return tag + data.base64EncodedString() + } + + public static func decode(_ text: String) -> (collection: String, id: Any, payload: [String: Any], vectors: [String: Any])? { + guard text.hasPrefix(tag), + let data = Data(base64Encoded: String(text.dropFirst(tag.count))), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let collection = object["collection"] as? String, let id = object["id"], + let payload = object["payload"] as? [String: Any] + else { return nil } + return (collection, id, payload, object["vectors"] as? [String: Any] ?? [:]) + } + + // The payload and which vectors the point holds. Vector values are not compared: Qdrant sorts a + // sparse vector's indices, stores float16 and uint8 vectors rounded and cosine vectors normalized, so + // a vector written exactly reads back different. + public static func landed(_ point: QdrantPoint?, payload: [String: Any], vectors: [String: Any]) -> Bool { + guard let point, (try? QdrantJSON.text(point.payload)) == (try? QdrantJSON.text(payload)) else { return false } + let names: Set + switch point.vector { + case is [Any]: names = [""] + case let named as [String: Any]: names = Set(named.keys) + default: names = [] + } + return names == Set(vectors.keys) + } +} + +public enum QdrantServerVersion { + public static func hasQueryAPI(_ version: String?) -> Bool { + isAtLeast(version, major: 1, minor: 10) + } + + // Measured: 1.15.5 and later honor `update_filter`; 1.13.6 and 1.14.1 ignore it and overwrite the + // point. `update_mode: insert_only` is ignored the same way on 1.16.3. + public static func insertsWithoutOverwrite(_ version: String?) -> Bool { + isAtLeast(version, major: 1, minor: 15) + } + + static func isAtLeast(_ version: String?, major: Int, minor: Int) -> Bool { + guard let version else { return false } + let parts = version.split(separator: ".").prefix(2).map { Int($0.prefix { $0.isNumber }) ?? 0 } + guard parts.count == 2 else { return false } + return parts[0] > major || (parts[0] == major && parts[1] >= minor) + } +} + +public struct QdrantRowRefusal: Error, LocalizedError, Equatable, Sendable { + public let rowIndex: Int + public let reason: String + + public var errorDescription: String? { reason } +} + +public enum QdrantRowWriter { + public static func writes( + collection: String, + changes: [QdrantRowChange], + serverVersion: String?, + typeName: (String) -> String?, + newID: () -> String = { UUID().uuidString.lowercased() } + ) throws -> [QdrantRowWrite] { + let points = "/collections/\(QdrantPathEncoding.segment(collection))/points" + let insertsSafely = QdrantServerVersion.insertsWithoutOverwrite(serverVersion) + var writes: [QdrantRowWrite] = [] + for change in changes { + do { + if change.kind == .insert, !insertsSafely { + throw Reason(text: String( + format: String(localized: "Qdrant %@ overwrites a point that already has the id. Insert it from the REST editor, or upgrade Qdrant to 1.15 or later."), + serverVersion ?? "?" + )) + } + writes += try self.writes(for: change, collection: collection, points: points, typeName: typeName, newID: newID) + } catch let error as QdrantValueError { + throw QdrantRowRefusal(rowIndex: change.rowIndex, reason: error.errorDescription ?? "") + } catch let reason as Reason { + throw QdrantRowRefusal(rowIndex: change.rowIndex, reason: reason.text) + } + } + return writes + } + + private struct Reason: Error { + let text: String + } + + private static func writes( + for change: QdrantRowChange, + collection: String, + points: String, + typeName: (String) -> String?, + newID: () -> String + ) throws -> [QdrantRowWrite] { + switch change.kind { + case .delete: + let id = try pointID(change.id) + return [try write("POST", "\(points)/delete?wait=true", ["points": [id]], change)] + case .insert: + let cells = change.cells.filter { $0.text != nil && $0.text != defaultMarker } + let idText = cells.first { $0.column == QdrantSchema.idColumn }?.text.flatMap { $0.isEmpty ? nil : $0 } + let id = try pointID(idText ?? newID()) + let values = try typedValues(cells, typeName: typeName) + let point: [String: Any] = ["id": id, "vector": values.vectors, "payload": values.payload] + let lookup = try request("POST", points, ["ids": [id], "with_payload": false, "with_vector": false]) + return [QdrantRowWrite( + request: try request("PUT", "\(points)?wait=true", ["points": [point], "update_filter": Self.matchesNoPoint]), + rowIndex: change.rowIndex, + mustFindNothing: lookup, + landedCheck: QdrantInsertCheck.encode(collection: collection, id: id, payload: values.payload, vectors: values.vectors) + )] + case .update: + let id = try pointID(change.id) + guard !change.cells.contains(where: { $0.column == QdrantSchema.idColumn }) else { + throw Reason(text: String(localized: "A point's id cannot change. Insert a new row with the new id instead.")) + } + let values = try typedValues(change.cells, typeName: typeName) + var writes: [QdrantRowWrite] = [] + if !values.payload.isEmpty { + writes.append(try write("POST", "\(points)/payload?wait=true", ["payload": values.payload, "points": [id]], change)) + } + if !values.vectors.isEmpty { + let body: [String: Any] = ["points": [["id": id, "vector": values.vectors]]] + writes.append(try write("PUT", "\(points)/vectors?wait=true", body, change)) + } + if !values.clearedVectors.isEmpty { + let body: [String: Any] = ["points": [id], "vector": values.clearedVectors] + writes.append(try write("POST", "\(points)/vectors/delete?wait=true", body, change)) + } + return writes + } + } + + // The host's marker for a cell the server fills; a duplicated row carries it in its key. + static let defaultMarker = "__DEFAULT__" + + // An upsert updates only existing points this filter matches, and it matches none. + private static var matchesNoPoint: [String: Any] { ["must": [["has_id": [Any]()]]] } + + private struct TypedValues { + var payload: [String: Any] = [:] + var vectors: [String: Any] = [:] + var clearedVectors: [String] = [] + } + + private static func typedValues(_ cells: [QdrantCell], typeName: (String) -> String?) throws -> TypedValues { + var values = TypedValues() + for cell in cells where cell.column != QdrantSchema.idColumn { + guard let type = typeName(cell.column) else { throw unknownType(cell.column) } + if let name = QdrantSchema.vectorName(ofColumn: cell.column) { + guard let kind = QdrantVectorSpec.kind(ofTypeName: type) else { throw unknownType(cell.column) } + if let text = cell.text { + values.vectors[name] = try QdrantValue.vector(text, kind: kind, column: cell.column) + } else { + values.clearedVectors.append(name) + } + continue + } + guard let fieldType = QdrantFieldType(rawValue: type.lowercased()) else { throw unknownType(cell.column) } + let key = QdrantSchema.payloadKey(forColumn: cell.column) + values.payload[key] = try cell.text.map { try QdrantValue.payload($0, type: fieldType, column: cell.column) } ?? NSNull() + } + return values + } + + private static func unknownType(_ column: String) -> Reason { + Reason(text: String(format: String(localized: "TablePro does not know what %@ holds. Reload the collection, then save again."), column)) + } + + private static func pointID(_ text: String?) throws -> Any { + guard let text, !text.isEmpty else { + throw Reason(text: String(localized: "This row has no point id, so Qdrant cannot tell which point to change.")) + } + guard let id = QdrantPointID.json(from: text) else { + throw Reason(text: String(QdrantFilterError.notAPointID(value: text).errorDescription ?? "")) + } + return id + } + + private static func write(_ method: String, _ path: String, _ body: [String: Any], _ change: QdrantRowChange) throws -> QdrantRowWrite { + QdrantRowWrite(request: try request(method, path, body), rowIndex: change.rowIndex, mustFindNothing: nil, landedCheck: nil) + } + + private static func request(_ method: String, _ path: String, _ body: [String: Any]) throws -> QdrantConsoleRequest { + QdrantConsoleRequest(method: method, path: path, body: try QdrantJSON.text(body)) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift new file mode 100644 index 0000000000..0894a0ae07 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -0,0 +1,378 @@ +import Foundation + +// The first seven are Qdrant's payload index schemas; the rest are read off sampled values. +public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { + case keyword + case integer + case float + case bool + case geo + case text + case datetime + case uuid + case string + case object + case array + case json + + public var isOrdered: Bool { + self == .integer || self == .float || self == .datetime + } + + public var holdsText: Bool { + switch self { + case .keyword, .text, .string, .uuid, .array, .json: return true + default: return false + } + } + + public static func indexed(_ schema: Any?) -> QdrantFieldType? { + let name: String? + if let object = schema as? [String: Any] { + name = object["data_type"] as? String + ?? (object["params"] as? [String: Any])?["type"] as? String + } else { + name = schema as? String + } + return name.flatMap { QdrantFieldType(rawValue: $0.lowercased()) } + } + + public static func observed(_ value: Any) -> QdrantFieldType? { + switch value { + case is NSNull: + return nil + case is String: + return .string + case let number as NSNumber: + if QdrantJSON.isBoolean(number) { return .bool } + return CFNumberIsFloatType(number) ? .float : .integer + case is [String: Any]: + return .object + case is [Any]: + return .array + default: + return .json + } + } + + func stores(_ stored: QdrantFieldType) -> Bool { + switch self { + case .keyword, .text, .uuid, .datetime, .string: return stored == .string + case .integer: return stored == .integer + case .float: return stored == .float || stored == .integer + case .bool: return stored == .bool + case .geo: return stored == .object + case .object, .array, .json: return stored == self + } + } + + func merged(with other: QdrantFieldType) -> QdrantFieldType { + if self == other { return self } + if Set([self, other]) == [.integer, .float] { return .float } + return .json + } +} + +public struct QdrantVectorSpec: Sendable, Equatable { + public enum Kind: Sendable, Equatable { + case dense(size: Int?) + case multi + case sparse + } + + // Empty for the collection's unnamed vector, which Qdrant also accepts under the name "". + public let name: String + public let kind: Kind + + public init(name: String, kind: Kind) { + self.name = name + self.kind = kind + } + + public var column: String { + name.isEmpty ? QdrantSchema.vectorColumn : "\(QdrantSchema.vectorColumn).\(name)" + } + + public var typeName: String { + switch kind { + case .dense(let size?): return "vector(\(size))" + case .dense: return "vector" + case .multi: return "multivector" + case .sparse: return "sparse_vector" + } + } + + public static func kind(ofTypeName typeName: String) -> Kind? { + let lower = typeName.lowercased().trimmingCharacters(in: .whitespaces) + switch lower { + case "sparse_vector": return .sparse + case "multivector": return .multi + case "vector": return .dense(size: nil) + default: + guard lower.hasPrefix("vector("), lower.hasSuffix(")") else { return nil } + return .dense(size: Int(lower.dropFirst("vector(".count).dropLast())) + } + } + + static func dense(name: String, config: [String: Any]) -> QdrantVectorSpec { + if config["multivector_config"] is [String: Any] { + return QdrantVectorSpec(name: name, kind: .multi) + } + return QdrantVectorSpec(name: name, kind: .dense(size: config["size"] as? Int)) + } +} + +public struct QdrantPayloadField: Sendable, Equatable { + public let name: String + // The index's type for an indexed field, which filters use; the sampled type otherwise. + public let type: QdrantFieldType + public let isIndexed: Bool + // False while every sampled value was null, so the first real value sets the type outright. + public let hasTypedValue: Bool + // What an indexed field's points actually store, which the index does not constrain: a keyword + // index still lets a point hold 42 or a list. + public let scalarType: QdrantFieldType? + public let holdsLists: Bool + + public init( + name: String, + type: QdrantFieldType, + isIndexed: Bool, + hasTypedValue: Bool = true, + scalarType: QdrantFieldType? = nil, + holdsLists: Bool = false + ) { + self.name = name + self.type = type + self.isIndexed = isIndexed + self.hasTypedValue = hasTypedValue + self.scalarType = scalarType + self.holdsLists = holdsLists + } + + public var storedType: QdrantFieldType? { + guard holdsLists else { return scalarType } + return scalarType == nil ? .array : .json + } + + // What a cell holds and is written back as, so a value saves as the type it was stored with. + // The index's name stands in only when every value seen fits it. + public var cellTypeName: String { + guard isIndexed, let storedType else { return type.rawValue } + return type.stores(storedType) ? type.rawValue : storedType.rawValue + } +} + +public struct QdrantPayloadIndex: Sendable, Equatable { + public let path: String + public let type: String + + public init(path: String, type: String) { + self.path = path + self.type = type + } +} + +public struct QdrantCollection: Sendable, Equatable { + public let name: String + public let vectors: [QdrantVectorSpec] + public let payloadFields: [QdrantPayloadField] + public let indexes: [QdrantPayloadIndex] + public let pointsCount: Int? + public let info: String + + public init( + name: String, + vectors: [QdrantVectorSpec], + payloadFields: [QdrantPayloadField], + indexes: [QdrantPayloadIndex] = [], + pointsCount: Int? = nil, + info: String = "{}" + ) { + self.name = name + self.vectors = vectors + self.payloadFields = payloadFields + self.indexes = indexes + self.pointsCount = pointsCount + self.info = info + } + + // Only an index states a field's type; a sample guesses it from 100 points. + public func cellTypeName(ofColumn column: String) -> String? { + field(ofColumn: column)?.cellTypeName + } + + // `id`, `score` and the vectors are the point's own; a payload key of that name is `payload.`. + private func field(ofColumn column: String) -> QdrantPayloadField? { + guard !QdrantSchema.isPointColumn(column) else { return nil } + let key = QdrantSchema.payloadKey(forColumn: column) + return payloadFields.first { $0.name == key } + } + + // A page's points widen the types a sample guessed, so a list first seen on page 2 is not saved as + // text. `addingKeys` is false for a grid page, whose columns stay the ones the table opened with. + public func including(_ points: [QdrantPoint], addingKeys: Bool = true) -> QdrantCollection { + var fields = payloadFields + var position = Dictionary(uniqueKeysWithValues: fields.enumerated().map { ($1.name, $0) }) + for point in points { + for key in point.payloadKeys { + let observed = point.payload[key].flatMap(QdrantFieldType.observed) + guard let index = position[key] else { + guard addingKeys else { continue } + position[key] = fields.count + fields.append(QdrantPayloadField( + name: key, type: observed ?? .json, isIndexed: false, hasTypedValue: observed != nil + )) + continue + } + let field = fields[index] + guard let observed else { continue } + if field.isIndexed { + let isList = observed == .array + fields[index] = QdrantPayloadField( + name: key, type: field.type, isIndexed: true, + scalarType: isList ? field.scalarType : field.scalarType.map { $0.merged(with: observed) } ?? observed, + holdsLists: field.holdsLists || isList + ) + continue + } + let type = field.hasTypedValue ? field.type.merged(with: observed) : observed + fields[index] = QdrantPayloadField(name: key, type: type, isIndexed: false) + } + } + return QdrantCollection( + name: name, vectors: vectors, payloadFields: fields, indexes: indexes, pointsCount: pointsCount, info: info + ) + } + + // `payload_schema` keys are JSON paths: only a top-level key is a column, while a nested path such as + // `metadata.source` or `tags[]` stays an index on the column that holds it. + public static func parse(name: String, info json: Any?, sample: [QdrantPoint] = []) -> QdrantCollection { + let result = (json as? [String: Any])?["result"] as? [String: Any] ?? [:] + let params = ((result["config"] as? [String: Any])?["params"] as? [String: Any]) ?? [:] + + var vectors: [QdrantVectorSpec] = [] + if let config = params["vectors"] as? [String: Any] { + // A named vector may itself be called `size`; only the unnamed one has a number there. + if config["size"] is NSNumber { + vectors.append(.dense(name: "", config: config)) + } else { + vectors += config.keys.sorted().map { key in + .dense(name: key, config: config[key] as? [String: Any] ?? [:]) + } + } + } + if let sparse = params["sparse_vectors"] as? [String: Any] { + vectors += sparse.keys.sorted().map { QdrantVectorSpec(name: $0, kind: .sparse) } + } + + let schema = result["payload_schema"] as? [String: Any] ?? [:] + var indexes: [QdrantPayloadIndex] = [] + var fields: [QdrantPayloadField] = [] + for path in schema.keys.sorted() { + guard let type = QdrantFieldType.indexed(schema[path]) else { continue } + indexes.append(QdrantPayloadIndex(path: path, type: type.rawValue)) + if let key = QdrantKeyPath.topLevelKey(ofIndexPath: path), !fields.contains(where: { $0.name == key }) { + fields.append(QdrantPayloadField(name: key, type: type, isIndexed: true)) + } + } + + return QdrantCollection( + name: name, + vectors: vectors, + payloadFields: fields, + indexes: indexes, + pointsCount: result["points_count"] as? Int, + info: (try? QdrantJSON.text(result, pretty: true)) ?? "{}" + ).including(sample) + } +} + +public struct QdrantColumn: Sendable, Equatable { + public let name: String + public let type: String + public let isPrimaryKey: Bool +} + +public enum QdrantSchema { + public static let idColumn = "id" + public static let vectorColumn = "vector" + public static let scoreColumn = "score" + static let payloadPrefix = "payload." + + public static func collectionNames(from json: Any?) -> [String] { + let result = (json as? [String: Any])?["result"] as? [String: Any] + let collections = result?["collections"] as? [[String: Any]] ?? [] + return collections.compactMap { $0["name"] as? String }.sorted() + } + + public static func isPointColumn(_ column: String) -> Bool { + column == idColumn || column == scoreColumn || isVectorColumn(column) + } + + public static func isVectorColumn(_ column: String) -> Bool { + column == vectorColumn || column.hasPrefix("\(vectorColumn).") + } + + public static func vectorName(ofColumn column: String) -> String? { + if column == vectorColumn { return "" } + guard column.hasPrefix("\(vectorColumn).") else { return nil } + return String(column.dropFirst(vectorColumn.count + 1)) + } + + // A payload key named `id`, `score`, `vector...` or the raw filter's `__RAW__` would take a name the + // grid gives another meaning, so it shows as `payload.`. A key already starting with `payload.` + // is prefixed too, so every column maps back to exactly one key. + public static func isReserved(_ key: String) -> Bool { + key == idColumn || key == scoreColumn || isVectorColumn(key) || key.hasPrefix(payloadPrefix) + || key == QdrantFilterBuilder.rawFilterColumn + } + + public static func column(forPayloadKey key: String) -> String { + isReserved(key) ? payloadPrefix + key : key + } + + public static func payloadKey(forColumn column: String) -> String { + column.hasPrefix(payloadPrefix) ? String(column.dropFirst(payloadPrefix.count)) : column + } + + public static func columns(for collection: QdrantCollection) -> [QdrantColumn] { + var columns = [QdrantColumn(name: idColumn, type: "id", isPrimaryKey: true)] + columns += collection.payloadFields.map { + QdrantColumn(name: column(forPayloadKey: $0.name), type: $0.cellTypeName, isPrimaryKey: false) + } + columns += collection.vectors.map { QdrantColumn(name: $0.column, type: $0.typeName, isPrimaryKey: false) } + return columns + } + + public static func typeName(of column: String, in collection: QdrantCollection) -> String { + if column == scoreColumn { return QdrantFieldType.float.rawValue } + if let known = knownTypeName(of: column, in: collection) { return known } + return isVectorColumn(column) ? "vector" : QdrantFieldType.json.rawValue + } + + public static func knownTypeName(of column: String, in collection: QdrantCollection) -> String? { + if column == idColumn { return "id" } + if let spec = collection.vectors.first(where: { $0.column == column }) { + return spec.typeName + } + return collection.cellTypeName(ofColumn: column) + } +} + +public enum QdrantKeyPath { + // Qdrant reads a filter key as a JSON path, so `a.b` would address a nested field. Quoting keeps + // any top-level key whole; a key holding a double quote has no spelling Qdrant accepts. + public static func quoted(_ key: String) -> String? { + guard !key.contains("\"") else { return nil } + return "\"\(key)\"" + } + + public static func topLevelKey(ofIndexPath path: String) -> String? { + if path.count >= 2, path.hasPrefix("\""), path.hasSuffix("\"") { + let inner = path.dropFirst().dropLast() + return inner.contains("\"") ? nil : String(inner) + } + return path.contains(where: { ".[]\"".contains($0) }) ? nil : path + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift new file mode 100644 index 0000000000..3945423068 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift @@ -0,0 +1,120 @@ +import Foundation + +public enum QdrantValueError: Error, LocalizedError, Equatable, Sendable { + case notAnInteger(column: String, value: String) + case notANumber(column: String, value: String) + case notABoolean(column: String, value: String) + case notAGeoPoint(column: String) + case notAnObject(column: String) + case notAnArray(column: String) + case notAVector(column: String) + case notAMultivector(column: String) + case notASparseVector(column: String) + + public var errorDescription: String? { + switch self { + case .notAnInteger(let column, let value): + return String(format: String(localized: "%@ holds integers, and %@ is not one."), column, value) + case .notANumber(let column, let value): + return String(format: String(localized: "%@ holds numbers, and %@ is not a number."), column, value) + case .notABoolean(let column, let value): + return String(format: String(localized: "%@ holds booleans, so it takes true or false, not %@."), column, value) + case .notAGeoPoint(let column): + return String(format: String(localized: "%@ holds geo points, written as {\"lon\": 13.4, \"lat\": 52.5}."), column) + case .notAnObject(let column): + return String(format: String(localized: "%@ holds JSON objects, written as {\"key\": \"value\"}."), column) + case .notAnArray(let column): + return String(format: String(localized: "%@ holds JSON arrays, written as [1, 2, 3]."), column) + case .notAVector(let column): + return String(format: String(localized: "%@ is a vector, written as an array of numbers such as [0.1, 0.2]."), column) + case .notAMultivector(let column): + return String(format: String(localized: "%@ is a multivector, written as arrays of numbers such as [[0.1, 0.2], [0.3, 0.4]]."), column) + case .notASparseVector(let column): + return String(format: String(localized: "%@ is a sparse vector, written as {\"indices\": [1, 5], \"values\": [0.2, 0.7]}."), column) + } + } +} + +// The grid's text becomes the JSON type its column holds. Parsing every cell as JSON would turn a +// keyword holding `42` into a number and stop `match` from finding it. +public enum QdrantValue { + // The grid's boolean menu offers 1 and 0 to an engine without a SQL dialect. + public static func bool(_ text: String) -> Bool? { + switch text.trimmingCharacters(in: .whitespaces).lowercased() { + case "true", "1": return true + case "false", "0": return false + default: return nil + } + } + + public static func integer(_ text: String) -> Int64? { + Int64(text.trimmingCharacters(in: .whitespaces)) + } + + public static func number(_ text: String) -> Double? { + let trimmed = text.trimmingCharacters(in: .whitespaces) + guard let value = Double(trimmed), value.isFinite else { return nil } + return value + } + + public static func payload(_ text: String, type: QdrantFieldType, column: String) throws -> Any { + switch type { + case .keyword, .text, .string, .uuid, .datetime: + return text + case .integer: + guard let value = integer(text) else { throw QdrantValueError.notAnInteger(column: column, value: text) } + return NSNumber(value: value) + case .float: + // A float column also holds integers, and a double rounds one past 2^53. + if let value = integer(text) { return NSNumber(value: value) } + guard let value = number(text) else { throw QdrantValueError.notANumber(column: column, value: text) } + return NSNumber(value: value) + case .bool: + guard let value = bool(text) else { throw QdrantValueError.notABoolean(column: column, value: text) } + return value + case .geo: + guard let point = QdrantJSON.parse(text) as? [String: Any], + point["lon"] is NSNumber, point["lat"] is NSNumber + else { throw QdrantValueError.notAGeoPoint(column: column) } + return point + case .object: + guard let object = QdrantJSON.parse(text) as? [String: Any] else { + throw QdrantValueError.notAnObject(column: column) + } + return object + case .array: + guard let array = QdrantJSON.parse(text) as? [Any] else { throw QdrantValueError.notAnArray(column: column) } + return array + case .json: + return QdrantJSON.parse(text) ?? text + } + } + + public static func vector(_ text: String, kind: QdrantVectorSpec.Kind, column: String) throws -> Any { + let parsed = QdrantJSON.parse(text) + switch kind { + case .dense: + guard let values = numbers(parsed), !values.isEmpty else { throw QdrantValueError.notAVector(column: column) } + return values + case .multi: + guard let rows = parsed as? [Any], !rows.isEmpty else { throw QdrantValueError.notAMultivector(column: column) } + return try rows.map { row in + guard let values = numbers(row), !values.isEmpty else { throw QdrantValueError.notAMultivector(column: column) } + return values + } + case .sparse: + guard let object = parsed as? [String: Any], + let indices = object["indices"] as? [Any], let values = numbers(object["values"]), + indices.count == values.count, + indices.allSatisfy({ ($0 as? NSNumber).map { !QdrantJSON.isBoolean($0) } ?? false }) + else { throw QdrantValueError.notASparseVector(column: column) } + return ["indices": indices, "values": values] + } + } + + private static func numbers(_ value: Any?) -> [NSNumber]? { + guard let array = value as? [Any] else { return nil } + let numbers = array.compactMap { $0 as? NSNumber }.filter { !QdrantJSON.isBoolean($0) } + return numbers.count == array.count ? numbers : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift new file mode 100644 index 0000000000..28e1f0c544 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift @@ -0,0 +1,140 @@ +import Foundation + +public struct QdrantScrollPage: @unchecked Sendable { + public let points: [QdrantPoint] + // Nil once the collection is exhausted. + public let nextCursor: Any? +} + +public final class QdrantClient: @unchecked Sendable { + public static let sampleSize = 100 + + public let settings: QdrantConnectionSettings + private let transport: QdrantTransport + private let timeout: @Sendable () -> TimeInterval + private let lock = NSLock() + private var version: String? + + public init( + settings: QdrantConnectionSettings, + transport: QdrantTransport, + timeout: @escaping @Sendable () -> TimeInterval + ) { + self.settings = settings + self.transport = transport + self.timeout = timeout + } + + public var serverVersion: String? { + lock.withLock { version } + } + + public func cancelAll() { + transport.cancelAll() + } + + public func connect(requestTimeout: @Sendable () -> TimeInterval) async throws { + let root = try await checked(send(method: "GET", path: "/", timeoutInterval: requestTimeout())) + if let found = (root.json as? [String: Any])?["version"] as? String { + lock.withLock { version = found } + } + _ = try await checked(send(method: "GET", path: "/collections", timeoutInterval: requestTimeout())) + } + + // `/healthz` answers without the key, so a revoked key would leave the session looking live. + public func ping() async throws { + _ = try await checked(send(method: "GET", path: "/collections")) + } + + public func collectionNames() async throws -> [String] { + QdrantSchema.collectionNames(from: try await checked(send(method: "GET", path: "/collections")).json) + } + + // The config and indexed fields alone: enough for a row count, the DDL and the index list. + public func collectionInfo(named name: String) async throws -> QdrantCollection { + let info = try await checked(send(method: "GET", path: collectionPath(name))) + return QdrantCollection.parse(name: name, info: info.json) + } + + // Fields nobody indexed are only visible in the points, so a sample of them fills in the columns. + public func collection(named name: String) async throws -> QdrantCollection { + let info = try await checked(send(method: "GET", path: collectionPath(name))) + let sample = try await scroll( + collection: name, + body: ["limit": Self.sampleSize, "with_payload": true, "with_vector": false] + ) + return QdrantCollection.parse(name: name, info: info.json, sample: sample.points) + } + + public func scroll(collection: String, body: [String: Any]) async throws -> QdrantScrollPage { + let response = try await post("\(collectionPath(collection))/points/scroll", body) + let result = (response.json as? [String: Any])?["result"] as? [String: Any] + let points = (result?["points"] as? [Any] ?? []).compactMap(QdrantPoint.parse) + let next = result?["next_page_offset"] + return QdrantScrollPage(points: points, nextCursor: next is NSNull ? nil : next) + } + + public func query(collection: String, body: [String: Any]) async throws -> [QdrantPoint] { + let response = try await post("\(collectionPath(collection))/points/query", body) + return QdrantPoint.list(fromResponse: response.json) ?? [] + } + + public func count(collection: String, filter: [String: Any]?) async throws -> Int { + var body: [String: Any] = ["exact": true] + if let filter { + body["filter"] = filter + } + let response = try await post("\(collectionPath(collection))/points/count", body) + let result = (response.json as? [String: Any])?["result"] as? [String: Any] + guard let count = result?["count"] as? Int else { + throw QdrantError.malformedResponse(String(localized: "Qdrant answered a count without a number.")) + } + return count + } + + public func execute(_ request: QdrantConsoleRequest) async throws -> QdrantHTTPResponse { + try await checked(send(method: request.method, path: request.path, body: request.body.map { Data($0.utf8) })) + } + + private func post(_ path: String, _ body: [String: Any]) async throws -> QdrantHTTPResponse { + try await checked(send(method: "POST", path: path, body: try QdrantJSON.data(body))) + } + + private func collectionPath(_ name: String) -> String { + "/collections/\(QdrantPathEncoding.segment(name))" + } + + private func send(method: String, path: String, body: Data? = nil) async throws -> QdrantHTTPResponse { + try await send(method: method, path: path, body: body, timeoutInterval: timeout()) + } + + private func send( + method: String, + path: String, + body: Data? = nil, + timeoutInterval: TimeInterval + ) async throws -> QdrantHTTPResponse { + guard let url = QdrantPathEncoding.resolve(path, against: try settings.baseURL()) else { + throw QdrantError.configuration(String(format: String(localized: "Invalid path: %@"), path)) + } + var headers = settings.headers + headers["Accept"] = "application/json" + if body != nil { + headers["Content-Type"] = "application/json" + } + return try await transport.send(QdrantHTTPRequest( + method: method, + url: url, + headers: headers, + body: body, + timeoutInterval: timeoutInterval + )) + } + + private func checked(_ response: QdrantHTTPResponse) throws -> QdrantHTTPResponse { + guard (200..<300).contains(response.statusCode) else { + throw QdrantError.from(status: response.statusCode, body: response.body) + } + return response + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift new file mode 100644 index 0000000000..28c0f493e8 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift @@ -0,0 +1,108 @@ +import Foundation + +// The sort checks a sorted browse has already passed, so only its first page pays for them. +public final class QdrantPagingCache: @unchecked Sendable { + private let lock = NSLock() + private var checkedSorts: Set = [] + + public init() {} + + public func isSortChecked(_ key: String) -> Bool { + lock.withLock { checkedSorts.contains(key) } + } + + public func markSortChecked(_ key: String) { + lock.withLock { _ = checkedSorts.insert(key) } + } + + public func removeAll() { + lock.withLock { checkedSorts.removeAll() } + } +} + +public extension QdrantClient { + static let walkBatchSize = 10_000 + + // A page in id order. The Query API with no query pages by id and applies the offset on the server, + // so a page is always the rows at that offset now; a cursor cached from an earlier read would start + // at the wrong row once another client added or removed a point before it. A node older than 1.10 + // has no Query API, so its pages walk the ids from the start. + func pointPage(collection: String, filter: [String: Any]?, offset: Int, limit: Int) async throws -> [QdrantPoint] { + if QdrantServerVersion.hasQueryAPI(serverVersion) { + var body: [String: Any] = ["limit": limit, "offset": offset, "with_payload": true, "with_vector": true] + if let filter { + body["filter"] = filter + } + return try await query(collection: collection, body: body) + } + var position = 0 + var cursor: Any? + while position < offset { + try Task.checkCancellation() + let page = try await scroll( + collection: collection, + body: Self.scrollBody(filter: filter, limit: min(offset - position, Self.walkBatchSize), cursor: cursor, withData: false) + ) + guard let next = page.nextCursor else { return [] } + position += page.points.count + cursor = next + } + return try await scroll( + collection: collection, + body: Self.scrollBody(filter: filter, limit: limit, cursor: cursor, withData: true) + ).points + } + + // Without vectors this reads payloads only, which is how an export learns every key before its header. + func pointPages( + collection: String, + pageSize: Int, + withVector: Bool = true + ) -> AsyncThrowingStream<[QdrantPoint], Error> { + AsyncThrowingStream { continuation in + let task = Task { + do { + var cursor: Any? + repeat { + try Task.checkCancellation() + let page = try await self.scroll( + collection: collection, + body: Self.scrollBody( + filter: nil, limit: max(pageSize, 1), cursor: cursor, withData: true, withVector: withVector + ) + ) + if !page.points.isEmpty { + continuation.yield(page.points) + } + cursor = page.nextCursor + } while cursor != nil + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { @Sendable _ in task.cancel() } + } + } + + static func scrollBody( + filter: [String: Any]?, + limit: Int, + cursor: Any?, + withData: Bool, + withVector: Bool? = nil + ) -> [String: Any] { + var body: [String: Any] = [ + "limit": limit, + "with_payload": withData, + "with_vector": withVector ?? withData + ] + if let filter { + body["filter"] = filter + } + if let cursor { + body["offset"] = cursor + } + return body + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift new file mode 100644 index 0000000000..302080324e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift @@ -0,0 +1,69 @@ +import Foundation + +// `order_by` walks Qdrant's range index, which holds only points with a value in it and holds a point +// once per value. So the grid's rows are the ordered points first, then the points without a value in +// id order, and a field holding a list in some point is refused: no offset can page it one row per point. +public extension QdrantClient { + func sortedPage( + collection: String, + filter: [String: Any]?, + column: String, + key: String, + ascending: Bool, + offset: Int, + limit: Int, + cache: QdrantPagingCache + ) async throws -> [QdrantPoint] { + let hasValue: [String: Any] = ["key": key, "range": [String: Any]()] + let ordered = QdrantFilterBuilder.allOf(filter, hasValue) + func orderBody(offset: Int, limit: Int, withData: Bool) -> [String: Any] { + var body: [String: Any] = [ + "query": ["order_by": ["key": key, "direction": ascending ? "asc" : "desc"]], + "limit": limit, + "offset": offset, + "with_payload": withData, + "with_vector": withData + ] + if let filter { + body["filter"] = filter + } + return body + } + + let checkKey = "\(collection)\u{0}\(key)\u{0}\((try? QdrantJSON.text(ordered)) ?? "")" + // Rechecked on every first page, so a refresh sees a list another client stored since. + if offset == 0 || !cache.isSortChecked(checkKey) { + // A field with no range index matches no point, so without asking Qdrant to order by it + // every point would land in the remainder and the page would look sorted by id. + _ = try await query(collection: collection, body: orderBody(offset: 0, limit: 1, withData: false)) + let listed = try await count( + collection: collection, + filter: QdrantFilterBuilder.allOf(ordered, ["key": key, "values_count": ["gt": 1]]) + ) + guard listed == 0 else { + throw QdrantError.unsupported(String( + format: String(localized: "%@ holds a list in %d points, and Qdrant sorts such a point once per value. Sort by a field that holds one value per point."), + column, listed + )) + } + cache.markSortChecked(checkKey) + } + + let orderedCount = try await count(collection: collection, filter: ordered) + var points: [QdrantPoint] = [] + if offset < orderedCount { + points = try await query( + collection: collection, + body: orderBody(offset: offset, limit: min(limit, orderedCount - offset), withData: true) + ) + } + guard points.count < limit else { return points } + let withoutValue = QdrantFilterBuilder.allOf(filter, QdrantFilterBuilder.negated([hasValue])) + return points + (try await pointPage( + collection: collection, + filter: withoutValue, + offset: max(offset - orderedCount, 0), + limit: limit - points.count + )) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift new file mode 100644 index 0000000000..b92ec486fa --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift @@ -0,0 +1,42 @@ +import Foundation + +public struct QdrantHTTPRequest: Sendable, Equatable { + public let method: String + public let url: URL + public let headers: [String: String] + public let body: Data? + public let timeoutInterval: TimeInterval + + public init(method: String, url: URL, headers: [String: String], body: Data?, timeoutInterval: TimeInterval) { + self.method = method + self.url = url + self.headers = headers + self.body = body + self.timeoutInterval = timeoutInterval + } +} + +public struct QdrantHTTPResponse: Sendable, Equatable { + public let statusCode: Int + public let body: Data + + public init(statusCode: Int, body: Data) { + self.statusCode = statusCode + self.body = body + } + + public var json: Any? { + try? JSONSerialization.jsonObject(with: body, options: [.fragmentsAllowed]) + } + + public var text: String { + String(data: body, encoding: .utf8) ?? "" + } +} + +// The URLSession transport lives in the plugin: the app links this module for the query classifier, +// and an Objective-C class compiled into both would be registered twice in one process. +public protocol QdrantTransport: Sendable { + func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse + func cancelAll() +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index 1ae428dc7c..8bfa07bd07 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -339,6 +339,7 @@ public struct SQLLexicalProfile: Sendable, Hashable { "Elasticsearch": commandLineFamily, "Typesense": commandLineFamily, "Weaviate": commandLineFamily, + "Qdrant": commandLineFamily, "Memcached": commandLineFamily, "Kafka": commandLineFamily, "Firestore": commandLineFamily, diff --git a/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/ServerCertificatePurpose.swift b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/ServerCertificatePurpose.swift new file mode 100644 index 0000000000..26f9ac028e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/ServerCertificatePurpose.swift @@ -0,0 +1,29 @@ +#if os(macOS) +import Foundation +import Security + +// Verify CA checks the chain with the basic X.509 policy, because Apple's SSL policy refuses a private +// CA's long-lived certificate as not standards compliant. The SSL policy was also what required a +// server certificate, so this puts that one check back: a client-only certificate from the same CA +// must not pass as the server. +public enum ServerCertificatePurpose { + private static let serverAuthentication = Data([0x2B, 0x06, 0x01, 0x05, 0x05, 0x07, 0x03, 0x01]) + private static let anyPurpose = Data([0x55, 0x1D, 0x25, 0x00]) + + public static func permitsServerAuthentication(_ certificate: SecCertificate) -> Bool { + let values = SecCertificateCopyValues(certificate, [kSecOIDExtendedKeyUsage] as CFArray, nil) as? [String: Any] + guard let usage = values?[kSecOIDExtendedKeyUsage as String] as? [String: Any], + let purposes = usage[kSecPropertyKeyValue as String] as? [Any] + else { return true } + return purposes.contains { purpose in + guard let oid = purpose as? Data else { return false } + return oid == serverAuthentication || oid == anyPurpose + } + } + + public static func leafPermitsServerAuthentication(_ trust: SecTrust) -> Bool { + guard let leaf = (SecTrustCopyCertificateChain(trust) as? [SecCertificate])?.first else { return false } + return permitsServerAuthentication(leaf) + } +} +#endif diff --git a/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift b/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift index 0fed9cb2d2..df41d8b00e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift @@ -1,5 +1,6 @@ import Foundation import Security +import TableProTLSClientIdentity public struct TrinoHeaderFields: Sendable, Equatable { private let storage: [String: String] @@ -395,6 +396,9 @@ struct TrinoTLSChallengeHandler { guard SecTrustEvaluateWithError(serverTrust, &error) else { return .refuse(Self.failureKind(of: error), message: error.map { CFErrorCopyDescription($0) as String } ?? "") } + if tls.mode == .caOnly, !ServerCertificatePurpose.leafPermitsServerAuthentication(serverTrust) { + return .refuse(.untrustedCertificate, message: "The server's certificate is not issued for a server.") + } return TrinoChallengeAnswer(disposition: .useCredential, credential: URLCredential(trust: serverTrust)) } diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift index 0705167481..7fa048ddc5 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift @@ -131,10 +131,8 @@ public enum WeaviateConsoleParser { let inlineBody = parts.count > 2 ? String(parts[2]).trimmingCharacters(in: .whitespacesAndNewlines) : "" - return WeaviateConsoleRequest( - method: method, - path: path, - body: body ?? (inlineBody.isEmpty ? nil : inlineBody) - ) + // A body can open on the request line and go on below it, as `POST /v1/graphql {`. + let joined = [inlineBody, body ?? ""].filter { !$0.isEmpty }.joined(separator: "\n") + return WeaviateConsoleRequest(method: method, path: path, body: joined.isEmpty ? nil : joined) } } diff --git a/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift b/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift index 2953d84f21..4eb2c2438b 100644 --- a/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift @@ -63,7 +63,7 @@ struct DatabaseTypeTests { @Test("allKnownTypes contains all expected types") func allKnownTypesComplete() { - #expect(DatabaseType.allKnownTypes.count == 35) + #expect(DatabaseType.allKnownTypes.count == 36) #expect(DatabaseType.allKnownTypes.contains(.mysql)) #expect(DatabaseType.allKnownTypes.contains(.tidb)) #expect(DatabaseType.allKnownTypes.contains(.databend)) @@ -81,6 +81,7 @@ struct DatabaseTypeTests { #expect(DatabaseType.allKnownTypes.contains(.kafka)) #expect(DatabaseType.allKnownTypes.contains(.cloudflareR2SQL)) #expect(DatabaseType.allKnownTypes.contains(.weaviate)) + #expect(DatabaseType.allKnownTypes.contains(.qdrant)) #expect(DatabaseType.allKnownTypes.contains(.memcached)) #expect(DatabaseType.allKnownTypes.contains(.firestore)) #expect(DatabaseType.allKnownTypes.contains(.firebaseRealtimeDatabase)) @@ -109,6 +110,13 @@ struct DatabaseTypeTests { #expect(DatabaseType.weaviate.pluginTypeId == "Weaviate") } + @Test("Qdrant resolves its icon and plugin type id") + func qdrantIdentity() { + #expect(DatabaseType.qdrant.rawValue == "Qdrant") + #expect(DatabaseType.qdrant.iconName == "qdrant-icon") + #expect(DatabaseType.qdrant.pluginTypeId == "Qdrant") + } + @Test("Memcached resolves its icon and plugin type id") func memcachedIdentity() { #expect(DatabaseType.memcached.rawValue == "Memcached") diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift new file mode 100644 index 0000000000..c927c42468 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift @@ -0,0 +1,151 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantClientTests { + @Test("On Qdrant 1.10 or later a page is one query with the offset applied on the server") + func pageByServerOffset() async throws { + let transport = RecordingTransport { request in + switch request.url.path { + case "/": return jsonResponse(["version": "1.20.0"]) + case "/collections": return jsonResponse(["result": ["collections": [Any]()]]) + default: return jsonResponse(["result": ["points": [["id": 101, "payload": [String: Any]()]]]]) + } + } + let client = makeClient(transport: transport) + try await client.connect(requestTimeout: { 5 }) + let filter: [String: Any] = ["must": [["has_id": [101]]]] + let page = try await client.pointPage(collection: "docs", filter: filter, offset: 100, limit: 10) + #expect(page.compactMap(\.idText) == ["101"]) + let request = try #require(transport.recorded.last) + #expect(request.url.path == "/collections/docs/points/query") + #expect(canonical(bodyObject(request)) == #"{"filter":{"must":[{"has_id":[101]}]},"limit":10,"offset":100,"with_payload":true,"with_vector":true}"#) + } + + @Test("An older node walks ids from the start every time, so no cached position can go stale") + func olderNodeWalksIDs() async throws { + let transport = scrollingTransport(count: 25_000) + let client = makeClient(transport: transport) + let page = try await client.pointPage(collection: "docs", filter: nil, offset: 20_000, limit: 10) + #expect(page.compactMap(\.idText) == (20_001...20_010).map(String.init)) + let walks = transport.scrolls.dropLast().map(bodyObject) + #expect(walks.map { $0["limit"] as? Int } == [10_000, 10_000]) + #expect(walks.allSatisfy { $0["with_payload"] as? Bool == false && $0["with_vector"] as? Bool == false }) + _ = try await client.pointPage(collection: "docs", filter: nil, offset: 20_000, limit: 10) + #expect(transport.scrolls.count == 6) + } + + @Test("An offset past the end answers an empty page") + func pastEnd() async throws { + let client = makeClient(transport: scrollingTransport(count: 5)) + let page = try await client.pointPage(collection: "docs", filter: nil, offset: 10, limit: 10) + #expect(page.isEmpty) + } + + @Test("Export walks every page until the cursor runs out") + func export() async throws { + let client = makeClient(transport: scrollingTransport(count: 7)) + var ids: [String] = [] + for try await page in client.pointPages(collection: "docs", pageSize: 3) { + ids += page.compactMap(\.idText) + } + #expect(ids == (1...7).map(String.init)) + } + + @Test("The API key travels in the api-key header and errors carry Qdrant's reason") + func headersAndErrors() async throws { + let transport = RecordingTransport { _ in + jsonResponse(["status": ["error": "Not found: Collection `nope` doesn't exist!"]], status: 404) + } + let client = makeClient(apiKey: "secret", transport: transport) + await #expect(throws: QdrantError.api(status: 404, message: "Not found: Collection `nope` doesn't exist!")) { + _ = try await client.collection(named: "nope") + } + #expect(transport.recorded.first?.headers["api-key"] == "secret") + #expect(QdrantError.from(status: 401, body: Data("Invalid api-key".utf8)) == .authentication("Invalid api-key")) + } + + @Test("Collection info is one GET, without the payload sample the columns need") + func infoOnly() async throws { + let transport = RecordingTransport { _ in + jsonResponse(["result": ["points_count": 12, "config": ["params": ["vectors": ["size": 2]]]], "status": "ok"]) + } + let collection = try await makeClient(transport: transport).collectionInfo(named: "docs") + #expect(collection.pointsCount == 12) + #expect(transport.recorded.map(\.method) == ["GET"]) + } +} + +// Ordered points 1...6 by `price`, then 7...9 without a price. +struct QdrantSortedPagerTests { + private func transport(listed: Int = 0) -> RecordingTransport { + RecordingTransport { request in + let body = bodyObject(request) + let text = canonical(body) + switch request.url.path { + case "/collections/docs/points/count": + if text.contains("values_count") { return jsonResponse(["result": ["count": listed]]) } + return jsonResponse(["result": ["count": 6]]) + case "/collections/docs/points/query": + let offset = body["offset"] as? Int ?? 0 + let limit = body["limit"] as? Int ?? 10 + let ids = Array((1...6).dropFirst(offset).prefix(limit)) + return jsonResponse(["result": ["points": ids.map { ["id": $0, "payload": ["price": $0]] }]]) + case "/collections/docs/points/scroll": + let limit = body["limit"] as? Int ?? 10 + let start = body["offset"] as? Int ?? 7 + let ids = Array(start.. [String] { + try await client.sortedPage( + collection: "docs", filter: nil, column: "price", key: "\"price\"", ascending: true, + offset: offset, limit: limit, cache: cache + ).compactMap(\.idText) + } + + @Test("Points with a value come first in order, then the points without one, and every point is reachable") + func orderedThenRemainder() async throws { + let transport = transport() + let client = makeClient(transport: transport) + let cache = QdrantPagingCache() + + #expect(try await page(client, offset: 0, limit: 4, cache: cache) == ["1", "2", "3", "4"]) + #expect(try await page(client, offset: 4, limit: 4, cache: cache) == ["5", "6", "7", "8"]) + #expect(try await page(client, offset: 8, limit: 4, cache: cache) == ["9"]) + + let query = try #require(transport.recorded.first { $0.url.path.hasSuffix("/query") }) + #expect(canonical(bodyObject(query)["query"] ?? [:]) == #"{"order_by":{"direction":"asc","key":"\"price\""}}"#) + let remainder = try #require(transport.recorded.first { $0.url.path.hasSuffix("/scroll") }) + #expect(canonical(bodyObject(remainder)["filter"] ?? [:]) + == #"{"must":[{"must_not":[{"key":"\"price\"","range":{}}]}]}"#) + #expect(transport.recorded.filter { canonical(bodyObject($0)).contains("values_count") }.count == 1) + } + + @Test("A field Qdrant cannot order by is refused, never shown in id order as if sorted") + func unorderableFieldRefused() async throws { + let transport = RecordingTransport { request in + guard request.url.path.hasSuffix("/query") else { return jsonResponse(["result": ["count": 0]]) } + return jsonResponse(["status": ["error": "Wrong input: No range index for `order_by` key: `city`."]], status: 400) + } + await #expect(throws: QdrantError.api(status: 400, message: "Wrong input: No range index for `order_by` key: `city`.")) { + _ = try await makeClient(transport: transport).sortedPage( + collection: "docs", filter: nil, column: "city", key: "\"city\"", ascending: true, + offset: 0, limit: 4, cache: QdrantPagingCache() + ) + } + } + + @Test("A field holding a list in some point is refused, since Qdrant would show that point once per value") + func listedValuesRefused() async throws { + let client = makeClient(transport: transport(listed: 2)) + await #expect(throws: QdrantError.self) { + _ = try await page(client, offset: 0, limit: 4, cache: QdrantPagingCache()) + } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift new file mode 100644 index 0000000000..33f6cb4555 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift @@ -0,0 +1,98 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantConsoleTests { + @Test("The console reads a request line with its body below, beside, or starting beside and going on below") + func parser() { + let below = QdrantConsoleParser.parse("POST /collections/docs/points/query\n{\"limit\": 3}") + #expect(below == QdrantConsoleRequest(method: "POST", path: "/collections/docs/points/query", body: "{\"limit\": 3}")) + let inline = QdrantConsoleParser.parse("post /collections/docs/points/count {\"exact\": true}") + #expect(inline?.method == "POST") + #expect(inline?.body == "{\"exact\": true}") + let split = QdrantConsoleParser.parse("POST /collections/docs/points/query {\n \"query\": [0.1, 0.2], \"limit\": 5 }") + #expect(split?.body == "{\n\"query\": [0.1, 0.2], \"limit\": 5 }") + let crlf = QdrantConsoleParser.parse("GET /collections\r\n") + #expect(crlf == QdrantConsoleRequest(method: "GET", path: "/collections", body: nil)) + #expect(QdrantConsoleParser.parse("DELETE\t/collections/docs")?.method == "DELETE") + #expect(QdrantConsoleParser.parse("SELECT 1") == nil) + #expect(QdrantConsoleParser.parse("GET collections") == nil) + } + + @Test("Reads, writes and removals are told apart by verb and route") + func requestKinds() { + func tier(_ method: String, _ path: String) -> QdrantRequestKind.Tier { + QdrantRequestKind.of(method: method, path: path).tier + } + #expect(tier("POST", "/collections/docs/points/query") == .read) + #expect(tier("POST", "/collections/docs/points") == .read) + #expect(tier("POST", "/collections/docs/facet") == .read) + #expect(tier("POST", "/collections/docs/points/facet") == .write) + #expect(tier("GET", "/collections") == .read) + #expect(tier("PUT", "/collections/docs/points?wait=true") == .write) + #expect(tier("POST", "/collections/docs/points/payload") == .write) + #expect(tier("POST", "/collections/docs/points/delete?wait=true") == .remove) + #expect(tier("POST", "/collections/docs/points/payload/clear") == .remove) + #expect(tier("POST", "/collections/docs/points/batch") == .remove) + #expect(tier("DELETE", "/collections/docs") == .remove) + #expect(tier("PUT", "/collections/docs/points/payload?wait=true") == .remove) + #expect(tier("POST", "/collections/docs/cluster") == .remove) + #expect(tier("POST", "/collections/aliases") == .remove) + } + + @Test("A snapshot touches the server's disk, and a dot segment is never read as another route") + func serverFilesAndDotSegments() { + #expect(QdrantRequestKind.of(method: "POST", path: "/collections/docs/snapshots").touchesServerFiles) + #expect(!QdrantRequestKind.of(method: "GET", path: "/collections").touchesServerFiles) + let dotted = QdrantRequestKind.of(method: "POST", path: "/collections/docs/points/query/../delete") + #expect(dotted == QdrantRequestKind(tier: .write, touchesServerFiles: true)) + #expect(QdrantPathEncoding.segments(of: "/collections/%2e%2e/x") == nil) + #expect(!QdrantRequestKind.of(method: "POST", path: "/collections/snapshots/points/query").touchesServerFiles) + #expect(QdrantRequestKind.of(method: "POST", path: "/snapshots").touchesServerFiles) + #expect(QdrantRequestKind.of(method: "PUT", path: "/collections/docs/shards/0/snapshots/recover").touchesServerFiles) + } + + @Test("A request path keeps its query string and never leaves the node") + func pathResolution() throws { + let base = try #require(URL(string: "http://localhost:6333")) + let url = QdrantPathEncoding.resolve("/collections/docs/points?wait=true", against: base) + #expect(url?.absoluteString == "http://localhost:6333/collections/docs/points?wait=true") + #expect(QdrantPathEncoding.resolve("//evil.example/x", against: base) == nil) + #expect(QdrantPathEncoding.resolve("/collections/docs/points/query/../delete", against: base) == nil) + } + + @Test("Browse requests round-trip through their tag with every sort column") + func browseRoundTrip() throws { + let request = QdrantBrowseRequest( + collection: "docs", + offset: 200, + limit: 100, + sorts: [QdrantSortSpec(column: "pop", ascending: false), QdrantSortSpec(column: "city", ascending: true)], + filters: [QdrantFilterSpec(column: "count", op: "BETWEEN", value: "1,5", secondValue: "5")], + logicMode: "OR" + ) + #expect(QdrantBrowseQuery.parse(QdrantBrowseQuery.encode(request)) == request) + } + + @Test("A sort Qdrant cannot honor is refused rather than ignored") + func sortPlans() throws { + #expect(try QdrantSortPlan.resolve([]) == .byID) + #expect(try QdrantSortPlan.resolve([QdrantSortSpec(column: "id", ascending: true)]) == .byID) + #expect(try QdrantSortPlan.resolve([QdrantSortSpec(column: "pop", ascending: false)]) + == .byPayload(column: "pop", key: "\"pop\"", ascending: false)) + #expect(throws: QdrantError.self) { try QdrantSortPlan.resolve([QdrantSortSpec(column: "id", ascending: false)]) } + #expect(throws: QdrantError.self) { try QdrantSortPlan.resolve([QdrantSortSpec(column: "vector.image", ascending: true)]) } + #expect(throws: QdrantError.self) { + try QdrantSortPlan.resolve([QdrantSortSpec(column: "pop", ascending: true), QdrantSortSpec(column: "city", ascending: true)]) + } + } + + @Test("Drop, truncate and export are request lines or tags the driver reads back") + func operations() { + #expect(QdrantOperations.deleteCollection(named: "my docs", objectType: "TABLE") == "DELETE /collections/my%20docs") + #expect(QdrantOperations.deleteCollection(named: "docs", objectType: "VIEW") == nil) + #expect(QdrantOperations.deleteAllPoints(collection: "docs") + == "POST /collections/docs/points/delete?wait=true\n{\"filter\":{}}") + #expect(QdrantOperations.decodeExport(QdrantOperations.encodeExport(collection: "docs")) == "docs") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift new file mode 100644 index 0000000000..d2a4a83fcb --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift @@ -0,0 +1,111 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantFilterBuilderTests { + private func condition(_ column: String, _ op: String, _ value: String = "", _ second: String? = nil) throws -> String { + canonical(try QdrantFilterBuilder.condition(for: QdrantFilterSpec(column: column, op: op, value: value, secondValue: second))) + } + + @Test("Text matches as text alone") + func textEquality() throws { + #expect(try condition("city", "=", "Berlin") == #"{"key":"\"city\"","match":{"value":"Berlin"}}"#) + } + + @Test("A value matches every type its text spells, whatever an index or a sample says the field holds") + func everySpelling() throws { + #expect(try condition("count", "=", "42") + == #"{"should":[{"key":"\"count\"","match":{"value":"42"}},{"key":"\"count\"","range":{"gte":42,"lte":42}}]}"#) + #expect(try condition("flag", "=", "1") + == #"{"should":[{"key":"\"flag\"","match":{"value":"1"}},{"key":"\"flag\"","range":{"gte":1,"lte":1}},{"key":"\"flag\"","match":{"value":true}}]}"#) + #expect(try condition("pop", "=", "3.5") + == #"{"should":[{"key":"\"pop\"","match":{"value":"3.5"}},{"key":"\"pop\"","range":{"gte":3.5,"lte":3.5}}]}"#) + #expect(try condition("count", "=", "9007199254740993") + == #"{"should":[{"key":"\"count\"","match":{"value":"9007199254740993"}},{"key":"\"count\"","match":{"value":9007199254740993}}]}"#) + } + + @Test("A key holding a dot stays one top-level key, and one holding a quote is refused") + func keyQuoting() throws { + #expect(try condition("a.b", "=", "x") == #"{"key":"\"a.b\"","match":{"value":"x"}}"#) + #expect(throws: QdrantFilterError.keyNotAddressable(column: "q\"x")) { try condition("q\"x", "=", "1") } + } + + @Test("Not equal nests the match under must_not") + func notEqual() throws { + #expect(try condition("city", "!=", "Berlin") == #"{"must_not":[{"key":"\"city\"","match":{"value":"Berlin"}}]}"#) + } + + @Test("Comparisons and BETWEEN become a range on numbers, or on dates for other text") + func ranges() throws { + #expect(try condition("pop", ">", "2") == #"{"key":"\"pop\"","range":{"gt":2}}"#) + #expect(try condition("count", "BETWEEN", "1,5", "5") == #"{"key":"\"count\"","range":{"gte":1,"lte":5}}"#) + #expect(try condition("seen", ">=", "2024-01-02T00:00:00Z") == #"{"key":"\"seen\"","range":{"gte":"2024-01-02T00:00:00Z"}}"#) + #expect(throws: QdrantFilterError.missingUpperBound(column: "count")) { try condition("count", "BETWEEN", "1", "") } + #expect(throws: QdrantFilterError.boundNotExact(column: "count", value: "9007199254740993")) { + try condition("count", ">=", "9007199254740993") + } + } + + @Test("IN collapses plain text to match any, and keeps every spelling of a number") + func inLists() throws { + #expect(try condition("city", "IN", "Berlin, Paris") == #"{"key":"\"city\"","match":{"any":["Berlin","Paris"]}}"#) + #expect(try condition("rank", "IN", "1") + == #"{"should":[{"should":[{"key":"\"rank\"","match":{"value":"1"}},{"key":"\"rank\"","range":{"gte":1,"lte":1}},{"key":"\"rank\"","match":{"value":true}}]}]}"#) + } + + @Test("NOT IN excludes every value the way != excludes one") + func notInLists() throws { + #expect(try condition("city", "NOT IN", "a, b") + == #"{"must_not":[{"key":"\"city\"","match":{"value":"a"}},{"key":"\"city\"","match":{"value":"b"}}]}"#) + } + + @Test("CONTAINS matches words in any field, indexed or not, and a value that is not text matches nothing") + func contains() throws { + #expect(try condition("body", "CONTAINS", "vector") == #"{"key":"\"body\"","match":{"text":"vector"}}"#) + #expect(try condition("rank", "CONTAINS", "1") == #"{"key":"\"rank\"","match":{"text":"1"}}"#) + } + + @Test("IS NULL matches a missing key as well as null, and IS EMPTY adds the empty string") + func nullness() throws { + #expect(try condition("note", "IS NULL") == #"{"is_empty":{"key":"\"note\""}}"#) + #expect(try condition("note", "IS NOT NULL") == #"{"must_not":[{"is_empty":{"key":"\"note\""}}]}"#) + #expect(try condition("note", "IS EMPTY") + == #"{"should":[{"is_empty":{"key":"\"note\""}},{"key":"\"note\"","match":{"value":""}}]}"#) + } + + @Test("The id column filters with has_id, keeping integer and UUID ids apart") + func ids() throws { + let uuid = "5c56c793-69f3-4fbf-87e6-c4bf54c28c26" + #expect(try condition("id", "IN", "1, \(uuid)") == #"{"has_id":[1,"\#(uuid)"]}"#) + #expect(try condition("id", "!=", "1") == #"{"must_not":[{"has_id":[1]}]}"#) + #expect(throws: QdrantFilterError.notAPointID(value: "abc")) { try condition("id", "=", "abc") } + #expect(try condition("id", "CONTAINS", "7") == #"{"has_id":[7]}"#) + #expect(try condition("id", "CONTAINS", "Berlin") == #"{"has_id":[]}"#) + } + + @Test("Vectors cannot be filtered, contains on one finds nothing, and unsupported operators say so") + func refusals() throws { + #expect(throws: QdrantFilterError.vectorNotFilterable(column: "vector")) { try condition("vector", "=", "1") } + #expect(try condition("vector.sp", "CONTAINS", "a") == #"{"has_id":[]}"#) + #expect(throws: QdrantFilterError.unsupportedOperator("STARTS WITH")) { try condition("city", "STARTS WITH", "B") } + } + + @Test("A raw row is a Qdrant condition in JSON") + func raw() throws { + #expect(try condition("__RAW__", "", #"{"key":"city","match":{"value":"X"}}"#) == #"{"key":"city","match":{"value":"X"}}"#) + #expect(throws: QdrantFilterError.rawFilterNotAnObject) { try condition("__RAW__", "", "city = 1") } + } + + @Test("AND rows go under must and OR rows under should") + func logic() throws { + let specs = [ + QdrantFilterSpec(column: "city", op: "=", value: "Berlin"), + QdrantFilterSpec(column: "count", op: "<", value: "3") + ] + let and = try #require(try QdrantFilterBuilder.filter(specs, logicMode: "AND")) + let or = try #require(try QdrantFilterBuilder.filter(specs, logicMode: "OR")) + #expect((and["must"] as? [Any])?.count == 2) + #expect((or["should"] as? [Any])?.count == 2) + #expect(try QdrantFilterBuilder.filter([], logicMode: "AND") == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift new file mode 100644 index 0000000000..cd0bbbdaab --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -0,0 +1,173 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantRowWriteTests { + private let types: [String: String] = [ + "id": "id", "city": "keyword", "count": "integer", "pop": "float", "active": "bool", + "note": "string", "meta": "object", "payload.id": "keyword", "mixed": "json", + "vector": "vector(2)", "vector.text": "multivector", "vector.sp": "sparse_vector" + ] + + private func writes(_ changes: [QdrantRowChange]) throws -> [String] { + try QdrantRowWriter.writes( + collection: "docs", + changes: changes, + serverVersion: "1.20.0", + typeName: { types[$0] }, + newID: { "00000000-0000-0000-0000-000000000001" } + ).map(\.request.text) + } + + private func refusal(_ change: QdrantRowChange) -> QdrantRowRefusal? { + do { + _ = try writes([change]) + return nil + } catch { + return error as? QdrantRowRefusal + } + } + + @Test("A payload edit is a set-payload request typed by the column the grid read") + func payloadUpdate() throws { + let change = QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [ + QdrantCell(column: "note", text: "42"), + QdrantCell(column: "count", text: "42"), + QdrantCell(column: "active", text: "1"), + QdrantCell(column: "payload.id", text: "ext"), + QdrantCell(column: "city", text: nil) + ]) + #expect(try writes([change]) == [ + "POST /collections/docs/points/payload?wait=true\n" + + #"{"payload":{"active":true,"city":null,"count":42,"id":"ext","note":"42"},"points":[7]}"# + ]) + } + + @Test("Vector edits update or remove vectors by name, the unnamed one under the empty name") + func vectorUpdate() throws { + let change = QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [ + QdrantCell(column: "vector", text: "[0.5, 1]"), + QdrantCell(column: "vector.sp", text: #"{"indices":[1],"values":[0.5]}"#), + QdrantCell(column: "vector.text", text: nil) + ]) + #expect(try writes([change]) == [ + "PUT /collections/docs/points/vectors?wait=true\n" + + #"{"points":[{"id":7,"vector":{"":[0.5,1],"sp":{"indices":[1],"values":[0.5]}}}]}"#, + "POST /collections/docs/points/vectors/delete?wait=true\n" + #"{"points":[7],"vector":["text"]}"# + ]) + } + + @Test("A value that does not fit its column refuses the row instead of changing the field's type") + func typeRefusals() { + let count = QdrantRowChange(kind: .update, rowIndex: 3, id: "7", cells: [QdrantCell(column: "count", text: "4.5")]) + #expect(refusal(count) == QdrantRowRefusal( + rowIndex: 3, reason: QdrantValueError.notAnInteger(column: "count", value: "4.5").errorDescription ?? "" + )) + let vector = QdrantRowChange(kind: .update, rowIndex: 1, id: "7", cells: [QdrantCell(column: "vector", text: "[\"a\"]")]) + #expect(refusal(vector)?.rowIndex == 1) + let sparse = QdrantRowChange(kind: .update, rowIndex: 1, id: "7", cells: [QdrantCell(column: "vector.sp", text: "[1]")]) + #expect(refusal(sparse)?.rowIndex == 1) + } + + @Test("A column whose type the grid did not report is refused, never written as text") + func unknownType() { + let change = QdrantRowChange(kind: .update, rowIndex: 2, id: "7", cells: [QdrantCell(column: "late", text: "43")]) + #expect(refusal(change)?.rowIndex == 2) + } + + @Test("A delete names the point, and a row without an id or with a changed id is refused") + func delete() throws { + let uuid = "5c56c793-69f3-4fbf-87e6-c4bf54c28c26" + #expect(try writes([QdrantRowChange(kind: .delete, rowIndex: 0, id: uuid)]) == [ + "POST /collections/docs/points/delete?wait=true\n{\"points\":[\"\(uuid)\"]}" + ]) + #expect(refusal(QdrantRowChange(kind: .delete, rowIndex: 4, id: nil))?.rowIndex == 4) + let idChange = QdrantRowChange(kind: .update, rowIndex: 5, id: "1", cells: [QdrantCell(column: "id", text: "2")]) + #expect(refusal(idChange)?.rowIndex == 5) + } + + @Test("A json column saves the JSON its cell spells, so a quoted string stays a string") + func jsonColumnValues() throws { + func written(_ text: String) throws -> String? { + try writes([QdrantRowChange(kind: .update, rowIndex: 0, id: "7", cells: [QdrantCell(column: "mixed", text: text)])]).first + } + let prefix = "POST /collections/docs/points/payload?wait=true\n" + #expect(try written(#""42""#) == prefix + #"{"payload":{"mixed":"42"},"points":[7]}"#) + #expect(try written("42") == prefix + #"{"payload":{"mixed":42},"points":[7]}"#) + #expect(try written(#""[]""#) == prefix + #"{"payload":{"mixed":"[]"},"points":[7]}"#) + #expect(try written(#"["a"]"#) == prefix + #"{"payload":{"mixed":["a"]},"points":[7]}"#) + } + + @Test("A duplicated row's default marker asks for a new id instead of failing as one") + func duplicatedRowGetsNewID() throws { + let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ + QdrantCell(column: "id", text: "__DEFAULT__"), QdrantCell(column: "city", text: "Hanoi") + ]) + #expect(try writes([change]).first?.contains(#""id":"00000000-0000-0000-0000-000000000001""#) == true) + } + + @Test("A server that would overwrite a taken id gets no grid insert") + func insertNeedsConditionalUpsert() { + let change = QdrantRowChange(kind: .insert, rowIndex: 6, id: nil, cells: [QdrantCell(column: "city", text: "Hanoi")]) + for version in ["1.14.1", nil] as [String?] { + #expect(throws: QdrantRowRefusal.self) { + _ = try QdrantRowWriter.writes(collection: "docs", changes: [change], serverVersion: version, typeName: { types[$0] }) + } + } + #expect(QdrantServerVersion.insertsWithoutOverwrite("1.15.0")) + #expect(QdrantServerVersion.insertsWithoutOverwrite("2.0.1-dev")) + #expect(!QdrantServerVersion.insertsWithoutOverwrite("1.9.7")) + } + + @Test("An insert carries its vectors, keeps an existing id untouched, and first checks that the id is free") + func insert() throws { + let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ + QdrantCell(column: "id", text: ""), + QdrantCell(column: "city", text: "Hanoi"), + QdrantCell(column: "active", text: "true"), + QdrantCell(column: "note", text: nil), + QdrantCell(column: "vector", text: "[1,2]") + ]) + let write = try #require(try QdrantRowWriter.writes( + collection: "docs", changes: [change], serverVersion: "1.20.0", typeName: { types[$0] }, + newID: { "00000000-0000-0000-0000-000000000001" } + ).first) + #expect(write.request.text == "PUT /collections/docs/points?wait=true\n" + + #"{"points":[{"id":"00000000-0000-0000-0000-000000000001","payload":{"active":true,"city":"Hanoi"},"vector":{"":[1,2]}}],"update_filter":{"must":[{"has_id":[]}]}}"#) + #expect(write.mustFindNothing?.text == "POST /collections/docs/points\n" + + #"{"ids":["00000000-0000-0000-0000-000000000001"],"with_payload":false,"with_vector":false}"#) + let check = try #require(write.landedCheck.flatMap(QdrantInsertCheck.decode)) + #expect(check.collection == "docs") + #expect(check.id as? String == "00000000-0000-0000-0000-000000000001") + #expect(canonical(check.payload) == #"{"active":true,"city":"Hanoi"}"#) + } + + @Test("The landed check passes only for the point the insert wrote") + func landedCheck() throws { + let written: [String: Any] = ["city": "Hanoi", "count": 3] + let same = QdrantPoint(id: 1, payload: ["count": 3, "city": "Hanoi"], vector: nil, score: nil) + let other = QdrantPoint(id: 1, payload: ["city": "Paris"], vector: nil, score: nil) + #expect(QdrantInsertCheck.landed(same, payload: written, vectors: [:])) + #expect(!QdrantInsertCheck.landed(other, payload: written, vectors: [:])) + #expect(!QdrantInsertCheck.landed(nil, payload: written, vectors: [:])) + + let sent: [String: Any] = ["": [3, 4], "sp": ["indices": [5, 1], "values": [0.5, 0.2]]] + let stored = QdrantPoint(id: 1, payload: [:], vector: ["": [0.6, 0.8], "sp": ["indices": [1, 5], "values": [0.2, 0.5]]], score: nil) + let missing = QdrantPoint(id: 1, payload: [:], vector: ["": [3, 4]], score: nil) + #expect(QdrantInsertCheck.landed(stored, payload: [:], vectors: sent)) + #expect(!QdrantInsertCheck.landed(missing, payload: [:], vectors: sent)) + } + + @Test("Typed values: text stays text, JSON columns parse, and malformed structures are refused") + func payloadValues() throws { + #expect(try QdrantValue.payload("42", type: .keyword, column: "c") as? String == "42") + #expect(try QdrantValue.payload("0", type: .bool, column: "c") as? Bool == false) + #expect((try QdrantValue.payload("9007199254740993", type: .float, column: "c") as? NSNumber)?.int64Value == 9_007_199_254_740_993) + #expect((try QdrantValue.payload("2.5", type: .float, column: "c") as? NSNumber)?.doubleValue == 2.5) + #expect(try QdrantValue.payload(#"["a"]"#, type: .array, column: "c") as? [String] == ["a"]) + #expect(try QdrantValue.payload("42", type: .json, column: "c") as? Int == 42) + #expect(try QdrantValue.payload("plain", type: .json, column: "c") as? String == "plain") + #expect(throws: QdrantValueError.notAnObject(column: "c")) { try QdrantValue.payload("not json", type: .object, column: "c") } + #expect(throws: QdrantValueError.notAGeoPoint(column: "c")) { try QdrantValue.payload("{\"x\":1}", type: .geo, column: "c") } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift new file mode 100644 index 0000000000..6ec0c041d5 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -0,0 +1,205 @@ +import Foundation +@testable import TableProQdrantCore +import Testing + +struct QdrantSchemaTests { + private func info(vectors: Any, sparse: [String: Any]? = nil, payloadSchema: [String: Any] = [:]) -> [String: Any] { + var params: [String: Any] = ["vectors": vectors] + params["sparse_vectors"] = sparse + return ["result": ["points_count": 4, "config": ["params": params], "payload_schema": payloadSchema]] + } + + private func point(_ json: String) throws -> QdrantPoint { + let object = try JSONSerialization.jsonObject(with: Data(json.utf8)) + return try #require(QdrantPoint.parse(object)) + } + + @Test("Columns are the id, indexed fields, sampled fields, then vectors typed by their size") + func columns() throws { + let sample = [ + try point(#"{"id":1,"payload":{"city":"Berlin","count":3,"meta":{"x":1}}}"#), + try point(#"{"id":2,"payload":{"count":3.5,"tags":["a"]}}"#) + ] + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": 4, "distance": "Cosine"], payloadSchema: ["city": ["data_type": "keyword"]]), + sample: sample + ) + let columns = QdrantSchema.columns(for: collection) + #expect(columns.map(\.name) == ["id", "city", "count", "meta", "tags", "vector"]) + #expect(columns.map(\.type) == ["id", "keyword", "float", "object", "array", "vector(4)"]) + #expect(collection.pointsCount == 4) + } + + @Test("A nested index path is an index, not a column that would always read NULL") + func nestedIndexPaths() throws { + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": 2], payloadSchema: [ + "metadata.source": ["data_type": "keyword"], + "tags[]": ["data_type": "keyword"], + "\"a.b\"": ["data_type": "integer"], + "rank": ["data_type": "integer"] + ]), + sample: [try point(#"{"id":1,"payload":{"metadata":{"source":"a.pdf"},"tags":["x"]}}"#)] + ) + #expect(QdrantSchema.columns(for: collection).map(\.name) == ["id", "a.b", "rank", "metadata", "tags", "vector"]) + #expect(collection.indexes.map(\.path) == ["\"a.b\"", "metadata.source", "rank", "tags[]"]) + #expect(QdrantSchema.typeName(of: "metadata", in: collection) == "object") + } + + @Test("A payload key named like a point column shows as payload. and maps back") + func reservedPayloadKeys() throws { + let row = try point(#"{"id":7,"payload":{"id":"external-1","vector":"text","city":"Berlin"},"vector":[1,2]}"#) + let collection = QdrantCollection.parse(name: "docs", info: info(vectors: ["size": 2]), sample: [row]) + let columns = QdrantSchema.columns(for: collection).map(\.name) + #expect(columns == ["id", "city", "payload.id", "payload.vector", "vector"]) + #expect(row.row(columns: columns) == ["7", "Berlin", "external-1", "text", "[1,2]"]) + #expect(QdrantSchema.payloadKey(forColumn: "payload.id") == "id") + } + + @Test("Every payload key maps to one column and back, including one that starts with payload.") + func payloadKeysRoundTrip() { + let keys = ["city", "id", "score", "vector", "vector.image", "payload.id", "payload.city", "payload.", "__RAW__"] + let columns = keys.map(QdrantSchema.column(forPayloadKey:)) + #expect(Set(columns).count == keys.count) + #expect(columns.map(QdrantSchema.payloadKey(forColumn:)) == keys) + #expect(QdrantSchema.column(forPayloadKey: "payload.id") == "payload.payload.id") + } + + @Test("A cell is typed by what points store; the index names it only when every value fits") + func indexedLists() throws { + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": 2], payloadSchema: [ + "tags": ["data_type": "keyword"], "mixed": ["data_type": "keyword"], "city": ["data_type": "keyword"], + "code": ["data_type": "keyword"], "price": ["data_type": "float"], "fresh": ["data_type": "integer"] + ]), + sample: [ + try point(#"{"id":1,"payload":{"tags":["a","b"],"mixed":["a"],"city":"Berlin","code":42,"price":3}}"#), + try point(#"{"id":2,"payload":{"tags":["c"],"mixed":"b","price":2.5}}"#) + ] + ) + let types = Dictionary(uniqueKeysWithValues: QdrantSchema.columns(for: collection).map { ($0.name, $0.type) }) + #expect(types["tags"] == "array") + #expect(types["mixed"] == "json") + #expect(types["city"] == "keyword") + #expect(types["code"] == "integer") + #expect(types["price"] == "float") + #expect(types["fresh"] == "integer") + } + + @Test("A page's points widen a sampled type without adding columns the table did not open with") + func pageWidensTypes() throws { + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": 2], payloadSchema: ["tags": ["data_type": "keyword"]]), + sample: [try point(#"{"id":1,"payload":{"tags":"a","note":"x"}}"#)] + ) + let page = [try point(#"{"id":101,"payload":{"tags":["a","b"],"note":["y"],"late":1}}"#)] + let typed = collection.including(page, addingKeys: false) + let types = Dictionary(uniqueKeysWithValues: QdrantSchema.columns(for: typed).map { ($0.name, $0.type) }) + #expect(types["tags"] == "json") + #expect(types["note"] == "json") + #expect(types["late"] == nil) + } + + @Test("A payload key named score never stands in for the similarity score column") + func scoreColumnIsThePoints() throws { + let hit = try point(#"{"id":1,"score":0.75,"payload":{"score":"high"}}"#) + let collection = QdrantCollection(name: "docs", vectors: [], payloadFields: [ + QdrantPayloadField(name: "score", type: .json, isIndexed: false) + ]) + let columns = QdrantPoint.columns(of: [hit], collection: collection) + #expect(columns == ["id", "score", "payload.score"]) + #expect(QdrantSchema.typeName(of: "score", in: collection) == "float") + #expect(QdrantSchema.typeName(of: "payload.score", in: collection) == "json") + #expect(hit.row(columns: columns, quotingTextIn: ["score", "payload.score"]) == ["1", "0.75", #""high""#]) + } + + @Test("A json column keeps a string's quotes, so 42 and \"42\" read apart") + func jsonCellsQuoteText() throws { + let text = try point(#"{"id":1,"payload":{"mixed":"42"}}"#) + let number = try point(#"{"id":2,"payload":{"mixed":42}}"#) + #expect(text.row(columns: ["mixed"], quotingTextIn: ["mixed"]) == [#""42""#]) + #expect(number.row(columns: ["mixed"], quotingTextIn: ["mixed"]) == ["42"]) + #expect(text.row(columns: ["mixed"], quotingTextIn: []) == ["42"]) + #expect(try QdrantValue.payload(#""42""#, type: .json, column: "mixed") as? String == "42") + } + + @Test("A named vector called size is not mistaken for the unnamed vector's dimension") + func vectorNamedSize() { + let collection = QdrantCollection.parse( + name: "docs", + info: info(vectors: ["size": ["size": 3, "distance": "Cosine"], "text": ["size": 2, "distance": "Dot"]]) + ) + #expect(collection.vectors.map(\.column) == ["vector.size", "vector.text"]) + } + + @Test("Keys outside the sample join the collection, and a key first seen as null takes its later type") + func includingLaterPoints() throws { + let sample = [try point(#"{"id":1,"payload":{"city":"Berlin","rank":null}}"#)] + let collection = QdrantCollection.parse(name: "docs", info: info(vectors: ["size": 2]), sample: sample) + let widened = collection.including([try point(#"{"id":2,"payload":{"late":true,"rank":3}}"#)]) + #expect(widened.payloadFields.map(\.name) == ["city", "rank", "late"]) + #expect(QdrantSchema.typeName(of: "rank", in: widened) == "integer") + #expect(QdrantSchema.typeName(of: "late", in: widened) == "bool") + } + + @Test("Named, multi and sparse vectors each get a column typed by their kind") + func namedVectors() throws { + let collection = QdrantCollection.parse( + name: "docs", + info: info( + vectors: ["image": ["size": 2], "text": ["size": 3, "multivector_config": ["comparator": "max_sim"]]], + sparse: ["keywords": [String: Any]()] + ) + ) + let columns = QdrantSchema.columns(for: collection) + #expect(columns.map(\.name) == ["id", "vector.image", "vector.text", "vector.keywords"]) + #expect(columns.map(\.type) == ["id", "vector(2)", "multivector", "sparse_vector"]) + + let row = try point(#"{"id":"5c56c793-69f3-4fbf-87e6-c4bf54c28c26","vector":{"image":[0.5,1],"keywords":{"indices":[1],"values":[0.5]}}}"#) + #expect(row.cell(for: "id") == "5c56c793-69f3-4fbf-87e6-c4bf54c28c26") + #expect(row.cell(for: "vector.image") == "[0.5,1]") + #expect(row.cell(for: "vector.keywords") == #"{"indices":[1],"values":[0.5]}"#) + #expect(row.cell(for: "vector.text") == nil) + } + + @Test("An unnamed vector beside sparse ones comes back under the empty name") + func unnamedBesideSparse() throws { + let row = try point(#"{"id":1,"vector":{"":[1,2],"sp":{"indices":[0],"values":[1]}}}"#) + #expect(row.cell(for: "vector") == "[1,2]") + #expect(QdrantPoint.columns(of: [row], collection: nil) == ["id", "vector", "vector.sp"]) + } + + @Test("A vector type name maps back to its kind") + func vectorKinds() { + #expect(QdrantVectorSpec.kind(ofTypeName: "vector(384)") == .dense(size: 384)) + #expect(QdrantVectorSpec.kind(ofTypeName: "VECTOR") == .dense(size: nil)) + #expect(QdrantVectorSpec.kind(ofTypeName: "multivector") == .multi) + #expect(QdrantVectorSpec.kind(ofTypeName: "sparse_vector") == .sparse) + #expect(QdrantVectorSpec.kind(ofTypeName: "keyword") == nil) + } + + @Test("A query result shows the score after the id") + func queryColumns() throws { + let json = try JSONSerialization.jsonObject(with: Data( + #"{"result":{"points":[{"id":1,"score":0.9,"payload":{"city":"Berlin"},"vector":[1,2]}]},"status":"ok"}"#.utf8 + )) + let points = try #require(QdrantPoint.list(fromResponse: json)) + let columns = QdrantPoint.columns(of: points, collection: nil) + #expect(columns == ["id", "score", "city", "vector"]) + #expect(points[0].row(columns: columns) == ["1", "0.9", "Berlin", "[1,2]"]) + #expect(QdrantPoint.list(fromResponse: ["result": ["count": 3]]) == nil) + #expect(QdrantPoint.list(fromResponse: ["result": [Any]()])?.isEmpty == true) + } + + @Test("Point ids parse back to the kind Qdrant expects") + func pointIDs() { + #expect(QdrantPointID.json(from: "42") as? NSNumber == 42) + #expect(QdrantPointID.json(from: "5C56C793-69F3-4FBF-87E6-C4BF54C28C26") as? String == "5c56c793-69f3-4fbf-87e6-c4bf54c28c26") + #expect(QdrantPointID.json(from: "-1") == nil) + #expect(QdrantPointID.json(from: "abc") == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift new file mode 100644 index 0000000000..7eed1272c5 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift @@ -0,0 +1,81 @@ +import Foundation +@testable import TableProQdrantCore + +final class RecordingTransport: QdrantTransport, @unchecked Sendable { + private let lock = NSLock() + private let handler: (QdrantHTTPRequest) throws -> QdrantHTTPResponse + private var requests: [QdrantHTTPRequest] = [] + + init(_ handler: @escaping (QdrantHTTPRequest) throws -> QdrantHTTPResponse) { + self.handler = handler + } + + func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse { + lock.withLock { requests.append(request) } + return try handler(request) + } + + func cancelAll() {} + + var recorded: [QdrantHTTPRequest] { lock.withLock { requests } } + + var scrolls: [QdrantHTTPRequest] { recorded.filter { $0.url.path.hasSuffix("/points/scroll") } } +} + +func jsonResponse(_ object: Any, status: Int = 200) -> QdrantHTTPResponse { + QdrantHTTPResponse(statusCode: status, body: (try? JSONSerialization.data(withJSONObject: object)) ?? Data()) +} + +func bodyObject(_ request: QdrantHTTPRequest) -> [String: Any] { + guard let body = request.body else { return [:] } + return (try? JSONSerialization.jsonObject(with: body) as? [String: Any]) ?? [:] +} + +func makeClient(apiKey: String = "", transport: RecordingTransport) -> QdrantClient { + QdrantClient( + settings: QdrantConnectionSettings(host: "localhost", port: 6_333, usesTLS: false, apiKey: apiKey), + transport: transport, + timeout: { 5 } + ) +} + +func scrollResponse(_ ids: [Int], next: Int?) -> QdrantHTTPResponse { + let points = ids.map { ["id": $0, "payload": ["n": $0]] as [String: Any] } + return jsonResponse(["result": ["points": points, "next_page_offset": next.map { $0 as Any } ?? NSNull()], "status": "ok"]) +} + +// Integer ids 1...count, answering `scroll` from `offset` (an id) the way Qdrant does. +func scrollingTransport(count: Int) -> RecordingTransport { + RecordingTransport { request in + if request.url.path.hasSuffix("/points/count") { + return jsonResponse(["result": ["count": count], "status": "ok"]) + } + let body = bodyObject(request) + let limit = body["limit"] as? Int ?? 10 + let start = body["offset"] as? Int ?? 1 + let ids = Array(start.. String { + (try? QdrantJSON.text(object)) ?? "" +} diff --git a/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/ServerCertificatePurposeTests.swift b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/ServerCertificatePurposeTests.swift new file mode 100644 index 0000000000..6e10a38aa0 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/ServerCertificatePurposeTests.swift @@ -0,0 +1,37 @@ +#if os(macOS) +import Foundation +import Security +import TableProTLSClientIdentity +import TableProTLSTestFixtures +import Testing + +struct ServerCertificatePurposeTests { + private func certificate(_ pem: String) throws -> SecCertificate { + try #require(SecCertificateCreateWithData(nil, TLSTestFixtures.der(pem) as CFData)) + } + + private func trust(_ leafPEM: String) throws -> SecTrust { + var trust: SecTrust? + SecTrustCreateWithCertificates(try certificate(leafPEM), SecPolicyCreateBasicX509(), &trust) + let created = try #require(trust) + SecTrustSetAnchorCertificates(created, [try certificate(TLSTestFixtures.caCertificate)] as CFArray) + SecTrustSetAnchorCertificatesOnly(created, true) + return created + } + + @Test("A client-only certificate chains to the CA under the X.509 policy but is not a server's") + func clientCertificateRefused() throws { + let trust = try trust(TLSTestFixtures.clientCertificate) + #expect(SecTrustEvaluateWithError(trust, nil)) + #expect(!ServerCertificatePurpose.leafPermitsServerAuthentication(trust)) + } + + @Test("A server certificate and a certificate with no key usage list both pass") + func serverCertificatePasses() throws { + let trust = try trust(TLSTestFixtures.serverCertificate) + #expect(SecTrustEvaluateWithError(trust, nil)) + #expect(ServerCertificatePurpose.leafPermitsServerAuthentication(trust)) + #expect(ServerCertificatePurpose.permitsServerAuthentication(try certificate(TLSTestFixtures.caCertificate))) + } +} +#endif diff --git a/apple/Plugins/QdrantDriverPlugin/Info.plist b/apple/Plugins/QdrantDriverPlugin/Info.plist new file mode 100644 index 0000000000..a9c5f4eb04 --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/Info.plist @@ -0,0 +1,12 @@ + + + + + TableProPluginKitVersion + TABLEPRO_PLUGINKIT_VERSION + TableProProvidesDatabaseTypeIds + + Qdrant + + + diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift new file mode 100644 index 0000000000..7b6a90473d --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift @@ -0,0 +1,116 @@ +import Foundation +import TableProPluginKit +import TableProQdrantCore + +final class QdrantPlugin: NSObject, TableProPlugin, DriverPlugin, PluginDefaultSortProvider { + static let pluginName = "Qdrant Driver" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Qdrant support over the REST API" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "Qdrant" + static let databaseDisplayName = "Qdrant" + static let iconName = "qdrant-icon" + static let defaultPort = QdrantConnectionSettings.defaultPort + static let isDownloadable = true + + static let navigationModel: NavigationModel = .standard + static let pathFieldRole: PathFieldRole = .database + static let requiresAuthentication = false + static let brandColorHex = "#DC244C" + static let queryLanguageName = "REST" + static let editorLanguage: EditorLanguage = .javascript + static let supportsForeignKeys = false + static let supportsSchemaEditing = false + static let supportsDatabaseSwitching = false + static let supportsImport = false + static let supportsExport = true + static let supportsSSH = true + static let supportsSSL = true + static let supportsReadOnlyMode = true + static let supportsForeignKeyDisable = false + static let supportsAddColumn = false + static let supportsModifyColumn = false + static let supportsDropColumn = false + static let supportsAddIndex = false + static let supportsDropIndex = false + static let supportsModifyPrimaryKey = false + static let databaseGroupingStrategy: GroupingStrategy = .flat + static let defaultGroupName = "default" + static let tableEntityName = "Collections" + static let containerEntityName = "Cluster" + static let immutableColumns: [String] = [QdrantSchema.idColumn] + static let defaultPrimaryKeyColumn: String? = QdrantSchema.idColumn + static let structureColumnFields: [StructureColumnField] = [.name, .type, .nullable] + static let sqlDialect: SQLDialectDescriptor? = nil + + static let columnTypesByCategory: [String: [String]] = qdrantColumnTypes + + static let additionalConnectionFields: [ConnectionField] = qdrantConnectionFields() + + static var statementCompletions: [CompletionEntry] { qdrantCompletions } + + // Points already come in id order, and Qdrant cannot list them by id descending. + func defaultSortHint(forTable table: String) -> DefaultSortHint { + .suppress + } + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + QdrantPluginDriver(config: config) + } +} + +func qdrantConnectionFields() -> [ConnectionField] { + [ + ConnectionField( + id: QdrantFieldID.apiKey, + label: String(localized: "API Key"), + placeholder: "Qdrant API key", + fieldType: .secure, + section: .authentication, + hidesPassword: true + ).withHidesUsername(true) + ] +} + +let qdrantCompletions: [CompletionEntry] = [ + CompletionEntry(label: "GET /collections", insertText: "GET /collections"), + CompletionEntry(label: "GET /collections/{name}", insertText: "GET /collections/docs"), + CompletionEntry( + label: "POST /points/scroll", + insertText: """ + POST /collections/docs/points/scroll + {"limit": 10, "with_payload": true} + """ + ), + CompletionEntry( + label: "POST /points/query", + insertText: """ + POST /collections/docs/points/query + {"query": [0.1, 0.2, 0.3, 0.4], "limit": 10, "with_payload": true} + """ + ), + CompletionEntry( + label: "POST /points/count", + insertText: """ + POST /collections/docs/points/count + {"exact": true} + """ + ), + CompletionEntry( + label: "POST /facet", + insertText: """ + POST /collections/docs/facet + {"key": "city"} + """ + ) +] + +let qdrantColumnTypes: [String: [String]] = [ + "Text": ["keyword", "text", "string", "uuid"], + "Numeric": ["integer", "float"], + "Boolean": ["bool"], + "Date": ["datetime"], + "Structured": ["geo", "object", "array", "json"], + "Vector": ["vector", "multivector", "sparse_vector"] +] diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift new file mode 100644 index 0000000000..4bf1ae92c0 --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -0,0 +1,198 @@ +import Foundation +import TableProPluginKit +import TableProQdrantCore + +extension QdrantPluginDriver { + static let exportPageSize = 500 + + func execute(query: String) async throws -> PluginQueryResult { + let started = Date() + let client = try requireClient() + let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + + if trimmed.lowercased() == "select 1" { + try await client.ping() + return PluginQueryResult( + columns: ["ok"], + columnTypeNames: ["integer"], + rows: [[.text("1")]], + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } + if let request = QdrantBrowseQuery.parse(trimmed) { + return try await executeBrowse(request, client: client, started: started) + } + if let check = QdrantInsertCheck.decode(trimmed) { + let response = try await client.execute(QdrantConsoleRequest( + method: "POST", + path: "/collections/\(QdrantPathEncoding.segment(check.collection))/points", + body: try QdrantJSON.text(["ids": [check.id], "with_payload": true, "with_vector": true]) + )) + let point = QdrantPoint.list(fromResponse: response.json)?.first + let missed = !QdrantInsertCheck.landed(point, payload: check.payload, vectors: check.vectors) + return PluginQueryResult( + columns: [QdrantSchema.idColumn], + columnTypeNames: ["id"], + rows: missed ? [[QdrantJSON.displayText(check.id).map(PluginCellValue.text) ?? .null]] : [], + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } + if let request = QdrantConsoleParser.parse(trimmed) { + return try await executeConsole(request, client: client, started: started) + } + throw QdrantError.malformedResponse( + String(localized: "Enter a request like GET /collections or POST /collections/docs/points/query.") + ) + } + + private func executeBrowse( + _ request: QdrantBrowseRequest, + client: QdrantClient, + started: Date + ) async throws -> PluginQueryResult { + let collection = try await collection(request.collection) + let filter = try QdrantFilterBuilder.filter(request.filters, logicMode: request.logicMode) + let points: [QdrantPoint] + switch try QdrantSortPlan.resolve(request.sorts) { + case .byID: + points = try await client.pointPage( + collection: request.collection, filter: filter, offset: request.offset, limit: request.limit + ) + case .byPayload(let column, let key, let ascending): + points = try await client.sortedPage( + collection: request.collection, filter: filter, column: column, key: key, ascending: ascending, + offset: request.offset, limit: request.limit, cache: paging + ) + } + let typed = collection.including(points, addingKeys: false) + if typed != collection { + remember(typed) + } + let columns = QdrantSchema.columns(for: typed).map(\.name) + return render(points, columns: columns, collection: typed, started: started) + } + + private func executeConsole( + _ request: QdrantConsoleRequest, + client: QdrantClient, + started: Date + ) async throws -> PluginQueryResult { + let response = try await client.execute(request) + if request.kind.tier != .read { + forgetAfterWrite() + } + // An empty point list is a grid with no rows, which is what a save's existence check counts. + let route = QdrantPathEncoding.segments(of: request.path) ?? [] + if let points = QdrantPoint.list(fromResponse: response.json), !points.isEmpty || route.contains("points") { + let name = route.count >= 2 && route[0] == "collections" ? Self.collectionName(in: request.path) : nil + let known = name.flatMap(rememberedCollection) + // Typed by the points that came back as well, so an export of a fresh query keeps numbers numbers. + let typed = (known ?? QdrantCollection(name: name ?? "", vectors: [], payloadFields: [])).including(points) + return render( + points, + columns: QdrantPoint.columns(of: points, collection: known), + collection: typed, + started: started + ) + } + return renderJSON(response, started: started) + } + + static func collectionName(in path: String) -> String? { + let segments = path.split(separator: "?").first?.split(separator: "/") ?? [] + guard segments.count >= 2, segments[0] == "collections" else { return nil } + return String(segments[1]).removingPercentEncoding + } + + private func render( + _ points: [QdrantPoint], + columns: [String], + collection: QdrantCollection, + started: Date + ) -> PluginQueryResult { + let typeNames = columns.map { QdrantSchema.typeName(of: $0, in: collection) } + return PluginQueryResult( + columns: columns, + columnTypeNames: typeNames, + rows: points.map(cells(columns: columns, typeNames: typeNames)), + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } + + private func cells(columns: [String], typeNames: [String]) -> (QdrantPoint) -> [PluginCellValue] { + let jsonColumns = Set(zip(columns, typeNames).filter { $0.1 == QdrantFieldType.json.rawValue }.map(\.0)) + return { point in + point.row(columns: columns, quotingTextIn: jsonColumns).map { $0.map(PluginCellValue.text) ?? .null } + } + } + + private func renderJSON(_ response: QdrantHTTPResponse, started: Date) -> PluginQueryResult { + let text: String + if let json = response.json, JSONSerialization.isValidJSONObject(json), + let pretty = try? QdrantJSON.text(json, pretty: true) { + text = pretty + } else { + text = response.text + } + return PluginQueryResult( + columns: ["response"], + columnTypeNames: ["json"], + rows: [[.text(text)]], + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } + + // MARK: - Export + + // The protocol's default `streamRows` runs `execute` once and buffers the whole collection. + func streamRows(query: String) -> AsyncThrowingStream { + AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + let task = Task { + do { + if let name = QdrantOperations.decodeExport(query) { + try await self.streamCollection(name, continuation: continuation) + } else { + let result = try await self.execute(query: query) + continuation.yield(.header(PluginStreamHeader( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + estimatedRowCount: nil + ))) + if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { @Sendable _ in task.cancel() } + } + } + + // The grid's columns come from a sample; an export carries every key, so it first reads payloads + // alone, which skips the vectors that make up most of a page. + private func streamCollection( + _ name: String, + continuation: AsyncThrowingStream.Continuation + ) async throws { + let client = try requireClient() + var collection = try await collection(name, refresh: true) + for try await points in client.pointPages(collection: name, pageSize: Self.exportPageSize, withVector: false) { + collection = collection.including(points) + } + let columns = QdrantSchema.columns(for: collection).map(\.name) + let typeNames = columns.map { QdrantSchema.typeName(of: $0, in: collection) } + continuation.yield(.header(PluginStreamHeader( + columns: columns, + columnTypeNames: typeNames, + estimatedRowCount: collection.pointsCount + ))) + for try await points in client.pointPages(collection: name, pageSize: Self.exportPageSize) { + continuation.yield(.rows(points.map(cells(columns: columns, typeNames: typeNames)))) + } + } +} diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift new file mode 100644 index 0000000000..ca822ad6e3 --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift @@ -0,0 +1,210 @@ +import Foundation +import TableProPluginKit +import TableProQdrantCore + +extension QdrantPluginDriver { + func fetchDatabases() async throws -> [String] { + [QdrantPlugin.defaultGroupName] + } + + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + try await requireClient().collectionNames().map { PluginTableInfo(name: $0, type: "TABLE") } + } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + QdrantSchema.columns(for: try await collection(table, refresh: true)).map { column in + PluginColumnInfo( + name: column.name, + dataType: column.type, + isNullable: !column.isPrimaryKey, + isPrimaryKey: column.isPrimaryKey + ) + } + } + + func fetchImmutableColumns(table: String, schema: String?) async throws -> Set { + [QdrantSchema.idColumn] + } + + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { + let collection = try await requireClient().collectionInfo(named: table) + let primary = PluginIndexInfo( + name: QdrantSchema.idColumn, + columns: [QdrantSchema.idColumn], + isUnique: true, + isPrimary: true, + type: "PRIMARY KEY" + ) + return [primary] + collection.indexes.map { index in + PluginIndexInfo(name: index.path, columns: [index.path], isUnique: false, isPrimary: false, type: index.type) + } + } + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + [] + } + + func fetchTableDDL(table: String, schema: String?) async throws -> String { + try await requireClient().collectionInfo(named: table).info + } + + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + throw QdrantError.configuration(String(localized: "Qdrant does not support views.")) + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table, engine: "Qdrant") + } + + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { + try await requireClient().collectionInfo(named: table).pointsCount + } + + func fetchFilteredRowCount(table: String, queryFilters: [PluginQueryFilter], logicMode: String) async throws -> Int? { + let filter = try QdrantFilterBuilder.filter(specs(queryFilters), logicMode: logicMode) + return try await requireClient().count(collection: table, filter: filter) + } + + func buildBrowseQuery( + table: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int + ) -> String? { + buildFilteredQuery( + table: table, + schema: nil, + queryFilters: [], + logicMode: "AND", + sortColumns: sortColumns, + columns: columns, + limit: limit, + offset: offset, + columnKinds: [:] + ) + } + + // Every sort column travels with the page, so one Qdrant cannot honor is refused instead of dropped. + func buildFilteredQuery( + table: String, + schema: String?, + queryFilters filters: [PluginQueryFilter], + logicMode: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int, + columnKinds: [String: PluginColumnKind] + ) -> String? { + let sorts = sortColumns.compactMap { sort -> QdrantSortSpec? in + guard columns.indices.contains(sort.columnIndex) else { return nil } + return QdrantSortSpec(column: columns[sort.columnIndex], ascending: sort.ascending) + } + return QdrantBrowseQuery.encode(QdrantBrowseRequest( + collection: table, + offset: offset, + limit: limit, + sorts: sorts, + filters: specs(filters), + logicMode: logicMode + )) + } + + func specs(_ filters: [PluginQueryFilter]) -> [QdrantFilterSpec] { + filters.map { + QdrantFilterSpec(column: $0.column, op: $0.op, value: $0.value, secondValue: $0.secondValue) + } + } + + // Typed by what the grid read each column as, so a cache a console write or a reconnect cleared + // can never turn a number into text. + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set, + context: PluginRowWriteContext + ) throws -> [PluginRowWrite]? { + let known = rememberedCollection(table) + let typeNames = context.columnTypeNames + let tracked = changes.compactMap { change in + rowChange(change, columns: columns, insertedRowData: insertedRowData, + deletedRowIndices: deletedRowIndices, insertedRowIndices: insertedRowIndices) + } + do { + return try QdrantRowWriter.writes(collection: table, changes: tracked, serverVersion: serverVersion) { column in + typeNames[column] ?? known.flatMap { QdrantSchema.knownTypeName(of: column, in: $0) } + }.map { write in + PluginRowWrite( + statement: write.request.text, + rowIndices: [write.rowIndex], + lookup: write.mustFindNothing.map { + PluginRowLookup(query: $0.text, parameters: [], rowCount: 0, pinsRows: false, check: write.landedCheck) + } + ) + } + } catch let refusal as QdrantRowRefusal { + throw PluginRowWriteRefusal(rowIndex: refusal.rowIndex, reason: refusal.reason) + } + } + + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try generateRowWrites( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, + insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices, context: PluginRowWriteContext() + ) + } + + private func rowChange( + _ change: PluginRowChange, + columns: [String], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) -> QdrantRowChange? { + switch change.type { + case .insert: + guard insertedRowIndices.contains(change.rowIndex) else { return nil } + let cells: [QdrantCell] + if let row = insertedRowData[change.rowIndex] { + cells = zip(columns, row).map { QdrantCell(column: $0, text: $1.asText) } + } else { + cells = change.cellChanges.map { QdrantCell(column: $0.columnName, text: $0.newValue.asText) } + } + return QdrantRowChange(kind: .insert, rowIndex: change.rowIndex, id: nil, cells: cells) + case .update: + let cells = change.cellChanges.map { QdrantCell(column: $0.columnName, text: $0.newValue.asText) } + return QdrantRowChange(kind: .update, rowIndex: change.rowIndex, id: pointID(from: change, columns: columns), cells: cells) + case .delete: + guard deletedRowIndices.contains(change.rowIndex) else { return nil } + return QdrantRowChange(kind: .delete, rowIndex: change.rowIndex, id: pointID(from: change, columns: columns)) + } + } + + private func pointID(from change: PluginRowChange, columns: [String]) -> String? { + guard let original = change.originalRow, + let index = columns.firstIndex(of: QdrantSchema.idColumn), + index < original.count + else { return nil } + return original[index].asText + } +} diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift new file mode 100644 index 0000000000..fe11918a98 --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift @@ -0,0 +1,122 @@ +import Foundation +import os +import TableProPluginKit +import TableProQdrantCore + +internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendable { + static let logger = Logger(subsystem: "com.TablePro", category: "QdrantPluginDriver") + + private let config: DriverConnectionConfig + private let lock = NSLock() + private var client: QdrantClient? + private var cachedCollections: [String: QdrantCollection] = [:] + let paging = QdrantPagingCache() + let queryTimeout = HttpQueryTimeoutBox() + + init(config: DriverConnectionConfig) { + self.config = config + } + + var serverVersion: String? { lock.withLock { client?.serverVersion } } + + var supportsTransactions: Bool { false } + + var capabilities: PluginCapabilities { [.cancelQuery] } + + var parameterStyle: ParameterStyle { .questionMark } + + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + + func connect() async throws { + let settings = try QdrantConnectionSettings.parse( + host: config.host, + port: config.port, + usesTLS: config.ssl.isEnabled, + fields: config.additionalFields + ) + let deadline = PluginConnectDeadline(milliseconds: PluginConnectTimeout.milliseconds( + in: config.additionalFields, + default: Int(HttpQueryTimeout().requestTimeoutInterval * 1_000) + )) + let timeout = queryTimeout + let client = QdrantClient( + settings: settings, + transport: URLSessionQdrantTransport( + resourceTimeout: HttpQueryTimeout.sessionResourceTimeout, + tls: config.ssl.isEnabled ? try QdrantTLS.make(from: config.ssl) : .systemDefault + ), + timeout: { timeout.requestTimeoutInterval } + ) + try await client.connect(requestTimeout: { deadline.remainingSeconds() }) + lock.withLock { + self.client = client + cachedCollections.removeAll() + } + paging.removeAll() + } + + func disconnect() { + lock.withLock { + client?.cancelAll() + client = nil + cachedCollections.removeAll() + } + paging.removeAll() + } + + func ping() async throws { + try await requireClient().ping() + } + + func cancelQuery() throws { + lock.withLock { client?.cancelAll() } + } + + func applyQueryTimeout(_ seconds: Int) async throws { + queryTimeout.set(serverTimeoutSeconds: seconds) + } + + func requireClient() throws -> QdrantClient { + guard let client = lock.withLock({ client }) else { + throw QdrantError.notConnected + } + return client + } + + func rememberedCollection(_ name: String) -> QdrantCollection? { + lock.withLock { cachedCollections[name] } + } + + func collection(_ name: String, refresh: Bool = false) async throws -> QdrantCollection { + if !refresh, let cached = rememberedCollection(name) { + return cached + } + let collection = try await requireClient().collection(named: name) + remember(collection) + return collection + } + + func remember(_ collection: QdrantCollection) { + lock.withLock { cachedCollections[collection.name] = collection } + } + + // A write moves points between pages and may add payload keys. + func forgetAfterWrite() { + lock.withLock { cachedCollections.removeAll() } + paging.removeAll() + } + + func defaultExportQuery(table: String) -> String? { + QdrantOperations.encodeExport(collection: table) + } + + func dropObjectStatement(name: String, objectType: String, schema: String?, cascade: Bool) -> String? { + QdrantOperations.deleteCollection(named: name, objectType: objectType) + } + + func truncateTableStatements(table: String, schema: String?, cascade: Bool) -> [String]? { + QdrantOperations.deleteAllPoints(collection: table).map { [$0] } + } +} diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift b/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift new file mode 100644 index 0000000000..1494bb383a --- /dev/null +++ b/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift @@ -0,0 +1,227 @@ +import Foundation +import Security +import TableProPluginKit +import TableProQdrantCore +import TableProTLSClientIdentity + +struct QdrantTLS: @unchecked Sendable { + enum Verification: Sendable, Equatable { + case none + case caOnly + case full + } + + let verification: Verification + let anchorDER: Data? + let clientCredential: URLCredential? + + static let systemDefault = QdrantTLS(verification: .full, anchorDER: nil, clientCredential: nil) + + // Preferred and Required encrypt without checking the certificate, as the SSL Mode picker says. + static func make(from ssl: SSLConfiguration) throws -> QdrantTLS { + let verification: Verification + switch ssl.mode { + case .disabled, .preferred, .required: verification = .none + case .verifyCa: verification = .caOnly + case .verifyIdentity: verification = .full + } + let anchor = try anchorCertificate(at: ssl.caCertificatePath, for: verification) + if verification == .caOnly, anchor == nil { + throw QdrantError.configuration(String(localized: """ + Verify CA needs a CA certificate. On the connection's Network tab, choose the CA certificate \ + that signed the server's certificate, or set SSL Mode to Verify Identity. + """)) + } + return QdrantTLS(verification: verification, anchorDER: anchor, clientCredential: try clientCredential(for: ssl)) + } + + private static func anchorCertificate(at path: String, for verification: Verification) throws -> Data? { + let trimmed = path.trimmingCharacters(in: .whitespaces) + guard verification != .none, !trimmed.isEmpty else { return nil } + guard let data = try? Data(contentsOf: URL(fileURLWithPath: trimmed)), + let der = PEMCertificateDecoder.certificateDER(from: data), + SecCertificateCreateWithData(nil, der as CFData) != nil + else { + throw QdrantError.configuration(String( + format: String(localized: "The CA certificate at %@ could not be read as a PEM or DER certificate."), + trimmed + )) + } + return der + } + + private static func clientCredential(for ssl: SSLConfiguration) throws -> URLCredential? { + let certificatePath = ssl.clientCertificatePath.trimmingCharacters(in: .whitespaces) + let keyPath = ssl.clientKeyPath.trimmingCharacters(in: .whitespaces) + guard ssl.isEnabled, !certificatePath.isEmpty else { return nil } + guard !keyPath.isEmpty else { + throw QdrantError.configuration(String(localized: """ + A client certificate needs its client key. On the connection's Network tab, choose the \ + client key, or clear the client certificate. + """)) + } + do { + return try TLSClientIdentity.credential( + certificateFile: URL(fileURLWithPath: certificatePath), + privateKeyFile: URL(fileURLWithPath: keyPath) + ) + } catch let failure as TLSClientIdentityError { + throw QdrantError.configuration(failure.message(certificatePath: certificatePath, keyPath: keyPath)) + } + } +} + +final class URLSessionQdrantTransport: QdrantTransport, @unchecked Sendable { + private let session: URLSession + private let tls: QdrantTLS + private let lock = NSLock() + private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] + + // No session delegate: URLSession then hands the trust and client certificate challenges to the + // per-task delegate, which can say why it refused instead of surfacing a bare cancel. + init(resourceTimeout: TimeInterval, tls: QdrantTLS) { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForResource = resourceTimeout + session = URLSession(configuration: configuration) + self.tls = tls + } + + deinit { + session.invalidateAndCancel() + } + + func cancelAll() { + let tasks = lock.withLock { Array(inFlight.values) } + tasks.forEach { $0.cancel() } + } + + func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = request.timeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + + let delegate = QdrantTaskDelegate(transport: self, tls: tls) + defer { delegate.finish() } + do { + let (data, response) = try await session.data(for: urlRequest, delegate: delegate) + guard let http = response as? HTTPURLResponse else { + throw QdrantError.transport(String(localized: "Qdrant answered with something other than HTTP.")) + } + return QdrantHTTPResponse(statusCode: http.statusCode, body: data) + } catch let error as URLError where error.code == .cancelled { + if let refusal = delegate.refusal { + throw QdrantError.transport(refusal) + } + throw QdrantError.cancelled + } catch let error as QdrantError { + throw error + } catch let error as URLError { + throw QdrantError.transport(error.localizedDescription) + } + } + + fileprivate func register(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = task } + } + + fileprivate func unregister(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = nil } + } +} + +private final class QdrantTaskDelegate: NSObject, URLSessionTaskDelegate, @unchecked Sendable { + private weak var transport: URLSessionQdrantTransport? + private let tls: QdrantTLS + private let lock = NSLock() + private var task: URLSessionTask? + private var refusalMessage: String? + + init(transport: URLSessionQdrantTransport, tls: QdrantTLS) { + self.transport = transport + self.tls = tls + } + + var refusal: String? { lock.withLock { refusalMessage } } + + func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { + lock.withLock { self.task = task } + transport?.register(task) + } + + func urlSession( + _ session: URLSession, + task: URLSessionTask, + didReceive challenge: URLAuthenticationChallenge, + completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void + ) { + switch challenge.protectionSpace.authenticationMethod { + case NSURLAuthenticationMethodServerTrust: + guard let trust = challenge.protectionSpace.serverTrust else { + completionHandler(.performDefaultHandling, nil) + return + } + if let failure = evaluate(trust) { + lock.withLock { refusalMessage = failure } + completionHandler(.cancelAuthenticationChallenge, nil) + } else if tls.verification == .full, tls.anchorDER == nil { + completionHandler(.performDefaultHandling, nil) + } else { + completionHandler(.useCredential, URLCredential(trust: trust)) + } + case NSURLAuthenticationMethodClientCertificate: + if let credential = tls.clientCredential { + completionHandler(.useCredential, credential) + } else { + completionHandler(.performDefaultHandling, nil) + } + default: + completionHandler(.performDefaultHandling, nil) + } + } + + // A redirect would carry the API key to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } + + func finish() { + guard let task = lock.withLock({ task }) else { return } + transport?.unregister(task) + } + + private func evaluate(_ trust: SecTrust) -> String? { + switch tls.verification { + case .none: + return nil + case .full where tls.anchorDER == nil: + return nil + case .caOnly, .full: + if let der = tls.anchorDER, let anchor = SecCertificateCreateWithData(nil, der as CFData) { + SecTrustSetAnchorCertificates(trust, [anchor] as CFArray) + SecTrustSetAnchorCertificatesOnly(trust, true) + } + if tls.verification == .caOnly { + SecTrustSetPolicies(trust, SecPolicyCreateBasicX509()) + } + var error: CFError? + guard SecTrustEvaluateWithError(trust, &error) else { + let detail = error.map { CFErrorCopyDescription($0) as String } ?? "" + return String(format: String(localized: "The server's certificate is not trusted. %@"), detail) + } + guard tls.verification != .caOnly || ServerCertificatePurpose.leafPermitsServerAuthentication(trust) else { + return String(localized: "The server's certificate is not issued for a server, so it cannot prove the server's identity.") + } + return nil + } + } +} diff --git a/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/Contents.json new file mode 100644 index 0000000000..f9d32664df --- /dev/null +++ b/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "qdrant.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/qdrant.svg b/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/qdrant.svg new file mode 100644 index 0000000000..41257b1911 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/qdrant-icon.imageset/qdrant.svg @@ -0,0 +1 @@ +Qdrant diff --git a/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift b/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift index d0bf6b9918..b0421096b8 100644 --- a/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift +++ b/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift @@ -289,13 +289,16 @@ enum DataWriteExecutor { logger.error( "Lookup on '\(table, privacy: .private(mask: .hash))' found \(rows.count, privacy: .public) rows, expected \(lookup.rowCount, privacy: .public)" ) - throw DataWriteError.changeRefused( - table: table, - kind: step.rowKind, - reason: rows.count < lookup.rowCount - ? String(localized: "No row holds the values it was loaded with. Another client may have changed or deleted it.") - : String(localized: "Other rows hold exactly the same values, so the change cannot reach this row alone.") - ) + let reason: String + if lookup.rowCount == 0 { + // A lookup that expects no row is an insert checking its key is free. + reason = String(localized: "A row with this key already exists, so the insert would not add one.") + } else if rows.count < lookup.rowCount { + reason = String(localized: "No row holds the values it was loaded with. Another client may have changed or deleted it.") + } else { + reason = String(localized: "Other rows hold exactly the same values, so the change cannot reach this row alone.") + } + throw DataWriteError.changeRefused(table: table, kind: step.rowKind, reason: reason) } // The row may have moved between the lookup and the write, or another row may now hold its old values, so a save diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift new file mode 100644 index 0000000000..95c510298b --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift @@ -0,0 +1,122 @@ +import Foundation +import TableProPluginKit + +extension PluginMetadataRegistry { + func qdrantPluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { + [ + ("Qdrant", PluginMetadataSnapshot( + displayName: "Qdrant", iconName: "qdrant-icon", defaultPort: 6_333, + requiresAuthentication: false, supportsForeignKeys: false, supportsSchemaEditing: false, + isDownloadable: true, primaryUrlScheme: "", parameterStyle: .questionMark, + navigationModel: .standard, explainVariants: [], pathFieldRole: .database, + supportsHealthMonitor: true, urlSchemes: [], postConnectActions: [], + brandColorHex: "#DC244C", + queryLanguageName: "REST", editorLanguage: .javascript, + connectionMode: .network, supportsDatabaseSwitching: false, + capabilities: PluginMetadataSnapshot.CapabilityFlags( + supportsSchemaSwitching: false, + supportsImport: false, + supportsExport: true, + supportsSSH: true, + supportsSSL: true, + supportsCascadeDrop: false, + supportsForeignKeyDisable: false, + supportsReadOnlyMode: true, + supportsQueryProgress: false, + requiresReconnectForDatabaseSwitch: false, + supportsDropDatabase: false, + supportsAddColumn: false, + supportsModifyColumn: false, + supportsDropColumn: false, + supportsAddIndex: false, + supportsDropIndex: false, + supportsModifyPrimaryKey: false, + supportsOpportunisticTLS: false, + verifiesServerWithSystemTrust: true, + columnsAreSampled: true, + supportsPrincipalConnectionLimit: false + ), + schema: PluginMetadataSnapshot.SchemaInfo( + defaultSchemaName: "", + defaultGroupName: "default", + tableEntityName: "Collections", + containerEntityName: "Cluster", + defaultPrimaryKeyColumn: "id", + immutableColumns: ["id"], + systemDatabaseNames: [], + systemSchemaNames: [], + fileExtensions: [], + databaseGroupingStrategy: .flat, + structureColumnFields: [.name, .type, .nullable] + ), + editor: PluginMetadataSnapshot.EditorConfig( + sqlDialect: nil, + statementCompletions: qdrantCompletions, + columnTypesByCategory: qdrantColumnTypes + ), + connection: PluginMetadataSnapshot.ConnectionConfig( + additionalConnectionFields: qdrantConnectionFields(), + category: .document, + tagline: String(localized: "Open-source vector search engine"), + hidesBuiltInPassword: true, + hidesBuiltInDatabase: true + ) + )) + ] + } +} + +private let qdrantCompletions: [CompletionEntry] = [ + CompletionEntry(label: "GET /collections", insertText: "GET /collections"), + CompletionEntry(label: "GET /collections/{name}", insertText: "GET /collections/docs"), + CompletionEntry( + label: "POST /points/scroll", + insertText: """ + POST /collections/docs/points/scroll + {"limit": 10, "with_payload": true} + """ + ), + CompletionEntry( + label: "POST /points/query", + insertText: """ + POST /collections/docs/points/query + {"query": [0.1, 0.2, 0.3, 0.4], "limit": 10, "with_payload": true} + """ + ), + CompletionEntry( + label: "POST /points/count", + insertText: """ + POST /collections/docs/points/count + {"exact": true} + """ + ), + CompletionEntry( + label: "POST /facet", + insertText: """ + POST /collections/docs/facet + {"key": "city"} + """ + ) +] + +private let qdrantColumnTypes: [String: [String]] = [ + "Text": ["keyword", "text", "string", "uuid"], + "Numeric": ["integer", "float"], + "Boolean": ["bool"], + "Date": ["datetime"], + "Structured": ["geo", "object", "array", "json"], + "Vector": ["vector", "multivector", "sparse_vector"] +] + +private func qdrantConnectionFields() -> [ConnectionField] { + [ + ConnectionField( + id: "qdApiKey", + label: String(localized: "API Key"), + placeholder: "Qdrant API key", + fieldType: .secure, + section: .authentication, + hidesPassword: true + ).withHidesUsername(true) + ] +} diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 8a29704f3f..f638edcd55 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -1047,7 +1047,7 @@ extension PluginMetadataRegistry { + kafkaPluginDefaults() + typesensePluginDefaults() + r2SQLPluginDefaults() + weaviatePluginDefaults() + hanaPluginDefaults() + firebasePluginDefaults() + bigtablePluginDefaults() + neo4jPluginDefaults() - + trinoPluginDefaults() + memcachedPluginDefaults() + influxDBPluginDefaults() + + trinoPluginDefaults() + memcachedPluginDefaults() + qdrantPluginDefaults() + influxDBPluginDefaults() } // swiftlint:enable function_body_length } diff --git a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift index e75a0d2393..7990c163f8 100644 --- a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift +++ b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift @@ -177,6 +177,7 @@ internal enum ExternalStatementGate { "ETCD_COUNT:", "ETCD_RANGE:", "ETCD_REFUSAL:", "FIREBASE_BROWSE:", "MEMCACHED_BROWSE:", "MEMCACHED_REFUSAL:", + "QDRANT_BROWSE:", "QDRANT_EXPORT:", "QDRANT_INSERT_CHECK:", "SPANNER_BROWSE:", "TYPESENSE_EXPORT:", "TYPESENSE_SEARCH:", "TYPESENSE_WRITE:", "WEAVIATE_EXPORT:", "WEAVIATE_SEARCH:", "WEAVIATE_WRITE:", diff --git a/apple/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift b/apple/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift index 23526d70ea..dbcf8d5171 100644 --- a/apple/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift @@ -53,7 +53,7 @@ enum CatalogChangeClassifier { // InfluxDB has no DDL: a write is what creates a measurement, so any statement that is not a read may change the list. private static let documentStoreTypes: Set = [ - .mongodb, .elasticsearch, .typesense, .weaviate, .etcd, .firestore, .firebaseRealtimeDatabase, .bigtable, .neo4j, + .mongodb, .elasticsearch, .typesense, .weaviate, .qdrant, .etcd, .firestore, .firebaseRealtimeDatabase, .bigtable, .neo4j, .influxdb, ] } diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index cb19c087d0..4a30495244 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import TableProQdrantCore import TableProRedisCore import TableProSQLGrammar import TableProUtilities @@ -1017,6 +1018,8 @@ private extension QueryClassifier { return worstRequest(typesenseClassification) case .weaviate: return worstRequest(weaviateClassification) + case .qdrant: + return worstRequest(qdrantClassification) case .memcached: return worstRequest(memcachedClassification) case .firestore, .firebaseRealtimeDatabase: @@ -1088,11 +1091,11 @@ private extension QueryClassifier { /// followed by a JSON body, and the body is the caller's data: scanning the whole statement /// let `POST /collections/c/documents/import` carrying `"note": "/multi_search"` in a field /// read as a search, which is a read-only mode and MCP gate bypass. A URL path holds no raw - /// space, so the path ends at the first one: that keeps a body written on the header line out - /// of it, and stops `POST /collections/c /multi_search` from ending in a read path. + /// whitespace, so the path ends at the first one: that keeps a body written on the header line + /// out of it, and stops `POST /collections/c /multi_search` from ending in a read path. static func typesenseRequestLine(_ trimmed: String) -> (verb: String, path: String) { - let header = trimmed.split(separator: "\n", maxSplits: 1).first.map(String.init) ?? "" - let parts = header.split(separator: " ", maxSplits: 1, omittingEmptySubsequences: true) + let header = trimmed.firstIndex(where: \.isNewline).map { String(trimmed[..<$0]) } ?? trimmed + let parts = header.split(maxSplits: 1, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) guard let verb = parts.first else { return ("", "/") } guard parts.count == 2 else { return (String(verb).uppercased(), "/") } @@ -1162,12 +1165,15 @@ private extension QueryClassifier { /// The driver takes a body from the rest of the request line as well as from the lines below /// it, so the gate has to read the same two places. static func weaviateConsoleBody(_ trimmed: String) -> String { - let lines = trimmed.split(separator: "\n", maxSplits: 1, omittingEmptySubsequences: false) - let header = lines.first.map(String.init) ?? "" - let following = lines.count > 1 ? String(lines[1]) : "" + let newline = trimmed.firstIndex(where: \.isNewline) + let header = newline.map { trimmed[..<$0] } ?? trimmed[...] + let following = newline.map { trimmed[trimmed.index(after: $0)...] } ?? "" let parts = header.split(maxSplits: 2, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) - let inline = parts.count > 2 ? String(parts[2]) : "" - return inline.isEmpty ? following : inline + let inline = parts.count > 2 ? parts[2] : "" + return [inline, following] + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .joined(separator: "\n") } static func weaviateClassification(_ trimmed: String) -> QueryClassification { @@ -1210,6 +1216,21 @@ private extension QueryClassifier { return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) } + // The driver runs what `QdrantConsoleParser` parses, so the gate reads the same request. + static func qdrantClassification(_ trimmed: String) -> QueryClassification { + if QdrantOperations.isTagged(trimmed) { + return .safe + } + guard let request = QdrantConsoleParser.parse(trimmed) else { return .safe } + let kind = request.kind + let tier: QueryTier = switch kind.tier { + case .read: .safe + case .write: .write + case .remove: .destructive + } + return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: kind.touchesServerFiles) + } + /// A grid page is a tagged spec and anything else a request line read the way the driver reads /// it. A write that leaves a node holding nothing deletes it, so it ranks with DELETE. static func firebaseClassification(_ trimmed: String, databaseType: DatabaseType) -> QueryClassification { diff --git a/apple/TablePro/Models/Connection/DatabaseType.swift b/apple/TablePro/Models/Connection/DatabaseType.swift index 63c0488f6d..0fbfdb1547 100644 --- a/apple/TablePro/Models/Connection/DatabaseType.swift +++ b/apple/TablePro/Models/Connection/DatabaseType.swift @@ -54,6 +54,7 @@ extension DatabaseType { static let trino = DatabaseType(rawValue: "Trino") static let presto = DatabaseType(rawValue: "Presto") static let weaviate = DatabaseType(rawValue: "Weaviate") + static let qdrant = DatabaseType(rawValue: "Qdrant") static let memcached = DatabaseType(rawValue: "Memcached") static let influxdb = DatabaseType(rawValue: "InfluxDB") static let bigtable = DatabaseType(rawValue: "Bigtable") diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 8aef509209..46807be8a2 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -3573,6 +3573,10 @@ } } }, + "%@ holds a list in %d points, and Qdrant sorts such a point once per value. Sort by a field that holds one value per point." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ holds an empty key, which the MongoDB client library will not insert. Set it to NULL, then set it on the saved document." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3597,6 +3601,30 @@ } } }, + "%@ holds booleans, so it takes true or false, not %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds geo points, written as {\"lon\": 13.4, \"lat\": 52.5}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds integers, and %@ is not one." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds JSON arrays, written as [1, 2, 3]." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds JSON objects, written as {\"key\": \"value\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "%@ holds numbers, and %@ is not a number." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ holds text in some rows and documents or arrays in others, so this value could be either. Change it with a query." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3701,6 +3729,10 @@ } } }, + "%@ is a multivector, written as arrays of numbers such as [[0.1, 0.2], [0.3, 0.4]]." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ is a numeric field, and %@ is not a number." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3735,6 +3767,10 @@ } } }, + "%@ is a sparse vector, written as {\"indices\": [1, 5], \"values\": [0.2, 0.7]}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ is a system collection. Its fields cannot be renamed or removed." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3759,6 +3795,10 @@ } } }, + "%@ is a vector, written as an array of numbers such as [0.1, 0.2]." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ is a view. Rename or remove the field in the collection it reads." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3995,6 +4035,10 @@ } } }, + "%@ is not a point id. Qdrant ids are unsigned integers or UUIDs." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ is not a Redis database index." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -16265,6 +16309,10 @@ } } }, + "A point's id cannot change. Insert a new row with the new id instead." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "a query" : { "localizations" : { "fr" : { @@ -16393,6 +16441,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "A raw filter for Qdrant is a JSON condition, such as {\"key\": \"city\", \"match\": {\"value\": \"Berlin\"}}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A raw Firestore filter is a JSON Filter object, such as %@." : { "extractionState" : "manual", "generatesSymbol" : false @@ -16508,6 +16560,9 @@ } } } + }, + "A row with this key already exists, so the insert would not add one." : { + }, "A row with this primary key already exists. Edit that row instead." : { "extractionState" : "manual", @@ -71912,6 +71967,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Enter a request like GET /collections or POST /collections/docs/points/query." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Enter a request like: GET /collections" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -121849,6 +121908,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Not connected to Qdrant." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Not connected to Redis" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -128339,6 +128402,9 @@ } } } + }, + "Open-source vector search engine" : { + }, "Opening the connection" : { "localizations" : { @@ -139328,6 +139394,54 @@ } } }, + "Qdrant %@ overwrites a point that already has the id. Insert it from the REST editor, or upgrade Qdrant to 1.15 or later." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant answered a count without a number." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant answered with something other than HTTP." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant cannot filter on %@, because a key holding a double quote has no path it accepts." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant cannot filter on %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant cannot filter with %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant cannot sort by %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant compares %@ as a 64-bit float, which cannot hold %@ exactly." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant does not support views." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant lists points by id in ascending order only." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant returned HTTP %d." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Qdrant sorts by one field at a time. Remove the sort from the other columns." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Queries" : { "comment" : "A section header for the list of queries in the favorites tab.", "isCommentAutoGenerated" : true, @@ -179836,6 +179950,10 @@ }, "TablePro does not have Local Network access, so it cannot reach the SSH server at %@." : { + }, + "TablePro does not know what %@ holds. Reload the collection, then save again." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "TablePro does not run a pre-connect script on its own." : { @@ -191121,6 +191239,14 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The server's certificate is not issued for a server, so it cannot prove the server's identity." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The server's certificate is not trusted. %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The server's socket file on this Mac. SELECT @@socket in the mysql client shows it." : { }, @@ -199148,6 +199274,10 @@ } } }, + "This row has no point id, so Qdrant cannot tell which point to change." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This row has no value for the key column %@, so it cannot be matched." : { }, diff --git a/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift b/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift index e5bce29c79..cc8ad67a43 100644 --- a/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift +++ b/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift @@ -55,16 +55,22 @@ private final class LookupDriver: PluginDriverStub, @unchecked Sendable { } struct DataWriteExecutorLookupTests { - private func step(_ sql: String, rowCount: Int = 1, check: Bool = true) -> DataWriteStep { + private func step( + _ sql: String, + rowCount: Int = 1, + check: Bool = true, + rowKind: RowWriteKind = .delete, + pinsRows: Bool = true + ) -> DataWriteStep { DataWriteStep( kind: .rowWrite, statement: ParameterizedStatement(sql: sql, parameters: ["a"]), tableName: "events", - rowKind: .delete, + rowKind: rowKind, lookup: RowLookup( query: ParameterizedStatement(sql: "LOOKUP \(sql)", parameters: ["a"]), rowCount: rowCount, - pinsRows: true, + pinsRows: pinsRows, check: check ? ParameterizedStatement(sql: "CHECK \(sql)", parameters: ["a"]) : nil ) ) @@ -115,6 +121,20 @@ struct DataWriteExecutorLookupTests { #expect(driver.writes.isEmpty) } + @Test("An insert whose key a row already holds refuses the save, saying the key is taken, before anything is written") + func takenKeyRefusesInsertBeforeAnyWrite() async { + let driver = LookupDriver(found: [1]) + + await #expect(throws: DataWriteError.changeRefused( + table: "events", + kind: .insert, + reason: String(localized: "A row with this key already exists, so the insert would not add one.") + )) { + try await run([step("INSERT keyed", rowCount: 0, check: false, rowKind: .insert, pinsRows: false)], on: driver) + } + #expect(driver.writes.isEmpty) + } + @Test("Each write reads its lookup again right before it runs, after the writes ahead of it") func lookupRunsAgainBeforeEachWrite() async { let driver = LookupDriver(found: [1, 1, 1, 2]) diff --git a/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift b/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift index b841a3bb27..00c68f911e 100644 --- a/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift +++ b/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift @@ -75,6 +75,9 @@ struct ExternalStatementGatePrivateRequestTests { (.weaviate, "WEAVIATE_SEARCH:"), (.weaviate, "WEAVIATE_WRITE:"), (.weaviate, "WEAVIATE_EXPORT:"), + (.qdrant, "QDRANT_BROWSE:"), + (.qdrant, "QDRANT_EXPORT:"), + (.qdrant, "QDRANT_INSERT_CHECK:"), (.memcached, "MEMCACHED_BROWSE:"), (.memcached, "MEMCACHED_REFUSAL:"), (.etcd, "ETCD_RANGE:"), diff --git a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift index 7a756f81ed..4649cc5a71 100644 --- a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift @@ -10,7 +10,7 @@ struct PluginMetadataRegistryTypeCountTests { "CockroachDB", "Dameng", "Databend", "DuckDB", "DynamoDB", "Elasticsearch", "etcd", "Kafka", "KingbaseES", "libSQL", "MariaDB", "MongoDB", "MySQL", "Oracle", "PGlite", "PostgreSQL", "Redis", "Redshift", "SAP HANA", "ScyllaDB", "Snowflake", "Spanner", "SQL Server", "SQLite", "SurrealDB", "Teradata", "TiDB", - "Trino", "Presto", "Turso", "Typesense", "Weaviate", "OceanBase", "StarRocks", "Firestore", "Firebase Realtime Database", "Bigtable", + "Trino", "Presto", "Turso", "Typesense", "Weaviate", "Qdrant", "OceanBase", "StarRocks", "Firestore", "Firebase Realtime Database", "Bigtable", "Neo4j", "Memcached", "InfluxDB" ] diff --git a/apple/TableProTests/Core/Plugins/PortForwardCapabilityTests.swift b/apple/TableProTests/Core/Plugins/PortForwardCapabilityTests.swift index dabb529605..89ce819b76 100644 --- a/apple/TableProTests/Core/Plugins/PortForwardCapabilityTests.swift +++ b/apple/TableProTests/Core/Plugins/PortForwardCapabilityTests.swift @@ -33,7 +33,7 @@ struct PortForwardCapabilityTests { @Test( "a self-hosted HTTP engine offers every transport that forwards a port", - arguments: [DatabaseType.weaviate, .typesense, .elasticsearch] + arguments: [DatabaseType.weaviate, .qdrant, .typesense, .elasticsearch] ) func httpEnginesOfferPortForwards(type: DatabaseType) { #expect(manager.supportsSSH(for: type)) diff --git a/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift b/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift index 3e6dfbd3e4..582610468c 100644 --- a/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift +++ b/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift @@ -28,7 +28,7 @@ struct AllTablesListingTests { } @Test("An engine that only highlights like MongoDB or Redis gets no shell command", arguments: [ - DatabaseType.elasticsearch, .typesense, .weaviate, .etcd, .memcached, .cassandra + DatabaseType.elasticsearch, .typesense, .weaviate, .qdrant, .etcd, .memcached, .cassandra ]) func noListingForOtherEngines(databaseType: DatabaseType) { #expect(AllTablesListing.statement(databaseType: databaseType, pluginListing: nil) == nil) diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index c7f4837e41..0bc9e4318b 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -895,10 +895,51 @@ struct QueryClassifierNonSqlTests { #expect(QueryClassifier.isWriteQuery("mutation { delete { Article } }", databaseType: .weaviate)) #expect(QueryClassifier.isWriteQuery("POST /v1/objects {}", databaseType: .weaviate)) #expect(QueryClassifier.classifyTier("DELETE /v1/objects/abc", databaseType: .weaviate) == .destructive) + #expect(QueryClassifier.classifyTier("DELETE /v1/schema/Article", databaseType: .weaviate) == .destructive) + #expect(QueryClassifier.classifyTier("DELETE\t/v1/schema/Article", databaseType: .weaviate) == .destructive) + let mutation = "POST /v1/graphql {\n\"query\": \"mutation { delete { Article } }\" }" + #expect(QueryClassifier.isWriteQuery(mutation, databaseType: .weaviate)) let search = "WEAVIATE_SEARCH:e30=" #expect(!QueryClassifier.isWriteQuery(search, databaseType: .weaviate)) } + @Test("Qdrant reads are safe, POST searches read, and point deletes are destructive") + func qdrantTiers() { + #expect(!QueryClassifier.isWriteQuery("QDRANT_BROWSE:e30=", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("GET /collections", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/points/query\n{\"limit\": 3}", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/points/scroll {}", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/points {\"ids\": [1]}", databaseType: .qdrant)) + #expect(QueryClassifier.classifyTier("PUT /collections/docs/points?wait=true\n{}", databaseType: .qdrant) == .write) + #expect(QueryClassifier.classifyTier("POST /collections/docs/points/payload?wait=true {}", databaseType: .qdrant) == .write) + #expect(QueryClassifier.classifyTier("POST /collections/docs/points/delete?wait=true {}", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.classifyTier("POST /collections/docs/points/payload/clear {}", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.classifyTier("POST /collections/docs/points/batch {}", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.classifyTier("DELETE /collections/docs", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.classifyTier( + "PUT /collections/docs/points/payload?wait=true\n{\"payload\":{},\"filter\":{}}", databaseType: .qdrant + ) == .destructive) + #expect(QueryClassifier.classifyTier( + "POST /collections/docs/cluster {\"drop_sharding_key\":{\"shard_key\":\"a\"}}", databaseType: .qdrant + ) == .destructive) + #expect(QueryClassifier.classifyTier("POST /collections/aliases {\"actions\":[]}", databaseType: .qdrant) == .destructive) + #expect(QueryClassifier.isWriteQuery("POST /collections/docs/points/query/../delete {}", databaseType: .qdrant)) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/facet {\"key\": \"city\"}", databaseType: .qdrant)) + #expect(QueryClassifier.classifyTier("DELETE\t/collections/docs", databaseType: .qdrant) == .destructive) + #expect(!QueryClassifier.isWriteQuery("POST /collections/docs/points/query {\n\"limit\": 3}", databaseType: .qdrant)) + } + + @Test("A Qdrant snapshot reaches the server's disk") + func qdrantSnapshots() { + #expect(QueryClassifier.reachesFilesystemOrExecutesCode( + "POST /collections/docs/snapshots", databaseType: .qdrant + )) + #expect(QueryClassifier.reachesFilesystemOrExecutesCode( + "PUT /collections/docs/snapshots/recover {\"location\": \"file:///etc\"}", databaseType: .qdrant + )) + #expect(!QueryClassifier.reachesFilesystemOrExecutesCode("GET /collections", databaseType: .qdrant)) + } + @Test("Memcached reads are safe, stores write, and delete and flush_all are destructive") func memcachedTiers() { #expect(!QueryClassifier.isWriteQuery("MEMCACHED_BROWSE:e30=", databaseType: .memcached)) diff --git a/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift b/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift index d3dff2c4db..fa31ee4f57 100644 --- a/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift +++ b/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift @@ -51,7 +51,7 @@ struct QueryTabBaseQueryTests { /// Their plugins build the browse query. Without one the tab must not be handed another engine's /// command: either the plugin builds it or the build throws. @Test("An engine that highlights like MongoDB or Redis is not browsed with their commands", arguments: [ - DatabaseType.elasticsearch, .typesense, .weaviate, .etcd, .memcached + DatabaseType.elasticsearch, .typesense, .weaviate, .qdrant, .etcd, .memcached ]) func otherEnginesNeverGetShellBrowseQueries(databaseType: DatabaseType) { let query = try? QueryTab.buildBaseTableQuery(tableName: "products", databaseType: databaseType) diff --git a/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift b/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift index e5b239c795..ddb9c261d9 100644 --- a/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift +++ b/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift @@ -13,7 +13,7 @@ extension GlobalStateSuites { "Only SQL reads :name as a bind parameter", arguments: [ (DatabaseType.mysql, true), (.postgresql, true), (.sqlite, true), - (.elasticsearch, false), (.typesense, false), (.weaviate, false), (.mongodb, false), + (.elasticsearch, false), (.typesense, false), (.weaviate, false), (.qdrant, false), (.mongodb, false), (.redis, false), (.etcd, false), (.memcached, false), (.surrealdb, false), (.kafka, false) ] ) diff --git a/apple/TableProTests/Views/SQLReviewSheetTests.swift b/apple/TableProTests/Views/SQLReviewSheetTests.swift index cf7c7ce451..5a3a86bf19 100644 --- a/apple/TableProTests/Views/SQLReviewSheetTests.swift +++ b/apple/TableProTests/Views/SQLReviewSheetTests.swift @@ -159,7 +159,7 @@ struct SQLReviewSheetTests { } @Test("A request body keeps a stored $oid as the JSON it is", arguments: [ - DatabaseType.elasticsearch, .typesense, .weaviate + DatabaseType.elasticsearch, .typesense, .weaviate, .qdrant ]) func requestBodyKeepsExtendedJSON(databaseType: DatabaseType) { let statement = """ diff --git a/apple/changelog.d/3972.added.md b/apple/changelog.d/3972.added.md new file mode 100644 index 0000000000..0ad2c302b7 --- /dev/null +++ b/apple/changelog.d/3972.added.md @@ -0,0 +1 @@ +Qdrant support: browse collections, filter and sort points, edit payloads and vectors, and run REST requests. diff --git a/apple/project.yml b/apple/project.yml index a7b429a9da..73a919362c 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -408,7 +408,7 @@ targets: - package: Sparkle - package: Yams - package: TableProCore - products: [TableProAnalytics, TableProAutocomplete, TableProConnectionLibrary, TableProGeometry, TableProGoogleCloud, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProRedisCore, TableProSQLGrammar, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme] + products: [TableProAnalytics, TableProAutocomplete, TableProConnectionLibrary, TableProGeometry, TableProGoogleCloud, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme] - package: TableProMacCore products: [TableProMCP, TableProSSHConfig, TableProUtilities] # Writes nothing for a version CHANGELOG.md has no section for, so a 0.0.0-dev build ships no notes. @@ -1298,6 +1298,17 @@ targets: - package: TableProCore product: TableProLogRedaction + QdrantDriverPlugin: + templates: [DriverPlugin] + templateAttributes: + folder: QdrantDriverPlugin + principalClass: QdrantPlugin + dependencies: + - package: TableProCore + product: TableProQdrantCore + - package: TableProCore + product: TableProTLSClientIdentity + WeaviateDriverPlugin: templates: [DriverPlugin] templateAttributes: @@ -1363,6 +1374,7 @@ aggregateTargets: - Neo4jDriverPlugin - OracleDriver - ParquetExport + - QdrantDriverPlugin - SnowflakeDriverPlugin - SpannerDriverPlugin - StarRocksDriverPlugin diff --git a/apple/scripts/probes/check-qdrant.sh b/apple/scripts/probes/check-qdrant.sh new file mode 100755 index 0000000000..1a4b551f33 --- /dev/null +++ b/apple/scripts/probes/check-qdrant.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Run TableProQdrantCore against a live Qdrant node: cursor paging, every filter kind, sorting with +# keyless points, typed payload and vector writes, the insert id check, facet, export, truncate and +# drop. The probe creates and drops a collection named tablepro_probe. +# +# Usage: +# scripts/probes/check-qdrant.sh [host] [port] +# +# QDRANT_API_KEY sends a key. Start a node with: +# docker run -d --name tablepro-qdrant-probe -p 6333:6333 qdrant/qdrant +# +# Exits 3 when no node answers. + +set -euo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-6333}" +HERE="$(cd "$(dirname "$0")" && pwd)" +CORE="$(cd "$HERE/../../Packages/TableProCore" && pwd)" + +if ! curl -fsS -o /dev/null ${QDRANT_API_KEY:+-H "api-key: $QDRANT_API_KEY"} "http://$HOST:$PORT/collections"; then + echo "No Qdrant node answers at $HOST:$PORT." >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/Sources/probe" +cp "$HERE/qdrant-probe.swift" "$WORK/Sources/probe/main.swift" +cat > "$WORK/Package.swift" < QdrantHTTPResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + request.headers.forEach { urlRequest.setValue($1, forHTTPHeaderField: $0) } + let (data, response) = try await URLSession.shared.data(for: urlRequest) + return QdrantHTTPResponse(statusCode: (response as? HTTPURLResponse)?.statusCode ?? 0, body: data) + } + + func cancelAll() {} +} + +let env = ProcessInfo.processInfo.environment +let settings = QdrantConnectionSettings( + host: env["QDRANT_HOST"] ?? "127.0.0.1", + port: Int(env["QDRANT_PORT"] ?? "") ?? 6_333, + usesTLS: false, + apiKey: env["QDRANT_API_KEY"] ?? "" +) +let client = QdrantClient(settings: settings, transport: ProbeTransport(), timeout: { 10 }) +try await client.connect(requestTimeout: { 10 }) +print("version", client.serverVersion ?? "nil") + +let name = "tablepro_probe" +let base = "/collections/\(name)" +func request(_ text: String?) -> QdrantConsoleRequest { + guard let text, let request = QdrantConsoleParser.parse(text) else { fatalError("not a request: \(text ?? "nil")") } + return request +} +func run(_ text: String?) async throws -> QdrantHTTPResponse { + try await client.execute(request(text)) +} + +_ = try? await run("DELETE \(base)") +_ = try await run(#"PUT \#(base) {"vectors":{"size":2,"distance":"Dot"},"sparse_vectors":{"sp":{}}}"#) +var points: [[String: Any]] = [] +for i in 1...250 { + var payload: [String: Any] = ["city": i % 2 == 0 ? "even" : "odd", "body": "Hello World \(i)", "f": Double(i) + 0.5] + if i <= 200 { payload["n"] = i } + if i <= 100 { payload["when"] = String(format: "2024-01-01T%02d:%02d:00Z", i / 60, i % 60) } + if i == 7 { payload["note"] = NSNull() } + if i == 8 { payload["note"] = "" } + if i == 9 { payload["city"] = 42 } + if i == 10 { payload["mixed"] = "42" } + if i == 11 { payload["mixed"] = 42 } + if i == 12 { payload["mixed"] = ["42"] } + payload["meta"] = ["source": "doc\(i).pdf"] + payload["a.b"] = "dotted" + points.append(["id": i, "vector": ["": [Double(i), 1]], "payload": payload]) +} +_ = try await run("PUT \(base)/points?wait=true\n" + (try QdrantJSON.text(["points": points]))) +for (field, schema) in [("n", "integer"), ("f", "float"), ("when", "datetime"), ("city", "keyword"), ("meta.source", "keyword")] { + _ = try await run(#"PUT \#(base)/index?wait=true {"field_name":"\#(field)","field_schema":"\#(schema)"}"#) +} + +check(try await client.collectionNames().contains(name), "lists collection") +let collection = try await client.collection(named: name) +let columns = QdrantSchema.columns(for: collection) +print("columns", columns.map { "\($0.name):\($0.type)" }) +check(collection.pointsCount == 250, "points_count") +check(!columns.contains { $0.name == "meta.source" } && collection.indexes.contains { $0.path == "meta.source" }, "nested index path is an index, not a column") +check(columns.contains { $0.name == "vector" && $0.type == "vector(2)" } && columns.contains { $0.name == "vector.sp" }, "vector columns typed") + +let cache = QdrantPagingCache() +let p0 = try await client.pointPage(collection: name, filter: nil, offset: 0, limit: 100) +let p1 = try await client.pointPage(collection: name, filter: nil, offset: 100, limit: 100) +let p2 = try await client.pointPage(collection: name, filter: nil, offset: 200, limit: 100) +check(p0.first?.idText == "1" && p1.first?.idText == "101" && p2.count == 50 && p2.last?.idText == "250", "paging forward") +let jump = try await client.pointPage(collection: name, filter: nil, offset: 170, limit: 10) +check(jump.map { $0.idText ?? "" } == (171...180).map(String.init), "jump walks to offset") +check(try await client.pointPage(collection: name, filter: nil, offset: 400, limit: 10).isEmpty, "past end") +_ = try await run(#"POST \#(base)/points/delete?wait=true {"points":[50]}"#) +let shifted = try await client.pointPage(collection: name, filter: nil, offset: 100, limit: 2) +check(shifted.map { $0.idText ?? "" } == ["102", "103"], "a page after another client's delete starts at the right row") +_ = try await run(#"PUT \#(base)/points?wait=true {"points":[{"id":50,"vector":[50,1],"payload":{"city":"even","body":"Hello World 50","f":50.5,"n":50,"meta":{"source":"doc50.pdf"},"a.b":"dotted","when":"2024-01-01T00:50:00Z"}}]}"#) + +func filtered(_ specs: [QdrantFilterSpec], _ mode: String = "AND") async throws -> Int { + let filter = try QdrantFilterBuilder.filter(specs, logicMode: mode) + return try await client.count(collection: name, filter: filter) +} +check(try await filtered([QdrantFilterSpec(column: "city", op: "=", value: "even")]) == 125, "match keyword") +check(try await filtered([QdrantFilterSpec(column: "city", op: "=", value: "42")]) == 1, "a number stored under a keyword index is found") +check(try await filtered([QdrantFilterSpec(column: "n", op: "BETWEEN", value: "10,19", secondValue: "19")]) == 10, "between") +check(try await filtered([QdrantFilterSpec(column: "n", op: ">", value: "190"), QdrantFilterSpec(column: "city", op: "!=", value: "even")]) == 5, "and with not") +check(try await filtered([QdrantFilterSpec(column: "n", op: "<", value: "3"), QdrantFilterSpec(column: "n", op: ">", value: "198")], "OR") == 4, "or") +check(try await filtered([QdrantFilterSpec(column: "body", op: "CONTAINS", value: "world")]) == 250, "contains without a text index matches words") +check(try await filtered([QdrantFilterSpec(column: "id", op: "IN", value: "1,2,3")]) == 3, "has_id") +let searchAll = [ + QdrantFilterSpec(column: "id", op: "CONTAINS", value: "Berlin"), + QdrantFilterSpec(column: "city", op: "CONTAINS", value: "even") +] +check(try await filtered(searchAll, "OR") == 125, "search all rows skips the id column") +let searchNumbers = [ + QdrantFilterSpec(column: "n", op: "CONTAINS", value: "5"), + QdrantFilterSpec(column: "vector.sp", op: "CONTAINS", value: "5"), + QdrantFilterSpec(column: "city", op: "CONTAINS", value: "even") +] +check(try await filtered(searchNumbers, "OR") == 125, "search all rows runs on number and vector columns") +let evenFilter = try QdrantFilterBuilder.filter([QdrantFilterSpec(column: "city", op: "=", value: "even")], logicMode: "AND") +let evenBefore = try await client.pointPage(collection: name, filter: evenFilter, offset: 10, limit: 2) +_ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"city":"odd"},"points":[2]}"#) +let evenAfter = try await client.pointPage(collection: name, filter: evenFilter, offset: 10, limit: 2) +_ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"city":"even"},"points":[2]}"#) +check(evenBefore.map { $0.idText ?? "" } == ["22", "24"] && evenAfter.map { $0.idText ?? "" } == ["24", "26"], "a filtered page follows another client's payload change") +check(try await filtered([QdrantFilterSpec(column: "n", op: "IN", value: "1,2,300")]) == 2, "match any") +check(try await filtered([QdrantFilterSpec(column: "f", op: "IN", value: "1.5, 2.5")]) == 2, "IN on floats") +check(try await filtered([QdrantFilterSpec(column: "f", op: "NOT IN", value: "1.5")]) == 249, "NOT IN on floats") +check(try await filtered([QdrantFilterSpec(column: "n", op: "IS NULL", value: "")]) == 50, "IS NULL counts missing keys") +check(try await filtered([QdrantFilterSpec(column: "note", op: "IS NULL", value: "")]) == 249, "IS NULL counts null and missing") +check(try await filtered([QdrantFilterSpec(column: "note", op: "IS EMPTY", value: "")]) == 250, "IS EMPTY adds the empty string") +check(try await filtered([QdrantFilterSpec(column: "a.b", op: "=", value: "dotted")]) == 250, "dotted top-level key") +check(try await filtered([QdrantFilterSpec(column: "mixed", op: "=", value: "42")]) == 3, "an unindexed value matches as text, number and list") +check(try await filtered([QdrantFilterSpec(column: "mixed", op: "NOT IN", value: "42, 7")]) == 247, "NOT IN excludes every spelling") + +func sorted(_ offset: Int, _ limit: Int, ascending: Bool = false) async throws -> [String] { + try await client.sortedPage( + collection: name, filter: nil, column: "n", key: "\"n\"", ascending: ascending, + offset: offset, limit: limit, cache: cache + ).map { $0.idText ?? "" } +} +check(try await sorted(5, 3) == ["195", "194", "193"], "order_by with offset") +check(try await sorted(198, 4) == ["2", "1", "201", "202"], "points without the key follow the ordered ones") +check(try await sorted(248, 10) == ["249", "250"], "every point reachable when sorted") +let byDate = { (offset: Int, limit: Int) in + try await client.sortedPage( + collection: name, filter: nil, column: "when", key: "\"when\"", ascending: false, + offset: offset, limit: limit, cache: cache + ).map { $0.idText ?? "" } +} +check(try await byDate(0, 2) == ["100", "99"], "datetime sort orders by date") +check(try await byDate(99, 3) == ["1", "101", "102"], "datetime sort puts undated points after") +_ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"n":[1,2]},"points":[3]}"#) +do { + _ = try await client.sortedPage(collection: name, filter: nil, column: "n", key: "\"n\"", ascending: true, offset: 0, limit: 5, cache: QdrantPagingCache()) + check(false, "list values refuse the sort") +} catch { + check(error.localizedDescription.contains("list"), "list values refuse the sort") +} +_ = try await run(#"POST \#(base)/points/payload?wait=true {"payload":{"n":3},"points":[3]}"#) +do { + _ = try await client.sortedPage(collection: name, filter: nil, column: "city", key: "\"city\"", ascending: true, offset: 0, limit: 5, cache: QdrantPagingCache()) + check(false, "a keyword field refuses the sort") +} catch { + check(error.localizedDescription.contains("range index"), "a keyword field refuses the sort") +} + +let types = Dictionary(uniqueKeysWithValues: columns.map { ($0.name, $0.type) }) +let writes = try QdrantRowWriter.writes(collection: name, changes: [ + QdrantRowChange(kind: .update, rowIndex: 0, id: "1", cells: [ + QdrantCell(column: "city", text: "edited"), QdrantCell(column: "n", text: "1000"), + QdrantCell(column: "vector", text: "[9, 9]"), QdrantCell(column: "vector.sp", text: #"{"indices":[2],"values":[0.5]}"#) + ]), + QdrantRowChange(kind: .delete, rowIndex: 1, id: "2"), + QdrantRowChange(kind: .insert, rowIndex: 2, id: nil, cells: [ + QdrantCell(column: "id", text: "999"), QdrantCell(column: "city", text: "new"), QdrantCell(column: "vector", text: "[1, 2]") + ]) +], serverVersion: client.serverVersion, typeName: { types[$0] }) +for write in writes { + if let lookup = write.mustFindNothing { + check(QdrantPoint.list(fromResponse: try await run(lookup.text).json)?.isEmpty == true, "insert lookup finds a free id") + } + _ = try await run(write.request.text) +} +let taken = try QdrantRowWriter.writes(collection: name, changes: [ + QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [QdrantCell(column: "id", text: "3")]) +], serverVersion: client.serverVersion, typeName: { types[$0] }) +check(QdrantPoint.list(fromResponse: try await run(taken[0].mustFindNothing?.text).json)?.count == 1, "insert lookup finds a taken id") +_ = try await run(taken[0].request.text) +let untouched = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[3],"with_payload":true,"with_vector":true}"#).json) +check(untouched?.first?.cell(for: "city") == "odd" && untouched?.first?.cell(for: "vector") == "[3,1]", "insert leaves a taken id untouched") + +let raced = try QdrantRowWriter.writes(collection: name, changes: [ + QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [QdrantCell(column: "id", text: "777"), QdrantCell(column: "city", text: "ours")]) +], serverVersion: client.serverVersion, typeName: { types[$0] })[0] +check(QdrantPoint.list(fromResponse: try await run(raced.mustFindNothing?.text).json)?.isEmpty == true, "raced insert passes its lookup") +_ = try await run(#"PUT \#(base)/points?wait=true {"points":[{"id":777,"vector":[7,7],"payload":{"city":"theirs"}}]}"#) +_ = try await run(raced.request.text) +guard let racedCheck = raced.landedCheck.flatMap(QdrantInsertCheck.decode) else { fatalError("insert has no landed check") } +let held = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[777],"with_payload":true,"with_vector":true}"#).json)?.first +check(!QdrantInsertCheck.landed(held, payload: racedCheck.payload, vectors: racedCheck.vectors), "an insert that lost a race to the same id is caught after the write") +_ = try await run(#"POST \#(base)/points/delete?wait=true {"points":[777]}"#) +let after = try await run(#"POST \#(base)/points {"ids":[1,2,999],"with_payload":true,"with_vector":true}"#) +let byID = Dictionary(uniqueKeysWithValues: (QdrantPoint.list(fromResponse: after.json) ?? []).map { ($0.idText ?? "", $0) }) +check(byID["1"]?.cell(for: "city") == "edited" && byID["1"]?.cell(for: "n") == "1000", "set payload typed") +check(byID["1"]?.cell(for: "vector") == "[9,9]" && byID["1"]?.cell(for: "vector.sp") == #"{"indices":[2],"values":[0.5]}"#, "vectors updated") +check(byID["2"] == nil, "delete") +check(byID["999"]?.cell(for: "city") == "new" && byID["999"]?.cell(for: "vector") == "[1,2]", "insert with vector") +let cleared = try QdrantRowWriter.writes(collection: name, changes: [ + QdrantRowChange(kind: .update, rowIndex: 0, id: "1", cells: [QdrantCell(column: "vector.sp", text: nil)]) +], serverVersion: client.serverVersion, typeName: { types[$0] }) +_ = try await run(cleared[0].request.text) +let afterClear = QdrantPoint.list(fromResponse: try await run(#"POST \#(base)/points {"ids":[1],"with_vector":true}"#).json) +check(afterClear?.first?.cell(for: "vector.sp") == nil, "NULL removes a vector") + +let facet = request(#"POST \#(base)/facet {"key":"city"}"#) +check(facet.kind.tier == .read, "facet is a read") +check(((try await client.execute(facet).json as? [String: Any])?["result"]) != nil, "facet route answers") + +let console = try await run("POST \(base)/points/query {\"query\": [1, 0],\n\"limit\": 2, \"with_vector\": true, \"with_payload\": true}") +let hits = QdrantPoint.list(fromResponse: console.json) ?? [] +check(hits.count == 2 && hits[0].score != nil, "vector query with a body over two lines has scores") + +var widened = collection +for try await page in client.pointPages(collection: name, pageSize: 60, withVector: false) { + widened = widened.including(page) +} +var exported = 0 +for try await page in client.pointPages(collection: name, pageSize: 60) { exported += page.count } +check(exported == 250, "export all") + +_ = try await run(QdrantOperations.deleteAllPoints(collection: name)) +check(try await client.count(collection: name, filter: nil) == 0, "truncate") +_ = try await run(QdrantOperations.deleteCollection(named: name, objectType: "TABLE")) +check(!(try await client.collectionNames().contains(name)), "drop") +exit(failures == 0 ? 0 : 1)