diff --git a/go/pkg/sdk/oapi/client.gen.go b/go/pkg/sdk/oapi/client.gen.go index cf5c6685c9..5b0b8c0160 100644 --- a/go/pkg/sdk/oapi/client.gen.go +++ b/go/pkg/sdk/oapi/client.gen.go @@ -7593,6 +7593,42 @@ func (e GetDocumentArtifactManifestParamsDetail) Valid() bool { } } +// Defines values for CreateTableStorageMigrationJSONBodyTarget. +const ( + CreateTableStorageMigrationJSONBodyTargetVectorStore CreateTableStorageMigrationJSONBodyTarget = "vector_store" +) + +// Valid indicates whether the value is a known member of the CreateTableStorageMigrationJSONBodyTarget enum. +func (e CreateTableStorageMigrationJSONBodyTarget) Valid() bool { + switch e { + case CreateTableStorageMigrationJSONBodyTargetVectorStore: + return true + default: + return false + } +} + +// Defines values for AdvanceTableStorageMigrationJSONBodyAction. +const ( + AdvanceTableStorageMigrationJSONBodyActionCancel AdvanceTableStorageMigrationJSONBodyAction = "cancel" + AdvanceTableStorageMigrationJSONBodyActionPublish AdvanceTableStorageMigrationJSONBodyAction = "publish" + AdvanceTableStorageMigrationJSONBodyActionStep AdvanceTableStorageMigrationJSONBodyAction = "step" +) + +// Valid indicates whether the value is a known member of the AdvanceTableStorageMigrationJSONBodyAction enum. +func (e AdvanceTableStorageMigrationJSONBodyAction) Valid() bool { + switch e { + case AdvanceTableStorageMigrationJSONBodyActionCancel: + return true + case AdvanceTableStorageMigrationJSONBodyActionPublish: + return true + case AdvanceTableStorageMigrationJSONBodyActionStep: + return true + default: + return false + } +} + // AgentDecision defines model for AgentDecision. type AgentDecision struct { // Answer User answer, scalar or structured depending on the question kind @@ -23392,6 +23428,29 @@ type UpdateSchemaParams struct { IfMatch string `json:"If-Match,omitempty,omitzero"` } +// CreateTableStorageMigrationJSONBody defines parameters for CreateTableStorageMigration. +type CreateTableStorageMigrationJSONBody struct { + Budget struct { + BatchBytes int64 `json:"batch_bytes,omitempty,omitzero"` + BatchRows int `json:"batch_rows,omitempty,omitzero"` + DiskReserveBytes int64 `json:"disk_reserve_bytes,omitempty,omitzero"` + TemporaryBytes int64 `json:"temporary_bytes,omitempty,omitzero"` + } `json:"budget,omitempty,omitzero"` + JobId string `json:"job_id"` + Target CreateTableStorageMigrationJSONBodyTarget `json:"target"` +} + +// CreateTableStorageMigrationJSONBodyTarget defines parameters for CreateTableStorageMigration. +type CreateTableStorageMigrationJSONBodyTarget string + +// AdvanceTableStorageMigrationJSONBody defines parameters for AdvanceTableStorageMigration. +type AdvanceTableStorageMigrationJSONBody struct { + Action AdvanceTableStorageMigrationJSONBodyAction `json:"action"` +} + +// AdvanceTableStorageMigrationJSONBodyAction defines parameters for AdvanceTableStorageMigration. +type AdvanceTableStorageMigrationJSONBodyAction string + // CleanupTransactionSessionsParams defines parameters for CleanupTransactionSessions. type CleanupTransactionSessionsParams struct { CutoffNs string `form:"cutoff_ns,omitempty" json:"cutoff_ns,omitempty,omitzero"` @@ -23535,6 +23594,12 @@ type PatchSchemaApplicationMergePatchPlusJSONRequestBody = TableSchemaPatch // UpdateSchemaJSONRequestBody defines body for UpdateSchema for application/json ContentType. type UpdateSchemaJSONRequestBody = TableSchema +// CreateTableStorageMigrationJSONRequestBody defines body for CreateTableStorageMigration for application/json ContentType. +type CreateTableStorageMigrationJSONRequestBody CreateTableStorageMigrationJSONBody + +// AdvanceTableStorageMigrationJSONRequestBody defines body for AdvanceTableStorageMigration for application/json ContentType. +type AdvanceTableStorageMigrationJSONRequestBody AdvanceTableStorageMigrationJSONBody + // BeginTransactionJSONRequestBody defines body for BeginTransaction for application/json ContentType. type BeginTransactionJSONRequestBody = TransactionBeginRequest @@ -32821,6 +32886,71 @@ type ClientInterface interface { // Corresponds with PUT /db/v1/tables/{tableName}/schema (the `UpdateSchema` operationId). UpdateSchema(ctx context.Context, tableName string, params *UpdateSchemaParams, body UpdateSchemaJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // CreateTableStorageMigrationWithBody Create or resume a table storage migration job + // + // Table-admin operation for local single-shard standalone tables. Target + // vector_store changes primary_lsm source ownership without changing models, + // dimensions, artifacts or logical indexes. Retry creation with the same + // job_id, target and budgets. The job is advanced explicitly through its + // job endpoint; the server does not schedule an unattended migration loop. + // Offline migration uses antfly storage migrate against a stopped server. + // + // Takes any type of body and a specified content type. + // + // Corresponds with POST /db/v1/tables/{tableName}/storage/migrations (the `CreateTableStorageMigration` operationId). + CreateTableStorageMigrationWithBody(ctx context.Context, tableName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + // CreateTableStorageMigration Create or resume a table storage migration job + // + // Table-admin operation for local single-shard standalone tables. Target + // vector_store changes primary_lsm source ownership without changing models, + // dimensions, artifacts or logical indexes. Retry creation with the same + // job_id, target and budgets. The job is advanced explicitly through its + // job endpoint; the server does not schedule an unattended migration loop. + // Offline migration uses antfly storage migrate against a stopped server. + // + // Takes a body of the `application/json` content type. + // + // Corresponds with POST /db/v1/tables/{tableName}/storage/migrations (the `CreateTableStorageMigration` operationId). + CreateTableStorageMigration(ctx context.Context, tableName string, body CreateTableStorageMigrationJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetTableStorageMigration Read a table storage migration receipt + // + // Table-admin observation only. Does not admit, advance, or publish a job. + // A phase of admitted means catalog admission is durable but DB preparation + // has not begun; retry creation or send a job action to recover it. The + // receipt is retained until a later migration replaces it; this endpoint + // is not a permanent job history. + // + // Corresponds with GET /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `GetTableStorageMigration` operationId). + GetTableStorageMigration(ctx context.Context, tableName string, jobId string, reqEditors ...RequestEditorFn) (*http.Response, error) + + // AdvanceTableStorageMigrationWithBody Advance, publish or cancel a table storage migration job + // + // Uses the job's durable configuration and budgets. Each step commits + // bounded progress. Publish is accepted only at ready; complete additionally + // certifies reference-only primary artifacts and native ANN serving. + // Cancellation is allowed only before publication. Repeating an action + // after an ambiguous response resumes the durable job. + // + // Takes any type of body and a specified content type. + // + // Corresponds with POST /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `AdvanceTableStorageMigration` operationId). + AdvanceTableStorageMigrationWithBody(ctx context.Context, tableName string, jobId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + // AdvanceTableStorageMigration Advance, publish or cancel a table storage migration job + // + // Uses the job's durable configuration and budgets. Each step commits + // bounded progress. Publish is accepted only at ready; complete additionally + // certifies reference-only primary artifacts and native ANN serving. + // Cancellation is allowed only before publication. Repeating an action + // after an ambiguous response resumes the durable job. + // + // Takes a body of the `application/json` content type. + // + // Corresponds with POST /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `AdvanceTableStorageMigration` operationId). + AdvanceTableStorageMigration(ctx context.Context, tableName string, jobId string, body AdvanceTableStorageMigrationJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // ListTransactionSessions List transaction sessions // // Corresponds with GET /db/v1/transactions (the `ListTransactionSessions` operationId). @@ -36766,6 +36896,121 @@ func (c *Client) UpdateSchema(ctx context.Context, tableName string, params *Upd return c.Client.Do(req) } +// CreateTableStorageMigrationWithBody Create or resume a table storage migration job +// +// Table-admin operation for local single-shard standalone tables. Target +// vector_store changes primary_lsm source ownership without changing models, +// dimensions, artifacts or logical indexes. Retry creation with the same +// job_id, target and budgets. The job is advanced explicitly through its +// job endpoint; the server does not schedule an unattended migration loop. +// Offline migration uses antfly storage migrate against a stopped server. +// +// Takes any type of body and a specified content type. +// +// Corresponds with POST /db/v1/tables/{tableName}/storage/migrations (the `CreateTableStorageMigration` operationId). +func (c *Client) CreateTableStorageMigrationWithBody(ctx context.Context, tableName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreateTableStorageMigrationRequestWithBody(c.Server, tableName, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +// CreateTableStorageMigration Create or resume a table storage migration job +// +// Table-admin operation for local single-shard standalone tables. Target +// vector_store changes primary_lsm source ownership without changing models, +// dimensions, artifacts or logical indexes. Retry creation with the same +// job_id, target and budgets. The job is advanced explicitly through its +// job endpoint; the server does not schedule an unattended migration loop. +// Offline migration uses antfly storage migrate against a stopped server. +// +// Takes a body of the `application/json` content type. +// +// Corresponds with POST /db/v1/tables/{tableName}/storage/migrations (the `CreateTableStorageMigration` operationId). +func (c *Client) CreateTableStorageMigration(ctx context.Context, tableName string, body CreateTableStorageMigrationJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreateTableStorageMigrationRequest(c.Server, tableName, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +// GetTableStorageMigration Read a table storage migration receipt +// +// Table-admin observation only. Does not admit, advance, or publish a job. +// A phase of admitted means catalog admission is durable but DB preparation +// has not begun; retry creation or send a job action to recover it. The +// receipt is retained until a later migration replaces it; this endpoint +// is not a permanent job history. +// +// Corresponds with GET /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `GetTableStorageMigration` operationId). +func (c *Client) GetTableStorageMigration(ctx context.Context, tableName string, jobId string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetTableStorageMigrationRequest(c.Server, tableName, jobId) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +// AdvanceTableStorageMigrationWithBody Advance, publish or cancel a table storage migration job +// +// Uses the job's durable configuration and budgets. Each step commits +// bounded progress. Publish is accepted only at ready; complete additionally +// certifies reference-only primary artifacts and native ANN serving. +// Cancellation is allowed only before publication. Repeating an action +// after an ambiguous response resumes the durable job. +// +// Takes any type of body and a specified content type. +// +// Corresponds with POST /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `AdvanceTableStorageMigration` operationId). +func (c *Client) AdvanceTableStorageMigrationWithBody(ctx context.Context, tableName string, jobId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewAdvanceTableStorageMigrationRequestWithBody(c.Server, tableName, jobId, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +// AdvanceTableStorageMigration Advance, publish or cancel a table storage migration job +// +// Uses the job's durable configuration and budgets. Each step commits +// bounded progress. Publish is accepted only at ready; complete additionally +// certifies reference-only primary artifacts and native ANN serving. +// Cancellation is allowed only before publication. Repeating an action +// after an ambiguous response resumes the durable job. +// +// Takes a body of the `application/json` content type. +// +// Corresponds with POST /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `AdvanceTableStorageMigration` operationId). +func (c *Client) AdvanceTableStorageMigration(ctx context.Context, tableName string, jobId string, body AdvanceTableStorageMigrationJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewAdvanceTableStorageMigrationRequest(c.Server, tableName, jobId, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // ListTransactionSessions List transaction sessions // // Corresponds with GET /db/v1/transactions (the `ListTransactionSessions` operationId). @@ -41553,6 +41798,148 @@ func NewUpdateSchemaRequestWithBody(server string, tableName string, params *Upd return req, nil } +// NewCreateTableStorageMigrationRequest calls the generic CreateTableStorageMigration builder with application/json body +func NewCreateTableStorageMigrationRequest(server string, tableName string, body CreateTableStorageMigrationJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewCreateTableStorageMigrationRequestWithBody(server, tableName, "application/json", bodyReader) +} + +// NewCreateTableStorageMigrationRequestWithBody constructs an http.Request for the CreateTableStorageMigration method, with any body, and a specified content type +func NewCreateTableStorageMigrationRequestWithBody(server string, tableName string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "tableName", tableName, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/db/v1/tables/%s/storage/migrations", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +// NewGetTableStorageMigrationRequest constructs an http.Request for the GetTableStorageMigration method +func NewGetTableStorageMigrationRequest(server string, tableName string, jobId string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "tableName", tableName, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithOptions("simple", false, "jobId", jobId, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/db/v1/tables/%s/storage/migrations/%s", pathParam0, pathParam1) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewAdvanceTableStorageMigrationRequest calls the generic AdvanceTableStorageMigration builder with application/json body +func NewAdvanceTableStorageMigrationRequest(server string, tableName string, jobId string, body AdvanceTableStorageMigrationJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewAdvanceTableStorageMigrationRequestWithBody(server, tableName, jobId, "application/json", bodyReader) +} + +// NewAdvanceTableStorageMigrationRequestWithBody constructs an http.Request for the AdvanceTableStorageMigration method, with any body, and a specified content type +func NewAdvanceTableStorageMigrationRequestWithBody(server string, tableName string, jobId string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "tableName", tableName, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithOptions("simple", false, "jobId", jobId, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/db/v1/tables/%s/storage/migrations/%s", pathParam0, pathParam1) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewListTransactionSessionsRequest constructs an http.Request for the ListTransactionSessions method func NewListTransactionSessionsRequest(server string) (*http.Request, error) { var err error @@ -44981,6 +45368,73 @@ type ClientWithResponsesInterface interface { // Corresponds with PUT /db/v1/tables/{tableName}/schema (the `UpdateSchema` operationId). UpdateSchemaWithResponse(ctx context.Context, tableName string, params *UpdateSchemaParams, body UpdateSchemaJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateSchemaResponse, error) + // CreateTableStorageMigrationWithBodyWithResponse Create or resume a table storage migration job + // + // Table-admin operation for local single-shard standalone tables. Target + // vector_store changes primary_lsm source ownership without changing models, + // dimensions, artifacts or logical indexes. Retry creation with the same + // job_id, target and budgets. The job is advanced explicitly through its + // job endpoint; the server does not schedule an unattended migration loop. + // Offline migration uses antfly storage migrate against a stopped server. + // + // Takes any type of body and a specified content type, and returns a wrapper object for the known response body format(s). + // + // Corresponds with POST /db/v1/tables/{tableName}/storage/migrations (the `CreateTableStorageMigration` operationId). + CreateTableStorageMigrationWithBodyWithResponse(ctx context.Context, tableName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateTableStorageMigrationResponse, error) + + // CreateTableStorageMigrationWithResponse Create or resume a table storage migration job + // + // Table-admin operation for local single-shard standalone tables. Target + // vector_store changes primary_lsm source ownership without changing models, + // dimensions, artifacts or logical indexes. Retry creation with the same + // job_id, target and budgets. The job is advanced explicitly through its + // job endpoint; the server does not schedule an unattended migration loop. + // Offline migration uses antfly storage migrate against a stopped server. + // + // Takes a body of the `application/json` content type, and returns a wrapper object for the known response body format(s). + // + // Corresponds with POST /db/v1/tables/{tableName}/storage/migrations (the `CreateTableStorageMigration` operationId). + CreateTableStorageMigrationWithResponse(ctx context.Context, tableName string, body CreateTableStorageMigrationJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateTableStorageMigrationResponse, error) + + // GetTableStorageMigrationWithResponse Read a table storage migration receipt + // + // Table-admin observation only. Does not admit, advance, or publish a job. + // A phase of admitted means catalog admission is durable but DB preparation + // has not begun; retry creation or send a job action to recover it. The + // receipt is retained until a later migration replaces it; this endpoint + // is not a permanent job history. + // + // Returns a wrapper object for the known response body format(s). + // + // Corresponds with GET /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `GetTableStorageMigration` operationId). + GetTableStorageMigrationWithResponse(ctx context.Context, tableName string, jobId string, reqEditors ...RequestEditorFn) (*GetTableStorageMigrationResponse, error) + + // AdvanceTableStorageMigrationWithBodyWithResponse Advance, publish or cancel a table storage migration job + // + // Uses the job's durable configuration and budgets. Each step commits + // bounded progress. Publish is accepted only at ready; complete additionally + // certifies reference-only primary artifacts and native ANN serving. + // Cancellation is allowed only before publication. Repeating an action + // after an ambiguous response resumes the durable job. + // + // Takes any type of body and a specified content type, and returns a wrapper object for the known response body format(s). + // + // Corresponds with POST /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `AdvanceTableStorageMigration` operationId). + AdvanceTableStorageMigrationWithBodyWithResponse(ctx context.Context, tableName string, jobId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*AdvanceTableStorageMigrationResponse, error) + + // AdvanceTableStorageMigrationWithResponse Advance, publish or cancel a table storage migration job + // + // Uses the job's durable configuration and budgets. Each step commits + // bounded progress. Publish is accepted only at ready; complete additionally + // certifies reference-only primary artifacts and native ANN serving. + // Cancellation is allowed only before publication. Repeating an action + // after an ambiguous response resumes the durable job. + // + // Takes a body of the `application/json` content type, and returns a wrapper object for the known response body format(s). + // + // Corresponds with POST /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `AdvanceTableStorageMigration` operationId). + AdvanceTableStorageMigrationWithResponse(ctx context.Context, tableName string, jobId string, body AdvanceTableStorageMigrationJSONRequestBody, reqEditors ...RequestEditorFn) (*AdvanceTableStorageMigrationResponse, error) + // ListTransactionSessionsWithResponse List transaction sessions // // Returns a wrapper object for the known response body format(s). @@ -51898,6 +52352,192 @@ func (r UpdateSchemaResponse) ContentType() string { return "" } +type CreateTableStorageMigrationResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON200 the response for an HTTP 200 `application/json` response + JSON200 *map[string]interface{} + // JSON400 the response for an HTTP 400 `application/json` response + JSON400 *BadRequest + // JSON404 the response for an HTTP 404 `application/json` response + JSON404 *NotFound + // JSON500 the response for an HTTP 500 `application/json` response + JSON500 *InternalServerError +} + +// GetJSON200 returns the response for an HTTP 200 `application/json` response +func (r CreateTableStorageMigrationResponse) GetJSON200() *map[string]interface{} { + return r.JSON200 +} + +// GetJSON400 returns the response for an HTTP 400 `application/json` response +func (r CreateTableStorageMigrationResponse) GetJSON400() *BadRequest { + return r.JSON400 +} + +// GetJSON404 returns the response for an HTTP 404 `application/json` response +func (r CreateTableStorageMigrationResponse) GetJSON404() *NotFound { + return r.JSON404 +} + +// GetJSON500 returns the response for an HTTP 500 `application/json` response +func (r CreateTableStorageMigrationResponse) GetJSON500() *InternalServerError { + return r.JSON500 +} + +// GetBody returns the raw response body bytes +func (r CreateTableStorageMigrationResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r CreateTableStorageMigrationResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r CreateTableStorageMigrationResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r CreateTableStorageMigrationResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + +type GetTableStorageMigrationResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON200 the response for an HTTP 200 `application/json` response + JSON200 *map[string]interface{} + // JSON400 the response for an HTTP 400 `application/json` response + JSON400 *BadRequest + // JSON404 the response for an HTTP 404 `application/json` response + JSON404 *NotFound + // JSON500 the response for an HTTP 500 `application/json` response + JSON500 *InternalServerError +} + +// GetJSON200 returns the response for an HTTP 200 `application/json` response +func (r GetTableStorageMigrationResponse) GetJSON200() *map[string]interface{} { + return r.JSON200 +} + +// GetJSON400 returns the response for an HTTP 400 `application/json` response +func (r GetTableStorageMigrationResponse) GetJSON400() *BadRequest { + return r.JSON400 +} + +// GetJSON404 returns the response for an HTTP 404 `application/json` response +func (r GetTableStorageMigrationResponse) GetJSON404() *NotFound { + return r.JSON404 +} + +// GetJSON500 returns the response for an HTTP 500 `application/json` response +func (r GetTableStorageMigrationResponse) GetJSON500() *InternalServerError { + return r.JSON500 +} + +// GetBody returns the raw response body bytes +func (r GetTableStorageMigrationResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r GetTableStorageMigrationResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetTableStorageMigrationResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r GetTableStorageMigrationResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + +type AdvanceTableStorageMigrationResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON200 the response for an HTTP 200 `application/json` response + JSON200 *map[string]interface{} + // JSON400 the response for an HTTP 400 `application/json` response + JSON400 *BadRequest + // JSON404 the response for an HTTP 404 `application/json` response + JSON404 *NotFound + // JSON500 the response for an HTTP 500 `application/json` response + JSON500 *InternalServerError +} + +// GetJSON200 returns the response for an HTTP 200 `application/json` response +func (r AdvanceTableStorageMigrationResponse) GetJSON200() *map[string]interface{} { + return r.JSON200 +} + +// GetJSON400 returns the response for an HTTP 400 `application/json` response +func (r AdvanceTableStorageMigrationResponse) GetJSON400() *BadRequest { + return r.JSON400 +} + +// GetJSON404 returns the response for an HTTP 404 `application/json` response +func (r AdvanceTableStorageMigrationResponse) GetJSON404() *NotFound { + return r.JSON404 +} + +// GetJSON500 returns the response for an HTTP 500 `application/json` response +func (r AdvanceTableStorageMigrationResponse) GetJSON500() *InternalServerError { + return r.JSON500 +} + +// GetBody returns the raw response body bytes +func (r AdvanceTableStorageMigrationResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r AdvanceTableStorageMigrationResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r AdvanceTableStorageMigrationResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r AdvanceTableStorageMigrationResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type ListTransactionSessionsResponse struct { Body []byte HTTPResponse *http.Response @@ -56590,6 +57230,103 @@ func (c *ClientWithResponses) UpdateSchemaWithResponse(ctx context.Context, tabl return ParseUpdateSchemaResponse(rsp) } +// CreateTableStorageMigrationWithBodyWithResponse Create or resume a table storage migration job +// +// Table-admin operation for local single-shard standalone tables. Target +// vector_store changes primary_lsm source ownership without changing models, +// dimensions, artifacts or logical indexes. Retry creation with the same +// job_id, target and budgets. The job is advanced explicitly through its +// job endpoint; the server does not schedule an unattended migration loop. +// Offline migration uses antfly storage migrate against a stopped server. +// +// Takes any type of body and a specified content type, and returns a wrapper object for the known response body format(s). +// +// Corresponds with POST /db/v1/tables/{tableName}/storage/migrations (the `CreateTableStorageMigration` operationId). +func (c *ClientWithResponses) CreateTableStorageMigrationWithBodyWithResponse(ctx context.Context, tableName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateTableStorageMigrationResponse, error) { + rsp, err := c.CreateTableStorageMigrationWithBody(ctx, tableName, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseCreateTableStorageMigrationResponse(rsp) +} + +// CreateTableStorageMigrationWithResponse Create or resume a table storage migration job +// +// Table-admin operation for local single-shard standalone tables. Target +// vector_store changes primary_lsm source ownership without changing models, +// dimensions, artifacts or logical indexes. Retry creation with the same +// job_id, target and budgets. The job is advanced explicitly through its +// job endpoint; the server does not schedule an unattended migration loop. +// Offline migration uses antfly storage migrate against a stopped server. +// +// Takes a body of the `application/json` content type, and returns a wrapper object for the known response body format(s). +// +// Corresponds with POST /db/v1/tables/{tableName}/storage/migrations (the `CreateTableStorageMigration` operationId). +func (c *ClientWithResponses) CreateTableStorageMigrationWithResponse(ctx context.Context, tableName string, body CreateTableStorageMigrationJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateTableStorageMigrationResponse, error) { + rsp, err := c.CreateTableStorageMigration(ctx, tableName, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseCreateTableStorageMigrationResponse(rsp) +} + +// GetTableStorageMigrationWithResponse Read a table storage migration receipt +// +// Table-admin observation only. Does not admit, advance, or publish a job. +// A phase of admitted means catalog admission is durable but DB preparation +// has not begun; retry creation or send a job action to recover it. The +// receipt is retained until a later migration replaces it; this endpoint +// is not a permanent job history. +// +// Returns a wrapper object for the known response body format(s). +// +// Corresponds with GET /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `GetTableStorageMigration` operationId). +func (c *ClientWithResponses) GetTableStorageMigrationWithResponse(ctx context.Context, tableName string, jobId string, reqEditors ...RequestEditorFn) (*GetTableStorageMigrationResponse, error) { + rsp, err := c.GetTableStorageMigration(ctx, tableName, jobId, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetTableStorageMigrationResponse(rsp) +} + +// AdvanceTableStorageMigrationWithBodyWithResponse Advance, publish or cancel a table storage migration job +// +// Uses the job's durable configuration and budgets. Each step commits +// bounded progress. Publish is accepted only at ready; complete additionally +// certifies reference-only primary artifacts and native ANN serving. +// Cancellation is allowed only before publication. Repeating an action +// after an ambiguous response resumes the durable job. +// +// Takes any type of body and a specified content type, and returns a wrapper object for the known response body format(s). +// +// Corresponds with POST /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `AdvanceTableStorageMigration` operationId). +func (c *ClientWithResponses) AdvanceTableStorageMigrationWithBodyWithResponse(ctx context.Context, tableName string, jobId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*AdvanceTableStorageMigrationResponse, error) { + rsp, err := c.AdvanceTableStorageMigrationWithBody(ctx, tableName, jobId, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseAdvanceTableStorageMigrationResponse(rsp) +} + +// AdvanceTableStorageMigrationWithResponse Advance, publish or cancel a table storage migration job +// +// Uses the job's durable configuration and budgets. Each step commits +// bounded progress. Publish is accepted only at ready; complete additionally +// certifies reference-only primary artifacts and native ANN serving. +// Cancellation is allowed only before publication. Repeating an action +// after an ambiguous response resumes the durable job. +// +// Takes a body of the `application/json` content type, and returns a wrapper object for the known response body format(s). +// +// Corresponds with POST /db/v1/tables/{tableName}/storage/migrations/{jobId} (the `AdvanceTableStorageMigration` operationId). +func (c *ClientWithResponses) AdvanceTableStorageMigrationWithResponse(ctx context.Context, tableName string, jobId string, body AdvanceTableStorageMigrationJSONRequestBody, reqEditors ...RequestEditorFn) (*AdvanceTableStorageMigrationResponse, error) { + rsp, err := c.AdvanceTableStorageMigration(ctx, tableName, jobId, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseAdvanceTableStorageMigrationResponse(rsp) +} + // ListTransactionSessionsWithResponse List transaction sessions // // Returns a wrapper object for the known response body format(s). @@ -62678,6 +63415,165 @@ func ParseUpdateSchemaResponse(rsp *http.Response) (*UpdateSchemaResponse, error return response, nil } +// ParseCreateTableStorageMigrationResponse parses an HTTP response from a CreateTableStorageMigrationWithResponse call +func ParseCreateTableStorageMigrationResponse(rsp *http.Response) (*CreateTableStorageMigrationResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &CreateTableStorageMigrationResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest map[string]interface{} + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest BadRequest + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest NotFound + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case rsp.StatusCode == 409: + break // No content-type + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest InternalServerError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + case rsp.StatusCode == 503: + break // No content-type + + } + + return response, nil +} + +// ParseGetTableStorageMigrationResponse parses an HTTP response from a GetTableStorageMigrationWithResponse call +func ParseGetTableStorageMigrationResponse(rsp *http.Response) (*GetTableStorageMigrationResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetTableStorageMigrationResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest map[string]interface{} + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest BadRequest + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest NotFound + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case rsp.StatusCode == 409: + break // No content-type + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest InternalServerError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + case rsp.StatusCode == 503: + break // No content-type + + } + + return response, nil +} + +// ParseAdvanceTableStorageMigrationResponse parses an HTTP response from a AdvanceTableStorageMigrationWithResponse call +func ParseAdvanceTableStorageMigrationResponse(rsp *http.Response) (*AdvanceTableStorageMigrationResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &AdvanceTableStorageMigrationResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest map[string]interface{} + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest BadRequest + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest NotFound + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case rsp.StatusCode == 409: + break // No content-type + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest InternalServerError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + case rsp.StatusCode == 503: + break // No content-type + + } + + return response, nil +} + // ParseListTransactionSessionsResponse parses an HTTP response from a ListTransactionSessionsWithResponse call func ParseListTransactionSessionsResponse(rsp *http.Response) (*ListTransactionSessionsResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -65939,681 +66835,696 @@ var swaggerSpec = []string{ "wLJBpS94MJBR5YSEEt02os6KVQY0LHiXu+XyhOwMr2FdqcLIhXJ6C7QY8QE8oZXo1jW9884zBqMXp751", "h9lyrzr/yWmvFV1dOFV19d+e/CKxBJz4Of/W6Z/DsPTdRtl/+5QXAZSaK5X5ekuTmWtZVZPFz2OvAnTp", "ZHmgQkCXrnniW1teF1PfaZUfW/E5jlTbnltndGllqMUmz7tPfDwtJyadLVo34sPL3UoUhainYw38oPWm", - "FOx5qduCYSPs4hUT0BDcrv8/9v62uY0byx+GvwqKs1Vje0lKdpLJRKmte2XJzug/dqKR7OS/O+0VwW6Q", - "xKgJ9DTQkjguV+2r+wNcNS+vTzef5Cqcc4BGk80HyaIkJ3qTWM1uPB6cJ5xzfjoTuWFPINKoKKURvXHJ", - "M/EUCh6+N8KwfYQpgjYp0JUdlAI0Le6+3T88oJDdyk7cU3JTsxOPA3a0/5aVOhds4P5rdrgscm7dUkB9", - "tgFVV3yFgensrRvU3rNnbCymUsleGG5vd/d5wN/vsq92v33BMjk1T933IZea5ilotSI7jxKQDKKVw+WJ", - "6bOXUnFI1M0kdxK0FFgrLVFRWORQgEqYc7eRTo6VGI3tmq21PpzHoU6NG/7E2sLs7eykbvn7Y9iLfqqn", - "Oxcwwh6XO2OhoFjchXB/ZTo1O2G2ZmcsbM8NtV4BM1+hMZNTzO3p7LnlqFUqR2Kmlzphw/PnnW4Htrqz", - "12lbVAz5ddPGnIfprDdOix49w18vZCbKzl4HB99ZqMaY1jRx1l4dyxfp44rtHx6w+gOsVTdyivoTpzTI", - "VPR4ChHOXW/Y/ENkPUcsoNiIKwQq9m897bP9HB6BukHeJSIU7GzCZQsaVLcD18Q/qXwW1RsKSxoXxoDV", - "XUyeDG/76+f6cJFr98m3f/hjlz3/5qs/wNiBaqH8UstGPO2v1JOb6ffN/W0OrXHySzH2HKFmAvvHRwGA", - "Pmrp9132e1G5ne1dCmOf//5pnx1wxXhuIA7fac0XkrMffvrphzevzg7e/PT+8OzNTwf7745++jGEwkOR", - "kEazrbl5RJPxpJaQ5+LCq6hQyFLWRmVDmtkzS7pYTBSLjsTHVetLL7Kjww3W6vjkp//z6uAdE+pCllqB", - "hnvBS7kUCK4+ex9jLDcrrpamsEJNjrXlsvz8QxUPEizz6T9hAH7DPiwVUT8gP9OfL6NqzkhCqq2IzeY8", - "h9gKI4aBHOdczIDrbMYXtnQI+8uq1CAk8vI0proOEoEnW+1XLeRne9DkJck6S4/fi/43vVHOoajmpkdv", - "7YGLG72Vw3YLp8WKKRQdrcpWuDwwjw0rucr0VAkDJrQnTa3Yk93+bu9Ff/dpwx4d5RpAEuriQFFtoN35", - "iluQVFqcnbeFlha9c2bcMsLVKi/5VFgMeWwBA9DFWdFSUKlKc1GZlmZw9M/Xjv75ytHfnFmciJKr8+vo", - "s47giBhqujvhkHMHZ+maWiuoElQIKeIlWB8SlbnjUsD0jLTCaXWJ6rFX4GNhhxTwOGOv1Fgq4UawxwZj", - "0Pc8x/EemRAeORPwMumDvJDG6YQD1+4PJVc2aMt7Xl2e/5JnU6kG7EmIGR7Mv1HikuACGvhzwApRTiWk", - "cD3FqYGeDZqqEVPuFPcebkiP+MF/5twKY4PK/bS78OaIG9vb3f16Q823oe/2eFH0hpXMM1Gi7kvjpmLr", - "+28/py0OqUk9cnE1tWavC6+e969AKz7FC1Y6LFHbUHTke2YEwBJacdWf8Wneqifgx9Hh6bPXPM8NA/fn", - "5rp2u8BZuwUt9UrowC8XOGsbvR1d705W9w6Vx2WsfDkLny9+uBli3S9i2PhwEahu/bE5bVPosK5v6/rt", - "Hx+/OUIr5ezg5NXhqx/fHe2/OW1dNEishLCFliiHsVtzHD7Wssb4hiXqSLvSOM71EAruriA0/+VKegiG", - "163R9Hria+2LGFRzmtyt1RmWMFw9WV8PkzZ1lPOLRhn4RksflvWvxgAr7XUJPwz6l/+tu2pHqR2G77bv", - "agtiYPd6Zk6jy51Il8FHXo3BC1/3tU+0aFs0J/hpvFjJJxClYbpM1IXPlroUQ9Bi+sxxUX+6CY9VGjZA", - "xjBgEBxp00moT5ioEIM2TzjYRqrhhtaRKwrvU2GrAtSl5/1V+pLTp+MmE/Wi7xQ1Z8M4iRgvFC+KnXpy", - "ifqqTxrTvHGHot9NA8sjYTHveGNvRV2J6yTWjHGeubUU/K+UT2Scp5JLMfTb6tcWsWde8egRm3BTF3Fq", - "tkIMPEKFgdypwvHXUnIrEuWb6c01AU5NgVjDWlGoCZSShYKnugjVZMFbG4mgn0kEGfbEX2+ap15Z/r/7", - "Z/vHR2d/fvVf7s/TVyfHr07iJ+/2fz5681/xk5cn+z+/ih/810/v4z/fHP345/fH8ZPVTL7byse67awU", - "dtUp1CWiAmHlqykvoH7g3sfOsOQXYploe+l+bOw/6CVLfTBXfO7lXKrzqlj2/hv4de4TI8oClNDWT07h", - "17lPLL+Q+WxpLWT4de4T0hqWfNKiDnQ7M10te/+/dDWnAngNYPYj1KmptY8FnZoX8qwV+uHYnw/HWM7F", - "DAk6LYWNCo2yY/g3ODGAJ5JYcYdA+bRidxaxGAoVV2zXrj5DTfFd4IBJS7xTjXLJYXGLtn94cIuKUetU", - "25oXKoMAohU7618BqVfKjKL9OJr7jYLmdbu+3r6zBYXaAENp35wjTkQtI315eXFlS55i3YMxwNG7JnEU", - "ocBdp61umh/FRI4nOSBSfNZAQjNUTApCZDcYRs7VuGotrYcHw9GffwcLTFMJZ39ZIBTcERj331H5+yZw", - "klDrteC1Ku/9n42mSOi6PVAMtfal/to25IKldcVr5y1xnwUgg+sVF7+Brv9QVnmVWRGbrxtZlMeR1ECP", - "+yo6j3zy81ReAX1X5+6/mZij8qoVGczwkfA2yjqkFFKJ3SeeArCEHd4+XB/MA9jHEkNoc2unnSbiS72m", - "MdW6aVZOha5sqK9Vo3gs4nj4cvL0jTMHpjLPJSVMr6+NudJJEeviqo7/mAeb8KZTiwe+rEjCBBi3SObv", - "0LrxErI2jVC2z05R4aCKalCJVjgLDU81hoYucuUHK1nr1WlvGMMlzzI95bLttuoQfmCZULNcGtvW6Ib4", - "OssEeSMIeCIg46iW0l5AQ2CHL1nUFJP9leJ63cx4nuvL25vavHbQNrtW0e8o7Xoz3EQTIFA6/2p/W9J9", - "I6HaCgDE1XkYp6EQ+XDaGf7lXSNBuPY7tyJEN53JJpII1hlf7G8kXdoow73Um5cnDdpvJ4WG+66NPdS+", - "gqaPrstMlU4YNyx20W28MFuQVptIpFYGWK4WRZtWdlxURRalykS0eXr8VUaiEtVjz56JK/7s2R57dcWZ", - "ElXJ8x1/rxF/vX98hK+jG8B9gSZ/PxMX3rd2Gt5lT4x0WkyJ+btP8Vv0B7hv0favfZH4jdM3pvIfhGJ6", - "sv8DfQe+EPcZ+D2iT/Dnma7cj/+lq36qp3GLEC8Im4tZfTSZS12ej3J9afB7dIa4JtDxMd9CtAxUcQZ4", - "fQiBwVaQMFwr13HCYqQd+Z/FFa8dLcF90iVfEHo6gu+muyrg4BeZZykvseb7ojIy1NqsBY1/CS/FyBUL", - "3VxSN+vRu8ObbQrUvJvmFq94lnpxPLV4J84TcktGTkAm1AW74OXT6+k6cctB3Xl/8qaRPlrKW/L4L1I9", - "nGz/WBp32H0xYTgLwF0DNftD0dOlFMoGIZGoQN9tzvZTOVasKhpDcBNGpzj4138QNqwuVG7IuJkMNS+z", - "dr84L2R/hk21+byBj2Ot6lO3/bi/+4X8s5jtV+gVk26NJlDdotPtYCnqzj5dYnsYL68TwZduK19yI1Pf", - "BJAWyC33tH7djRJeFrwU5eLb8Hj+9U+gco10C5y0htJxBKIC4QOcRL2dCLoSAVkkyr7/c//4iMpEu+cZ", - "JWwOdrLhzsXzAV7X40v+F/cEf5OKXJNzL0j8+S2kB2cS9ZnGK9OcWlcZXP5jHGouRyKdpflce+EF4z4C", - "aSYtWLM0B8K72T8+Qg6GIbCdi+c8Lyb8OaGhK17Izl7nq/5u/ysqWg/bjePdSSfc7tQVPuCngljafC1w", - "ofaPmtWzuY2qg5jaT3p6+GdKRE55nidqsNDPgKCr6+jfSO0bciPcOccQwoDPfZRhSq49iMaLzFEY+1Jn", - "MzIPva3B66Cinb8RdA4yvHUc+8jvMQVICl/F/1OTHVOooQ/bg+V7sbu7zXEQBiPwOHFld8SFULZnbCn4", - "9DMaPphU6hynt5ADHe9yHaLITgTWQcW4F10ypRUNBIuzYE5RN1GOJBAi4NTJ8FcXUAvcfbL4OnuCz/YA", - "Fv+po4FP3c7X21hUhHBpmfORgnx/PyYcwPM7HMB+IzUjyhME3oGDiwPgLkWEoJNLkeGQv7rDIZ+Iqba1", - "g31IRc9CSc9wuv0bAS8B0/BxxF/f4Yghng5WbaQrhUv2/C6X7C2ksPj1EFepEJmZXy4oHADouG6A39zx", - "OcAINRKfiN6Dw3hxf6Q1EtaZrFzmSOffIJ239REY8867kivj9LIDXvBU2hl++e0dTuOHOg7Z265L9nwq", - "prqcUR1XVNgQMSnIYQZsORKEQUGG14N0d0x9qUg/LXJZZ1dZjXWWRYm4GIZKbwcjVpdsJK9E1sNaFoSd", - "UYf7J+p3v8NkMIN//I69du+zA//qE55f8pmpUWHALD0FAxfjWnqIIwIAGdbXJgIXZ84L9/J7QwkG/5F0", - "YDRJxz1+zQ2appmwopxKJY2VqR/GTz/++H/DyHrs1M8ozAF7hYqYxJzkVObcsSf3AX7qtELd87G7vora", - "AOd/lskS11uUZmeAY4rvUGCwdY9yRPGQjo5pvdgBQHNBtMtJBMSaUhVH5QnDSc4XbCpVBeFBB1DPwFkH", - "IcR4qO3EO14AccLtMU2uXa2q1Pk7cWW3rVBBR/etTdEggiq1cFTfwXLBbmYRsEI+e9RFbqaLPBi5dTNR", - "0eDAQD0tXDNkTUccGJLWlnPg/Vxyw/TI249RNilm2e6nqSgsCgji/PUrQYyYCS8EOSFQJw8IeQtmW6L8", - "TJlQFyLXheiz104VL3hpRA+wHXOfxdVlg4xb/lf5oR/6HRDeFbyfKEqbJMCUKAeXs0y4biBBBZFa+uwQ", - "HskpHwvK6AUWh66GXiizlSjA78k8vPWYu4YxNm48LsWYW8Gw8l/WK+SVyEla+gKcyFxzzTPAd0IVIlHv", - "T95AQQpWaEsX5RKhdcAPQB+DhvE9kwrqZnqUygqD82TJeGorngdsMCcP25iqt61C5t7WzVXo6b65Kw1i", - "OXet1yMkwD1y2UeL7zdg8b0ijgHFC7rEBkNKOkJ5L/A3BvzNkLEwv7iPpuEWTMOmvMfw9kjuPuG13I4k", - "9tMFyY9M/xo+1agTb9D1WawENGBv8RYkUdyQnP+9YYM5NYLtHx91vZ974rhGw2M9p3GsUAQETyeoDQyY", - "tGLKjJV57qQijmxQKwiJgpuuLhtWFlCOILomVhvYtbSGRG2oNrD1WkOirq02sNVaQ6I+Q21gIW8k3HCN", - "dJmorNRFTyrmaQOLiYE1THrg6eGfTaspB+QaZOyjytFQOYL/+lHHeNQxHnWMRx3jweoYC8pBU73AiMkV", - "zl2YXy8r5YUIaRBQ98gtJVXy3j8K0yp4aSHVEPGs7UQkKuVKKyiCW4TLZrzpg2QcKjFbilSPFYADdOla", - "V8c9dhMFN5WZTitIP0tzbowc0X51KaNx/ikAENiySq2jwUSFz+uWwUVy5Ajc1zCP5hkUE1CZnPjuocT2", - "UC1wMnqQ59O7FBifmSinPUDVBeIgjDudqhQ8G3SbWgIUnKtwTUuW6Uvl9IY+eyeuLBQCTVQ0nFQrU02F", - "YVoJVndSKdnqkH1F+7sd6f0qjOueJHc8gBVSO14+j5T86Ch4FOKPV8OPV8N3JpuJCzFfq7wWcqY7J7XM", - "nNhq3AV4X+dyme0dxnQni/evuXYC+M2bt3X1T0h5+kFMp5x99dRZ7Y4XssiNAHZidGewcFvsoxsbN7b+", - "mrMU6686x75ynNkZMG6Zsby0VdH3hRBEcDDgR4PavrWaQf5FuAWeu0I+9TFB7u9TYdkgjggaYFZAKiQE", - "MC9EFT05PX31FBcgqtKfKFQysCR8n0F+PoROAe0AHDNng2Z0GVz0DMhH0U/UfgNjHcbkZmWwN3CN7LG/", - "Hv7046sPgz47GsWVx0KsU6IuSwlX23DzW6O4sj+9e3cczFNcUNMNxRBEGbwkYiqtSRRXbABT2MPTN2Cj", - "0qk6lxNtBCYegcelyLlUUJgU32NTATkomCaZqDTXQVXCSQVAIDag6WDTfoeOAMfuNXmfqITXkgBB9FHh", - "ElHHhmD7aeXBKte5SNQTMzNWTLsMayS5k2UsV/ZpHC7eZy00MR9LRmcbBnxImhmJSHLPiNhMcgsFjNQJ", - "oWpaYWhiPgt4YpPgdHOUBH/n+hIzQp/v7rK38mU3Zs2DKb868yrhmWPUWFsYoz//9b//vISlHwRFsB/C", - "SvsI5unBWENtbrh2K7SRbmj/+t9/Pv+D6xXSH0KwJfKvPtsHQJRoQE59xsHgPFkmDVaAUFqJaWGpoO6G", - "Li9ShBPV7vKi15a7vWoKQ6eMtBQGGlUu8dBBQ53NqEUqsIP+UEEaPDkBkW9B4m/DHRj8fR4agCxd4HLB", - "U4hAd1EJ/Od/IIyvQpT4iXcnJqqx+988fwH7oEvW2BEe9opda5e/Zxw2L1Hh13TGjLCEh9jwuY6kklaE", - "MaRC5uC3BPKD1TkLlv8gUbD1Bj25tBoiG4s+e8tzd079khqftP717m6XIDmiwrll7E11uokx4eL56+df", - "LfGlHiHiUKDSyAgCWA8TiAqPJJBWlxkotQYRQHiGiRn6Dr/Z/SpQE3xCKUfLIl1CwCspKI8BxI8BxI9W", - "4qOV+OuyEgtR9uhurdXdGyKHHk3Ix+jiyNb13Dw2QJ3pucYb7U3bHUCuWW7gnlTKsGmVW1nkYo7djxcm", - "YBhHd6mZqXRSaqUrw6CHfqIOajuDYATCVz73Cxy/qIU4bU9cibSycLtR6mo8QX2TPkbInYz9+Wc6I+X3", - "rFIhIx8OFNx6hzJrkL+PNSIYN4kKbwAdm1op4aCUMSOHeSynML7Pzc3y1HFqZ8MjZ4bBhPC8oOqAgfhi", - "dxcMVyoZoKcCqgbyPIdbeQP99dlBLkFGTisD1+uFYwWg+Lm3fu9+cFp5PkMELSMvRKIGnqIHICYG3qyU", - "Is9MlxV5ZdiAiGXQZSWHJHY74Yq5bQQN1Q0yUW6U3Hh7GXZEQClB13ufveaWzr1MBdTC9XN1o69KYVDT", - "TVRlkE5eXF3NmXXvGpfmEEMO4Ze49RHFOM2WaZXP+ux1ZSun97ufPTxvomC5MdgclevFSAsMeIAwSzT2", - "HVntYUMYqfkS9iwiYk5RG74WJBorEKBQZVLPX7y8JTs0UUuiGRpadSZNwcH0IqeQp7Y40BMO2lRnpHRD", - "DIGslXuumFSZKITKYBwN+vUNJmoF/S5VrmEx7ljDhj4fippNg1l+tbFALr4ihqz5ty6BDcGmhHOBfIfK", - "12BWAIbofujjcb0/bbfJtr5UpfdONbiXTU4ffJFTDc4MrtjzF39scCHzawunb6ockFy/KPARVJBA6iLF", - "A7mfG4TTZRb1DZKcuXR6gR5FPudw1d4NSTldVgoqVt5lwcHd9ZfK7p+lAKAR+BfPwEmpMmbdFNNSDkU5", - "58v2GGAGi4RAIhJx7AWXukfAovShv1RcWajhARXDnWSb8AvBBj35xwEz1Wgkr3zeEOUdYSf7mGQVcoFC", - "ohR7AqjQPakg7+rYiXOuZqtHFec0QWceBoJ6+xH1J11iM6X/dbGl8BPOD386m3IlR8LYvqPaQVNQpVwh", - "IGbOrehB5TS6DgZaqft6MnBvnEVvDLpskOp8KEo7gNm+AienHDGl/Vxj9yhMLcDh0NyiS5eVFyCbtv/K", - "ExK1X99vz0VbYndOC0GI9SyEHIylsSUkpR35/f3hjfzx1Un4UpfgP+yZibY+RANTQxYiM8KOIlHTqE7F", - "31+cir8vnbk/AzTxd9902es3+z/23D9e7p+8wzPx5u1ffmBwYF2fkaCbRhmC7AR9t9jxz9JIrdb0zesT", - "8q786eCkyw61qmyXvc41ojAjXNdPByeYEOjDRiDs1ViuMrpa+t3v2Lv64PrJF0Kkk5VjiE47DeSXiTSF", - "Ywa/8IsXP4v0RZf9qXr56uQduqigyZ7VcAczd9EW3bHxujI23abhtQdk/7VpW2+ksdhU5y50G+xqlVJD", - "86J6J/ceq/GSP6bN3aacv6NhvJuIUHK9Ub4FgdrqFDfprERuK6qjy5euuA9DhyAxlXk0dsdMZv05deRN", - "29V0rHK4r5Z7OIil0w0+8A66UkFnyiKPy+W5WMrKIFrzWF69OIWQgs+7sw/M8zYv7Hvs2TMY/bNne+zY", - "WdeO82KlNyePs50JVxlBqjLkgO4bmKr75rCVQaOZXcf/MV3ZorLsCVz9F9Z0wa43VNisXjHX5Ftn+PYs", - "N+eAK++FDnsCi5xy2CzQ8n7+yz61UC+ya+FnaSqe1/KLK3OJxflgXIrbquR5LxReLko9LSyVQXurtcpK", - "wacwPfQmITI4jqb+DHfE394qb5kD7YC3xReOjaBM0RvjBZib47HvPFGnehp0CHJeMVgHGiBsTCZHcAbj", - "UEr4yOz5HYXdYQc/nRy6OQySanf3q9ScpbrMehcv4E+6daf9YAUvDQjW+vNDnf78l/1GA5lOL/7O8Xv/", - "zC8yPu33+/jDzvwv/n3ci3gMF7hbf9mfJwZGdEn9/3RwEn+GIRGRNjT38QESStTAwf4xYA5EjVB8dc0E", - "2shpgV78zOj4D35xFGBnBeD0BZVFIpf7/w1a6GoZCVKDhwJVFGSTMMj+IITLRnevdFePbjMxGokU9Pks", - "BE30IAyArpYd56Br7kKUifJxs/WtbqUkVmLgluWCGwseXHhaOFK/1CwTaQ7hx8gZEd8kdJ0o3xeGITOq", - "pO0RDCNPNrisYD4Y1IA37/iEgLfAx+uOWQ/tSHzxyYtvcBLDGavB0urIjGh9ZPDgzcVcXDN6o8/2Q5sT", - "XhRCmY3CicFLDDFC/lG3Je6h9hm2X74fNmIgEvW5QRCNCJhEbSsIgkUxEIm6ZhDEOwTU9HEjbFzqS8O4", - "1VN0ticKi+nNUXq99bSU9XDb4yqQpkyi5uMqvmfT5XEVaJy0BlUwjKmAwoE+qAKnAz7pZv4aRamD0K/z", - "qaHgWpdS6WpXR8MnHDk7kKAGPquAatTV8MhwY+goj17oUq0VZUSiagzf/eMj02fX9oRjvAf4wbuJAtUy", - "Dt2P1gVDiGifYAT4QM4xtfgQJAovfkJSAQKGm2hzdNm863Ej8W14T3ubAeasV+CnW0/sdz3dt28bx7Dc", - "+jvyJM4fg/Qfwy8eg/Qfg/TvMEjfMadFm9sZfDvtHsCnTZveicIVcQuiZ1JA6CtK0SuFa0hQf17vDjXE", - "SpGLC65SKG3KCdMOrX/V5ixfLKKGl/eMmK30/nC89RYsxZ+nlbF7CNhHGQrUsyUbkYoF0/SxlvAJjB3c", - "/lZMC0BYBWi+U6GyltlhSWWIxBjQTAfh9j0TRo4VOxeiMFFdXGO5FbkwBmV8nju9K6WABKtZWhmnhP3D", - "qWeuL8xyGMt00b1R117bzMHRuGE4KLUxPVQayui2AAr/BmkfRYOwEHyMc3dbXsJFwJr7B19U/0Dn4PZ1", - "trwuoCZ9HS6w/K5jxhbvLxyRhBsM8EzjiA37e7gY8gkh1Hz4oa+VusJrj/3K6in34SUFABjETYS7JRld", - "jrWEV1BABky8h8EUJ4LnbsZ/uRSqZ+wsF6xNu/RQLM4aGGH8RbNuFXk/ytJpfYDSRafXW9c9RyMXIovS", - "Y4FSXoOdhNRNhQPrXmHd4RzEqaghugPJsWdkJhI1yOVwJ3w6YAVPz904LicynbAJV1kOio0nVlIN3ZoC", - "scRL1a4nuqbJTbN9VdF1dv/KIo5iubpIy+FP0aPOeDs64z1rYL+myASk4VrAz2rRvqg8nNW8bxM9IuKU", - "QUAvVSFA2oAe0RL5xuerA9Ha95wZG5xhHu8XJHlsm6uIrfYTFZLX63BKJ8yGoVS+nIK5boUTAZR7L3jZ", - "iMCP8n+cMgASpVGZn/SgpvBYuOSvmTmnDCcvy3OuMicDlvTPc7jaYWWlfFf/Lce0FkUpilI7TuMLAYxL", - "CapPwXFR+om6ThaUz1QJNQWCZw5c+IuVh1pTkFr9dktlydtAPncqVepuH758OQzH6mFKmHmvE+z8o6vi", - "0VXx5RUFukt6fdtmYgThQrVo/kGX/l5eADKJW/4QJi4yNhP21+oLAb2lXcVYqsRAUNcq1QVewOgnCKmP", - "UkTmQsaetMSECZv2n36+gd8IPZvrFsIjacvBBO76XI6eVNLSo/reAF9iI5kLuLk/mCugsFkI23sj2AE3", - "wvi787/4L+qUHAAz83Gu9d0vzA+vs3tAEVeWFVyWJjRUzpa1QshTTit0dvO8qwfuoHnJi0nJ3Ra5b2kL", - "fenqKA7gUpeZ8bEcXJk89Of/pK+Gwl4KoQLApGnVUKCbu6gmT13dvzJCw1hTUR7J1wLU56Ot+2jrPihb", - "N+INbUw9FhZ1BOxyeVGH1hq65rW60fpCoC0ErrSE0uqSomk/L/qtGba7NASukDcIgKNRO4YZCvHSsyAT", - "QRzn0vHNHBDm0C9bg/dnwiIKLkWk+RVwrb7mqRhqfe7axcfsRX83tP1KjXNpJr2RTisjMmoAV23ucwpM", - "9l8akY96pipEeSHxS1jifdgxqPMCASjwJ2JZT3SVO3MVvBV/+Jr861nfO6gRdHTKrWGYXMa0wjge1+Ue", - "xEvv/wyBfHo6FRAR0GMlv2THB29h5G+PvwL8lDf7B/Dn1/s7++6fFEMPqO7ur8Fg4A5Zoj4mirGkg9f9", - "nT2WIPKd3LnEPehZqWZJp4uvATXia+/P8x9OZL/fTzqJ+gQtuoZ/cdI/7MtEKrt3y73Rj74P/F2oeBgL", - "crU+UbAd25atdXf3LV7jkSyXsEij9Sl/dCg/CtkHJmRrQp4TiU+aOSL+bhoxR3emYlWCG6RaOF5roZ6Y", - "T36nmLp8FsfFiwxKa/VbEnftAX7w3gD66med6iZy8FRY7kQHfJdlEiOaj+N3PnU7qspBtCJfacE2LkRJ", - "4V3QaoDqX7Xhx+GbNgx/txQIbfuxI1CsdPY6f9MTlWmxCarwIv2c1gXiwgLf9slfSrzvFSeE3tvPHll/", - "YrBAD/Mv1iDDAD4dQQP/9cOnbhNvGJ/EgMB//fDpQzNJ1HqKbqFngPgem87eXztAvh8ax8dUsF3rs0Qr", - "Jf9eCebfZ/7aGVRJ1w8GNBxwM5SK1fTYZaW+7CF+f5duFHRVOC0XHI5SmH5r8pab66kf3WeeuY3OQ9Th", - "4oHYnJp/RYSF6T6VnYRtj4jpROdiCTHtfKR/fdqpN389iZX6kuG7gTRYJkuRAqO2FpH+4NKPyIy6CSEm", - "HGolOqMIiKydsGiPT/TlaxrYXVBX6O6VsuXskcBqAqu3HQU0Vw2aa5AcrWHng5N5vORTgZS1BMG/fmWH", - "dv2Y28mxf9r59GFz8t35CJe1n5B+c4H+4EUH9oVAOmzS8qydkuESuBApJvR5cob4ctdZv8V/6HqYJ+FF", - "Cv66ZXD1kEpopc0I+Hr7pBWNY2vK7D3TNe5STAQthO1Mf477vJTGV/JLKLy6JTr7Qdj1RHZ7+zXPHa+j", - "PD6S7G0pkZ9Pr7fBk7sL3lKOhcymgj3RJfv9s9/j8PIcR2OeOvqV7tWC20mn20HDpePH2nTLdKMNqS0b", - "qCdkWgybD91OUbVhiAgLUZilKHKeLmH6WLHFKSVdVEnAY6uhNFhTh1k8gqdLjuDN3FrLjcu1dhvlmWOs", - "dqhHihN1eny4WevcpQdsPcuITq0Rdqser5VlFshF6GvTS/J90dDcgj79lTGT01tgJrFmBvblBn4ejlcR", - "egTMAa1SqajonYHqeich4DqbNgzVdmPhvblNA6Hp+/lsF8vNbYh7cbncZiDU0k5f63Ios0wo1ouC6+f2", - "+ldpp3uKX+3wgVd2Prr//cinq82ZQ3iOwJSgO6ZLPKT4Yrt3tMUScS8y7DR7MGzZn0YMYkUp9vTONExY", - "kl+rbonkwfgSf2T3Ou77tbT4gwCm/XLmyHubNguM/lo895Gwf4VGU4XMDCi0jbavZxK9J74876fqLokk", - "QWxEx6KVuMSxBHyjMaAbhu131n7BjbnUZQbQukQVCiNALT/HFM5UZFDQYQl4buDyt3+9XnfQuFVvIagU", - "KiprFS38Osvj+dYPfj2yByTWvLUBwNA1M9jxlMCm0ky5TSfEEr7b/igPtBrlMm0wqJCWAlmm5tdmEBGE", - "aX1Kr6ek7fBC9s7FbP0ditMC94+PmHuZ6UtVh/7X7j+Qm+xUpKUgqHAlEEILimZmS+7jYMJ3dBUHfX2B", - "Bs6jQL0FSybQL+p7c8eFaOOuRSsNqsZPmDtOWHtA8BKz4+FwocZFMAiQNwjOCMySQ5yfJUKWJrlNMYtd", - "rBC0fsb3KmtxlL9IO0F+tXagD1X0Pr0HDwiNg2gIa8L5MCGfuASLpsvfm/jHR/Pg9qU+UWkbJ9tI9O98", - "PBezo2ylw+ZYlFOuMLItI+dNk1nRKPrstBoaRxfK1qnBGHwODMqdJ0D8GAqqCdamF6BhHzGrdZ4ff1SX", - "O3/ugOb8IH7t7ha1iuZuSXp21374Z0e1q8IuFijemybgMb8tEd92hfe+yLg/IsEe2sjJhF+67o79YLcj", - "q7Ej38kKaf2juKznsLmkvj2iJ038LcLYthY18cOriuxBCWoVrx0AvkZes0eX2a0yJyRotPmL+uxcxxie", - "C3ZeH4wVDnP9JSurXFCOKQ4mqi9CZUqD18wZoctCseoo6telnpJzbI5rLZZwh6apZm/oz5sW0SitRgEM", - "4VohygFu4uswB9/AtSIdzsLd6/wtY9t4ffXlWx3vO9fzqjGvvP6PG4EojfXax3G0/Usj4O6HBcG41JiC", - "LGr6qa2IeNUeEHtiumQnOhfRE08X6H8O1PmrDPGLaD9EwkfsLMqx2OCei+d5w0S7zn3XccQU78JFtyp7", - "5PFO7PFOrIwpefmJ2K4Lbz/LvP+uKaTWnqr9LKtH+k5v8dIrPkgtOnM9bH8L7oafZXfqhdtAt6/HyaEk", - "+gO//3o89bd76vezbO6IrZKEK9X7UudiQ8UeY92bST9sKqZDUZqJLGL9fpn+7nSXTTX3k0Y/PpLRatIm", - "l+q7euMoZPfqnhWKA+ADz3AJFyOSN8n2yMWD1XLdNO/44OkSyeRXn3Lizdq580fpemtSSqSaiFJakWFD", - "IV2zzvus7+SWB8yewAm+LS10jvw/R9l85PK3cD0Lyl1JW7xAXnei0K3l+yuIdD/L3GC3qtO5DvaNkWM1", - "dW215iYsESUcPntg2h2M9mHqdfcgSn7NOhwcqUXtrSXZu0Vv2zzVm+d5a7r3kuiKVknzxSZwP1LqdvLG", - "F4n25qmJywTTpifgOtni7Xm87UdhfW74l5kUXuvov9XkcNze7eSEX5+WfhCPid+/LbJsSwDfmCZvK5bk", - "i0z8vuGZIwxO1GqHOpt5TNeFfOvW3PDHpPDfQlL4o664jVz0DfmaU/ayoVP1+BimBNTeG1Yyz0S5vLzv", - "eyMM44q9efMWiq+EMuGES12XT6VC5RIL3kuFV1M1sHt8yvoIcOdRbDM6e4RAUxmR1bVfKOkdiqQTpfUd", - "jZ9ZcWXPqNdwXs+wJQTG67cCnFVGOAk80iUUqX3plsDHK4QS65QKF5YCqqb7Eu1qzIjp1h/BXsAO/N4w", - "pBfAZRtDCTtW4s04z918TvZ/YIUsRC5Va2V1mOtL3BpoYEu+nbifeyr+2hyCqfJWDo0V8h2x3hObm8d0", - "IVYHAXae5HUZLgWADu6O6ZFycX9cL66i6tfGcYzU6TeIevF04+qqQPHvxLTQJS9lPnuvQj3uW+eoQHmO", - "T9UoCzNkAPP8LeKr8NZZOLKmjb2G876OtbLD1/sEasiLotQ8neA1HEbS1MrHXqLSnJdyNGP/+v//P8yI", - "XKT2zFjHQMf4rBQjqQSxQPcAEbGhiPazZ38WM/ZauGkJs/fsGdYOB5iVnm/l2bM9diqm3LGsLnv59sU3", - "XWZLARV/eDHpMl/jFWr/TGbDUmYxhsUxMTXXzsGESxX4I2TyAiqlUE7RJCJxjfvFhzLt7vTA+hsaBiuB", - "KxiqdX4AS0B0jHXYUetDGJQewK47ZoyLkEnDp0M5ruB9bOJEcKOV6wiGiIgZaHIaKwoD6cOKZRWBjdFG", - "4iKe4krJsIQDP9DBHvtZpFaXzMipdMO0Mz81zEaoYS3hw+H0xTeDPfa6yvMeZVrBy7BUbu0BmFSqMbzt", - "V36wx05roYrAl7TI8J5b0cEeO7JAnRcCl1jxCznG1CdoHk+A/Ac+enLMx+JIZeIKIdpgrQew5QO3OAgT", - "YiayIEq1Jb8QpXGL0mMDpIPBHjvQ06FUgpmwSNjbyclrtDJKrs5pHU9fsVcXjm2+mxXRYlpRnAEhCNeg", - "pyfYGDYUY67YEyMEOz19dWpFcYpvkrx9WjdRlHpcCmNcG/RPGIpUoJSIotFMeGWhHcegIL1jYTAjqaSZ", - "iKzR0IF/vdFSmnNjAtEO9lCXYc3HROfY3EHjp3ceigf+QimJLZeelJEqBJ8igj/RN1BVOqnUOXuCtiB+", - "NtJ5ri+rYlDjzGQMH/aqoobCaXw0kdbRlcrkhcyqCGyoMXKY2p8kjc9qnZ9NdeYo8p3WeYBSdc+IhUVL", - "6F55qzOiP3foBnvs1QXPK8LjcWcxNfi+ex6vRaaVAGr1ipbfO3w9PAfxsvChN1gpC9bi2RlJx1mGTqiK", - "rAesnQl1IXJdYMzwUNsJmozOGHZEnWplqqlAbJ8BSEQ3CZCMOoXaxvWE4fHTVmydeLTbutqbW5J7UQDb", - "9gULRIkruyMuAL0XSLvZ5rwjwykgvVO3QcBWHC93H/XZKS31BW5eNXUblyinmUDrEBCOFZ/tRMiSpbrE", - "2YJVQONEiMZ5v0oL7JenPiSWMgAZ3LmiOj+QCJ3g6/VK2I/avp6DsFv9ARz8Yz7LNc/eaf2Gl2Oc9YsX", - "G378XhFeJ2p67tPvNvz0hFvxRk6lFfes/EZAcxsM+yXPfuBWXGKZ+s9Xjl0bX2/YN3X8Tk6FruwWbgY8", - "/cGIWW0JR2LDv4JlXiL5sKmyPeTpeVUsV7HrvHd805fTQ5er00lqlyY+67NXoIGTIU+fUTkJAw4DtLa8", - "PsaeIKV0mXQKlDBdZia8zAKEIg9QP/t5Tr/hh25znNohMlyBfxiLwDz7TGnVQ+OSRlBUw9zpG24uaV4Z", - "K8peLi5EzjzYPGbl85F1tHghylmi6mxpmE+YjXHKbe24DS2QDESpl+rpVNpEEfqwygB0scxMLVjFVQEC", - "fCeoSdRRqitlTZ/ts0HhlHqeDxKlSzZAW3DAuLVuelRSABK2e/hpJvlYaWOdnB9WlmVaGDBtaQUYd/My", - "VsMEIjDLALrP9v0K4bIqXWsatPHUbaIGX+/uDuA1XVl2WUowQAK5uLGPeGrBkfQSnx0dYqEROZ1W5P0+", - "ERWBDLOjQ0Qw9jVgJlDK3e9e5mw4HFpYdL8Eg693vxvAOueC+7tUKCEDUPTYeaXSCVdjkfW9OaJLPhbs", - "jUbm5hVp93eOiIdQLHKPDdwfezs7OwW3kx2r6ewAvuL+lP9DK3b61R4bmK/2dnaGVXoubI+q68y97/ql", - "tQiWCHQLcEYfcxrKp51E/euf/++//vm///rn/7KP+PmZzD71PP36I9R3HJkx9mR+bZ66Fv5JLdS4jD2T", - "6iLQmn/ZUFUokEBmGbgSDnxb1Shw/NjHPelTc2NYjqNEW+hPYjjCfXakHFe0PkMOWnD2ELeVM6gGtVGU", - "KGkYeOIit1yXXU5kLuqjz+KTX4pClzY68YlyP1WlMKCPMXcUxxPqHL1t0hqRj9glN4xUhLm7/X6ifnLc", - "Lxpag2NGDGOexpxiVytnqwXnS16nLYcCU6s/8QWiIrVk9QderUCN9tW1cIlwS9/Ssdqq34yIpylLm1w2", - "EuK06GdTrvgYcHlbxPjyUKs30h1vrYQ3xVjhmJ4ezYlCaodJRXD0YZd752LmJFAmSsbt3GWm51j94AcK", - "wp0PnVwQTiXAxhOFugBWlxB+9r7eVJdZORXG8mmBCDF063EhSijMy465Ac7uhBhV0xkocWXP0qo0uhzU", - "HL7hBXRdKYSNHYs2uHi3RC9pGdeE/nuZ4aftOiLXE5i02Ep/22Kk041uruH16Qw/2OGwaD0ayc6SXAQ/", - "/pWX42tTgn8En8lAXOGxO5N64FQ3hRCNrK6KBYvjhHooLEIyOZDPknHWrX3eSN/yKzmtpoHKAwFJBWcD", - "SGPZWgGId7dhP4+4M7X3nu/udjtTbBv+cn9KRX8Gm1cqK8ZwQ3/VG+uee9oz57LoaXLA9kBRFGVnb8Rz", - "I1omcKCVlYp8OUjv9SSotBsWbQRIpRXTwY8b85nyqzdCjR2zev7ijzCF8Lc7ENbtb2ev8z9/3e/9N+/9", - "Y7f33Yf6n/2z3odn/7Zg4W8+2Q9bFOl4riFOdIU8f+PLhQfwUs9Wbybebi6ttlISemFW15AuNp0stxGP", - "RQkw5/Aak8qI0pqurzmEXDyAoRvG01Ibg7cNRe4FgBMLjDNnBTijxOqpTPErDicf1HVn/9WGrG8pkqHO", - "rECzy0lRbIU75QrNi0w6shxW1tkRL3rFhBv/Pnvy4vjgaZ+9koDHwJs9garkhJMunWUpWKbJfnjvWuBG", - "mGfP0LaFiUIEJdp6HgicRvgE14dYjl8lD3AOYBAI7cqlslwqf+Xj9EC4Jx3DtQhN3a9dj+3jitEdDuwz", - "VlZpux6HO6uXbrt+ASz77SjzdS/X0uSfb2UAy4/9gVvJnrfvHQmHxYpUZHegX6Bf6i7HhrSZiVQagiYl", - "x8P37EIaOZS5owZdMjAVUllw8AsBtjPEkBkroZAA+GJvrKdf1995i5Vj39U8AFdjJXh8/bLTPOFqayhS", - "XhnBuBONGYAhdpnVhc71eMZGQqWiO7+CzlQT4IBypkfDJ7p6FU7dh7kwJhrJa7TOIiMkWhhw0hc5l3NL", - "stZRDjUmtS4zqbjVJdh2lYq4vBjp0s06bdLQ96ATI23wkdhCnbulxykWOc44WOaUJHG0NnHEW6ITwXM7", - "YUVeGXAM9pTORJeVzgroQmimTHmENxntvi5Zxs1kqHmZsQspLk075CuNaPtuh3c0tFZuQPMNw/fWTVuc", - "/h2XvGW9GG4Ur2OJzz8kVQjgUeeWcXNFqDZElpvar1Q1hQthgw6UlHnLKLKKTHBuC0J9l4Y5qmXOOi8T", - "ZXVBJvkg+mqwx6RHZmZFqS9kBmWBLsXQG6D100SFjo9+CvF7pI8dHB74g4GCDsry+Hg/wci4qb1X0g+5", - "58FeXTP8QsvMMJPry9AzXEG4lgAcnuz6/0AY/S56AhYn4drHIIyM5XBxrEsmrUkijkZI/JcTUQoydehj", - "cVVoEyB/nKa3f3zUZwf1yiUKnMojLnN0C8D1JOiL6E/zVwjopGNJB26jko7TZ63gGdMj9LN5v8XlBJPq", - "qXYv2/ef6hFLOvX2Jh02FVyRzyIyjLG4ObjoHe9W2sLMe0Wph2iTNqsDL/FZRJNc57dw8pP3jHAvWVho", - "NHiiQZ1LlUFVEdq3REGcEUs6Yc+6kaXfTbM06Tzts0OkFyxIkueJgob6DGJHfWIWWaiBppKOeyvpYBAO", - "TrDNWIX75c61TP1lUxVXBVdQhqbPTquCdv6C55Uwe2yAJDZgVicKNgODoGp6hRc8laHD3Pt13dTrDYxO", - "X3SA++yNI+5LXZ47iteF7TmzR2Vk+PRGuRxPLCscD8Br0eXLQlt0vYU5FZBZnHSc8p10oEjbrCCfGjMT", - "XVq6j+0ZmTnbJp0I8tzQXRguTWMp3zQeQC0/ssaMKCXPQUJY7TSwC6EsnAOPUl2KUSnMhPFpkYeIHaKc", - "cHc10mUKxXMdj8TDBfdhwc9YilzzFTREvaxcrG26IKJjulrhD2IhlhaPgv5GPo9IyLadxhsJ/p2P9R9n", - "Mvu0E5jDzsfAnT8t95YcqQt9jikBNVupLU47KcPtTbgOqHvsJyqA1g3qnp8N4Oo1LkHXD8H2tEVww+dk", - "XKIGGNqzxxwTGJALsSFSf2/Y0oig75HDI3Bl2FWSpeSM5E6wjdzwMsxaapFeuBJHfhIHsZd3To61pHU1", - "duHz/MJhRS3YVEEYprzgZGGTZOgn1e7uV2mYB/wpBksyz+oUxZXZZ6qaOhKE4NZOt+NzOuCjkqvzTrcD", - "IYjg77clTzGc7TK4bjgUTXUGZ1rKoeh8WFIm6DbzwnAWLZlht3tRe53OrxvxBm84da4+hwLp/HIiVFQp", - "n/SW+pDSHXl/k4C2o0VdtyWmrT0GLaJFXbZimtyUTxP4wLKhzjMTp6xQagb4SuYOShDTmUhzTtq57y46", - "RlFw1waLhFfXeO298Q1uaAkcMFIoe8ALnkLntyxikH9BaXW8bnMD4lY6W6Umqcbl1YbiZnXaw0mlmMCY", - "Xl0aZz7SolFYlA/GwdwFDMfxSWPvQ+oWswJNpbqprv+3e5wiXjglD8CuoztHhHBicoHXkWoUXvzsGXtC", - "uw8OF5Wd2bKyk34pcnHBlT1zRiMGRJFe4548BQd6KZz92IV7LHBYdZnK0nGXTcuyy6a8wE7fvHnb46b3", - "tyobi7Z+8QdUAHzD0Hkquo6m7GRU5UoY0w2BG/6vCdk54B+zsy6biLyI3i5LkVr6A+KctTr7e8UdgbdI", - "OYq+3paTHYO47yVQpo4fb+N7UdR5Hdx2r6lnNeHOZ6GFUnSeZWH249OHkweG5BwSwW47LdZylfFcKxEv", - "kncZbRpLihbXUrb1ClOpjOdG8fWdZwzhHs9ZxBMO92e+FLuTKZTlinsB7Od3v6NkkFcYCmGQPcznBPnw", - "OrdfifqYKOZMYIie6uyxpHMpz2UhMsmTThd/nM+Qde99THAF8JuhzmZ7bpcrK8qk84k+hEgB98bz3UR9", - "ojA6CDX0CVmbDQmuI9yE/Ih8nlQ0IHxrJFN/O5jncgxiJyLSugUKsHVf/jXpWGlzceZmcRYSq5LOh/lp", - "vJibxp8gWcn7G5+cnLx+um4qRamzKrXmGoub88Lqgo1BQ6sXt3UNJnI8ccoKZPi4yeNXzO/NsgWgYbXN", - "Ps6FXhzcvxelTMUeGgMvdnd32b9LdWasTs/30K/yKbTjGAn158yCpNOFBZEp/TM6JC2L//yb5uKDTzUU", - "Vli37lCUqF50nFNRipG8whewuugeT6di7xqbw1MrL8TcVMWV02pQGLauGvrJ9gjup+XAfDNHaT8e+loS", - "U25hrq+dFlLl5z5hr8uMUFkdS+CTJblh9DE4dQcHyDF672aF2IvPxs5VT2Vu+Qb9REHgOiSoTStjHf+j", - "z5NEDfZ8eG69wsg23Lq10GXS2T+Cn2qa+2vijDzY5TDpbz7NNZnpFLasbRui9Uw6trK6lDyHhQztPd/9", - "tCxy94dcD3n+F/KLbUMZwR7gDnRU5XHBARBZbevebBy3urPXGUrFYZgt9tUdwtg0Z+J7WZpfH4ULdDsT", - "KNkLYzoUTptF221RnX99wL779utv4lIPWf2B0+lFn/mh+HhIMYXbEbgiwJ7QauXRIIKaA/UgLFzrYLsi", - "SxSmpxJxCTNAped7lnKllUx5XnuNnDgOj3ta5TPvnc4h6bMqR1D8Rk+lZdKis6etvs5/Pv/2j99+u/vC", - "6X2NmLL//Otu77sP//5vbTt+13oiKhnlNeOUgQLiYOUvNelrgw+XhlhvMW8LvvsC8rZ8NBxnY+CGzF9E", - "bKRDY5y9WGH84ws+9MtXNfd32hjb12cHXFHMvpiLMXcMYmiEDcY7vvRWZ3USubMzzuToDK8oB3tsf6hL", - "y+SIcTVj1hEpqepzUM3sCV0cUwb4uSwa7Zyey6JOyYmUfT9Wuh+08Lm+ECV4oiA5m4+xBjUcUm6Fr7kF", - "qXB1bovpukZUHPznE5/sREyZBvuqOYRw7e2HIaGwcEUZUmam0kmpla4M+5seNot1+ewjzBuDS96/6SGF", - "/7mGClEaCX5EN3rId67bdq/SLqms0VOiLnV5LkrwqwdS77Nj7bYzGiv2FsEw+dT9fqIOuOW5HlMkAsqT", - "EEM2Y+lEpOcYiAsV+/2eKhi40jZRpSgQSBXn5zrlpe2zoxHLUcBNZMEw4t74oCO4a68b77pmKCSNZ7qA", - "VIR8xrhi4oqntotRaj1UK+tNwKowoXaa056VzytTddbB96xSpchhlI7AoVyF26t5QoPb/zTXBrK/9mnl", - "uJN+bEAhniIbxHmAgMITqIwiJgzFDPg0BBxuSOpzYwv3kgADbBobhvOCM5qKPPe7kqhUE1e4EM2w0cb+", - "QQYHd2pun52IqbaComuxoFyijOVjN2tIlq+3REQ5gujfhdi3PjuOGjdWF4YpcZkongUQD5WxrORSGVZf", - "2sLew0/QWBk231zyoqAoiURhFapewe0kOqT91ox9ZH1rky9gBudiRmqNBj9dPqO4toBQDAhlGEDhz0mf", - "/VnM6F6aMuB0KFpAnnJ3EkqpUlnwHBUfYq2BRQH76/uibSb4W2NdLI3RXh2dkbezKhWT1kSFuwCA9URU", - "ho6NmxjRnmKVOlf6UrF6L3RlUz0V3zNTFYUjU/hAKmkl8DoIboGzhqIh18ZHrLgJlqIBNw2XVhOPLUHX", - "VkeZk7JWqHTWQyzX1iSBF9/8YT5J4LNi/7eWTUiEdS0v6YtbLM8Avf8fPWwvd1Azcc+VeZqKwsJdYGRD", - "zO9Ki2ZALNYjk6Nz3UQHAWu/rQ4Y6fhM2LYecmRO70/eeABAL8japNG6nk7cge3tjyjWc77U63hMQNUa", - "S55kIudQzc6IVKvMNJtfkXHTsCG2mqBIavuWi5XFoQEkD12PwKD9fmRwu0PpCyiPuVLasiHlACLrk8ax", - "qozyC2Dd7iLPksh+37O1bWZa+iM2l3DiNeehz2dedTfnWBnl1NXL1VPaOjI7pCU/ODygGGNeTJob4G+l", - "YCpOSdk/PuoBp29c0fY7i6bAzt/0cIMa+k7QCGN7I1ka2w1Zd5jujZ6/KOuzFI5aRBYfW1oRd6iD9oIq", - "3pKS+4GxmSUBG5un0H0zl0HXXX2cFyI4fir43yuxzZS4F7urMuKe9777AB6Mj7vd5999+rd2KNW2XiEH", - "qdMWD/L3SlQgBMpKKZSnQUftdDt0L97tpKhGiqwt7mNJr6D9tPbqS6D6GPwP3TsNk6uJypFYa63hOsIh", - "Jt4b5+htwK0ch5Op2CaTgnC1Fll6jRi1mF/sfPybHp7JbA3kACmwQ8c4xGjkTJo0Mkr67C9AgyFeNZAa", - "k9OpyCS3Ip8lCnRbJFJ8Fa0IDnUHMOvaqejecpmB9SjLUkBSdzTjRMWGjjeogmWRNg0mpofghMzA4ifN", - "Y+qkYKJicw59+PjxWYjqoeC3NksEDbNIcbsTgm8NvCRw9Fi7wmuUG+Rl3bpcRQdIbLUGWzjaqXUUvAbE", - "wGcp1KvQ86sAUfdBVmH5PShMMSMRkCi0u205a5QccpK7h7KNDb7Z/WrAKmVljlI0n0UppK6rCTeJgh2m", - "Ugd5PvuepbnEmMiJrvLMJ1WFZAFmwQ4F04vXeRSJ4gbtwaEB67vtnkbYh0B7J7dBcw+Hw/4gWhmsn9pa", - "Il0fkYosd2WsZ5xj/93GCsUfWxWKDzXrNyIthV1d/gNcWVFANH4DgApx7T5KYnmCP/dwnUYyF10m1AW7", - "4CWGgmk7edpP1I+OhwcfKLWJ6RTsX//7T3Sl1X00SnpRX8sSW05pUtu834Mulmka+KtfkVx6m/Ax4H6p", - "BkOEuOGhWiDgnY/nYrYBTBLUm4bNCSZLRNmXpUQf4QIN9xkUOapTyrDOjY99mupMQIA9UvM3u1+5FzCl", - "HhJP6IUFcj2E0SK9bIS0RKRFoRAPPanjpjx/u+MnmdFIeu55uqjpDOKSGzs+t6G3fhKQGgKNXlu0tJLK", - "uZgR+A0GBepCKC77vJBn52L2tD0B4VzMVoqjWOLw3j/2e/9NlVxaL+KXYuDg/ZVfeoKbuNcDeVzZ6DTe", - "vmcZG4fiGfcUfIsjWA6U5WWXLpeH3V7PFfrAxN5vmcM0T9w1xCyK5XXORPC98Xy+vgKZ+VZOhVfd2myX", - "Uy/7t10qgTpaUSiBVLfH7MnPKZNwXV2OahfWRLao2r/z5Q1XSj6EXvLu+iHJPwxf9WIQg3jPks7TRjU8", - "eLjEz4sNLL/V/Ka7PmNwYWilGIsrRvK0Htz/wED6z84uKL5tfqD/UxlRnvWfLRsrCeglg/1md/f23bMb", - "AeDCDp4Gr8A69Nv9kIBP1PGrqKgW4qqiwwEPVh8NgoMFmNimrTNnVZS6eBfACxevp9ddS5IP622FOUr7", - "/loZKlm1GCVYljsrdVHcjs7wYLbLrWQLvFvLXnXbmdYPwi7ZiduTao0jtQyiKhOWy/wBeH9/CBGAfkhr", - "1nUlq3dnwUcU+F1aBuXp3t3UydbKzZ0ds6bYvRKXNDu4PPCBK4RlwTIxgogbraKi9Qv+LZ+f9N4IdsCN", - "T006ldNwCw1xGpWKI7+55a1pHaqanmFKFOSFNDMl3tVjHW2S+dRs7CufXIOge5C4oRhzjzzWwBlRafQj", - "q3OUGk9bW/I/uL3AbBBM26akk/qFIqR5L3wNCTxZy+Nmw7jPcw3jS+ESHyrKUFbOuZhd6jJLOh+aH3zq", - "zncOCVO33r/bK0wF2nwkQ53NtjSQ+b7jPz/NbVZohkrR9KTq8TyP88twZjjeqOnQLP3Dt5z4mICzeh6B", - "yNwrnxYzuQKxIsGfyexqjkzrtkJSDbZG/X9aepoulgCSbXCqvqGRNhK83BiOMe2MpRSci3WnmsmBddLf", - "tc8kZbXd3Zn0aXT3ejYxt+4BHM257b6DE9pd3BDILlzTuaqmQ1Fu3jnUtZPpbXKIOiGymQV5Mz4RCH89", - "n6BkvVWcIsIarJeIEjrxjcaYo1fwS7e0cyfQ17agw5bnfMob6590oMwMsb08702lkvl07p2qpDcm1hZ7", - "OztwJT3Rxu49f/71V197zlav2QKL+93v2EthLDsueWplippJjx0C+qVXcpxqkqKjVuRUFG7qpiDiXFv3", - "HYAIkvYhLkXpU7ddC2bqzCSn1RhhDXvyvPcV5IY4pWgquJJqPKrIjqKLwoZpT2nDUTJrIzUa2jrQykDN", - "EA4bhjkWtR4ESSPIx4kaYGSTOI+5LQoEtMF3EXr8rccn1z3ckxf5HV21L7E2MIq9aQviPs9dJtO6RkWS", - "P8M4/XozszETV/7rOVf1XQThHrbF2V4rxPara07yDmIkDuKshfXW8n1ExS66UHY8xtR6xzZ8RHVV6wwM", - "/z0TqpTpBLCCY+IGZPJEwcddVqOXxK9T4SN3EiCBfDjzZ4IqGUoqABo6c+RZ6rxX5FwJKD38PcMkkBqy", - "uB5aqaF0BhTmBGyhRE25rWsr1hhOiKNO2CwpFXYNLS4Lttinjl7Vc9q6r2Oxz6Uxn4s7FMVjXPty+iE5", - "ESnjroUEo6MHpHR/DpUVkUerjuPOR/9P/KGe2/oID1OPfPl5rd0wfQLaITysUqRapTKXlEFfQlKb7uVa", - "jUXZy4SBKj8RNZViLAF0GRgTZNJLC+BcELlqNcs0M3p58MciMXc+E9mgpbbe+lNBESV3V+j/m/UfvBV2", - "orMftd2HAM3sTjJM7kBa+jgP1XZ8H+7p7S7JqGw5YK6fJbUt46N9PeaxJIrkBE4g5JRe59x75Mg6Q7gx", - "ep8kPgcGOcUU1T47AGxvJzKNEQ326wTsjLmHg7oSC2ylL5tqtWsnGiBYG1JRTikhoCbK11KvLRJoZknc", - "yhJGsoXKcaEDLDq8DXSWG/GwkijhkY3dCRvzB49B5hSclhvxtJspBaWg8iUYj/ZgmGLcW0A8mOMR22CN", - "yyp+AgQzYTcGngE1egouS6i4jqq/mtE40XsiSjA5E0XV9sgeKPUloDw6awtAHfvgkUEkxXMxG9SRtaFc", - "PzeYTwEJPcSRfm/YwL1J36CrvMiFr2dPGXlpBFzXT9RhjcPlJ0DmUO4EAKVUUOEMNgB3EuE7mnpkUHqB", - "hgbAtVjEHvH9fMXAeLCE/FBNBcJQQLuJcrRcCOXs13yGup9xlrwTFDkvxwJfrBGuW4sXEBkfkqHlGdsJ", - "V+Nt+Y/m+yJPEo0kciltM7993SBW4OcSOQcWgBmaxgAeBr+uR+hRBnyODKAtYHyO2/kinWSolkTOm8A4", - "3UgM9Hy686MsWBek4LOZKiPK3oinWMEWuOnf9JAcQX683gFU68dYxghy0TKoi2ya4ANu0gEtWJQ9BN7H", - "DhJVlHIqIecPqrAMZ2xQb+IAoyGoUlVIx46lQKKInzOj2UQbgjkIMiDFAlaOV7fwYBqGSZQvvpLqPOcF", - "shCqf4SqeMCUpHJcTuJh1218HK4yFni4n5fPx9s+K4+7hDHdEzdvTL01SdAzDkdIjm/TncEj275Lf/38", - "KY8cx/EGbZ1xQ975EaadLwukW3u6tuRz/jzSjjNgt0/VDyoQfEPiWgwUX6Sx7kMX6wtzvG253gI+E6+h", - "hFu0kcSiK+1JxkfZXXjr2w/2DukKX4CG9mvcypUOAq1qhS12C4BxjLZ0KERY5eAHrSspLGhC+7jRXya3", - "BgOSKpNaUU6l4reO4Px5I6Rz9Kgn3YWeRLT8ABWlHayU8shOHxQ7xQJAhFRSrScXKGjEo2fAd6VJlGdC", - "VB6p2/So+roZkC/tK6tSISVgy98nar7okS9Eg0VqLXZV14R15rRrbv/4iCkqzlHoEpB0o7o2UgWoURAR", - "UW2n5RWRvkxZUBet0uWCUOjft1RoFAEOxam+byUJ2KoJYQXPhKXixRnjzdJYoc7Wb85cwdW8Cz5PZSOv", - "zbcBcNeXnLy9iJxlnOwllgc9OsSqx3I6rSzWEfcX51yFQ0F10kXGjg4Dbxp8vfvdINQ4hkLfoaCwv06n", - "qvMLbAN732YILfZwrejZ691gFxHs5MdOvet1Cm8D9GOxyO3aK3DaIk+VWMrZ3lbVhmuriRtUvIUNxWHH", - "xW+3rilil2+piNs2FUbak8WU0WtzCZtObsAkRgFtMcaQ3Qa7mD+wNp38EoBlt3FebTrZ4nHdoO8VF5LN", - "FY/hfG5ZS1g7FAyQZxlhYNagDeJ7diGNJFhi7Uz80spUFhwiZ6gEuDQIqcAKoTLY8ztlH9EaASRvkXPZ", - "Hhm0HDf33UQQhgedhJRYja/3aZzdiFdebhfcnAHQIPMItR6jiKUQToD4tKj4CXY5EYBf7a+HeuNSVwU7", - "4SMLajIgKQR9uc/2VQQsQUX4CSQjhIhC9MyMDXOpMook8BUofalMqtKttOpJX9DdEn6DLqcQW7Aphs+h", - "UEacgKPH8auiFMZU5eYFvAFSKhfGAFQuB4za11zmoY2vbmsvgaNADDvYQEEpQ5vOziJMYV1mUnHI1THM", - "1nA/jSI8VA/WGzThmGwNzAcpUDqCIjgsDLQF9N3PlhJRUkSvUbj7BpLjcqKN8HWfGtkWFFUzFIjBArWj", - "7z7H/kSEYjgmDpOslcmDwwOf7lDnjZTC6Nyd5XhKiapjP1Oe5wBb742mGrw6pGDUB67m8dIgHksmskRB", - "Nn4xLnkWocb4JCgiupYC+81q6+ySm0RJZTH+KsO7cNe0uHKcStp8FhBbSgQ2qJvSl4owdQgEJFFR9pA0", - "rNTu2GZ4e4+zZkqIzABIiIpBuykEikK+KDY+ILbMvSlAcAAAesXzxpjQYmiNegqbCTrhYbQ5n+sQaCre", - "deUqX6i8rqnUUqO8S3XMW2Enaqquy51T+x820NcPV218DeJBZ+ymgvcOypsdIPHkPD03fiZOp2hQDQWR", - "zBHD9myLO0fT+PxEvgfi+9h3FOc5P57wxpwgbzN2K9WTuXn9oh0vy+85ZmupvDlNuTLsXMwg3JVHVV3c", - "n6ou7XIuZhjdFkE1UTyp4+dKXOZSiV4mckRMZAAv+wRhZp/2WQ0eG7Q8ju8gK4E+UR0cnMlsMK8FjSTi", - "d4UR5TNWlPpvCPMSXsZk7DqpsAqA/omSFpgzSEQM5OJ5vjD3CAPQbQf87NcDV6A1MCrl6s9iZrZVUJOa", - "XxnmtEqAtOPJzlFg6x52MRgYtk62bZojk6QD5S1w+TvdzYBqm70D9NgI2MkTY0vBpxBGSEDFT39zvlq3", - "5YGuW09nfSY/Q7f2HdS1pdvLEmp9XhUId7aSjTnLnPeMcC85EvKF7aIqCVjqwhfCDXhriToaob2IyblO", - "NYxvgEZVnteJuuy0KuDiZg9KLGBUPXay52sRdVFqJx33yo+ElsXtxHjw7z7PMoCCTKWd4Wu/yDxLeYmt", - "nKWTSp2b/jP87ZWH8IYfe+HXfgyRDsNxM+B5NJ76DdM9w02WwnSpCfdVo+5hPPyuh2Xowx63V0LEnjrX", - "vEDkEFQK8JsqnQUoyRw2G1Von501KAXPMMFrUKdbI8MuSumIFr53fII7lQ/DULljxgNjeS4Gjt3qS8dC", - "JtpiHePhDBoBR0TUtcdGMXwkeq4R9ub0R2S7rdBE9Rza8ZM69eDdMpOK3HiIcBdnuSC8ITfibaD78CyT", - "KMCOIw0e5fda3foEUVOBSzbh7/5vD4Gvez+L0rSi09EPzOpzoWirpYkO1HvIrpcG8Qnc8fzp4AAtMXR+", - "JAq2AgqVWA3OIkzU9xrTVGdyRFMH0NjObtJhUmUAUIKn2PGsTAv0DIFROw+HvQLg+rfB+U+oHq+zLLxe", - "MwTcwi8nePDPYhYiJPwcwAWIU7tZPfbrhe7Nibbr1MFYFdcR1ZIgum9UaAdNky/UlUCFNHdHxyPjZNIg", - "EGkjI9ejOSWKKlnAJaQP0vd9g0OwlOOxKMnHG1KElpWwmL95f+unsU6iez88VbJkkHfsmDoS7KAW0rMi", - "KrXhJb5W+ayfqEHJL50EMEEBMEwjLJ3/AupvC9Qv0cgAQjDBNTPvwEFE/2a9aGmYUO79bLm8wIm04rv5", - "U9jtlPzyrtHdlm3R2rofYc1vXPXjNx3/huhy6w/8l8N+j0WZCmV7QqU6E9k8OwJU5XvlwXPhf+tr7nsK", - "h2jdOh8SWUZ7LlfYwW6iTJVOnDUZC6RepaRlEylKXqaTGRNXtuTgTiDs0+PD1132p3dv3wAwZpc51g4F", - "CoC1Ec69U4XhQnHCy6zHL3kpvvcZW5kocj1DAw5fpLzckG8L4+GWM7xfAzA0fUkyqG3X2gHRlnGOR97+", - "0Hn7Znx9VPPpR8a+OWM/QhVqZcDbtD4qj8z9QUY+346IebClNH5z27nsLuJIXfBcZsFV4N0K4aiiDPHi", - "Q/h8nbqSh3aydVjJPGtJRvferQXT7EAXs66vU6TLLvo5RNlF50dayqH7g9QDXeKNddAi6LkTUH4kEE2a", - "KJ7nzKCbsj1ufGlZjM595DCvCrg6iYtQLNafeBRKt1BM4uZefF9fdhWo0BG9cxfwN9DXTeBv4gLENff9", - "8stYOk4Qz20xJuqLqmFJU9n5CP/YCKnniPzsbeGr82IgE1e3ibNzd+xpoxCORtXi2wkQ3woekEKK9cbw", - "RgS7rKjBkt2/PfOqwXEWOQzR1I3RgR5YyQHZmM6XU0oz7s1fvLX0FrjK7UTtYxmQmgS3Vakeerin4H0c", - "QYaTXHoAeJbdlKWuKij/hfDXO4mX+2F5PO51wuY2inA/tbrkY3Ei0KR5dTXhlbF3pCnfdfH9/SyrZRIU", - "rlovkVZW4F+1U7dUer9FT9qBgO3eVNhSpmbnI/4Dftr7iGbkpwec7o7h5rfOu1f2GWENYPe4Zu391+v5", - "eQP4KSoiSta9I7qiyGfebx8PhtVwAyvS0bChlQOrg1NGpTATeBc8Gp2u16+dSK8MgS1W02VhKssSDKDd", - "LqNmu5Sq0WXQaBfL3EJhuSmH0H8oDYFGS70TiYpnj1chtBINkD9WiikEe+pLhXXw5qgI8MthSOTYQfeN", - "jxMXCkoM+MItXRwb3LTHw4N0JLyrgA74eFyKMbdUF88jHfs03Usu4V7dzQrPowGYiEtdnvcTNcAlGbA0", - "F7w0rAklgbOsfVF+qJk0mBHBK6un3Mo0UdEY+2yf5a4hVi5sQb3mRGql6OFNB8FiUKeAociVz0QmJBLy", - "XGHt2AUf06srkVZWALd7C83s11S4JW18obNVrqX9sFeNA0VbhmkesKepbaT6ze6rBupdKBB+w1OUR7xs", - "wqyH8C0CHN9CcqFPxy90rseQSglB8z3IwIloDnPeKa0Qqm4YPRUe48pvUbSD30MU0MydP2ni+MMeJAoi", - "F6gM5DfNME+z5768EAwZYa0ebXcnEDPDoJIFioLTsnAz4tQ7cTt61y3rS3TuGW8eK70g1z6jbPlUlGPx", - "UDMbZiqdlFrJfyBNnQtRMAySZFIxM1Mphq+LK9wMvIf3m5zLc8FOJ7qQo1k3Ucfa2HEpTJedfgWUDrQJ", - "jn1IKy79vU2f7edGs3OlLxXjZo9aDYOBte8myj0eciPgJ4BNKmHJHcXBE9FL9XQqyhRfAbTcl9pO/P0F", - "ZlnWiUfTyoBpQYMiSQtlas/FrM8oVxOq2ArRu+QzBtsH8dNHlLrpxAlOx6DfBzsDRLoCb2bouPvXEBgP", - "kz1pdX0DfGggJaNuB+IoCObXZ9eypNrdffEHdhTSD58922M/atwhFLNDYS+FUPC56bPTkFBsLC9totBF", - "pWbwApMjyFksy6rAexfXX0AZdq072qDLGkTAoc0wjWXfPz4y7ImnAfaL1gf001PaQJbzc8HEFQSiA1lc", - "8lJMNABVkfjWflkgOlpf9pwSoNJZE2CPRvnL/smPRz/+gCtAmdMINFgH2bpNo3QlfSHKHLE6MBHA9BN1", - "SqXlIcwQV3H/+MjHarTEBirBy7dwkq/rHaGAdbx4+IeEyh9RBoOjPYj4GOqMgPEOsMHeK5VqyCKFz7od", - "iDl2n+fc2DOYY3Ym3fhgUEBQlPboNm1vF30nxDre8MJq1wxAe3b2vvvuu/53333qRq+/iF5/q1F9prdf", - "wMsgAGphGWbxXjkWXgpjfFITjM5fQT6UsX/qbijuov2+J1TFxghMlbfGwMDPzH0NiE33cw/wfAM/yDGf", - "5Zpn77R+w90Zgu828G+9V+Fq+K3IJH/30BImYwEKzA64bJCWXlDWDDKWkU8ZlU+ihBA8NDe/4cQorxuV", - "k/IBYtsvD/MX19M2yzmB0BxVOXRUn97uBql+G2Xk3R0PmJuJ76WFEcAbrFFFKsp7ORRFKdJQDWLO1/H6", - "gH337dffMGPLKrUV1nwIHziqFn3mh+LltJhKC1Au0jDsiYIRo0GEkiVQ3R9jTbBdkSVqABr3GUp2YQYY", - "Lvm9M921kinP6/gVsujxcc/JaF+WBXQjU5UjgOfSU2mZXMiVqTPG/vP5t3/89tvdF44vdh2Zu2Pa2ev8", - "z3/+dbf33Yd//7e2HX9IVbNgnxvu+k24L3zVxoJfvNjw4/eKwjFCqMGLTUd7wq14g6myG3Nu+HCp333T", - "Qb/k2Q/ciks+29hfj8ypNlsbvnrXxtcbtkEdv5NToSt762IHz3sE0Trv5gd+fg3RgbWedzyg7P0DqCw1", - "Wfehdqnxnlh0CaK1aa2YFrZZx9Jjz+QeHxxiyJ0J8K7kF6I0WBHFV2QByBEgOPDh+Pj1UOw6wkZJuWLD", - "OgUJa4PWhUJDqYRSjKucl/GnQmWFlsqaPqPanVKNsbgqrD/V1NaFIdewr9wpavdspTINygPVcqVPYWaQ", - "swilcAB7DTtjRQmVWQzTuePXOE0yYcEWLEpBmVTUWB2Fzw3DqkrlzE+E/MDt8CcewYnL8gDb2h7uSWtf", - "84And46IjgNaUrH1/zj6CWp7VpW+BBFWbsx9ae9bLy27fmgnNZkGv6Q7XFRrnA0rC2mpjeq3v9Lq47dt", - "LDiajECX8Jo2OvM3V/2Jf0tjKvFQOfcbAA/wkycukomhjaP9KAkUZ64DJpVnYv4CyzBxBTCuUQC46dZR", - "0PgnLjBkjHbJZ47IU8LtLz2zE1FiWa5mvg3bJN0G4J19vg0j8mYjmVsKcIbYZmDQVMoh+HpopqWAdPrE", - "W7qGDaxT0Ox/+MkgJmKQUH6KeOcnlIWS3XC557+kCgTuM1wCD5DIVYax48sSYCP+cIS0tB2eHXXhur1x", - "yZYbc7/QdevVTkyhcKbAafrI6W4ASx+vovlsBveAFdNFUL3GAng4PUSiAze3Gvd8WfamTuVB9Qj8LoDR", - "OXbF07SaVjlHsLtKIaNBLdcjt5CwTlTK89xEVSuCLiiVsYJnbpX8tYuuDBuUlYoOyYDqj+LoaixTH6Wt", - "yer32fpKwJ2HLWcwcz/uTbTEO1EPNwLCe9QDH/XAu9UD5/nErXDJTWD0Ws7fvR2FiN4ITeq3iJPHr0cM", - "DypwPdrBO4egayH8Bwozt+VVWg3/btlUG7sU5c1nlAEc0UF9nR7AfeHOnpQTMFuiQpfLEYiWAsM9UP5T", - "Q+J44caKCTf3LIV9kCjBSenSL/Kj5L0BiNtWZe7DRGS7L87jr/Gafnkn6+Z4UMCcYq/BIeTGkvXA2EjU", - "3/SwxmwQJVZSsejDj8vAM6lqyHFwmXd9LGyi4NIAYpXTCXi6fRQyDQE9QtycG8A0woBKDOa0ms2gok1t", - "nvkTqRwvIEZRKStzLFFjwwWDV/FNlxnNeKIG4Z5gUCO5TQVXBhLWgPuUItUKL2XB+STzHBTuENDlbwK4", - "ClXhmxHSACnngdx8PjV9bsjawOKUYahdv7xx4Dg2api09YLxPJ99X5dp0wlk7YABaLUXGBizXQi3GFBp", - "fhGnbDlS3AMVDncLBbd+aHPgb0iRUkVOA3fSSFT0YqQ/iFeObE5vjkTHDm/ZavL4DaPB3brAKCv1wKOC", - "dWUAHxKuVQ2h07jpkwpC/BY8e/0Wt3UoJiNMUgdyRU56qTLj/fI5n5lQ6gGBKx3DkbVXazhL1ACqbg8a", - "9dWhAjsU0nJHTqqKQu6r0ngHFVcakXqoa+/SolhYU9emrOfhy/jCi4iMDCeDXOpuNBBqT7UnJzzTlziR", - "FIKUuyz1YkZaVhU+I8qryDRXGErOZ8y446JSQTIIYIuggru5pOhVtxzKakQtSsX3bCJ4biezRHmHHFw4", - "nEsoClApiI4ZjHSZiv9wxDCAEmx0H0HdqBkrtJFO0NWrC3ceUDiGW19VM547rkm4G/dApIQRWinKF8zw", - "dsfXuc5n4cKldkS23bkk6po1zuYKxjR8mFu9wDip1N37EqHTFQhkdFwaDAv4fvA0PhoMG9ehqVSkpy4s", - "6ecIAXDu3yxakz4GbnAnQZsn2KEP21w54lMc5rmYUeCfhhB9yHmw5QyxkAAwaETpWTQb8DoAwgGwsVQX", - "IvNMM2IATq8vpUplwXPkJx6MlOwMrxfHwYkIwES5gODTrv0WjtNRqpbIMA/kRKCng8NNsodRUqxSlLOS", - "BdQjhHH7HvnqjAAHyH2PabCEqIeBNrkOcFNuPmVU29/vJA653suQ9JHOeggusARD6w/dzlQq//fz+VjG", - "bueqN9Y997DnhETPo4T0gBWLsrM34rkRVONxGzwTNvpaUTkvbrv3JWr0YQhLCMRYp042omnnt6PF4CUE", - "RUcJIzCB8I6+Jng0fVYjEUBq3ZKYXeH0/QvB3p+8CaXpWibg0/HW9XTiDmZvfwQ0sHCeq/GYYCE0BKqx", - "TDh9RSpmnImamWbzU6nktJrGBOioayzKz4mmfYiwT079QTTfGq8na4OCAkZyJ3hQG8a50lnY92xsu6Xc", - "kCC9CQVsMMBXLxWfK8tS+NN6cHgQAfxtozpFBCZ3r4aaBwFuD4blip28PmDffvXdHzDACDODjgFycs7u", - "wF3A3vrsJ8DRSNRUTIc+ZAmzs5zcqkojL5zoBluBqSrPCfmiFFN9IYi08eN+oig7oC5R6b67QFAN45Fs", - "KJTWa6M0lMjiaVPqYSqnuERrFZBSq7Fv9tU7PiZJjxmfHAJhpa6Mf2VKBVq6hLAI6QJskNCW9HaTzqDP", - "9tlUGvAcBsvz693v4kgKx/UvS1nH0PolryAtdLmAH/Xewv62Zy3E42imLYRfMH0h6bQlMHzYYlgFbgjs", - "zUKKDVBRDwj33z+76buP321j+DgmBnOaT7dr6AiO6FokKRImUOQooNUSPLAnRjot/XXE8ByIoS1b5cUm", - "GRIHvl9fnWj/zutEXFOkPxibELkqD6izxnOlFfiH3U5Rtd6NgLvIeFJAhtjk0DUEEiQ4hdL7ESh0onjp", - "WXKG4V7H++8O/oQ4nNIAoBdF2QLWN88XxQQxqRbei1ntj8z3QTLfh8YccbkfmeNvlDkSR7sme4xU8BpK", - "bXWp5ghw/lQgRvdW7wgXusO49RUlm/D2OJoQM36gDy9Wu22UGzo3ox3bGYox1oZo9yO+dD9HS7mtkN+6", - "B+hxpY/++Ra7XU4d7xYXnMHF/I2ZzIMhKJi7YwCLU7wJSaW54KoqlhPVAb7QzhDmlJVWZMzK6tHoTK1G", - "Bv1wp8yF5rSKgugVsqi/dKKB2bCqYOKqcIrMrXEk1CFi6lnE4pWWcbxzhivTnw4O2JOfCiun0liZ1pGQ", - "6YxRhunTeHy+LpEuz0e5vnz2bC9Rz/sMoGNrcGKs9eGzgQnDNWQCd1nKCxuCMRLFGBvMA5ZGwUJ0rwE6", - "MwetmGeJetFnB3paVFZgITrDco2XyENuROZx+QGiFPw4JlFf9dlpNXRrgO5IXA+f+BulMSOsKXsCaN//", - "Htw6T2Gs/mIXQePhvSGo6/UesUKUqBU8dQv2ro7liiFNOAIB4BC95wgK3GHYVvBjYWgpACJzNfPvsgk3", - "dbU9im0KhCQN40OMQkBUbrAivCKVqLC85M9FbzdUEHcfVpZdTmQ6Ac9+Sl9R9Sw3jDyvhwyD7fptcPaZ", - "wGJzjbv9C8nZi+MDaKGu9uXDqlBHcrOQJdOXKmwCRshRojq+lSEiSw/wZWc93IdLIknTDaG7Lecqykdv", - "i8YCgrhToY1d3lf29uI4NhPjkbUyV4fpViPCPn94vjQoHIELaeRQ5tJCOBh4B1JZcAjNoHusQqgMhOCd", - "Gj/3u1qeTWQV3Lh77pJe1+YKhzpq/DWXeVWK6LbmtuqRphqrNFiosmca5VARMhd8I8DhM5FKU5cahdgg", - "Prr9+x8vYBegsm8izT9Gf53JNdlNC1rVoj7Ytnn1KzH5HGWdO9YBD1HwXNeKaAB4PLoqVudb2RULeAvk", - "uQNsZLnpsu9+/nLpFKpPr6JPmo/npo80uTZBxS3U7dnOC/S4aA2tUfZukyLvVmf8dLcG+1rtwp+FoIdt", - "UTO8+aB+o8rhDRfs+vrgnfhSvEtjW1ykxrBr5yKnlo9FtLqHNSbHg+YhMG4c7P0bn+ulK46UGTfqR9m6", - "SdmFsZg7FQEt5vMPRSl4tvmROHFvfxkHwg31QRwHP5RVCLw8u5/zcG8C6ec5zwSUFoLU/PRLOYSxr/dW", - "jqLhFwIdmis0XcgIiFfcf/SlWF9+vCvVFf8SZUA8ion12pNPFWloTxFx3CZ97nwM/0YhovN8yNPz5XR7", - "Qm9sg3K7H9uCluMRXjdu+cEcBLdsIoO488cjsDaIR+c5LFW7DYHIkrd8JJxc2Fx9+nLdEg/Wmjh9NCNu", - "rME0L81v5UDABe7mB+IXeP3LMChgrF/CkfgFgxkeD8aND8YlkeXq8yCurFAQNuHOhVTGcietV8Z/Hvm3", - "XoWPPzf+U1oxNetB9Of7dZtAqg8vSz5rhxanr1g9V0w9mwvElK0v1gsYGlq9dDsfnfK28nq0ZSLt4XJN", - "bVCty15bmUrpuzoKBai2rCe2bddG2zO/O1CdUbXtz7LtWZZMT+88jGW/fW4/P717YvQ33nioilTOemWl", - "KLZynhToI0imSc+B6RmswrWGItYf2B0Ehga5X7XezxG49AM7vbdPRmGmXwYhnYgIgX0pE/HZES1cpAkJ", - "/hkURBDby1XHQ3zhkf9HNym4JCt2jt5YIgJCzZGs1FirCaHcQ/nTm+5lqVcEvh+WuvjV8wE3yTYWsM0j", - "P9fnihu4UhdLhAT8dD2FYQOCwLJXy0nilXo82s0twhVZcbLxBcZZ5pnA7W6ZhgoL5prGzE/01Re6bxuZ", - "UqHrt1A/YhM7KnxCNScYrW6rLUW/ES8ezm7/OGKy6/LjiJnTv3oejdO8ay692fl/T/nI19Dv8ZPPphYy", - "Dlaf/DD2Y//2XZw86uwtV3JEjqp1J28/BJJ7o2dKX7cfPj7/fsONEWa7auE2cGLML9+vUc4tbNbi5hz7", - "LRGWZ9zyfuR6vJ0aWH7WS4th+SEobdlIVyprdaAsElEY8c2IY8cnXO18pH9di15+DlEH90I27ReudSjE", - "w7hrvQYB+jyZiBAXiaCGrZX/H3nfutzIcaz5KhWUTwigAfA2I/tAofDhDDkSj3gzQVnnrDDBLnQXgBIb", - "Ve2qbnKgifHPfYB9xH2SjcrMujTQJEeW7Y2N/SNx0N11zcrK65erVVO36CFBeHiCHFalI4OVLkSZstZN", - "dRxBQuqlYLeGFxKRDtnFOauMKGReQ7HWmpca4SYyapeeZqOpuvYvYvpeqXnhsVeyabO/f5SvyrtCGvhb", - "7OFPbgvpB0QpFh8qbZ0qGFBwpyrgtxyfMZiIH8pTZfniUHb+qRfqXBihchG7e075SdYHEQ4fOlLfXv0D", - "8+vD+J5kQsdP4QQjEBkvZZHg7Vn2KAxyLI+KnDhU/kVD9p4ZnwsrNoD8/kXDuF0K1ijHlwpf4z0ttGDC", - "eZIWrBqNfQGZGQqEydpCHIFQhXuE5Qt4se6UGJ46p3YnOff069P53IjYW/MZpFnHw96rjRBMKCtWs1IM", - "WCmV4AYKR0/+ctFnGoB+MWV5PlVzwevGOH4EQxixC2A4v4ETTFWougHKiHuOWE1Zi6dlAf6mBCVmHm/N", - "qaoSnoQZ19IwP1ba7U74PFq2f5bJv804/q9ZajeH8SLz8mgsr/7FZx5ZkU+y7614Oddm5QhDF+uB39Eh", - "FHgM0Ff9/0cZ6j9SEH15yOFaiqIoDOLgX8lO3xDypqaKmT3kBOxgf39/nxn9aPv/v/F4YsxPMHkc7z8g", - "HAGr2kMDjSl3xjt7ENdCwuTmbfHnRub3TKra6IIK7dSaoDpAQMNlOr4+A4iGL75gbwGbgZ3LmeFGCut+", - "prcqox9kISxUyRlGSPHJyfc2YvZdVUIdX58FIRgWdeyaGbLd3W/17u6YZQtZL5vZKNerPQTBLWb0x95C", - "71X3iz1b3Gf4ye26EhOYE3z6H/See4H1VLXq42vX63qpFbxCuLr4xvX6+oxeuWlsvf1CbqBitNQA23Fc", - "ljghdxMiljifYx24e1HVAAu9VnkEDnFzrZdGN4sly3UhPNa6B06BWk8mX8pa5HVjeAlQfg9SPAY+CNCI", - "3EirFRXtbqxgObeCLRpZcKiaYQUCXPy04lIF3BXo531vWdeVHe/RGo6k3it0bvtwU5LWtRB1LdXizqMw", - "DXY+DAtpq5KvL/GNb/ENNknf0Lnl8yHwccQs+2nnR8AImZx8z+xSN2XBzmC87lpfAZX8LPL6TzuDne/0", - "Iys0O2vB27dWzr3149Ix1ZwrdsbmUhW4ioR6Z/+083671NeFVtJxQLdUBCKy1o3x5JyXja2F+dJSBRF4", - "D098JHN4hX0HL3i8FvrQfxaqeATKN4KXw1quhGcgGmFRqJIx9LPkprBjpDiqYOKI7hjqWxWttyJ6DC/x", - "g0IsjJO+3BcT7frBF1tIB00dRjpvVJ5+36iky7dGQikrNkdEBsv4fC5yxJDE5iD5ET8FyRw+w7aHj7IQ", - "4VNaN1p5qKzlPsP1Y/lS5PdhvcYs+/b0lu3hUH6Bc3xt9ErUS9FYpzwbmVv/Fv0zcyJqpU3NXh3u77vW", - "sayzsLGcChTpAbR1EyhpFUbE7NrWYkV1pEthYKJSzQ3HOmONIfxPj4yFE02x+rZPhaeUi42XOg5GoHdc", - "Djl358HvlPTUuEai57V7F6Dt5RyuttrTHdKWsFAyrXWQ4tyJ1GEJ4tp2nxaC2QH0b8fAOs7KiN3iYwQz", - "hzcithNQLBY4YqumrGVVhirZ/kQhPiYA/eCNzQoxlwqZVlh/FOvXFZYLKHjNw4BofHg57e4CgCs2tbuL", - "88wbW+uVH3kSQDZVPy5lKZiv+DDAvql0EaEa+zNsxYNwbHgmlJjL2oa7yV00bMLnoobTc6osnBkYZK6V", - "lRaqIvhq5GF58GsE1PpFFOzMLYxUC9fIu6Ys2a34UOOvDLbVzd/RpTYrFDkAuraqjK6MdJv7phQPgq2w", - "LhK1/0bUjowmwl0mrmk33CF/5Ia2Aoh95aYJk3SvMSNKQSUhVcEqYaBPqJ3u2vxzI8ya0chhKDhx3Am8", - "/bwtC1DlErgr2vZkm9jE73IiNMCEbc1VwU3Rejvd0riXsBtZljnhcKo+ThVj0x2/1HckDE53xgweuYeQ", - "tV2K5Df3K77Z+tH97ObkfpzuoPdsujNIH1cBjHjjS/hW1hvdbLeKBstWq/hKQPsH4ncv/zQFMJzpzoBN", - "d+7F+lGbYrrzPv3w06A9Aqc9/UMHsKxX5Qt9OsnyhT7BqdLRJfhq8NuOEX76vCF+3sp4g+wLI+3Y9Bc3", - "nqisqZfaPD2Z9hetfyb/+NSiNl54XeG61T2UqWm9GVZGqrxsCjGUasjL0pMREuYgUEhYKuoa/vdpqj7B", - "0UI4v5O14iuZJ9yK9bKuEWV9EGayF8absSG7UuWaaqjNpSgLfwcUjslncY0z1jMi16uVgLpTjv0lvKn/", - "QndY621IHNWJX74znBIgz/XmpfggvbhkS/0oDDR8NkcFHvmaKAb+s4SL2hAsgWt1iQViKGgBABCHaNrq", - "Iq4MWZvCj1rTdjMNd2EBV4tritr3Dv1Cqy+dprEURtYse2LzM9S5Ku5EB9fKsTuGcK9SQ+PtMdJRpSFm", - "T53RcE7YaDT6lBGvJ5YeI9Lx9y+SIcLJzaYKxuKGQt/AlYeHxdLar91SL/UjynSwlbiRI3bhZQyUFKi6", - "VZA84K1wPYb7e4KV/Nw3oOU9y3TfOwrCXMjkfnLPsHlssj2Ql1tts/K0iwxadTJlxnru5T5cyfHnEX1H", - "Wu+Pbn/wFh9yO1zrZvh5E3NDwO/uuL1b6+YOt/WF0QzSodwdLgxfZe3fjui3jWHfwd7dVUbM5QfiLSBJ", - "vdWrGZWrYTdNKfy5yVx/6DzK3P2TYVGWpm7g4IoPTiSVD4L18qXWVkBpSm5ZZeSKGySlPqoU8QfLetju", - "gBrtg0Y5A916BiwIEV+4kdyJtL3Mr/eAZdvrlSET6njAfs/Cp4756EdqWzd1a4ysB3YSXhSW5gwGhs2z", - "EoraJhVEY8EEnpPEPSRy38MhTZVdalMvuSpG7PSDe83mvOQG7O4g5SPWKJxD1AdmAo4bnTIm7VRhY44J", - "Oi5ZaGCNusJ6xvSaogI7brgA+DrEEjlOU+ALMVXeAs5moP17Q1OmTSHM3Wydjbb5BLSdoY6XFnhuHW1g", - "l8TbYTYBoaeGOtgC561NPZqqiahZ5v52fNtfEuD1oN0aMNWshJH5nmpWM2H2qGzXYKpmWpeCqz33/4Hj", - "ysLp+Hvujz33l635qgJfSinVvR/eyDM3LHVqsda09A6vsF4ERfs1ywqd3+E/oFYplKDGG65cw9JzVjWz", - "UuZuZ8AYg0VI/XoAWa0aW1PtNtgXul/WX9qWeA39TJX7xFEIra9/OXBPVvN714bIRSGgpvWDMOFKrMWq", - "KsE85lQsoSIP9iI5nb5C5CVYzJyyx1fuOZbygP2seL0coDdnqpS75kvQlKrl2oKBIowGJjcTVME95+Wo", - "Sx+gZP87Xrc0gSel4OkO6lr40G0rGHGS523C3LoP3SfEWonCvCCCotUAx7WpITw7pK4u1bbISlw9PsID", - "saXZhGtnY+jh9ydH/6ygOHE6P5RzJIMc4zP9QLaXLG4EMXRYgXCVDRByWECpgHLtDgzjipdrt/l0D3hm", - "xJQAypNG5HW5Zn6oo6k6LqAK8dbprjXyDmnB0uPrNbtTtEFYU4WYySN25Wg8IjlD+UKoc5yCgc24MVKY", - "tLA9YRZP1YqvsSZ0wuDCqANLiJDPuS4TQgdUMb5wl9pUzaWxNdhO3drNAYY5LyGcAhwTWB9aKj+Mr73V", - "0QjuxF1g11P1yM3KddOsFFbgh4W6K+Vc5Ou8FHcQpp8xN3K8bFhG57XIkor+U0Wjhlz4Js+FKGwH78bK", - "0VP1Brmm2wYSl6MIjcV4YG/HrfPbdSLaCk4hPgSlItLh5hBakrC3OQ1L8SBKuvT0nGgLb0SQMuErvBwF", - "iqG8ZNkdStPwshs42HfwWJIINt7mQb9SJXN/JWY50NH87EBtcqdhiDDmnQLNgL0gsQxY5m6nrM/QJASj", - "ojrYGnqAywIFFJxcz9052ShpNxH9osSnjftXS84LglR/1FYObkEgpQ0Dwxf9EgS/IRjFhiBq4zSAdNAt", - "oO+FIlMUFfDm5dpKi5EEQFVo7rEDliyno/vYFPkbR9ApipjPdfrd7cU5q/kC9BO62X1v8KzVXpD9hih3", - "DRG23es2sKSex/VjQ2cnduA6AST8Wiy0kW4Oc1nWwtjRk9LmkCx+qRIQr0tYkKbWnkOMOiT83yDNh45w", - "fEBebMh+uDnfA0GIqIiWxE/b9APTuESJC3rx7CLSA8ljrsnLBusz9kgoQ5vzvNTckVhYRQgYsANHxrlb", - "PptrI2h0JMS51m5NI/aAgZDsFQZ04kS6W7kSyTC8vAdfemmP5M6ka7zq9nwBKODuAyYeBALLY/vfCm0r", - "DhXIYgcLocF94To457Wsm0LslVot4C8Gz2JHJdXFdZTi8a79HBdC2yVHungLe/6BLYSG+qAyZ/AstgSF", - "WDkSmhELMpXjQCfE/OIoxWom4KJ1jf8F4oJY+I0IG23+YM4BG3WyNVauZMmNuz6RjmlXSj1zDb6RyvEd", - "MKanTSUtyBVf0Jlw/5vhF/HwAZsB1TLEAlBlgi2dihz1iZEHeSpyxrkwxgc4pSJzBkfOW7u2L6qM/e//", - "+b9YltgQtl5F1cK96pjm1JcGTr8lqu/8nGg4fT389D6YYZpar3gtc/YORcFobef+EWgUoHqiB81rURrs", - "ZUF4J3s6eSuzu6DsZLu7Yzgs4OyMV2MU+/2b1n+LoRBSWPgWbWWeQ6x45e5jNPOE+5iCJxiaQ1zzx2dJ", - "XEFokOYNs2WXfOUEjLdaPaDyBI/byknc7IQCvMOLGGS0Lm7Yk/BSsM18Lj8IvzTX6b0JplK8zNwuo70B", - "jG2WaSMXMjYKUXLQwvfEJOniBEPOU8afSbfd5x9loDlFUf5XGbOeMSvRipJxxwk+ZFSKF0J8FGbacW0k", - "b9EEWe9wCDIBs0u07nW+fEQvH33Oy+0brieKhWBq6Fpg+GP6GRDeG2Frdm14XrtbB82NYFklDQZqlWxK", - "H4m85ks/5jWWTQGFZEgGMk+tYOvwNmtwtkm1GCfcZtO2KMhNN9P1ks2DdINB2WAYIUvRkB0XRbd0sSk/", - "JPEEFBcA80QZCkSPTaEIzrGTm0C4YQWW+PF6gGthgtbe5+zYwPwrp92plng2D/ztiy/YdfQQuNNvZZG6", - "IrvsisT0lvxBsKVcLIWJCkqube2rX0RatdK2hTxqwrGSOQc/frBWo8VMfVkH17ivPQqbTLdq16oDX07p", - "LozKaYdLKHXUvpBhUniVJiNqc3swnIbAXZhTug0bNvVKmGDgcwfYkvLGrPwF9dEwKDCV4B6cePZ/e3vO", - "ek5YGt7q4bl8wJpHPuoO18PGAWLZqyjWAy4tJKvN0zCDuVtgHpk2oHVTJNVJfG1r6lQaFiuwznh+vzBQ", - "g55qq7Gf9YyZRkFAANWIuuHzmpVQZSrIRFhL/W0KVOAeBagGmDVo5sDyMLwArQ697PpqcsueLLSe9aeK", - "SjJldV1mrNKlzNesUYUwLMN7OENrH7bobg4GtW1N7Uuv/3Bz3ukf33J2T3fquiRTkF9DlDAOXy2j6/VZ", - "zzo0jDh0W8amTu/6Z7haX3C0TncaK8ydLH6F0zh6fTs6f9IP+xkO0u/0I5M11Jiy491dd4YiDWIZN6LY", - "YExo0WwIiZQmSkvEUi1LZS06iLM1K8ScN2WNroOWIAd1tZhU1i2eVnCjvdmmdNMo63QSs2ZH+8yKXKvC", - "dtC8+/yUStEVrZOFGqlnI+4iksIyuVqJQkJ+D+tRqR3qE4ZyQgca25hRBfHewf7+fto+oLbLlUDn10KH", - "yEkYWq6VFco2wX7wFqND3LG7ESTyoxjo48O4jyCp24pbWjo5XepffXzCUWgfoz8U2zZZfJLapom4Pve0", - "gS75rztrm1b0J0xzT5jOt08iDP/ON/JymMRmDIf3I5EQmg7v/a87ujfYEhAdHd2ElEjcJ09DZYR1N5pU", - "LdXI4p1J3gjFbt69PTo6+neGq8F6YrQYDVh2uH/4erh/MNw/uD04HO/vj/f3/wf6DiOv8M7B6FN7lCUG", - "ZbJocQ0WCs8+3uG6q6lyg/bkZ8FejBftl+hCWggzDAHzCfuB70ds4v2KU9UoSac8U9ZpDw38dwX/xT/B", - "3uiViIKU4aN963T4yFHg19cr9+NrtpKqqUncOXy1dD8evmJL3Rj87Q9gLPsDK/jaQp4SJyfnwVd/XKJo", - "7v5yLx2wRyHu/VK4A/6WWxS5nVaE8KwxEhP29rnT/OQ1eLAk4o4Ht+S2vuOQ3udo8FNCU77gqmf7BzA7", - "4v5Z68NsxCi/HnymgStphYUs8T0nRULMG1aVpGuWCm3eCicLO23zLQdl4e+fJt72yUxci4IJVQNXp+n4", - "3aIJhUuGxnOuF+xGY6j53z+Uo/2NRT3Xiy5ZDksWFDSWo30gmyCitZSAJXc6mTBQxpSIJIZhTtA/Teff", - "naFos8Aw1zRrlfWycZ2R2s/uxdqiReKqd9CnaqZAzZcaTUaFsMJIXnpjtRKiwMsftAu6jm3OFXz2t4P9", - "/aG7DT8EPWLJFTwHqdTpb9G4MtOFxAN1IZVc8dI724OCwHp/O9pns3VNorz/tI+rgIGd7+Aq9wQ0ZBQg", - "cy/Wvj5rrX3odpTPe39bydxoOul9mrTEWAStKBC05BAMC83mvESDzkqWJZyTVKzHEflyvm/Ekj9IbWhI", - "N53ySi+AhWE11SE7B6llCH6MEIPKepVB+ysrGswrwpqg8MnrITbHFobnwq2S1AUTFNWbFC6FZF5mBLVQ", - "oGiFEkwR5Jogy9g+o25hNx6Xolz5fBqMQPfE2g6WdxSYVIvFTMvFwkemeJLx0fF0Ss4u310xTMlwnbhG", - "Uq2mrss7f86+OXwVmrmDMIQHXn5ztG99K7AHokjLEEfZDEzr37w6ZElzr1duW2te3tH73xwcHf2h5ZO7", - "gHqsfmhbutPuLjlv4fzpDWctEUHCMG0SG9xWlcCOAUl3mI+tOXmVN42atn378hLStcKMIdiIm3uKOwyC", - "bdhqL0Ks9AMN3B8fUdOIas0y1YDjcQiJHcnxqYx2HB7yEWqNTOPaiAepG1uuOxYeXbJ0SpZcLaAqL61h", - "e4Vc7yP/KMPIwcd443NanlRY71qc0zhaI3Je5k0JVzKG3sctSky8uNnnciXr1ObizrXO7yElaml0cN15", - "UW53F2MQL2+vwVC0VjnL3RfWR9H79AhIjfFNIn0Ti4GQ9K1V89eEoxSp2N9a3IMa8GeA0r8gw4FcxC00", - "Qoimvz0P67e5Zt5hjKET2FZUN0DMck14adGLitr4Py+50mniCTT0fNoJDPZzkk5ONDuD2wf88JhyEZIe", - "SFOjG93RlZ9TOz+r8I5ulvMKs4L8QuCsfc7Kl7aVspILNhP1oxCKBeOjN+uCnakzIYUucXQGJSwR7YJu", - "mJiojsO2VI/L+qR2thJmIYCYBlM14/l9U+0ZAVf6wJuu5IMsGl4ml13IVMHk1atWJsPpfC5zKVQNAcyu", - "W8BRwgpUW0FPQBGcWbxPKc0ZjTcw8qkKqWMYqEVWWppQ8IfcizUF01VcGtuPtjFhWS+EMYA04kd/Ip0y", - "NYNK2wnad3eiJlTipngGyCSjsJcUm52OPU+r62IgwlQdDqsl1toBAPje4fVbd/vpWue6HDHwugRYgXAt", - "24orG1cNU9i8A2aqNs2WUDbcMlkDziwrkgluV6HfssocjNgFBf/HdF/woQrLvjt/m8jgkMYkSoixTksJ", - "wwCpunxSYBglhHYlnxwCCZNKgSRJUCwf1ps/RjtmWfpffTTNoNUtLmqrh6l6NWLXaescE87RtMQ9n8W7", - "pNbMCKvLh6T71yN242sXllpXQeDBRsi1HCKbwH9NsnY6Nkr7wzV/h2mqXpXe3Q0eSMf2LnWIpDm+Pgus", - "nw3Zzw2FN9X+YIRMPmoH6NNnR3qRDC8GSKvE5EeiPm2w0Cqr9ULUSzRiuZsag6pmJXjJ3m7PAldxyVVR", - "igJq0pt0iejaCSzAtfK3w/2V9TdQEhSEh6jeOHe7u4le0hK26QM0iNkxQ+wM9jo2DsaJJ9reGAekL6DX", - "Bfa9AQnBt5ncgI/a2HoICcS9remSivBtww1XtQjbmmwBkSusuLdU9JCZyHrdR+fV1jIzaX1PaBNX4pHM", - "jYBEt9qkQszHKJy6uzEp1osCfZtwwanL54JmEUwEqaDk2JkRKNPgkQ1URRRF5pvGCuPY2VyWgv0e3H94", - "q+HQwGUypLBCYt0Qhwy2I79IKTW+acp7cmVZOgnAW8tyqM1QaSerLJgVK668ygpiFVxrF3Ct9U4cJ5us", - "VQ5TnATJyiehi8q7slu56OIDxR0jJ9SNyYVlpbwXbLLUlZyvB1N1rW29MO4qnRxBXBdXGGkZVoq+HLHj", - "0mp2r/SjYtyOqdW2mDeYKvezuyvwNkZfEuSll/SLGEKej8nxFeDib9xdiP3AnGg63iJH45mtYzTxvViP", - "vO7v9sAIMXzka5QExlPld8dRHO34gGGkjmUY+lnEB/6GDbQxAwsgGLxp+k9fNqhah9WmacAWVHzhEx56", - "cRL3wp2YwxGbCHf/4O8VZOTqtkQTmeNRCCSFJ3bMmgpn6M2VW5Oh97vmVEGY/it3M1SC18DOlBPXYBRD", - "pkgyhzjRyMC9ZOfesl3HjUIjiKb2LtaTP59DoMLk9Pz07S3bZe9uri58ToBlVzcnpzfszX8zWbDzs4uz", - "W8ALYVfv3k1Ob9l+NlWMDYH00zyCkzduoWh6bujL9czIIri33Tc3jSIF36d/LXVjyvVewWW57qOTmocT", - "EzKG8GC4ewvGDfnovFhJtccruXe4f/hquH+w5ycw+tlq9SeI+v1GFo7tHn5VOpXom8PXrdEnRO9B2Cjb", - "1VgoNOzpBL+huc1Erlchdxdh47Bn9MWGUR8x4BHn/F4gTgCQFJilIHaK9TJ3Lvf29/cPYMzZgIVfDv0v", - "o9Goj/2jQIxsCykGTvsjN2Kp3f0tPgBHw7evUcGlrqRisCADZgVgN5DEA6Q1WxPt4cDdqGdG5ve2g0oK", - "Udb8DpltIBQnvUdKaa3xxhApezxQCr5J4Melfhz6u5z0HEhziJ9jJp4j8Tc+LdwLOxN3KkphLQk7uInW", - "cU+wnREywdBdXzX3K+gYK87dtREvOdgwPhcovAFKq5yjpmqaiqxPG8LIpDaCrxKUkIE7soarwnWEBmW0", - "P+IUfjy+uTy7/BbHW8N1SdGTKm8MIA0gW0GqdEtRUoaJcazSaUokvmDYO7IHJ91pVa6jHuU0L+DgN6h9", - "gbcck/VQLYMNeeBG6sYGGyah6VDAkCOj8d7eXsXr5V6t9/BDCJXUgF3hyIzMakOW2aPx3t6sye9FDZ+4", - "F49X/Bet2OTI9U9DSfVKH+FPNAL7g90wq3hllzpEcLLvHcV5c+9UnUBgf6KokS4J60mwgBTw4jXOqCTi", - "DSYNa5T8a4O3WGoG2KwWtG0FgHN+tfHG08gTT6hOIBDeu6OKZ6Ol+9twmGBdPDcH4Dhkjy/r/hsKIAbv", - "pldbq8dZpBvS2qOVocNacIoKYnBFkyTXgouYbwR3DwjYbjsqdTBV8QoBzAXuJHNKEwTbg5NO4GDciFxW", - "Bqjwhqt79q4B/1Pv5uZdUMdRHGhFvUOkOYS+T+iKAo8/cXk0n9u1qvkHzHzWj8LMm5Il45dqAacDfN1t", - "DAbppIFspov12DGEphYGLIE+vBp3QRv31vHliWP9Vzfuv5dXt/DijTviSVNrwc0Y8bsO9w/3Ea1laZxQ", - "F1+a7mBMWQUPpjvB/jshWTbM9ZIjylHJ1aLhsSfaqa19GaNpFfIQo5geoilCEJTHG4EpEJSXF6Thyqcm", - "26bOiQ9FCtaI2CDiEfqJfIckgSPBPZ4qCtGzG/F1vl9/meDUusmlI3XENXbnGrsjGQb9ZUAZPg442V7v", - "a5/u+H6T7xD/wl0WHM0NZSkXEBq9Qx/5dUsSU+5c+3fK6XOtRBQKsyWvEUz1FNNx0Sx+g+ZFhBWn2yOQ", - "7zx4m8JBgF/YNURVUpoLorAlTPJxqS2wRkjDNrW3/Qeix9iBjSwi7IyiOHEdnDI3hhADmIunEXAZQfOg", - "C9F7B4dH6ACGf716/dV0Z2PUcLCn6riqSkDWbx1fLIh0fHni7lSMI+4eYNjT9v7+nnJA1uNa5EulS71Y", - "s99vnER0l8JkojsDlCRMOQFGVUN6qPRuOciawAnAD8FxGvbRT+wUwwWTnfCxqVuz7V1e3caJ9jdmKnzL", - "T002zLUQlRHgYmNXN5TLNgZ29+A97iGEI3X0XupaeCMRJcyg4xgxGL2V2+Y6kK3h6l6qxYAQcdzPOJcE", - "Z8Kz8Jvjb1nvBu9yXg6Pm4VbD1GwbwOSWx/DpjcuIlV4sDcRg9WJGZyfX6CAs82BjO/KtwSvgYxnw9ni", - "lmExw+HECWCn6ObsTSan/ZQDrri5L5yOHvuH8yOVu35ZHt01Id449gEXfgyahwvVr0kY4/EC4CWm6iww", - "l5rW0ugGbC14YkOsJz6MMHiwDG9Lbq1j6Taun2VAJxaDBJzE4WPM3Wkc+vV3wkhVoxsc+W2nKJAuS0wi", - "CGisrOcu2yA19VmazXB2Qs8pIR7tP35Pcho72qAG3sNhB34gAy/MuGWlFTwLhEdXyo3HYuqy1HtQKmAH", - "TljEhEWEc4poTq4D9tcGorMDmz0YbV5ePrI8SiK/Dxd1HwPngHlkmzcRRRpv3DNZsHGNnKSP9GkEinbk", - "NnQM92lpyU1mxks4zd4/lLCb9EYNgkFgXYeuVzrQgSGTPVaoXBfCEMw46lSQFBaIHFMqfdzCMGx5gMYa", - "uaG75j1bWYFEmueNQa80V1tCQwIqAxm3ZOReQVoveZyHIZLSD4WsjrWu2GuIBul3Zrd33vILTHspeQVO", - "ZX+5g/nBvXGwT78YmksrnJHozJDEgBpyTEWHSeGzlfwgipmbwJDLvdWHGZdDbBJ8QsOHgzSrPkZPtXJb", - "wws5VwXkQd9BeAEOdH8rYs/t8pGnLXZtGgVbTTexbmrQ4Inu486uWSFqwjPETV/wCjQXQk9gK6nkqlmB", - "xdAudVl044t1rvgKgq8EKwU3Kqbuu8Vs1Mb6rqS6gxHcActzz/ZHr+P68g/0eMGru0qYnAI9j/ZH26sx", - "ZNlGe9mYfS9EhbKMn3647iCm19bsv/7Nx4nXoWwXttbVfTZmE/ceQuBi7G3e1PJBxLVk4gMkgbeaBtsu", - "HjGnSfj9ezUiA/akdudmAffaW0RwWerHpyRok/ATD9WChgFj5hnrUUR0f/yUNjb0jKUYoKGWPQrwvLGV", - "tjWgqGIYorHzzLVScpjkBKYQGgmoFHivPwq5WNIiCxshGsu1n+9rbwoO8jIy1iyV/DKfjJVIqxA+UhlB", - "/GFDekkklqkCJp1ILHTNeEy/GWU+gHZk6Pc6kegEN2Su2d1tJTGho4zSj+KV425Af9XSxozB6D2JknnL", - "+sp6cVt/HzYVzd3IqL28VmumuDH6EW0IGmNejjD70/MsCFQPtGvBZo3N4JlDBr9yV6Ju6lIKY1N7Ciz5", - "8wYVj3j4okXleYPHs+pgNHYIu7FcaFa4eQeU2kb23LzqaZNfNr448RVtcC2RrdOWQsa5BDeXtENMP/t8", - "Kwo24UTHlS4A16KVpYvQbMEqspkXVD/qgESAWUcQLBIkmiiXhMz9xJ6CrU/Vm4vD14SYsDlyEsIhR3kU", - "E2bBToCIlBgfswnC2c4EpZPzdiNWaPv+iIHuYeDhtoAMUAyCx1QwWXwID1dideeYegC8aMexXyG4gQfO", - "8m9nrOeRTvtjdjYHOJSBByHGBZVQQk6bNV4avcYKUNK1YbWA+K5+27lPnbhF9doT4+VCG1kvV0/bolgP", - "8yqjMarfaY1ivW1rVH/LHMV6W+ao/rY9ivW27VH9kFmDW/ul3c4KjzvsMzxJaQCIGvtChuI2Kkgg15gm", - "n+TOT+KhIYI9riqjP8iVO4L3QyW4cTxZuatmBkt0/P3lZd8DKIIYsyUVtwn8YTNNf7StPW9s8HfcFI/c", - "iCHPc1GSllRIW8PLPhowCQyanF2cQDqNafKgS37441d7xxcnY3b8l/86RCfuX/5r+PrgEO8PchAjW6FB", - "ptCxQ3Z8czFml6dXl/Dx5OKU9SY5LynqrTbyQ8QZ7LMwVjBEOaJ5I+s/O01I1T7mEC3yRZOLwpP+XOu6", - "MhCGBQA7EGkY02IvNfv2+oc0YkWH6HHX2tvrH4i/FKIq9ToJZ+5IFXyBM8Q96mAN4WGLOzwrXvuPnpX0", - "dVnyFe+W9HlZDp18XK6S542hp8u6rsZ7e6WTt5ba1uODg1dHrzClaSPbBpn9hS6IxtxvGLhY+OsFMua9", - "z9wDhg58uhucXboj9KOyiboVrQghH8f6fGLEkIMUfI9Tlm2/5VcJbd8CUAlWXDUA/JMeXIp9weo0PPG5", - "7u6eem9/93QAjQom8d+QFhJrXHzGdCJORZjSIJkQou+0JnEiFPhk22OioBTfUiFXeHjgE2RJKGW7edZC", - "RaQtn4OHghmpx+21mfucu7izuOuIQOEvKBz+mJAaao0zJz7yn5Ory2teL73vw6dQbRB3Bj+FAmskWPjf", - "fzdqrDCjmdOxCKMxLNWYfQchXjPu5E36FVk3Jn9VRq+qemMT4tqO2Sn92Q4Phib8R4HpEnZMWOYxu6o8", - "ngSEjiobpKuvN0BHAE/N9yXnTK/clhTtSx70Mw/uvXGfb5Bl115skOfWaG8819saLfpZK26sR1rDIG78", - "yU8/lMAJ9NMimY0RIs0S8MRv4ZR4dO9gwC1m6efroe3o5zBlULf/+KrNusL4Jji3f9wAcbE+Z4T4pv+x", - "Pb4fjQSSu0CGdRo6pl2+eIaRubMOsh+GINNOb9JFuum4SX5/AZcOIIXB/3eXznrMPnbuxZj9tD86GLD9", - "0aH7z9F79ilDDIgW6YC5TN8LNUQ920ncL/WSLig++AOYk0avB2y68+rQ/eNwtM8+UZcB8pKCuEHsx6yN", - "XzMf9wG1eOVOKQARJt/+hNGXeFmA1Po+QwxCpYe6+pq5EWiMfPRDwYoIkICGK0JaU2A/13SJg7DRukoJ", - "eQAA+kiZomo2jj/idQ9/VUJx6fM4Z6IwOr/Ppirl9VVjKm0TiP8M+IgfBQo32Yhd1UsMk8QxUTbEVM2M", - "htjObOML9M2EciWYs+UGbOY8F97jZxoF2EI5V1NFQeBgQw12X5IIErCnzSP5jFmzJeq8OT4+25stxNCu", - "nNQj1PDhYPTaCTNtG+QVrB/rYeQJlsTrv9RtS8Z6XsL6LPnKD+laV1DjDy3OY5bF5twOg/d2CIQ49ECB", - "aK7F36AwV5xZJdTx2YtTAarpmgqgEwaqHx7hSoY3eSXv7gX5++z9cDQapVO58FPobiZjvYPXR18V/UHH", - "GzgP1jva/8Nh1xu84MP9/cPQRqCeHyfsDdL9S9Om49G5hRBaNKplzVWy2NEMPt1BZDXvgx4KbuvhQefs", - "n27Mbd6TTw/d01wvhREj/F2oRSntcvhwtPUIbDKlVIuGl+65r9sTTTVnARRmqrJ4o8DP/gQHjkU4w5xl", - "5GD8RZhsxN45IflRD20NIW6JIcijxgyia5RDZAuERkVBD3GleRQ4ScQMGUsEPsMxqJAAWNDbnfJDvuBO", - "N0WRneDjz+bIoFJB3KcBgkCOmfju3qcbaUAorE9cj1MFNmFMnVmKVbfIEyDqSPQcvgHd8RqkTmDlCEIg", - "f0GLeBt3I9fqQUAgp0ckoqWI2gPK0R1CbrdX41mBZVMu+jyV03eIb1xjdOqYffwINqBPn6ZT9dZ7+9nH", - "j97zDw9OYnvuWdL8p0//DJ22Q1P1O8MmEFAYQzxBI9nd7dIg3JKH+mrL8PhnC9XrFo0sxB4lGFK6PqQT", - "+p3pyHDPOaQWY0RwkuiMoEqkE48jsIJv3WlCH+p260bkQj7gwWjnkHOLRtkPdWqxwJnSxLOPHwNM3KdP", - "2ZidYeYb2rgIQBxe+0ISmO6nT6PR6OPHPTmHD976oBBeslIvZO7fR6wDJz36L9wv2ElNbAFkS/9Bo0ph", - "bYwy8Z/h7zS6B2EgGS10Sl//B5zST58cN/z48T/uxTr8DRjL4V8lh3+M2bnWFULx+aCH3d03jSzroVTs", - "O1FWwvgY94MR2921uWlm39WrcneXDdkN+iCQgPdsvYbwiYUl8LXa8LzGTGyqKOGUvu9uL86BZLMsi3QE", - "v3z8GNpny3pV3pHW++mT/wD+7zu2viwvDgDFTnI10QM3JP87VPDB74+LApIkSogDwbSvGWTwihLhUliv", - "GrBCPgzY8mC4/GrASjlgos4xVpzFEIuq5JKmh3FgRkD6XQHo1KGyHLiCdnfFX2HhTr0PN8Yy+8hkv6X2", - "yTVyRNgTf/Ug2NMdzACf7vQ/fTqGP4kwN9539AB6FiSpwese/RRoOv2KWMWRGzMkUcOwvxXqe5DjazIh", - "wCOyoLjjB7UPt7whT84EP29M+Q1gnp7wmv9wcxYGHh/XS2lH8M5dY8qOF1CQdKyJ8NmBK8EXo5+rBQZV", - "bX0DQdQ+28RHU+NHlXrqo81A7e1OaPEYcyw1VLP44eYcAVYpawSoCNIRxgAMy6346hXDQA4sicN+uDkL", - "0RL4JnS293MlFl/P4IPBaDTKPE1mJE1nbA//tvCPIftRzFz/djNEKYAXrivhgwgwG4PFkPSuGHQoZeBP", - "EwajA0bzESDw8BrQG32gew+dS2N6cXPB78UaS4LAgkVU17c0uOuAJBDXbXf3DFByAZ5VPyqs1T1gRlgw", - "W/fknFIB+oO2SBEWNrR0ffLO4m3yofZcC5P+EJgeUjiMUAUEsXCLAL3hc8fP3Oc3cNwdB2y3g9LKhf5F", - "liWntwJTQBqhcrgwY6PLQB9+Znh7Vka7DcJsKr9zvpSuD/VwIhCyOCrJ4pQq9riUtSilrenhtZEP7u45", - "u0a2B5d7ABGZTG7eMV7XPL+3nrT8UBAXEcJtbHJJH+zvX7zx7353e3vNijB2UG91U0coG5+fGZHevma/", - "CBPw88nHK3jhGDS1ChvOgry2PYbD/Vd/rD4MsCwvEYKnq4kQY9ZdHXavZeT8wi/ocJhrNDy8AsYN5/JW", - "a4UMHP4ZCjfeXl1dBkisW7DoXBmJ0ZOEAHxJyEH9J7lh7AJqM48Q4TLwoO7HrBRqUS8vuLkX5hvE+nZS", - "gKq/efXSp4WANRTmm+nOdFp38i+Y2TvCdohxeru7R/vDr/b/Dc1X6G9C5w9daXjY/nNydekJ8i2V8SGz", - "Nx6LbIxOxQ0EAE/DkPmPM0SwEgjndxf5T18cvR8zLgfkiF+VXja45TOwF9B2uNYbJSPQQqg75nu5ajm7", - "zs8vWMWNJc2QMbJnh4OVpSuesd5M67I/BmzZLzxyrlPpYOwI1J4QqdOmArvGjcoA7b0/Bo2UShkzW/Ec", - "TmUk7/BZ2LeM9TAqvO99DQTkpA36b2gpUMQE5zfLaK8z/4b1Z4SMvu6oVk09bolc6NtmZxsVtL2LN6Qg", - "MqwPOGb/yZVgJxoPb/eGbdFawDZMLkigv0mrhnY4x0n17FprNcQdh7/p6281O3NzCiWiOz/mZXUv7L1U", - "ewuNH0/VjwRWx4NyCYV7grcNiVWCDbcIPiTwUtWILCZN4kaSlnmcLFGMWkDep3hTuBWF29n7h3wxZt+j", - "29iVAM+tN3luupIQXxmbUwvC/Eepm6rwRcTzcAp+vc6cKMfBP/WyY/Yp9fVpzXVDaw2zJUINLp5QZgQq", - "F+iKsG1C/boM0xvIW8ZCnUECZYf7ojKiEu5Oyn43ypjE4oiUZOwLMf5u1OGOY4mJPW4dvQ9VBX/afz9C", - "v0gWeJpMjE+3UFXIiwtIWD4HnFPax8gDbZOPj7QFrHwAKW4hgAczMkMaV7taYCz83fLs+RUN2UwSOAFI", - "RME09vdTSVitf7p141LXYpw6+NzJI7ciBjTKskQoy01HZGpXpjjq3tEfX8UiHu3VwMPq4TS9Peg3GJ8O", - "Xh991WleClal32YuSm3an2nV7gxQvo43AFrT2UTUTfUPmnpCNVQw6J86w/Buy3L/u49X16eXx2d3x9dn", - "d9+f/venzqXoxMPf3aVY3yioPkorSoDv+g7B18MjG8IdMcw+X0NAfUAiJwVqj2LKAKydUFqgKHwwf3XU", - "U01DWzd4ANqcV4I7BjFvSgoIoJZR1Xuw7G2pG0AuI3cQOsgfRKkrd4XsVU6JyNcDlrsXE8cYZHMGKtmz", - "OfdwYSSkpNc5JHZvhLJALHa09waEqbmPp+APWhbscalLETMZWkCUMfsEb3/XznGK8D8JoWu3kGCCr7Wq", - "AOxt1x1tFwagIJ+DETvBjMAUtm8b8R7U7E7k0GcLe9NS/vMLe3vn/T+hrndnyYzfAEp+6Cs8JSGvcKd2", - "Lm+S+hlWNY1CfaLkYBLE+vRIjnz8O2y6UwaGpAT4eqV1vuwcVXf6q+dd7uM7+HhjfElGYcmrFvx2u4BO", - "yArtLJr4k9/v90/NDYTPrroNQMu+fkysMVJ47cc2s7RaKWZp+qIQUlVNPVUcJGeD1fXRkUaWu4GfmK8f", - "gf+udYWkVPLKcXW0xoY0veAoOyEEix9d10teVUJZZChr3QSAQIMf/Mk711EZt/53xDFF3xaAjTGhjMyX", - "7owONpHbCFYhyc32QImxBMOIfSeM+NKNhNcIAFqJvMaBxU4RSIj1qAKEZ39KPJKbD8A2CBuzZbfoh6Dx", - "Y//VpXikiKhas1OPZnmL4I0hGL1j5u3pPbUOkfk51RdLviYHcMbtcqqwGsR2HYg9Wqm9j/AH/PZ5gkNL", - "EBiNRtuM4TQMMex/a0bk9Qkr0vY/4gSxtld7Dz3OaBJXFF+OUKObML0JEOoTML2QFwrww1R6BA377o5G", - "5FzXxsIIawPLudh40F51wO/53EWnpTvX+p7xmmUADXaHY6CSPXFBfX8AtgxvxsxqY7RBx6GvfHkMJBP3", - "I1So6ooEZe2eoAzoYPN8CteJRShjwCsee5ja2qw3QZXfcVmKImm0u2ZKbaR3Q4oPiB7vGJXbeD2fP7Od", - "EVJwY0M9/DW2jVizcWZOG5Iq5BlBVFGSq/QgAArH6pWARaBw04ChDNNKYAg7KKeXbVPA5u5/yhDYG1aU", - "7FSI0atXol6KxnoIZlgVKzY3p4HMNwK8p6XeRPNNMZweJN+8rMFMH6Iair7PM1mttPLTZJNcKG6kbvkh", - "oxp2fH3GflAhzCoYjpIFp23Y2HhgvVaYB5kLj9dng7V7Yx6+4F/PFzGInKnfmTNEDYHW7k7BHdgkxikx", - "+gQ4L92ndOpLiwDiIxTApjv8mCXYlyRliwg9aCCw28bKvqNALuTfCZDynFoM8vOSe4hJTBivYeI1ZUkB", - "Mrg2NcvCO3dGVFyaO584kDGrqdGY75Jzd8G598iob+SDKBBjbRT8ojfu6ACsMgQOePtv8AXRYaTEv9qs", - "g1EZC2CRORnCyLy+EvQEcLOju8CbfE1UshwB/bXRNYntkTUF7dI7QiGACfkcVOiEohUo0AxSv0g/TOA8", - "ATZPkuwRcVs9CFvLBU9cbjdEqbbWFW3Tin9gvHaqXr1JnaFkvruqQh1XoM4Wy4USMl5qQIlgC6D8VhM+", - "oI+vTF/A5McToysqgU03J174LDs5PT+9PX2R4cBe3wiUHHFRnGSDXY1Z1iEsdDVyNErubnDA2C2ePvQ3", - "M7IUMAvt7ibFb72k1kty7PpeCGohh2sVg02hFG/a1VR5UC1kJ6j/4UpSFj0vxMgbymOvSbxUn7ZrE9Z2", - "W4iMBboAOeoNMokL/rM2iQjs3nLyNEQMwgpZUduBx5si1hLlvfHnyGv4cRDR8J9UoWq6g20Oqc0hVYOJ", - "2flUypV0yKPx3t5qTd5fAkgL4IKb0ajsW8MrynWkpOTTYgFBQu4FfBrXFUrAwpLCgQD4U6mVXcoKjXo+", - "mK423DH8iD4yYufyPsGBGTBoCwrU2anyxe06a7l5GbkSBit3Y4GC9AzF0rhxPuzqwd0/4nF7JogHhn4F", - "J8O4QYRU12SMME8P6yIJ6ZQws1RBoKtuJEoQBhrC8oU2oCgIrkUwFw+XuvIe/mEChJSuJsK6OUk9Cata", - "xDl0qrxRsoc3u5JDdS55eUePN22PfuefK06XbJB7Oy1JIEw6Qu+CTGtHZC94Q54athG5UDVWm/CAyCFY", - "MGk/Vs16scjaKWz5ryywFjhVq9Laby2QxnM8BiAi8ARJ/F60C8b03bHCcAJfuf/RSRVY6ygttoYEvV1o", - "betYbqpWz1VYo0Z/ZXW1QFQPtlXYEXcm/WV3l/XqWD2hj4g+QX4FERdcCWlpw7RoAl0QaQ2PF4ssbJfJ", - "0ybWe4NSGGg2pVQemo0bK3Ljp8faJn3Xc3o8kH2lY8Xy+ZrWGQbK64hQlnwcBg5ks02VrEeJH1IxoB9E", - "Bx6yE48wkbYmUIyEAqYBgoK1yoE9UzYs7C+kvgQs5QAe/ULlMMZtlN1ub8//FaXAnq37BReF/Tv5K8YK", - "3lGlreeY1EGL/966c83wa8QL9018TeTQ4lRUGQxqXcNHHsgIcOrdSh7Tr+ydEMXfOZnfwnUjkIyjiJLb", - "GmquuSHD9NJGv2a6dEoG8ZeuOl1bNctuBGAfK6xq9/duV20E3OJ3yQ3+zBShztnmpvk2QsxY0tbXzIi5", - "EXbJACHa3eklYQbpsmjtrN++VFP7e4mwkndu9X4t+TlVDb6Dq9xt03PEB06xUpKzVhX6MdEEUaX+rGJq", - "gX/4u7PNR2KuvIdR7eAYGyXZqNwZ3fQACAkl0pBdtu5X275gKV8/NL9Zfy0Uj3Tt9g76sFLQKtWhMALi", - "CdsCwmeVZYNWIkD0r6+t1hbH0rJuHpQ6VlfbkgOerrBGAvqvr64G2+eLI21u2tPl1p4uqobQCyHZ5rkC", - "aki2bnNAhOl310oLpNddMC0p24dw5W2i6yqcJjaJ+Wc985fsnVPJ7+q6/Obo19ZPw+lg7bSDwz/E4ml/", - "/Pet2mmvXx0dBtH3gmyMIDeKYlNGh8v2FrC20oNeMCvdaaWqGjORu62kOHi3b46d+xkGvoIUpNzQH2xr", - "6MH+uVnMLVU1n6zhtqn1DNlbhBcHUwScR8sKoxFaHE1YoXJ2cJxEmZ9q3SSpBChMo2mCmsIjfl1yxXie", - "QxGSBRUGSIoGbAF2dNRzA7tOMipvAvLGpLaUGEq9kUaa6xXQ/4orMjc9Xb7tuZ7AFLTdVbD0iO6leaYk", - "23VQxreqnIFGk8q6TrGRtWX6UcH2UjQ1gdAXC18vLdaCo5JuXRIu60GSsz+M8RvKYLrUYCgQQVIfJysK", - "FbXSodklp4w/4Fep1Eug+vCaJ6mNqnNvuXKDWQFFIytRnZT7TBk7jG4nOr68vQYiC8CQaUn7zlp2ZLPa", - "CpZxEi3yrxb6VkvAt5/tzw8keNeyUjzp3A8WhnZpahIpvcz60uep8/0JgXOzfV46grhDT+Bvav9g+XRg", - "gtMdQthPW5VqO8PjcqeWEqe6yq0NgdgOd+W2lnjU8lAFtlvqhTdl/AgRAFCzRC6WKcdNigPRd8BfChBV", - "UlJnch5a9gVCOJvpui6FEvk9abfoDLBLbaCS10YBREK1kSsxtEJZ6ZUZj46IErwNKaeRkdJE/iKMO0ab", - "hqUZiRshYgG9+xHlCRr1o0mBDIHzGUGBpojI2LqQhuwUvIwdhhHCTdy8gg2GIqeAgLDzz9dZJPCil+ss", - "vgwIeBcAAaOR2xMfwvjJvMvP4ratKQt25rakDQpIRnm+IAswOk3BLuRefMsVVW/0NB7tZVS2KhihOvEA", - "r9IiGVgwq9YYoBJX8QcrTMfKwc+f26TjUhL15tjwdfixo/nk4ed24tQkJ4C3Npz6Oq7k92Ld0Y/7yD35", - "3E6MfqQ7MaQgEcZl7OxGP6J+0NHfjX4k5eGzu3zL7UwqZjSFGy2MhkLBq5kwjhOlHZeis89SdPQ2kQsF", - "pihb8xLBOqCYAM9rS6W/Qj2dUbJtPL93FNm1af5RR9FPLCwy+fP5EBNmqVtRYCV0ym2ai3ydlyLp7sy/", - "13l6fRsBLM7ufHrvXqz54lu3Tu7wfgxUAHPZGezUfAGnmq7sNo/oKM+6XaqlA2x0i93AifNdN/Uy6ZgO", - "VesIBBqN9JO2EIBQYjPp42QFYjfJXsV1fP/p/af/EwAA//8=", + "FOx5qduCYSPs4hUT0BDcrv8/9v52uY0bywPGbwXF2arYXpKS35KJXFv/pSU7qx3b0Uh2srvTXhHsBklE", + "TaCngZbEuFy1n/4X8NR8fK5uruQpnHOARpNNirL1lkRfEqvZjdeD84Zzzk9nIjfsAUQaFaU0ojcpeSYe", + "QsHDD0YYNkCYImiTAl3ZbilA0+Lu28HeLoXsVnbqnpKbmh16HLD9wVtW6lywofuv2eKyyLl1SwH12YZU", + "XfEVBqazt25QO48esYmYSSV7Ybi97e3HAX+/y55uf/eEZXJmHrrvQy41zVPQakV2HiUgGUQrh8sT02cv", + "peKQqJtJ7iRoKbBWWqKisMiRAJUw524jnRwrMRrbNVtrfTiPPZ0aN/yptYXZ2dpK3fL3J7AX/VTPtk5h", + "hD0utyZCQbG4U+H+ynRqtsJszdZE2J4bar0CZrFCYyZnmNvT2XHLUatUjsRML3XChuePO90ObHVnp9O2", + "qBjy66aNOQ+zeW+SFj16hr+eykyUnZ0ODr6zVI0xrWniuL06li/SxxUb7O2y+gOsVTd2ivoDpzTIVPR4", + "ChHOXW/Y/CqyniMWUGzEOQIV+7ce9tkgh0egbpB3iQgFO5ty2YIG1e3ANfGPKp9H9YbCksaFMWB1l5Mn", + "w9v++rk+XOTaffDdt3/ussfPn34LYweqhfJLLRvxsL9WT26m3zf3tzm0xskvxcRzhJoJDA72AwB91NI3", + "XfaNqNzO9s6EsY+/edhnu1wxnhuIw3da86nk7Icff/zhzavj3Tc/ftg7fvPj7uD9/o/vQig8FAlpNNua", + "m0c0GU9qBXkuL7yKCoWsZG1UNqSZPbOii+VEsehIfFq3vvQi29/bYK0ODn/8z1e775lQp7LUCjTcU17K", + "lUBw9dn7FGO5WXG+MoUVanJcWC7Lzz9U8SDBspj+EwbgN+zjShH1A/Iz/fUyquaMJKTaithsznOIrTBi", + "GMhxTsQcuM5mfOGaDmF/VZUahERencZU10Ei8GSr/aqF/GwPmrwiWWfl8XvSf94b5xyKam569C48cHGj", + "V3LYruC0WDGDoqNV2QqXB+axYSVXmZ4pYcCE9qSpFXuw3d/uPelvP2zYo+NcA0hCXRwoqg20vVhxC5JK", + "i+OTttDSonfCjFtGuFrlJZ8JiyGPLWAAujguWgoqVWkuKtPSDI7+8YWjf7x29F/OLA5FydXJZfRZR3BE", + "DDXdHXLIuYOzdEmtFVQJKoQU8RKsD4nK3EEpYHpGWuG0ukT12CvwsbA9Cnics1dqIpVwI9hhwwnoe57j", + "eI9MCI+cC3iZ9EFeSON0wqFr94eSKxu05R2vLi9+ybOZVEP2IMQMDxffKHFJcAEN/DlkhShnElK4HuLU", + "QM8GTdWIGXeKew83pEf84N9zboWxQeV+2F16c8yN7W1vP9tQ823ouz1eFL1RJfNMlKj70rip2Prg7de0", + "xSE1qUcurqbW7HXh9fP+HWjFR3jBSoclahuKjrxgRgAsoRXn/Tmf5a16An4cHZ4+e83z3DBwf26ua7cL", + "nAu3oKVeCR341QLnwkavRte7kdW9QeVxFStfzcIXix9uhlj3sxg1PlwGqrv42By1KXRY17d1/QYHB2/2", + "0Uo53j18tffq3fv9wZuj1kWDxEoIW2iJcpi4NcfhYy1rjG9YoY60K42TXI+g4O4aQvNfrqWHYHhdGU1f", + "THytfRGDak6Tu7U6xhKG6yfr62HSpo5zftooA99o6eOq/tUEYKW9LuGHQf/yv3XX7Si1w/Dd9l1tQQzs", + "Xs7MaXS5Feky+MirMXjh6772iRZti+YEP40XK/kEojRMl4k69dlSZ2IEWkyfOS7qTzfhsUrDhsgYhgyC", + "I206DfUJExVi0BYJB9tINdzQOnJF4X0kbFWAuvS4v05fcvp03GSinvSdouZsGCcR44XiRbFVTy5RT/uk", + "MS0adyj63TSwPBIW84439krUlbhOYs0YF5lbS8H/SvlExkUqORMjv61+bRF75hWPHrEpN3URp2YrxMAj", + "VBjInSocfy0ltyJRvpneQhPg1BSINawVhZpAKVkoeKqLUE0WvLWRCPqJRJBhD/z1pnnoleX/GhwPDvaP", + "//Lqv92fR68OD14dxk/eD37af/Pf8ZOXh4OfXsUP/vvHD/Gfb/bf/eXDQfxkPZPvtvKxbjsrhV11CnWJ", + "qEBY+WrGC6gfuPOpMyr5qVgl2l66Hxv7D3rJSh/MOV94OZfqpCpWvf8Gfl34xIiyACW09ZMj+HXhE8tP", + "ZT5fWQsZfl34hLSGFZ+0qAPdzlxXq97/b10tqABeA5i/gzo1tfaxpFPzQh63Qj8c+PPhGMuJmCNBp6Ww", + "UaFRdgD/BicG8EQSK+4QKJ9W7M4iFkOh4ort2tVXqCm+CxwwaYk3qlGuOCxu0QZ7u1eoGLVOta15oTII", + "IFqzs/4VkHqlzCjaj6O53yhoXrfr6+07W1CoDTCUBuYEcSJqGenLy4tzW/IU6x5MAI7eNYmjCAXuOm11", + "0/wopnIyzQGR4qsGEpqhYlIQIrvBMHKuJlVraT08GI7+/DtYYJpKOPvLAqHgjsC4/47Lb5rASUJdrAVf", + "qPLe/tloioSu2wPFUGtf6a9tQy5YWVe8dt4S91kCMrhccfEv0PXvyiqvMyti83Uji/IgkhrocV9H55FP", + "fpHKK6Dv6sT9NxMLVF61IoMZPhbeRrkIKYVUYveJpwAsYYe3D5cH8wD2scIQ2tzaaaeJ+FKvaUy1bpqV", + "M6ErG+pr1Sgeyzgevpw8fePMgZnMc0kJ0xfXxlzrpIh1cVXHfyyCTXjTqcUDX1YkYQKMWyTzt2jdeAlZ", + "m0Yo22dHqHBQRTWoRCuchYanGkNDl7nynZWs9eq0N4zhkseZnnHZdlu1Bz+wTKh5Lo1ta3RDfJ1VgrwR", + "BDwVkHFUS2kvoCGww5csaorJ/lpxfdHMeJ7rs6ub2qJ20Da7VtHvKO1yM9xEEyBQOv9q/7qk+0ZCtRUA", + "iKuTME5DIfLhtDP8y7tGgnDtd65EiG46k00kEawzvtjfSLq0UYZ7qbcoTxq0304KDfddG3uofQVNH12X", + "mSqdMm5Y7KLbeGGuQVptIpFaGWC5XhRtWtlxWRVZlipT0ebp8VcZiUpUjz16JM75o0c77NU5Z0pUJc+3", + "/L1G/PXgYB9fRzeA+wJN/n4mTr1v7Si8yx4Y6bSYEvN3H+K36A9w36LtX/si8Runb8zkr4Riejj4gb4D", + "X4j7DPwe0Sf481xX7sf/1lU/1bO4RYgXhM3FrD6azJkuT8a5PjP4PTpDXBPo+FhsIVoGqjgDvD6EwGAr", + "SBiulcs4YTHSjvzP4pzXjpbgPumSLwg9HcF3010XcPCzzLOUl1jzfVkZGWltLgSNfwkvxcgVS92cUTcX", + "o3eHN9sUqEU3zRVe8az04nhq8U6cB+SWjJyATKhTdsrLh5fTdeKWg7rz4fBNI320lFfk8V+mejjZ/rE0", + "7rD7YsJwFoC7Bmr2h6KnSymUDUIiUYG+25ztR3KiWFU0huAmjE5x8K//IGxYXajckHEzHWleZu1+cV7I", + "/hybavN5Ax/HWtVHbvtxfweF/IuYDyr0ikm3RlOobtHpdrAUdWdAl9gexsvrRPCl28qX3MjUNwGkBXLL", + "Pa1fd6OElwUvRbn8NjxefP0zqFxj3QInraF0HIGoQPgAJ1Fvp4KuREAWibLv/xwc7FOZaPc8o4TN4VY2", + "2jp9PMTrenzJ/+Ke4G9SkWty4QWJP7+F9OBMoj7TeGWWU+sqg8t/jEPN5Vik8zRfaC+8YNxHIM2kBWuW", + "5kB4N4ODfeRgGALbOX3M82LKHxMauuKF7Ox0nva3+0+paD1sN453K51yu1VX+ICfCmJpi7XAhRrsN6tn", + "cxtVBzG1n/Ro7y+UiJzyPE/UcKmfIUFX19G/kdo34ka4c44hhAGfez/DlFy7G40XmaMw9qXO5mQeeluD", + "10FFW78QdA4yvIs49r7fYwqQFL6K/+cmO6ZQQx+2B8v3ZHv7OsdBGIzA48S53RKnQtmesaXgs69oeHda", + "qROc3lIOdLzLdYgiOxRYBxXjXnTJlFY0ECzOgjlF3UQ5kkCIgCMnw1+dQi1w98ny6+wBPtsBWPyHjgY+", + "dzvPrmNREcKlZc77CvL9/ZhwAI9vcACDRmpGlCcIvAMHFwfAnYkIQSeXIsMhP73BIR+Kmba1g31ERc9C", + "Sc9wuv0bAS8B0/BxxM9ucMQQTwerNtaVwiV7fJNL9hZSWPx6iPNUiMwsLhcUDgB0XDfA5zd8DjBCjcQn", + "ovfgMJ7cHmmNhXUmK5c50vlzpPO2PgJj3npfcmWcXrbLC55KO8cvv7vBafxQxyF723XFns/ETJdzquOK", + "ChsiJgU5zIAtR4IwKMjwepDujqmvFOlHRS7r7Cqrsc6yKBEXw1Dp7WDE6pKN5bnIeljLgrAz6nD/RP3p", + "T5gMZvCPP7HX7n226199wPMzPjc1KgyYpUdg4GJcSw9xRAAgw/raRODizHnhXv5gKMHg35IOjCbpuMev", + "uUHTNBNWlDOppLEy9cP48d27/woj67EjP6MwB+wVKmISc5IzmXPHntwH+KnTCnXPx+76KmpDnP9xJktc", + "b1GarSGOKb5DgcHWPcoxxUM6Oqb1YrsAzQXRLocREGtKVRyVJwwnOZ+wmVQVhAftQj0DZx2EEOORtlPv", + "eAHECbfHNLl2tapSJ+/Fub1uhQo6um1tigYRVKmlo/oelgt2M4uAFfL5vS7yZbrInZFbXyYqGhwYqKeF", + "a4as6YgDQ9Laag48yCU3TI+9/Rhlk2KW7SBNRWFRQBDnr18JYsRMeSHICYE6eUDIWzLbEuVnyoQ6Fbku", + "RJ+9dqp4wUsjeoDtmPssri4bZtzyv8mP/dDvkPCu4P1EUdokAaZEObicZcJ1AwkqiNTSZ3vwSM74RFBG", + "L7A4dDX0QpmtRAF+T+bhrSfcNYyxcZNJKSbcCoaV/7JeIc9FTtLSF+BE5pprngG+E6oQifpw+AYKUrBC", + "W7oolwitA34A+hg0jBdMKqib6VEqKwzOkyXjqa14HrDBnDxsY6retgqZe9durkJPt81daRCruWu9HiEB", + "7p7L3lt8fwCL7xVxDChe0CU2GFLSEcp7ib8x4G+GjIXFxb03Da/BNGzKewxvj+TuA17L7UhiP1yS/Mj0", + "L+FTjTrxBl2fxUpAA/YWb0ESxQ3J+W8MGy6oEWxwsN/1fu6p4xoNj/WCxrFGERA8naI2MGTSihkzVua5", + "k4o4smGtICQKbrq6bFRZQDmC6JpYbWCX0hoStaHawC7WGhJ1abWBrdcaEvUVagMLeSPhhmusy0RlpS56", + "UjFPG1hMDKxh0gOP9v5iWk05INcgY+9VjobKEfzX9zrGvY5xr2Pc6xh3VsdYUg6a6gVGTK5x7sL8elkp", + "T0VIg4C6R24pqZL3YD9Mq+ClhVRDxLO2U5GolCutoAhuES6b8aYPknGoxGwpUj1RAA7QpWtdHffYTRTc", + "VGY6rSD9LM25MXJM+9WljMbFpwBAYMsqtY4GExU+r1sGF8m+I3BfwzyaZ1BMQGVy4ruHEttDtcDJ6EGe", + "T+9MYHxmopz2AFUXiIMw7nSqUvBs2G1qCVBwrsI1LVmmz5TTG/rsvTi3UAg0UdFwUq1MNROGaSVY3Uml", + "ZKtD9hXt7/VI71dhXLckueMBrJHa8fJ5pOR7R8G9EL+/Gr6/Gr4x2UxciPla5bWQM90FqWUWxFbjLsD7", + "OlfLbO8wpjtZvH/NtRPAb968rat/QsrTD2I24+zpQ2e1O17IIjcC2InRncHSbbGPbmzc2PprzlJcfNU5", + "8ZXjzNaQccuM5aWtir4vhCCCgwE/Gtb2rdUM8i/CLfDCFfKRjwlyfx8Jy4ZxRNAQswJSISGAeSmq6MHR", + "0auHuABRlf5EoZKBJeH7DPLzIXQKaAfgmDkbNqPL4KJnSD6KfqIGDYx1GJOblcHewDWyw/629+O7Vx+H", + "fbY/jiuPhVinRJ2VEq624ea3RnFl//H+/UEwT3FBTTcUQxBl8JKImbQmUVyxIUxhB0/fkI1Lp+qcTbUR", + "mHgEHpci51JBYVJ8j80E5KBgmmSi0lwHVQknFQCB2JCmg037HdoHHLvX5H2iEl4rAgTRR4VLRB0bgu2n", + "lQerXOciUQ/M3Fgx6zKskeROlrFc2YdxuHiftdDEYiwZnW0Y8B5pZiQiyT0jYjPJLRQwUieEqlmFoYn5", + "POCJTYPTzVES/J3rM8wIfby9zd7Kl92YNQ9n/PzYq4THjlFjbWGM/vzn//3jDJZ+GBTBfggr7SOYpwdj", + "DbW54dqt0Ea6of3z//7x+FvXK6Q/hGBL5F99NgBAlGhATn3GweA8WSYNVoBQWolZYamg7oYuL1KEE9Xu", + "8qLXVru9agpDp4y0FAYaVS7x0EEjnc2pRSqwg/5QQRo8OQGRb0Hib8MdGPx9HhqALF3gcsFTiEB3UQn8", + "x98SxlchSvzEuxMT1dj954+fwD7okjV2hIe9Ypfa5ReMw+YlKvyazpkRlvAQGz7XsVTSijCGVMgc/JZA", + "frA6x8HyHyYKtt6gJ5dWQ2QT0Wdvee7OqV9S45PWn21vdwmSIyqcW8beVKebGBMunp89frrCl7qPiEOB", + "SiMjCGA9TCAqPJJAWl1moNQaRADhGSZm6Dt8vv00UBN8QilHqyJdQsArKSj3AcT3AcT3VuK9lfj7shIL", + "Ufbobq3V3Rsih+5NyPvo4sjW9dw8NkCd6XmBN9qbtluAXLPawD2slGGzKreyyMUCu58sTcAwju5SM1fp", + "tNRKV4ZBD/1E7dZ2BsEIhK987hc4flELcdqeOBdpZeF2o9TVZIr6Jn2MkDsZ+8tPdEbKF6xSISMfDhTc", + "eocya5C/jzUiGDeJCm8AHZtaKeGglDEjR3kspzC+z83N8tRxamfDI2eGwYTwvKDqgIH4ZHsbDFcqGaBn", + "AqoG8jyHW3kD/fXZbi5BRs4qA9frhWMFoPi5t75xPzitPJ8jgpaRpyJRQ0/RQxATQ29WSpFnpsuKvDJs", + "SMQy7LKSQxK7nXLF3DaChuoGmSg3Sm68vQw7IqCUoOu9z15zS+depgJq4fq5utFXpTCo6SaqMkgnT87P", + "F8y6941Lc4ghh/BL3PqIYpxmy7TK5332urKV0/vdzx6eN1Gw3Bhsjsr1cqQFBjxAmCUa+46sdrAhjNR8", + "CXsWETGnqA1fCxKNFQhQqDKpFy9e3pIdmqgV0QwNrTqTpuBgepFTyFNbHOgJB22mM1K6IYZA1so9V0yq", + "TBRCZTCOBv36BhO1hn5XKtewGDesYUOfd0XNpsGsvtpYIhdfEUPW/FuXwIZgU8K5QL5D5WswKwBDdD/2", + "8bjenrbbZFu/VaX3RjW4l01OH3yRMw3ODK7Y4yd/bnAh83sLp2+qHJBcvyzwEVSQQOoixQO5nxuE02WW", + "9Q2SnLl0eoEeRz7ncNXeDUk5XVYKKlbeZcHB3fWXyu6fpQCgEfgXz8BJqTJm3RTTUo5EueDL9hhgBouE", + "QCIScewll7pHwKL0ob9WXFmo4QEVw51km/JTwYY9+echM9V4LM993hDlHWEnA0yyCrlAIVGKPQBU6J5U", + "kHd14MQ5V/P1o4pzmqAzDwNBvb1D/UmX2Ezpf11uKfyE88OfjmdcybEwtu+odtgUVClXCIiZcyt6UDmN", + "roOBVuq+HgzdG8fRG8MuG6Y6H4nSDmG2r8DJKcdMaT/X2D0KUwtwODS36NJl7QXIpu2/8oRE7df32wvR", + "ltid00IQYj0LIQcTaWwJSWn7fn9/eCPfvToMX+oS/Ic9M9XWh2hgashSZEbYUSRqGtWR+PuTI/H3lTP3", + "Z4Am/v55l71+M3jXc/94OTh8j2fizdu//sDgwLo+I0E3izIE2SH6brHjn6SRWl3QN69PyPvyx93DLtvT", + "qrJd9jrXiMKMcF0/7h5iQqAPG4GwV2O5yuhq6U9/Yu/rg+snXwiRTteOITrtNJCfp9IUjhn8zE+f/CTS", + "J132H9XLV4fv0UUFTfashjuYhYu26I6N15Wx6TYNrz0g+69N23ojjcWmOjeh22BX65QamhfVO7n1WI2X", + "/D5t7irl/A0N4/1UhJLrjfItCNRWp7hJZyVyW1EdXb5yxX0YOgSJqcyjsTtmMu8vqCNv2q6mY5XDfbXa", + "w0EsnW7wgXfQlQo6U5Z5XC5PxEpWBtGaB/L8yRGEFHzdnX1gnld5Yd9jjx7B6B892mEHzrp2nBcrvTl5", + "nG1NucoIUpUhB3TfwFTdN3utDBrN7Dr+j+nKFpVlD+Dqv7CmC3a9ocJm9Yq5Jt86w7dnuTkBXHkvdNgD", + "WOSUw2aBlvfTXwfUQr3IroWfpKl4XssvrswZFueDcSluq5LnvVB4uSj1rLBUBu2t1iorBZ/B9NCbhMjg", + "OJr6M9wRf3urvGUOtAPeFl84NoIyRW+MF2Bujge+80Qd6VnQIch5xWAdaICwMZkcwxmMQynhI7PjdxR2", + "h+3+eLjn5jBMqu3tp6k5TnWZ9U6fwJ906077wQpeGhCs9ed7Ov3pr4NGA5lOT//O8Xv/zC8yPu33+/jD", + "1uIv/n3ci3gMp7hbfx0sEgMjuqT+f9w9jD/DkIhIG1r4eBcJJWpgd3AAmANRIxRfXTOBNnJaohc/Mzr+", + "w58dBdh5ATh9QWWRyOX+f8MWulpFgtTgnkAVBdkkDLI/DOGy0d0r3dWj20yMxyIFfT4LQRM9CAOgq2XH", + "OeiauxBlonzcbH2rWymJlRi4ZbngxoIHF54WjtTPNMtEmkP4MXJGxDcJXSfK94VhyIwqaXsEw8iTDS4r", + "mA8GNeDNOz4h4C3w8bpj1kM7El988OQ5TmI0ZzVYWh2ZEa2PDB68hZiLS0Zv9NkgtDnlRSGU2SicGLzE", + "ECPkH3Vb4h5qn2H75fteIwYiUV8bBNGIgEnUdQVBsCgGIlGXDIJ4j4CaPm6ETUp9Zhi3eobO9kRhMb0F", + "Sq+3npayHm57XAXSlEnUYlzFCzZbHVeBxklrUAXDmAooHOiDKnA64JNu5q9RlDoI/TqfGgqudSmVrnZ1", + "NHzCkbMDCWroswqoRl0Njww3ho7y6IUu1VpRRiSqxvAdHOybPru0JxzjPcAP3k0UqJZx6H60LhhCRPsE", + "I8AHcoGpxYcgUXjxE5IKEDDcRJujy+ZdjxuJb8N72tsMMGe9Aj+99sR+19Nt+7ZxDKutv31P4vw+SP8+", + "/OI+SP8+SP8Gg/Qdc1q2uZ3Bt9XuAXzYtOmdKFwTtyB6JgWEvqIUvVK4hgT15/XuUEOsFLk45SqF0qac", + "MO3Q+ldtzvLlImp4ec+I2UrvD8dbb8FS/HlWGbuDgH2UoUA9W7IRqVgwTR9rCR/C2MHtb8WsAIRVgOY7", + "EiprmR2WVIZIjCHNdBhu3zNh5ESxEyEKE9XFNZZbkQtjUMbnudO7UgpIsJqllXFK2K9OPXN9YZbDRKbL", + "7o269tpmDo7GDcNuqY3podJQRrcFUPg3SPsoGoSF4GOcu9vyEi4CLrh/8EX1d3UObl9ny+sCatLX4QKr", + "7zrmbPn+whFJuMEAzzSO2LC/h4shnxBCzYcf+lqpc7z2GFRWz7gPLykAwCBuItwtyehyrCW8ggIyYOI9", + "DKY4FDx3M/7rmVA9Y+e5YG3apYdicdbAGOMvmnWryPtRlk7rA5QuOr3euu45GjkVWZQeC5TyGuwkpG4q", + "HFj3CusO5yBORQ3RHUiOPSMzkahhLkdb4dMhK3h64sZxNpXplE25ynJQbDyxkmro1hSIJV6qdj3RNU1u", + "mutXFV1nt68s4ihWq4u0HP4U3euMV6Mz3rIG9nuKTEAargX8vBbty8rDcc37NtEjIk4ZBPRKFQKkDegR", + "LZFvfLE6EK19z5mxwRnm8X5Bkse2uYrYaj9RIXm9Dqd0wmwUSuXLGZjrVjgRQLn3gpeNCPwo/8cpAyBR", + "GpX5SQ9qCo+lS/6amXPKcPKyPOcqczJgRf88h6sdVlbKd/U/ckJrUZSiKLXjNL4QwKSUoPoUHBeln6jL", + "ZEH5TJVQUyB45sCFv1x5qDUFqdVvt1KWvA3kc6NSpe727suXvXCs7qaEWfQ6wc7fuyruXRW/vaJAN0mv", + "b9tMjCBcqBbNr3Tp7+UFIJO45Q9h4iJjc2F/r74Q0FvaVYyVSgwEda1TXeAFjH6CkPooRWQhZOxBS0yY", + "sGn/4dcb+I3Qs4VuITySthxM4K7P5ehJJS09qu8N8CU2lrmAm/vdhQIKm4WwfTCC7XIjjL87/6v/ok7J", + "ATAzH+da3/3C/PA6uwcUcW5ZwWVpQkPlfFUrhDzltEJnNy+6euAOmpe8mJbcbZH7lrbQl66O4gDOdJkZ", + "H8vBlclDf/5P+mok7JkQKgBMmlYNBbq5iWry1NXtKyM0jAsqyiP5WoD6vLd1723dO2XrRryhjanHwqKO", + "gF0tL+rQWkPXvFY3Wl8KtIXAlZZQWl1SNO3XRb81w3ZXhsAV8gsC4GjUjmGGQrz0LMhEEMe5dHwzB4Q5", + "9MvW4P2ZsIiCSxFpfgVcq695KkZan7h28TF70t8Obb9Sk1yaaW+s08qIjBrAVVv4nAKT/ZdG5OOeqQpR", + "nkr8EpZ4ADsGdV4gAAX+RCzrqa5yZ66Ct+LbZ+Rfz/reQY2gozNuDcPkMqYVxvG4LncgXnrwEwTy6dlM", + "QERAj5X8jB3svoWRvz14Cvgpbwa78OezwdbA/ZNi6AHV3f01HA7dIUvUp0QxlnTwur+zwxJEvpNbZ7gH", + "PSvVPOl08TWgRnztw0n+w6Hs9/tJJ1GfoUXX8M9O+od9mUpld664N/rR94G/CxUPY0mu1icKtuO6ZWvd", + "3W2L13gkqyUs0mh9yu8dyvdC9o4J2ZqQF0Tig2aOiL+bRszRrZlYl+AGqRaO11qoJ+aT3ymmLp/HcfEi", + "g9Ja/ZbEXbuLH3wwgL76Vae6iRw8E5Y70QHfZZnEiOaD+J3P3Y6qchCtyFdasI0LUVJ4F7QaoPrXbfhB", + "+KYNw98tBULbfuoIFCudnc4veqoyLTZBFV6mn6O6QFxY4Ks++SuJ94PihNB79dkjF58YLNDD/Is1yDCA", + "T0fQwH/7+LnbxBvGJzEg8N8+fv7YTBK1nqJb6Bkgviems/O3DpDvx8bxMRVs18VZopWSf68E8+8zf+0M", + "qqTrBwMadrkZScVqeuyyUp/1EL+/SzcKuiqclgsORylMvzV5y831yI/uK8/cRuch6nD5QGxOzb8jwsJ0", + "n8pOw7ZHxHSoc7GCmLY+0b8+b9WbfzGJlfqM4buBNFgmS5ECo7YWkf7g0o/IjLoJISYcaiU6owiIrJ2w", + "aI8P9dlrGthNUFfo7pWy5fyewGoCq7cdBTRXDZprkBytYeejk3m85DOBlLUCwb9+ZYt2/YDb6YF/2vn8", + "cXPy3foEl7WfkX5zgf7gZQf2qUA6bNLyvJ2S4RK4ECkm9Hlyhvhy11m/xX/oelgk4WUKftYyuHpIJbTS", + "ZgQ8u37SisZxbcrsLdM17lJMBC2E7Ux/jvu8ksbX8ksovHpNdPaDsBcT2dXt1yJ3vIzyeE+yV6VEfj29", + "XgVP7i55SzkWMpsJ9kCX7JtH3+Dw8hxHYx46+pXu1YLbaafbQcOl48fadMt0ow2pLRuoJ2RaDJuP3U5R", + "tWGICAtRmKUocp6uYPpYscUpJV1UScBjq6E0WFOHWT6CRyuO4Je5tVYblxfabZRnjrHaoR4pTtTp8eFm", + "rXOTHrCLWUZ0ao2w1+rxWltmgVyEvja9JN8XDc0t6MPfGTM5ugJmEmtmYF9u4OfheBWhx8Ac0CqViore", + "GaiudxgCrrNZw1BtNxY+mKs0EJq+n692sXy5DXErLperDIRa2elrXY5klgnFelFw/cJe/y7tdE/x6x0+", + "8MrWJ/e/d3y23pzZg+cITAm6Y7rCQ4ovtntHWywR9yLDTrM7w5b9acQgVpRiD29Mw4Ql+b3qlkgejK/w", + "R3Yv476/kBZ/EMC0X84deV+nzQKjvxTPvSfs36HRVCEzAwpto+3LmUQfiC8v+qm6KyJJEBvRsWglznAs", + "Ad9oAuiGYfudtV9wY850mQG0LlGFwghQy08whTMVGRR0WAGeG7j81V+v1x00btVbCCqFispaRQt/keXx", + "+NoPfj2yOyTWvLUBwNA1M9jylMBm0sy4TafEEr6//lHuajXOZdpgUCEtBbJMze/NICII0/qUXk5J2+KF", + "7J2I+cV3KE4LHBzsM/cy02eqDv2v3X8gN9mRSEtBUOFKIIQWFM3MVtzHwYRv6CoO+voNGjj3AvUKLJlA", + "v6jvLRwXoo2bFq00qBo/YeE4Ye0BwUvMjofDhRoXwSBA3iA4IzBLDnF+VghZmuR1ilnsYo2g9TO+VVmL", + "o/xZ2inyqwsHeldF78Nb8IDQOIiGsCacDxPyiUuwaLr8xsQ/3psHVy/1iUrbONlGon/r04mY72drHTYH", + "opxxhZFtGTlvmsyKRtFnR9XIOLpQtk4NxuBzYFDuPAHix0hQTbA2vQAN+4hZXeT58Ud1tfPnBmjOD+L3", + "7m5R62juiqRn98IP/+Kodl3YxRLFe9MEPOZXJeLbrvA+FBn3RyTYQxs5mfBL192BH+z1yGrsyHeyRlq/", + "E2f1HDaX1FdH9KSJv0UY29aiJn54VZHdKUGt4rUDwNfIa3bvMrtS5oQEjTZ/UZ+dyxjDC8HOFwdjhcNc", + "f8nKKheUY4qDieqLUJnS4DVzRuiqUKw6ivp1qWfkHFvgWssl3KFpqtkb+vOmRTRKq1EAQ7hWiHKAm/g6", + "zME3cKlIh+Nw97p4y9g2Xl99+UrH+971vG7Ma6//40YgSuNi7eMg2v6VEXC3w4JgXGpCQRY1/dRWRLxq", + "d4g9MV2yQ52L6ImnC/Q/B+r8XYb4RbQfIuEjdhblWGxwz8XzvGGiXea+6yBiijfholuXPXJ/J3Z/J1bG", + "lLz6RFyvC2+QZd5/1xRSF56qQZbVI32vr/HSKz5ILTpzPWx/C+6Gn2U36oXbQLevx8mhJPodv/+6P/VX", + "e+oHWbZwxNZJwrXqfalzsaFij7HuzaQfNhOzkSjNVBaxfr9Kf3e6y6aa+2GjHx/JaDVpkyv1Xb1xFLJ7", + "dccKxQHwgWe4hMsRyZtke+Tizmq5bpo3fPB0iWTyu0858WbtwvmjdL0LUkqkmopSWpFhQyFds877rO/k", + "VgfMHsIJviotdIH8v0bZvOfyV3A9C8pdSVu8RF43otBdyPfXEOkgy9xgr1Wncx0MjJETNXNtteYmrBAl", + "HD67Y9odjPZu6nW3IEp+zzocHKll7a0l2btFb9s81ZvneWu694roilZJ85tN4L6n1OvJG18m2i9PTVwl", + "mDY9AZfJFm/P420/Chfnhv82k8JrHf2PmhyO23s9OeGXp6UfxH3i9x+LLNsSwDemyauKJflNJn5/4Zkj", + "DE7Uakc6m3tM16V869bc8Puk8D9CUvi9rngduegb8jWn7GUjp+rxCUwJqL03qmSeiXJ1ed8PRhjGFXvz", + "5i0UXwllwgmXui6fSoXKJRa8lwqvpmpg9/iU9RHgzqPYZnT2CIGmMiKra79Q0jsUSSdK6zsaP7bi3B5T", + "r+G8HmNLCIzXbwU4q4xwEnisSyhS+9ItgY9XCCXWKRUuLAVUTfcl2tWEEdOtP4K9gB34xjCkF8Blm0AJ", + "O1bizTjP3XwOBz+wQhYil6q1sjrM9SVuDTRwTb6duJ9bKv7aHIKp8lYOjRXyHbHeEptbxHQhVgcBdp7k", + "dRkuBYAObo7pkXJxe1wvrqLq18ZxjNTpN4h68XDj6qpA8e/FrNAlL2U+/6BCPe4r56hAeY5P1SgLc2QA", + "i/wt4qvw1nE4sqaNvYbzfhFrZXuvBwRqyIui1Dyd4jUcRtLUysdOotKcl3I8Z//8//8/zIhcpPbYWMdA", + "J/isFGOpBLFA9wARsaGI9qNHfxFz9lq4aQmz8+gR1g4HmJWeb+XRox12JGbcsawue/n2yfMus6WAij+8", + "mHaZr/EKtX+m81EpsxjD4oCYmmtnd8qlCvwRMnkBlVIop2gSkbjG/eJDmXZ3emD9DQ2DlcAVDNU634Ul", + "IDrGOuyo9SEMSg9g1x0zxkXIpOGzkZxU8D42cSi40cp1BENExAw0OY0VhYH0YcWyisDGaCNxEY9wpWRY", + "wqEf6HCH/SRSq0tm5Ey6Ydq5nxpmI9SwlvDhaPbk+XCHva7yvEeZVvAyLJVbewAmlWoCb/uVH+6wo1qo", + "IvAlLTK851Z0uMP2LVDnqcAlVvxUTjD1CZrHEyB/xUcPDvhE7KtMnCNEG6z1ELZ86BYHYULMVBZEqbbk", + "p6I0blF6bIh0MNxhu3o2kkowExYJezs8fI1WRsnVCa3j0Sv26tSxzffzIlpMK4pjIAThGvT0BBvDRmLC", + "FXtghGBHR6+OrCiO8E2Stw/rJopST0phjGuD/glDkQqUElE0mgmvLLXjGBSkdywNZiyVNFORNRra9a83", + "Wkpzbkwg2uEO6jKs+ZjoHJvbbfz03kPxwF8oJbHl0pMyUoXgM0TwJ/oGqkqnlTphD9AWxM/GOs/1WVUM", + "a5yZjOHDXlXUUDiNj6bSOrpSmTyVWRWBDTVGDlP7D0njs1rnxzOdOYp8r3UeoFTdM2Jh0RK6V97qjOjP", + "HbrhDnt1yvOK8HjcWUwNvu+ex2uRaSWAWr2i5fcOXw/PQbwsfegNVsqCtXh2xtJxlpETqiLrAWtnQp2K", + "XBcYMzzSdoomozOGHVGnWplqJhDbZwgS0U0CJKNOobZxPWF4/LAVWyce7XVd7S0sya0ogG37ggWixLnd", + "EqeA3guk3Wxz0ZHhFJDekdsgYCuOl7uP+uyIlvoUN6+auY1LlNNMoHUICMeKz3YqZMlSXeJswSqgcSJE", + "46JfpQX2y1MfEksZgAxuXFFdHEiETvDsYiXsnbavFyDs1n8AB/+Az3PNs/dav+HlBGf95MmGH39QhNeJ", + "mp779PsNPz3kVryRM2nFLSu/EdDcBsN+ybMfuBVnWKb+65Vj18azDfumjt/LmdCVvYabAU9/MGJWW8KR", + "2PCvYJmXSD5sqmyPeHpSFatV7DrvHd/05fTQ5ep0ktqlic/67BVo4GTI02dUTsKAwwCtLa+PsQdIKV0m", + "nQIlTJeZKS+zAKHIA9TPIM/pN/zQbY5TO0SGK/CrsQjMM2BKqx4alzSCohrlTt9wc0nzylhR9nJxKnLm", + "weYxK5+PraPFU1HOE1VnS8N8wmyMU25rx21ogWQgSr1Uz2bSJorQh1UGoItlZmrBKs4LEOBbQU2ijlJd", + "KWv6bMCGhVPqeT5MlC7ZEG3BIePWuulRSQFI2O7hp5nkE6WNdXJ+VFmWaWHAtKUVYNzNy1gNE4jALAPo", + "Phv4FcJlVbrWNGjjqdtEDZ9tbw/hNV1ZdlZKMEACubixj3lqwZH0Ep/t72GhETmbVeT9PhQVgQyz/T1E", + "MPY1YKZQyt3vXuZsOBxaWHS/BMNn298PYZ1zwf1dKpSQASh67LxS6ZSricj63hzRJZ8I9kYjc/OKtPs7", + "R8RDKBa5w4buj52tra2C2+mW1XR2AF9xMOO/asWOnu6woXm6s7U1qtITYXtUXWfhfdcvrUWwRKBbgDP6", + "lNNQPm8l6p//+H//+Y//++c//o99ws+PZfa55+nXH6G+48iMsQeLa/PQtfAPaqHGZeyZVBeB1vzLhqpC", + "gQQyq8CVcODXVY0Cx4993JI+tTCG1ThKtIX+JIYj3Gf7ynFF6zPkoAVnD3FbOYNqWBtFiZKGgScucst1", + "2dlU5qI++iw++aUodGmjE58o91NVCgP6GHNHcTKlztHbJq0R+ZidccNIRVi42+8n6kfH/aKhNThmxDAW", + "acwpdrVytl5wvuR12nIoMLX+E18gKlJL1n/g1QrUaF9dCpcIt/QtHatr9ZsR8TRlaZPLRkKcFv14xhWf", + "AC5vixhfHWr1RrrjrZXwphgrHNPT4wVRSO0wqQiOPuxy70TMnQTKRMm4XbjM9ByrH/xAQbjzkZMLwqkE", + "2HiiUBfA6hLCz97Xm+oyK2fCWD4rECGGbj1ORQmFedkBN8DZnRCjajpDJc7tcVqVRpfDmsM3vICuK4Ww", + "sRPRBhfvluglLeMFof9eZvhpu47I9QQmLbbSv24x0ulGN9fw+myOH2xxWLQejWRrRS6CH//ay/ELU4Lf", + "gc9kKM7x2B1LPXSqm0KIRlZXxYLFcUI9FBYhmRzIZ8U469a+bqRv+bmcVbNA5YGApIKzAaSxaq0AxLvb", + "sJ/H3JnaO4+3t7udGbYNf7k/paI/g80rlRUTuKE/7010zz3tmRNZ9DQ5YHugKIqyszPmuREtE9jVykpF", + "vhyk93oSVNoNizYCpNKa6eDHjfnM+PkboSaOWT1+8meYQvjbHQjr9rez0/nfvw16/8N7v273vv9Y/7N/", + "3Pv46F+WLPzNJ/vxGkU6nmuIE10jz9/4cuEBvNSz1S8Tb18ura6lJPTSrC4hXWw6XW0jHogSYM7hNSaV", + "EaU1XV9zCLl4AEM3jKelNgZvG4rcCwAnFhhnzgpwRonVM5niVxxOPqjrzv6rDVnfUiRDnVmBZpeTotgK", + "d8oVmheZdGQ5qqyzI570iik3/n324MnB7sM+eyUBj4E3ewJVyQknXTrLUrBMk/3wwbXAjTCPHqFtCxOF", + "CEq09TwQOI3wAa4PsRy/Sh7gHMAgENqVS2W5VP7Kx+mBcE86gWsRmrpfux4b4IrRHQ7sM1ZWabsehzur", + "l267fgYs++tR5uteLqXJP76WAaw+9rtuJXvevnckHBYrUpHdgX6CfqmbHBvSZiZSaQialBwPL9ipNHIk", + "c0cNumRgKqSy4OAXAmxniCEzVkIhAfDFfrGefll/5xVWjn1f8wBcjbXg8fXLTvOEq62RSHllBONONGYA", + "hthlVhc615M5GwuViu7iCjpTTYADypkeDZ/o+lU4ch/mwphoJK/ROouMkGhhwElf5FwuLMmFjnKoMal1", + "mUnFrS7BtqtUxOXFWJdu1mmThl6AToy0wcfiGurcrTxOschxxsEqpySJowsTR7wlOhU8t1NW5JUBx2BP", + "6Ux0WemsgC6EZsqUR3iT0e7rkmXcTEealxk7leLMtEO+0oiu3+3wnobWyg1ovmH43rppi9O/4ZK3rBfD", + "jeJ1LPH5u6QKATzqwjJurgjVhshqU/uVqmZwIWzQgZIybxlFVpEJzm1BqO/SMEe1zFnnZaKsLsgkH0Zf", + "DXeY9MjMrCj1qcygLNCZGHkDtH6aqNDx/o8hfo/0sd29XX8wUNBBWR4f7ycYGTe190r6Ifc82Ktrhp9q", + "mRlmcn0WeoYrCNcSgMOTXf9vCKPfRU/A8iRc+xiEkbEcLo51yaQ1ScTRCIn/bCpKQaYOfSzOC20C5I/T", + "9AYH+322W69cosCpPOYyR7cAXE+Cvoj+NH+FgE46lnTgNirpOH3WCp4xPUY/m/dbnE0xqZ5q97KB/1SP", + "WdKptzfpsJnginwWkWGMxc3BRe94t9IWZt4rSj1Cm7RZHXiFzyKa5EV+Cyc/ec8I95KFhUaDJxrUiVQZ", + "VBWhfUsUxBmxpBP2rBtZ+t00S5POwz7bQ3rBgiR5nihoqM8gdtQnZpGFGmgq6bi3kg4G4eAE24xVuF/u", + "XMrUXzVVcV5wBWVo+uyoKmjnT3leCbPDhkhiQ2Z1omAzMAiqpld4wVMZOsy9X9dNvd7A6PRFB7jP3jji", + "PtPliaN4XdieM3tURoZPb5zLydSywvEAvBZdvSy0RZdbmCMBmcVJxynfSQeKtM0L8qkxM9WlpfvYnpGZ", + "s23SqSDPDd2F4dI0lvJN4wHU8iNrzIhS8hwkhNVOAzsVysI58CjVpRiXwkwZnxV5iNghygl3V2NdplA8", + "1/FIPFxwHxb8jKXINV9DQ9TL2sW6ThdEdEzXK/xBLMTS4l7Qf5HPIxKybafxiwT/1qf6j2OZfd4KzGHr", + "U+DOn1d7S/bVqT7BlICardQWp52W4fYmXAfUPfYTFUDrhnXPj4Zw9RqXoOuHYHvaIrjhczIuUUMM7dlh", + "jgkMyYXYEKnfGLYyIugFcngErgy7SrKUnJHcCbaxG16GWUst0gtXYt9PYjf28i7IsZa0rsYufJ1fOKyo", + "BZsqCMOUF5wsbJIM/aTa3n6ahnnAn2K4IvOsTlFcm32mqpkjQQhu7XQ7PqcDPiq5Oul0OxCCCP5+W/IU", + "w9nOguuGQ9FUZ3CmpRyJzscVZYKuMi8MZ9GSGXa1F7WX6fyyEW/whlPn6nMokM7PpkJFlfJJb6kPKd2R", + "9zcJaNtf1nVbYtraY9AiWtRlK6bJl/JpAh9YNdRFZuKUFUrNAF/JwkEJYjoTac5JO/fdRccoCu7aYJHw", + "6hqvvTe+wQ0tgQNGCmV3ecFT6PyKRQzyLyitjtdtbkDcSmer1CTVuLzaUNysT3s4rBQTGNOrS+PMR1o0", + "CovywTiYu4DhOD5p7ENI3WJWoKlUN9X1/3aPU8QLp+QB2HV054gQTkwu8DpSjcKLHz1iD2j3weGismNb", + "VnbaL0UuTrmyx85oxIAo0mvck4fgQC+Fsx+7cI8FDqsuU1k66bJZWXbZjBfY6Zs3b3vc9H6psolo6xd/", + "QAXANwydp6LraMpOx1WuhDHdELjh/5qSnQP+MTvvsqnIi+jtshSppT8gzlmr479X3BF4i5Sj6OvrcrJj", + "EPetBMrU8eNtfC+KOq+D22419awm3MUstFCKzrMszH58eHfywJCcQyLYVafFWq4ynmsl4kXyLqNNY0nR", + "4lrJtl5hKpXx3Ci+vvOMIdzjOYt4yuH+zJdidzKFslxxL4D9/OlPlAzyCkMhDLKHxZwgH17n9itRnxLF", + "nAkM0VOdHZZ0zuSJLEQmedLp4o+LGbLuvU8JrgB+M9LZfMftcmVFmXQ+04cQKeDeeLydqM8URgehhj4h", + "a7MhwXWEm5Afkc+TigaEb41l6m8H81xOQOxERFq3QAG27su/JR0rbS6O3SyOQ2JV0vm4OI0nC9P4D0hW", + "8v7GB4eHrx9eNJWi1FmVWnOJxc15YXXBJqCh1YvbugZTOZk6ZQUyfNzk8Svm92bVAtCw2mYf50IvD+5f", + "i1KmYgeNgSfb29vsX6U6NlanJzvoV/kc2nGMhPpzZkHS6cKCyJT+GR2SlsV//Ly5+OBTDYUVLlp3KEpU", + "LzrOqSjFWJ7jC1hddIenM7Fzic3hqZWnYmGq4txpNSgMW1cN/WQ7BPfTcmCeL1Dauz1fS2LGLcz1tdNC", + "qvzEJ+x1mREqq2MJfLIkN4w+BqfucBc5Ru/9vBA78dnYOu+pzC3fsJ8oCFyHBLVZZazjf/R5kqjhjg/P", + "rVcY2YZbtxa6TDqDffipprm/Jc7Ig10Ok37+eaHJTKewZW3bEK1n0rGV1aXkOSxkaO/x9udVkbs/5HrE", + "87+SX+w6lBHsAe5Ax1UeFxwAkdW27s3Gcas7O52RVByG2WJf3SCMTXMmvpeV+fVRuEC3M4WSvTCmPeG0", + "WbTdltX517vs+++ePY9LPWT1B06nF33mh+LjIcUMbkfgigB7QquVR4MIag7Ug7BwrYPtiixRmJ5KxCXM", + "EJWeFyzlSiuZ8rz2GjlxHB73tMrn3judQ9JnVY6h+I2eScukRWdPW32df3/83Z+/+277idP7GjFl//63", + "7d73H//1X9p2/Kb1RFQyykvGKQMFxMHKv9Wkrw0+XBlifY15W/DdbyBvy0fDcTYBbsj8RcRGOjTG2Ys1", + "xj++4EO/fFVzf6eNsX19tssVxeyLhRhzxyBGRthgvONLb3VWJ5E7O+NYjo/xinK4wwYjXVomx4yrObOO", + "SElVX4BqZg/o4pgywE9k0Wjn6EQWdUpOpOz7sdL9oIXP9akowRMFydl8gjWo4ZByK3zNLUiFq3NbTNc1", + "ouLgP5/4ZKdixjTYV80hhGtvPwwJhYUrypAyc5VOS610ZdgvetQs1uWzjzBvDC55f9EjCv9zDRWiNBL8", + "iG70kO9ct+1epV1SWaOnRJ3p8kSU4FcPpN5nB9ptZzRW7C2CYfKp+/1E7XLLcz2hSASUJyGGbM7SqUhP", + "MBAXKvb7PVUwcKVtokpRIJAqzs91ykvbZ/tjlqOAm8qCYcS98UFHcNdeN951zVBIGs90AakI+ZxxxcQ5", + "T20Xo9R6qFbWm4BVYULtNKc9K59XpuqsgxesUqXIYZSOwKFchdurRUKD2/801wayvwa0ctxJPzakEE+R", + "DeM8QEDhCVRGEROGYgZ8GgIONyT1ubGFe0mAATaNDcN5wRlNRZ77XUlUqokrnIpm2Ghj/yCDgzs1t88O", + "xUxbQdG1WFAuUcbyiZs1JMvXWyKiHEH070LsW58dRI0bqwvDlDhLFM8CiIfKWFZyqQyrL21h7+EnaKwM", + "m2/OeFFQlESisApVr+B2Gh3SfmvGPrK+C5MvYAYnYk5qjQY/XT6nuLaAUAwIZRhA4c9Jn/1FzOlemjLg", + "dChaQJ5ydxJKqVJZ8BwVH2KtgUUB++v7om0m+FtjXSyN0V4dnZG3syoVk9ZEhbsAgPVQVIaOjZsY0Z5i", + "lTpR+kyxei90ZVM9Ey+YqYrCkSl8IJW0EngdBLfAWUPRkGvjI1bcBEvRgJuGS6upx5aga6v9zElZK1Q6", + "7yGWa2uSwJPn3y4mCXxV7P+1ZRMSYV3KS/rkCsszQO//qUft5Q5qJu65Mk9TUVi4C4xsiMVdadEMiMV6", + "ZHJ0rpvoIGDtt/UBIx2fCdvWQ47M6cPhGw8A6AVZmzS6qKdDd2B7gzHFei6Wep1MCKhaY8mTTOQcqtkZ", + "kWqVmWbzazJuGjbEtSYoktp+zcXK4tAAkoeuR2DQfj8yuN2h9AWUx1wpbdmIcgCR9UnjWFVG+QWwbjeR", + "Z0lkP/Bs7TozLf0RW0g48ZrzyOczr7ubc6yMcurq5eopbR2Z7dGS7+7tUowxL6bNDfC3UjAVp6QMDvZ7", + "wOkbV7T9zrIpsPWLHm1QQ98JGmFsbyxLY7sh6w7TvdHzF2V9lsJRi8jiY0sr4g510F5QxVtRcj8wNrMi", + "YGPzFLrnCxl03fXHeSmC48eC/70S15kS92R7XUbc4973H8GD8Wm7+/j7z//SDqXa1ivkIHXa4kH+XokK", + "hEBZKYXyNOionW6H7sW7nRTVSJG1xX2s6BW0n9ZefQlUH4P/sXujYXI1UTkSa601XEc4xMT7xTl6G3Ar", + "x+FkKq6TSUG4WossvUSMWswvtj79okfHMrsAcoAU2JFjHGI8diZNGhklffZXoMEQrxpIjcnZTGSSW5HP", + "EwW6LRIpvopWBIe6A5h17VR0b7nMwXqUZSkgqTuacaJiQ8cbVMGySJsGE9MjcEJmYPGT5jFzUjBRsTmH", + "Pnz8+DhE9VDwW5slgoZZpLjdCMG3Bl4SOHqsXeE1yhfkZV25XEUHSGy1Bls42qmLKPgCEAOfpVCvQs+v", + "AkTdB1mF5fegMMWcRECi0O625bxRcshJ7h7KNjZ8vv10yCplZY5SNJ9HKaSuqyk3iYIdplIHeT5/wdJc", + "YkzkVFd55pOqQrIAs2CHgunF6zyKRHGD9uDIgPXddk8j7F2gvcOroLm7w2F/EK0M1k/tQiK9OCIVWe7a", + "WM84x/77jRWKP7cqFB9r1m9EWgq7vvwHuLKigGj8BgAV4tp9lMTyAH/u4TqNZS66TKhTdspLDAXTdvqw", + "n6h3jocHHyi1iekU7J//9w90pdV9NEp6UV+rEluOaFLXeb8HXazSNPBXvyK59DbhfcD9Sg2GCHHDQ7VE", + "wFufTsR8A5gkqDcNmxNMloiyz0qJPsIlGu4zKHJUp5RhnRsf+zTTmYAAe6Tm59tP3QuYUg+JJ/TCErnu", + "wWiRXjZCWiLSolCIu57U8aU8/3rHTzKjkfTc83RR0xnEJTd2fGFDr/wkIDUEGr20aGkllRMxJ/AbDArU", + "hVBc9nkhj0/E/GF7AsKJmK8VR7HE4b1fB73/oUourRfxKzFw8P7KLz3BTdzqgTyobHQar96zjI1D8Yxb", + "Cr7FEawGyvKyS5erw24v5wq9Y2Lvj8xhmifuEmIWxfJFzkTwvfF8sb4CmflWzoRX3dpslyMv+6+7VAJ1", + "tKZQAqlu99mTX1Mm4bK6HNUurIlsWbV/78sbrpV8CL3k3fUjkn8YvurFIAbxHiedh41qePBwhZ8XG1h9", + "q/m8e3HG4NLQSjER54zkaT24/4WB9B8dn1J82+JA/7cyojzuP1o1VhLQKwb7fHv76t2zGwHgwg4eBa/A", + "Rei3g5CAT9Txu6ioFuKqosMBD9YfDYKDBZjYpq2zYFWUungfwAuXr6cvupYkH9bbCnOUBv5aGSpZtRgl", + "WJY7K3VRXI3OcGe2y61kC7xby15125nWD8Ku2Imrk2qNI7UKoioTlsv8Dnh/fwgRgH5IF6zrWlbvzoKP", + "KPC7tArK0727qZOtlZs7O+aCYvdKnNHs4PLAB64QlgXLxBgibrSKitYv+bd8ftIHI9guNz416UjOwi00", + "xGlUKo785pa3pnWoanaMKVGQF9LMlHhfj3W8SeZTs7GnPrkGQfcgcUMx5h55rIFjotLoR1bnKDWetrbk", + "f3B7gdkgmLZNSSf1C0VI8176GhJ4spbHzYZxnxcaxpfCJT5UlKGsnBMxP9NllnQ+Nj/43F3sHBKmrrx/", + "t1eYCrT5SEY6m1/TQBb7jv/8vLBZoRkqRdOTqsfzPM4vw5nheKOmQ7P0D99y4mMCjut5BCJzr3xezuQK", + "xIoEfyyz8wUyrdsKSTXYGvX/eeVpOl0BSLbBqXpOI20keLkxHGDaGUspOBfrTjWTA+ukv0ufScpqu7kz", + "6dPobvVsYm7dHTiaC9t9Aye0u7whkF14Qeeqmo1EuXnnUNdOplfJIeqEyGYW5JfxiUD4F/MJStZbxyki", + "rMF6iSihE99ojDl6Bb90S7twAn1tCzpsec5nvLH+SQfKzBDby/PeTCqZzxbeqUp6Y2ptsbO1BVfSU23s", + "zuPHz54+85ytXrMlFvenP7GXwlh2UPLUyhQ1kx7bA/RLr+Q41SRFR63IqSjczE1BxLm27jsAESTtQ5yJ", + "0qduuxbMzJlJTqsxwhr24HHvKeSGOKVoJriSajKuyI6ii8KGaU9pw1EyayM1Gtra1cpAzRAOG4Y5FrUe", + "BEkjyMeJGmBk0ziPuS0KBLTB9xF6/JXHJ9c93JIX+T1dta+wNjCKvWkL4j4vXCbTukZFkr/COH22mdmY", + "iXP/9YKr+iaCcPfa4mwvFWL79JKTvIEYid04a+Fia/k2omKXXShbHmPqYsc2fER1VesMDP89E6qU6RSw", + "gmPiBmTyRMHHXVajl8SvU+EjdxIggXw092eCKhlKKgAaOnPkWeq8V+RcCSg9/IJhEkgNWVwPrdRQOgMK", + "cwK2UKJm3Na1FWsMJ8RRJ2yWlAq7hhZXBVsMqKNX9Zyu3dex3OfKmM/lHYriMS59OX2XnIiUcddCgtHR", + "A1K6PYfKmsijdcdx65P/J/5Qz+3iCA9Tj3z1ea3dMH0C2iE8rFKkWqUyl5RBX0JSm+7lWk1E2cuEgSo/", + "ETWVYiIBdBkYE2TSSwvgXBC5ajXLNDN6dfDHMjF3vhLZoKW23sWngiJKbq7Q//OLP3gr7FRn77QdQIBm", + "diMZJjcgLX2ch2o7vnf39HZXZFS2HDDXz4ralvHRvhzzWBFFcggnEHJKL3PuPXJknSHcGL1PEl8Ag5xh", + "imqf7QK2txOZxogG+3UCds7cw2FdiQW20pdNtdq1Ew0QrA2pKKeUEFAT5Wup1xYJNLMibmUFI7mGynGh", + "Ayw6fB3oLF/Ew0qihHs2diNszB88BplTcFq+iKd9mVJQCipfgvFod4Ypxr0FxIMFHnEdrHFVxU+AYCbs", + "xsAzoEZPwWUJFddR9VdzGid6T0QJJmeiqNoe2QOlPgOUR2dtAahjHzwyiKR4IubDOrI2lOvnBvMpIKGH", + "ONI3hg3dm/QNusqLXPh69pSRl0bAdf1E7dU4XH4CZA7lTgBQSgUVzmBDcCcRvqOpRwalF2hoAFyLRewR", + "389XDIwHS8gP1UwgDAW0myhHy4VQzn7N56j7GWfJO0GR83Ii8MUa4bq1eAGR8R4ZWp6xHXI1uS7/0WJf", + "5EmikUQupevMb79oEGvwc4mcAwvADE1jAA+DX9YjdC8DvkYG0BYwvsDtfJFOMlRLIudNYJy+SAz0fLrz", + "vSy4KEjBZzNVRpS9MU+xgi1w01/0iBxBfrzeAVTrx1jGCHLRMqiLbJrgA27SAS1YlD0E3scOElWUciYh", + "5w+qsIzmbFhv4hCjIahSVUjHjqVAooifM6PZVBuCOQgyIMUCVo5Xt/BgGoZJlC++kuo85wWyEKp/hKp4", + "wJSkclxO4mHXbXwcrjKWeLifl8/Hu35WHncJY7olbt6YemuSoGccjpAc36Y7g3u2fZP++sVTHjmO4w26", + "dsYNeef7mHa+KpDuwtN1TT7nryPtOAP2+qn6TgWCb0hcy4HiyzTWvetifWmOVy3XW8Bn4jWUcIs2llh0", + "pT3JeD+7CW99+8HeIl3hN6Ch/R63cq2DQKtaYYvdAmAcoy0dChFWOfhB60oKS5rQADf6t8mtwYCkyqRW", + "lDOp+JUjOH/dCOkc3etJN6EnES3fQUVpCyul3LPTO8VOsQAQIZVUF5MLFDTi0TPgu9IkyjMhKo/UbXpU", + "fd0MyJf2lVWpkBKw5ReJWix65AvRYJFai13VNWGdOe2aGxzsM0XFOQpdApJuVNdGqgA1CiIiqu20uiLS", + "b1MW1EWrdLkkFPq3LRUaRYBDcaoXrSQBWzUlrOC5sFS8OGO8WRor1Nn6w5kruJo3weepbOSl+TYA7vqS", + "k1cXkbOKk73E8qD7e1j1WM5mlcU64v7inKtwKKhOusjY/l7gTcNn298PQ41jKPQdCgr763SqOr/ENrD3", + "6wyhxR4uFT17uRvsIoKd/NSpd71O4W2AfiwXub3wCpy2yFMllnK2V1W14dJq4gYVb2FDcdhx8dtr1xSx", + "y7dUxO06FUbak+WU0UtzCZtOv4BJjAPaYowhex3sYvHA2nT6cwCWvY7zatPpNR7XDfpecyHZXPEYzueK", + "tYQLh4IB8iwjDMwatEG8YKfSSIIl1s7EL61MZcEhcoZKgEuDkAqsECqDPb9R9hGtEUDyFjmX7ZFBq3Fz", + "308FYXjQSUiJ1fh6n8bZjXjl5XbBzRkADTKPUOsxilgK4QSIT4uKn2BnUwH41f56qDcpdVWwQz62oCYD", + "kkLQl/tsoCJgCSrCTyAZIUQUomfmbJRLlVEkga9A6UtlUpVupVVP+oLulvAbdDmD2IJNMXz2hDLiEBw9", + "jl8VpTCmKjcv4A2QUrkwBqByOWDUvuYyD208vaq9BI4CMexgAwWlDG06O48whXWZScUhV8cwW8P9NIrw", + "UD1Yb9CEY3JtYD5IgdIRFMFhYaAtoO9+tZSIkiJ6jcLdXyA5zqbaCF/3qZFtQVE1I4EYLFA7+uZz7A9F", + "KIZj4jDJWpnc3dv16Q513kgpjM7dWY6nlKg69jPleQ6w9d5oqsGrQwpGfeBqHi8N4rFkIksUZOMXk5Jn", + "EWqMT4IiomspsN+sts7OuEmUVBbjrzK8C3dNi3PHqaTN5wGxpURgg7opfaYIU4dAQBIVZQ9Jw0rtjm2G", + "t/c4a6aEyAyAhKgYtJtCoCjki2LjA2LLwpsCBAcAoFc8b4wJLYbWqKewmaAT7kWb87UOgabiXVeu8oXK", + "65pKLTXKu1THvBV2oqbqutw5tf9xA319b93G1yAedMa+VPDeQHmzXSSenKcnxs/E6RQNqqEgkgViuD7b", + "4sbRNL4+ke+O+D4GjuI858cT3pgT5G3GbqV6Ml9ev2jLy/JbjtlaKW+OUq4MOxFzCHflUVUX96eqS7uc", + "iDlGt0VQTRRP6vi5Eme5VKKXiRwRExnAyz5AmNmHfVaDxwYtj+M7yEqgT1QHh8cyGy5qQWOJ+F1hRPmc", + "FaX+BWFewsuYjF0nFVYB0D9R0gJzBomIgVw8z5fmHmEAuu2An/164Aq0BkalXP1FzM11FdSk5teGOa0T", + "IO14sgsU2LqHXQwGhq2TbZvmyCTpQHkLXP5OdzOg2mbvAD02BnbywNhS8BmEERJQ8cM/nK/WbXmg69bT", + "WZ/Jr9CtfQd1ben2soRan1QFwp2tZWPOMuc9I9xLjoR8YbuoSgKWuvCFcAPeWqL2x2gvYnKuUw3jG6Bx", + "led1oi47qgq4uNmBEgsYVY+d7PhaRF2U2knHvfKO0LK4nRoP/t3nWQZQkKm0c3ztZ5lnKS+xleN0WqkT", + "03+Ev73yEN7wYy/82o8h0mE4bgY8j8ZTv2G6x7jJUpguNeG+atQ9jIff9bAMfdjj9kqI2FPnkheIHIJK", + "AX5TpfMAJZnDZqMK7bOzhqXgGSZ4Det0a2TYRSkd0cL3jk9wp/JhGCp3zHhoLM/F0LFbfeZYyFRbrGM8", + "mkMj4IiIuvbYKIaPRc81wt4cvUO22wpNVM+hHT+pUw/eLTOpyI2HCHdxnAvCG3Ijvg50H55lEgXYQaTB", + "o/y+ULc+RNRU4JJN+Lv/6iHwde8nUZpWdDr6gVl9IhRttTTRgfoA2fXSID6BO54/7u6iJYbOj0TBVkCh", + "EqvBWYSJ+l5jmulMjmnqABrb2U46TKoMAErwFDuelWmBniEwahfhsNcAXP8xOP8h1eN1loXXa0aAW/jb", + "CR78i5iHCAk/B3AB4tS+rB775UL3FkTbZepgrIvriGpJEN03KrSDpsmX6kqgQpq7o+ORcTJpEIi0kZHr", + "0ZwSRZUs4BLSB+n7vsEhWMrJRJTk4w0pQqtKWCzevL/107hIons/PFWyZJB37Jg6EuywFtLzIiq14SW+", + "Vvm8n6hhyc+cBDBBATBMIyyd/wLqbwvUL9HIAEIwwTWz6MBBRP9mvWhpmFDu/Wy1vMCJtOK7+VPY7ZT8", + "7KbR3VZt0YV1P8Kaf3HVjz90/Buiy1184H877PdAlKlQtidUqjORLbIjQFW+VR68EP53cc19T+EQrVvn", + "QyLLaM/lCjvYTZSp0qmzJmOB1KuUtGwqRcnLdDpn4tyWHNwJhH16sPe6y/7j/ds3AIzZZY61Q4ECYG2E", + "c+9UYbhQnPIy6/EzXooXPmMrE0Wu52jA4YuUlxvybWE83HKG92sAhqbPSAa17Vo7INoqznHP2+86b9+M", + "r49rPn3P2Ddn7PuoQq0NeJvVR+Weud/JyOerETF3tpTGH247V91F7KtTnsssuAq8WyEcVZQhXnwIn69T", + "V/LQTraOKplnLcno3ru1ZJrt6mLe9XWKdNlFP4cou+j8SEs5cn+QeqBLvLEOWgQ9dwLKjwSiSRPF85wZ", + "dFO2x42vLIvRuY0c5nUBV4dxEYrl+hP3QukKikl8uRff15ddByq0T+/cBPwN9PUl8DdxAeKa+/72y1g6", + "ThDPbTkm6jdVw5KmsvUJ/rERUs8++dnbwlcXxUAmzq8SZ+fm2NNGIRyNqsVXEyB+LXhACinWG8MbEeyq", + "ogYrdv/qzKsGx1nmMERTX4wOdMdKDsjGdH47pTTj3vzFW0tvgatcTdQ+lgGpSfC6KtVDD7cUvI8jyHCS", + "Kw8Az7IvZanrCsr/RvjrjcTL/bA6HvcyYXMbRbgfWV3yiTgUaNK8Op/yytgb0pRvuvj+IMtqmQSFqy6W", + "SGsr8K/bqSsqvd+iJ21BwHZvJmwpU7P1Cf8BP+18QjPy8x1Od8dw8yvn3Wv7jLAGsHtcs/b+6/X8ugH8", + "GBURJeveEV1R5HPvt48Hw2q4gTXpaNjQ2oHVwSnjUpgpvAsejU7X69dOpFeGwBar2aowlVUJBtBul1Gz", + "XUrV6DJotItlbqGw3IxD6D+UhkCjpd6JRMWzx6sQWokGyB8rxQyCPfWZwjp4C1QE+OUwJHLsoPvGx4kL", + "BSUGfOGWLo4Nbtrj4UE6Et5VQAd8MinFhFuqi+eRjn2a7hmXcK/uZoXn0QBMxJkuT/qJGuKSDFmaC14a", + "1oSSwFnWvig/1EwazIjgldUzbmWaqGiMfTZguWuIlUtbUK85kVopenjTQbAY1ClgKHLlM5EJiYQ8V1g7", + "dsnH9OpcpJUVwO3eQjODmgqvSRtf6myda2kQ9qpxoGjLMM0D9jS1jVS/+W3VQL0JBcJveIryiJdNmPUQ", + "vkWA49eQXOjT8Qud6wmkUkLQfA8ycCKaw5x3SiuEqhtGz4THuPJbFO3gC4gCmrvzJ00cf9iDREHkApWB", + "/KY55mn23JengiEjrNWj690JxMwwqGSBouC0LNyMOPVOXI3edcX6Ep17xpvHSi/Jta8oWz4T5UTc1cyG", + "uUqnpVbyV6SpEyEKhkGSTCpm5irF8HVxjpuB9/B+k3N5ItjRVBdyPO8m6kAbOymF6bKjp0DpQJvg2Ie0", + "4tLf2/TZIDeanSh9phg3O9RqGAysfTdR7vGIGwE/AWxSCUvuKA6eiF6qZzNRpvgKoOW+1Hbq7y8wy7JO", + "PJpVBkwLGhRJWihTeyLmfUa5mlDFVojeGZ8z2D6In96n1E0nTnA6Bv0+2Bkg0hV4M0PH3b+GwHiY7Emr", + "6xvgIwMpGXU7EEdBML8+u5Yl1fb2k2/Zfkg/fPRoh73TuEMoZkfCngmh4HPTZ0chodhYXtpEoYtKzeEF", + "JseQs1iWVYH3Lq6/gDLsWne0QZc1iIBDm2Eayz442DfsgacB9rPWu/TTQ9pAlvMTwcQ5BKIDWZzxUkw1", + "AFWR+NZ+WSA6Wp/1nBKg0nkTYI9G+fPg8N3+ux9wBShzGoEG6yBbt2mUrqRPRZkjVgcmAph+oo6otDyE", + "GeIqDg72faxGS2ygErx8Cyf5st4RCljHi4dfJVT+iDIYHO1BxMdIZwSMt4sN9l6pVEMWKXzW7UDMsfs8", + "58YewxyzY+nGB4MCgqK0R7dpO9voOyHW8YYXVrtmANqzs/P999/3v//+czd6/Un0+luN6jO9/QReBgFQ", + "C8swiw/KsfBSGOOTmmB0/gryroz9c3dDcRft9y2hKjZGYKq8NQYGfmbua0Bsup17gMcb+EEO+DzXPHuv", + "9RvuzhB8t4F/64MKV8NvRSb5+7uWMBkLUGB2wGWDtPSCsmaQsYx8yKh8EiWE4KH58htOjPL6onJSPkDs", + "+svD/NX1dJ3lnEBojqscOqpPb3eDVL+NMvJujgcszMT30sII4A3WqCIV5b3siaIUaagGseDreL3Lvv/u", + "2XNmbFmltsKaD+EDR9Wiz/xQvJwWM2kBykUahj1RMGI0iFCyBKr7Y6wJtiuyRA1B4z5GyS7MEMMlXzjT", + "XSuZ8ryOXyGLHh/3nIz2ZVlANzJVOQZ4Lj2TlsmlXJk6Y+zfH3/35+++237i+GLXkbk7pp2dzv/++9+2", + "e99//Nd/advxu1Q1C/a54a7fhPvCV20s+MmTDT/+oCgcI4QaPNl0tIfcijeYKrsx54YPV/rdNx30S579", + "wK044/ON/fXInGqzteGrd20827AN6vi9nAld2SsXO3jeI4jWRTc/8PNLiA6s9bzlAWVvH0Blpck6gNql", + "xnti0SWI1qa1YlbYZh1Ljz2Te3xwiCF3JsD7kp+K0mBFFF+RBSBHgODAh+Pj10Ox6wgbJeWKjeoUJKwN", + "WhcKDaUSSjGpcl7GnwqVFVoqa/qMandKNcHiqrD+VFNbF4Zcw75yp6jds5XKNCgPVMuVPoWZQc4ilMIB", + "7DXsjBUlVGYxTOeOX+M0yYQFW7AoBWVSUWN1FD43DKsqlXM/EfIDt8OfeAQnLstdbOv6cE9a+1oEPLlx", + "RHQc0IqKrf/p6Ceo7VlV+hJEWLkx96W9r7y07MVDO6zJNPgl3eGiWuNsVFlIS21Uv/2dVh+/amPB0WQE", + "uoTXtNGZ/3LVn/i3NKYSd5VzvwHwAD954iKZGNk42o+SQHHmOmBSeSbmL7AME+cA4xoFgJtuHQWNf+IC", + "Q8Zol3zmiDwl3P7SMzsVJZblaubbsE3SbQDe2efbMCJvNpa5pQBniG0GBk2lHIKvh2ZaCkinT7yla9jQ", + "OgXN/pufDGIiBgnlp4h3fkJZKNkNl3v+S6pA4D7DJfAAiVxlGDu+KgE24g/7SEvXw7OjLly3X1yy5Yu5", + "X+i69WonplA4U+A0ved0XwBLH6+i+WoGd4cV02VQvcYCeDg9RKIDN7ea9HxZ9qZO5UH1CPwugNE5dsXT", + "tJpVOUewu0oho0Et1yO3kLBOVMrz3ERVK4IuKJWxgmdulfy1i64MG5aVig7JkOqP4uhqLFMfpa3J6vfZ", + "+krAnYct5zBzP+5NtMQbUQ83AsK71wPv9cCb1QMX+cSVcMlNYPRazt+tHYWI3ghN6o+Ik8cvRwx3KnA9", + "2sEbh6BrIfw7CjN3zau0Hv7dspk2diXKm88oAzii3fo6PYD7wp09KSdgtkSFLlcjEK0Ehruj/KeGxPHC", + "jRVTbm5ZCvsgUYKT0qVf5HvJ+wUgbtcqc+8mItttcR5/jdf0yztZt8CDAuYUew0OITeWrAfGRqJ+0aMa", + "s0GUWEnFog8/LgPPpKohx8Fl3vWxsImCSwOIVU6n4On2Ucg0BPQIcXNiANMIAyoxmNNqNoeKNrV55k+k", + "cryAGEWlrMyxRI0NFwxexTddZjTjiRqGe4JhjeQ2E1wZSFgD7lOKVCu8lAXnk8xzULhDQJe/CeAqVIVv", + "RkgDpJwHcvP51PS5IWsDi1OGoXb98saB49ioYdLWC8bzfP6iLtOmE8jaAQPQai8wMGa7EG4xoNL8Mk7Z", + "aqS4OyocbhYK7uKhLYC/IUVKFTkN3EkjUdGLkf4gXjmyOb05Eh07vGWryeMPjAZ35QKjrNQdjwrWlQF8", + "SLhWNYRO46ZPKgjxW/Ds9Vvc1qGYjDBJHcgVOemlyoz3y+d8bkKpBwSudAxH1l6t0TxRQ6i6PWzUV4cK", + "7FBIyx05qSoKua9K4x1UXGlE6qGuvUuLYmFNXZuynocv4wsvIjIynAxyqbvRQKg91Z6c8kyf4URSCFLu", + "stSLGWlZVfiMKK8i01xhKDmfM+OOi0oFySCALYIK7uaMolfdciirEbUoFS/YVPDcTueJ8g45uHA4kVAU", + "oFIQHTMc6zIV/+aIYQgl2Og+grpRc1ZoI52gq1cX7jygcAy3vqpmPHdck3A37oFICSO0UpQvmOHtjq9z", + "nc/DhUvtiGy7c0nUJWucLRSMafgwr/UC47BSN+9LhE7XIJDRcWkwLOD7wdN4bzBsXIemUpGeurSkXyME", + "wLn/ZdGa9DFwgxsJ2jzEDn3Y5toRH+EwT8ScAv80hOhDzoMt54iFBIBBY0rPotmA1wEQDoCNpboQmWea", + "EQNwen0pVSoLniM/8WCkZGd4vTgOTkQAJsoFBJ927bdwnI5StUSGeSCHAj0dHG6SPYySYpWinJUsoB4h", + "jNsL5KtzAhwg9z2mwRKiHgba5DrATbn5lFFtf7+TOOR6L0PSRzrvIbjACgytb7udmVT+78eLsYzdznlv", + "onvuYc8JiZ5HCekBKxZlZ2fMcyOoxuN18EzY6EtF5Ty56t5XqNF7ISwhEGOdOtmIpl3cjhaDlxAUHSWM", + "wQTCO/qa4NH0WY9EAKl1K2J2hdP3TwX7cPgmlKZrmYBPx7uop0N3MHuDMdDA0nmuJhOChdAQqMYy4fQV", + "qZhxJmpmms3PpJKzahYToKOuiSi/Jpr2LsI+OfUH0XxrvJ6sDQoKGMmN4EFtGOdKZ2Hg2dj1lnJDgvQm", + "FLDBAF+9UnyuLUvhT+vu3m4E8Hcd1SkiMLlbNdQ8CHB7MCxX7PD1Lvvu6fffYoARZgYdAOTkgt2Bu4C9", + "9dmPgKORqJmYjXzIEmZnOblVlUaeOtENtgJTVZ4T8kUpZvpUEGnjx/1EUXZAXaLSfXeKoBrGI9lQKK3X", + "RmkokcXTptTDVI5wiS5UQEqtJr7ZV+/5hCQ9ZnxyCISVujL+lRkVaOkSwiKkC7BhQlvS2046wz4bsJk0", + "4DkMluez7e/jSArH9c9KWcfQ+iWvIC10tYAf997C/rZnLcTjaKYthF8wfSHptCUwfLzGsArcENibpRQb", + "oKIeEO6/fnXTNx+/28bwcUwM5rSYbtfQERzRtUhSJEygyHFAqyV4YE+MdFr6FxHDYyCGtmyVJ5tkSOz6", + "fn11osGN14m4pEi/MzYhclUeUGeN50pr8A+7naJqvRsBd5HxpIAMscmhawgkSHAKpfcjUOhE8dKz5AzD", + "vQ4G73f/A3E4pQFAL4qyBaxvni+LCWJSLbwXs9rvme+dZL53jTnict8zxz8ocySOdkn2uF4Fx+IyWzM5", + "8dbBsjp+/bciQPg9TAqosb+xYAXUwMGSEqGWlsp4rpWvcdFn7+FaIVGnIrW6PEajCAuFGI8OeJybUAak", + "RvH2Di142fG3mc5EbrqJyuRMKFCu69wHg8HFE8ihpYuBPjts+t5CdprhMwGX68cy6zK8+gCtfVRlE2EN", + "OvkhZsDUwZ8xCjnBxEI+cJzc9iK+rQ9odm7hswoy7VilHDcFz2rYW5ZrXfQT9eN4DGiq9Q9U7QgMDF9v", + "CH8VjE+c8W0Zh6w5KIkM/bbeLoMrEDkoNvPW9/EVdwXtkIXgT+su4JDjyl72Iyejjkdzi38G1MZnj79/", + "9nT7WYQmK5X99lmn25nxc/TDfPvd4+0///nbZ93aNfNs+/tvl70zXeql1GfNTh5vP3kWN/j8+dNvu2sd", + "Pd1OJs3JcSlgH1rG/Xj7u6ffPXv85ydtQ19uzZeSmrc09e2fv3v8/bPvvv3ODerCtj4vQUh2O0j+yFWC", + "fP/boPc/vPfrdu/7497HT4+7j5/8+XOLiHd8raT99HUJ4yPeVniwiR9PvYeG2gHkr1bCfxXCpve+1Gez", + "FKmQhcU03IxkNj0krD6299LpfnAV6Jq5WUm6iL+LktPxUogryuVYpPM0FxFn1yWWEKQ7R7iSviLn3BJ8", + "FyVMeH8txs2QF7m+axhzmVfl1XvokCPGBRa9CdJgs24IXxX4sCzIW4LU1wjdkWMmtDkqn/fZnpcqQHNd", + "L5+6Ye+g/OMvetRP1ABjOJ2pECgUI65SbnmuJ9FCRwnho8ouUG6iphx7HYlJ5Sv1RddaJTMC3GV4h+DL", + "ovprIAlAl4BwhucDbgQslwou7a3MGacimPEBI0NRglyNUnkgAxHWAK7tuaNE6Hkq3XqvAkNbLf/uuco9", + "V7kaS4BnazgJ7ePl0huuMjj2i6JeL68frDYoPvi41V/06Jua4zRLAzdU8Vc8nTJjRUGWsUlUCHX1uXbs", + "gBifjAqLQslRbgFrfP6i9jTVBzafJyp1J3YsheNHY1EKlQqothOwy2sTww1L4R3k4N07jEgCWN1GOKI0", + "CGfuR0BnNjoAzjApHONUEx9x6zhsuHDns5GcVLoycZwRRqPGl57I4temGdyOrk/VQ2OARSugkB3uUqfb", + "oVjxC9VEaupeO7zn4zfKxwdeqfIaVShzczV6Yo2dvx6b63394pGAKZtrDQpf6g4LFayp0Y3pAtGEmPED", + "vXvJ+W2jvPyObY3EBIuBtgeOvXQ/R0t5XTnedQ/Q49qgzMfX2O1q6ni/vOAMMjG+2Kt8ZwgK5u6YwfIU", + "v4Sk0lxwVRWriWoXX2hnCG0a4wL8cVpZPR4fw+trHcE3yFxoTusoiF6hEIrfOtHAbFhVMHFeOM3lyjgS", + "qsYx9SyKd/c7uIp9de0fd3fZgx8LK2fSWJnWqa/pnFFJsYfx+Hwhal2ejHN99ujRTqIeO1WW15Cnhoq7", + "+vJvudYnbra+9FuXpbywIfsmUYyx4X/1MIKm9xNedEXZYRTICpekHK5BeZaoJ322q2dFZQUiDxi8jHC6", + "NjegdtM9Mc8wcMck6mmfHVUjtwYYf4br4Su9RXXreMaMsOzBiZgb9q8hjuchjNVH8kO3+B54j2sdybBC", + "lKghPHQL9r6+DogxbDkiP+IQfagQIBpgnl4IXMJc4kTtjyFtgd5lU25qeAVKZguE5CyQEaadwMQ4XBt7", + "DS9RYXkpgA/DGwEyzn1YWXY2len/x967LkduI+uir4KQZ4dLmrro0u2ZXQ7HLHVLbWtZt5Hk8VrH5RBR", + "JKoKFgvgAKTU5Y6en+cBziOeJ9mBzAQIsliS2m7b47X9x1YXQRCXRCKvXy4glDOltwgu3Q0jz+shw2D7", + "fhu4EUxgdYFGMse95Gz/8jX0UMO7+zw6dIq5WUjD9IMKm4ApkYRMiK0yLME7WOpMzlYD3IcHIknbD7na", + "HecqAiDscpAAQfyqlzZ+8reC61sfx/Ou8cg93QLe/qgpgD9/eL4WDByBe2nlVOayhPw/CAdJZcEhF4eU", + "ikKojJwbv6IW9tuulmcTWQUpFp67pB/qZA+HOur8jVfNgo73sQrQpBphOUsoq2Ab9W9Is+aew2cilbau", + "LQPJYHz2C7gT6IJVcK+WDT7ywbf5u+hft/IJOJs1qWpdHuzavLpJTD4n2davLAMe4cXzoVpEo2LrH7Ep", + "jwPslI8s4EcgzxGwkc2qy6F7/PulUyg39hh90nw8N/2DJp+0rrmF+ni68xo9rmtDTwh7H5Mif12Z8f2v", + "q7A/KV34sxDksF9QMvzpg/q/VDj8iQv24fLgr2JL8SaNX4qLUFHVjVzkuuRzEa3uUV2E9d+ah8C4cbC/", + "vfL59O2KI2XWjfqPu/U5OJtz0ToVoTzwzz8URvDs+UfiyrX+fRwIN9R/i+Pgh7L5RIDJ9Tc5D7/ZhfSP", + "lmUCwr0BizH9vRzC2Nb7UY6i5fcCDZqPSLoY9x2tuH/p96J9+fE+Kq74RgR58cc18dzw15b0FBHHx6TP", + "0bvwN14iOs+nPL3bTLdX1OKXoNzuMLh4hB+aO/NvcxDcsokMgAb+OAJPxmrqPIel6tYhANLxYx8Jdy88", + "X3z6/Zol/m21ies/1IifLME0neYf5UCAA/f5B+JbaP77UChgrL+HI/EtBjP8cTB+8sF4ILJ8/DyItyXl", + "rbpzIZUtubutH43/PPGtjsPLPzf+U5ZiaZ8ioPXvbtU5hNwYvuqipPAWq+eKWEOtQEzZ2bBewNDR40s3", + "eueEt0fdox0TeU6ChXoqt+JR7Cz/qZOAOP4Ly4ld2/Ws7WnvDpTjUF37s2l7NqEnUpt/j2X/+Ny+Pb3f", + "iNH/5I0HGGyzGphKUWxlmxToJUBPSe+A6VmEXX+CIp4+sCNM94F7v+r0z2E2kPg3O70fn4zCTH8fhHQl", + "fKLWY0zEw2F0cJFmotfPoKBMYlnujaLjETb4g/9HnhRckkd2jlpsuAICJkdmNIJzQ7nout7NT91Lox8J", + "fD8yuvgfzwfcJLtYwC955FvffMQDZ3Sx4ZKARx8mMDyDIBDnfDNJHKs/jnZzi3BFHjnZ2IBxlnkm8HG3", + "DFMj7QcqMxf01u90356lSoVPnwFg6HP0qPAKgYwyWt1OXYqeES+erj7+cUR0s83HEaHy/sfzaJzmr82l", + "n3f+vyEAug+Q7/GVn00tpBw8fvLD2C9961/j5NHHzriSMzJUPXXyDkMguVd6lvR29+Hj7fYNM0aY7WML", + "9wwjRnv5/ifec2ubtb45l35LRMkzXvJhZHr8OKDnftYb0c/9EJQu2UxXKus0oKwTURjxTyOOkU+4Gr2j", + "vz6IXv4Rog5+E7LpdrjWoRD/Hr7WDyBAnycTEeI6ETBbiFTOZMrkclmVDXqIID03kMMyd2SAmIcbgaKu", + "CBW2XAh2Y7jHr2Bnp6wwIpNpCbAFiPQ004Yl1C89TYYTdekbYvpernnmwXaTSbW7e5Au89tMGvhbjPAn", + "t4X0A5alEm8LbQGB0Jc9mqgA2Ht4guCNfihd+XeOm9ZD2fpFL1SCeak/95jyE60PlrS470h9e/ER8+vD", + "+DYyocNNhaEQeZ7nMosKLFj2IAxyLF8GK3Ko/EpD9p4ZnwsrWvAfv9IwbhaCVcrxpYwRbH9cWdOE8yQt", + "WDUq+0QpLoBKk6WFOAKhMka4ZIA51CkxbDqndis69/Tr5nxuLNFU8imkWdeHvVcaIZhQViynueizXCrB", + "DYDQXP/jbJtpqOyEKcuziZoJXlbG8SMYwpCdAcP5GZxgokKZVVBG3HME504aPC0JeMc5KDGz+tacqCLi", + "SZhxLQ3zY6Xd7qyXQMv2S5n8m4zjN7PUtofxJPPyaCwvfuUzj6zIJ9n3ljyfabN0hKGzVd/v6CDVlSoD", + "1vn275ShfkxB9Okhh2upFkVhEHu/Jjt9RaVWNMu5mQvWQ07A9nZ3d3eZ0Q92+/82Hk+MeQOTx/F+hHAE", + "mBDK8JXJt8ZbI4hrIWGyfVv8vZLpHZOqNDqrAjZnFYqy0TIdXp4ARMMnn7DXgM3ATuXUcCOFdT9Tq8Lo", + "e5kJC2WRB3UNueujr21dpOGiEOrw8iQIwbCoY9fNgO3sfKl3dsYsmctyUU2HqV6OEOM6m9Ifo7keFXfz", + "kc3uEnzlZlWIa5gTvPof1M41YD1VLLex2eWqXGgFTaiQEra4XF2eUJOrypbrDVI3CzuUGmA7DvMcJ+Ru", + "Qiwex2dY+P9OFCXUAVuptAYOcXP1qOCpzoQvrueBU6C4t0kXshRpWRmeQ+2GeykeAh8EhEJupEVkc5UB", + "mkXKrWDzSmaIR2YFAlx8t+RSBdwV+M73vUVZFnY8ojUcSj3KdGq34aYkrWsuylKq+a1HYepvvR1k0hY5", + "X51jiy+xBbuOW+jU8tkA+Dhiln239S1ghFwffc3sQld5xk5gvO5aXwKV/CDS8m9b/a2v9APLNDtp1DNs", + "rJxr9e3CMdWUK3bCZlJluIpU5sD+bev79druZ1pJxwHdUhGIyEpXxpNzmle2FOZTSyVjoR2e+JrMoQn7", + "Chp4vBZ60b8WyrYGyjeC54NSLoVnIBphUQD7GmUWQMS3Y6Q4KlnriO4QCppnjVY1egzP8YVMzI2Tvtwb", + "19p9Bxs2kA6qMox0Vqk0fr9S0SdfGwm1yz1YnmV8NhMI9UfdQfIjvgqSObyGfQ8eZCbCq7RutPJQSt29", + "huvH0oVI78J6jVny5fENG+FQfoRzfGn0UpQLUVmnPBuZWt+K/pk4EbXQpmQv9nd3Xe/fIKSLrevnQlVm", + "wDY3gZKWYUTMrmwplrjAPBcGJirVzHAsLF8ZKvjikbFwonFxhvVT4SnlrNWo42AEesflkDN3HvxOSU+N", + "KyR6Xrq2AOspZ3C1lZ7ukLaEBcTmxkGq506kDktQr233aSGYHYCodgys46wM2Q0+xkIN0KLGdgKKxYrW", + "bFnlpSxyEYotEGVgQRQA+sEbm2ViJhUyrbD+KNavCqwPmfGShwHR+PBy2tmBij3Y1c4OzjOtbKmXfuRR", + "ANlEfbuQAFiJZ6GP36Za1VTGyp9hK+6FY8NTocRMljbcTe6iYdd8Jko4PcfKwpmBQaZaWWmhDCbjqdHW", + "1suDbyOg1o8iYyduYaSau07eVHnObsTbEn/Fsg5u/o4uAbzfl9XgRWF0YaTb3Fe5uBdsiYWwqf9XonRk", + "dC3cZeK6dsMd8AduaCuA2JdumjBJ14wZkQvAs4R1L4SBb6pUYJ9/r4RZMRo5DAUnjjuBt5+3ZQGqXAR3", + "RdsebRO79rscCQ0wYawPYrJG63hL672E3UiSxAmHE/VuohibbPmlviVhcLI1ZvDIPYSs7VxEv7lfsWXj", + "R/ezm5P7cUKQrpOtfvy4hrNtvQnvyrL1mfVe0WDZ6BWbhPKOQPyu8XcTAMOZbPXZZOtOrB60ySZb38cv", + "vu83R+C0p486gEW5zJ/4ppMsn/gmOFU6Pgm+Gny3Y4TvnzfE562MN8g+MdKOTX9y44nKqnKhzebJNN9o", + "/DP6x/sGtXUBD7tuEVs5bhlWRqo0rzIxkGrA89yTERJmP1BIWCr6NPzv/US9h6OFcH5HK8WXMo24Fesl", + "XSNKtkGYSZ4Yb8IG7ELlKyqaP5Miz/wdkDkmn9RrnLCeEaleLrEcjmN/EW/afuJzWNx/QBzViV/+Yzgl", + "QJ7rzXLxVnpxyeb6QRjo+GSGCjzyNZH1/WsRF7UhWALX6hwrAlPQAgAgDtC01UVcCbI2hS81pu1mGu7C", + "DK4W1xX17x36mVafOk1jIYwsWbJh8xPUuQruRAfXy6E7hnCvUkfj9THSUaUhJpvOaDgnbDgcvk+I1xNL", + "ryPS8fdPoiHCyU0mCsbihkLvwJWHh8XS2q/cUi/0A8p0sJW4kUN25mUMlBSonHmQPKBVuB7D/X0NJang", + "HdDyHmW63zsKwlzI6H5yz7B77LI5kKd7bbLy+BMJ9OpkyoT1XONtuJLrn4f0Hmm937r9wVt8wO1gpavB", + "8ybmhoDv3XJ7u9LVLW7rE6Ppx0O53Z8bvkyavx3Qb61h38Le3RZGzORb4i0gSb3WyynVJ2ZXVS78uUnc", + "99B5lLj7J8EqvFVZwcEVb51IKu8F66ULra1gWkEEsofld9PZRpWi/sGyHvbbp063QaOcgm49BRaEiC/c", + "SO5E2l7i17vPkvX1SpAJdTxgf2bh1YD37+Mj4zGyHthJeJZZmjMYGNpnhRmxhFraJGCLzE3WV8jkKUnc", + "AyL3EQ5pouxCm3LBVTZkx29dM5vynBuwu4OUj1ijcA5RH5gKOG50ypi0E4WdOSbouGSmgTXqooTvUzNF", + "FZXdcAHwdYA1kQkGfaK8BZxNQfv3hqZEm0yY2+kqGa7zCeg7QR0PQsuxwlrzaAO7JN4OswkIPaWTeZnA", + "eWtTDifqWpQscX87vu0vCfB60G71maqWwsh0pKrlVJgRFcnqT9RU61xwNXL/7zuuLJyOP3J/jNxftuTL", + "AnwpuVR3fnhDz9wyYeS9U5HdMknv8ArrRVC0n7Mk0+kt/iNxl4wEsygVpPBVbQBm3+0MGGOwIJ1fDyCr", + "ZWVLqnUB+0L3y+pT2xCv4TsT5V5xFELr6xsH7slKfuf6EKnIhFMUoGqPvxJLsSxyMI85FUuomgd7kZxO", + "XybSHCxmVGSPZRprt8J+Frxc9NGbM1HKXfM5aErFYmXBQBFGA5ObCoa+pJTnwy59gJL9b3nZ0AQ2SsET", + "KpSGD922ghEnet4kzLX70L1CrJUozAsiKFr1cVxtDeHRIXV9Uq2LrMTV60d4INY0m3DttIYeft84+kcF", + "xWun87td9QY5xqf6nmwvSb0RxNBhBcJV1kfIYQGlAvKVOzCMK56v3ObTPeCZEVMCKE8akZb5ivmhDifq", + "MMsc9a+d7lIj75AWLD0YaSGtO0UtwpooxEwesgtH4zWSMxSXmFYyz2IwsCk3RgoTKsfYvscsnqglXzFp", + "bSUiBhdGHVhCDfmc6jwidEAV43N3qU3UTBpbgu3Urd0MYJjTHMIpwDEhS+QSfhife6ujEdyJu8CuJ+qB", + "m6X7TLVUts+sxoW6DXU8biFMP2Fu5HjZsITOa5ZQDS7HXieKRg258FWaCpHZDt4NF34yUa+Qa7ptIHG5", + "FqGx+jLs7bhxfrtORFPBycTboFTUdNgeQkMS9janQS7uRU6Xnp4RbeGNCFImvIWXo0AxlOcsuUVpGhq7", + "gYN9B48liWDjdR70gSqZ+ysyy4GO5mcHapM7DQOEMe8UaPrsCYmlzxJ3OyXbDE1CMKoM5TWoR4SXBQoo", + "OLmeu3OSYdRvJPrVEp827l8NOS8IUtvDpnJwAwIpbRgYvuiXIPgNwCg2AFEbpwGkg24BfScUmaLQjOu4", + "hZUWIwmAqtDcY/ssWk5H93VX5G8cwkdRxHzso1/dnJ2yks9BP6Gb3X8NnjX6C7LfAOWuAcK2e90GltTz", + "uO26o5Mj23cfAST8Usy1kW4OM5mXwtjhRmlzQBa/WAmor0tYkKrUnkMMOyT8nyHNhw/h+IC82IB9c3U6", + "AkGIqIiWxE/bbAemcY4SF3zFs4uaHkgec12eg0hmWY+EMrQ5z3LNHYmFVYSAAdt3ZJy65bOpNoJGR0Kc", + "6+3GVGIEDIRkrzCgIyfS3ciliIbh5T1400t7JHdGn8arbuQrfgN37zNxLxBYHvv/UmhbcCg5X39gLjS4", + "L9wHTnkpyyoTo1yrOfzF4Fn9oVyjz9RRise79nOcC20XHOniNez5WzYXem54sZApg2d1T1AtjSOhGTEn", + "UzkO9JqYXz1KsZwKuGhd5/+AuCAWfiPCRps/mHPARh1tjZVLmXPjrk+kY9qVXE9dh6+kcnwHjOlxV1EP", + "csnndCbc/6b4Rn34gM2AahliAagywZpORY76yMiDPBU540wY4wOcYpE5gSPnrV3rF1XC/v//9/9jSWRD", + "WGuKqoVr6pjmxBfijd8lqu98nWg4bh5++j6YYapSL3kpU/YGRcHa2s79I9AoQPVED5rXojTYy4LwTvZ0", + "8lYmt0HZSXZ2xnBYwNlZX4212O9bWv8uhkJIYeFdtJV5DrHkhbuP0cwT7mMKnmBoDnHdH55EcQWhQ5o3", + "zJad86UTMF5rdY/KEzxuKif1ZkcU4B1exCBr62LLnoSXgq1mM/lW+KW5jO9NMJXiZeZ2Ge0NYGyzTBs5", + "l3WnECUHPXxNTJIuTjDkbDL+XHfbfT6WgeYYRfkPMmY9YlaiFSXjjhN8yKhUXwj1ozDTjmsjakUTZL39", + "AcgEzC7QutfZ+IAaHzyncfOG64lsLpgauB4Y/hi/BoT3StiSXRqelu7WQXMjWFZJg4FaJW3pI5LXsIwN", + "GCtAWACFZEAGMk+tYOvwNmtwtkk1H0fcpm1bFOSmm+pywWZBusGgbDCMkKVowA6zrFu6aMsPUTwBxQXA", + "PFGGAtGjLRTBOXZyEwg3LMMSP14PcD1co7X3MTs2MP/CaXeqIZ7NAn/75BN2WXsI3Om3MotdkV12RWJ6", + "C34v2ELOF8LUCkqqbemrX9S0aqVtCnnUhWMlMw5+/GCtRouZ+rQMrnEwT3Eq5EO3ateqA1+O6S6MymmH", + "Cyh11LyQYVJ4lUYjanJ7MJyGwF2YU7wNLZt6IUww8LkDbEl5Y1b+iPpoGBSYSnAPjjz7v7k5ZT0nLA1u", + "9OBU3mPNIx91h+th6wFi2atarAdcWkhWm8VhBlgHtWbagNZNkVRHdbO1qRux1PeUAwnBenPjRB9GtdWg", + "QLSpFAQEUI2oKz4rWQ5VpoJMBBF57HUMVOAeBagGmDVo5lj+GsIL0OrQSy4vrm/YxmrgyfZEUUmmpCzz", + "hBU6l+mKVSoThiV4Dydo7cMe3c3h5BhHVG6Z3KC/uTrt9I+vObsnW2WZkynIryFKGPsvFrXr9VHPOnSM", + "OHRrxqZO7/ozXK1POFonW5UV5lZmH+A0rr2+HR/f6Id9hoP0K/3AZAk1pux4Z8edoZoGsYwbUWwwJjRo", + "NoRESlNLS8RSLYtlLTqI0xXLxIxXeYmug4YgB3W1mFTWLZ5WcKO9Wqd0UynrdBKzYge7zIpUq8x20Lx7", + "/ZhK0WWNk4UaqWcj7iKSwjK5XIpMQn4P61GpHfomDOWIDjT2AXkGImO9vd3d3bh/QG2XS4HOr7kOkZMw", + "tFQrK5Stgv3gNUaHuGN35Ws0oxjo48O4jyApm4obpLYKnrmTEy/1Bx+fcBSax+gv2bpNFp/Etmkiruee", + "NtAlf72z1raibzDNbTCdr59EGP6t7+TpMIl2DIf3I5EQGg/v+w87ulfYExAdHd2IlEjcJ09DYYR1N5pU", + "DdXI4p1J3gjFrt68Pjg4+N8MV4P1xHA+7LNkf3f/5WB3b7C7d7O3P97dHe/u/j/oO6x5hXcO1j61B5lj", + "UCarLa7BQuHZxxtcdzVRbtCe/CzYi/Gi/RRdSHNhBiFgPmI/8P6QXXu/4kRVStIpT5R12kMF/13Cf/FP", + "sDd6JSIjZfhg1zodvuYo8OvLpfvxJVtKVZUk7uy/WLgf91+wha4M/vYXMJb9hWV8ZSFPiZOTc++zvy5Q", + "NHd/uUZ77EGIO78U7oC/5hZFbqcVITxrHYkJe/vYad54De4tiLjrg5tzW95ySO9zNPg+oilfcNWz/T2Y", + "HXH/pPFiMmSUXw8+08CVtMJCltjOSZEQ84ZVJemapUKbN8LJwk7bfM1BWfjp08TbPpqJ61EwoUrg6jQd", + "v1s0oXDJ0HhO9ZxdaQw1/+lDOdhtLeqpnnfJcliyIKOxHOwC2QQRraEELLjTyYSBMqZEJHUYJpXAp/Pv", + "zlBts8Aw1zhrlfWScZmQ2s/uxMqiReKit7dN1UyBms81mowyYYWRPPfGaiVEhpc/aBd0HduUK3jtX3u7", + "uwN3G74NesSCK3gOUqnT32rjylRnEg/UmVRyyfNQc9wrCKz3r4NdNl2VJMr7V7dxFTCw8w1c5Z6ABowC", + "ZO7EytdnLbUP3a7l896/ljI1mk76Nk1aYiyCVhQImnMIhoVuU56jQWcp8xzOSSzW44h8Od9XYsHvpTY0", + "pKtOeaUXwMKwmuqAnYLUMgA/Rl2WvlcYtL+yrMK8IqwJCq+8HGB3bG54KtwqSZ0xQVG9UeFSSOZlRlAP", + "GYpWKMFkQa4JsozdZvRZ2I2HhciXPp8GI9A9sTaD5R0FRtViMdNyPveRKZ5kfHQ8nZKT8zcXDFMy3Edc", + "J7FWU5b5rT9nX+y/CN3cQhjCPc+/ONi1vhfYA5HFZYhr2QxM61+82GdRdy+XbltLnt9S+y/2Dg7+0vDJ", + "nUE9Vj+0Nd1pZ4ect3D+dMtZS0QQMUwbxQY3VSWwY0DSHeZja05e5bZR0zZvX55DulaYMQQbcXNHcYdB", + "sA1b7UWIpb6ngfvjI0oaUalZoipwPA4gsSM6PoXRjsNDPkKpkWlcGnEvdWXzVcfCo0uWTsmCqzlU5aU1", + "bK6Q+/rQP0owcvChvvE5LU8srHctznE9WiNSnqdVDlcyht7XWxSZeHGzT+VSlrHNxZ1rnd5BStTC6OC6", + "86Lczg7GIJ7fXIKhaKVSlro3rI+i9+kRkBrju0T6JhYDIelrq+avCUcpUrF/NbgHdeDPAKV/QYYDuYgb", + "aIQQTX9zGtavvWbeYYyhE9hXrW6AmOW68NKiFxW18X+ec6XjxBPo6PG0Exjsc5JOjjQ7gdsH/PCYchGS", + "HkhToxvd0ZWfUzM/K/OObpbyArOC/ELgrH3Oyqe2kbKSCjYV5YMQigXjozfrgp2pMyGFLnF0BkUsEe2C", + "bpiYqI7DtlSPy/qkdrYUZi6AmPoTNeXpXVWMjIArve9NV/JeZhXPo8suZKpg8upFI5PheDaTqRSqhABm", + "91nAUcIKVGtBT0ARnFm8TynNGY03MPKJCqljGKhFVlqaUPCH3IkVBdMVXBq7XdvGhGW9EMYA0ogf/ZF0", + "ytQUKm1HaN/diZpQiZviGSCTjMJeYmx2OvY8rq6LgQgTtT8oFlhrBwDge/uXr93tp0ud6nzIwOsSYAXC", + "tWwLrmy9apjC5h0wE9U2W0LZcMtkCTizLIsmuF6Ffs0qszdkZxT8X6f7gg9VWPbV6etIBoc0JpFDjHVc", + "ShgGSNXlowLDKCE0K/mkEEgYVQokSYJi+bDe/CHaMfPc/+qjafqNz+KiNr4wUS+G7DLunWPCOZqWuOez", + "eJeUmhlhdX4fff7lkF352oW51kUQeLATci2HyCbwX5OsHY+N0v5wzd9gmqpXpXd2ggfSsb1zHSJpDi9P", + "AutnA/ZDReFNpT8YIZOP+gH69NmRXiTDiwHSKjH5kahPGyy0yko9F+UCjVjupsagqmkOXrLX67PAVVxw", + "leUig5r0Jl4iunYCC3C9/Gt/d2n9DRQFBeEhKlvnbmcn0ksawja9gAYxO2aIncFe1p2DcWJD361xQPoC", + "el1g3yuQEHyf0Q34oI0tB5BA3FubLqkIX1bccFWKsK3RFhC5wop7S0UPmYksV9vovFpbZiat/xLaxJV4", + "IHMjINEt21SI+RiZU3dbk2K9WqBvEi44dflM0CyCiSAWlBw7MwJlGjyygaqIosh8U1lhHDubyVywP4P7", + "D281HBq4TAYUVkisG+KQwXbkFymmxldVfkeuLEsnAXhrng+0GSjtZJU5s2LJlVdZQayCa+0MrrXekeNk", + "1yuVwhSvg2Tlk9BF4V3ZjVx08ZbijpET6sqkwrJc3gl2vdCFnK36E3WpbTk37iq9PoC4Lq4w0jKsFL05", + "ZIe51exO6QfFuB1Tr00xrz9R7md3V+BtjL4kyEvP6RcxgDwfk2IT4OKv3F2I34E50XS8RY7GM13V0cR3", + "YjX0ur/bAyPE4IGvUBIYT5TfHUdxtON9hpE6lmHoZ1Y/8DdsoI0pWADB4E3T33zZoGodVpumAVtQ8LlP", + "eOjVk7gT7sTsD9m1cPcP/l5ARq5uSjQ1czwIgaTwxI5ZVeAMvblybTLUvmtOBYTpv3A3QyF4CexMOXEN", + "RjFgiiRziBOtGbiX7Fwr23XcKDSCaGp0trr++ykEKlwfnx6/vmE77M3VxZnPCbDs4uro+Iq9+m8mM3Z6", + "cnZyA3gh7OLNm+vjG7abTBRjAyD9OI/g6JVbKJqeG/piNTUyC+5t985VpUjB9+lfC12ZfDXKuMxX2+ik", + "5uHEhIwhPBju3oJxQz46z5ZSjXghR/u7+y8Gu3sjP4HhD1arv0HU7xcyc2x3/7PcqURf7L9sjD4ieg/C", + "RtmuxkKhYU8n+A7NbSpSvQy5uwgbh19GX2wY9QEDHnHK7wTiBABJgVkKYqdYL3HncrS7u7sHY076LPyy", + "738ZDofb+H0UiJFtIcXAaX/gRiy0u7/FW+Bo2PoSFVz6lFQMFqTPrADsBpJ4gLSmK6I9HLgb9dTI9M52", + "UEkm8pLfIrMNhOKk95pSGmvcGiJljwdKwZYEfpzrh4G/y0nPgTSH+nXMxHMk/sqnhXth59qdilxYS8IO", + "bqJ13BNsZ4RMMHDXV8n9CjrGinN3fdSXHGwYnwkU3gClVc5QUzVVQdanljByXRrBlxFKSN8dWcNV5j6E", + "BmW0P+IUvj28Oj85/xLHW8J1SdGTKq0MIA0gW0GqdEuRU4aJcazSaUokvmDYO7IHJ91pla9qPcppXsDB", + "r1D7Am85JuuhWgYbcs+N1JUNNkxC06GAIUdG49FoVPByMSr1CF+EUEkN2BWOzMisNmCJPRiPRtMqvRMl", + "vOIaHi75j1qx6wP3fRpKrFf6CH+iEdgf/Ayzihd2oUMEJ/vaUZw3907UEQT2R4oa6ZKwngQLSAEvXuOs", + "lUS8waRhlZL/rPAWi80A7WpB61YAOOcXrRabkSc2qE4gEN65o4pno6H723CYYF08NwfgOGSPT+v+LQUQ", + "g3fjq63xxWlNN6S111aGDmvBMSqIwRVNklwDLmLWCu7uE7DdelRqf6LqKwQwF7iTzClNEGwPTjqBg3El", + "UlkYoMIrru7Ymwr8T72rqzdBHUdxoBH1DpHmEPp+TVcUePyJy6P53K5Uyd9i5rN+EGZW5Swav1RzOB3g", + "625iMEgnDSRTna3GjiFUpTBgCfTh1bgL2rhWh+dHjvVfXLn/nl/cQMMrd8SjrlaCmzHid+3v7u8iWsvC", + "OKGubjTZwpiyAh5MtoL995pk2TDXc44oRzlX84rXX6KdWtuXMZpWIQ+xFtNDNEUIgvJ4IzAFgvLygjRc", + "+dRl09R57UORgjWi7hDxCP1EvkKSwJHgHk8UhejZVnyd/66/THBq3eTSkTriOrt1nd2SDIP+MqAMHwcc", + "ba/3tU+2/Hej9xD/wl0WHM0NeS7nEBq9RS/5dYsSU25d/7fK6XONRBQKsyWvEUz1GNNx0Sx+heZFhBWn", + "2yOQ7yx4m8JBgF/YJURVUpoLorBFTPJhoS2wRkjDNqW3/Qeix9iBVhYRfoyiOHEdnDI3hhADmIunEXAZ", + "QfegC1G7vf0DdADDv168/Gyy1Ro1HOyJOiyKHJD1G8cXCyIdnh+5OxXjiLsHGPa0ub9/phyQ1bgU6ULp", + "XM9X7M+tk4juUphM7c4AJQlTToBRlZAeKr1bDrImcALwQ3Cchn30EzvGcMFoJ3xs6tpse+cXN/VEt1sz", + "Fb7nTZMNc81EYQS42NjFFeWyjYHd3XuPewjhiB2957oU3khECTPoOEYMRm/ltqkOZGu4upNq3idEHPcz", + "ziXCmfAs/OrwS9a7wruc54PDau7WQ2Tsy4Dkto1h062LSGUe7E3UwerEDE5Pz1DAWedAxn/K9wTNQMaz", + "4Wxxy7CY4eDaCWDH6ObsXV8fb8cccMnNXeZ09Pr7cH6kctcvS2t3TYg3rr8BF34dNA8Xql+TMMbDOcBL", + "TNRJYC4lraXRFdha8MSGWE98WMPgwTK8zrm1jqXbev0sAzqxGCTgJA4fY+5O48CvvxNGihLd4MhvO0WB", + "eFnqJIKAxsp67rINUtM2i7MZTo7oOSXEo/3H70lKY0cbVN97OGzfD6TvhRm3rLSCJ4Hw6Eq58lhMXZZ6", + "D0oF7MAJi5iwiHBONZqT+wD7ZwXR2YHN7g3bl5ePLK8lkT+Hi3obA+eAeSTtm4gijVv3TBJsXEMn6SN9", + "GoGiHbkNHcPdLC25yUx5DqfZ+4cidhPfqEEwCKxr332VDnRgyGSPFSrVmTAEM446FSSFBSLHlEoftzAI", + "Wx6gsYZu6K57z1aWIJGmaWXQK83VmtAQgcpAxi0ZuZeQ1kse50GIpPRDIatjqQv2EqJBtjuz2ztv+Tmm", + "veS8AKeyv9zB/OBa7O3SL4bm0ghnJDozJDGghlynosOk8NlSvhXZ1E1gwOVo+XbK5QC7BJ/Q4H4vzqqv", + "o6caua2hQcpVBnnQtxBegAPdXYvYc7t84GmLXZpKwVbTTayrEjR4ovt6Z1csEyXhGeKmz3kBmguhJ7Cl", + "VHJZLcFiaBc6z7rxxTpXfAnBV4LlghtVp+67xaxUa32XUt3CCG6B5blnu8OX9fryt/R4zovbQpiUAj0P", + "dofrqzFgSau/ZMy+FqJAWcZPP1x3ENNrS/Zf/8vHiZehbBf21vX5ZMyuXTuEwMXY27Qq5b2o15KJt5AE", + "3ugabLt4xJwm4ffvxZAM2NelOzdzuNdeI4LLQj9skqBNxE88VAsaBoyZJaxHEdHb403a2MAzlqyPhlr2", + "IMDzxpbaloCiimGIxs4S10vOYZLXMIXQSUClwHv9Qcj5ghZZ2BqiMV/5+b70puAgLyNjTWLJL/HJWJG0", + "CuEjhRHEH1rSSySxTBQw6UhioWvGY/pNKfMBtCNDv5eRRCe4IXPNzk4jiQkdZZR+VF857gb0Vy1tzBiM", + "3te1ZN6wvrJeva1/DpuK5m5k1F5eKzVT3Bj9gDYEjTEvB5j96XkWBKoH2rVgs8Zu8Mwhg1+6K1FXZS6F", + "sbE9BZb8cYOKRzx80qLyuMHjUXWwNnYI21ouNCtcvQFKbSJ7tq962uSnjS9OfEUbXENk67SlkHEuws0l", + "7RDTz55vRcEunOi41BngWjSydBGaLVhF2nlB5YMOSASYdQTBIkGiqeWSkLkf2VOw94l6dbb/khAT2iMn", + "IRxylId1wizYCRCREuNj2iCczUxQOjmvW7FC6/dHHegeBh5uC8gAxSB4TAWT2dvwcCmWt46pB8CLZhz7", + "BYIbeOAs3zphPY90uj1mJzOAQ+l7EGJcUAkl5LRZ4aXRq6wAJV0bVgqI79puOvfpI25RvfbEeD7XRpaL", + "5WZbFOthXmVtjNrutEax3ro1anvNHMV6a+ao7XV7FOut26O2Q2YNbu2ndj0rvN5hn+FJSgNA1NgnMhTX", + "UUECudZp8lHu/HV9aIhgD4vC6Ldy6Y7g3UAJbhxPVu6qmcISHX59fr7tARRBjFmTipsEft9O0x+ua8+t", + "Df6Km+yBGzHgaSpy0pIyaUto7KMBo8Cg65OzI0inMVUadMm3f/1sdHh2NGaH//ivfXTi/uO/Bi/39vH+", + "IAcxshUaZAwdO2CHV2djdn58cQ4vX58ds951ynOKeiuNfFvjDG6zMFYwRDmieSXLvztNSJU+5hAt8lmV", + "isyT/kzrsjAQhgUAOxBpWKfFnmv25eU3ccSKDtHjrrfXl98Qf8lEketVFM7ckSr4BGeo96iDNYSHDe7w", + "qHjtX3pU0td5zpe8W9LneT5w8nG+jJ5Xhp4uyrIYj0a5k7cW2pbjvb0XBy8wpamVbYPM/kxnRGPuNwxc", + "zPz1Ahnz3mfuAUP7Pt0Nzi7dEfpB2Ujdqq0IIR/H+nxixJCDFHyPU5ast/KrhLZvAagES64qAP6JDy7F", + "vmB1Gh75XHd2jr23v3s6gEYFk/hvSAupa1w8Yzo1TkWYUj+aEKLvNCZxJBT4ZJtjoqAU31Mml3h44BVk", + "SShlu3mWQtVIWz4HDwUzUo+bazPzOXf1zuKuIwKFv6Bw+GNCaig1zpz4yH9eX5xf8nLhfR8+hapF3An8", + "FAqskWDhf//TsLLCDKdOxyKMxrBUY/YVhHhNuZM36Vdk3Zj8VRi9LMrWJtRrO2bH9GczPBi68C8FpkvY", + "MWGZx+yi8HgSEDqqbJCuPm+BjgCemv+WnDG9dFuSNS950M88uHfrPm+RZddetMhzbbRXnuutjRb9rAU3", + "1iOtYRA3/uSnH0rgBPppkExrhEizBDzxczglHt1bGHCDWfr5emg7+jlMGdTtv75osq4wvmuc28cbIC7W", + "c0aILf2PzfF9aySQ3BkyrOPwYdrls0cYmTvrIPthCDLtdJsu4k3HTfL7C7h0ACkM/r/beNZj9q5zL8bs", + "u93hXp/tDvfdfw6+Z+8TxIBokA6Yy/SdUAPUs53E/dRX4gXFB38Bc9LwZZ9Ntl7su3/sD3fZe/pkgLyk", + "IG4Q+zFr40Pm416gHi/cKQUgwujd7zD6Ei8LkFq/TxCDUOmBLj5nbgQaIx/9ULAiAiSg4YqQ1hTYzyVd", + "4iBsNK5SQh4AgD5SpqiajeOPeN3DX4VQXPo8zqnIjE7vkomKeX1RmULbCOI/AT7iR4HCTTJkF+UCwyRx", + "TJQNMVFToyG2M2m9gb6ZUK4Ec7bcgM2Mp8J7/EylAFso5WqiKAgcbKjB7ksSQQT21D6Sj5g1G6LOq8PD", + "k9F0LgZ26aQeoQb3e8OXTphp2iAvYP1YDyNPsCTe9lOfbchYj0tYz5Kv/JAudQE1/tDiPGZJ3Z3bYfDe", + "DoAQBx4oEM21+BsU5qpnVgh1ePLkVIBquqYC6ISB6gcHuJKhJS/k7Z0gf5+9GwyHw3gqZ34K3d0krLf3", + "8uCzbLvf0QLnwXoHu3/Z72rBMz7Y3d0PfQTq+faavUK6f2radDw6txBCi4alLLmKFrs2g0+2EFnN+6AH", + "gttysNc5+82duc3b+HTfPU31QhgxxN+FmufSLgb3B2uPwCaTSzWveO6e+7o9tanmJIDCTFRS3yjwsz/B", + "gWMRzjBnCTkYfxQmGbI3Tkh+0ANbQohbZAjyqDH92jXKIbIFQqNqQQ9xpXktcJKIGTKWCHyGY1AhAbCg", + "tzvmh3zOnW6KIjvBx5/MkEHFgrhPAwSBHDPx3b1PN1KfUFg3XI8TBTZhTJ1ZiGW3yBMg6kj0HLwC3fES", + "pE5g5QhCIH9Ei3gTdyPV6l5AIKdHJKKlqLUHlKM7hNxur8ajAktbLnqeyuk/iC0uMTp1zN69AxvQ+/eT", + "iXrtvf3s3Tvv+YcHR3V/7lnU/fv3v4RO26Gp+p1h1xBQWId4gkays9OlQbglD/XVFuHxDxaq180rmYkR", + "JRhSuj6kE/qd6chwTzmkFmNEcJTojKBKpBOPa2AF37vThN6Wzd6NSIW8x4PRzCHnFo2yb8vYYoEzpYkn", + "794FmLj375MxO8HMN7RxEYA4NPtEEpju+/fD4fDdu5GcwQuvfVAIz1mu5zL17RHrwEmP/g33C36kJLYA", + "sqV/oVK5sLaOMvGv4e80unthIBktfJTe/g84pe/fO2747t1/3IlV+BswlsO/cg7/GLNTrQuE4vNBDzs7", + "ryqZlwOp2FciL4TxMe57Q7azY1NTTb8ql/nODhuwK/RBIAGPbLmC8Im5JfC10vC0xExsqijhlL6vbs5O", + "gWSTJKnpCH559y70zxblMr8lrff9e/8C/N9/2PqyvDgAFDvJ1UQP3JD871DBB98/zDJIksghDgTTvqaQ", + "wStyhEthvaLPMnnfZ4u9weKzPstln4kyxVhxVodYFDmXND2MAzMC0u8yQKcOleXAFbSzI/4JC3fsfbh1", + "LLOPTPZbajeukSPCnvinB8GebGEG+GRr+/37Q/iTCLPV3tED6FmQpAbNPfop0HT8FrGKAzdmSKKGYX8p", + "1Ncgx5dkQoBHZEFxxw9qH655QzbOBF+vTP4FYJ4e8ZJ/c3USBl4/LhfSDqHNbWXyjgYoSDrWRPjswJXg", + "jeEPxRyDqtbegSBqn23io6nxpUJteqkdqL3+EVo8xhxLDdUsvrk6RYBVyhoBKoJ0hDEAw3IrPnvBMJAD", + "S+Kwb65OQrQEtoSPjX4oxPzzKbzQHw6HiafJhKTphI3wbwv/GLBvxdR937ZDlAJ44aoQPogAszFYHZLe", + "FYMOpQz8acJgdMBoPgAEHl4CeqMPdO+hc2lMDdsLfidWWBIEFqxGdX1Ng7sMSAL1uu3snABKLsCz6geF", + "tbr7zAgLZuuenFEqwHa/KVKEhQ09XR69sXibvC0918KkPwSmhxQOI1QGQSzcIkBveN3xM/f6FRx3xwGb", + "/aC0cqZ/lHnOqVVgCkgjVA4XZmx0HujDzwxvz8Jot0GYTeV3zpfS9aEeTgRCFkclWZxSxR4WshS5tCU9", + "vDTy3t09J5fI9uByDyAi19dXbxgvS57eWU9afiiIiwjhNja6pPd2d89e+bZf3dxcsiyMHdRbXZU1lI3P", + "z6yR3j5nPwoT8PPJxyt45hg09QobzoK8tj6G/d0Xfy3e9rEsLxGCp6trIcasuzrsqGHk/MQv6GCQajQ8", + "vADGDefyRmuFDBz+GQo33lxcnAdIrBuw6FwYidGThAB8TshB2xu5Yf0JqM08RITLwIO6H7NcqHm5OOPm", + "TpgvEOvbSQGq/OLFU69mAtZQmC8mW5NJ2cm/YGZvCNuhjtPb2TnYHXy2+7/QfIX+JnT+0JWGh+0/ry/O", + "PUG+pjI+ZPbGY5GM0anYQgDwNAyZ/zhDBCuBcH53kX/3ycH3Y8Zlnxzxy9zLBjd8CvYC2g7Xe6VkDbQQ", + "6o75r1w0nF2np2es4MaSZsgY2bPDwUriFU9Yb6p1vj0GbNlPPHKuU+lg7AjUHhGp06YCu8aNSgDtfXsM", + "GimVMma24Cmcypq8w2th3xLWw6jwbe9rICAnbdB/Q0uBIiY4v1lCe534FtafETL6uqNaVOW4IXKhb5ud", + "tCpoexdvSEFkWB9wzP6TK8GONB7e7g1bo7WAbRhdkEB/140a2uEcR9WzS63VAHcc/qa3v9TsxM0plIju", + "fJnnxZ2wd1KN5hpfnqhvCayOB+USCvcEbxsSqwQbbhZ8SOClKhFZTJrIjSQt8zhZIhs2gLyP8aZwKwq3", + "s/cP+WLM/otuY5cCPLfe5Nl2JSG+Mnan5oT5j1I3VeGrEc/DKfhwnTlSjoN/6mnH7Cb1dbPm2tJaw2yJ", + "UIOLJ5QZgcoFuiBsm1C/LsH0BvKWsVBnkEDZ4b4ojCiEu5OSPw0TJrE4IiUZ+0KMfxp2uONYZGKvt47a", + "Q1XB73a/H6JfJAk8TUbGpxuoKuTFBSQsnwPOKe1j6IG2ycdH2gJWPoAUtxDAgxmZIY2rWS2wLvzd8Oz5", + "FQ3ZTBI4AUhEwTT206kkrNYvbt0416UYxw4+d/LIrYgBjTLPEcqy7YiM7coUR907+OuLuohHczXwsHo4", + "TW8P+hnGp72XB591mpeCVennmYtim/YzrdqdAcqX9Q2A1nR2Lcqq+EhTj6iGCgb9ojMMbRuW+z+9u7g8", + "Pj88uT28PLn9+vi/33cuRSce/s4OxfrWguqDtCIH+K6vEHw9PLIh3BHD7NMVBNQHJHJSoEYUUwZg7YTS", + "AkXhg/mro55qHNra4gFoc14K7hjErMopIIB6RlXv3rLXua4AuYzcQeggvxe5LtwVMiqcEpGu+ix1DSPH", + "GGRzBioZ2ZR7uDASUuLrHBK7W6EsEItd23sDwtTMx1Pwey0z9rDQuagzGRpAlHX2Cd7+rp/DGOH/OoSu", + "3UCCCTZrVAEYrdcdbRYGoCCfvSE7wozAGLZvHfEe1OxO5NBHC3vTUv7yhb298/4XqOvdWTLjZ4CS7/sK", + "T1HIK9ypncsbpX6GVY2jUDeUHIyCWDeP5MDHv8OmO2VgQEqAr1dapovOUXWnv3re5V6+hZdb44syCnNe", + "NOC3mwV0QlZoZ9HE7/x+f79pbiB8dtVtAFr29WPqGiOZ135sNY2rlWKWpi8KIVVRlRPFQXI2WF0fHWlk", + "uev7ifn6EfjvUhdISjkvHFdHa2xI0wuOsiNCsPjWfXrBi0IoiwxlpasAEGjwhb955zoq49b/jjim6NsC", + "sDEmlJHpwp3Rfhu5jWAVotxsD5RYl2AYsq+EEZ+6kfASAUALkZY4sPqjCCTEelQBwrM/JR7IzQdgG4SN", + "2bBbbIeg8UP/1rl4oIioUrNjj2Z5g+CNIRi9Y+bN6W1ah5r5OdUXS75GB3DK7WKisBrEeh2IEa3U6B38", + "Ab89T3BoCALD4XCdMRyHIYb9b8yIvD5hRZr+R5wg1vZq7qHHGY3iiurGNdRoG6Y3AkLdANMLeaEAP0yl", + "R9Cw7+5oRM51fcyNsDawnLPWg+aqA37Pcxedlu5U6zvGS5YANNgtjoFK9tQL6r8HYMvQss6sNkYbdBz6", + "ypeHQDL1foQKVV2RoKz5JSgD2m+fT+E+YhHKGPCKxx6mtjSrNqjyGy5zkUWddtdMKY30bkjxFtHjHaNy", + "G69ns0e2s4YUbG2oh7/GvhFrtp6Z04akCnlGEFUU5SrdC4DCsXopYBEo3DRgKMO0IhjCDsrpJesU0N79", + "9wkCe8OKkp0KMXr1UpQLUVkPwQyrYkV7cyrIfCPAe1rqNppvjOF0L3n7sgYzfYhqyLZ9nslyqZWfJrtO", + "heJG6oYfslbDDi9P2DcqhFkFw1G04LQNrY0H1muFuZep8Hh9Nli7W/PwBf96vohBzZm2O3OGqCPQ2t0p", + "uAWbxDgmRp8A56X7mE59aRFAfIQC2HSHH7II+5KkbFFDDxoI7LZ1Zd9hIBfy7wRIeU49Bvl5wT3EJCaM", + "lzDxkrKkABlcm5Iloc2tEQWX5tYnDiTMauq0zndJubvgXDsy6ht5LzLEWBsGv+iVOzoAqwyBA97+G3xB", + "dBgp8a80q2BUxgJYZE6GMDKvrwQ9Adzs6C7wJl9TK1mOgP5Z6ZLE9po1Be3SO0IhgAn5HFTohKIVKND0", + "Y7/IdpjAaQRsHiXZI+K2uhe2lHMeudyuiFJtqQvapiV/y3jpVL2yTZ2hZL67qkIdV6DOBsuFEjJeakCJ", + "YA2g/EYTPqCPr4wbYPLjkdEFlcCmmxMvfJYcHZ8e3xw/yXBgr68ESo64KE6ywU+NWdIhLHR1cjCM7m5w", + "wNg1nj7wNzOyFDAL7exExW+9pNaLcuy2vRDUQA7Xqg42hVK88acmyoNqITtB/Q9XkrLoeSaG3lBefzWK", + "l9qm7WrD2q4LkXWBLkCOeoVM4oz/oE0kArtWTp6GiEFYIStK2/d4U8Raanlv/Bx5DV8OIhr+kypUTbaw", + "zwH1OaBqMHV2PpVyJR3yYDwaLVfk/SWAtAAu2I5GZV8aXlCuIyUlH2dzCBJyDfBpva5QAhaWFA4EwJ9K", + "rexCFmjU88F0peGO4dfoI0N2Ku8iHJg+g76gQJ2dKF/crrOWm5eRC2GwcjcWKIjPUF0at54Pu7h39494", + "WJ8J4oGhX8HJMG4QIdU1GiPM08O6SEI6JcwslRHoqhuJEoSBhrB8oQ8oCoJrEczFg4UuvId/EAEhxauJ", + "sG5OUo/Cqub1HDpV3lqyh5ZdyaE6lTy/pcdt26Pf+ceK00Ub5FrHJQmEiUfoXZBx7YjkCW/IpmEbkQpV", + "YrUJD4gcggWj/uuqWU8WWTuGLf/AAmuBUzUqrf3cAmk8xWMAIgKPkMTvRLNgzLY7VhhO4Cv3PzipAmsd", + "xcXWkKDXC62tHcu2avVYhTXq9AOrqwWiureNwo64M/EvOzusV9bVE7YR0SfIryDigishLm0YF02gCyKu", + "4fFkkYX1Mnna1PXeoBQGmk0plYdm48aK3HjzWJuk774cHw9kX/FYsXy+pnWGgfKyRiiLXg4DB7JZp0rW", + "o8QPqRjQD6IDD9iRR5iIexMoRkIB0wBBwRrlwB4pGxb2F1JfApZyAI9+onIY47aW3W5uTn+NUmCP1v2C", + "i8L+RP6KsYK3VGnrMSa11+C/N+5cM3wb8cJ9F58TOTQ4FVUGg1rX8JIHMgKcereSh/QreyNE9hMn83O4", + "bg0k4ygi57aEmmtuyDC9uNPPmc6dkkH8patO11rNsisB2McKq9r91O0qjYBb/Da6wR+ZItQ5a2+a7yPE", + "jEV9fc6MmBlhFwwQot2dnhNmkM6zxs767Ys1tZ9KhIW8dav3oeTnVDV4D65yt02PER84xXJJzlqV6YdI", + "E0SV+lnF1AL/8Hdnk4/UufIeRrWDY7RKslG5M7rpARASSqQhu2zcr7Z5wVK+fui+XX8tFI90/fb2tmGl", + "oFeqQ2EExBM2BYRnlWWDXmqA6A+vrdYUx+Kybh6Uuq6utiYHbK6wRgL6h1dXg+3zxZHam7a53NrmomoI", + "vRCSbR4roIZk6zYHRJjt7lppgfS6C6ZFZfsQrrxJdF2F00SbmH/QU3/J3jqV/LYs8y8OPrR+Gk4Ha6ft", + "7f+lLp721/+9Vjvt5YuD/SD6npGNEeRGkbVldLhsbwBrKz7oGbPSnVaqqjEVqdtKioN3++bYuZ9h4CtI", + "QcoN/d42hh7sn+1ibrGqubGGW1vrGbDXCC8Opgg4j5ZlRiO0OJqwQuXs4DipZX6qdROlEqAwjaYJ6gqP", + "+GXOFeNpCkVI5lQYICoasAbY0VHPDew60ai8Ccgbk5pSYij1RhppqpdA/0uuyNy0uXzbY18CU9D6p4Kl", + "R3QvzSMl2S6DMr5W5Qw0mljWdYqNLC3TDwq2l6KpCYQ+m/t6aXUtOCrp1iXhsh4kOfvDWL9DGUznGgwF", + "Ikjq42hFoaJWPDS74JTxB/wqlnoJVB+aeZJqVZ17zZUbzBIoGlmJ6qTcR8rYYXQ70fH5zSUQWQCGjEva", + "d9ayI5vVWrCMk2iRfzXQtxoCvn22Pz+Q4G3DSrHRuR8sDM3S1CRSepn1qddj5/sGgbPdP88dQdyiJ/Bn", + "9b+32ByY4HSHEPbTVKWazvB6uWNLiVNd5dqGQGyHu3IbSzxseKgC28313JsyvoUIAKhZIueLmONGxYHo", + "PeAvGYgqMakzOQs9+wIhnE11WeZCifSOtFt0BtiFNlDJq1UAkVBt5FIMrFBWemXGoyOiBG9DymnNSGki", + "/xDGHaO2YWlK4kaIWEDvfo3yBJ360cRAhsD5jKBAU0RkbFxIA3YMXsYOwwjhJravYIOhyDEgIOz843UW", + "Cbzo6TqLTwMC3gZAwNrI7YkPYfxk2uVncdtW5Rk7cVvSBAUkozyfkwUYnaZgF3INX3NF1Rs9jdf2Mipb", + "FYxQnXiAF3GRDCyYVWoMUKlX8RsrTMfKwc/P7dJxKYl6c93xZfixo/vo4XM/4tQkJ4A3Npy+dVjIr8Wq", + "4zvuJffkuR8x+oHuxJCCRBiX9ceu9APqBx3fu9IPpDw8+5OvuZ1KxYymcKO50VAoeDkVxnGi+MO56Pxm", + "Ljq+di3nCkxRtuQ5gnVAMQGelpZKf4V6OsNo23h65yiya9P8o46in1hY5PrvpwNMmKXPigwroVNu00yk", + "qzQX0edOfLvO0+v7CGBxduv9965hyedfunVyh/ddoAKYy1Z/q+RzONV0ZTd5REd51vVSLR1go2vsBk6c", + "/3RVLqIP06FqHIFAozX9xD0EIJS6m/hxtAL1Z6K9qtfx+/ffv/8/AQAA//8=", } // decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, diff --git a/openapi.yaml b/openapi.yaml index e873cbe469..18fea57ced 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4719,6 +4719,196 @@ paths: - BasicAuth: [] - ApiKeyAuth: [] - BearerAuth: [] + /db/v1/tables/{tableName}/storage/migrations: + parameters: + - name: tableName + in: path + required: true + schema: + type: string + post: + operationId: createTableStorageMigration + summary: Create or resume a table storage migration job + description: > + Table-admin operation for local single-shard standalone tables. Target + + vector_store changes primary_lsm source ownership without changing models, + + dimensions, artifacts or logical indexes. Retry creation with the same + + job_id, target and budgets. The job is advanced explicitly through its + + job endpoint; the server does not schedule an unattended migration loop. + + Offline migration uses antfly storage migrate against a stopped server. + tags: + - data_operations + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: false + required: + - job_id + - target + properties: + job_id: + type: string + pattern: ^[A-Za-z0-9_-]{1,128}$ + target: + type: string + enum: + - vector_store + budget: + type: object + additionalProperties: false + properties: + batch_bytes: + type: integer + format: int64 + minimum: 4096 + maximum: 67108864 + default: 4194304 + batch_rows: + type: integer + minimum: 1 + maximum: 65536 + default: 1024 + temporary_bytes: + type: integer + format: int64 + default: 68719476736 + disk_reserve_bytes: + type: integer + format: int64 + default: 1073741824 + responses: + '200': + description: Durable migration receipt, or admitted receipt before DB preparation + content: + application/json: + schema: + type: object + additionalProperties: true + '400': + $ref: '#/components/responses/BadRequest' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: Conflicting job, lifecycle operation or publication state + '503': + description: Retryable resource or recovery admission failure + '500': + $ref: '#/components/responses/InternalServerError' + security: + - BasicAuth: [] + - ApiKeyAuth: [] + - BearerAuth: [] + /db/v1/tables/{tableName}/storage/migrations/{jobId}: + parameters: + - name: tableName + in: path + required: true + schema: + type: string + - name: jobId + in: path + required: true + schema: + type: string + pattern: ^[A-Za-z0-9_-]{1,128}$ + get: + operationId: getTableStorageMigration + summary: Read a table storage migration receipt + description: > + Table-admin observation only. Does not admit, advance, or publish a job. + + A phase of admitted means catalog admission is durable but DB preparation + + has not begun; retry creation or send a job action to recover it. The + + receipt is retained until a later migration replaces it; this endpoint + + is not a permanent job history. + tags: + - data_operations + responses: + '200': + description: Durable migration receipt, or admitted receipt before DB preparation + content: + application/json: + schema: + type: object + additionalProperties: true + '400': + $ref: '#/components/responses/BadRequest' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: Conflicting job, lifecycle operation or publication state + '503': + description: Retryable resource or recovery admission failure + '500': + $ref: '#/components/responses/InternalServerError' + security: + - BasicAuth: [] + - ApiKeyAuth: [] + - BearerAuth: [] + post: + operationId: advanceTableStorageMigration + summary: Advance, publish or cancel a table storage migration job + description: > + Uses the job's durable configuration and budgets. Each step commits + + bounded progress. Publish is accepted only at ready; complete additionally + + certifies reference-only primary artifacts and native ANN serving. + + Cancellation is allowed only before publication. Repeating an action + + after an ambiguous response resumes the durable job. + tags: + - data_operations + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: false + required: + - action + properties: + action: + type: string + enum: + - step + - publish + - cancel + responses: + '200': + description: Durable migration receipt, or admitted receipt before DB preparation + content: + application/json: + schema: + type: object + additionalProperties: true + '400': + $ref: '#/components/responses/BadRequest' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: Conflicting job, lifecycle operation or publication state + '503': + description: Retryable resource or recovery admission failure + '500': + $ref: '#/components/responses/InternalServerError' + security: + - BasicAuth: [] + - ApiKeyAuth: [] + - BearerAuth: [] /db/v1/tables/{tableName}/repair/run: parameters: - name: tableName diff --git a/py/packages/sdk/src/antfly/client_generated/api/data_operations/advance_table_storage_migration.py b/py/packages/sdk/src/antfly/client_generated/api/data_operations/advance_table_storage_migration.py new file mode 100644 index 0000000000..201b2a2301 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/api/data_operations/advance_table_storage_migration.py @@ -0,0 +1,238 @@ +from http import HTTPStatus +from typing import Any, cast +from urllib.parse import quote + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.advance_table_storage_migration_body import AdvanceTableStorageMigrationBody +from ...models.advance_table_storage_migration_response_200 import AdvanceTableStorageMigrationResponse200 +from ...models.error import Error +from ...types import Response + + +def _get_kwargs( + table_name: str, + job_id: str, + *, + body: AdvanceTableStorageMigrationBody, +) -> dict[str, Any]: + headers: dict[str, Any] = {} + + _kwargs: dict[str, Any] = { + "method": "post", + "url": "/db/v1/tables/{table_name}/storage/migrations/{job_id}".format( + table_name=quote(str(table_name), safe=""), + job_id=quote(str(job_id), safe=""), + ), + } + + _kwargs["json"] = body.to_dict() + + headers["Content-Type"] = "application/json" + + _kwargs["headers"] = headers + return _kwargs + + +def _parse_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> AdvanceTableStorageMigrationResponse200 | Any | Error | None: + if response.status_code == 200: + response_200 = AdvanceTableStorageMigrationResponse200.from_dict(response.json()) + + return response_200 + + if response.status_code == 400: + response_400 = Error.from_dict(response.json()) + + return response_400 + + if response.status_code == 404: + response_404 = Error.from_dict(response.json()) + + return response_404 + + if response.status_code == 409: + response_409 = cast(Any, None) + return response_409 + + if response.status_code == 500: + response_500 = Error.from_dict(response.json()) + + return response_500 + + if response.status_code == 503: + response_503 = cast(Any, None) + return response_503 + + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Response[AdvanceTableStorageMigrationResponse200 | Any | Error]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + table_name: str, + job_id: str, + *, + client: AuthenticatedClient, + body: AdvanceTableStorageMigrationBody, +) -> Response[AdvanceTableStorageMigrationResponse200 | Any | Error]: + """Advance, publish or cancel a table storage migration job + + Uses the job's durable configuration and budgets. Each step commits + bounded progress. Publish is accepted only at ready; complete additionally + certifies reference-only primary artifacts and native ANN serving. + Cancellation is allowed only before publication. Repeating an action + after an ambiguous response resumes the durable job. + + Args: + table_name (str): + job_id (str): + body (AdvanceTableStorageMigrationBody): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[AdvanceTableStorageMigrationResponse200 | Any | Error] + """ + + kwargs = _get_kwargs( + table_name=table_name, + job_id=job_id, + body=body, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + table_name: str, + job_id: str, + *, + client: AuthenticatedClient, + body: AdvanceTableStorageMigrationBody, +) -> AdvanceTableStorageMigrationResponse200 | Any | Error | None: + """Advance, publish or cancel a table storage migration job + + Uses the job's durable configuration and budgets. Each step commits + bounded progress. Publish is accepted only at ready; complete additionally + certifies reference-only primary artifacts and native ANN serving. + Cancellation is allowed only before publication. Repeating an action + after an ambiguous response resumes the durable job. + + Args: + table_name (str): + job_id (str): + body (AdvanceTableStorageMigrationBody): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + AdvanceTableStorageMigrationResponse200 | Any | Error + """ + + return sync_detailed( + table_name=table_name, + job_id=job_id, + client=client, + body=body, + ).parsed + + +async def asyncio_detailed( + table_name: str, + job_id: str, + *, + client: AuthenticatedClient, + body: AdvanceTableStorageMigrationBody, +) -> Response[AdvanceTableStorageMigrationResponse200 | Any | Error]: + """Advance, publish or cancel a table storage migration job + + Uses the job's durable configuration and budgets. Each step commits + bounded progress. Publish is accepted only at ready; complete additionally + certifies reference-only primary artifacts and native ANN serving. + Cancellation is allowed only before publication. Repeating an action + after an ambiguous response resumes the durable job. + + Args: + table_name (str): + job_id (str): + body (AdvanceTableStorageMigrationBody): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[AdvanceTableStorageMigrationResponse200 | Any | Error] + """ + + kwargs = _get_kwargs( + table_name=table_name, + job_id=job_id, + body=body, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + table_name: str, + job_id: str, + *, + client: AuthenticatedClient, + body: AdvanceTableStorageMigrationBody, +) -> AdvanceTableStorageMigrationResponse200 | Any | Error | None: + """Advance, publish or cancel a table storage migration job + + Uses the job's durable configuration and budgets. Each step commits + bounded progress. Publish is accepted only at ready; complete additionally + certifies reference-only primary artifacts and native ANN serving. + Cancellation is allowed only before publication. Repeating an action + after an ambiguous response resumes the durable job. + + Args: + table_name (str): + job_id (str): + body (AdvanceTableStorageMigrationBody): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + AdvanceTableStorageMigrationResponse200 | Any | Error + """ + + return ( + await asyncio_detailed( + table_name=table_name, + job_id=job_id, + client=client, + body=body, + ) + ).parsed diff --git a/py/packages/sdk/src/antfly/client_generated/api/data_operations/create_table_storage_migration.py b/py/packages/sdk/src/antfly/client_generated/api/data_operations/create_table_storage_migration.py new file mode 100644 index 0000000000..d43baa59c6 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/api/data_operations/create_table_storage_migration.py @@ -0,0 +1,228 @@ +from http import HTTPStatus +from typing import Any, cast +from urllib.parse import quote + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.create_table_storage_migration_body import CreateTableStorageMigrationBody +from ...models.create_table_storage_migration_response_200 import CreateTableStorageMigrationResponse200 +from ...models.error import Error +from ...types import Response + + +def _get_kwargs( + table_name: str, + *, + body: CreateTableStorageMigrationBody, +) -> dict[str, Any]: + headers: dict[str, Any] = {} + + _kwargs: dict[str, Any] = { + "method": "post", + "url": "/db/v1/tables/{table_name}/storage/migrations".format( + table_name=quote(str(table_name), safe=""), + ), + } + + _kwargs["json"] = body.to_dict() + + headers["Content-Type"] = "application/json" + + _kwargs["headers"] = headers + return _kwargs + + +def _parse_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Any | CreateTableStorageMigrationResponse200 | Error | None: + if response.status_code == 200: + response_200 = CreateTableStorageMigrationResponse200.from_dict(response.json()) + + return response_200 + + if response.status_code == 400: + response_400 = Error.from_dict(response.json()) + + return response_400 + + if response.status_code == 404: + response_404 = Error.from_dict(response.json()) + + return response_404 + + if response.status_code == 409: + response_409 = cast(Any, None) + return response_409 + + if response.status_code == 500: + response_500 = Error.from_dict(response.json()) + + return response_500 + + if response.status_code == 503: + response_503 = cast(Any, None) + return response_503 + + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Response[Any | CreateTableStorageMigrationResponse200 | Error]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + table_name: str, + *, + client: AuthenticatedClient, + body: CreateTableStorageMigrationBody, +) -> Response[Any | CreateTableStorageMigrationResponse200 | Error]: + """Create or resume a table storage migration job + + Table-admin operation for local single-shard standalone tables. Target + vector_store changes primary_lsm source ownership without changing models, + dimensions, artifacts or logical indexes. Retry creation with the same + job_id, target and budgets. The job is advanced explicitly through its + job endpoint; the server does not schedule an unattended migration loop. + Offline migration uses antfly storage migrate against a stopped server. + + Args: + table_name (str): + body (CreateTableStorageMigrationBody): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any | CreateTableStorageMigrationResponse200 | Error] + """ + + kwargs = _get_kwargs( + table_name=table_name, + body=body, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + table_name: str, + *, + client: AuthenticatedClient, + body: CreateTableStorageMigrationBody, +) -> Any | CreateTableStorageMigrationResponse200 | Error | None: + """Create or resume a table storage migration job + + Table-admin operation for local single-shard standalone tables. Target + vector_store changes primary_lsm source ownership without changing models, + dimensions, artifacts or logical indexes. Retry creation with the same + job_id, target and budgets. The job is advanced explicitly through its + job endpoint; the server does not schedule an unattended migration loop. + Offline migration uses antfly storage migrate against a stopped server. + + Args: + table_name (str): + body (CreateTableStorageMigrationBody): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Any | CreateTableStorageMigrationResponse200 | Error + """ + + return sync_detailed( + table_name=table_name, + client=client, + body=body, + ).parsed + + +async def asyncio_detailed( + table_name: str, + *, + client: AuthenticatedClient, + body: CreateTableStorageMigrationBody, +) -> Response[Any | CreateTableStorageMigrationResponse200 | Error]: + """Create or resume a table storage migration job + + Table-admin operation for local single-shard standalone tables. Target + vector_store changes primary_lsm source ownership without changing models, + dimensions, artifacts or logical indexes. Retry creation with the same + job_id, target and budgets. The job is advanced explicitly through its + job endpoint; the server does not schedule an unattended migration loop. + Offline migration uses antfly storage migrate against a stopped server. + + Args: + table_name (str): + body (CreateTableStorageMigrationBody): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any | CreateTableStorageMigrationResponse200 | Error] + """ + + kwargs = _get_kwargs( + table_name=table_name, + body=body, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + table_name: str, + *, + client: AuthenticatedClient, + body: CreateTableStorageMigrationBody, +) -> Any | CreateTableStorageMigrationResponse200 | Error | None: + """Create or resume a table storage migration job + + Table-admin operation for local single-shard standalone tables. Target + vector_store changes primary_lsm source ownership without changing models, + dimensions, artifacts or logical indexes. Retry creation with the same + job_id, target and budgets. The job is advanced explicitly through its + job endpoint; the server does not schedule an unattended migration loop. + Offline migration uses antfly storage migrate against a stopped server. + + Args: + table_name (str): + body (CreateTableStorageMigrationBody): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Any | CreateTableStorageMigrationResponse200 | Error + """ + + return ( + await asyncio_detailed( + table_name=table_name, + client=client, + body=body, + ) + ).parsed diff --git a/py/packages/sdk/src/antfly/client_generated/api/data_operations/get_table_storage_migration.py b/py/packages/sdk/src/antfly/client_generated/api/data_operations/get_table_storage_migration.py new file mode 100644 index 0000000000..3d9404ac10 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/api/data_operations/get_table_storage_migration.py @@ -0,0 +1,217 @@ +from http import HTTPStatus +from typing import Any, cast +from urllib.parse import quote + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.error import Error +from ...models.get_table_storage_migration_response_200 import GetTableStorageMigrationResponse200 +from ...types import Response + + +def _get_kwargs( + table_name: str, + job_id: str, +) -> dict[str, Any]: + + _kwargs: dict[str, Any] = { + "method": "get", + "url": "/db/v1/tables/{table_name}/storage/migrations/{job_id}".format( + table_name=quote(str(table_name), safe=""), + job_id=quote(str(job_id), safe=""), + ), + } + + return _kwargs + + +def _parse_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Any | Error | GetTableStorageMigrationResponse200 | None: + if response.status_code == 200: + response_200 = GetTableStorageMigrationResponse200.from_dict(response.json()) + + return response_200 + + if response.status_code == 400: + response_400 = Error.from_dict(response.json()) + + return response_400 + + if response.status_code == 404: + response_404 = Error.from_dict(response.json()) + + return response_404 + + if response.status_code == 409: + response_409 = cast(Any, None) + return response_409 + + if response.status_code == 500: + response_500 = Error.from_dict(response.json()) + + return response_500 + + if response.status_code == 503: + response_503 = cast(Any, None) + return response_503 + + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Response[Any | Error | GetTableStorageMigrationResponse200]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + table_name: str, + job_id: str, + *, + client: AuthenticatedClient, +) -> Response[Any | Error | GetTableStorageMigrationResponse200]: + """Read a table storage migration receipt + + Table-admin observation only. Does not admit, advance, or publish a job. + A phase of admitted means catalog admission is durable but DB preparation + has not begun; retry creation or send a job action to recover it. The + receipt is retained until a later migration replaces it; this endpoint + is not a permanent job history. + + Args: + table_name (str): + job_id (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any | Error | GetTableStorageMigrationResponse200] + """ + + kwargs = _get_kwargs( + table_name=table_name, + job_id=job_id, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + table_name: str, + job_id: str, + *, + client: AuthenticatedClient, +) -> Any | Error | GetTableStorageMigrationResponse200 | None: + """Read a table storage migration receipt + + Table-admin observation only. Does not admit, advance, or publish a job. + A phase of admitted means catalog admission is durable but DB preparation + has not begun; retry creation or send a job action to recover it. The + receipt is retained until a later migration replaces it; this endpoint + is not a permanent job history. + + Args: + table_name (str): + job_id (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Any | Error | GetTableStorageMigrationResponse200 + """ + + return sync_detailed( + table_name=table_name, + job_id=job_id, + client=client, + ).parsed + + +async def asyncio_detailed( + table_name: str, + job_id: str, + *, + client: AuthenticatedClient, +) -> Response[Any | Error | GetTableStorageMigrationResponse200]: + """Read a table storage migration receipt + + Table-admin observation only. Does not admit, advance, or publish a job. + A phase of admitted means catalog admission is durable but DB preparation + has not begun; retry creation or send a job action to recover it. The + receipt is retained until a later migration replaces it; this endpoint + is not a permanent job history. + + Args: + table_name (str): + job_id (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any | Error | GetTableStorageMigrationResponse200] + """ + + kwargs = _get_kwargs( + table_name=table_name, + job_id=job_id, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + table_name: str, + job_id: str, + *, + client: AuthenticatedClient, +) -> Any | Error | GetTableStorageMigrationResponse200 | None: + """Read a table storage migration receipt + + Table-admin observation only. Does not admit, advance, or publish a job. + A phase of admitted means catalog admission is durable but DB preparation + has not begun; retry creation or send a job action to recover it. The + receipt is retained until a later migration replaces it; this endpoint + is not a permanent job history. + + Args: + table_name (str): + job_id (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Any | Error | GetTableStorageMigrationResponse200 + """ + + return ( + await asyncio_detailed( + table_name=table_name, + job_id=job_id, + client=client, + ) + ).parsed diff --git a/py/packages/sdk/src/antfly/client_generated/models/__init__.py b/py/packages/sdk/src/antfly/client_generated/models/__init__.py index 6d0a38a10c..868074a9f4 100644 --- a/py/packages/sdk/src/antfly/client_generated/models/__init__.py +++ b/py/packages/sdk/src/antfly/client_generated/models/__init__.py @@ -1,5 +1,8 @@ """Contains all the data models used in inputs/outputs""" +from .advance_table_storage_migration_body import AdvanceTableStorageMigrationBody +from .advance_table_storage_migration_body_action import AdvanceTableStorageMigrationBodyAction +from .advance_table_storage_migration_response_200 import AdvanceTableStorageMigrationResponse200 from .agent_decision import AgentDecision from .agent_question import AgentQuestion from .agent_question_kind import AgentQuestionKind @@ -137,6 +140,10 @@ from .create_index_common import CreateIndexCommon from .create_table_request import CreateTableRequest from .create_table_request_indexes import CreateTableRequestIndexes +from .create_table_storage_migration_body import CreateTableStorageMigrationBody +from .create_table_storage_migration_body_budget import CreateTableStorageMigrationBodyBudget +from .create_table_storage_migration_body_target import CreateTableStorageMigrationBodyTarget +from .create_table_storage_migration_response_200 import CreateTableStorageMigrationResponse200 from .create_user_request import CreateUserRequest from .create_user_request_metadata_type_0 import CreateUserRequestMetadataType0 from .created_algebraic_index import CreatedAlgebraicIndex @@ -323,6 +330,7 @@ from .get_current_user_response_200 import GetCurrentUserResponse200 from .get_current_user_response_200_metadata_type_0 import GetCurrentUserResponse200MetadataType0 from .get_document_artifact_manifest_detail import GetDocumentArtifactManifestDetail +from .get_table_storage_migration_response_200 import GetTableStorageMigrationResponse200 from .global_stateful_query_request import GlobalStatefulQueryRequest from .google_embedder_config import GoogleEmbedderConfig from .google_embedder_config_provider import GoogleEmbedderConfigProvider @@ -1060,6 +1068,9 @@ from .you_search_config import YouSearchConfig __all__ = ( + "AdvanceTableStorageMigrationBody", + "AdvanceTableStorageMigrationBodyAction", + "AdvanceTableStorageMigrationResponse200", "AgentDecision", "AgentQuestion", "AgentQuestionKind", @@ -1216,6 +1227,10 @@ "CreateIndexCommon", "CreateTableRequest", "CreateTableRequestIndexes", + "CreateTableStorageMigrationBody", + "CreateTableStorageMigrationBodyBudget", + "CreateTableStorageMigrationBodyTarget", + "CreateTableStorageMigrationResponse200", "CreateUserRequest", "CreateUserRequestMetadataType0", "Credentials", @@ -1381,6 +1396,7 @@ "GetCurrentUserResponse200", "GetCurrentUserResponse200MetadataType0", "GetDocumentArtifactManifestDetail", + "GetTableStorageMigrationResponse200", "GlobalStatefulQueryRequest", "GoogleEmbedderConfig", "GoogleEmbedderConfigProvider", diff --git a/py/packages/sdk/src/antfly/client_generated/models/advance_table_storage_migration_body.py b/py/packages/sdk/src/antfly/client_generated/models/advance_table_storage_migration_body.py new file mode 100644 index 0000000000..0adaf1fdfb --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/advance_table_storage_migration_body.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define + +from ..models.advance_table_storage_migration_body_action import AdvanceTableStorageMigrationBodyAction + +T = TypeVar("T", bound="AdvanceTableStorageMigrationBody") + + +@_attrs_define +class AdvanceTableStorageMigrationBody: + """ + Attributes: + action (AdvanceTableStorageMigrationBodyAction): + """ + + action: AdvanceTableStorageMigrationBodyAction + + def to_dict(self) -> dict[str, Any]: + action = self.action.value + + field_dict: dict[str, Any] = {} + + field_dict.update( + { + "action": action, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + action = AdvanceTableStorageMigrationBodyAction(d.pop("action")) + + advance_table_storage_migration_body = cls( + action=action, + ) + + return advance_table_storage_migration_body diff --git a/py/packages/sdk/src/antfly/client_generated/models/advance_table_storage_migration_body_action.py b/py/packages/sdk/src/antfly/client_generated/models/advance_table_storage_migration_body_action.py new file mode 100644 index 0000000000..6be50b23ea --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/advance_table_storage_migration_body_action.py @@ -0,0 +1,10 @@ +from enum import StrEnum + + +class AdvanceTableStorageMigrationBodyAction(StrEnum): + CANCEL = "cancel" + PUBLISH = "publish" + STEP = "step" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/advance_table_storage_migration_response_200.py b/py/packages/sdk/src/antfly/client_generated/models/advance_table_storage_migration_response_200.py new file mode 100644 index 0000000000..759c2eaaa2 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/advance_table_storage_migration_response_200.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="AdvanceTableStorageMigrationResponse200") + + +@_attrs_define +class AdvanceTableStorageMigrationResponse200: + """ """ + + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + advance_table_storage_migration_response_200 = cls() + + advance_table_storage_migration_response_200.additional_properties = d + return advance_table_storage_migration_response_200 + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_body.py b/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_body.py new file mode 100644 index 0000000000..0c9c3eb022 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_body.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define + +from ..models.create_table_storage_migration_body_target import CreateTableStorageMigrationBodyTarget +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.create_table_storage_migration_body_budget import CreateTableStorageMigrationBodyBudget + + +T = TypeVar("T", bound="CreateTableStorageMigrationBody") + + +@_attrs_define +class CreateTableStorageMigrationBody: + """ + Attributes: + job_id (str): + target (CreateTableStorageMigrationBodyTarget): + budget (CreateTableStorageMigrationBodyBudget | Unset): + """ + + job_id: str + target: CreateTableStorageMigrationBodyTarget + budget: CreateTableStorageMigrationBodyBudget | Unset = UNSET + + def to_dict(self) -> dict[str, Any]: + job_id = self.job_id + + target = self.target.value + + budget: dict[str, Any] | Unset = UNSET + if not isinstance(self.budget, Unset): + budget = self.budget.to_dict() + + field_dict: dict[str, Any] = {} + + field_dict.update( + { + "job_id": job_id, + "target": target, + } + ) + if budget is not UNSET: + field_dict["budget"] = budget + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.create_table_storage_migration_body_budget import CreateTableStorageMigrationBodyBudget + + d = dict(src_dict) + job_id = d.pop("job_id") + + target = CreateTableStorageMigrationBodyTarget(d.pop("target")) + + _budget = d.pop("budget", UNSET) + budget: CreateTableStorageMigrationBodyBudget | Unset + if isinstance(_budget, Unset): + budget = UNSET + else: + budget = CreateTableStorageMigrationBodyBudget.from_dict(_budget) + + create_table_storage_migration_body = cls( + job_id=job_id, + target=target, + budget=budget, + ) + + return create_table_storage_migration_body diff --git a/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_body_budget.py b/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_body_budget.py new file mode 100644 index 0000000000..7110868f76 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_body_budget.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="CreateTableStorageMigrationBodyBudget") + + +@_attrs_define +class CreateTableStorageMigrationBodyBudget: + """ + Attributes: + batch_bytes (int | Unset): Default: 4194304. + batch_rows (int | Unset): Default: 1024. + temporary_bytes (int | Unset): Default: 68719476736. + disk_reserve_bytes (int | Unset): Default: 1073741824. + """ + + batch_bytes: int | Unset = 4194304 + batch_rows: int | Unset = 1024 + temporary_bytes: int | Unset = 68719476736 + disk_reserve_bytes: int | Unset = 1073741824 + + def to_dict(self) -> dict[str, Any]: + batch_bytes = self.batch_bytes + + batch_rows = self.batch_rows + + temporary_bytes = self.temporary_bytes + + disk_reserve_bytes = self.disk_reserve_bytes + + field_dict: dict[str, Any] = {} + + field_dict.update({}) + if batch_bytes is not UNSET: + field_dict["batch_bytes"] = batch_bytes + if batch_rows is not UNSET: + field_dict["batch_rows"] = batch_rows + if temporary_bytes is not UNSET: + field_dict["temporary_bytes"] = temporary_bytes + if disk_reserve_bytes is not UNSET: + field_dict["disk_reserve_bytes"] = disk_reserve_bytes + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + batch_bytes = d.pop("batch_bytes", UNSET) + + batch_rows = d.pop("batch_rows", UNSET) + + temporary_bytes = d.pop("temporary_bytes", UNSET) + + disk_reserve_bytes = d.pop("disk_reserve_bytes", UNSET) + + create_table_storage_migration_body_budget = cls( + batch_bytes=batch_bytes, + batch_rows=batch_rows, + temporary_bytes=temporary_bytes, + disk_reserve_bytes=disk_reserve_bytes, + ) + + return create_table_storage_migration_body_budget diff --git a/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_body_target.py b/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_body_target.py new file mode 100644 index 0000000000..a449ebd881 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_body_target.py @@ -0,0 +1,8 @@ +from enum import StrEnum + + +class CreateTableStorageMigrationBodyTarget(StrEnum): + VECTOR_STORE = "vector_store" + + def __str__(self) -> str: + return str(self.value) diff --git a/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_response_200.py b/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_response_200.py new file mode 100644 index 0000000000..3987c88d1b --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/create_table_storage_migration_response_200.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="CreateTableStorageMigrationResponse200") + + +@_attrs_define +class CreateTableStorageMigrationResponse200: + """ """ + + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + create_table_storage_migration_response_200 = cls() + + create_table_storage_migration_response_200.additional_properties = d + return create_table_storage_migration_response_200 + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/py/packages/sdk/src/antfly/client_generated/models/get_table_storage_migration_response_200.py b/py/packages/sdk/src/antfly/client_generated/models/get_table_storage_migration_response_200.py new file mode 100644 index 0000000000..3f100b4606 --- /dev/null +++ b/py/packages/sdk/src/antfly/client_generated/models/get_table_storage_migration_response_200.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="GetTableStorageMigrationResponse200") + + +@_attrs_define +class GetTableStorageMigrationResponse200: + """ """ + + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + get_table_storage_migration_response_200 = cls() + + get_table_storage_migration_response_200.additional_properties = d + return get_table_storage_migration_response_200 + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/specs/openapi/antfly/metadata.yaml b/specs/openapi/antfly/metadata.yaml index d61f0c3dd9..8a1039fd70 100644 --- a/specs/openapi/antfly/metadata.yaml +++ b/specs/openapi/antfly/metadata.yaml @@ -12157,6 +12157,159 @@ paths: $ref: "#/components/responses/MethodNotAllowed" "500": $ref: "#/components/responses/InternalServerError" + /tables/{tableName}/storage/migrations: + parameters: + - name: tableName + in: path + required: true + schema: { type: string } + post: + operationId: createTableStorageMigration + summary: Create or resume a table storage migration job + description: | + Table-admin operation for local single-shard standalone tables. Target + vector_store changes primary_lsm source ownership without changing models, + dimensions, artifacts or logical indexes. Retry creation with the same + job_id, target and budgets. The job is advanced explicitly through its + job endpoint; the server does not schedule an unattended migration loop. + Offline migration uses antfly storage migrate against a stopped server. + tags: [data_operations] + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: false + required: [job_id, target] + properties: + job_id: + type: string + pattern: '^[A-Za-z0-9_-]{1,128}$' + target: + type: string + enum: [vector_store] + budget: + type: object + additionalProperties: false + properties: + batch_bytes: + type: integer + format: int64 + minimum: 4096 + maximum: 67108864 + default: 4194304 + batch_rows: + type: integer + minimum: 1 + maximum: 65536 + default: 1024 + temporary_bytes: + type: integer + format: int64 + default: 68719476736 + disk_reserve_bytes: + type: integer + format: int64 + default: 1073741824 + responses: + "200": + description: Durable migration receipt, or admitted receipt before DB preparation + content: + application/json: + schema: + type: object + additionalProperties: true + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "409": + description: Conflicting job, lifecycle operation or publication state + "503": + description: Retryable resource or recovery admission failure + "500": + $ref: "#/components/responses/InternalServerError" + /tables/{tableName}/storage/migrations/{jobId}: + parameters: + - name: tableName + in: path + required: true + schema: { type: string } + - name: jobId + in: path + required: true + schema: + type: string + pattern: '^[A-Za-z0-9_-]{1,128}$' + get: + operationId: getTableStorageMigration + summary: Read a table storage migration receipt + description: | + Table-admin observation only. Does not admit, advance, or publish a job. + A phase of admitted means catalog admission is durable but DB preparation + has not begun; retry creation or send a job action to recover it. The + receipt is retained until a later migration replaces it; this endpoint + is not a permanent job history. + tags: [data_operations] + responses: + "200": + description: Durable migration receipt, or admitted receipt before DB preparation + content: + application/json: + schema: + type: object + additionalProperties: true + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "409": + description: Conflicting job, lifecycle operation or publication state + "503": + description: Retryable resource or recovery admission failure + "500": + $ref: "#/components/responses/InternalServerError" + post: + operationId: advanceTableStorageMigration + summary: Advance, publish or cancel a table storage migration job + description: | + Uses the job's durable configuration and budgets. Each step commits + bounded progress. Publish is accepted only at ready; complete additionally + certifies reference-only primary artifacts and native ANN serving. + Cancellation is allowed only before publication. Repeating an action + after an ambiguous response resumes the durable job. + tags: [data_operations] + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: false + required: [action] + properties: + action: + type: string + enum: [step, publish, cancel] + responses: + "200": + description: Durable migration receipt, or admitted receipt before DB preparation + content: + application/json: + schema: + type: object + additionalProperties: true + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "409": + description: Conflicting job, lifecycle operation or publication state + "503": + description: Retryable resource or recovery admission failure + "500": + $ref: "#/components/responses/InternalServerError" /tables/{tableName}/repair/run: parameters: - name: tableName diff --git a/ts/packages/sdk/src/public-api.d.ts b/ts/packages/sdk/src/public-api.d.ts index defdb626d2..105a1c5d0f 100644 --- a/ts/packages/sdk/src/public-api.d.ts +++ b/ts/packages/sdk/src/public-api.d.ts @@ -1234,6 +1234,68 @@ export interface paths { patch?: never; trace?: never; }; + "/db/v1/tables/{tableName}/storage/migrations": { + parameters: { + query?: never; + header?: never; + path: { + tableName: string; + }; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Create or resume a table storage migration job + * @description Table-admin operation for local single-shard standalone tables. Target + * vector_store changes primary_lsm source ownership without changing models, + * dimensions, artifacts or logical indexes. Retry creation with the same + * job_id, target and budgets. The job is advanced explicitly through its + * job endpoint; the server does not schedule an unattended migration loop. + * Offline migration uses antfly storage migrate against a stopped server. + */ + post: operations["createTableStorageMigration"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/db/v1/tables/{tableName}/storage/migrations/{jobId}": { + parameters: { + query?: never; + header?: never; + path: { + tableName: string; + jobId: string; + }; + cookie?: never; + }; + /** + * Read a table storage migration receipt + * @description Table-admin observation only. Does not admit, advance, or publish a job. + * A phase of admitted means catalog admission is durable but DB preparation + * has not begun; retry creation or send a job action to recover it. The + * receipt is retained until a later migration replaces it; this endpoint + * is not a permanent job history. + */ + get: operations["getTableStorageMigration"]; + put?: never; + /** + * Advance, publish or cancel a table storage migration job + * @description Uses the job's durable configuration and budgets. Each step commits + * bounded progress. Publish is accepted only at ready; complete additionally + * certifies reference-only primary artifacts and native ANN serving. + * Cancellation is allowed only before publication. Repeating an action + * after an ambiguous response resumes the durable job. + */ + post: operations["advanceTableStorageMigration"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/db/v1/tables/{tableName}/repair/run": { parameters: { query?: never; @@ -18295,6 +18357,165 @@ export interface operations { 500: components["responses"]["InternalServerError"]; }; }; + createTableStorageMigration: { + parameters: { + query?: never; + header?: never; + path: { + tableName: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + job_id: string; + /** @enum {string} */ + target: "vector_store"; + budget?: { + /** + * Format: int64 + * @default 4194304 + */ + batch_bytes?: number; + /** @default 1024 */ + batch_rows?: number; + /** + * Format: int64 + * @default 68719476736 + */ + temporary_bytes?: number; + /** + * Format: int64 + * @default 1073741824 + */ + disk_reserve_bytes?: number; + }; + }; + }; + }; + responses: { + /** @description Durable migration receipt, or admitted receipt before DB preparation */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: unknown; + }; + }; + }; + 400: components["responses"]["BadRequest"]; + 404: components["responses"]["NotFound"]; + /** @description Conflicting job, lifecycle operation or publication state */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 500: components["responses"]["InternalServerError"]; + /** @description Retryable resource or recovery admission failure */ + 503: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + getTableStorageMigration: { + parameters: { + query?: never; + header?: never; + path: { + tableName: string; + jobId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Durable migration receipt, or admitted receipt before DB preparation */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: unknown; + }; + }; + }; + 400: components["responses"]["BadRequest"]; + 404: components["responses"]["NotFound"]; + /** @description Conflicting job, lifecycle operation or publication state */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 500: components["responses"]["InternalServerError"]; + /** @description Retryable resource or recovery admission failure */ + 503: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + advanceTableStorageMigration: { + parameters: { + query?: never; + header?: never; + path: { + tableName: string; + jobId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + /** @enum {string} */ + action: "step" | "publish" | "cancel"; + }; + }; + }; + responses: { + /** @description Durable migration receipt, or admitted receipt before DB preparation */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: unknown; + }; + }; + }; + 400: components["responses"]["BadRequest"]; + 404: components["responses"]["NotFound"]; + /** @description Conflicting job, lifecycle operation or publication state */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 500: components["responses"]["InternalServerError"]; + /** @description Retryable resource or recovery admission failure */ + 503: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; runTableRepair: { parameters: { query?: never; diff --git a/zig/DB.md b/zig/DB.md index aab531e562..83b1e3a291 100644 --- a/zig/DB.md +++ b/zig/DB.md @@ -1135,6 +1135,17 @@ their physical proofs match their storage engines: complete source outcomes and enough live sparse documents to cover produced sources; chunking may make the physical count larger. +For repeatable ANN comparisons on an unchanged corpus, also wait for +`status.hbc_posting.refresh_pending == false` on a fresh, complete index status. +This is a separate optimization-convergence signal: a bounded clean sweep has +verified the current mutation epoch. Writes invalidate it, and reopen starts +pending until another sweep. Dirty postings can serve exact member scoring +while clean postings use approximate quantized candidates, so ordinary idle +refresh can change the neighbor list without a source-vector change. A status +probe alone may inspect and retire a cold owner; activate the table with a query +before waiting for its background work. This signal does not freeze the index +against concurrent writes or change query readiness. + Correctness and latency use separate verification gates. The 500-document regression requires complete coverage, idempotent replay, a complete second index, and a stable first index. A deterministic partial-publication test proves diff --git a/zig/VECTOR_STORAGE_MIGRATION_FINDINGS.md b/zig/VECTOR_STORAGE_MIGRATION_FINDINGS.md new file mode 100644 index 0000000000..1c1fa38b3b --- /dev/null +++ b/zig/VECTOR_STORAGE_MIGRATION_FINDINGS.md @@ -0,0 +1,494 @@ +# Source ownership migration qualification + +The public interface is `antfly storage migrate` and the table-scoped +`/storage/migrations` job API documented in [VECTOR_STORE.md](VECTOR_STORE.md). +Qualification compares existing primary-LSM tables migrated online/offline with +fresh vector-store tables. It does not change the table-creation default. + +## Protocol + +[`scripts/qualify_vector_migration.py`](scripts/qualify_vector_migration.py) retains +inputs, binary hashes, databases, progress, and results. The screen uses 768-D +normalized synthetic vectors, cosine distance, default ANN settings, 256-document +write batches, 1,000 updates and 1,000 deletes. Online churn happens after capture +admission, and a semantic query checks availability every 16 migration steps. +The reported online duration includes those queries. Every query cell warms up +for 128 requests; the first screen measures 4,096 requests +at concurrency 1, 8, and 32. Workloads are semantic, full-text, and an alternating +50/50 mix. Memory sampling includes the server and offline migration process; +disk includes retained physical roots. After the restarted query cells, the +harness waits up to 180 seconds for source reclamation. That additional wait is +not the elapsed reclamation time since job completion. +The final runner fails if source collection does not reach the expected live +payload count. For the large-vector synthetic corpus (at least 512 dimensions), +it also rejects migrated primary SSTables larger than half the raw vector plane. +This broad regression guard catches inline-sized retention; it is not a proof +that every obsolete byte has been removed. Exact bytes remain in each receipt. + +This is a sequential screen on the available host. It is not a repeated ABBA +promotion qualification. The isotropic synthetic corpus has low absolute recall +with default ANN settings; its QPS cannot establish equivalent search quality. +Restart is warm, and this harness does not measure lock waits directly. + +## Initial 50K screen: forced-flush regression + +Preserved receipts are under +`.benchmark-results/vector-migration-implementation/compare-50k/` in the worktree. +All three modes completed, retained the expected 49,000 payloads after churn, +and reopened successfully. They did **not** have equivalent query performance. + +| Measurement | Fresh vector-store | Online migration | Offline migration | +|---|---:|---:|---:| +| Initial ready, seconds | 25.68 | 25.07 | 25.53 | +| Churn plus migration, seconds | 1.07 | 26.94 | 30.50 | +| Semantic QPS C1 | 330.6 | 22.3 | 21.4 | +| Semantic QPS C8 | 1,658.6 | 105.9 | 98.5 | +| Semantic QPS C32 | 560.5 | 127.1 | 114.3 | +| Full-text QPS C8 | 1,265.8 | 1,230.1 | 1,277.7 | +| Mixed QPS C8 | 1,777.5 | 189.2 | 182.3 | +| Recall@10 | 0.169 | 0.147 | 0.138 | +| Warm restart, seconds | 0.46 | 1.47 | 1.24 | +| Allocated disk, MiB | 197.9 | 199.6 | 350.2 | + +The first screen ran alongside compilation, and the fresh C32 tail was an +outlier. Its offline RSS sampler omitted the command's peak; that harness bug +is fixed for subsequent runs. These are diagnostic results, not promotion data. + +The online primary store recorded 1,304 flushes, 1,448 output runs and 1,762 +manifest writes, leaving 112 L0 runs. Snapshot-rotation counters stayed at zero. +The actual cause of the tiny files was `sync(true)` after each migration page: +that API synchronizes the WAL **and** flushes pending memtables. Verification +pages often update only the small job receipt. A short sample showed ANN +identity lookups and Snappy decompression in primary-LSM point reads; the sample +was too small to assign a percentage of query time. + +Page commits now use the existing WAL durability barrier. Payload preparation +still precedes the atomic primary reference/progress commit, and ownership +publication retains its full storage barrier. A regression checks that 32 +single-row pages create no additional flushes or SSTables. Recovery-boundary +tests pass with this change. A production test additionally kills the process +after an acknowledged backfill page, verifies its exact receipt after reopen, +and completes the migration and queries both models. All ten production +migration/vector-store tests pass. + +## Separating restart from migration + +A follow-up probe reopens each preserved 50K table with the same original +binary, warms 128 queries and measures 256 semantic queries per cell. Receipts +and the probe script are retained under the same implementation result root. + +| Reopened table | C1 QPS | C8 QPS | +|---|---:|---:| +| Fresh vector-store | 22.6 | 120.4 | +| Online migration | 22.2 | 116.2 | +| Offline migration | 22.3 | 107.1 | + +The fresh table loses its ingestion-time advantage after restart. Therefore the +large initial gap is not evidence that migration uniquely damages steady-state +throughput. The forced page flushes are independently undesirable, but fixing +them cannot be assumed to fix the shared restart/identity-lookup cost. The +corrected harness measures semantic throughput after restart in every arm and +captures LSM counters before queries. The corrected 50K/1M screen uses 1,024 +measured queries per cell, keeping all three arms matched within each screen. + +## Corrected 50K screen + +Receipts: `.benchmark-results/vector-migration-implementation/wal-50k/`. +All modes completed and observed exactly 49,000 retained source payloads with +no pending source collection bytes. The binary includes the WAL page barrier +and rebuilding-index error identity fix. + +| Measurement | Fresh vector-store | Online migration | Offline migration | +|---|---:|---:|---:| +| Initial ready, seconds | 25.47 | 25.49 | 25.53 | +| Churn plus migration, seconds | 1.56 | 15.37 | 13.88 | +| Reopened semantic QPS C1 | 22.7 | 22.7 | 22.7 | +| Reopened semantic QPS C8 | 124.1 | 107.8 | 118.5 | +| Reopened semantic QPS C32 | 132.7 | 117.0 | 134.5 | +| Full-text QPS C8, before restart | 1,217.9 | 1,236.1 | 1,244.1 | +| Recall@10, before restart | 0.188 | 0.194 | 0.191 | +| Warm restart, seconds | 1.43 | 1.47 | 1.47 | +| Peak process RSS, MiB | 1,078.0 | 1,340.1 | 1,046.5 | +| Final allocated disk, MiB | 197.7 | 199.4 | 199.8 | + +Online migration plus churn fell from 26.9 to 15.4 seconds, and offline from +30.5 to 13.9 seconds. This is a single before/after screen, not a confidence +interval. Query measurements have different fixed counts across these screens; +compare matched arms within each screen. + +Immediately after online completion, the primary store had 12 runs and 187 MB +of SSTables, including superseded inline bytes. Existing background compaction +reduced this to 23 MB during queries and 20 MB after restart. No manual flush or +compaction was injected into the comparison. Final total disk was within about +2 MB of fresh storage. Logical completion and physical reclamation are distinct. + +Fresh pre-restart throughput itself varied substantially between screens. The +matched reopened results are more informative: C1 agrees, while online C8/C32 +are about 13%/12% below fresh in this run. The screen neither establishes a +migration-specific order-of-magnitude loss nor proves complete performance +equivalence. All arms retain the shared post-churn query bottleneck. + +### No-churn control + +The short fresh-only control in `no-churn-50k/` uses the same binary, 50K rows, +no updates/deletes, and 128 measured queries per cell. Reopened semantic QPS is +287 at C1, 1,599 at C8 and 1,745 at C32. Reopening alone does not reproduce the +22-QPS result; the churn history matters. This control changes both updates and +deletes together, so it does not experimentally distinguish them. + +The code provides a concrete next hypothesis: live-document constraints map +nonvisible document ordinals to ANN member IDs before search. Missing mappings +fall through primary and legacy identity lookups and are not cached as misses. +The saved sample visits this path. A follow-up should isolate deletes from +updates, count those lookups per query, and preserve generation-aware visibility +when avoiding repeated mapping of absent ANN members. It must also retain +correct one-to-many behavior for chunk and multi-source indexes; treating every +document ordinal as its vector ID is not a general solution. + +## 1M screen: explicit primary reclamation + +The first WAL-barrier 1M fresh and online arms completed and recovered, with +999,000 retained source payloads after churn. Online completion did not ensure +physical primary reclamation: it retained 2,595,108,632 bytes of primary SSTables, +versus 379,544,230 for fresh storage. Total allocated disk was 6,233,559,040 versus +3,987,898,368 bytes. Source collection was complete in both, so the difference +was old inline primary values rather than orphan source payloads. These receipts +are retained under `wal-1m/`. All three arms subsequently completed. + +| Measurement | Fresh vector-store | Online migration | Offline migration | +|---|---:|---:|---:| +| Initial ready, seconds | 572.8 | 593.5 | 595.4 | +| Churn plus migration, seconds | 5.2 | 351.1 | 1,792.6 | +| Semantic QPS C1/C8/C32 | 13.2 / 44.8 / 53.9 | 18.8 / 87.8 / 107.9 | 17.9 / 81.3 / 100.9 | +| Restarted semantic QPS C1/C8/C32 | 13.5 / 66.5 / 80.9 | 17.7 / 76.7 / 100.3 | 18.9 / 76.5 / 80.9 | +| Recall@10 | 0.094 | 0.097 | 0.091 | +| Warm restart, seconds | 16.7 | 2.2 | 5.6 | +| Sampled peak process RSS, GiB | 5.89 | 10.28 | 7.20 | +| Primary SSTable bytes, GB | 0.380 | 2.595 | 3.614 | +| Allocated disk, GB | 3.988 | 6.234 | 7.240 | + +All source collections completed with exactly 999,000 retained payloads. The +primary stores reported no obsolete paths or ordinary compaction backlog: +old inline values remained in active lower-level runs. Offline conversion took +about 30 minutes. Compilation overlapped portions of this screen; these single +sequential arms do not establish performance equivalence or causal speedups. +The following changes are intended to address its disk and conversion costs. + +The migration now has an explicit `reclaiming` phase. After candidate cleanup, +it flushes replacements once and durably requests overlap rewrites through the +existing LSM GC planner. Requests qualify even without tombstones and survive +partial level jobs, splits and restart. Only a validated full overlap rewrite +clears them. Job completion waits for that work; reader pins, retention windows +and source collection can still retain files afterward. + +Explicit steps bypass optional query-idle deferral, which otherwise starved +reclamation in a test that queried between every step. They retain ordinary +memory/I/O admission and streaming-work yielding, and execute outside the table +apply lock. This is production migration behavior, not a benchmark-only force +compaction. Metadata request preparation reserves its working set before +cloning the directory and publishes its intent before the job receipt. + +The regression checks bounded input I/O, an old reader, restart after partial +progress, and removal of superseded values in a store with zero tombstones. +The LSM suite passed 493 tests (23 skipped), and all 12 DB migration tests passed, +including recovery at the manifest-request and primary-receipt boundaries. +The final binary at `a494f01b21` passes all 11 production migration/vector-store +checks, including abrupt process death after the reclamation receipt. The +combined 50K/1M qualification is running. + +### Descriptor-cache hit cost + +A short offline final-verification sample found primary point reads repeatedly +allocating a filename in the process-wide descriptor cache. The sample contains +only 17 main-thread stacks and cannot establish a percentage of migration time. +Inspection confirmed that even a cache hit duplicated and freed the path using +the page allocator before returning the existing descriptor. + +The hit path now retains the existing descriptor under its shard mutex before +allocating anything. Misses retain the existing outside-lock allocation, +entry recheck and mutation-epoch fencing. Five descriptor-cache tests pass, +including mutation races and admission limits. A new allocation-failure check +proves that a populated-cache hit needs no allocation. + +A Debug microbenchmark with the process pool's page-allocator shape alternates +baseline/candidate/candidate/baseline three times. Each cell performs 20,000 +cached descriptor acquisitions. Baseline averaged 48.081 ms (46.870–51.090 ms) +and 20,000 allocations; the candidate averaged 7.394 ms (7.324–7.425 ms) and zero +allocations: about 6.5× faster for this operation. This does not establish an +end-to-end migration speedup. Logs, results, source snapshots and test binaries +are retained under `fd-cache-hit/` in the implementation result root. The +baseline intentionally fails the newly added allocation-free assertion. + +The offline candidate also now supplies a 64 MiB shared LSM block cache when +none was supplied by the caller. Standalone serving already supplies a shared +cache, but a direct offline DB open did not. Repeated current-value and candidate +checks could therefore reload/decompress adjacent blocks for each artifact. +The operator cache uses the same standalone resource policy, participates in +memory admission, and is destroyed after the candidate closes. Caller caches +and resource managers take precedence. All 12 DB migration tests pass with this +change, including offline resume/copy/publication faults. Its end-to-end effect +is evaluated in the final comparison; it is distinct from the descriptor microbench. + +## Final 50K screen + +Receipts: `.benchmark-results/vector-migration-implementation/final-50k/`. +The final binary at `a494f01b21` includes explicit primary reclamation, +allocation-free descriptor cache hits, and the bounded offline block cache. +All arms completed, reopened, and collected down to 49,000 source payloads. + +| Measurement | Fresh vector-store | Online migration | Offline migration | +|---|---:|---:|---:| +| Initial ready, seconds | 24.58 | 24.72 | 24.53 | +| Churn plus migration, seconds | 1.55 | 18.49 | 11.52 | +| Semantic QPS C1/C8/C32 | 48.0 / 324.0 / 246.2 | 23.1 / 116.2 / 128.2 | 22.9 / 106.1 / 117.9 | +| Restarted semantic QPS C1/C8/C32 | 23.5 / 130.0 / 140.9 | 23.0 / 111.2 / 116.5 | 22.8 / 111.1 / 122.9 | +| Full-text QPS C8, before restart | 1,216.6 | 1,256.5 | 1,194.8 | +| Mixed QPS C8, before restart | 207.7 | 177.8 | 193.8 | +| Recall@10 | 0.197 | 0.159 | 0.181 | +| Warm restart, seconds | 0.19 | 1.19 | 2.24 | +| Sampled peak process RSS, MiB | 1,089.8 | 1,471.1 | 1,161.0 | +| Final primary SSTable bytes, MB | 18.59 | 19.04 | 17.24 | +| Final allocated disk, MB | 206.29 | 207.69 | 204.96 | + +Online primary SSTables were already down to 19.04 MB before query measurement, +compared with 187 MB at that point in the WAL-only screen. Its longer conversion +now includes explicit primary reclamation. Total final disk is within about 1% +of fresh storage. Reopened C1 is close, while migrated C8/C32 remain lower in +this single screen. The shared post-churn slowdown is still present. These +results do not prove query-performance equivalence. + +### Same-index neighbor preservation + +The independently built 50K arms have different recall. A separate production +regression isolates conversion from ANN build variation: create 4,096 normalized +64-D vectors, restart the primary-LSM table, record 32 top-10 queries, migrate +that same table, and repeat before and after another restart. Both online and +offline conversion preserve all ordered neighbor lists exactly. The two checks +pass in 9.21 seconds with the final binary. This covers native ANN conversion; +it does not claim identical graph construction for legacy ANN rebuilding or +equivalent recall for independently built million-vector tables. +The complete production migration/vector-store suite, including these two +checks, subsequently passed all 13 tests in 43.11 seconds. + +Linux CI exposed two unit fixtures that exhausted a fixed number of tight +migration steps. Step counts do not bound timed GC admission retries. Their +completion driver now uses a 30-second deadline, yields during reclamation, +and reports the durable phase/counters on timeout. The old-reader fixture +explicitly injects a 250 ms GC retry. All 12 focused DB migration tests pass +with that case and no leaks. Production admission behavior is unchanged. + +## Final 1M screen + +Receipts: `.benchmark-results/vector-migration-implementation/final-1m/`. +The same final binary completed all three arms, including restart, source +collection and the primary-size regression guard. Each retained exactly +999,000 source payloads with zero pending source collection bytes. + +| Measurement | Fresh vector-store | Online migration | Offline migration | +|---|---:|---:|---:| +| Initial ready, seconds | 557.2 | 546.5 | 534.1 | +| Churn plus migration, seconds | 4.3 | 449.4 | 530.7 | +| Semantic QPS C1/C8/C32 | 63.0 / 150.8 / 130.0 | 19.7 / 91.8 / 90.3 | 18.7 / 90.4 / 92.6 | +| Restarted semantic QPS C1/C8/C32 | 18.0 / 83.2 / 103.9 | 19.0 / 86.8 / 98.3 | 20.1 / 92.1 / 91.6 | +| Restarted semantic C8 p99, ms | 140.0 | 120.7 | 120.2 | +| Full-text QPS C8, before restart | 82.8 | 86.8 | 81.1 | +| Mixed QPS C8, before restart | 98.0 | 89.3 | 89.6 | +| Recall@10 | 0.1094 | 0.1063 | 0.0875 | +| Warm restart, seconds | 15.55 | 2.46 | 2.47 | +| Sampled peak process RSS, GiB | 6.84 | 11.67 | 7.14 | +| Primary SSTable bytes, GB | 0.388 | 0.349 | 0.379 | +| Allocated disk, GB | 3.980 | 3.977 | 4.006 | + +Both migrated tables finish within 1% of fresh total disk. Before queries, +online primary SSTables were already down to 349 MB and offline to 450 MB; +ordinary later maintenance reduced offline primary storage further. The +earlier retained-inline results were 2.595 GB online and 3.614 GB offline. +Explicit migration completion now includes removal of those superseded primary +values, while source GC and reader retirement remain separate lifecycle work. + +Offline conversion plus churn fell from 1,792.6 to 530.7 seconds (29.9 to 8.85 +minutes), including the newly required primary reclamation. This measures the +combined implementation changes, not the isolated effect of the block cache. +Online completion rose from 351.1 to 449.4 seconds with that added reclamation +work. Its sampled peak RSS is about 71% above fresh storage, so online operators +still need temporary memory headroom within normal resource admission. + +After a matched restart, online semantic QPS is within about 6% of fresh across +the three concurrencies; offline is within about 12%. Before restart, fresh +still has the transient ingestion-time advantage. Offline recall is lower in +this independently built arm, and all absolute recall scores are low. The +small same-index preservation checks above do not establish unchanged recall +at 1M. These results qualify conversion/recovery and reclamation for this +screen, not equivalent query quality or performance. A quality comparison +would need before/after measurements on the same large built index and repeated +matched runs. The shared post-churn identity-lookup cost remains a separate +query-engine follow-up. + +Focused test compilation overlapped part of the offline ingestion. There are +no confidence intervals, cold-cache resets or quiet-host guarantees. The +packaged Linux Antfly and inference E2E jobs passed for this engine revision; +rerunning CI with the unit-driver fixes requires the repository's new human +approval gate for the final PR commit. + +## PR recovery review follow-up + +Catalog-only admission can now be cancelled without starting the source store +or passing the migration's disk reserve. The DB writes a durable cancelled +receipt under the same apply lock as startup before the API clears admission. +Exact retries return that receipt; a conflicting active job remains protected. +The regression covers an impossible reserve, restart before cancellation, a +lost cancellation response, a failed catalog reconciliation, another rejected +admission with an older cancelled receipt, and a replacement job. + +Offline copying now syncs the whole directory chain, including publication of +the shadow root, before acknowledging a file's first chunk. Retries sync existing +directories too, because a failed attempt may have created them without making +their parent entries durable. Later chunks rely on the directory chain already +acknowledged by the cursor. + +The copy recovery tests use the production chunk function with a stricter +VoprIo directory-sync adapter. The default model persists the entire namespace +on any directory sync, which cannot expose this bug. The adapter persists only +immediate entries and removes descendants whose parent links are lost on a +simulated power failure. A negative control demonstrates the old sequence +retaining its cursor while losing a copied subtree. Recovery cases cover empty +and multi-chunk files, intermediate-directory sync failures, retry without a +crash, and power loss before/after cursor publication and between later chunks. +These simulated failures complement the production process-restart tests; they +are not physical power-cut tests on a host filesystem. + +Validation: `vector-migration-test` passed 27/27; the focused API +observation/cancellation test passed; the ReleaseFast executable built; the +production migration/vector-store suite passed 14/14 on its complete rerun. + +The first production run hit an intermittent offline ANN-neighbor mismatch +after restart. The preserved pre-fix executable reproduced the identical +query-15 difference on the second additional control attempt: `doc:000221` +disappeared from the top ten and `doc:002725` entered. The fixed run first +matched after migration and differed after another restart; the control +differed at the first post-migration check. Both logs record deferred posting +maintenance, but this comparison does not establish its causal role. The +assertions remain unchanged; this is an unresolved pre-existing ANN +stability/qualification issue, not a clean repeated end-to-end result. +Logs, both failed database roots, the tested binary, and a control runner are +preserved under `.benchmark-results/vector-migration-review-20260915/`. + +## Delayed commands and cancellation before copying + +Online mutation handlers now acquire a per-table command slot from the shared +standalone catalog owner before reading admission. They retain it across the +synchronous DB command and catalog reconciliation. This prevents a paused start +from resuming after another handler cancels and replaces its admission. A +contending migration mutation returns retryable HTTP 503; status reads and +commands for other tables remain available. This applies to the supported +single-process standalone deployment, whose catalog process lock excludes +another server or offline operator. Persisted admission still governs recovery +after process loss. + +The regression reenters through a second API handler between catalog admission +and DB startup, with a real DB behind both handlers. It checks that observation +works, cancellation/replacement cannot pass the slot, and subsequent commands +release the slot and retain the correct catalog job. The existing test also +checks slot release after DB rejection and failed catalog publication. + +Offline cancellation now persists a source-identity-bound receipt even when +admission failed before the copy fence existed. Exact retries preserve the +terminal cancellation; mismatched budgets and a replacement's active fence +remain protected. If an exact CLI retry temporarily reinstalls admission before +reading that receipt, it clears the marker before returning the cancelled +error. The packaged CLI regression uses an incorrect replica root to leave +catalog-only admission, cancels against the correct root, models a lost catalog +publication, retries both cancellation and startup, and admits a replacement. + +Validation: the migration/recovery target passed 29/29 and the focused API +target passed 2/2, with no leaks. The packaged Debug executable built and passed +14/15 migration/vector-store E2E cases. The remaining online neighbor test +exhausted 256 tight requests during reclamation; its completion helper now uses +a 120-second deadline and yields during serving/reclamation, as timed GC work +cannot be bounded by request count. This changes no production scheduling or +completion/neighbor assertions. + +With that helper corrected, the migration suite passed 8/9. The online neighbor +case completed migration, then showed the identical query-15 neighbor difference +documented above for the pre-fix offline control. The offline neighbor case and +both cancellation cases passed. This remains an unresolved ANN qualification +issue, not a clean end-to-end result. Logs, the tested Debug executable, and both +failed database roots are retained under +`.benchmark-results/vector-migration-review-20260915/review-followup/fixes/`. + +## Large rows and backups after cancellation + +Migration pages now retain only cursor keys for unrelated primary values. +Draining hashes borrowed inline embeddings into compact references while +scanning, then validates those references against the committed candidate map. +The DB retains apply-exclusive admission across capture and commit. A vector +larger than the page budget, captured after verification, consumes one page +without retaining a full payload copy. Prepublication dense-artifact admission +remains bounded by the configured byte budget. The regression inserts a large +document and an oversized embedding after `ready`, publishes, restarts during +draining, and verifies exact preservation after completion. + +Snapshot eligibility now distinguishes retained source objects from published +ownership. A terminal cancelled job on primary ownership can back up immediately; +its source object remains alive for old readers and retirement. Active, +cancelling, replacement and published jobs remain ineligible. Both native and +portable capture recheck eligibility after their admission/maintenance boundary. +The focused regression holds an old source reader through cancellation and +native backup, then verifies that starting a replacement closes admission again. + +The focused migration/recovery suite passes 31/31 with no leaks. The packaged +Debug executable builds. Its migration/vector-store E2E run passes 17/18, +including the large-document case and immediate native/portable backup, restore +and restart after cancellation. The remaining online ANN case reproduces the +same query-15 neighbor difference documented above; its assertions are unchanged. +The tested binary, logs and failed database root are retained under +`.benchmark-results/vector-migration-review-20260916/fixes/`. + + +## ANN comparison baseline and posting-refresh convergence + +The query-15 mismatch is reproducible without migration. On the preserved +pre-fix executable, three immediate primary-LSM restarts retained `doc:000221`; +five seconds idle then replaced it with `doc:002725`, exactly as in the migration +failure. A phase-by-phase online run first changed during backfill, before +ownership publication. Logs show one deferred posting repair. The same-index +comparison after that maintenance settled preserved all 32 ordered top-ten +lists through conversion and restart. The control scripts, logs and databases +are under `.benchmark-results/vector-migration-review-20260916/`. + +The fixture confused replay completion with convergence of optional ANN work. +Dirty posting payloads fall back to exact member scoring; clean postings can use +quantized candidate selection. Consequently, the same corpus can have different +approximate neighbors across an idle repair. Restart alone does not drain it. +The earlier failures did not demonstrate missing source vectors or migration +corruption, and repeated immediate queries were not a sufficient baseline gate. + +Index status now includes `hbc_posting.refresh_pending`. The bounded refresh +scanner publishes an atomic certificate only after a clean sweep at the current +mutation epoch. A partial/changed sweep stays pending, a write or abort +invalidates the certificate, and reopen starts uncertified. Lightweight status, +cached-status overlays and detailed diagnostics expose the same constant-cost +observation; they do not scan the corpus to count dirty postings. Shard +aggregation remains pending if any reported shard is pending or lacks the new +observation. Read-only verification progress also invalidates runtime status so +the final clean transition can be published without another repair. Query +readiness and optional-maintenance scheduling remain separate. + +The E2E fixture first activates the lazily opened owner with a query, then waits +for fresh, complete status and a clean refresh certificate. Status-only cold +inspection can retire its temporary owner, so polling it alone does not request +resident background maintenance. The fixture retains exact ordered-neighbor +assertions, adds a no-migration restart control, and checks two post-operation +restarts. Unit regressions cover partial sweeps, writes behind the cursor, +aborted writes, reopen, cached observations and conservative shard aggregation. + +Validation after merging origin/main `1faa190bd2`: the packaged Debug build +succeeded; all 13 focused refresh/status/stable-tip and merge-fixture tests +passed, along with their 43 storage-owner boundary tests. The migration/recovery +suite passed 31/31 with no leaks. The full production migration/vector-store +suite passed 19/19, followed by three additional restart/online/offline rounds +(9/9). All ordered-neighbor assertions remain exact. Earlier overlapping Zig +targets collided on existing fixed `/tmp` fixture paths; the final combined +unit invocation runs those dependencies once and passes. The tested binary, +source patch, command driver and final logs are recorded in +`.benchmark-results/vector-migration-review-20260916/REFRESH_QUALIFICATION.md`. diff --git a/zig/VECTOR_STORE.md b/zig/VECTOR_STORE.md index adf961a120..354a313889 100644 --- a/zig/VECTOR_STORE.md +++ b/zig/VECTOR_STORE.md @@ -83,142 +83,199 @@ owner tests, 13 benchmark-tool tests, and 11 API checks with vector/HBC override cleared. A separate float16 override/reopen check also passed. No new throughput measurement is claimed. -## Existing-table migration plan (not yet implemented) +## Existing-table migration -Treat the new ANN physical storage and source-vector ownership as separate -transitions. The public logical index type remains `embeddings`; changing a -model, dimension, distance metric or source definition is a separate index -configuration/re-embedding operation. A format conversion must preserve the -existing logical contract and exact source vectors. +Source ownership and ANN format are separate transitions. The logical index +remains `embeddings`: migration preserves the table/document incarnation, every +artifact/producer identity, exact vector bytes, models, dimensions, metrics, +chunks and index definitions. It does not re-embed documents. Dense artifacts +with no ANN consumers migrate too, and dropping the last consumer preserves +source ownership. -| Transition | Authority that changes | Completion evidence | -| --- | --- | --- | -| ANN LSM projection to native generation | Per-index serving generation and its posting/vector manifests | Validated generation, mutation coverage fence, matching index incarnation/configuration and restart recovery | -| `primary_lsm` to `vector_store` | Table-wide source payload ownership | Every live artifact resolves through a durable source reference, with concurrent mutations caught up and all protected generations retained | - -The existing native-storage phases (`legacy`, `native_building`, -`native_validating`, `native_authoritative`) describe ANN authority. Preserve -that machinery and its supported pre-PR compatibility path. Those phases are -not proof that source ownership has migrated: a native ANN index can still -serve a `primary_lsm` table. Do not add decoders for discarded experimental -formats from this PR. Unsupported source/native formats must fail closed. - -### Durable job and admission - -Introduce a separately versioned storage-migration job, rather than allowing a -PATCH of the immutable ownership field. Persist the job ID/idempotency key, -table incarnation, source and target ownership, ownership epoch, target format -capabilities, artifact/schema configuration revision, snapshot fence, replay -cursor, backfill cursor, candidate manifests, publication decision and error. -Expose distinct source and per-index progress. Retries consult the same durable -job; an ambiguous response must not start a second migration. - -Start with the same qualified local single-shard deployment. Reject overlapping -restore, split/move, ownership migration and incompatible schema/index changes; -serialize index create/drop and producer-definition changes for the first online -implementation. Ordinary document updates, deletes and enrichment completion -must continue under version fencing. Admit only when the binary can read both -representations, sufficient disk exists for the temporary overlap plus journals, -and the operator's I/O, memory and journal-lag budgets can be enforced. No -automatic conversion on open, and no new HA/replication admission by implication. - -### Preparation, backfill and publication - -1. **Prepare.** Establish a durable primary snapshot fence and a durable mutation - capture cursor atomically with respect to writes. Pin their recovery inputs. - The old ownership and serving generations remain authoritative while the - candidate is built. Reuse the source store and its prepare-before-primary- - commit protocol; do not create a third permanent vector corpus. -2. **Backfill.** Stream all table-owned dense artifacts, including chunk artifacts, - externally supplied embeddings and artifacts with zero ANN consumers. Read in - bounded batches, bypass one-pass cache admission, and persist restartable - progress only after candidate payloads and reference mappings are durable. - Bind references to the complete artifact identity/version: document and shard - identity, artifact/producer identity, source hash, dimensions and model/config - identity. Never key migration by docid alone or assume one vector per document. -3. **Catch up.** Replay committed mutations in order, including tombstones and - producer removals. Compare artifact versions when applying snapshot work so - an old row cannot overwrite a newer update or resurrect a deletion. Stale - enrichment completion must pass the normal producer/source-version checks. - Prepared but uncommitted payloads are orphans, not visible documents. Bound - replay retention and pause backfill or apply backpressure if it falls behind. -4. **Prepare serving generations.** Reuse a compatible native index generation - where its immutable reference bindings remain valid; otherwise construct a - candidate version map/posting generation against the migrated source snapshot. - Keep the healthy old index queryable during replacement. Validate every - index's incarnation, config, artifact coverage and mutation fence, then check - representative query results and recall. Counts alone do not prove coverage. -5. **Cut over.** Use a short write-admission fence to drain admitted mutations, - apply the final replay suffix and durably publish the selected ownership - epoch, source manifest/reference root and required serving manifests. This - needs a recoverable publication decision across catalog and DB state, not - independent flag/file renames. On reopen, resolve an uncertain decision before - admitting writes or GC. New writes then prepare durable source payloads before - committing primary references, as fresh vector-store tables already do. -6. **Drain and reclaim.** Convert remaining inline primary values in bounded - version-checked batches. A transitional resolver must accept the proven old - inline representation and new references until conversion completes. Mark - the migration complete only after a full coverage check proves no live inline - payload remains and every committed reference resolves. Retain old files and - versions for old query/transaction snapshots, serving generations, recovery - journals and pinned backups. Reclaim only after those owners release them. - -For online conversion, the candidate reference root in step 5 must cover the -entire cutover snapshot, including artifacts not yet rewritten in the primary -LSM. Readers resolve against their captured ownership epoch; they must never -combine an old primary snapshot with a new mutable reference map. Either retain -the inline bytes for those readers or pin the corresponding immutable mapping. -The migration state must explicitly distinguish published ownership from fully -rewritten/reclaimed storage. This avoids an unbounded cutover transaction while -preserving snapshot correctness. - -The temporary mapping/journal is migration machinery. Retire it once primary -references, serving bindings and the durable publication record suffice for -recovery; include its bytes in accounting until then. Source GC must protect -candidate preparations and migration snapshot/replay inputs. Cancellation before -publication releases only candidate-owned data after proving no references were -published. Once publication may have committed, cancellation requires resolving -that outcome first. - -### Rollback, restore and delivery order - -Before cutover, rollback discards the unreferenced candidate and leaves the old -table authoritative. After cutover, returning to `primary_lsm` requires a reverse -backfill and mutation replay, or restoration of a consistent pre-migration -backup with an explicit data-loss boundary. Changing the setting or booting an -older binary is not rollback. Do not retain two payload copies indefinitely to -make downgrade appear free. - -Backups must capture catalog ownership/job state, primary references, all source -files needed for reference closure and the selected ANN manifests at one proven -fence. Restore either reproduces that state and resumes the job or rejects it -before exposing the table. Until this is implemented, reject migration-overlap -backups/restores; copying the primary LSM alone cannot back up a reference table. - -Deliver in this order: - -1. A bounded, resumable offline migration command with exclusive table admission, - a streaming shadow-root copy that preserves internal document identities, - versions, all artifact namespaces and catalog definitions, and native ANN - rebuild with recoverable atomic publication. A public document-only export - is insufficient. Validate it on real legacy tables; this - provides an initial conversion route without immediately adding online replay. -2. Durable online mutation capture, mixed-representation reads, candidate reference - roots and version-checked conversion. Reuse the offline builder and verifier; - add fault injection at every durability boundary before enabling publication. -3. Online generation publication, cancellation and reverse conversion, followed - by backup/restore and broader topology support as separately qualified work. - -Require restart/crash tests between preparation, payload sync, reference commit, -manifest publication and WAL retirement; repeated restarts and ambiguous retries; -same-dimension different models and multiple indexes per artifact; updates, -deletes and stale completions racing backfill; zero indexes and last-index drop -followed by rebuild; old readers across cutover/GC; disk exhaustion, cancellation, -backup/restore and rollback. Report progress, replay lag, lock-wait tails, -temporary/retained/orphan bytes, primary/vector/journal write I/O, readiness, -recall and query latency throughout. Run 50K then 1M with fixed-count churn, and -compare migrated tables with freshly created vector-store tables to detect a -permanent migration tax. +The first implementation supports local, single-shard, single-replica standalone +tables moving from `primary_lsm` to `vector_store`. It is an explicit operation, +not a setting PATCH or an automatic conversion on open. HA, replication, Lite, +serverless, schema migration, restore and topology changes are not admitted. +Index/producer/schema changes and table deletion are fenced while a job is +active. Existing native generation repair handles legacy ANN conversion; +discarded experimental formats do not gain compatibility decoders. + +### Online operator + +Use the same binary for the server and its compiled runtime libraries: + +```sh +zig/zig-out/bin/antfly storage migrate \ + --url http://127.0.0.1:8080 --table documents --to vector-store --job vectors-20260915 +``` + +The command creates a job with `POST /db/v1/tables/{table}/storage/migrations`, +requiring table admin permission when authentication is enabled. `ANTFLY_API_KEY` +supplies its Bearer token. Creation takes `{"job_id":"...","target":"vector_store", +"budget":{...}}`. `GET /db/v1/tables/{table}/storage/migrations/{job}` observes the +receipt; `POST` on that job takes `{"action":"step|publish|cancel"}` and uses its +durable budgets. GET never admits work or reconciles catalog publication. An +`admitted` receipt means the catalog marker exists but DB preparation has not +begun; retry creation or send a job action to recover that boundary. + +The CLI defaults to `--action run`, which creates/resumes the job, advances +bounded steps, and publishes when verification reaches `ready`. Actions `start`, +`step`, `publish`, `status` and `cancel` provide explicit operator control. Ctrl-C +stops the driver; durable capture continues, and running the identical command +resumes it. The server does not schedule an unattended migration loop. +Use a migration-capable server throughout the job; do not downgrade between +admission and completion or cancellation. Older binaries do not maintain the +candidate map required by an active job. + +Job ID, target and budgets form the creation idempotency contract. Keep them +equal when retrying creation, including after a timeout. Job actions use the +persisted configuration, so callers do not have to repeat budgets. DB publication +is authoritative if its response or the catalog update is lost. Opening the DB +can bridge that specific stale catalog setting using the matching durable job +and table identity. A creation/action retry reconciles the catalog decision. +The table retains its current receipt until a later job replaces it; this is +not a permanent job-history service. + +Defaults are 4 MiB and 1,024 primary rows per step, a 64 GiB temporary allowance, +and a 1 GiB free-space reserve in addition to normal resource admission. The +driver accepts `--batch-bytes`, `--batch-rows`, `--temporary-bytes` and +`--disk-reserve-bytes`. Before publication, an individual dense artifact must +fit the byte budget. Unrelated values contribute only their cursor keys to a +page. Draining hashes borrowed inline vectors into compact references before +retaining the page; an oversized vector captured after verification consumes +one page by itself, without copying its payload into page memory. +Preparation charges a conservative eight times payload/reference/metadata size, +including concurrent embedding writes; the source also checks retained candidate +bytes, covering failed preparations. This is an admission allowance, not a +measurement of physical disk usage. It deliberately overestimates preparation +cost and does not promise an exact filesystem quota. Free space is checked +before preparation. Resource rejection preserves progress and reports the +reason. Before publication, cancel and start a new ID if a larger allowance is +needed; after publication, finish draining to retire the migration allowance. +Reads/deletes continue when preparation is backpressured. + +The durable phases are: + +| Phase | Authority and work | +| --- | --- | +| `backfill` | Inline primary values remain authoritative; prepare candidate source payloads in bounded pages. | +| `verifying` / `ready` | Verify exact identity/version bindings and byte equality for the cutover corpus. Concurrent writes keep the candidate current. | +| `draining` | Ownership and the publication fence are durable. New writes use references; replace old inline values with already-prepared references. | +| `final_verification` | Prove that every live dense artifact is a valid, resolvable reference. | +| `serving` | Convert any legacy ANN generations and consolidate serving vectors into source references, retaining healthy query generations during replacement. | +| `cleanup` | Delete temporary candidate mappings. | +| `reclaiming` | Flush the final replacements once, durably request primary overlap rewrites, and advance bounded streaming compaction until those requests are discharged. | +| `complete` | Reference, serving and primary rewrite closure are certified. Source GC and reader retirement can finish reclaiming retained versions. | +| `cancelling` / `cancelled` | Before publication only: disable capture, remove candidate mappings, retain inline authority and a durable receipt. | + +Progress includes the ownership epoch, snapshot/publication fences, an exclusive +hex-encoded primary cursor, scanned/prepared/verified/rewritten counts, preparation +bytes, charged temporary allowance and the last admission error. Existing table +and index status endpoints provide source-store accounting, index readiness and +repair status. `primary_reclamation_requested` records the durable primary +rewrite request. `complete` includes discharge of those requests, but old readers, +retention windows and source GC may still hold files; it does not mean all old +files or cache pages have already been reclaimed. The request uses persistent +run metadata and ordinary admitted streaming GC. Partial level jobs and splits +carry the request even when their outputs contain no tombstones. Only a +validated full overlap rewrite clears it. A crash between the manifest request +and its job receipt safely repeats the request after reopening. + +### Mutation, reader and recovery protocol + +A compacted candidate map replaces an additional payload replay journal. Each +dense mutation prepares the payload and co-commits its full artifact-key/version +reference with the authoritative inline primary value, reference epoch and +allowance ledger. Deletes remove the candidate in the same transaction. There +is no asynchronous capture lag. The backfill compares exact current bytes before +installing a candidate, so it cannot overwrite an update or resurrect a deleted +artifact. Normal enrichment producer/source-version fencing remains in force. + +Publication commits the table setting and migration decision in one primary +transaction under write admission. Its candidate map covers the entire cutover +corpus. Mixed readers continue accepting inline bytes until draining finishes; +stable old snapshots retain their original payloads and source leases protect +reference snapshots. A live probe admitted before activation retries if it +encounters a reference without a source lease. An ambiguous preparation/commit +fences the shared DocStore, including transaction-recovery owners, until reopen. + +Draining validates and reuses the durable candidate reference; it does not +append the same payload again or create another permanent corpus. ANN format +conversion uses native generation publication and coverage checks independently +of the source rewrite. The healthy serving generation remains queryable while +its replacement is staged. The source retains candidates throughout the active +job, including cancellation, while checkpoints and memory admission continue. +Once the job finishes, ordinary snapshot/ANN ownership and journal retirement +control reclamation. Transaction and replay journals are included in total-disk +qualification; old inline payloads are not retained indefinitely for rollback. +After cancellation reaches `cancelled`, native and portable backups are eligible +again without restarting. Retained source objects may still protect existing +readers; snapshot eligibility checks durable cancellation and inline authority, +and rechecks under capture admission before selecting a snapshot. + +### Offline operator + +The same `antfly storage migrate` subcommand supports stopped-server migration. +The offline candidate uses a 64 MiB shared LSM block cache for repeated +verification point reads when the caller has not supplied a cache. It shares the +normal standalone memory budget and is released after the candidate closes. +Stop standalone, then run: + +```sh +zig/zig-out/bin/antfly storage migrate \ + --catalog /data/metadata/local-metadata.json \ + --replica-root /data/data/replicas \ + --table documents --to vector-store --job vectors-offline-20260915 +``` + +Use the actual configured catalog and replica-root paths. The command and the +new standalone runtime lock the same stable catalog sibling inode. Older +running binaries do not participate in this new operator lock: stop them first. +The command preserves unknown catalog fields and extension records. It records +offline admission before copying; standalone refuses to start while that marker +is present. `--once` executes one bounded unit and leaves a resumable candidate; +retry the same command and budgets to continue. `--cancel` discards only the +unpublished candidate, persists a cancellation receipt and clears admission. +It cannot cancel an already-published generation. + +Under exclusive generation admission, the command inventories and streams the +whole physical database root into a durable sibling, recording a synced file and +byte cursor. It preserves opaque internal namespaces, identity/version records, +artifacts and ANN state; document-only export would lose required information. +It rejects symlinks and storage configurations whose physical state is outside +the lifecycle-owned root. The shadow replays committed derived work, runs the +same source conversion/verifier and native ANN lifecycle, syncs, seals and +publishes through the existing recoverable generation exchange. Repeated restart +or a lost publication response resolves the same selected generation. Old roots +are retired by the generation lifecycle after their readers release them. + +### Qualification and remaining scope + +Recovery checks cover preparation/commit/publication boundaries, interrupted +physical copies, repeated restart, ambiguous retries, old readers, concurrent +updates/deletes, distinct models, no ANN indexes, last-index drop/rebuild, +resource rejection, cancellation and catalog fencing. The production HTTP and +offline-command suites additionally check compiled-owner routing, admission, +catalog recovery and queries across serving conversion. + +Performance qualification must compare migrated and fresh vector-store tables +at 50K and then 1M, with fixed-count churn, restart, readiness, recall, QPS/tails, +lock waits, memory and complete disk accounting. Report retained/orphan bytes and +reclamation separately from logical completion. A passing migration correctness +suite is not evidence of equivalent steady-state throughput. + +The [migration qualification findings](VECTOR_STORAGE_MIGRATION_FINDINGS.md) +record the initial screen, the WAL-only page durability fix, and the shared +restart cost found in both fresh and migrated tables. Page durability must not +force one SSTable per progress update. Query comparisons include a matched +restart in every arm so ingestion-time identity caches do not confound them. + +Reverse migration, migration-overlap backup/restore, HA/replication and broader +topology remain separately qualified work. Migration-overlap backups/restores +are rejected; a primary-only backup cannot capture reference closure. After +publication, changing the setting or booting an older binary is not rollback. +Returning to primary ownership requires a reverse conversion or a consistent +pre-migration backup with an explicit data-loss boundary. ## September 13: compiled-owner maintenance and fresh ownership comparison @@ -3690,10 +3747,10 @@ for the experimental mode until their lifecycle contracts are implemented and validated. Supported backup/restore paths must preserve reference closure; any unimplemented path must reject the operation explicitly. -Switching an existing table requires the separate migration protocol planned -above. Until it is implemented, select ownership explicitly on a fresh table -and reload the complete source data. A runtime toggle is not a rollback -mechanism for reference-only artifacts. +Switching an existing table uses the explicit offline or online protocol in +[Existing-table migration](#existing-table-migration). Direct configuration +changes remain rejected. A runtime toggle is not a rollback mechanism for +reference-only artifacts. ## Implementation sequence and acceptance diff --git a/zig/e2e/antfly/test_vector_migration.py b/zig/e2e/antfly/test_vector_migration.py new file mode 100644 index 0000000000..354313bb39 --- /dev/null +++ b/zig/e2e/antfly/test_vector_migration.py @@ -0,0 +1,590 @@ +# Copyright 2026 Antfly, Inc. +# +# Licensed under the Elastic License 2.0 (ELv2); you may not use this file +# except in compliance with the Elastic License 2.0. You may obtain a copy of +# the Elastic License 2.0 at +# +# https://www.antfly.io/licensing/ELv2-license +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# Elastic License 2.0 for the specific language governing permissions and +# limitations. + +"""Source ownership migration through the production compiled owner and catalog.""" + +import json +import math +import random +import subprocess +import time + +import pytest +import requests +from helpers import wait_until +from test_vector_store import hit_ids + + +def command(api, table, job, action="start"): + path = f"/tables/{table}/storage/migrations" + if action == "start": + return api.post( + path, + { + "job_id": job, + "target": "vector_store", + "budget": { + "batch_rows": 8, + "batch_bytes": 4096, + "disk_reserve_bytes": 0, + }, + }, + ) + if action == "status": + return api.get(f"{path}/{job}") + return api.post(f"{path}/{job}", {"action": action}) + + +def seed(api, table): + api.create_table(table, storage={"dense_embeddings": "primary_lsm"}) + for name in ("model_a", "model_b"): + api.create_index( + table, + name, + { + "name": name, + "type": "embeddings", + "external": True, + "dimension": 3, + }, + ) + api.batch_write( + table, + inserts={ + "a": { + "text": "alpha", + "_embeddings": {"model_a": [1, 0, 0], "model_b": [0, 1, 0]}, + }, + "b": { + "text": "beta", + "_embeddings": {"model_a": [0, 1, 0], "model_b": [1, 0, 0]}, + }, + }, + sync_level="full_index", + ) + assert wait_until( + lambda: nearest(api, table, "model_a", [1, 0, 0]) == ["a", "b"], timeout_s=90 + ) + + +def nearest(api, table, index, vector): + return hit_ids( + api.query_table( + table, {"embeddings": {index: vector}, "indexes": [index], "limit": 2} + ) + ) + + +def finish(api, table, job, status=None, check=None): + status = status or command(api, table, job) + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + if check: + check() + if status["phase"] in ("complete", "cancelled"): + return status + status = command( + api, table, job, "publish" if status["phase"] == "ready" else "step" + ) + # Reclamation includes timed GC admission retries and asynchronous work; + # a fixed number of tight HTTP requests is not a completion deadline. + if status["phase"] in ("serving", "reclaiming"): + time.sleep(0.01) + pytest.fail(f"migration did not finish: {status}") + + +def wait_for_ann_refresh(api, table, index, count, query): + """Activate the lazy owner, then wait for a verified clean posting sweep.""" + api.query_table( + table, {"embeddings": {index: query}, "indexes": [index], "limit": 1} + ) + + def settled(): + status = api.get_index(table, index).get("status", {}) + return ( + status.get("runtime_fresh") is True + and status.get("total_indexed") == count + and status.get("readiness", {}).get("complete") is True + and status.get("hbc_posting", {}).get("refresh_pending") is False + ) + + assert wait_until(settled, timeout_s=90, interval_s=0.1), json.dumps( + api.get_index(table, index), indent=2 + ) + + +@pytest.mark.parametrize("mode", ["restart", "online", "offline"]) +def test_vector_migration_preserves_native_ann_neighbors(stateful_api, mode): + """Compare the same built ANN before/after ownership conversion and restart.""" + api = stateful_api + table = f"neighbors_migrate_{mode}_{time.time_ns()}" + rng = random.Random(728) + + def vector(): + values = [rng.gauss(0, 1) for _ in range(64)] + length = math.sqrt(sum(x * x for x in values)) + return [x / length for x in values] + + api.create_table(table, storage={"dense_embeddings": "primary_lsm"}) + api.create_index( + table, + "model", + {"name": "model", "type": "embeddings", "external": True, "dimension": 64}, + ) + for first in range(0, 4096, 256): + api.batch_write( + table, + inserts={ + f"doc:{i:06d}": { + "text": f"document {i}", + "_embeddings": {"model": vector()}, + } + for i in range(first, first + 256) + }, + sync_level="full_index", + ) + assert wait_until( + lambda: ( + api.get_index(table, "model").get("status", {}).get("total_indexed") == 4096 + ), + timeout_s=90, + ) + queries = [vector() for _ in range(32)] + + def neighbors(): + return [ + hit_ids( + api.query_table( + table, + {"embeddings": {"model": q}, "indexes": ["model"], "limit": 10}, + ) + ) + for q in queries + ] + + # Replay completion and restart do not drain optional posting refresh. + # Dirty leaves use exact member scoring; repaired leaves use RaBitQ, so + # their approximate candidates can differ even with no storage migration. + # Certify a clean sweep after reopen before comparing the same ANN state. + api.restart_server() + wait_for_ann_refresh(api, table, "model", 4096, queries[0]) + before = neighbors() + assert all(len(hits) == 10 for hits in before) + assert neighbors() == before + if mode == "online": + status = api.post( + f"/tables/{table}/storage/migrations", + { + "job_id": "neighbors", + "target": "vector_store", + "budget": { + "batch_rows": 8192, + "batch_bytes": 8 * 1024 * 1024, + "disk_reserve_bytes": 0, + }, + }, + ) + assert finish(api, table, "neighbors", status=status)["phase"] == "complete" + elif mode == "offline": + server = api._server + api.pause_server() + try: + completed = subprocess.run( + [ + str(server.binary), + "storage", + "migrate", + "--to", + "vector-store", + "--catalog", + str(server.root / "metadata/local-metadata.json"), + "--replica-root", + str(server.replica_root), + "--table", + table, + "--job", + "neighbors", + "--disk-reserve-bytes", + "0", + ], + capture_output=True, + text=True, + timeout=180, + ) + assert completed.returncode == 0, completed.stderr + finally: + api.resume_server() + assert neighbors() == before + api.restart_server() + wait_for_ann_refresh(api, table, "model", 4096, queries[0]) + assert neighbors() == before + # A second reopen also checks the no-migration control and catches a + # baseline captured before deferred maintenance was actually certified. + api.restart_server() + wait_for_ann_refresh(api, table, "model", 4096, queries[0]) + assert neighbors() == before + + +def test_online_vector_migration_cancels_rejected_admission(stateful_api): + api = stateful_api + table = f"rejected_migrate_{time.time_ns()}" + api.create_table(table, storage={"dense_embeddings": "primary_lsm"}) + path = f"/tables/{table}/storage/migrations" + request = { + "job_id": "rejected", + "target": "vector_store", + "budget": {"disk_reserve_bytes": 2**64 - 1}, + } + with pytest.raises(requests.HTTPError) as rejected: + api.post(path, request) + assert rejected.value.response.status_code == 503 + assert "VectorMigrationDiskReserve" in rejected.value.response.text + api.restart_server() + assert command(api, table, "rejected", "status")["phase"] == "admitted" + cancelled = command(api, table, "rejected", "cancel") + assert cancelled["phase"] == "cancelled" + api.restart_server() + assert command(api, table, "rejected", "cancel") == cancelled + assert api.post(path, request) == cancelled + # A second rejected admission encounters the previous cancelled DB receipt. + request["job_id"] = "second" + with pytest.raises(requests.HTTPError) as rejected: + api.post(path, request) + assert rejected.value.response.status_code == 503 + assert command(api, table, "second", "status")["phase"] == "admitted" + assert command(api, table, "second", "cancel")["phase"] == "cancelled" + replacement = command(api, table, "replacement") + assert replacement["phase"] == "backfill" + assert finish(api, table, "replacement", status=replacement)["phase"] == "complete" + api.delete_table(table) + + +def test_online_vector_migration_restart_concurrent_models_and_rebuild(stateful_api): + api = stateful_api + table = f"online_migrate_{time.time_ns()}" + seed(api, table) + job = "online" + status = command(api, table, job) + assert status["phase"] == "backfill" + assert command(api, table, job) == status + assert command(api, table, job, "status") == status + assert command(api, table, job, "status") == status + with pytest.raises(requests.HTTPError) as missing: + command(api, table, "missing", "status") + assert missing.value.response.status_code == 404 + with pytest.raises(requests.HTTPError) as drop: + api.delete_table(table) + assert drop.value.response.status_code in (400, 409) + with pytest.raises(requests.HTTPError) as duplicate: + command(api, table, "different") + assert duplicate.value.response.status_code == 409 + command(api, table, job, "step") + api.batch_write( + table, + inserts={ + "a": { + "text": "new version", + "_embeddings": {"model_a": [0, 0, 1], "model_b": [1, 0, 0]}, + }, + "0": { + "text": "behind cursor", + "_embeddings": {"model_a": [1, 0, 0], "model_b": [0, 0, 1]}, + }, + }, + deletes=["b"], + sync_level="full_index", + ) + api.restart_server() + + def check(): + assert nearest(api, table, "model_a", [0, 0, 1]) == ["a", "0"] + assert nearest(api, table, "model_b", [0, 0, 1]) == ["0", "a"] + + assert wait_until( + lambda: nearest(api, table, "model_a", [0, 0, 1]) == ["a", "0"], timeout_s=90 + ) + status = finish(api, table, job, check=check) + assert status["phase"] == "complete" + assert status["publication_fence"] >= status["snapshot_fence"] + assert api.get_table(table)["storage"]["dense_embeddings"] == "vector_store" + with pytest.raises(requests.HTTPError) as cancel: + command(api, table, job, "cancel") + assert cancel.value.response.status_code == 409 + api.restart_server() + assert command(api, table, job, "status")["phase"] == "complete" + check() + api.delete_index(table, "model_a") + api.delete_index(table, "model_b") + api.restart_server() + api.create_index( + table, + "model_a", + {"name": "model_a", "type": "embeddings", "external": True, "dimension": 3}, + ) + assert wait_until( + lambda: nearest(api, table, "model_a", [0, 0, 1]) == ["a", "0"], timeout_s=90 + ) + + +@pytest.mark.parametrize("crash_phase", ["backfill", "reclaiming"]) +def test_online_vector_migration_page_receipt_survives_process_crash( + stateful_api, crash_phase +): + api = stateful_api + table = f"crash_migrate_{time.time_ns()}" + seed(api, table) + job = "wal-page" + state = command(api, table, job) + for _ in range(256): + action = "publish" if state["phase"] == "ready" else "step" + state = command(api, table, job, action) + if crash_phase == "backfill" and state["prepared_artifacts"]: + break + if crash_phase == "reclaiming" and state.get("primary_reclamation_requested"): + break + else: + pytest.fail(f"migration never reached {crash_phase} receipt") + # Do not allow graceful shutdown to flush the WAL-only page receipt. + api._server.proc.kill() + api._server.proc.wait(timeout=10) + api.restart_server() + recovered = command(api, table, job, "status") + for field in ( + "phase", + "cursor", + "scanned_rows", + "prepared_artifacts", + "primary_reclamation_requested", + ): + assert recovered[field] == state[field] + assert finish(api, table, job, status=recovered)["phase"] == "complete" + assert nearest(api, table, "model_a", [1, 0, 0]) == ["a", "b"] + assert nearest(api, table, "model_b", [0, 1, 0]) == ["a", "b"] + + +def test_online_vector_migration_cancellation_reopens_inline_authority(stateful_api): + api = stateful_api + table = f"cancel_migrate_{time.time_ns()}" + seed(api, table) + command(api, table, "cancel") + command(api, table, "cancel", "step") + command(api, table, "cancel", "cancel") + api.restart_server() + assert finish(api, table, "cancel")["phase"] == "cancelled" + api.restart_server() + assert api.get_table(table)["storage"]["dense_embeddings"] == "primary_lsm" + assert nearest(api, table, "model_a", [1, 0, 0]) == ["a", "b"] + # The packaged command drives the same authenticated HTTP contract. + server = api._server + completed = subprocess.run( + [ + server.binary, + "storage", + "migrate", + "--to", + "vector-store", + "--url", + server.url, + "--table", + table, + "--job", + "second", + "--batch-rows", + "8", + "--batch-bytes", + "4096", + "--disk-reserve-bytes", + "0", + ], + capture_output=True, + text=True, + timeout=180, + ) + assert completed.returncode == 0, completed.stderr + assert command(api, table, "second", "status")["phase"] == "complete" + + +def test_online_vector_migration_skips_large_documents_after_publication(stateful_api): + api = stateful_api + table = f"migration_large_document_{time.time_ns()}" + seed(api, table) + status = command(api, table, "large") + for _ in range(128): + if status["phase"] == "ready": + break + status = command(api, table, "large", "step") + assert status["phase"] == "ready" + document = {"text": "x" * 6000} + api.batch_write(table, inserts={"large": document}, sync_level="full_index") + status = command(api, table, "large", "publish") + assert status["phase"] == "draining" + api.restart_server() + assert finish(api, table, "large")["phase"] == "complete" + assert api.lookup_key(table, "large") == document + assert nearest(api, table, "model_a", [1, 0, 0]) == ["a", "b"] + + +@pytest.mark.parametrize("backup_format", ["native", "portable"]) +def test_cancelled_vector_migration_allows_backup_without_restart( + stateful_api, tmp_path, backup_format +): + api = stateful_api + table = f"migration_cancel_backup_{time.time_ns()}" + seed(api, table) + command(api, table, "cancel") + command(api, table, "cancel", "step") + command(api, table, "cancel", "cancel") + assert finish(api, table, "cancel")["phase"] == "cancelled" + assert api.get_table(table)["storage"]["dense_embeddings"] == "primary_lsm" + location = tmp_path.resolve().as_uri() + assert ( + api.backup_table( + table, backup_id="cancelled", location=location, backup_format=backup_format + )["backup"] + == "successful" + ) + api.delete_table(table) + assert api.restore_table(table, backup_id="cancelled", location=location) == { + "restore": "triggered" + } + assert api.lookup_key(table, "a")["text"] == "alpha" + api.restart_server() + assert api.lookup_key(table, "a")["text"] == "alpha" + assert nearest(api, table, "model_a", [1, 0, 0]) == ["a", "b"] + + +def test_offline_vector_migration_cancels_before_copy_fence(stateful_api): + api = stateful_api + table = f"offline_cancel_admission_{time.time_ns()}" + api.create_table(table, storage={"dense_embeddings": "primary_lsm"}) + server = api._server + api.pause_server() + catalog_path = server.root / "metadata/local-metadata.json" + + def invoke(root, job="cancel-before-fence", *extra): + return subprocess.run( + [ + str(server.binary), + "storage", + "migrate", + "--to", + "vector-store", + "--catalog", + str(catalog_path), + "--replica-root", + str(root), + "--table", + table, + "--job", + job, + "--disk-reserve-bytes", + "0", + *extra, + ], + text=True, + capture_output=True, + timeout=60, + ) + + def record(): + return next( + t + for t in json.loads(catalog_path.read_text())["tables"] + if t["name"] == table + ) + + try: + rejected = invoke( + server.root / "wrong-replica-root", "cancel-before-fence", "--once" + ) + assert rejected.returncode != 0 and "FileNotFound" in rejected.stderr + admitted_catalog = catalog_path.read_text() + assert ( + record()["storage_migration"]["request"]["job_id"] == "cancel-before-fence" + ) + cancelled = invoke(server.replica_root, "cancel-before-fence", "--cancel") + assert cancelled.returncode == 0, cancelled.stderr + assert record().get("storage_migration") is None + # Model a lost catalog publication after the durable DB cancellation. + catalog_path.write_text(admitted_catalog) + cancelled = invoke(server.replica_root, "cancel-before-fence", "--cancel") + assert cancelled.returncode == 0, cancelled.stderr + assert record().get("storage_migration") is None + retried = invoke(server.replica_root, "cancel-before-fence", "--once") + assert retried.returncode != 0 and "VectorMigrationCancelled" in retried.stderr + assert record().get("storage_migration") is None + assert record()["storage"]["dense_embeddings"] == "primary_lsm" + replacement = invoke(server.replica_root, "replacement", "--once") + assert replacement.returncode == 0, replacement.stderr + cancelled = invoke(server.replica_root, "replacement", "--cancel") + assert cancelled.returncode == 0, cancelled.stderr + assert record().get("storage_migration") is None + finally: + api.resume_server() + assert api.get_table(table)["storage"]["dense_embeddings"] == "primary_lsm" + api.delete_table(table) + + +def test_offline_vector_migration_lock_resume_catalog_and_native_queries(stateful_api): + api = stateful_api + table = f"offline_migrate_{time.time_ns()}" + seed(api, table) + server = api._server + assert server is not None and hasattr(server, "root") + argv = [ + str(server.binary), + "storage", + "migrate", + "--to", + "vector-store", + "--catalog", + str(server.root / "metadata/local-metadata.json"), + "--replica-root", + str(server.replica_root), + "--table", + table, + "--job", + "offline", + "--batch-bytes", + "4096", + "--disk-reserve-bytes", + "0", + ] + locked = subprocess.run(argv, capture_output=True, text=True, timeout=30) + assert locked.returncode != 0 and "VectorMigrationCatalogInUse" in locked.stderr + api.pause_server() + try: + pending = subprocess.run( + argv + ["--once"], capture_output=True, text=True, timeout=60 + ) + assert pending.returncode == 0, pending.stderr + catalog = json.loads((server.root / "metadata/local-metadata.json").read_text()) + record = next(t for t in catalog["tables"] if t["name"] == table) + assert record["storage"]["dense_embeddings"] == "primary_lsm" + assert record["storage_migration"]["request"]["job_id"] == "offline" + complete = subprocess.run(argv, capture_output=True, text=True, timeout=180) + assert complete.returncode == 0, complete.stderr + assert "migration complete" in complete.stderr + retry = subprocess.run(argv, capture_output=True, text=True, timeout=30) + assert retry.returncode == 0, retry.stderr + catalog = json.loads((server.root / "metadata/local-metadata.json").read_text()) + record = next(t for t in catalog["tables"] if t["name"] == table) + assert record["storage"]["dense_embeddings"] == "vector_store" + assert record.get("storage_migration") is None + finally: + api.resume_server() + assert wait_until( + lambda: nearest(api, table, "model_a", [1, 0, 0]) == ["a", "b"], timeout_s=90 + ) + assert nearest(api, table, "model_b", [1, 0, 0]) == ["b", "a"] diff --git a/zig/pkg/antfly/build/api_tests.zig b/zig/pkg/antfly/build/api_tests.zig index e602ada70f..8e9303afc0 100644 --- a/zig/pkg/antfly/build/api_tests.zig +++ b/zig/pkg/antfly/build/api_tests.zig @@ -297,6 +297,8 @@ pub fn addTests(b: *std.Build, options: AddTestsOptions) AddTestsResult { lib_resolution_source_test_step.dependOn(&run_lib_resolution_source_tests.step); const lib_api_auth_default_filters = [_][]const u8{ + "storage migration job observation preserves admitted and unpublished catalog state", + "storage migration command admission fences delayed starts across handlers", "api http server requires auth on public routes when enabled", "continuous HA rejects non-replicated public mutations before handlers", "HA mutation middleware fails closed for unregistered HTTP methods", diff --git a/zig/pkg/antfly/build/tests.zig b/zig/pkg/antfly/build/tests.zig index 4f673b4947..dd10021d51 100644 --- a/zig/pkg/antfly/build/tests.zig +++ b/zig/pkg/antfly/build/tests.zig @@ -2224,6 +2224,13 @@ pub fn addTests(b: *std.Build, options: AddTestsOptions) AddTestsResult { const lsm_backend_test_step = b.step("lsm-backend-test", "Run LSM backend unit tests only"); lsm_backend_test_step.dependOn(&run_lsm_backend_tests.step); + const vector_migration_tests = b.addTest(.{ + .root_module = antfly_test_mod, + .filters = &.{"source vector migration"}, + .test_runner = .{ .path = b.path("pkg/antfly/src/test_runner.zig"), .mode = .simple }, + }); + b.step("vector-migration-test", "Run source ownership migration and recovery tests").dependOn(&b.addRunArtifact(vector_migration_tests).step); + const resource_budget_runtime_filters = [_][]const u8{ "default tokenizer cache budget is aligned with its resource slice", "default lake range cache queue budget is aligned with its terminal resource slice", @@ -4971,6 +4978,7 @@ pub fn addTests(b: *std.Build, options: AddTestsOptions) AddTestsResult { "storage.db.transform.", "storage.db.typed_doc_values_coverage.", "storage.db.types.", + "storage.vector_migration_offline.", }, &.{"storage.hot_standby."}, &.{ @@ -5192,7 +5200,7 @@ pub fn addTests(b: *std.Build, options: AddTestsOptions) AddTestsResult { unit_storage_db_core_tests.step.dependOn(&unit_storage_shard_audit.step); @import("test_support.zig").addOwnerTestRuns(b, db_test_step, &.{ .{ .artifact = unit_storage_support_tests, .filters = &.{"storage.db."} }, - .{ .artifact = unit_storage_db_core_tests, .filters = &.{"storage.db."}, .skip_filters = unit_storage_support_compile_filters }, + .{ .artifact = unit_storage_db_core_tests, .filters = &.{ "storage.db.", "storage.vector_migration_offline." }, .skip_filters = unit_storage_support_compile_filters }, }, &release_scale_test_filters); const unit_storage_compile_step = b.step( "unit-storage-compile", diff --git a/zig/pkg/antfly/src/api/http_routes.zig b/zig/pkg/antfly/src/api/http_routes.zig index c20eaa5860..e079c425ac 100644 --- a/zig/pkg/antfly/src/api/http_routes.zig +++ b/zig/pkg/antfly/src/api/http_routes.zig @@ -551,6 +551,22 @@ pub const Routes = struct { return matchTableArtifactRepairWithSuffix(path, artifact_repair_suffix); } + pub fn matchTableStorageMigration(path: []const u8) ?TableArtifactRepair { + return matchTableArtifactRepairWithSuffix(path, "/storage/migrations"); + } + + pub const TableStorageMigrationJob = struct { table_name: []const u8, job_id: []const u8 }; + pub fn matchTableStorageMigrationJob(path: []const u8) ?TableStorageMigrationJob { + if (!std.mem.startsWith(u8, path, tables_prefix)) return null; + const tail = path[tables_prefix.len..]; + const separator = std.mem.indexOfScalar(u8, tail, '/') orelse return null; + const suffix = "/storage/migrations/"; + if (separator == 0 or !std.mem.startsWith(u8, tail[separator..], suffix)) return null; + const job_id = tail[separator + suffix.len ..]; + if (job_id.len == 0 or std.mem.indexOfScalar(u8, job_id, '/') != null) return null; + return .{ .table_name = tail[0..separator], .job_id = job_id }; + } + pub fn matchTableArtifactRepairRun(path: []const u8) ?TableArtifactRepair { return matchTableArtifactRepairWithSuffix(path, artifact_repair_run_suffix); } diff --git a/zig/pkg/antfly/src/api/http_server.zig b/zig/pkg/antfly/src/api/http_server.zig index 85ba6c71c2..80d9a57956 100644 --- a/zig/pkg/antfly/src/api/http_server.zig +++ b/zig/pkg/antfly/src/api/http_server.zig @@ -1423,6 +1423,9 @@ pub const StatusSource = struct { free_routing_snapshot: ?*const fn (ptr: *anyopaque, snapshot: *metadata_api.CatalogRoutingSnapshot) void = null, create_table: ?*const fn (ptr: *anyopaque, alloc: std.mem.Allocator, table_name: []const u8, req: tables_api.CreateTableRequest) anyerror!void = null, replace_table_definition: ?*const fn (ptr: *anyopaque, expected: metadata_table_manager.TableRecord, replacement: metadata_table_manager.TableRecord) anyerror!void = null, + publish_vector_migration_table: ?*const fn (ptr: *anyopaque, expected: metadata_table_manager.TableRecord, replacement: metadata_table_manager.TableRecord) anyerror!void = null, + begin_vector_migration_command: ?*const fn (ptr: *anyopaque, table_name: []const u8) anyerror!void = null, + end_vector_migration_command: ?*const fn (ptr: *anyopaque, table_name: []const u8) void = null, replace_table_definition_stamped: ?*const fn (ptr: *anyopaque, expected: metadata_table_manager.TableRecord, replacement: metadata_table_manager.TableRecord) anyerror!?metadata_api.CatalogMutationStamp = null, restore_table: ?*const fn ( ptr: *anyopaque, @@ -1533,6 +1536,22 @@ pub const StatusSource = struct { return try BoundaryAbi.call("replace_table_definition", self.boundary_dispatch, fn_ptr, .{ self.ptr, expected, replacement }); } + pub fn publishVectorMigrationTable(self: StatusSource, expected: metadata_table_manager.TableRecord, replacement: metadata_table_manager.TableRecord) !void { + const callback = self.vtable.publish_vector_migration_table orelse return error.VectorStoreRequiresLocalSingleShardTable; + return try BoundaryAbi.call("publish_vector_migration_table", self.boundary_dispatch, callback, .{ self.ptr, expected, replacement }); + } + + pub fn beginVectorMigrationCommand(self: StatusSource, table_name: []const u8) !void { + const callback = self.vtable.begin_vector_migration_command orelse return error.VectorStoreRequiresLocalSingleShardTable; + if (self.vtable.end_vector_migration_command == null) return error.VectorStoreRequiresLocalSingleShardTable; + try BoundaryAbi.call("begin_vector_migration_command", self.boundary_dispatch, callback, .{ self.ptr, table_name }); + } + + pub fn endVectorMigrationCommand(self: StatusSource, table_name: []const u8) void { + BoundaryAbi.call("end_vector_migration_command", self.boundary_dispatch, self.vtable.end_vector_migration_command.?, .{ self.ptr, table_name }) catch + @panic("failed to release migration command admission"); + } + pub fn replaceTableDefinitionStamped(self: StatusSource, expected: metadata_table_manager.TableRecord, replacement: metadata_table_manager.TableRecord) !?metadata_api.CatalogMutationStamp { if (self.vtable.replace_table_definition_stamped) |fn_ptr| { if (try BoundaryAbi.call("replace_table_definition_stamped", self.boundary_dispatch, fn_ptr, .{ self.ptr, expected, replacement })) |stamp| @@ -12405,6 +12424,7 @@ pub const ApiHttpServer = struct { }; defer self.source.freeAdminSnapshot(&authoritative_snapshot); const record = tables_api.findTableByName(&authoritative_snapshot, table_name) orelse return error.NotFound; + if (record.storage_migration != null) return error.UnsupportedBackupMigrationState; admitted_fence = backups_api.tableBackupFence(&authoritative_snapshot, record); if (expected_fence) |expected| { if (!expected.matches(admitted_fence)) return error.CatalogChanged; @@ -14073,6 +14093,10 @@ pub const ApiHttpServer = struct { continue; }; + if (table.storage_migration != null) { + statuses[i].@"error" = "storage migration active"; + continue; + } self.backupOwnedTableWithArtifactId( backup_io, table, @@ -15224,6 +15248,7 @@ pub const ApiHttpServer = struct { fn executeMcpDropTable(self: *ApiHttpServer, table_name: []const u8) !contextual_operations.OwnedResponse { var drop_result = self.source.dropTableExact(self.alloc, table_name) catch |err| return switch (err) { + error.VectorMigrationActive => try contextual_operations.textAlloc(self.alloc, 409, "table storage migration is active"), error.InvalidTableName => try contextual_operations.textAlloc(self.alloc, 400, "invalid table name"), error.TableNotFound => try contextual_operations.textAlloc(self.alloc, 404, "not found"), error.MetadataTopologyCommandTooLarge => try contextual_operations.textAlloc(self.alloc, 413, "table topology exceeds the 3 MiB metadata command limit; reduce the initial shard count or table definition size"), @@ -15692,6 +15717,129 @@ pub const ApiHttpServer = struct { return response; } + pub fn createStorageMigration(self: *ApiHttpServer, table_name: []const u8, body: []const u8) ![]u8 { + const migration = @import("../common/vector_migration.zig"); + var create = try std.json.parseFromSlice(migration.CreateRequest, self.alloc, body, .{}); + defer create.deinit(); + const command = migration.Command{ .action = .start, .request = .{ .job_id = create.value.job_id, .mode = .online, .budget = create.value.budget } }; + const encoded = try std.json.Stringify.valueAlloc(self.alloc, command, .{}); + defer self.alloc.free(encoded); + return self.executeVectorMigration(table_name, encoded); + } + + pub fn getStorageMigration(self: *ApiHttpServer, table_name: []const u8, job_id: []const u8) ![]u8 { + const migration = @import("../common/vector_migration.zig"); + const command = migration.Command{ .action = .status, .request = .{ .job_id = job_id, .mode = .online } }; + const encoded = try std.json.Stringify.valueAlloc(self.alloc, command, .{}); + defer self.alloc.free(encoded); + return self.executeVectorMigration(table_name, encoded); + } + + pub fn advanceStorageMigration(self: *ApiHttpServer, table_name: []const u8, job_id: []const u8, body: []const u8) ![]u8 { + const migration = @import("../common/vector_migration.zig"); + var action = try std.json.parseFromSlice(migration.JobCommand, self.alloc, body, .{}); + defer action.deinit(); + const status = try self.getStorageMigration(table_name, job_id); + defer self.alloc.free(status); + var request = try std.json.parseFromSlice(migration.Request, self.alloc, status, .{ .ignore_unknown_fields = true }); + defer request.deinit(); + const encoded = try std.json.Stringify.valueAlloc(self.alloc, migration.Command{ + .action = switch (action.value.action) { + .step => .step, + .publish => .publish, + .cancel => .cancel, + }, + .request = request.value, + }, .{}); + defer self.alloc.free(encoded); + return self.executeVectorMigration(table_name, encoded); + } + + /// Durable admission is persisted before touching the table owner. If the + /// response is lost, the same command resumes the original admitted job. + pub fn executeVectorMigration(self: *ApiHttpServer, table_name: []const u8, body: []const u8) ![]u8 { + const migration = @import("../common/vector_migration.zig"); + if (!self.cfg.deployment_mode.isStandalone() or self.source.vtable.publish_vector_migration_table == null) + return error.VectorStoreRequiresLocalSingleShardTable; + var command = try std.json.parseFromSlice(migration.Command, self.alloc, body, .{}); + defer command.deinit(); + try command.value.request.validate(); + if (command.value.request.mode != .online) return error.VectorStoreRequiresOfflineCommand; + // Acquire before the snapshot, and retain through both owner mutation + // and catalog reconciliation. A delayed command must not keep an old + // admission alive after other commands cancel and replace that job. + const mutating = command.value.action != .status; + if (mutating) try self.source.beginVectorMigrationCommand(table_name); + defer if (mutating) self.source.endVectorMigrationCommand(table_name); + var snapshot = try self.source.adminSnapshot() orelse return error.UnsupportedOperation; + defer self.source.freeAdminSnapshot(&snapshot); + var table = blk: { + for (snapshot.tables) |record| if (std.mem.eql(u8, record.name, table_name)) break :blk record; + return error.TableNotFound; + }; + if (table.desired_replica_count != 1 or table.read_schema_json.len != 0 or table.restore_backup_id.len != 0) + return error.VectorStoreRequiresLocalSingleShardTable; + var replication = try std.json.parseFromSlice(std.json.Value, self.alloc, table.replication_sources_json, .{}); + defer replication.deinit(); + if (replication.value != .array or replication.value.array.items.len != 0) return error.VectorStoreRequiresLocalSingleShardTable; + var group: ?u64 = null; + for (snapshot.ranges) |range| if (range.table_id == table.table_id) { + if (group != null or range.start_key.len != 0 or (range.end_key != null and range.end_key.?.len != 0) or + range.restore_backup_id.len != 0 or range.restore_snapshot_path.len != 0) + return error.VectorStoreRequiresLocalSingleShardTable; + group = range.group_id; + }; + const group_id = group orelse return error.TableNotFound; + const source = self.table_writes orelse return error.UnsupportedOperation; + // GET observes only. It neither admits a DB job nor publishes a catalog + // decision. Explicit creation/action retries recover those boundaries. + if (command.value.action == .status) { + if (table.storage_migration) |admission| { + if (!std.mem.eql(u8, admission.request.job_id, command.value.request.job_id)) return error.VectorMigrationNotFound; + } + return source.vectorMigrationGroupLocal(self.alloc, group_id, table_name, body) catch |err| switch (err) { + error.VectorMigrationNotFound => if (table.storage_migration) |admission| + try std.json.Stringify.valueAlloc(self.alloc, .{ .job_id = admission.request.job_id, .mode = admission.request.mode, .budget = admission.request.budget, .target = "vector_store", .phase = "admitted" }, .{}) + else + return err, + else => return err, + } orelse return error.UnsupportedOperation; + } + if (table.storage_migration) |admission| { + if (!admission.eql(.{ .request = command.value.request })) return error.VectorMigrationIdempotencyConflict; + } else if (command.value.action == .start and table.storage.dense_embeddings == .primary_lsm) { + var admitted = table; + admitted.storage_migration = .{ .request = command.value.request }; + try self.source.publishVectorMigrationTable(table, admitted); + table = admitted; + } + // A durable marker with no DB job means admission committed before a + // crash. Cancellation must also work when startup admission (for + // example its disk reserve) cannot succeed. The DB records that + // cancellation durably without preparing a source store. + if (table.storage_migration != null and command.value.action != .cancel) { + var start = command.value; + start.action = .start; + const start_body = try std.json.Stringify.valueAlloc(self.alloc, start, .{}); + defer self.alloc.free(start_body); + const receipt = try source.vectorMigrationGroupLocal(self.alloc, group_id, table_name, start_body) orelse return error.UnsupportedOperation; + self.alloc.free(receipt); + } + const result = try source.vectorMigrationGroupLocal(self.alloc, group_id, table_name, body) orelse return error.UnsupportedOperation; + errdefer self.alloc.free(result); + var job = try std.json.parseFromSlice(migration.Job, self.alloc, result, .{}); + defer job.deinit(); + try job.value.validate(); + if (table.storage_migration != null) { + var reconciled = table; + if (job.value.published()) reconciled.storage = .{ .dense_embeddings = .vector_store }; + if (!job.value.active()) reconciled.storage_migration = null; + if (!metadata_table_manager.tableDefinitionsEqual(table, reconciled)) + try self.source.publishVectorMigrationTable(table, reconciled); + } + return result; + } + const PublicRepairListRequest = struct { target: []const u8 = "artifact", kind: ?db_mod.types.ArtifactRepairKind = null, @@ -20072,6 +20220,8 @@ pub fn requiredPermissionForRequest(alloc: std.mem.Allocator, method: http_commo .POST => .admin, .GET, .PUT, .DELETE => return null, }); + if (routes.Routes.matchTableStorageMigration(path)) |table| return try tablePermission(alloc, table.table_name, .admin); + if (routes.Routes.matchTableStorageMigrationJob(path)) |job| return try tablePermission(alloc, job.table_name, .admin); if (routes.Routes.matchTableArtifactRepairRun(path)) |artifact| return try tablePermission(alloc, artifact.table_name, switch (method) { .POST => .admin, .GET, .PUT, .DELETE => return null, @@ -20582,6 +20732,218 @@ fn graphResolverValueDestinationsAllowed( return true; } +test "storage migration job observation preserves admitted and unpublished catalog state" { + const migration = @import("../common/vector_migration.zig"); + const alloc = std.testing.allocator; + const Fake = struct { + commands: migration.CommandAdmissions = .{}, + table: metadata_table_manager.TableRecord = .{ .table_id = 10, .name = "docs", .desired_replica_count = 1, .storage_migration = .{ .request = .{ .job_id = "job", .mode = .online, .budget = .{ .batch_rows = 7 } } } }, + range: metadata_table_manager.RangeRecord = .{ .group_id = 101, .table_id = 10, .start_key = "", .end_key = null }, + job: ?migration.Job = null, + mutations: usize = 0, + publications: usize = 0, + reject_start: bool = false, + fail_publication: bool = false, + fn beginMigration(ptr: *anyopaque, table_name: []const u8) !void { + try from(ptr).commands.begin(std.testing.allocator, table_name); + } + fn endMigration(ptr: *anyopaque, table_name: []const u8) void { + from(ptr).commands.end(std.testing.allocator, table_name); + } + fn from(ptr: *anyopaque) *@This() { + return @ptrCast(@alignCast(ptr)); + } + fn status(_: *anyopaque) !metadata_api.MetadataStatus { + return .{ .metadata_group_id = 1, .metrics = .{} }; + } + fn snapshot(ptr: *anyopaque) !metadata_api.AdminSnapshot { + return .{ .status = try status(ptr), .tables = @as(*[1]metadata_table_manager.TableRecord, @ptrCast(&from(ptr).table)), .ranges = @as(*[1]metadata_table_manager.RangeRecord, @ptrCast(&from(ptr).range)), .stores = &.{}, .placement_intents = &.{}, .split_transitions = &.{}, .merge_transitions = &.{} }; + } + fn free(_: *anyopaque, _: *metadata_api.AdminSnapshot) void {} + fn publish(ptr: *anyopaque, expected: metadata_table_manager.TableRecord, replacement: metadata_table_manager.TableRecord) !void { + const self = from(ptr); + try std.testing.expect(metadata_table_manager.tableDefinitionsEqual(self.table, expected)); + if (self.fail_publication) { + self.fail_publication = false; + return error.InputOutput; + } + self.table = replacement; + self.publications += 1; + } + fn batch(_: *anyopaque, _: std.mem.Allocator, _: []const u8, _: db_mod.types.BatchRequest) anyerror!?void { + return null; + } + fn command(ptr: *anyopaque, allocator: std.mem.Allocator, _: u64, _: []const u8, raw: []const u8) anyerror!?[]u8 { + const self = from(ptr); + var parsed = try std.json.parseFromSlice(migration.Command, allocator, raw, .{}); + defer parsed.deinit(); + const cmd = parsed.value; + if (cmd.action == .status) { + if (self.job == null) return error.VectorMigrationNotFound; + } else { + self.mutations += 1; + try std.testing.expectEqual(@as(u32, 7), cmd.request.budget.batch_rows); + if (cmd.action == .start and self.reject_start) return error.VectorMigrationDiskReserve; + if ((cmd.action == .start or cmd.action == .cancel) and self.job == null) self.job = .{ + .job_id = "job", + .mode = .online, + .budget = cmd.request.budget, + .table_identity = "table", + .configuration_hash = 1, + .ownership_epoch = 1, + .snapshot_fence = 1, + .replay_cursor = 1, + }; + if (cmd.action == .cancel) self.job.?.phase = .cancelled; + if (cmd.action == .step and self.job.?.phase == .backfill) self.job.?.phase = .verifying; + } + return try std.json.Stringify.valueAlloc(allocator, self.job.?, .{}); + } + }; + var fake = Fake{}; + defer fake.commands.deinit(alloc); + var server = ApiHttpServer.init(alloc, .{ .deployment_mode = .standalone }, .{ .ptr = &fake, .vtable = &.{ + .status = Fake.status, + .admin_snapshot = Fake.snapshot, + .free_admin_snapshot = Fake.free, + .publish_vector_migration_table = Fake.publish, + .begin_vector_migration_command = Fake.beginMigration, + .end_vector_migration_command = Fake.endMigration, + } }, null, .{ .ptr = &fake, .vtable = &.{ .batch = Fake.batch, .vector_migration_group_local = Fake.command } }); + defer server.deinit(); + const admitted = try server.getStorageMigration("docs", "job"); + defer alloc.free(admitted); + try std.testing.expect(std.mem.indexOf(u8, admitted, "admitted") != null); + try std.testing.expect(fake.job == null); + try std.testing.expectEqual(@as(usize, 0), fake.mutations); + try std.testing.expectEqual(@as(usize, 0), fake.publications); + try std.testing.expectError(error.VectorMigrationNotFound, server.getStorageMigration("docs", "other")); + try std.testing.expectError(error.VectorMigrationIdempotencyConflict, server.createStorageMigration("docs", "{\"job_id\":\"job\",\"target\":\"vector_store\"}")); + const advanced = try server.advanceStorageMigration("docs", "job", "{\"action\":\"step\"}"); + defer alloc.free(advanced); + try std.testing.expectEqual(migration.Phase.verifying, fake.job.?.phase); + fake.job.?.phase = .draining; + fake.job.?.publication_fence = 1; + const count = fake.mutations; + const published = try server.getStorageMigration("docs", "job"); + defer alloc.free(published); + try std.testing.expectEqual(count, fake.mutations); + try std.testing.expectEqual(@as(usize, 0), fake.publications); + try std.testing.expectEqual(.primary_lsm, fake.table.storage.dense_embeddings); + const reconciled = try server.advanceStorageMigration("docs", "job", "{\"action\":\"step\"}"); + defer alloc.free(reconciled); + try std.testing.expectEqual(@as(usize, 1), fake.publications); + try std.testing.expectEqual(.vector_store, fake.table.storage.dense_embeddings); + + // Catalog admission exists but DB startup cannot pass its reserve check. + // Cancellation must reach the owner directly, retain its receipt if the + // catalog update fails, and reconcile that receipt on an exact retry. + fake.commands.deinit(alloc); + fake = .{ .reject_start = true, .fail_publication = true }; + try std.testing.expectError(error.InputOutput, server.advanceStorageMigration("docs", "job", "{\"action\":\"cancel\"}")); + try std.testing.expectEqual(.cancelled, fake.job.?.phase); + try std.testing.expect(fake.table.storage_migration != null); + const cancelled = try server.advanceStorageMigration("docs", "job", "{\"action\":\"cancel\"}"); + defer alloc.free(cancelled); + try std.testing.expect(fake.table.storage_migration == null); + try std.testing.expectEqual(.primary_lsm, fake.table.storage.dense_embeddings); + const repeated = try server.advanceStorageMigration("docs", "job", "{\"action\":\"cancel\"}"); + defer alloc.free(repeated); + try std.testing.expectEqualSlices(u8, cancelled, repeated); +} + +test "storage migration command admission fences delayed starts across handlers" { + const migration = @import("../common/vector_migration.zig"); + const Db = @import("../storage/db/db.zig").DB; + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try std.fmt.allocPrint(alloc, ".zig-cache/tmp/{s}/review-db", .{tmp.sub_path}); + defer alloc.free(path); + var db = try Db.open(alloc, path, .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }); + defer db.close(); + const Fake = struct { + commands: migration.CommandAdmissions = .{}, + db: *Db, + server: *ApiHttpServer = undefined, + interleave: bool = true, + owns_table: bool = false, + table: metadata_table_manager.TableRecord = .{ .table_id = 10, .name = "docs", .desired_replica_count = 1 }, + range: metadata_table_manager.RangeRecord = .{ .group_id = 101, .table_id = 10, .start_key = "", .end_key = null }, + fn beginMigration(ptr: *anyopaque, table_name: []const u8) !void { + try from(ptr).commands.begin(std.testing.allocator, table_name); + } + fn endMigration(ptr: *anyopaque, table_name: []const u8) void { + from(ptr).commands.end(std.testing.allocator, table_name); + } + fn from(ptr: *anyopaque) *@This() { + return @ptrCast(@alignCast(ptr)); + } + fn status(_: *anyopaque) !metadata_api.MetadataStatus { + return .{ .metadata_group_id = 1, .metrics = .{} }; + } + fn snapshot(ptr: *anyopaque) !metadata_api.AdminSnapshot { + const tables = try std.testing.allocator.alloc(metadata_table_manager.TableRecord, 1); + tables[0] = try metadata_table_manager.cloneTable(std.testing.allocator, from(ptr).table); + return .{ .status = try status(ptr), .tables = tables, .ranges = @as(*[1]metadata_table_manager.RangeRecord, @ptrCast(&from(ptr).range)), .stores = &.{}, .placement_intents = &.{}, .split_transitions = &.{}, .merge_transitions = &.{} }; + } + fn free(_: *anyopaque, snapshot_value: *metadata_api.AdminSnapshot) void { + metadata_table_manager.freeTable(std.testing.allocator, snapshot_value.tables[0]); + std.testing.allocator.free(snapshot_value.tables); + } + fn publish(ptr: *anyopaque, expected: metadata_table_manager.TableRecord, replacement: metadata_table_manager.TableRecord) !void { + const self = from(ptr); + if (!metadata_table_manager.tableDefinitionsEqual(self.table, expected)) return error.TableGenerationChanged; + const owned = try metadata_table_manager.cloneTable(std.testing.allocator, replacement); + if (self.owns_table) metadata_table_manager.freeTable(std.testing.allocator, self.table); + self.table = owned; + self.owns_table = true; + } + fn batch(_: *anyopaque, _: std.mem.Allocator, _: []const u8, _: db_mod.types.BatchRequest) anyerror!?void { + return null; + } + fn command(ptr: *anyopaque, allocator: std.mem.Allocator, _: u64, _: []const u8, raw: []const u8) anyerror!?[]u8 { + const self = from(ptr); + var parsed = try std.json.parseFromSlice(migration.Command, allocator, raw, .{}); + defer parsed.deinit(); + if (self.interleave and parsed.value.action == .start) { + self.interleave = false; + // Reenter through a separate handler at the old race boundary. + // GET can observe admission, but cancellation/replacement must + // not pass the shared catalog owner's per-table command slot. + const observed = try self.server.getStorageMigration("docs", "A"); + defer allocator.free(observed); + try std.testing.expect(std.mem.indexOf(u8, observed, "admitted") != null); + try std.testing.expectError(error.StorageBusy, self.server.advanceStorageMigration("docs", "A", "{\"action\":\"cancel\"}")); + try std.testing.expectError(error.StorageBusy, self.server.createStorageMigration("docs", "{\"job_id\":\"B\",\"target\":\"vector_store\"}")); + try std.testing.expect(self.table.storage_migration != null); + } + return try self.db.vectorMigrationCommand(allocator, parsed.value); + } + }; + var fake = Fake{ .db = &db }; + defer fake.commands.deinit(alloc); + defer if (fake.owns_table) metadata_table_manager.freeTable(alloc, fake.table); + var server = ApiHttpServer.init(alloc, .{ .deployment_mode = .standalone }, .{ .ptr = &fake, .vtable = &.{ .status = Fake.status, .admin_snapshot = Fake.snapshot, .free_admin_snapshot = Fake.free, .publish_vector_migration_table = Fake.publish, .begin_vector_migration_command = Fake.beginMigration, .end_vector_migration_command = Fake.endMigration } }, null, .{ .ptr = &fake, .vtable = &.{ .batch = Fake.batch, .vector_migration_group_local = Fake.command } }); + defer server.deinit(); + var peer = ApiHttpServer.init(alloc, .{ .deployment_mode = .standalone }, server.source, null, server.table_writes); + defer peer.deinit(); + fake.server = &peer; + const result = try server.createStorageMigration("docs", "{\"job_id\":\"A\",\"target\":\"vector_store\",\"budget\":{\"disk_reserve_bytes\":0}}"); + defer alloc.free(result); + var job = try std.json.parseFromSlice(migration.Job, alloc, result, .{}); + defer job.deinit(); + try std.testing.expectEqual(.backfill, job.value.phase); + try std.testing.expectEqualStrings("A", fake.table.storage_migration.?.request.job_id); + alloc.free(try peer.advanceStorageMigration("docs", "A", "{\"action\":\"cancel\"}")); + alloc.free(try peer.advanceStorageMigration("docs", "A", "{\"action\":\"step\"}")); + try std.testing.expect(fake.table.storage_migration == null); + alloc.free(try peer.createStorageMigration("docs", "{\"job_id\":\"B\",\"target\":\"vector_store\",\"budget\":{\"disk_reserve_bytes\":0}}")); + try std.testing.expectError(error.VectorMigrationIdempotencyConflict, server.createStorageMigration("docs", "{\"job_id\":\"A\",\"target\":\"vector_store\",\"budget\":{\"disk_reserve_bytes\":0}}")); + try std.testing.expectEqualStrings("B", fake.table.storage_migration.?.request.job_id); + try std.testing.expectEqual(@as(usize, 0), fake.commands.tables.count()); +} + test "document artifact routes declare read and admin permissions" { { const required = (try requiredPermissionForRequest(std.testing.allocator, .GET, "/tables/docs/documents/doc%2Fa/artifacts")).?; @@ -20618,6 +20980,19 @@ test "document artifact routes declare read and admin permissions" { try std.testing.expectEqualStrings("docs", required.resource); try std.testing.expectEqual(usermgr.PermissionType.admin, required.permission_type); } + { + const required = (try requiredPermissionForRequest(std.testing.allocator, .POST, "/tables/docs/storage/migrations")).?; + defer required.deinit(std.testing.allocator); + try std.testing.expectEqual(usermgr.ResourceType.table, required.resource_type); + try std.testing.expectEqualStrings("docs", required.resource); + try std.testing.expectEqual(usermgr.PermissionType.admin, required.permission_type); + } + inline for (.{ http_common.Method.GET, http_common.Method.POST }) |method| { + const required = (try requiredPermissionForRequest(std.testing.allocator, method, "/tables/docs/storage/migrations/job")).?; + defer required.deinit(std.testing.allocator); + try std.testing.expectEqualStrings("docs", required.resource); + try std.testing.expectEqual(usermgr.PermissionType.admin, required.permission_type); + } { const required = (try requiredPermissionForRequest(std.testing.allocator, .POST, "/tables/docs/repair/run")).?; defer required.deinit(std.testing.allocator); diff --git a/zig/pkg/antfly/src/api/httpx_handler.zig b/zig/pkg/antfly/src/api/httpx_handler.zig index 8802c7c284..a410548395 100644 --- a/zig/pkg/antfly/src/api/httpx_handler.zig +++ b/zig/pkg/antfly/src/api/httpx_handler.zig @@ -5298,6 +5298,7 @@ pub const AntflyApiHandler = struct { while (true) { metadata_drop_attempts += 1; drop_result = self.api_server.source.dropTableExact(alloc, decoded_table_name) catch |err| switch (err) { + error.VectorMigrationActive => return textResponse(ctx, 409, "table storage migration is active"), error.TableNotFound => { _ = ctx.status(404); return ctx.text("not found"); @@ -5984,6 +5985,44 @@ pub const AntflyApiHandler = struct { return try self.listTableRepairIssues(ctx, table_name); } + pub fn createTableStorageMigration(self: *AntflyApiHandler, ctx: *httpx.Context, table_name: []const u8) !httpx.Response { + return self.storageMigrationResponse(ctx, table_name, null, false); + } + + pub fn getTableStorageMigration(self: *AntflyApiHandler, ctx: *httpx.Context, table_name: []const u8, job_id: []const u8) !httpx.Response { + return self.storageMigrationResponse(ctx, table_name, job_id, true); + } + + pub fn advanceTableStorageMigration(self: *AntflyApiHandler, ctx: *httpx.Context, table_name: []const u8, job_id: []const u8) !httpx.Response { + return self.storageMigrationResponse(ctx, table_name, job_id, false); + } + + fn storageMigrationResponse(self: *AntflyApiHandler, ctx: *httpx.Context, table_name: []const u8, job_path: ?[]const u8, observe: bool) !httpx.Response { + var identity: ?AuthenticatedIdentity = null; + defer if (identity) |*owned| owned.deinit(self.api_server.alloc); + if (try self.authorizeRequest(ctx, &identity)) |response| return response; + const name = (try decodePathParamOrBadRequest(ctx, table_name)) orelse return textResponse(ctx, 400, "invalid table name"); + defer ctx.allocator.free(name); + const job = if (job_path) |path| (try decodePathParamOrBadRequest(ctx, path)) orelse return textResponse(ctx, 400, "invalid job ID") else null; + defer if (job) |id| ctx.allocator.free(id); + const body = if (observe) "" else (try ctx.body()) orelse return textResponse(ctx, 400, "missing migration command"); + const result = (if (job) |id| + if (observe) self.api_server.getStorageMigration(name, id) else self.api_server.advanceStorageMigration(name, id, body) + else + self.api_server.createStorageMigration(name, body)) catch |err| { + const code: u16 = switch (err) { + error.TableNotFound, error.NotFound, error.VectorMigrationNotFound => 404, + error.VectorMigrationIdempotencyConflict, error.VectorMigrationAlreadyExists, error.VectorMigrationAlreadyPublished, error.VectorMigrationNotReady, error.VectorMigrationActive, error.VectorMigrationConfigurationChanged, error.TableGenerationChanged => 409, + error.VectorMigrationRecoveryRequired, error.VectorMigrationDiskReserve, error.VectorMigrationTemporaryBudgetExceeded, error.ResourceBudgetExceeded, error.StorageBusy, error.GenerationTransitionActive => 503, + error.InvalidVectorMigrationId, error.InvalidVectorMigrationBudget, error.VectorMigrationRowExceedsBudget, error.VectorStoreLifecycleUnsupported, error.VectorStoreRequiresLocalSingleShardTable, error.VectorStoreRequiresOfflineCommand, error.UnsupportedOperation, error.SyntaxError, error.UnexpectedToken, error.UnknownField, error.MissingField, error.InvalidEnumTag => 400, + else => 500, + }; + return textResponse(ctx, code, @errorName(err)); + }; + defer self.api_server.alloc.free(result); + return jsonResponse(ctx, 200, result); + } + pub fn runTableRepair(self: *AntflyApiHandler, ctx: *httpx.Context, table_name: []const u8) !httpx.Response { var authenticated_identity: ?AuthenticatedIdentity = null; defer if (authenticated_identity) |*identity| identity.deinit(self.api_server.alloc); diff --git a/zig/pkg/antfly/src/api/indexes.zig b/zig/pkg/antfly/src/api/indexes.zig index f01810afc4..a90bbe0439 100644 --- a/zig/pkg/antfly/src/api/indexes.zig +++ b/zig/pkg/antfly/src/api/indexes.zig @@ -2036,7 +2036,7 @@ const AggregatedIndexStatus = struct { catch_up_target_sequence: u64 = 0, text_merge: db_mod.types.TextMergeStats = .{}, hbc_cache: db_mod.types.HbcCacheStats = .{}, - hbc_posting: db_mod.types.HbcPostingStats = .{}, + hbc_posting: db_mod.types.HbcPostingStats = .{ .refresh_pending = false }, async_indexing: db_mod.types.AsyncIndexingStats = .{}, enrichment: db_mod.types.EnrichmentStats = .{}, enrichment_observation_count: u64 = 0, @@ -3044,6 +3044,7 @@ fn aggregateHbcCacheStats(dst: *db_mod.types.HbcCacheStats, src: db_mod.types.Hb } fn aggregateHbcPostingStats(dst: *db_mod.types.HbcPostingStats, src: db_mod.types.HbcPostingStats) void { + dst.refresh_pending = dst.refresh_pending or src.refresh_pending; dst.scanned_nodes += src.scanned_nodes; dst.scanned_postings += src.scanned_postings; dst.dirty_postings += src.dirty_postings; @@ -4859,6 +4860,8 @@ fn appendHbcPostingStatus(alloc: std.mem.Allocator, out: *std.ArrayListUnmanaged try appendIntValue(alloc, out, stats.lazy_payload_deferrals); try out.appendSlice(alloc, ",\"lazy_ancestor_deferrals\":"); try appendIntValue(alloc, out, stats.lazy_ancestor_deferrals); + try out.appendSlice(alloc, ",\"refresh_pending\":"); + try out.appendSlice(alloc, if (stats.refresh_pending) "true" else "false"); try out.append(alloc, '}'); } @@ -10623,3 +10626,21 @@ fn consumerTests() type { comptime { if (@import("builtin").is_test) _ = consumer_tests; } + +test "posting refresh status aggregates unknown and pending shards conservatively" { + const alloc = std.testing.allocator; + var aggregate: AggregatedIndexStatus = .{}; + aggregateHbcPostingStats(&aggregate.hbc_posting, .{ .refresh_pending = false }); + try std.testing.expect(!aggregate.hbc_posting.refresh_pending); + // A missing observation has the same conservative default as old senders. + aggregateHbcPostingStats(&aggregate.hbc_posting, .{}); + aggregateHbcPostingStats(&aggregate.hbc_posting, .{ .refresh_pending = false }); + try std.testing.expect(aggregate.hbc_posting.refresh_pending); + var encoded: std.ArrayListUnmanaged(u8) = .empty; + defer encoded.deinit(alloc); + try appendHbcPostingStatus(alloc, &encoded, aggregate.hbc_posting); + try std.testing.expect(std.mem.indexOf(u8, encoded.items, "\"refresh_pending\":true") != null); + encoded.clearRetainingCapacity(); + try appendHbcPostingStatus(alloc, &encoded, .{ .refresh_pending = false }); + try std.testing.expect(std.mem.indexOf(u8, encoded.items, "\"refresh_pending\":false") != null); +} diff --git a/zig/pkg/antfly/src/api/kernel_owner_source.zig b/zig/pkg/antfly/src/api/kernel_owner_source.zig index 5b0ff05b68..ca497f3017 100644 --- a/zig/pkg/antfly/src/api/kernel_owner_source.zig +++ b/zig/pkg/antfly/src/api/kernel_owner_source.zig @@ -359,6 +359,7 @@ pub const ProvisionedKernelOwnerSource = struct { .reprocess_document_artifact_group_local = reprocessDocumentArtifactGroupLocal, .reprocess_document_artifact_range_group_local = reprocessDocumentArtifactRangeGroupLocal, .list_artifact_repair_issues_group_local = listArtifactRepairIssuesGroupLocal, + .vector_migration_group_local = vectorMigrationGroupLocal, .graph_metric_maintenance_group_local = graphMetricMaintenanceGroupLocal, .repair_artifact_issues_group_local = repairArtifactIssuesGroupLocal, .repair_artifact_issues_group_local_controlled = repairArtifactIssuesGroupLocalControlled, @@ -2984,6 +2985,19 @@ pub const ProvisionedKernelOwnerSource = struct { return kernel_error_identity.statusFromError(err); } + fn vectorMigrationGroupLocal(ptr: *anyopaque, alloc: std.mem.Allocator, group_id: u64, table_name: []const u8, request_json: []const u8) !?[]u8 { + const self: *ProvisionedKernelOwnerSource = @ptrCast(@alignCast(ptr)); + var lease = try self.acquire(group_id, table_name); + defer lease.deinit(); + var response = lease.owner().vectorMigrationJson(table_name, request_json) catch |err| { + if (err == error.VectorMigrationRecoveryRequired or err == error.VectorPayloadStorePoisoned) + lease.retireAfterConfigurationFailure(); + return err; + }; + defer response.deinit(); + return try alloc.dupe(u8, response.bytes()); + } + fn executeArtifactOperation( self: *ProvisionedKernelOwnerSource, group_id: u64, diff --git a/zig/pkg/antfly/src/api/request_admission_policy.zig b/zig/pkg/antfly/src/api/request_admission_policy.zig index c7e69d1ec4..792a93edf5 100644 --- a/zig/pkg/antfly/src/api/request_admission_policy.zig +++ b/zig/pkg/antfly/src/api/request_admission_policy.zig @@ -93,6 +93,9 @@ pub const public_operation_policies = [_]PublicOperationPolicy{ .{ .operation_id = "getTableRepairJob", .class = .none }, .{ .operation_id = "advanceTableRepairJob", .class = .none }, .{ .operation_id = "cancelTableRepairJob", .class = .none }, + .{ .operation_id = "createTableStorageMigration", .class = .none }, + .{ .operation_id = "getTableStorageMigration", .class = .none }, + .{ .operation_id = "advanceTableStorageMigration", .class = .none }, .{ .operation_id = "runTableRepair", .class = .none }, .{ .operation_id = "restoreTable", .class = .none }, .{ .operation_id = "reauthorizeTableDestinations", .class = .none }, diff --git a/zig/pkg/antfly/src/api/table_write_source.zig b/zig/pkg/antfly/src/api/table_write_source.zig index 4e789dccdb..2deaae517b 100644 --- a/zig/pkg/antfly/src/api/table_write_source.zig +++ b/zig/pkg/antfly/src/api/table_write_source.zig @@ -77,6 +77,8 @@ pub const TableWriteSource = struct { boundary_dispatch: BoundaryAbi.Dispatch = BoundaryAbi.local_dispatch, pub const VTable = struct { + vector_migration_group_local: ?*const fn (ptr: *anyopaque, alloc: std.mem.Allocator, group_id: u64, table_name: []const u8, request_json: []const u8) anyerror!?[]u8 = null, + /// Committed replication has distinct transaction and entry-identity /// semantics from an ordinary request batch. Prepared application may /// only borrow an already configured owner, never consult the catalog. @@ -1193,6 +1195,11 @@ pub const TableWriteSource = struct { return try BoundaryAbi.call("reprocess_document_artifact_range", self.boundary_dispatch, fn_ptr, .{ self.ptr, alloc, table_name, artifact_name, req }); } + pub fn vectorMigrationGroupLocal(self: TableWriteSource, alloc: std.mem.Allocator, group_id: u64, table_name: []const u8, request_json: []const u8) !?[]u8 { + const callback = self.vtable.vector_migration_group_local orelse return null; + return try BoundaryAbi.call("vector_migration_group_local", self.boundary_dispatch, callback, .{ self.ptr, alloc, group_id, table_name, request_json }); + } + pub fn listArtifactRepairIssues( self: TableWriteSource, alloc: std.mem.Allocator, diff --git a/zig/pkg/antfly/src/api/table_writes.zig b/zig/pkg/antfly/src/api/table_writes.zig index 175382e005..03038e9d4e 100644 --- a/zig/pkg/antfly/src/api/table_writes.zig +++ b/zig/pkg/antfly/src/api/table_writes.zig @@ -6088,6 +6088,7 @@ pub const BoundTableWriteSource = struct { .repair_artifact_issues = repairArtifactIssues, .repair_artifact_issues_controlled = repairArtifactIssuesControlled, .list_artifact_repair_issues_group_local = listArtifactRepairIssuesGroupLocal, + .vector_migration_group_local = vectorMigrationGroupLocal, .repair_artifact_issues_group_local = repairArtifactIssuesGroupLocal, .repair_artifact_issues_group_local_controlled = repairArtifactIssuesGroupLocalControlled, .update_document_artifact_child_range_placement = updateDocumentArtifactChildRangePlacement, @@ -6184,6 +6185,15 @@ pub const BoundTableWriteSource = struct { return try (try self.activeDb()).repairArtifactIssuesWithRequestOptions(alloc, req, options); } + fn vectorMigrationGroupLocal(ptr: *anyopaque, alloc: std.mem.Allocator, group_id: u64, table_name: []const u8, request_json: []const u8) !?[]u8 { + const self: *BoundTableWriteSource = @ptrCast(@alignCast(ptr)); + _ = group_id; + if (!std.mem.eql(u8, table_name, self.table_name)) return null; + var command = try std.json.parseFromSlice(@import("../common/vector_migration.zig").Command, alloc, request_json, .{}); + defer command.deinit(); + return try (try self.activeDb()).vectorMigrationCommand(alloc, command.value); + } + fn listArtifactRepairIssuesGroupLocal( ptr: *anyopaque, alloc: std.mem.Allocator, @@ -20628,6 +20638,7 @@ pub const ProvisionedTableWriteSource = struct { .reprocess_document_artifact_group_local = reprocessDocumentArtifactGroupLocal, .reprocess_document_artifact_range_group_local = reprocessDocumentArtifactRangeGroupLocal, .list_artifact_repair_issues_group_local = listArtifactRepairIssuesGroupLocal, + .vector_migration_group_local = vectorMigrationGroupLocal, .repair_artifact_issues_group_local = repairArtifactIssuesGroupLocal, .repair_artifact_issues_group_local_controlled = repairArtifactIssuesGroupLocalControlled, .update_document_artifact_child_range_placement_group_local = updateDocumentArtifactChildRangePlacementGroupLocal, @@ -24887,6 +24898,19 @@ pub const ProvisionedTableWriteSource = struct { return result; } + fn vectorMigrationGroupLocal(ptr: *anyopaque, alloc: std.mem.Allocator, group_id: u64, table_name: []const u8, request_json: []const u8) !?[]u8 { + const self: *ProvisionedTableWriteSource = @ptrCast(@alignCast(ptr)); + if (comptime !control_only_storage_sources) { + if (self.localWriteOwnerSource()) |owner_source| return try owner_source.vectorMigrationGroupLocal(alloc, group_id, table_name, request_json); + } + self.beginTableRequest(table_name); + defer self.endTableRequest(table_name); + self.beginGroupOperation(table_name, group_id); + defer self.endGroupOperation(table_name, group_id); + const owner_source = self.groupLocalWriteSource() orelse return error.StorageKernelOwnerUnavailable; + return try owner_source.vectorMigrationGroupLocal(alloc, group_id, table_name, request_json); + } + fn listArtifactRepairIssuesGroupLocal( ptr: *anyopaque, alloc: std.mem.Allocator, @@ -25744,6 +25768,7 @@ pub const HostedProvisionedTableWriteSource = struct { .reprocess_document_artifact_group_local = reprocessDocumentArtifactGroupLocal, .reprocess_document_artifact_range_group_local = reprocessDocumentArtifactRangeGroupLocal, .list_artifact_repair_issues_group_local = listArtifactRepairIssuesGroupLocal, + .vector_migration_group_local = vectorMigrationGroupLocal, .repair_artifact_issues_group_local = repairArtifactIssuesGroupLocal, .repair_artifact_issues_group_local_controlled = repairArtifactIssuesGroupLocalControlled, .update_document_artifact_child_range_placement_group_local = updateDocumentArtifactChildRangePlacementGroupLocal, @@ -27465,6 +27490,12 @@ pub const HostedProvisionedTableWriteSource = struct { return result; } + fn vectorMigrationGroupLocal(ptr: *anyopaque, alloc: std.mem.Allocator, group_id: u64, table_name: []const u8, request_json: []const u8) !?[]u8 { + const self: *HostedProvisionedTableWriteSource = @ptrCast(@alignCast(ptr)); + const owner_source = self.groupLocalWriteSource() orelse return error.StorageKernelOwnerUnavailable; + return try owner_source.vectorMigrationGroupLocal(alloc, group_id, table_name, request_json); + } + fn listArtifactRepairIssuesGroupLocal( ptr: *anyopaque, alloc: std.mem.Allocator, @@ -49587,8 +49618,8 @@ fn implementationTests() type { const FakeEmbeddingProvider = struct { request_count: std.atomic.Value(u32) = .init(0), - rate_limited_count: std.atomic.Value(u32) = .init(0), - allow_all: std.atomic.Value(bool) = .init(false), + entered: std.Io.Event = .unset, + release: std.Io.Event = .unset, fn vectorForInput(input: std.json.Value) []const u8 { if (jsonValueContainsText(input, "alpha")) return "[1,0,0]"; @@ -49641,17 +49672,8 @@ fn implementationTests() type { defer parsed_req.deinit(); _ = self.request_count.fetchAdd(1, .monotonic); - if (!self.allow_all.load(.acquire)) { - _ = self.rate_limited_count.fetchAdd(1, .monotonic); - const body = try arena.dupe(u8, - \\{"error":{"message":"rate limited","type":"rate_limit_exceeded"}} - ); - return .{ - .status = 429, - .content_type = try arena.dupe(u8, "application/json"), - .body = body, - }; - } + self.entered.set(std.testing.io); + self.release.waitUncancelable(std.testing.io); const body = try successBody(arena, parsed_req.value.input); return .{ @@ -49662,7 +49684,7 @@ fn implementationTests() type { } fn allowAll(self: *@This()) void { - self.allow_all.store(true, .release); + self.release.set(std.testing.io); } }; @@ -49739,6 +49761,9 @@ fn implementationTests() type { source.read_cache = &read_cache; source.write_cache = &write_cache; source.backend_runtime = backend_runtime.ptr(); + // Release blocked HTTP work before source/cache shutdown, including + // assertion failures while the initial cached reader is inspected. + defer embedding_provider.allowAll(); _ = try source.source().batch(alloc, "docs", .{ .writes = &.{ @@ -49749,11 +49774,9 @@ fn implementationTests() type { .sync_level = .write, }); - var attempts: usize = 0; - while (attempts < 100 and embedding_provider.rate_limited_count.load(.monotonic) == 0) : (attempts += 1) { - sleepNs(50 * std.time.ns_per_ms); - } - try std.testing.expect(embedding_provider.rate_limited_count.load(.monotonic) > 0); + // Hold the response until the stale reader exists. Returning 429 + // here couples cache invalidation to unrelated provider backoff. + try embedding_provider.entered.waitTimeout(std.testing.io, .{ .duration = .{ .raw = .fromSeconds(30), .clock = .awake } }); const db_path = try metadata_mod.groupDbPathFromReplicaRoot(alloc, path, 7001); defer alloc.free(db_path); @@ -49770,14 +49793,15 @@ fn implementationTests() type { .limit = 3, }); defer initial.deinit(); - try std.testing.expect(initial.total_hits < 3); + try std.testing.expectEqual(@as(u32, 0), initial.total_hits); } embedding_provider.allowAll(); var ready = false; - attempts = 0; - while (attempts < 200) : (attempts += 1) { + var last_total_hits: u32 = 0; + const deadline_ns = platform_time.monotonicNs() + 30 * std.time.ns_per_s; + while (platform_time.monotonicNs() < deadline_ns) { { var read_lease = try read_cache.getOrOpen(db_path, FakeCatalog.iface(), 7001, 0, "docs"); defer read_lease.release(); @@ -49791,6 +49815,7 @@ fn implementationTests() type { .limit = 3, }); defer result.deinit(); + last_total_hits = result.total_hits; if (result.total_hits == 3 and result.hits.len == 3) { try std.testing.expectEqualStrings("doc:a", result.hits[0].id); ready = true; @@ -49801,6 +49826,7 @@ fn implementationTests() type { sleepNs(25 * std.time.ns_per_ms); } + if (!ready) std.debug.print("managed dense visibility timed out: provider_requests={d} cached_total_hits={d}\n", .{ embedding_provider.request_count.load(.monotonic), last_total_hits }); try std.testing.expect(ready); } diff --git a/zig/pkg/antfly/src/capi/db.zig b/zig/pkg/antfly/src/capi/db.zig index af33d853d7..c671316cac 100644 --- a/zig/pkg/antfly/src/capi/db.zig +++ b/zig/pkg/antfly/src/capi/db.zig @@ -6029,6 +6029,25 @@ fn storageOwnerArtifactJsonResponse( return .ok; } +pub fn storageOwnerVectorMigrationJson( + owner: ?*anyopaque, + request: *const kernel_owner_abi.JsonOperationRequest, + out_response: *kernel_owner_abi.OwnedBytes, +) callconv(.c) kernel_owner_abi.Status { + out_response.* = .{}; + if (request.version != kernel_owner_abi.abi_version) return .invalid_abi; + const handle = asHandle(owner) orelse return .invalid_argument; + _ = storageOwnerTableName(handle, request.table_name) orelse return .invalid_argument; + var parsed = std.json.parseFromSlice(antfly.vector_migration.Command, handle.alloc, request.request_json.slice(), .{}) catch return .invalid_argument; + defer parsed.deinit(); + // Offline publication owns a separate exclusive root transition; it may + // never run against a serving compiled owner through this online endpoint. + if (parsed.value.request.mode != .online) return .invalid_argument; + const result = handle.db.vectorMigrationCommand(handle.alloc, parsed.value) catch |err| return storageOwnerStatusFromError(err); + out_response.* = .{ .ptr = result.ptr, .len = @intCast(result.len) }; + return .ok; +} + pub fn storageOwnerArtifactOperationJson( owner: ?*anyopaque, request: *const kernel_owner_abi.ArtifactOperationRequest, @@ -6193,7 +6212,7 @@ pub fn storageOwnerRuntimeStatusJson( }; defer status.deinit(handle.alloc); status.replaceMetadata(.{ - .updated_at_ns = @import("antfly_platform").time.monotonicNs(), + .updated_at_ns = antfly.platform_time.monotonicNs(), .source = .live_writer_publish, .freshness = .fresh, .lsm_root_generation = handle.storage_owner_root_generation, @@ -6212,7 +6231,9 @@ pub fn storageOwnerRuntimeStatusJson( test "storage owner runtime status does not wait behind apply writer" { const alloc = std.testing.allocator; - const path = try tempTestPath(alloc, "storage-owner-runtime-status-busy"); + var test_tmp = try TestDirectory.init("storage-owner-runtime-status-busy"); + defer test_tmp.cleanup(); + const path = try tempTestPath(alloc, test_tmp.path(), "db"); defer alloc.free(path); cleanupTestDir(path); defer cleanupTestDir(path); @@ -6367,10 +6388,10 @@ pub fn storageOwnerMaintenance( out_result.deferred = 1; return .ok; } - const started = @import("antfly_platform").time.monotonicNs(); + const started = antfly.platform_time.monotonicNs(); var pass: usize = 0; out_result.deferred = 1; - while (pass < 64 and @import("antfly_platform").time.monotonicNs() -| started < 50 * std.time.ns_per_ms) : (pass += 1) { + while (pass < 64 and antfly.platform_time.monotonicNs() -| started < 50 * std.time.ns_per_ms) : (pass += 1) { const page = handle.db.refreshDensePostingPayloadPageBestEffort() catch |err| return storageOwnerStatusFromError(err); out_result.dense_steps += page.repaired; diff --git a/zig/pkg/antfly/src/capi_root.zig b/zig/pkg/antfly/src/capi_root.zig index 5982279bf1..d4fd00f2ba 100644 --- a/zig/pkg/antfly/src/capi_root.zig +++ b/zig/pkg/antfly/src/capi_root.zig @@ -18,6 +18,7 @@ pub const aggregation = @import("search/aggregation.zig"); pub const backup_codec = @import("storage/backup_codec.zig"); +pub const vector_migration = @import("common/vector_migration.zig"); pub const common_config = @import("common/config.zig"); pub const common_secrets = @import("common/secrets.zig"); pub const data_snapshot = @import("data/storage/shard_state_store.zig"); diff --git a/zig/pkg/antfly/src/cmd/storage.zig b/zig/pkg/antfly/src/cmd/storage.zig new file mode 100644 index 0000000000..a9a23eaba9 --- /dev/null +++ b/zig/pkg/antfly/src/cmd/storage.zig @@ -0,0 +1,254 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Storage migration operator: drive online HTTP jobs or migrate a stopped +//! table under the catalog lock shared with standalone startup. Offline mode +//! must not run alongside an older binary that does not acquire that lock. +const std = @import("std"); +const antfly = struct { + const vector_migration = @import("../common/vector_migration.zig"); + const migration_files = @import("../common/migration_files.zig"); + const metadata = @import("../metadata/mod.zig"); + const vector_migration_offline = @import("../storage/vector_migration_offline.zig"); +}; +const migration = antfly.vector_migration; + +pub fn runFromIterator(init: std.process.Init, iterator: *std.process.Args.Iterator) !void { + const kind = iterator.next() orelse { + printUsage(); + return; + }; + if (std.mem.eql(u8, kind, "--help") or std.mem.eql(u8, kind, "-h")) { + printUsage(); + return; + } + if (!std.mem.eql(u8, kind, "migrate")) return error.InvalidArguments; + const alloc = init.arena.allocator(); + var arguments: std.ArrayListUnmanaged([]const u8) = .empty; + while (iterator.next()) |arg| try arguments.append(alloc, arg); + const args = arguments.items; + var url: ?[]const u8 = null; + var action: []const u8 = "run"; + var catalog_path: ?[]const u8 = null; + var replicas: ?[]const u8 = null; + var table_name: ?[]const u8 = null; + var job_id: ?[]const u8 = null; + var target: ?[]const u8 = null; + var budget: migration.Budget = .{}; + var once = false; + var cancelling = false; + var i: usize = 0; + while (i < args.len) : (i += 1) { + if (std.mem.eql(u8, args[i], "--help") or std.mem.eql(u8, args[i], "-h")) { + printUsage(); + return; + } + if (std.mem.eql(u8, args[i], "--cancel")) { + cancelling = true; + continue; + } + if (std.mem.eql(u8, args[i], "--once")) { + once = true; + continue; + } + if (i + 1 == args.len) return error.MissingOptionValue; + const value = args[i + 1]; + if (std.mem.eql(u8, args[i], "--to")) target = value else if (std.mem.eql(u8, args[i], "--url")) url = value else if (std.mem.eql(u8, args[i], "--action")) action = value else if (std.mem.eql(u8, args[i], "--catalog")) catalog_path = value else if (std.mem.eql(u8, args[i], "--replica-root")) replicas = value else if (std.mem.eql(u8, args[i], "--table")) table_name = value else if (std.mem.eql(u8, args[i], "--job")) job_id = value else if (std.mem.eql(u8, args[i], "--batch-bytes")) budget.batch_bytes = try std.fmt.parseInt(u64, value, 10) else if (std.mem.eql(u8, args[i], "--batch-rows")) budget.batch_rows = try std.fmt.parseInt(u32, value, 10) else if (std.mem.eql(u8, args[i], "--temporary-bytes")) budget.temporary_bytes = try std.fmt.parseInt(u64, value, 10) else if (std.mem.eql(u8, args[i], "--disk-reserve-bytes")) budget.disk_reserve_bytes = try std.fmt.parseInt(u64, value, 10) else return error.UnknownOption; + i += 1; + } + if (!std.mem.eql(u8, target orelse return error.InvalidArguments, "vector-store")) return error.InvalidArguments; + if (url) |base| { + if (catalog_path != null or replicas != null or once or cancelling) return error.InvalidArguments; + return runOnline(init, base, table_name orelse return error.InvalidArguments, .{ + .job_id = job_id orelse return error.InvalidArguments, + .mode = .online, + .budget = budget, + }, action); + } + if (!std.mem.eql(u8, action, "run") or (once and cancelling)) return error.InvalidArguments; + const path = catalog_path orelse return error.ExpectedCatalogReplicaRootTableAndJob; + const root = replicas orelse return error.ExpectedCatalogReplicaRootTableAndJob; + const name = table_name orelse return error.ExpectedCatalogReplicaRootTableAndJob; + const request = migration.Request{ .job_id = job_id orelse return error.ExpectedCatalogReplicaRootTableAndJob, .mode = .offline, .budget = budget }; + try request.validate(); + const lock = try antfly.migration_files.lockCatalog(alloc, init.io, path); + defer lock.close(init.io); + const raw = try std.Io.Dir.cwd().readFileAlloc(init.io, path, alloc, .limited(64 * 1024 * 1024)); + // Preserve unknown fields and extension catalogs, changing only this + // table's ownership/marker and the catalog epoch. + var catalog = try std.json.parseFromSlice(std.json.Value, alloc, raw, .{ .allocate = .alloc_always }); + defer catalog.deinit(); + const json_alloc = catalog.arena.allocator(); + const table_values = catalog.value.object.getPtr("tables") orelse return error.InvalidCatalog; + var table_value: *std.json.Value = blk: { + for (table_values.array.items) |*entry| { + const n = entry.object.get("name") orelse continue; + if (n == .string and std.mem.eql(u8, n.string, name)) break :blk entry; + } + return error.TableNotFound; + }; + const table_json = try std.json.Stringify.valueAlloc(alloc, table_value.*, .{}); + var table = try std.json.parseFromSlice(antfly.metadata.TableRecord, alloc, table_json, .{ .ignore_unknown_fields = true }); + defer table.deinit(); + if (table.value.desired_replica_count != 1 or table.value.min_ranges != 1 or + table.value.read_schema_json.len != 0 or table.value.restore_backup_id.len != 0) return error.VectorStoreRequiresLocalSingleShardTable; + var replication = try std.json.parseFromSlice(std.json.Value, alloc, table.value.replication_sources_json, .{}); + defer replication.deinit(); + if (replication.value != .array or replication.value.array.items.len != 0) return error.VectorStoreRequiresLocalSingleShardTable; + const ranges_json = try std.json.Stringify.valueAlloc(alloc, catalog.value.object.get("ranges") orelse return error.InvalidCatalog, .{}); + var ranges = try std.json.parseFromSlice([]const antfly.metadata.RangeRecord, alloc, ranges_json, .{ .ignore_unknown_fields = true }); + defer ranges.deinit(); + const range = blk: { + var selected: ?antfly.metadata.RangeRecord = null; + for (ranges.value) |entry| if (entry.table_id == table.value.table_id) { + if (selected != null or entry.start_key.len != 0 or (entry.end_key != null and entry.end_key.?.len != 0) or entry.restore_backup_id.len != 0) + return error.VectorStoreRequiresLocalSingleShardTable; + selected = entry; + }; + break :blk selected orelse return error.TableNotFound; + }; + if (table.value.storage_migration) |admitted| { + if (!admitted.eql(.{ .request = request })) return error.VectorMigrationIdempotencyConflict; + } else if (!cancelling and table.value.storage.dense_embeddings == .primary_lsm) { + const admission_json = try std.json.Stringify.valueAlloc(alloc, migration.Admission{ .request = request }, .{}); + const admitted = try std.json.parseFromSliceLeaky(std.json.Value, json_alloc, admission_json, .{ .allocate = .alloc_always }); + try table_value.object.put(json_alloc, "storage_migration", admitted); + try publishCatalog(json_alloc, init.io, path, &catalog.value); + } + const db_path = try antfly.metadata.groupDbPathFromReplicaRoot(alloc, root, range.group_id); + if (cancelling) { + try antfly.vector_migration_offline.cancel(std.heap.smp_allocator, init.io, db_path, request, .{ .identity_namespace = .{ + .table_id = table.value.table_id, + .shard_id = antfly.metadata.table_manager.rangeDocIdentityShardId(range), + .range_id = antfly.metadata.table_manager.rangeDocIdentityRangeId(range), + } }); + _ = table_value.object.swapRemove("storage_migration"); + try publishCatalog(json_alloc, init.io, path, &catalog.value); + std.debug.print("offline vector migration cancelled\n", .{}); + return; + } + const result = antfly.vector_migration_offline.run(std.heap.smp_allocator, init.io, db_path, request, .{ + .open = .{ + .identity_namespace = .{ + .table_id = table.value.table_id, + .shard_id = antfly.metadata.table_manager.rangeDocIdentityShardId(range), + .range_id = antfly.metadata.table_manager.rangeDocIdentityRangeId(range), + }, + }, + .max_steps = if (once) 1 else 0, + .progress_fn = printProgress, + }) catch |err| { + // An exact retry can briefly readmit a terminal cancelled job. Its + // durable receipt proves no migration remains; do not strand the + // stopped server behind the marker installed by this invocation. + if (err == error.VectorMigrationCancelled) { + _ = table_value.object.swapRemove("storage_migration"); + try publishCatalog(json_alloc, init.io, path, &catalog.value); + } + return err; + }; + if (result == .complete) { + var storage = std.json.ObjectMap{}; + try storage.put(json_alloc, "dense_embeddings", .{ .string = "vector_store" }); + try table_value.object.put(json_alloc, "storage", .{ .object = storage }); + _ = table_value.object.swapRemove("storage_migration"); + try publishCatalog(json_alloc, init.io, path, &catalog.value); + } + std.debug.print("offline vector migration {s}\n", .{@tagName(result)}); +} +fn printProgress(_: ?*anyopaque, raw: []const u8) !void { + std.debug.print("{s}\n", .{raw}); +} +fn publishCatalog(alloc: std.mem.Allocator, io: std.Io, path: []const u8, value: *std.json.Value) !void { + const epoch = value.object.get("epoch") orelse std.json.Value{ .integer = 0 }; + try value.object.put(alloc, "epoch", .{ .integer = try std.math.add(i64, epoch.integer, 1) }); + const encoded = try std.json.Stringify.valueAlloc(alloc, value.*, .{}); + defer alloc.free(encoded); + try antfly.migration_files.writeAtomic(alloc, io, path, encoded); +} + +fn printUsage() void { + std.debug.print( + \\usage: antfly storage migrate --table NAME --to vector-store --job ID [options] + \\ + \\Online: --url http://HOST:PORT [--action run|start|step|publish|cancel|status] + \\Offline: --catalog PATH --replica-root PATH [--once | --cancel] + \\Budgets: --batch-bytes N --batch-rows N --temporary-bytes N --disk-reserve-bytes N + \\ + \\Online defaults to run: advance bounded steps and publish verified coverage. + \\ANTFLY_API_KEY supplies a Bearer token. Stop the driver to pause; capture continues. + \\Offline requires a stopped standalone server. --once advances one bounded step. + \\Resume with the same job ID and budgets; cancellation is prepublication only. + \\ + , .{}); +} + +fn runOnline(init: std.process.Init, base: []const u8, table: []const u8, request: migration.Request, action: []const u8) !void { + try request.validate(); + const drive = std.mem.eql(u8, action, "run"); + var next: migration.Action = if (drive) .start else std.meta.stringToEnum(migration.Action, action) orelse return error.InvalidArguments; + const alloc = init.gpa; + var encoded: std.Io.Writer.Allocating = .init(alloc); + defer encoded.deinit(); + for (table) |byte| { + if (std.ascii.isAlphanumeric(byte) or std.mem.indexOfScalar(u8, "-._~", byte) != null) { + try encoded.writer.writeByte(byte); + } else { + try encoded.writer.print("%{X:0>2}", .{byte}); + } + } + const trimmed = std.mem.trimEnd(u8, base, "/"); + const api_root = if (std.mem.endsWith(u8, trimmed, "/db/v1")) "" else "/db/v1"; + const url = try std.fmt.allocPrint(alloc, "{s}{s}/tables/{s}/storage/migrations", .{ trimmed, api_root, encoded.written() }); + defer alloc.free(url); + const job_url = try std.fmt.allocPrint(alloc, "{s}/{s}", .{ url, request.job_id }); + defer alloc.free(job_url); + const token = init.environ_map.get("ANTFLY_API_KEY"); + const authorization = if (token) |key| try std.fmt.allocPrint(alloc, "Bearer {s}", .{key}) else null; + defer if (authorization) |value| alloc.free(value); + const headers: [1][2][]const u8 = .{.{ "Authorization", authorization orelse "" }}; + var http = @import("httpx").Client.initWithConfig(alloc, init.io, .{}); + defer http.deinit(); + while (true) { + // Free each response before the next bounded step: a long migration + // must not retain its entire progress history in the process arena. + const body = if (next == .start) + try std.json.Stringify.valueAlloc(alloc, migration.CreateRequest{ .job_id = request.job_id, .target = .vector_store, .budget = request.budget }, .{}) + else + try std.json.Stringify.valueAlloc(alloc, .{ .action = next }, .{}); + defer alloc.free(body); + var response = try http.request(if (next == .status) .GET else .POST, if (next == .start) url else job_url, .{ + .json = if (next == .status) null else body, + .headers = if (authorization != null) &headers else null, + .timeout_ms = 300_000, + }); + defer response.deinit(); + const raw = response.body orelse return error.InvalidVectorMigrationState; + if (response.status.code >= 300) { + std.debug.print("migration HTTP {d}: {s}\nRetry with the same job ID and budgets after resolving the error.\n", .{ response.status.code, raw }); + return error.MigrationRequestFailed; + } + if (!drive) { + std.debug.print("{s}\n", .{raw}); + return; + } + var job = try std.json.parseFromSlice(migration.Job, alloc, raw, .{}); + defer job.deinit(); + try job.value.validate(); + std.debug.print("{s}\n", .{raw}); + if (!drive or job.value.phase == .complete or job.value.phase == .cancelled) return; + next = if (job.value.phase == .ready) .publish else .step; + if (job.value.phase == .serving) try init.io.sleep(.fromMilliseconds(250), .awake); + } +} diff --git a/zig/pkg/antfly/src/common/migration_files.zig b/zig/pkg/antfly/src/common/migration_files.zig new file mode 100644 index 0000000000..08ec88bbf9 --- /dev/null +++ b/zig/pkg/antfly/src/common/migration_files.zig @@ -0,0 +1,44 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +const std = @import("std"); +const fs = @import("fs_paths.zig"); + +pub fn writeAtomic(alloc: std.mem.Allocator, io: std.Io, path: []const u8, bytes: []const u8) !void { + const temp = try std.fmt.allocPrint(alloc, "{s}.migration-tmp", .{path}); + defer alloc.free(temp); + var file = try fs.createFilePortable(io, temp, .{ .truncate = true }); + defer file.close(io); + try file.writePositionalAll(io, bytes, 0); + try file.sync(io); + try std.Io.Dir.rename(std.Io.Dir.cwd(), temp, std.Io.Dir.cwd(), path, io); + try fs.syncDirPortable(io, std.fs.path.dirname(path) orelse "."); +} + +/// Shared by standalone metadata and stopped-server tooling. Lock a stable +/// sibling inode because atomic catalog publication replaces the catalog file. +pub fn lockCatalog(alloc: std.mem.Allocator, io: std.Io, path: []const u8) !std.Io.File { + const lock_path = try std.fmt.allocPrint(alloc, "{s}.operator-lock", .{path}); + defer alloc.free(lock_path); + if (std.fs.path.dirname(lock_path)) |parent| try fs.createDirPathPortable(io, parent); + return std.Io.Dir.cwd().createFile(io, lock_path, .{ + .read = true, + .truncate = false, + .lock = .exclusive, + .lock_nonblocking = true, + }) catch |err| switch (err) { + error.WouldBlock => error.VectorMigrationCatalogInUse, + else => err, + }; +} diff --git a/zig/pkg/antfly/src/common/mod.zig b/zig/pkg/antfly/src/common/mod.zig index 303e6f4325..05ab95acef 100644 --- a/zig/pkg/antfly/src/common/mod.zig +++ b/zig/pkg/antfly/src/common/mod.zig @@ -14,6 +14,7 @@ pub const provider_registry = @import("provider_registry.zig"); pub const config = @import("config.zig"); +pub const vector_migration = @import("vector_migration.zig"); pub const table_storage = @import("table_storage.zig"); pub const http = @import("http/mod.zig"); pub const audio_runtime = @import("audio_runtime.zig"); @@ -36,6 +37,7 @@ test { _ = provider_registry; _ = config; _ = table_storage; + _ = vector_migration; _ = http; _ = audio_runtime; _ = secrets; diff --git a/zig/pkg/antfly/src/common/topology_records.zig b/zig/pkg/antfly/src/common/topology_records.zig index 9a6ba574b7..4520363dd0 100644 --- a/zig/pkg/antfly/src/common/topology_records.zig +++ b/zig/pkg/antfly/src/common/topology_records.zig @@ -12,6 +12,7 @@ const std = @import("std"); pub const TableRecord = struct { storage: @import("table_storage.zig").Settings = .{}, + storage_migration: ?@import("vector_migration.zig").Admission = null, table_id: u64, name: []const u8, description: []const u8 = "", diff --git a/zig/pkg/antfly/src/common/vector_migration.zig b/zig/pkg/antfly/src/common/vector_migration.zig new file mode 100644 index 0000000000..a96df8c4db --- /dev/null +++ b/zig/pkg/antfly/src/common/vector_migration.zig @@ -0,0 +1,202 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Durable source-ownership migration contract, independent of ANN format. +const std = @import("std"); + +pub const format_version: u32 = 1; +pub const offline_fence_file = "VECTOR-MIGRATION-OFFLINE.json"; +pub const job_key = "\x00\x00__metadata__:vector_migration"; +pub const accounting_key = "\x00\x00__metadata__:vector_migration_bytes"; +pub const candidate_prefix = "\x00\x00__metadata__:vector_migration_candidate:"; +pub const Mode = enum { offline, online }; +pub const Phase = enum { backfill, verifying, ready, draining, final_verification, serving, cleanup, reclaiming, complete, cancelling, cancelled }; + +/// Shared by all API handlers using one standalone catalog owner. Hold a table +/// slot from before reading admission through the DB command and catalog +/// reconciliation. The catalog's process lock excludes another server/offline +/// operator; after process loss the persisted marker is the recovery authority. +pub const CommandAdmissions = struct { + mutex: std.atomic.Mutex = .unlocked, + tables: std.StringHashMapUnmanaged(void) = .empty, + + pub fn begin(self: *CommandAdmissions, alloc: std.mem.Allocator, table: []const u8) !void { + while (!self.mutex.tryLock()) std.atomic.spinLoopHint(); + defer self.mutex.unlock(); + if (self.tables.contains(table)) return error.StorageBusy; + const owned = try alloc.dupe(u8, table); + errdefer alloc.free(owned); + try self.tables.put(alloc, owned, {}); + } + + pub fn end(self: *CommandAdmissions, alloc: std.mem.Allocator, table: []const u8) void { + while (!self.mutex.tryLock()) std.atomic.spinLoopHint(); + defer self.mutex.unlock(); + const removed = self.tables.fetchRemove(table) orelse unreachable; + alloc.free(removed.key); + } + + pub fn deinit(self: *CommandAdmissions, alloc: std.mem.Allocator) void { + std.debug.assert(self.tables.count() == 0); + self.tables.deinit(alloc); + } +}; + +pub const Budget = struct { + batch_bytes: u64 = 4 * 1024 * 1024, + batch_rows: u32 = 1024, + temporary_bytes: u64 = 64 * 1024 * 1024 * 1024, + disk_reserve_bytes: u64 = 1024 * 1024 * 1024, + + pub fn validate(self: Budget) !void { + if (self.batch_rows == 0 or self.batch_rows > 65536 or + self.batch_bytes < 4096 or self.batch_bytes > 64 * 1024 * 1024 or + self.temporary_bytes < self.batch_bytes) + return error.InvalidVectorMigrationBudget; + } +}; + +pub const Request = struct { + job_id: []const u8, + mode: Mode, + budget: Budget = .{}, + + pub fn validate(self: Request) !void { + if (self.job_id.len == 0 or self.job_id.len > 128) return error.InvalidVectorMigrationId; + for (self.job_id) |byte| { + if (!std.ascii.isAlphanumeric(byte) and byte != '-' and byte != '_') + return error.InvalidVectorMigrationId; + } + try self.budget.validate(); + } +}; + +/// Catalog admission survives a lost response before the DB job exists. The +/// marker blocks incompatible definition/topology changes until the DB's +/// durable publication or cancellation has been reconciled. +pub const Admission = struct { + request: Request, + + pub fn eql(a: Admission, b: Admission) bool { + return std.mem.eql(u8, a.request.job_id, b.request.job_id) and + a.request.mode == b.request.mode and std.meta.eql(a.request.budget, b.request.budget); + } +}; + +pub fn admissionsEqual(a: ?Admission, b: ?Admission) bool { + if (a) |left| return if (b) |right| left.eql(right) else false; + return b == null; +} + +pub const Job = struct { + version: u32 = format_version, + source: @import("table_storage.zig").DenseEmbeddings = .primary_lsm, + target: @import("table_storage.zig").DenseEmbeddings = .vector_store, + job_id: []const u8, + mode: Mode, + phase: Phase = .backfill, + budget: Budget, + /// Full persisted document namespace, not a process-local table name. + table_identity: []const u8, + configuration_hash: u64, + ownership_epoch: u64, + snapshot_fence: u64, + replay_cursor: u64, + publication_fence: ?u64 = null, + /// Hex-encoded exclusive primary key; empty at a phase boundary. + cursor: []const u8 = "", + scanned_rows: u64 = 0, + prepared_artifacts: u64 = 0, + prepared_bytes: u64 = 0, + charged_temporary_bytes: u64 = 0, + verified_artifacts: u64 = 0, + rewritten_artifacts: u64 = 0, + primary_reclamation_requested: bool = false, + last_error: ?[]const u8 = null, + + pub fn published(self: Job) bool { + return self.phase == .draining or self.phase == .final_verification or self.phase == .serving or self.phase == .cleanup or self.phase == .reclaiming or self.phase == .complete; + } + + pub fn active(self: Job) bool { + return self.phase != .complete and self.phase != .cancelled; + } + + pub fn captures(self: Job) bool { + return self.phase == .backfill or self.phase == .verifying or self.phase == .ready; + } + + pub fn validate(self: Job) !void { + if (self.version != 1) return error.UnsupportedVectorMigrationVersion; + if (self.source != .primary_lsm or self.target != .vector_store) return error.UnsupportedVectorMigrationDirection; + try (Request{ .job_id = self.job_id, .mode = self.mode, .budget = self.budget }).validate(); + if (self.table_identity.len == 0 or self.ownership_epoch == 0 or self.replay_cursor < self.snapshot_fence) + return error.InvalidVectorMigrationState; + if (self.published() != (self.publication_fence != null)) return error.InvalidVectorMigrationState; + if (self.publication_fence) |fence| if (fence < self.snapshot_fence or fence > self.replay_cursor) return error.InvalidVectorMigrationState; + } +}; + +pub fn candidateKeyAlloc(alloc: std.mem.Allocator, artifact_key: []const u8) ![]u8 { + return std.mem.concat(alloc, u8, &.{ candidate_prefix, artifact_key }); +} + +/// Each step is bounded and durable; the operator may resume with the same ID. +pub const Action = enum { start, step, publish, cancel, status }; +pub const Command = struct { + action: Action, + request: Request, +}; + +/// Public job creation is distinct from the internal replay command. Execution +/// mode is selected by transport; the HTTP API admits online jobs only. +pub const CreateRequest = struct { + job_id: []const u8, + target: enum { vector_store }, + budget: Budget = .{}, +}; +pub const JobCommand = struct { action: enum { step, publish, cancel } }; + +test "source vector migration validates durable identity and publication fencing" { + const alloc = std.testing.allocator; + try std.testing.expectError(error.InvalidVectorMigrationId, (Request{ .job_id = "../other", .mode = .offline }).validate()); + try std.testing.expectError(error.InvalidVectorMigrationBudget, (Budget{ .batch_rows = 0 }).validate()); + var job: Job = .{ .job_id = "job", .mode = .online, .budget = .{}, .table_identity = "identity", .configuration_hash = 42, .ownership_epoch = 1, .snapshot_fence = 2, .replay_cursor = 3 }; + try job.validate(); + job.phase = .draining; + try std.testing.expectError(error.InvalidVectorMigrationState, job.validate()); + job.publication_fence = 1; + try std.testing.expectError(error.InvalidVectorMigrationState, job.validate()); + job.publication_fence = 3; + try job.validate(); + const encoded = try std.json.Stringify.valueAlloc(alloc, job, .{}); + defer alloc.free(encoded); + var reopened = try std.json.parseFromSlice(Job, alloc, encoded, .{}); + defer reopened.deinit(); + try reopened.value.validate(); + try std.testing.expect(reopened.value.published()); + try std.testing.expectEqual(job.publication_fence, reopened.value.publication_fence); +} + +test "source vector migration command admission isolates tables and releases slots" { + var commands: CommandAdmissions = .{}; + defer commands.deinit(std.testing.allocator); + try commands.begin(std.testing.allocator, "A"); + try std.testing.expectError(error.StorageBusy, commands.begin(std.testing.allocator, "A")); + try commands.begin(std.testing.allocator, "B"); + commands.end(std.testing.allocator, "A"); + try commands.begin(std.testing.allocator, "A"); + commands.end(std.testing.allocator, "B"); + commands.end(std.testing.allocator, "A"); +} diff --git a/zig/pkg/antfly/src/completion.zig b/zig/pkg/antfly/src/completion.zig index f187065319..7107801ab0 100644 --- a/zig/pkg/antfly/src/completion.zig +++ b/zig/pkg/antfly/src/completion.zig @@ -14,6 +14,7 @@ pub const Route = enum { data, inference, metadata, + storage, serverless, standalone, standby, @@ -79,6 +80,7 @@ pub const commands = [_]Command{ .{ .name = "agents", .description = "Run AI agents", .route = .cli, .subcommands = &agents_subcommands }, .{ .name = "backup", .description = "Back up tables", .route = .cli }, .{ .name = "restore", .description = "Restore tables", .route = .cli }, + .{ .name = "storage", .description = "Manage table storage", .route = .storage, .subcommands = &.{"migrate"} }, .{ .name = "auth", .description = "Manage users and authorization", .route = .cli, .subcommands = &auth_subcommands }, .{ .name = "internal", .description = "Run internal cluster commands", .route = .cli, .subcommands = &internal_subcommands }, .{ .name = "cloud", .description = "Delegate to the Antfly Cloud CLI", .route = .cloud }, @@ -266,6 +268,8 @@ fn writeFish(writer: *std.Io.Writer) !void { test "command table drives routes and completion entries" { try std.testing.expectEqual(Route.standalone, findCommand("swarm").?.route); try std.testing.expectEqual(Route.cli, findCommand("table").?.route); + try std.testing.expectEqual(Route.storage, findCommand("storage").?.route); + try std.testing.expectEqualStrings("migrate", findCommand("storage").?.subcommands[0]); try std.testing.expectEqual(Route.completion, findCommand("completion").?.route); try std.testing.expectEqual(Route.standby, findCommand("standby").?.route); try std.testing.expectEqual(Route.standby, findCommand("ha").?.route); diff --git a/zig/pkg/antfly/src/data/runtime.zig b/zig/pkg/antfly/src/data/runtime.zig index 11991bfdbb..b4a2c2255b 100644 --- a/zig/pkg/antfly/src/data/runtime.zig +++ b/zig/pkg/antfly/src/data/runtime.zig @@ -8155,6 +8155,11 @@ pub const DataServer = struct { self.dense_posting_maintenance_next_eligible_ns.store(posting_now_ns +| next_delay_ns, .release); if (posting.repaired > 0) { std.log.info("dense posting maintenance repaired steps={d} scanned={d} pending={}", .{ posting.repaired, posting.scanned, posting.pending }); + } + // A read-only sweep can certify refresh completion without + // repairing anything (especially after reopen). Publish that + // status transition too; certified idle rounds scan zero rows. + if (posting.repaired > 0 or posting.scanned > 0) { self.runtime_status_dirty.store(true, .release); self.markStoreStatusDirtyImmediate(); } diff --git a/zig/pkg/antfly/src/main.zig b/zig/pkg/antfly/src/main.zig index f49b08b178..88f3c903d6 100644 --- a/zig/pkg/antfly/src/main.zig +++ b/zig/pkg/antfly/src/main.zig @@ -85,6 +85,7 @@ fn mainImpl(init: std.process.Init) !void { return runRuntimeUnit(.inference, subcommand, init, &args); }, .metadata => return runRuntimeUnit(.metadata, subcommand, init, &args), + .storage => return runRuntimeUnit(.storage, subcommand, init, &args), .serverless => return runRuntimeUnit(.serverless, subcommand, init, &args), .standalone => return runRuntimeUnit(.standalone, subcommand, init, &args), .cloud => { @@ -97,13 +98,14 @@ fn mainImpl(init: std.process.Init) !void { } } -const RuntimeRole = enum { cli, data, inference, metadata, serverless, standalone, standby }; +const RuntimeRole = enum { cli, data, inference, metadata, storage, serverless, standalone, standby }; extern fn antfly_runtime_cli(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_data(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_standby(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_inference(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_metadata(context: *const runtime_bridge.Context) callconv(.c) c_int; +extern fn antfly_runtime_storage(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_serverless(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_standalone(context: *const runtime_bridge.Context) callconv(.c) c_int; extern fn antfly_runtime_lite(context: *const runtime_bridge.Context) callconv(.c) c_int; @@ -141,6 +143,7 @@ pub fn runRuntimeUnit( .standby => antfly_runtime_standby(&context), .inference => antfly_runtime_inference(&context), .metadata => antfly_runtime_metadata(&context), + .storage => antfly_runtime_storage(&context), .serverless => antfly_runtime_serverless(&context), .standalone => if (std.mem.eql(u8, command, "lite")) if (argument_views.items.len > 0 and std.mem.eql(u8, argument_views.items[0].slice(), "serve")) @@ -257,6 +260,7 @@ fn printUsage(argv0: []const u8) void { \\ agents Run AI agents (retrieval, query-builder) \\ backup Backup tables \\ restore Restore tables from backup, including Lite *.aflite input + \\ storage Manage table storage (migrate) \\ auth Manage data-plane users, roles, permissions, row filters, and API keys \\ internal Internal cluster management \\ cloud Delegate to the separate Antfly Cloud CLI diff --git a/zig/pkg/antfly/src/metadata/runtime.zig b/zig/pkg/antfly/src/metadata/runtime.zig index c6de6232b8..3fbb5b67bf 100644 --- a/zig/pkg/antfly/src/metadata/runtime.zig +++ b/zig/pkg/antfly/src/metadata/runtime.zig @@ -2762,7 +2762,7 @@ test "metadata ownership excludes colliding data placements across control round try server.bootstrapLocal(svc.metadata_group_id, 3); if (boot == 0) { try svc.upsertNode(.{ .node_id = 3 }); - try svc.raft.runRaftRoundOnly(); + try svc.runRaftRoundOnly(); try svc.upsertStore(.{ .store_id = 3, .node_id = 3, @@ -2777,7 +2777,7 @@ test "metadata ownership excludes colliding data placements across control round .store_id = 3, .peer_node_ids = &.{}, }, null, 0, false); - for (0..8) |_| try svc.raft.runRaftRoundOnly(); + for (0..8) |_| try svc.runRaftRoundOnly(); } for (0..8) |_| try server.runRound(); std.debug.print("OWNERSHIP_RED placements boot={d} expects absent foreign group\n", .{boot}); @@ -2900,7 +2900,7 @@ fn exerciseMetadataOwnershipProjection(case: MetadataOwnershipProjectionCase) !v try server.bootstrapLocal(svc.metadata_group_id, 3); if (boot == 0) { try svc.upsertNode(.{ .node_id = 3 }); - try svc.raft.runRaftRoundOnly(); + try svc.runRaftRoundOnly(); var groups = [_]antfly.metadata.table_manager.GroupStatusReport{.{ .group_id = data_group_id, .doc_count = 1, @@ -2944,7 +2944,9 @@ fn exerciseMetadataOwnershipProjection(case: MetadataOwnershipProjectionCase) !v }); } } - for (0..8) |_| try svc.raft.runRaftRoundOnly(); + // start() also runs the restore supervisor, which requests ReadIndex. + // Drive Raft through the service so both share its runtime mutex. + for (0..8) |_| try svc.runRaftRoundOnly(); switch (case) { .progress => { try expectMetadataOwnershipRemoteProgress(svc, boot, "before_control"); diff --git a/zig/pkg/antfly/src/metadata/table_manager.zig b/zig/pkg/antfly/src/metadata/table_manager.zig index 273d093e80..0e7473468e 100644 --- a/zig/pkg/antfly/src/metadata/table_manager.zig +++ b/zig/pkg/antfly/src/metadata/table_manager.zig @@ -44,7 +44,8 @@ pub const TableRecord = topology_records.TableRecord; pub const TableDefinition = TableRecord; pub fn tableDefinitionsEqual(lhs: TableDefinition, rhs: TableDefinition) bool { - return lhs.storage.dense_embeddings == rhs.storage.dense_embeddings and + return @import("../common/vector_migration.zig").admissionsEqual(lhs.storage_migration, rhs.storage_migration) and + lhs.storage.dense_embeddings == rhs.storage.dense_embeddings and lhs.table_id == rhs.table_id and std.mem.eql(u8, lhs.name, rhs.name) and std.mem.eql(u8, lhs.description, rhs.description) and @@ -71,6 +72,16 @@ fn hashTableDefinitionPart(hasher: *std.crypto.hash.sha2.Sha256, value: []const pub fn tableDefinitionFingerprint(table: TableDefinition) TableDefinitionFingerprint { var hasher = std.crypto.hash.sha2.Sha256.init(.{}); hasher.update("antfly-table-definition-v1"); + if (table.storage_migration) |migration| { + hashTableDefinitionPart(&hasher, "vector-migration-v1"); + hashTableDefinitionPart(&hasher, migration.request.job_id); + hashTableDefinitionPart(&hasher, @tagName(migration.request.mode)); + inline for (std.meta.fields(@TypeOf(migration.request.budget))) |field| { + var bytes: [8]u8 = undefined; + std.mem.writeInt(u64, &bytes, @field(migration.request.budget, field.name), .little); + hasher.update(&bytes); + } + } // Preserve fingerprints of existing default-mode tables. if (table.storage.dense_embeddings != .primary_lsm) hashTableDefinitionPart(&hasher, @tagName(table.storage.dense_embeddings)); @@ -1369,6 +1380,33 @@ pub const TableManager = struct { } pub fn upsertTable(self: *TableManager, record: TableRecord) !void { + if (self.tables.get(record.table_id)) |existing| { + if (existing.storage_migration != null and !tableDefinitionsEqual(existing, record)) + return error.VectorMigrationActive; + } + return self.upsertTableUnchecked(record); + } + + pub fn publishVectorMigrationTable(self: *TableManager, expected: TableRecord, record: TableRecord) !void { + const current = self.tables.get(expected.table_id) orelse return error.UnknownTable; + if (!tableDefinitionsEqual(current, expected)) return error.TableGenerationChanged; + var contract = record; + contract.storage = expected.storage; + contract.storage_migration = expected.storage_migration; + if (!tableDefinitionsEqual(contract, expected)) return error.VectorMigrationConfigurationChanged; + if (record.storage_migration) |admission| try admission.request.validate(); + if (expected.storage.dense_embeddings == .vector_store and record.storage.dense_embeddings != .vector_store) + return error.UnsupportedVectorMigrationDirection; + if (expected.storage_migration) |active| { + if (record.storage_migration) |next| if (!active.eql(next)) return error.VectorMigrationIdempotencyConflict; + } else if (!tableDefinitionsEqual(expected, record)) { + if (expected.storage.dense_embeddings != .primary_lsm or record.storage.dense_embeddings != .primary_lsm or + record.storage_migration == null) return error.InvalidVectorMigrationState; + } + return self.upsertTableUnchecked(record); + } + + fn upsertTableUnchecked(self: *TableManager, record: TableRecord) !void { const owned = try cloneTable(self.alloc, record); errdefer freeTable(self.alloc, owned); if (self.tables.getPtr(record.table_id)) |existing| { @@ -1382,10 +1420,25 @@ pub const TableManager = struct { pub fn upsertRange(self: *TableManager, record: RangeRecord) !void { try group_ids.requireDataGroupId(record.group_id); const table = self.tables.get(record.table_id) orelse return error.UnknownTable; - _ = table; + var normalized = record; + if (normalized.range_id == 0) normalized.range_id = normalized.group_id; + if (table.storage_migration != null) { + const existing = self.ranges.get(record.group_id) orelse return error.VectorMigrationActive; + if (!rangeRecordsEqual(existing, normalized)) return error.VectorMigrationActive; + } + + try self.installProjectedRange(normalized); + } + // Loading a complete durable projection reconstructs an already-admitted + // topology. It must not apply the live topology-change fence to its first + // range, while ordinary upserts still reject changes during migration. + fn installProjectedRange(self: *TableManager, record: RangeRecord) !void { + try group_ids.requireDataGroupId(record.group_id); + if (!self.tables.contains(record.table_id)) return error.UnknownTable; var normalized = record; if (normalized.range_id == 0) normalized.range_id = normalized.group_id; + const owned = try cloneRange(self.alloc, normalized); errdefer freeRange(self.alloc, owned); if (self.ranges.getPtr(record.group_id)) |existing| { @@ -1409,7 +1462,7 @@ pub const TableManager = struct { pub fn replaceTopology(self: *TableManager, tables: []const TableRecord, ranges: []const RangeRecord) !void { self.clearTopology(); for (tables) |record| try self.upsertTable(record); - for (ranges) |record| try self.upsertRange(record); + for (ranges) |record| try self.installProjectedRange(record); } pub const ProjectedTopologyLoadResult = struct { @@ -1426,7 +1479,7 @@ pub const TableManager = struct { result.skipped_orphan_ranges += 1; continue; } - try self.upsertRange(record); + try self.installProjectedRange(record); } return result; } @@ -1507,6 +1560,7 @@ pub const TableManager = struct { } pub fn requestSplit(self: *TableManager, intent: SplitIntent) !void { + if (self.tables.get(intent.table_id)) |table| if (table.storage_migration != null) return error.VectorMigrationActive; try group_ids.requireDataGroupId(intent.source_group_id); try group_ids.requireDataGroupId(intent.destination_group_id); const source = self.ranges.getPtr(intent.source_group_id) orelse return error.UnknownSourceRange; @@ -2091,6 +2145,9 @@ fn freeOwnedOptional(alloc: std.mem.Allocator, value: ?[]const u8) void { } pub fn cloneTable(alloc: std.mem.Allocator, record: TableRecord) !TableRecord { + var storage_migration = record.storage_migration; + if (storage_migration) |*migration| migration.request.job_id = try alloc.dupe(u8, migration.request.job_id); + errdefer if (storage_migration) |migration| alloc.free(migration.request.job_id); const name = try alloc.dupe(u8, record.name); errdefer alloc.free(name); const description = try alloc.dupe(u8, record.description); @@ -2111,6 +2168,7 @@ pub fn cloneTable(alloc: std.mem.Allocator, record: TableRecord) !TableRecord { errdefer alloc.free(restore_location); return .{ .storage = record.storage, + .storage_migration = storage_migration, .table_id = record.table_id, .name = name, .description = description, @@ -2166,6 +2224,7 @@ pub fn cloneRoutingTable(alloc: std.mem.Allocator, record: TableRecord) !TableRe } pub fn freeTable(alloc: std.mem.Allocator, record: TableRecord) void { + if (record.storage_migration) |migration| alloc.free(migration.request.job_id); alloc.free(record.name); alloc.free(record.description); alloc.free(record.schema_json); diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/client.zig b/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/client.zig index ae2072945b..e7ee856b04 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/client.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_client_openapi/client.zig @@ -1503,6 +1503,47 @@ pub const Client = struct { return ApiResponse(types.Table).fromResponse(self.allocator, &resp); } + /// Create or resume a table storage migration job + /// POST /db/v1/tables/{tableName}/storage/migrations + pub fn createTableStorageMigration(self: *@This(), table_name: []const u8, body: std.json.Value) !ApiResponse(std.json.ArrayHashMap(std.json.Value)) { + const encoded_table_name = try httpx.PercentEncoding.encode(self.allocator, table_name); + defer self.allocator.free(encoded_table_name); + const url = try std.fmt.allocPrint(self.allocator, "{s}/db/v1/tables/{s}/storage/migrations", .{ self.base_url, encoded_table_name }); + defer self.allocator.free(url); + const json_body = try httpx.json.Json.stringifyRequest(self.allocator, body); + defer self.allocator.free(json_body); + var resp = try self.http.post(url, .{ .json = json_body, .headers = self.authHeaders() }); + return ApiResponse(std.json.ArrayHashMap(std.json.Value)).fromResponse(self.allocator, &resp); + } + + /// Read a table storage migration receipt + /// GET /db/v1/tables/{tableName}/storage/migrations/{jobId} + pub fn getTableStorageMigration(self: *@This(), table_name: []const u8, job_id: []const u8) !ApiResponse(std.json.ArrayHashMap(std.json.Value)) { + const encoded_table_name = try httpx.PercentEncoding.encode(self.allocator, table_name); + defer self.allocator.free(encoded_table_name); + const encoded_job_id = try httpx.PercentEncoding.encode(self.allocator, job_id); + defer self.allocator.free(encoded_job_id); + const url = try std.fmt.allocPrint(self.allocator, "{s}/db/v1/tables/{s}/storage/migrations/{s}", .{ self.base_url, encoded_table_name, encoded_job_id }); + defer self.allocator.free(url); + var resp = try self.http.get(url, .{ .headers = self.authHeaders() }); + return ApiResponse(std.json.ArrayHashMap(std.json.Value)).fromResponse(self.allocator, &resp); + } + + /// Advance, publish or cancel a table storage migration job + /// POST /db/v1/tables/{tableName}/storage/migrations/{jobId} + pub fn advanceTableStorageMigration(self: *@This(), table_name: []const u8, job_id: []const u8, body: std.json.Value) !ApiResponse(std.json.ArrayHashMap(std.json.Value)) { + const encoded_table_name = try httpx.PercentEncoding.encode(self.allocator, table_name); + defer self.allocator.free(encoded_table_name); + const encoded_job_id = try httpx.PercentEncoding.encode(self.allocator, job_id); + defer self.allocator.free(encoded_job_id); + const url = try std.fmt.allocPrint(self.allocator, "{s}/db/v1/tables/{s}/storage/migrations/{s}", .{ self.base_url, encoded_table_name, encoded_job_id }); + defer self.allocator.free(url); + const json_body = try httpx.json.Json.stringifyRequest(self.allocator, body); + defer self.allocator.free(json_body); + var resp = try self.http.post(url, .{ .json = json_body, .headers = self.authHeaders() }); + return ApiResponse(std.json.ArrayHashMap(std.json.Value)).fromResponse(self.allocator, &resp); + } + /// List transaction sessions /// GET /db/v1/transactions pub fn listTransactionSessions(self: *@This()) !ApiResponse(types.TransactionSessionListResponse) { diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/server.zig b/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/server.zig index b6b1bf7310..565a579eca 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/server.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_metadata_openapi/server.zig @@ -486,6 +486,33 @@ pub fn parsePatchSchemaBody(allocator: std.mem.Allocator, body: []const u8) !std return std.json.parseFromSlice(types.TableSchemaPatch, allocator, body, .{ .ignore_unknown_fields = true }); } +/// Create or resume a table storage migration job +pub const CreateTableStorageMigrationPathParams = struct { + table_name: []const u8, +}; + +/// Parse the JSON request body for createTableStorageMigration. +pub fn parseCreateTableStorageMigrationBody(allocator: std.mem.Allocator, body: []const u8) !std.json.Parsed(std.json.Value) { + return std.json.parseFromSlice(std.json.Value, allocator, body, .{ .ignore_unknown_fields = true }); +} + +/// Read a table storage migration receipt +pub const GetTableStorageMigrationPathParams = struct { + table_name: []const u8, + job_id: []const u8, +}; + +/// Advance, publish or cancel a table storage migration job +pub const AdvanceTableStorageMigrationPathParams = struct { + table_name: []const u8, + job_id: []const u8, +}; + +/// Parse the JSON request body for advanceTableStorageMigration. +pub fn parseAdvanceTableStorageMigrationBody(allocator: std.mem.Allocator, body: []const u8) !std.json.Parsed(std.json.Value) { + return std.json.parseFromSlice(std.json.Value, allocator, body, .{ .ignore_unknown_fields = true }); +} + /// Parse the JSON request body for beginTransaction. pub fn parseBeginTransactionBody(allocator: std.mem.Allocator, body: []const u8) !std.json.Parsed(types.TransactionBeginRequest) { return std.json.parseFromSlice(types.TransactionBeginRequest, allocator, body, .{ .ignore_unknown_fields = true }); @@ -638,6 +665,9 @@ pub const routes = [_]Route{ .{ .method = "POST", .path = "/tables/{tableName}/restore", .operation_id = "restoreTable", .request_body = .buffered, .streaming_response = false }, .{ .method = "PUT", .path = "/tables/{tableName}/schema", .operation_id = "updateSchema", .request_body = .buffered, .streaming_response = false }, .{ .method = "PATCH", .path = "/tables/{tableName}/schema", .operation_id = "patchSchema", .request_body = .buffered, .streaming_response = false }, + .{ .method = "POST", .path = "/tables/{tableName}/storage/migrations", .operation_id = "createTableStorageMigration", .request_body = .buffered, .streaming_response = false }, + .{ .method = "GET", .path = "/tables/{tableName}/storage/migrations/{jobId}", .operation_id = "getTableStorageMigration", .request_body = .none, .streaming_response = false }, + .{ .method = "POST", .path = "/tables/{tableName}/storage/migrations/{jobId}", .operation_id = "advanceTableStorageMigration", .request_body = .buffered, .streaming_response = false }, .{ .method = "GET", .path = "/transactions", .operation_id = "listTransactionSessions", .request_body = .none, .streaming_response = false }, .{ .method = "POST", .path = "/transactions/begin", .operation_id = "beginTransaction", .request_body = .buffered, .streaming_response = false }, .{ .method = "POST", .path = "/transactions/cleanup", .operation_id = "cleanupTransactionSessions", .request_body = .none, .streaming_response = false }, @@ -719,6 +749,9 @@ pub fn ServerRouter(comptime Impl: type) type { if (!@hasDecl(Impl, "restoreTable")) @compileError("ServerRouter: Impl missing required method 'restoreTable'"); if (!@hasDecl(Impl, "updateSchema")) @compileError("ServerRouter: Impl missing required method 'updateSchema'"); if (!@hasDecl(Impl, "patchSchema")) @compileError("ServerRouter: Impl missing required method 'patchSchema'"); + if (!@hasDecl(Impl, "createTableStorageMigration")) @compileError("ServerRouter: Impl missing required method 'createTableStorageMigration'"); + if (!@hasDecl(Impl, "getTableStorageMigration")) @compileError("ServerRouter: Impl missing required method 'getTableStorageMigration'"); + if (!@hasDecl(Impl, "advanceTableStorageMigration")) @compileError("ServerRouter: Impl missing required method 'advanceTableStorageMigration'"); if (!@hasDecl(Impl, "listTransactionSessions")) @compileError("ServerRouter: Impl missing required method 'listTransactionSessions'"); if (!@hasDecl(Impl, "beginTransaction")) @compileError("ServerRouter: Impl missing required method 'beginTransaction'"); if (!@hasDecl(Impl, "cleanupTransactionSessions")) @compileError("ServerRouter: Impl missing required method 'cleanupTransactionSessions'"); @@ -798,6 +831,9 @@ pub fn ServerRouter(comptime Impl: type) type { try server.post("/tables/:tableName/restore", httpx.Handler.bind(self.impl, restoreTable)); try server.put("/tables/:tableName/schema", httpx.Handler.bind(self.impl, updateSchema)); try server.patch("/tables/:tableName/schema", httpx.Handler.bind(self.impl, patchSchema)); + try server.post("/tables/:tableName/storage/migrations", httpx.Handler.bind(self.impl, createTableStorageMigration)); + try server.get("/tables/:tableName/storage/migrations/:jobId", httpx.Handler.bind(self.impl, getTableStorageMigration)); + try server.post("/tables/:tableName/storage/migrations/:jobId", httpx.Handler.bind(self.impl, advanceTableStorageMigration)); try server.get("/transactions", httpx.Handler.bind(self.impl, listTransactionSessions)); try server.post("/transactions/begin", httpx.Handler.bind(self.impl, beginTransaction)); try server.post("/transactions/cleanup", httpx.Handler.bind(self.impl, cleanupTransactionSessions)); @@ -1241,6 +1277,29 @@ pub fn ServerRouter(comptime Impl: type) type { return impl.patchSchema(ctx, table_name); } + /// Create or resume a table storage migration job + /// POST /tables/{tableName}/storage/migrations + fn createTableStorageMigration(impl: *Impl, ctx: *httpx.Context) anyerror!httpx.Response { + const table_name = ctx.param("tableName") orelse return ctx.status(400).json(.{ .@"error" = "missing_path_param", .message = "Missing path parameter: tableName" }); + return impl.createTableStorageMigration(ctx, table_name); + } + + /// Read a table storage migration receipt + /// GET /tables/{tableName}/storage/migrations/{jobId} + fn getTableStorageMigration(impl: *Impl, ctx: *httpx.Context) anyerror!httpx.Response { + const table_name = ctx.param("tableName") orelse return ctx.status(400).json(.{ .@"error" = "missing_path_param", .message = "Missing path parameter: tableName" }); + const job_id = ctx.param("jobId") orelse return ctx.status(400).json(.{ .@"error" = "missing_path_param", .message = "Missing path parameter: jobId" }); + return impl.getTableStorageMigration(ctx, table_name, job_id); + } + + /// Advance, publish or cancel a table storage migration job + /// POST /tables/{tableName}/storage/migrations/{jobId} + fn advanceTableStorageMigration(impl: *Impl, ctx: *httpx.Context) anyerror!httpx.Response { + const table_name = ctx.param("tableName") orelse return ctx.status(400).json(.{ .@"error" = "missing_path_param", .message = "Missing path parameter: tableName" }); + const job_id = ctx.param("jobId") orelse return ctx.status(400).json(.{ .@"error" = "missing_path_param", .message = "Missing path parameter: jobId" }); + return impl.advanceTableStorageMigration(ctx, table_name, job_id); + } + /// List transaction sessions /// GET /transactions fn listTransactionSessions(impl: *Impl, ctx: *httpx.Context) anyerror!httpx.Response { @@ -1391,6 +1450,9 @@ pub fn ServerRouter(comptime Impl: type) type { // fn restoreTable(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response // fn updateSchema(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response // fn patchSchema(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response +// fn createTableStorageMigration(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response +// fn getTableStorageMigration(self: *Impl, ctx: *httpx.Context, table_name: []const u8, job_id: []const u8) !httpx.Response +// fn advanceTableStorageMigration(self: *Impl, ctx: *httpx.Context, table_name: []const u8, job_id: []const u8) !httpx.Response // fn listTransactionSessions(self: *Impl, ctx: *httpx.Context) !httpx.Response // fn beginTransaction(self: *Impl, ctx: *httpx.Context) !httpx.Response // fn cleanupTransactionSessions(self: *Impl, ctx: *httpx.Context, params: CleanupTransactionSessionsParams) !httpx.Response diff --git a/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/server.zig b/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/server.zig index 478bfa3903..7de9392995 100644 --- a/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/server.zig +++ b/zig/pkg/antfly/src/openapi/generated/antfly_public_openapi/server.zig @@ -672,6 +672,33 @@ pub fn parsePatchSchemaBody(allocator: std.mem.Allocator, body: []const u8) !std return std.json.parseFromSlice(types.TableSchemaPatch, allocator, body, .{ .ignore_unknown_fields = true }); } +/// Create or resume a table storage migration job +pub const CreateTableStorageMigrationPathParams = struct { + table_name: []const u8, +}; + +/// Parse the JSON request body for createTableStorageMigration. +pub fn parseCreateTableStorageMigrationBody(allocator: std.mem.Allocator, body: []const u8) !std.json.Parsed(std.json.Value) { + return std.json.parseFromSlice(std.json.Value, allocator, body, .{ .ignore_unknown_fields = true }); +} + +/// Read a table storage migration receipt +pub const GetTableStorageMigrationPathParams = struct { + table_name: []const u8, + job_id: []const u8, +}; + +/// Advance, publish or cancel a table storage migration job +pub const AdvanceTableStorageMigrationPathParams = struct { + table_name: []const u8, + job_id: []const u8, +}; + +/// Parse the JSON request body for advanceTableStorageMigration. +pub fn parseAdvanceTableStorageMigrationBody(allocator: std.mem.Allocator, body: []const u8) !std.json.Parsed(std.json.Value) { + return std.json.parseFromSlice(std.json.Value, allocator, body, .{ .ignore_unknown_fields = true }); +} + /// Parse the JSON request body for beginTransaction. pub fn parseBeginTransactionBody(allocator: std.mem.Allocator, body: []const u8) !std.json.Parsed(types.TransactionBeginRequest) { return std.json.parseFromSlice(types.TransactionBeginRequest, allocator, body, .{ .ignore_unknown_fields = true }); @@ -848,6 +875,9 @@ pub const routes = [_]Route{ .{ .method = "POST", .path = "/tables/{tableName}/restore", .operation_id = "restoreTable", .request_body = .buffered, .streaming_response = false }, .{ .method = "PUT", .path = "/tables/{tableName}/schema", .operation_id = "updateSchema", .request_body = .buffered, .streaming_response = false }, .{ .method = "PATCH", .path = "/tables/{tableName}/schema", .operation_id = "patchSchema", .request_body = .buffered, .streaming_response = false }, + .{ .method = "POST", .path = "/tables/{tableName}/storage/migrations", .operation_id = "createTableStorageMigration", .request_body = .buffered, .streaming_response = false }, + .{ .method = "GET", .path = "/tables/{tableName}/storage/migrations/{jobId}", .operation_id = "getTableStorageMigration", .request_body = .none, .streaming_response = false }, + .{ .method = "POST", .path = "/tables/{tableName}/storage/migrations/{jobId}", .operation_id = "advanceTableStorageMigration", .request_body = .buffered, .streaming_response = false }, .{ .method = "GET", .path = "/transactions", .operation_id = "listTransactionSessions", .request_body = .none, .streaming_response = false }, .{ .method = "POST", .path = "/transactions/begin", .operation_id = "beginTransaction", .request_body = .buffered, .streaming_response = false }, .{ .method = "POST", .path = "/transactions/cleanup", .operation_id = "cleanupTransactionSessions", .request_body = .none, .streaming_response = false }, @@ -944,6 +974,9 @@ pub const routes = [_]Route{ // fn restoreTable(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response // fn updateSchema(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response // fn patchSchema(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response +// fn createTableStorageMigration(self: *Impl, ctx: *httpx.Context, table_name: []const u8) !httpx.Response +// fn getTableStorageMigration(self: *Impl, ctx: *httpx.Context, table_name: []const u8, job_id: []const u8) !httpx.Response +// fn advanceTableStorageMigration(self: *Impl, ctx: *httpx.Context, table_name: []const u8, job_id: []const u8) !httpx.Response // fn listTransactionSessions(self: *Impl, ctx: *httpx.Context) !httpx.Response // fn beginTransaction(self: *Impl, ctx: *httpx.Context) !httpx.Response // fn cleanupTransactionSessions(self: *Impl, ctx: *httpx.Context, params: CleanupTransactionSessionsParams) !httpx.Response diff --git a/zig/pkg/antfly/src/root.zig b/zig/pkg/antfly/src/root.zig index ce9551bba1..c6afda80e6 100644 --- a/zig/pkg/antfly/src/root.zig +++ b/zig/pkg/antfly/src/root.zig @@ -145,6 +145,9 @@ pub const metadata_table_workflow = @import("metadata/table_workflow.zig"); pub const metadata_replication_backfill = @import("metadata/replication_backfill.zig"); pub const metadata_placement_planner = @import("metadata/placement_planner.zig"); pub const data = @import("data/mod.zig"); +pub const vector_migration = @import("common/vector_migration.zig"); +pub const vector_migration_offline = @import("storage/vector_migration_offline.zig"); +pub const migration_files = @import("common/migration_files.zig"); pub const standalone = @import("standalone/mod.zig"); pub const inference_runtime = @import("inference_runtime/runtime.zig"); pub const usermgr = @import("usermgr/mod.zig"); diff --git a/zig/pkg/antfly/src/runtime_error_abi.zig b/zig/pkg/antfly/src/runtime_error_abi.zig index 8acc0b3668..2e46738e25 100644 --- a/zig/pkg/antfly/src/runtime_error_abi.zig +++ b/zig/pkg/antfly/src/runtime_error_abi.zig @@ -365,6 +365,31 @@ pub const Detail = enum(c_int) { generation_transition_active, storage_busy, storage_kernel_failure, + invalid_vector_migration_budget, + invalid_vector_migration_id, + invalid_vector_migration_state, + unsupported_vector_migration_direction, + unsupported_vector_migration_version, + vector_migration_active, + vector_migration_already_exists, + vector_migration_already_published, + vector_migration_configuration_changed, + vector_migration_coverage_mismatch, + vector_migration_disk_reserve, + vector_migration_idempotency_conflict, + vector_migration_identity_mismatch, + vector_migration_inline_payload_remains, + vector_migration_not_found, + vector_migration_not_ready, + vector_migration_read_epoch_changed, + vector_migration_recovery_required, + vector_migration_row_exceeds_budget, + vector_migration_temporary_budget_exceeded, + vector_migration_offline_admission, + vector_migration_catalog_in_use, + vector_migration_copy_mismatch, + vector_migration_unsupported_file, + vector_store_requires_offline_command, }; pub const Status = extern struct { @@ -706,6 +731,31 @@ pub fn statusFromError(err: anyerror) Status { error.ImmutableTableStorageSettings => status(.conflict, .immutable_table_storage_settings), error.VectorStoreLifecycleUnsupported => status(.unsupported, .vector_store_lifecycle_unsupported), error.VectorStoreReferenceFormatRequired => status(.unsupported, .vector_store_reference_format_required), + error.InvalidVectorMigrationBudget => status(.invalid_argument, .invalid_vector_migration_budget), + error.InvalidVectorMigrationId => status(.invalid_argument, .invalid_vector_migration_id), + error.InvalidVectorMigrationState => status(.corrupt, .invalid_vector_migration_state), + error.UnsupportedVectorMigrationDirection => status(.unsupported, .unsupported_vector_migration_direction), + error.UnsupportedVectorMigrationVersion => status(.unsupported, .unsupported_vector_migration_version), + error.VectorMigrationActive => status(.conflict, .vector_migration_active), + error.VectorMigrationAlreadyExists => status(.conflict, .vector_migration_already_exists), + error.VectorMigrationAlreadyPublished => status(.conflict, .vector_migration_already_published), + error.VectorMigrationConfigurationChanged => status(.conflict, .vector_migration_configuration_changed), + error.VectorMigrationCoverageMismatch => status(.corrupt, .vector_migration_coverage_mismatch), + error.VectorMigrationDiskReserve => status(.retryable, .vector_migration_disk_reserve), + error.VectorMigrationIdempotencyConflict => status(.conflict, .vector_migration_idempotency_conflict), + error.VectorMigrationIdentityMismatch => status(.corrupt, .vector_migration_identity_mismatch), + error.VectorMigrationInlinePayloadRemains => status(.corrupt, .vector_migration_inline_payload_remains), + error.VectorMigrationNotFound => status(.not_found, .vector_migration_not_found), + error.VectorMigrationNotReady => status(.retryable, .vector_migration_not_ready), + error.VectorMigrationReadEpochChanged => status(.retryable, .vector_migration_read_epoch_changed), + error.VectorMigrationRecoveryRequired => status(.retryable, .vector_migration_recovery_required), + error.VectorMigrationRowExceedsBudget => status(.invalid_argument, .vector_migration_row_exceeds_budget), + error.VectorMigrationTemporaryBudgetExceeded => status(.retryable, .vector_migration_temporary_budget_exceeded), + error.VectorMigrationOfflineAdmission => status(.conflict, .vector_migration_offline_admission), + error.VectorMigrationCatalogInUse => status(.conflict, .vector_migration_catalog_in_use), + error.VectorMigrationCopyMismatch => status(.conflict, .vector_migration_copy_mismatch), + error.VectorMigrationUnsupportedFile => status(.conflict, .vector_migration_unsupported_file), + error.VectorStoreRequiresOfflineCommand => status(.conflict, .vector_store_requires_offline_command), error.StorageKernelFailure => status(.internal, .storage_kernel_failure), else => status(.internal, .none), }; @@ -757,6 +807,31 @@ fn detailErrorName(comptime detail: Detail) []const u8 { .immutable_table_storage_settings => "ImmutableTableStorageSettings", .vector_store_lifecycle_unsupported => "VectorStoreLifecycleUnsupported", .vector_store_reference_format_required => "VectorStoreReferenceFormatRequired", + .invalid_vector_migration_budget => "InvalidVectorMigrationBudget", + .invalid_vector_migration_id => "InvalidVectorMigrationId", + .invalid_vector_migration_state => "InvalidVectorMigrationState", + .unsupported_vector_migration_direction => "UnsupportedVectorMigrationDirection", + .unsupported_vector_migration_version => "UnsupportedVectorMigrationVersion", + .vector_migration_active => "VectorMigrationActive", + .vector_migration_already_exists => "VectorMigrationAlreadyExists", + .vector_migration_already_published => "VectorMigrationAlreadyPublished", + .vector_migration_configuration_changed => "VectorMigrationConfigurationChanged", + .vector_migration_coverage_mismatch => "VectorMigrationCoverageMismatch", + .vector_migration_disk_reserve => "VectorMigrationDiskReserve", + .vector_migration_idempotency_conflict => "VectorMigrationIdempotencyConflict", + .vector_migration_identity_mismatch => "VectorMigrationIdentityMismatch", + .vector_migration_inline_payload_remains => "VectorMigrationInlinePayloadRemains", + .vector_migration_not_found => "VectorMigrationNotFound", + .vector_migration_not_ready => "VectorMigrationNotReady", + .vector_migration_read_epoch_changed => "VectorMigrationReadEpochChanged", + .vector_migration_recovery_required => "VectorMigrationRecoveryRequired", + .vector_migration_row_exceeds_budget => "VectorMigrationRowExceedsBudget", + .vector_migration_temporary_budget_exceeded => "VectorMigrationTemporaryBudgetExceeded", + .vector_migration_offline_admission => "VectorMigrationOfflineAdmission", + .vector_migration_catalog_in_use => "VectorMigrationCatalogInUse", + .vector_migration_copy_mismatch => "VectorMigrationCopyMismatch", + .vector_migration_unsupported_file => "VectorMigrationUnsupportedFile", + .vector_store_requires_offline_command => "VectorStoreRequiresOfflineCommand", .out_of_memory => "OutOfMemory", .invalid_argument => "InvalidArgument", @@ -1157,6 +1232,7 @@ test "stable detail detection distinguishes private errors" { test "every classified boundary outcome retains its identity" { const classified = comptime blk: { + @setEvalBranchQuota(@typeInfo(Detail).@"enum".fields.len * 8); const details = std.meta.tags(Detail)[1..]; var errors: [details.len]anyerror = undefined; for (details, 0..) |detail, index| { diff --git a/zig/pkg/antfly/src/runtime_failure_abi.zig b/zig/pkg/antfly/src/runtime_failure_abi.zig index db170780ff..3fc7959a0f 100644 --- a/zig/pkg/antfly/src/runtime_failure_abi.zig +++ b/zig/pkg/antfly/src/runtime_failure_abi.zig @@ -420,6 +420,34 @@ pub const Status = enum(u32) { storage_unavailable = 462, transaction_too_large = 463, unsupported_operation = 464, + invalid_vector_migration_budget = 465, + invalid_vector_migration_id = 466, + invalid_vector_migration_state = 467, + unsupported_vector_migration_direction = 468, + unsupported_vector_migration_version = 469, + vector_migration_active = 470, + vector_migration_already_exists = 471, + vector_migration_already_published = 472, + vector_migration_configuration_changed = 473, + vector_migration_coverage_mismatch = 474, + vector_migration_disk_reserve = 475, + vector_migration_idempotency_conflict = 476, + vector_migration_identity_mismatch = 477, + vector_migration_inline_payload_remains = 478, + vector_migration_not_found = 479, + vector_migration_not_ready = 480, + vector_migration_read_epoch_changed = 481, + vector_migration_recovery_required = 482, + vector_migration_row_exceeds_budget = 483, + vector_migration_temporary_budget_exceeded = 484, + vector_migration_offline_admission = 485, + vector_migration_catalog_in_use = 486, + vector_migration_copy_mismatch = 487, + vector_migration_unsupported_file = 488, + vector_store_requires_empty_table = 489, + vector_store_requires_local_single_shard_table = 490, + vector_store_requires_offline_command = 491, + index_rebuilding = 492, }; /// Lossless failure metadata for compiled operation and per-item boundaries. diff --git a/zig/pkg/antfly/src/runtime_failure_identity.zig b/zig/pkg/antfly/src/runtime_failure_identity.zig index 730662f8f9..d8da30479e 100644 --- a/zig/pkg/antfly/src/runtime_failure_identity.zig +++ b/zig/pkg/antfly/src/runtime_failure_identity.zig @@ -29,6 +29,7 @@ const Mapping = struct { }; const mappings = [_]Mapping{ + .{ .status = .index_rebuilding, .err = error.IndexRebuilding }, .{ .status = .invalid_abi, .err = error.InvalidAbiVersion }, .{ .status = .invalid_argument, .err = error.InvalidArgument }, .{ .status = .invalid_arguments, .err = error.InvalidArguments }, @@ -427,6 +428,33 @@ const mappings = [_]Mapping{ .{ .status = .restore_dense_checkpoint_incomplete, .err = error.RestoreDenseCheckpointIncomplete }, .{ .status = .restore_index_availability_incomplete, .err = error.RestoreIndexAvailabilityIncomplete }, .{ .status = .provider_internal, .err = error.Internal }, + .{ .status = .invalid_vector_migration_budget, .err = error.InvalidVectorMigrationBudget }, + .{ .status = .invalid_vector_migration_id, .err = error.InvalidVectorMigrationId }, + .{ .status = .invalid_vector_migration_state, .err = error.InvalidVectorMigrationState }, + .{ .status = .unsupported_vector_migration_direction, .err = error.UnsupportedVectorMigrationDirection }, + .{ .status = .unsupported_vector_migration_version, .err = error.UnsupportedVectorMigrationVersion }, + .{ .status = .vector_migration_active, .err = error.VectorMigrationActive }, + .{ .status = .vector_migration_already_exists, .err = error.VectorMigrationAlreadyExists }, + .{ .status = .vector_migration_already_published, .err = error.VectorMigrationAlreadyPublished }, + .{ .status = .vector_migration_configuration_changed, .err = error.VectorMigrationConfigurationChanged }, + .{ .status = .vector_migration_coverage_mismatch, .err = error.VectorMigrationCoverageMismatch }, + .{ .status = .vector_migration_disk_reserve, .err = error.VectorMigrationDiskReserve }, + .{ .status = .vector_migration_idempotency_conflict, .err = error.VectorMigrationIdempotencyConflict }, + .{ .status = .vector_migration_identity_mismatch, .err = error.VectorMigrationIdentityMismatch }, + .{ .status = .vector_migration_inline_payload_remains, .err = error.VectorMigrationInlinePayloadRemains }, + .{ .status = .vector_migration_not_found, .err = error.VectorMigrationNotFound }, + .{ .status = .vector_migration_not_ready, .err = error.VectorMigrationNotReady }, + .{ .status = .vector_migration_read_epoch_changed, .err = error.VectorMigrationReadEpochChanged }, + .{ .status = .vector_migration_recovery_required, .err = error.VectorMigrationRecoveryRequired }, + .{ .status = .vector_migration_row_exceeds_budget, .err = error.VectorMigrationRowExceedsBudget }, + .{ .status = .vector_migration_temporary_budget_exceeded, .err = error.VectorMigrationTemporaryBudgetExceeded }, + .{ .status = .vector_migration_offline_admission, .err = error.VectorMigrationOfflineAdmission }, + .{ .status = .vector_migration_catalog_in_use, .err = error.VectorMigrationCatalogInUse }, + .{ .status = .vector_migration_copy_mismatch, .err = error.VectorMigrationCopyMismatch }, + .{ .status = .vector_migration_unsupported_file, .err = error.VectorMigrationUnsupportedFile }, + .{ .status = .vector_store_requires_empty_table, .err = error.VectorStoreRequiresEmptyTable }, + .{ .status = .vector_store_requires_local_single_shard_table, .err = error.VectorStoreRequiresLocalSingleShardTable }, + .{ .status = .vector_store_requires_offline_command, .err = error.VectorStoreRequiresOfflineCommand }, }; pub fn statusFromError(err: anyerror) abi.Status { @@ -664,5 +692,10 @@ pub fn validateForTest() !void { } test "registered storage-kernel errors are unique and round trip without losing identity" { + // A newly created/rebuilt ANN index has no serving generation yet. This + // expected state must survive both compiled query boundaries as a retry, + // rather than becoming an unregistered StorageKernelFailure (HTTP 500). + try std.testing.expectEqual(abi.Status.index_rebuilding, statusFromError(error.IndexRebuilding)); + try std.testing.expectError(error.IndexRebuilding, statusToError(.index_rebuilding)); try validateForTest(); } diff --git a/zig/pkg/antfly/src/runtime_native_abi.zig b/zig/pkg/antfly/src/runtime_native_abi.zig index 7cfcab19b5..b388a15858 100644 --- a/zig/pkg/antfly/src/runtime_native_abi.zig +++ b/zig/pkg/antfly/src/runtime_native_abi.zig @@ -10,7 +10,7 @@ const std = @import("std"); const builtin = @import("builtin"); -pub const abi_version: u32 = 6; +pub const abi_version: u32 = 7; pub const zig_compiler_id: u64 = stableId(builtin.zig_version_string); pub const TypeContract = extern struct { diff --git a/zig/pkg/antfly/src/runtime_storage_kernel_root.zig b/zig/pkg/antfly/src/runtime_storage_kernel_root.zig index 67a775eebd..289fdd52c7 100644 --- a/zig/pkg/antfly/src/runtime_storage_kernel_root.zig +++ b/zig/pkg/antfly/src/runtime_storage_kernel_root.zig @@ -37,6 +37,14 @@ fn liteEntry(context: *const bridge.Context) callconv(.c) c_int { return runtimeEntry(context, "lite", runLite); } +fn runStorage(init: std.process.Init, _: []const u8, args: *std.process.Args.Iterator) !void { + return @import("cmd/storage.zig").runFromIterator(init, args); +} + +fn storageEntry(context: *const bridge.Context) callconv(.c) c_int { + return runtimeEntry(context, "storage", runStorage); +} + comptime { // The kernel owns physical DB and local-query compilation plus // the C API. Product-mode orchestration stays in the distributed @@ -45,6 +53,7 @@ comptime { _ = storage_kernel_exports; exportInternal(&storage_kernel_exports.storageOwnerMergeArtifactsPage, "antfly_storage_owner_merge_artifacts_page"); exportInternal(&liteEntry, "antfly_runtime_lite"); + exportInternal(&storageEntry, "antfly_runtime_storage"); exportInternal(&restore_staging_exports.create, "antfly_restore_staging_create"); exportInternal(&restore_staging_exports.destroy, "antfly_restore_staging_destroy"); exportInternal(&@import("storage/db/enrichment/enrichment_types.zig").interactiveActivity, "antfly_storage_interactive_activity"); @@ -154,6 +163,7 @@ comptime { exportInternal(&storage_kernel_exports.storageOwnerGraphEdgesJson, "antfly_storage_owner_graph_edges_json"); exportInternal(&storage_kernel_exports.storageOwnerDocumentArtifactManifestJson, "antfly_storage_owner_document_artifact_manifest_json"); exportInternal(&storage_kernel_exports.storageOwnerDocumentArtifactManifestsJson, "antfly_storage_owner_document_artifact_manifests_json"); + exportInternal(&storage_kernel_exports.storageOwnerVectorMigrationJson, "antfly_storage_owner_vector_migration_json"); exportInternal(&storage_kernel_exports.storageOwnerArtifactOperationJson, "antfly_storage_owner_artifact_operation_json"); exportInternal(&storage_kernel_exports.storageOwnerRuntimeStatusJson, "antfly_storage_owner_runtime_status_json"); exportInternal(&storage_kernel_exports.storageOwnerObservedDynamicFieldCapabilitySetsJson, "antfly_storage_owner_observed_dynamic_field_capability_sets_json"); diff --git a/zig/pkg/antfly/src/standalone/runtime.zig b/zig/pkg/antfly/src/standalone/runtime.zig index 16bc9bafab..5cd4bbe5f5 100644 --- a/zig/pkg/antfly/src/standalone/runtime.zig +++ b/zig/pkg/antfly/src/standalone/runtime.zig @@ -731,6 +731,7 @@ const UnifiedServerLifecycle = antfly.common.runtime_lifecycle.HttpServerLifecyc const LocalStandaloneMetadata = struct { alloc: std.mem.Allocator, mutex: std.atomic.Mutex = .unlocked, + vector_migration_commands: @import("../common/vector_migration.zig").CommandAdmissions = .{}, manager: antfly.metadata.TableManager, extension_catalog: antfly.extensions.ExtensionCatalog, local_node_id: u64, @@ -738,6 +739,7 @@ const LocalStandaloneMetadata = struct { api_url: []const u8, replica_root_dir: []const u8, catalog_path: []const u8, + operator_lock: ?std.Io.File = null, catalog_store: ?*antfly.storage_backend_erased.Store, backend_runtime: *antfly.db.background_runtime.BackendRuntime, storage_engine: antfly.common.config.StorageEngine = .local, @@ -830,11 +832,18 @@ const LocalStandaloneMetadata = struct { owned_replica_root_dir = null; owned_catalog_path = null; errdefer self.deinit(); + if (catalog_store == null) self.operator_lock = try @import("../common/migration_files.zig").lockCatalog( + alloc, + backend_runtime.filesystemIo() orelse return error.MissingBackendRuntimeIo, + catalog_path, + ); try self.loadPersistedCatalog(); return self; } fn deinit(self: *LocalStandaloneMetadata) void { + self.vector_migration_commands.deinit(self.alloc); + if (self.operator_lock) |file| file.close(self.backend_runtime.filesystemIo().?); self.extension_catalog.deinit(); self.manager.deinit(); self.alloc.free(self.catalog_path); @@ -880,6 +889,9 @@ const LocalStandaloneMetadata = struct { .free_routing_snapshot = catalogFreeRoutingSnapshot, .create_table = createTable, .replace_table_definition = replaceTableDefinition, + .publish_vector_migration_table = publishVectorMigrationTable, + .begin_vector_migration_command = beginVectorMigrationCommand, + .end_vector_migration_command = endVectorMigrationCommand, .restore_table = restoreTable, .drop_table = dropTable, .drop_table_exact = dropTableExact, @@ -1248,6 +1260,8 @@ const LocalStandaloneMetadata = struct { const current = self.findTableByNameLocked(replacement.name) orelse return error.TableNotFound; if (!antfly.metadata.table_manager.tableDefinitionsEqual(current.*, expected) or replacement.table_id != expected.table_id) return error.TableGenerationChanged; + if (current.storage_migration != null and !antfly.metadata.table_manager.tableDefinitionsEqual(current.*, replacement)) + return error.TableTransitionActive; try antfly.public_api.indexes.validateArtifactEnrichmentsForTableIndexesJson(self.alloc, replacement.indexes_json); try antfly.inference.managed_embedder.validateEmbeddingProducerOwnershipJson(self.alloc, replacement.indexes_json); const previous = try antfly.metadata.table_manager.cloneTable(self.alloc, current.*); @@ -1265,6 +1279,29 @@ const LocalStandaloneMetadata = struct { }; } + fn publishVectorMigrationTable(ptr: *anyopaque, expected: antfly.metadata.TableRecord, replacement: antfly.metadata.TableRecord) !void { + const self: *LocalStandaloneMetadata = @ptrCast(@alignCast(ptr)); + if (!self.vector_source_storage_allowed or self.storage_engine != .local) + return error.VectorStoreRequiresLocalSingleShardTable; + lockAtomic(&self.mutex); + defer self.mutex.unlock(); + var mutation = try self.beginCatalogMutationLocked(); + defer mutation.deinit(self); + try self.manager.publishVectorMigrationTable(expected, replacement); + self.epoch +|= 1; + try mutation.commit(self); + } + + fn beginVectorMigrationCommand(ptr: *anyopaque, table_name: []const u8) !void { + const self: *LocalStandaloneMetadata = @ptrCast(@alignCast(ptr)); + try self.vector_migration_commands.begin(self.alloc, table_name); + } + + fn endVectorMigrationCommand(ptr: *anyopaque, table_name: []const u8) void { + const self: *LocalStandaloneMetadata = @ptrCast(@alignCast(ptr)); + self.vector_migration_commands.end(self.alloc, table_name); + } + fn restoreTable( ptr: *anyopaque, alloc: std.mem.Allocator, @@ -1321,6 +1358,7 @@ const LocalStandaloneMetadata = struct { lockAtomic(&self.mutex); defer self.mutex.unlock(); const table = self.findTableByNameLocked(table_name) orelse return error.TableNotFound; + if (table.storage_migration != null) return error.VectorMigrationActive; const table_id = table.table_id; const ranges = try self.manager.listRanges(alloc); defer self.manager.freeRanges(alloc, ranges); @@ -1774,6 +1812,9 @@ const LocalStandaloneMetadata = struct { }); defer parsed.deinit(); + for (parsed.value.tables) |table| if (table.storage_migration) |admission| { + if (admission.request.mode == .offline) return error.VectorMigrationOfflineAdmission; + }; _ = try self.manager.replaceProjectedTopology(parsed.value.tables, parsed.value.ranges); try self.extension_catalog.loadProjectedRows( parsed.value.extension_packages, diff --git a/zig/pkg/antfly/src/storage/artifact_payload.zig b/zig/pkg/antfly/src/storage/artifact_payload.zig index 63fec4f5cf..50c70deb9e 100644 --- a/zig/pkg/antfly/src/storage/artifact_payload.zig +++ b/zig/pkg/antfly/src/storage/artifact_payload.zig @@ -6,6 +6,7 @@ const std = @import("std"); const codec = @import("db/enrichment/artifact_codec.zig"); const keys = @import("internal_keys.zig"); +const migration = @import("../common/vector_migration.zig"); const Allocator = std.mem.Allocator; pub const Stats = struct { @@ -318,6 +319,10 @@ pub const Session = struct { refs: std.atomic.Value(usize) = .init(1), arena: std.heap.ArenaAllocator, store: Store, + /// Pre-publication writes retain inline authority while atomically + /// maintaining the migration's durable candidate reference root. + capture_inline: bool = false, + migration_allowance: ?u64 = null, // Keep preparations contiguous for one durable append, indexed by their // immutable identity for reads. The index owns no second payload copy and // is released with this transaction, including on abort. @@ -355,6 +360,9 @@ pub const Session = struct { pub fn retain(self: *Session) void { _ = self.refs.fetchAdd(1, .monotonic); } + pub fn primaryValue(self: *const Session, inline_value: []const u8, prepared_value: []const u8) []const u8 { + return if (self.capture_inline) inline_value else prepared_value; + } pub fn release(self: *Session) void { if (self.refs.fetchSub(1, .acq_rel) != 1) return; if ((self.prepared_once or (self.reference_mutated and self.primary_commit_attempted)) and !self.committed) { @@ -474,8 +482,24 @@ pub const Session = struct { /// existing replay WAL. No independent journal can outlive its checkpoint. /// Called only after the physical artifact mutation succeeded. pub fn recordOwnership(self: *Session, txn: anytype, key: []const u8, value: ?[]const u8) !void { - if (!ownershipEnabled() or !isEmbeddingKey(key)) return; + if (!isEmbeddingKey(key)) return; errdefer self.ownership_failed = true; + if (self.capture_inline) { + const candidate_key = try migration.candidateKeyAlloc(self.alloc, key); + defer self.alloc.free(candidate_key); + if (value) |raw| { + if (isReference(raw)) { + try txn.put(candidate_key, raw); + return; + } + } + txn.delete(candidate_key) catch |err| switch (err) { + error.NotFound => {}, + else => return err, + }; + return; + } + if (!ownershipEnabled()) return; var owner_key: [ownership_prefix.len + 32]u8 = undefined; @memcpy(owner_key[0..ownership_prefix.len], ownership_prefix); std.crypto.hash.sha2.Sha256.hash(key, owner_key[ownership_prefix.len..], .{}); @@ -515,6 +539,26 @@ pub const Session = struct { pub fn stageReferenceEpoch(self: *Session, txn: anytype) !void { if (self.ownership_failed) return error.VectorOwnershipMutationFailed; if (!self.reference_mutated or self.reference_epoch_staged) return; + if (self.migration_allowance) |limit| { + const before = txn.get(migration.accounting_key) catch |err| switch (err) { + error.NotFound => null, + else => return err, + }; + var charged: u64 = if (before) |raw| blk: { + if (raw.len != 8) return error.InvalidVectorMigrationState; + break :blk std.mem.readInt(u64, raw[0..8], .little); + } else 0; + // Conservative overlap allowance includes source WAL/segments, + // primary candidate/reference rows and transaction/replay copies. + for (self.prepared.keys()) |item| { + charged = try std.math.add(u64, charged, try std.math.mul(u64, item.artifact.len + reference_len + 1024, 8)); + } + if (charged > limit) return error.VectorMigrationTemporaryBudgetExceeded; + var encoded: [8]u8 = undefined; + std.mem.writeInt(u64, &encoded, charged, .little); + try txn.put(migration.accounting_key, &encoded); + } + const previous = txn.get(reference_epoch_key) catch |err| switch (err) { error.NotFound => null, else => return err, diff --git a/zig/pkg/antfly/src/storage/db/catalog/index_manager.zig b/zig/pkg/antfly/src/storage/db/catalog/index_manager.zig index b6bef3fe99..2c178d8b46 100644 --- a/zig/pkg/antfly/src/storage/db/catalog/index_manager.zig +++ b/zig/pkg/antfly/src/storage/db/catalog/index_manager.zig @@ -4453,7 +4453,28 @@ pub const IndexManager = struct { } pub fn ensureVectorBlockBaseAtAppliedSequence(self: *IndexManager, name: []const u8, applied_sequence: u64) !void { + return self.ensureVectorBlockBaseAtAppliedSequenceWithEncoding(name, applied_sequence, false); + } + + fn vectorBlockUsesDesiredEncoding(self: *IndexManager) bool { + const generation = self.acquireVectorBlockGeneration() orelse return false; + defer generation.release(); + return generation.opened.usesBaseEncoding(self.vectorBlockStorageEncoding()); + } + + pub fn sourceMigrationServingComplete(self: *IndexManager) bool { + if (self.dense_indexes.items.len == 0) return true; + if (self.source_payload_store == null or !self.vectorBlockUsesDesiredEncoding()) return false; + for (self.dense_indexes.items) |*entry| { + if (!entry.native_physical_v2 or !self.vectorBlockReadyForDenseIndex(entry.config.name) or + self.repairUnavailable(entry.config.name)) return false; + } + return true; + } + + fn ensureVectorBlockBaseAtAppliedSequenceWithEncoding(self: *IndexManager, name: []const u8, applied_sequence: u64, require_storage_encoding: bool) !void { if (self.vector_block_storage == null) return; + const converting = require_storage_encoding and !self.vectorBlockUsesDesiredEncoding(); const entry = self.denseIndex(name) orelse return error.IndexNotFound; if (entry.index.experimentalPostingDurableAppliedSequence()) |posting_sequence| { // A stable source snapshot may legitimately be ahead of the last @@ -4507,7 +4528,7 @@ pub const IndexManager = struct { // maintenance lane. The empty-bootstrap exception remains enforced // by vectorBlockGenerationReadyAtSequenceAndCount, so initial // publication still produces a cardinality-certified base. - if (self.vectorBlockReadyAtSequenceAndCount( + if (!converting and self.vectorBlockReadyAtSequenceAndCount( applied_sequence, entry, entry.index.stats().active_count, @@ -4521,7 +4542,7 @@ pub const IndexManager = struct { // without cloning/scanning primary LSM artifacts. Missing, lossy, or // sequence-mismatched native state falls through to the pinned primary // snapshot repair path below. - if (try self.compactVectorBlockGenerationAtStableTip(entry, applied_sequence)) return; + if (!converting and try self.compactVectorBlockGenerationAtStableTip(entry, applied_sequence)) return; // Only one snapshot builder may reserve the next immutable generation. // Ordinary WAL appends continue under vector_block_build_mu while the @@ -4611,7 +4632,7 @@ pub const IndexManager = struct { defer store.deinit(); if (store.manifest != null and store.covered_source_sequence == applied_sequence) { try self.loadVectorBlockGenerationIfPresent(false); - if (self.vectorBlockReadyAtSequenceAndCount( + if (!converting and self.vectorBlockReadyAtSequenceAndCount( applied_sequence, entry, entry.index.stats().active_count, @@ -10400,7 +10421,7 @@ pub const IndexManager = struct { for (self.dense_indexes.items) |*entry| { if (!entry.apply_mutex.tryLock()) continue; defer entry.apply_mutex.unlock(); - if (entry.index.posting_refresh_clean_epoch == entry.index.published_mutation_epoch.load(.acquire)) continue; + if (!entry.index.postingRefreshPending()) continue; if (entry.index.resource_manager) |resources| if (resources.shouldDeferPostingRefreshForForegroundWrites()) continue; if (entry.index.treeLinkRepairPending() or (if (entry.index.resource_manager) |resources| resources.dense_posting_row_deltas else @import("../../dense_perf_experiments.zig").enabled("ANTFLY_EXPERIMENT_POSTING_ROW_DELTAS"))) @@ -10583,6 +10604,10 @@ pub const IndexManager = struct { } pub const OnlineVectorBlockPublicationOptions = struct { + /// Source ownership conversion calls this only after physical primary + /// values are all references. Preserve healthy serving while replacing + /// the old full-payload exact-vector plane. + require_storage_encoding: bool = false, cancel_check: ?types.RepairCancelCheck = null, /// The repair owner has verified terminal source-outcome coverage for /// this rebuilding index. This permits staging only; serving and the @@ -10626,7 +10651,9 @@ pub const IndexManager = struct { if (options.only_index) |name| if (!std.mem.eql(u8, name, entry.config.name)) continue; if (self.repairUnavailable(entry.config.name) and !(options.covered_rebuilding_index != null and std.mem.eql(u8, options.covered_rebuilding_index.?, entry.config.name))) continue; - if (self.vectorBlockReadyForDenseIndex(entry.config.name) and !entry.index.nativePostingAccelerationPending()) continue; + const healthy = self.vectorBlockReadyForDenseIndex(entry.config.name); + if (healthy and !entry.index.nativePostingAccelerationPending() and + (!options.require_storage_encoding or self.vectorBlockUsesDesiredEncoding())) continue; const sequence = entry.index.experimentalPostingDurableAppliedSequence() orelse continue; if (sequence != primary.lastReplaySequence(0)) { deferred = true; @@ -10646,16 +10673,20 @@ pub const IndexManager = struct { continue; } } - if (self.vector_block_stable_tip_finalizing.cmpxchgStrong(false, true, .acq_rel, .acquire) != null) - return .{ .published = published, .deferred = true }; - self.vector_block_stable_tip_index.store(@intFromPtr(entry), .release); - self.vector_block_stable_tip_sequence.store(sequence, .release); - defer { + // Healthy replacements are serialized by the DB projection owner + // and base-staging reservation. They do not close serving admission. + if (!healthy) { + if (self.vector_block_stable_tip_finalizing.cmpxchgStrong(false, true, .acq_rel, .acquire) != null) + return .{ .published = published, .deferred = true }; + self.vector_block_stable_tip_index.store(@intFromPtr(entry), .release); + self.vector_block_stable_tip_sequence.store(sequence, .release); + } + defer if (!healthy) { self.vector_block_stable_tip_sequence.store(0, .release); self.vector_block_stable_tip_index.store(0, .release); self.vector_block_stable_tip_finalizing.store(false, .release); - } - self.ensureVectorBlockBaseAtAppliedSequence(entry.config.name, sequence) catch |err| switch (err) { + }; + self.ensureVectorBlockBaseAtAppliedSequenceWithEncoding(entry.config.name, sequence, options.require_storage_encoding) catch |err| switch (err) { error.PostingCheckpointSequenceMismatch, error.VectorBlockSnapshotAdvancedWithoutWal, error.VectorBlockGenerationReservationLost, diff --git a/zig/pkg/antfly/src/storage/db/db.zig b/zig/pkg/antfly/src/storage/db/db.zig index 0be6f8286f..e508c2aac5 100644 --- a/zig/pkg/antfly/src/storage/db/db.zig +++ b/zig/pkg/antfly/src/storage/db/db.zig @@ -32,6 +32,7 @@ const backend_types = @import("../backend_types.zig"); const backup_codec = @import("../backup_codec.zig"); const docstore_mod = @import("../docstore.zig"); const table_storage_mod = @import("../../common/table_storage.zig"); +pub const vector_migration = @import("../vector_migration.zig"); const vector_payload_store_mod = @import("../vector_payload_store.zig"); const segment_mod = @import("../../segment.zig"); const backend_erased_mod = @import("../backend_erased.zig"); @@ -211,7 +212,7 @@ fn standaloneResourceManagerOptionsForTotal(alloc: Allocator, total: u64) resour return options; } -fn standaloneResourceManagerOptions(alloc: Allocator) resource_manager_mod.Options { +pub fn standaloneResourceManagerOptions(alloc: Allocator) resource_manager_mod.Options { return standaloneResourceManagerOptionsForTotal(alloc, process_memory_mod.systemEnvelope().limit_bytes); } @@ -595,6 +596,9 @@ pub const OpenOptions = struct { hbc_cache: ?*hbc_mod.Cache = null, lsm_root_generation: u64 = 0, staged_generation: ?*const generation_lifecycle.StagedGeneration = null, + /// Exclusive offline tooling may inspect a fenced source root. The caller + /// must close it before publication; ordinary open cannot bypass the fence. + exclusive_generation: ?*const generation_lifecycle.ExclusiveTransition = null, resource_manager: ?*resource_manager_mod.ResourceManager = null, /// Optional storage-backend capacity probe. BackendRuntime configurators /// may install this while composing a DB open; it is resource policy input, @@ -5042,7 +5046,10 @@ const GraphRestoreParseCache = struct { pub const DB = struct { table_storage: table_storage_mod.Settings = .{}, - source_vectors: ?*vector_payload_store_mod.Store = null, + vector_migration_offline_candidate: bool = false, + vector_migration_active: std.atomic.Value(bool) = .init(false), + vector_migration_reopen_required: std.atomic.Value(bool) = .init(false), + source_vectors: std.atomic.Value(?*vector_payload_store_mod.Store) = .init(null), source_vector_storage: ?*lsm_backend_mod.NativeStorage = null, closed: bool = false, stable_address: bool = false, @@ -5270,6 +5277,7 @@ pub const DB = struct { } fn enforcePortableRuntimeGate(self: *const DB) !void { + if (self.vector_migration_reopen_required.load(.acquire)) return error.VectorMigrationRecoveryRequired; try enforcePortableRuntimeGateOptional(&self.async_context.portable_runtime_activation_pending); } @@ -5679,11 +5687,23 @@ pub const DB = struct { var generation_read_lease = if (opts.staged_generation) |staged_generation| staged_blk: { try staged_generation.validatePath(path); break :staged_blk null; + } else if (opts.exclusive_generation) |transition| exclusive_blk: { + try transition.validate(path); + break :exclusive_blk null; } else if (opts.physical_root_mode == .external_backend) null else try generation_lifecycle.acquirePublishedGenerationReadWithRuntime(alloc, path, backend_runtime); errdefer if (generation_read_lease) |*lease| lease.deinit(); + if (opts.physical_root_mode == .filesystem_managed and opts.exclusive_generation == null) { + const fence = try std.fs.path.join(alloc, &.{ path, vector_migration.contract.offline_fence_file }); + defer alloc.free(fence); + const io = backend_runtime.filesystemIo() orelse return error.MissingBackendRuntimeIo; + if (std.Io.Dir.cwd().access(io, fence, .{})) |_| { + return error.VectorMigrationOfflineAdmission; + } else |err| if (err != error.FileNotFound) return err; + } + const open_started_ns = monotonicTimeNs(); const ha_write_gate = if (opts.ha_write_gate) |gate| gate.pinned() else null; var profile = OpenProfile{}; @@ -6149,6 +6169,8 @@ pub const DB = struct { } fn initializeTableStorage(self: *DB, requested: ?table_storage_mod.Settings) !void { + var migration_job = try vector_migration.load(self.alloc, self.core.store); + defer if (migration_job) |*job| job.deinit(); const raw = self.core.store.get(self.alloc, &internal_keys.table_storage_settings_key) catch |err| switch (err) { error.NotFound => null, else => return err, @@ -6158,7 +6180,14 @@ pub const DB = struct { var parsed = try std.json.parseFromSlice(table_storage_mod.Settings, self.alloc, value, .{}); defer parsed.deinit(); if (requested) |settings| { - if (settings.dense_embeddings != parsed.value.dense_embeddings) return error.ImmutableTableStorageSettings; + if (settings.dense_embeddings != parsed.value.dense_embeddings) { + // Only this table's durable ownership publication can + // bridge a catalog update interrupted after DB commit. + const job = if (migration_job) |job| job.value else return error.ImmutableTableStorageSettings; + try self.validateVectorMigrationIdentity(job); + if (!job.published() or settings.dense_embeddings != .primary_lsm or + parsed.value.dense_embeddings != .vector_store) return error.ImmutableTableStorageSettings; + } } self.table_storage = parsed.value; if (self.table_storage.dense_embeddings == .vector_store) { @@ -6182,10 +6211,381 @@ pub const DB = struct { if (settings.dense_embeddings != .primary_lsm) return error.MissingTableStorageSettings; } else try self.configureTableStorage(settings); } + if (migration_job) |job| { + try self.validateVectorMigrationIdentity(job.value); + if (job.value.published() != (self.table_storage.dense_embeddings == .vector_store)) + return error.InvalidVectorMigrationState; + if (job.value.phase == .cancelled) { + if (self.table_storage.dense_embeddings != .primary_lsm) return error.InvalidVectorMigrationState; + // No primary reference was ever published by a cancelled job. + // At open there are no local sessions/workers to race teardown; + // other process read mappings retain their own file leases. + if (!openModeRequiresReadOnlyBackends(self.open_mode)) { + const root = try std.fs.path.join(self.alloc, &.{ self.core.path, "source-vectors" }); + defer self.alloc.free(root); + const io = self.backend_runtime.filesystemIo() orelse return error.MissingBackendRuntimeIo; + std.Io.Dir.cwd().deleteTree(io, root) catch |err| { + std.log.warn("cancelled source candidate cleanup deferred err={s}", .{@errorName(err)}); + }; + } + } else if (job.value.active()) { + try self.openSourceVectors(false); + self.installVectorMigrationRuntime(job.value); + } + } + } + + fn validateVectorMigrationIdentity(self: *DB, job: vector_migration.contract.Job) !void { + const identity = try std.json.Stringify.valueAlloc(self.alloc, self.core.identity_namespace, .{}); + defer self.alloc.free(identity); + if (!std.mem.eql(u8, identity, job.table_identity)) return error.VectorMigrationIdentityMismatch; + } + + fn vectorMigrationConfigurationHash(self: *DB) !u64 { + const indexes = try self.core.listIndexes(self.alloc); + defer types.freeIndexConfigs(self.alloc, indexes); + const enrichments = try self.core.listEnrichments(self.alloc); + defer types.freeEnrichmentConfigs(self.alloc, enrichments); + const encoded = try std.json.Stringify.valueAlloc(self.alloc, .{ + .schema = self.core.schema, + .indexes = indexes, + .enrichments = enrichments, + }, .{}); + defer self.alloc.free(encoded); + return std.hash.Wyhash.hash(0, encoded); + } + + fn installVectorMigrationRuntime(self: *DB, job: vector_migration.contract.Job) void { + self.vector_migration_active.store(job.active(), .release); + const source = self.source_vectors.load(.acquire) orelse return; + source.setMigrationRetention(job.active()); + source.migration_disk_reserve.store(if (job.active()) job.budget.disk_reserve_bytes else 0, .release); + source.migration_temporary_limit.store(if (job.active()) job.budget.temporary_bytes else 0, .release); + self.core.store.configurePayloadPolicy( + if (job.phase == .cancelling or job.phase == .cancelled) null else source.interface(), + job.captures(), + if (job.active()) job.budget.temporary_bytes else null, + ); + } + + pub fn authorizeOfflineVectorMigrationCandidate(self: *DB, stage: *const generation_lifecycle.StagedGeneration) !void { + try stage.validatePath(self.core.path); + self.vector_migration_offline_candidate = true; + } + + pub fn vectorMigrationCommand(self: *DB, alloc: Allocator, command: vector_migration.contract.Command) ![]u8 { + try command.request.validate(); + if (command.action != .start and command.action != .cancel) { + const raw = try self.vectorMigrationStatus(alloc) orelse return error.VectorMigrationNotFound; + defer alloc.free(raw); + var prior = try std.json.parseFromSlice(vector_migration.contract.Job, alloc, raw, .{}); + defer prior.deinit(); + if (command.action == .status) { + if (!std.mem.eql(u8, prior.value.job_id, command.request.job_id)) return error.VectorMigrationNotFound; + return try alloc.dupe(u8, raw); + } + if (!std.mem.eql(u8, prior.value.job_id, command.request.job_id) or + prior.value.mode != command.request.mode or !std.meta.eql(prior.value.budget, command.request.budget)) + return error.VectorMigrationIdempotencyConflict; + } + switch (command.action) { + .start => try self.startVectorMigration(command.request), + .step => try self.advanceVectorMigration(command.request.job_id), + .publish => try self.publishVectorMigration(command.request.job_id), + .cancel => try self.cancelVectorMigrationImpl(command.request.job_id, command.request), + .status => {}, + } + const result = try self.vectorMigrationStatus(alloc) orelse return error.VectorMigrationNotFound; + errdefer alloc.free(result); + var parsed = try std.json.parseFromSlice(vector_migration.contract.Job, alloc, result, .{}); + defer parsed.deinit(); + if (!std.mem.eql(u8, parsed.value.job_id, command.request.job_id)) return error.VectorMigrationIdempotencyConflict; + return result; + } + + pub fn vectorMigrationStatus(self: *DB, alloc: Allocator) !?[]u8 { + lockApplyShared(self); + defer self.core.unlockApplyShared(); + var job = (try vector_migration.load(alloc, self.core.store)) orelse return null; + defer job.deinit(); + if (self.core.store.get(alloc, vector_migration.contract.accounting_key)) |bytes| { + defer alloc.free(bytes); + if (bytes.len != 8) return error.InvalidVectorMigrationState; + job.value.charged_temporary_bytes = std.mem.readInt(u64, bytes[0..8], .little); + } else |err| if (err != error.NotFound) return err; + return try std.json.Stringify.valueAlloc(alloc, job.value, .{}); + } + + pub fn startVectorMigration(self: *DB, request: vector_migration.contract.Request) !void { + try request.validate(); + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + if (request.mode == .offline and !self.vector_migration_offline_candidate) return error.VectorStoreRequiresOfflineCommand; + var structural = self.beginIndexStructuralMutation("source ownership migration", "*"); + defer structural.deinit(); + try self.lockApplyForPortableRuntime(); + defer self.core.unlockApply(); + var ownership_epoch: u64 = 1; + if (try vector_migration.load(self.alloc, self.core.store)) |existing| { + var job = existing; + defer job.deinit(); + try self.validateVectorMigrationIdentity(job.value); + if (std.mem.eql(u8, job.value.job_id, request.job_id)) { + if (job.value.mode != request.mode or !std.meta.eql(job.value.budget, request.budget)) + return error.VectorMigrationIdempotencyConflict; + return; + } + if (job.value.phase != .cancelled) return error.VectorMigrationAlreadyExists; + ownership_epoch = try std.math.add(u64, job.value.ownership_epoch, 1); + } + if (self.table_storage.dense_embeddings != .primary_lsm) return error.VectorMigrationAlreadyPublished; + if (self.core.splitState() != null) return error.VectorStoreLifecycleUnsupported; + for (self.core.index_manager.dense_indexes.items) |entry| { + if (!entry.native_physical_v2 and !try self.core.index_manager.denseNativePhysicalMigrationRequired(entry.config.name)) + return error.VectorStoreLifecycleUnsupported; + } + const identity = try std.json.Stringify.valueAlloc(self.alloc, self.core.identity_namespace, .{}); + defer self.alloc.free(identity); + const configuration_hash = try self.vectorMigrationConfigurationHash(); + const disk = try @import("antfly_platform").filesystem.capacity(self.core.path); + if (disk.available_bytes < request.budget.disk_reserve_bytes +| request.budget.batch_bytes * 8) + return error.VectorMigrationDiskReserve; + // The source manifest is durable before the primary job admits any + // capture. A crash before the job commit leaves only orphan data. + try self.openSourceVectors(true); + try self.source_vectors.load(.acquire).?.beginMigrationRetention(); + errdefer self.requireVectorMigrationRecovery(); + const raw_epoch = self.core.store.get(self.alloc, @import("../artifact_payload.zig").reference_epoch_key) catch |err| switch (err) { + error.NotFound => null, + else => return err, + }; + defer if (raw_epoch) |value| self.alloc.free(value); + const epoch: u64 = if (raw_epoch) |value| blk: { + if (value.len != 8) return error.InvalidVectorReferenceEpoch; + break :blk std.mem.readInt(u64, value[0..8], .little); + } else 0; + const job: vector_migration.contract.Job = .{ + .job_id = request.job_id, + .mode = request.mode, + .budget = request.budget, + .table_identity = identity, + .configuration_hash = configuration_hash, + .ownership_epoch = ownership_epoch, + .snapshot_fence = epoch, + .replay_cursor = epoch, + }; + var txn = try self.core.store.runtime_store.beginWrite(); + var committed = false; + defer if (!committed) txn.abort(); + try vector_migration.save(self.alloc, &txn, job); + try txn.put(vector_migration.contract.accounting_key, &(@as([8]u8, @splat(0)))); + try txn.commit(); + committed = true; + try self.core.store.runtime_store.sync(true); + self.installVectorMigrationRuntime(job); + } + + pub fn advanceVectorMigration(self: *DB, job_id: []const u8) anyerror!void { + // Staging pins index/catalog lifetime but never holds table apply over + // corpus-sized ANN work. Completion is checked again under apply. + const status = try self.vectorMigrationStatus(self.alloc) orelse return error.VectorMigrationNotFound; + defer self.alloc.free(status); + var observed = try std.json.parseFromSlice(vector_migration.contract.Job, self.alloc, status, .{}); + defer observed.deinit(); + if (!std.mem.eql(u8, observed.value.job_id, job_id)) return error.VectorMigrationIdempotencyConflict; + if (observed.value.phase == .serving) { + const legacy = blk: { + var lease = self.tryAcquireIndexCatalogReadLease() orelse return; + defer lease.release(); + for (self.core.index_manager.dense_indexes.items) |entry| { + if (try self.core.index_manager.denseNativePhysicalMigrationRequired(entry.config.name)) + break :blk try self.alloc.dupe(u8, entry.config.name); + } + break :blk null; + }; + if (legacy) |name| { + defer self.alloc.free(name); + var repair = try self.repairArtifactIssuesWithRequest(self.alloc, .{ .target = .index, .index_name = name, .limit = 1 }); + defer repair.deinit(self.alloc); + } + _ = try self.publishVectorBlockBasesOnlineReported(.{ .require_quiescence = false, .require_storage_encoding = true }); + } + if (observed.value.phase == .reclaiming and observed.value.primary_reclamation_requested) { + // Offline operators have no maintenance worker. Online execution + // uses the same bounded scheduler without holding the table apply + // lock over streaming compaction I/O. + _ = try self.core.primary_store_owner.lsm.handle.backend.runValueReclamationStep(); + } + try self.lockApplyForPortableRuntime(); + defer self.core.unlockApply(); + var job = (try vector_migration.load(self.alloc, self.core.store)) orelse return error.VectorMigrationNotFound; + defer job.deinit(); + if (!std.mem.eql(u8, job.value.job_id, job_id)) return error.VectorMigrationIdempotencyConflict; + try self.validateVectorMigrationIdentity(job.value); + if (job.value.configuration_hash != try self.vectorMigrationConfigurationHash()) return error.VectorMigrationConfigurationChanged; + if (!job.value.active() or job.value.phase == .ready) return; + errdefer |err| switch (@as(anyerror, err)) { + error.VectorMigrationTemporaryBudgetExceeded, error.VectorMigrationDiskReserve, error.VectorMigrationRowExceedsBudget => {}, + else => self.requireVectorMigrationRecovery(), + }; + if (job.value.phase == .reclaiming) { + const backend = self.core.primary_store_owner.lsm.handle.backend; + if (!job.value.primary_reclamation_requested) { + // One final flush includes all replacements and candidate + // deletes before requesting the old overlap closures. Persist + // the manifest request before its primary receipt. A crash + // between them safely repeats the request after reopening. + try self.core.store.runtime_store.sync(true); + try backend.requestValueReclamation(); + try vector_migration.boundary(.reclamation_request); + job.value.primary_reclamation_requested = true; + } else { + if (try backend.hasValueReclamationRequests()) return; + // Fence the manifest that discharged the last request before + // reporting completion. Pinned readers may retain old files. + try backend.persistManifest(); + job.value.phase = .complete; + } + var txn = try self.core.store.runtime_store.beginWrite(); + var committed = false; + defer if (!committed) txn.abort(); + try vector_migration.save(self.alloc, &txn, job.value); + try txn.commit(); + committed = true; + try self.core.store.runtime_store.syncReplayState(); + try vector_migration.boundary(.reclamation_receipt); + } else if (job.value.phase == .serving) { + if (!self.core.index_manager.sourceMigrationServingComplete()) return; + job.value.phase = .cleanup; + var txn = try self.core.store.runtime_store.beginWrite(); + var committed = false; + defer if (!committed) txn.abort(); + try vector_migration.save(self.alloc, &txn, job.value); + try txn.commit(); + committed = true; + try self.core.store.runtime_store.sync(true); + } else vector_migration.advance(self.alloc, self.core.store, self.source_vectors.load(.acquire).?.interface(), job.value) catch |err| { + switch (err) { + error.VectorMigrationTemporaryBudgetExceeded, error.VectorMigrationDiskReserve, error.VectorMigrationRowExceedsBudget => { + // Known pre-preparation admission failure: preserve progress + // and expose the reason without requiring a DB restart. + job.value.last_error = @errorName(err); + var txn = try self.core.store.runtime_store.beginWrite(); + var committed = false; + defer if (!committed) txn.abort(); + try vector_migration.save(self.alloc, &txn, job.value); + try txn.commit(); + committed = true; + try self.core.store.runtime_store.sync(true); + }, + else => {}, + } + return @as(anyerror!void, err); + }; + var next = (try vector_migration.load(self.alloc, self.core.store)).?; + defer next.deinit(); + self.installVectorMigrationRuntime(next.value); + } + + pub fn publishVectorMigration(self: *DB, job_id: []const u8) !void { + var structural = self.beginIndexStructuralMutation("source ownership publication", "*"); + defer structural.deinit(); + try self.lockApplyForPortableRuntime(); + defer self.core.unlockApply(); + var job = (try vector_migration.load(self.alloc, self.core.store)) orelse return error.VectorMigrationNotFound; + defer job.deinit(); + if (!std.mem.eql(u8, job.value.job_id, job_id)) return error.VectorMigrationIdempotencyConflict; + try self.validateVectorMigrationIdentity(job.value); + if (job.value.configuration_hash != try self.vectorMigrationConfigurationHash()) return error.VectorMigrationConfigurationChanged; + if (job.value.published()) return; + if (job.value.phase != .ready) return error.VectorMigrationNotReady; + errdefer self.requireVectorMigrationRecovery(); + try vector_migration.publish(self.alloc, self.core.store, job.value); + self.table_storage = .{ .dense_embeddings = .vector_store }; + self.core.store.configurePayloadPolicy(self.source_vectors.load(.acquire).?.interface(), false, job.value.budget.temporary_bytes); + self.core.index_manager.table_owns_embedding_artifacts = true; + self.core.index_manager.source_payload_store = self.source_vectors.load(.acquire); + try self.refreshSourceVectorOwnershipScopes(); + try self.core.index_manager.refreshSourcePayloadGeneration(); + } + + pub fn cancelVectorMigration(self: *DB, job_id: []const u8) !void { + return self.cancelVectorMigrationImpl(job_id, null); + } + + fn cancelVectorMigrationImpl(self: *DB, job_id: []const u8, admitted: ?vector_migration.contract.Request) !void { + if (openModeRequiresReadOnlyBackends(self.open_mode)) return error.ReadOnly; + try self.lockApplyForPortableRuntime(); + defer self.core.unlockApply(); + var prior = try vector_migration.load(self.alloc, self.core.store); + defer if (prior) |*value| value.deinit(); + // A catalog admission can outlive a rejected/ambiguous DB start. Under + // the same apply lock as startup, save a terminal receipt before the + // caller releases the catalog fence. No source payload was committed + // without a DB job, so there is no candidate cleanup to drive here. + if (prior == null or (prior.?.value.phase == .cancelled and !std.mem.eql(u8, prior.?.value.job_id, job_id))) { + const request = admitted orelse return error.VectorMigrationNotFound; + if (self.table_storage.dense_embeddings != .primary_lsm) return error.VectorMigrationAlreadyPublished; + if (prior) |value| try self.validateVectorMigrationIdentity(value.value); + const identity = try std.json.Stringify.valueAlloc(self.alloc, self.core.identity_namespace, .{}); + defer self.alloc.free(identity); + const receipt: vector_migration.contract.Job = .{ + .job_id = request.job_id, + .mode = request.mode, + .budget = request.budget, + .phase = .cancelled, + .table_identity = identity, + .configuration_hash = try self.vectorMigrationConfigurationHash(), + .ownership_epoch = if (prior) |value| try std.math.add(u64, value.value.ownership_epoch, 1) else 1, + .snapshot_fence = 0, + .replay_cursor = 0, + }; + var txn = try self.core.store.runtime_store.beginWrite(); + var committed = false; + defer if (!committed) txn.abort(); + errdefer self.requireVectorMigrationRecovery(); + try vector_migration.save(self.alloc, &txn, receipt); + try txn.put(vector_migration.contract.accounting_key, &(@as([8]u8, @splat(0)))); + try txn.commit(); + committed = true; + try self.core.store.runtime_store.sync(true); + self.installVectorMigrationRuntime(receipt); + return; + } + const job = &prior.?; + if (!std.mem.eql(u8, job.value.job_id, job_id)) return error.VectorMigrationIdempotencyConflict; + if (admitted) |request| { + if (job.value.mode != request.mode or !std.meta.eql(job.value.budget, request.budget)) + return error.VectorMigrationIdempotencyConflict; + } + try self.validateVectorMigrationIdentity(job.value); + if (job.value.published()) return error.VectorMigrationAlreadyPublished; + if (job.value.phase == .cancelled or job.value.phase == .cancelling) return; + job.value.phase = .cancelling; + job.value.cursor = ""; + var txn = try self.core.store.runtime_store.beginWrite(); + var committed = false; + defer if (!committed) txn.abort(); + errdefer self.requireVectorMigrationRecovery(); + try vector_migration.save(self.alloc, &txn, job.value); + try txn.commit(); + committed = true; + try self.core.store.runtime_store.sync(true); + self.installVectorMigrationRuntime(job.value); + } + + fn requireVectorMigrationRecovery(self: *DB) void { + self.vector_migration_reopen_required.store(true, .release); + // Transaction-recovery owners share this DocStore but have copied DB + // wrapper fields. Fence their admission too after an ambiguous commit. + self.core.store.payload_recovery_required.store(true, .release); + } + + fn enforceVectorMigrationConfigurationGate(self: *const DB) !void { + if (self.vector_migration_active.load(.acquire)) return error.VectorMigrationActive; } fn openSourceVectors(self: *DB, create: bool) !void { - if (self.source_vectors != null) return; + if (self.source_vectors.load(.acquire) != null) return; if (self.primary_backend != .lsm or self.physical_root_mode != .filesystem_managed or self.ha_write_gate != null or self.ha_async_batch_mirror != null or self.ha_async_effect_mirror != null) return error.VectorStoreRequiresLocalSingleShardTable; @@ -6210,10 +6610,12 @@ pub const DB = struct { source.enableBackgroundCollection(); source.ann_reference_root = try std.fs.path.join(source.alloc, &.{ self.core.index_manager.base_path, "vector-blocks" }); self.source_vector_storage = storage; - self.source_vectors = source; - self.core.store.payload_store = source.interface(); - self.core.index_manager.table_owns_embedding_artifacts = true; - self.core.index_manager.source_payload_store = source; + self.source_vectors.store(source, .release); + if (self.table_storage.dense_embeddings == .vector_store) { + self.core.store.configurePayloadPolicy(source.interface(), false, null); + self.core.index_manager.table_owns_embedding_artifacts = true; + self.core.index_manager.source_payload_store = source; + } } /// Creation/provisioning-only configuration. Existing persisted authority @@ -6247,20 +6649,26 @@ pub const DB = struct { // A failed primary append/sync may have persisted the marker. Fence // source writes until reopen resolves that outcome; never continue // creating references under an unconfirmed table mode. - errdefer if (self.source_vectors) |source| { + errdefer if (self.source_vectors.load(.acquire)) |source| { source.poison(); }; try self.core.store.put(&internal_keys.table_storage_settings_key, encoded); try self.core.store.sync(true); self.table_storage = settings; + if (settings.dense_embeddings == .vector_store) { + const source = self.source_vectors.load(.acquire).?; + self.core.store.configurePayloadPolicy(source.interface(), false, null); + self.core.index_manager.table_owns_embedding_artifacts = true; + self.core.index_manager.source_payload_store = source; + } } pub fn sourceVectorStats(self: *DB) ?vector_payload_store_mod.Stats { - return if (self.source_vectors) |source| source.tryStatsSnapshot() else null; + return if (self.source_vectors.load(.acquire)) |source| source.tryStatsSnapshot() else null; } fn refreshSourceVectorOwnershipScopes(self: *DB) !void { - const source = self.source_vectors orelse return; + const source = self.source_vectors.load(.acquire) orelse return; const configs = try self.core.listIndexes(self.alloc); defer types.freeIndexConfigs(self.alloc, configs); const scopes = try self.core.index_manager.sourcePayloadScopeHashesAlloc(configs); @@ -6274,7 +6682,7 @@ pub const DB = struct { /// requires quiescent readers and sufficient per-call work/memory budgets; /// disabling index workers alone does not drain startup cleanup readers. pub fn collectSourceVectorGarbage(self: *DB) !bool { - const source = self.source_vectors orelse return false; + const source = self.source_vectors.load(.acquire) orelse return false; try source.advanceMarkingSnapshot(); lockApply(self); defer self.core.unlockApply(); @@ -7399,11 +7807,11 @@ pub const DB = struct { self.runtime_alloc.destroy(self.async_context); // A bounded source mark owns a read transaction on core's backend. // Workers are stopped; release it before core destroys that backend. - if (self.source_vectors) |source| source.cancelMarking(); + if (self.source_vectors.load(.acquire)) |source| source.cancelMarking(); const core = self.core; core.deinit(); self.alloc.destroy(core); - if (self.source_vectors) |source| { + if (self.source_vectors.load(.acquire)) |source| { source.deinit(); self.alloc.destroy(source); } @@ -9381,7 +9789,7 @@ pub const DB = struct { const apply_lock_wait_start_ns = monotonicTimeNs(); try self.lockApplyForPortableRuntime(); if (profile) |active_profile| active_profile.apply_lock_wait_ns += monotonicTimeNs() - apply_lock_wait_start_ns; - if (self.source_vectors) |source| source.recordBatchLockWait(monotonicTimeNs() -| apply_lock_wait_start_ns); + if (self.source_vectors.load(.acquire)) |source| source.recordBatchLockWait(monotonicTimeNs() -| apply_lock_wait_start_ns); var apply_mutex_held = true; var apply_lock_acquired_ns = monotonicTimeNs(); errdefer if (apply_mutex_held) unlockProfiledApply(self, profile, &apply_mutex_held, apply_lock_acquired_ns); @@ -19801,7 +20209,7 @@ pub const DB = struct { ); // Repair candidates share the table's immutable source owner too. // Otherwise their native base build silently recreates payload copies. - shadow_manager.source_payload_store = self.source_vectors; + shadow_manager.source_payload_store = self.core.index_manager.source_payload_store; shadow_manager.setIo(self.backend_runtime.io()); shadow_manager.setAppliedSequenceCheckpointPath(shadow_checkpoint_path); shadow_manager.registerReplacementIndex(self.core.store, cfg) catch |err| { @@ -21119,7 +21527,7 @@ pub const DB = struct { } pub fn setSplitState(self: *DB, state: ?types.SplitState) !void { - if (self.source_vectors != null) return error.VectorStoreLifecycleUnsupported; + if (self.source_vectors.load(.acquire) != null) return error.VectorStoreLifecycleUnsupported; var ha_mutation = self.acquireHAMutationShared(); defer if (ha_mutation) |*lease| lease.release(); try self.enforceHAWriteGate(); @@ -21628,7 +22036,7 @@ pub const DB = struct { dest_dir2: []const u8, prepare_only: bool, ) !void { - if (self.source_vectors != null) return error.VectorStoreLifecycleUnsupported; + if (self.source_vectors.load(.acquire) != null) return error.VectorStoreLifecycleUnsupported; var ha_mutation = self.acquireHAMutationShared(); defer if (ha_mutation) |*lease| lease.release(); try self.enforceHAWriteGate(); @@ -21742,6 +22150,24 @@ pub const DB = struct { }; var test_snapshot_fence_hook: ?SnapshotFenceTestHook = null; + fn ensurePrimaryOnlySnapshotLocked(self: *DB) !void { + if (self.source_vectors.load(.acquire) == null) return; + // Cancellation retains the source object for already-admitted readers + // and background retirement. Its presence is not published authority: + // a terminal cancelled job has removed all candidate roots and kept + // every live artifact inline in the primary store. + var job = (try vector_migration.load(self.alloc, self.core.store)) orelse return error.VectorStoreLifecycleUnsupported; + defer job.deinit(); + if (self.table_storage.dense_embeddings != .primary_lsm or job.value.phase != .cancelled) + return error.VectorStoreLifecycleUnsupported; + } + + fn ensurePrimaryOnlySnapshot(self: *DB) !void { + lockApplyShared(self); + defer self.core.unlockApplyShared(); + try self.ensurePrimaryOnlySnapshotLocked(); + } + fn snapshotInternal( self: *DB, id: []const u8, @@ -21749,7 +22175,7 @@ pub const DB = struct { cancellation: types.CancellationToken, maintenance_deadline_ns: ?u64, ) !u64 { - if (self.source_vectors != null) return error.VectorStoreLifecycleUnsupported; + try self.ensurePrimaryOnlySnapshot(); // Serialize only snapshot construction/publication. Normal writes can // resume before native manifest hashing, while same-ID captures cannot // race the fresh-directory check or atomic rename. @@ -21803,6 +22229,8 @@ pub const DB = struct { try self.lockApplyForPortableRuntime(); var apply_held = true; defer if (apply_held) self.core.unlockApply(); + // Migration may have started while portable maintenance drained. + try self.ensurePrimaryOnlySnapshotLocked(); try self.core.syncStore(true); try self.core.index_manager.syncAll(true); var primary_snapshot = try self.core.pinPortableSnapshot(); @@ -21834,6 +22262,9 @@ pub const DB = struct { else => return err, }; defer capture.release(); + // Migration may have won admission after the optimistic entry check. + // Its structural mutation uses this same snapshot fence. + try self.ensurePrimaryOnlySnapshot(); if (builtin.is_test) { if (test_snapshot_fence_hook) |hook| hook.after_capture_admission(hook.ptr); } @@ -22064,11 +22495,11 @@ pub const DB = struct { } pub fn sync(self: *DB, full: bool) !void { - if (full) if (self.source_vectors) |source| try source.advanceMarkingSnapshot(); + if (full) if (self.source_vectors.load(.acquire)) |source| try source.advanceMarkingSnapshot(); lockApply(self); defer self.core.unlockApply(); try self.core.syncStore(full); - if (self.source_vectors) |source| { + if (self.source_vectors.load(.acquire)) |source| { try source.checkpoint(); if (full) { try self.refreshSourceVectorOwnershipScopes(); @@ -23714,6 +24145,7 @@ pub const DB = struct { try self.lockApplyForPortableRuntime(); var apply_held = true; errdefer if (apply_held) self.core.unlockApply(); + try self.enforceVectorMigrationConfigurationGate(); const reconciled_row_count = try self.validateStorageModeCompatibilityLocked(table_schema); if (durable_ha_schema_outbox_key != null) self.durable_ha_outbox_maybe.store(true, .release); _ = try self.core.commitPreparedSchemaMetadata( @@ -25660,6 +26092,7 @@ pub const DB = struct { try self.enforceHAWriteGate(); var structural_guard = self.beginIndexStructuralMutation("index creation", cfg.name); defer structural_guard.deinit(); + try self.enforceVectorMigrationConfigurationGate(); // Generated artifact namespaces can be shared across differently named // indexes. Cleanup is durable and owner-driven; never turn index // admission into an unbounded corpus scan. Metadata reconciliation can @@ -25725,6 +26158,7 @@ pub const DB = struct { try self.enforceHAWriteGate(); try self.lockApplyForPortableRuntime(); defer self.core.unlockApply(); + try self.enforceVectorMigrationConfigurationGate(); try self.core.addEnrichment(cfg); } @@ -25735,6 +26169,7 @@ pub const DB = struct { try self.enforceHAWriteGate(); try self.lockApplyForPortableRuntime(); defer self.core.unlockApply(); + try self.enforceVectorMigrationConfigurationGate(); return try self.core.upsertEnrichment(cfg); } @@ -27082,6 +27517,7 @@ pub const DB = struct { try self.enforceHAWriteGate(); var structural_guard = self.beginIndexStructuralMutation("index deletion", name); defer structural_guard.deinit(); + try self.enforceVectorMigrationConfigurationGate(); const restart_enrichment = self.quiesceEnrichmentForStructuralMutation(); const removed = self.deleteIndexWhileEnrichmentQuiesced(name) catch |delete_err| { if (restart_enrichment) self.restartEnrichmentAfterStructuralMutation("failed index deletion", name) catch |restart_err| { @@ -27179,6 +27615,7 @@ pub const DB = struct { try self.enforceHAWriteGate(); try self.lockApplyForPortableRuntime(); defer self.core.unlockApply(); + try self.enforceVectorMigrationConfigurationGate(); return try self.core.deleteEnrichment(kind, name); } @@ -27769,7 +28206,7 @@ pub const DB = struct { if (openModeRequiresReadOnlyBackends(self.open_mode)) return false; // The mark owns immutable primary/ANN/source leases. Scan before // taking apply; only setup, planning, and publication need that fence. - if (self.source_vectors) |source| try source.advanceMarkingSnapshot(); + if (self.source_vectors.load(.acquire)) |source| try source.advanceMarkingSnapshot(); return self.runArtifactRepairMetadataMaintenanceAfterScan(); } @@ -27780,7 +28217,7 @@ pub const DB = struct { var more = try self.core.index_manager.runGraphOwnershipCleanupStep(); more = (try self.rebuildArtifactRepairSummaryIfMissing(self.alloc)) or more; more = (try self.rebuildArtifactRepairKindIndexIfMissing(self.alloc)) or more; - if (self.source_vectors) |source| { + if (self.source_vectors.load(.acquire)) |source| { const step_bytes = source.backgroundCollectionStepBytes(); if (step_bytes != 0) { try self.refreshSourceVectorOwnershipScopes(); @@ -27840,11 +28277,11 @@ pub const DB = struct { if (self.artifact_repair_metadata_stop.load(.acquire)) return null; self.runIndependentMaintenancePass(); const artifact_active = self.artifact_repair_metadata_pending or - (if (self.source_vectors) |source| source.collectionPending() else false); + (if (self.source_vectors.load(.acquire)) |source| source.collectionPending() else false); const active = (platform_time.monotonicNs() >= self.artifact_metadata_retry_after_ns and artifact_active) or (self.relational_column_maintenance.pending.load(.acquire) and !self.relational_column_maintenance.backing_off.load(.acquire)); - const scan_pause = if (self.source_vectors) |source| source.activeScanPauseNs() else null; - if (self.source_vectors) |source| if (source.background_checkpoint and scan_pause == null and !active) return 50; + const scan_pause = if (self.source_vectors.load(.acquire)) |source| source.activeScanPauseNs() else null; + if (self.source_vectors.load(.acquire)) |source| if (source.background_checkpoint and scan_pause == null and !active) return 50; return std.math.divCeil(u64, scan_pause orelse if (active) artifact_repair_metadata_active_poll_ns else artifact_repair_metadata_poll_ns, std.time.ns_per_ms) catch unreachable; } @@ -28144,13 +28581,13 @@ pub const DB = struct { } fn runArtifactRepairMaintenanceTurn(self: *DB) !void { - if (self.source_vectors) |source| try source.checkpointMaintenance(); - const independent = if (self.source_vectors) |source| source.independent_scan else false; + if (self.source_vectors.load(.acquire)) |source| try source.checkpointMaintenance(); + const independent = if (self.source_vectors.load(.acquire)) |source| source.independent_scan else false; const now = monotonicTimeNs(); if (!independent or now >= self.artifact_repair_metadata_due_ns) self.artifact_repair_metadata_pending = self.artifactRepairMetadataRebuildPending(); if (independent) { - const source = self.source_vectors.?; + const source = self.source_vectors.load(.acquire).?; try source.advanceMarkingSnapshot(); // Immutable scan turns bypass apply/catalog work until metadata // is due. State survives scheduler yields, not a pinned thread. @@ -29374,7 +29811,7 @@ pub const DB = struct { // them without populating the new generation. // Table-owned sources can predate even an ordinary ANN admission, // including after its last consumer was dropped. Bootstrap them too. - if (disposition == .managed_rebuild or (self.source_vectors != null and try self.externalCoverageHasStoredArtifacts(cfg))) { + if (disposition == .managed_rebuild or (self.source_vectors.load(.acquire) != null and try self.externalCoverageHasStoredArtifacts(cfg))) { try self.deleteDenseArtifactCounterMetadata(cfg.name); return; } @@ -32562,6 +32999,7 @@ pub const DB = struct { item.node_count = hbc_stats.node_count; item.root_node = hbc_stats.root_node; item.hbc_cache = dbHbcCacheStats(entry.index.hbcCacheStats()); + item.hbc_posting.refresh_pending = entry.index.postingRefreshPending(); } try self.populateConfiguredDerivedCoverageCounts(item.name, item); visible_doc_count = @max(visible_doc_count, item.doc_count); @@ -33419,6 +33857,7 @@ pub const DB = struct { item.serving_snapshot_owner_id = self.backend_owner_id; serving_observed = true; item.hbc_cache = dbHbcCacheStats(entry.index.hbcCacheStats()); + item.hbc_posting.refresh_pending = entry.index.postingRefreshPending(); visible_doc_count = @max(visible_doc_count, item.doc_count); try self.markDenseCoverageRegressionIfNeeded(alloc, cfg.name, &item); } @@ -33668,6 +34107,7 @@ pub const DB = struct { item.root_node = hbc_stats.root_node; item.hbc_cache = dbHbcCacheStats(entry.index.hbcCacheStats()); item.hbc_posting = dbHbcPostingStats(try entry.index.postingBacklogStats(), entry.index.getWriteProfile()); + item.hbc_posting.refresh_pending = entry.index.postingRefreshPending(); try self.markDenseCoverageRegressionIfNeeded(alloc, cfg.name, &item); if (async_indexing.dense_catch_up.active) { item.catch_up_active = true; @@ -68377,45 +68817,69 @@ test "relational columnar shared pages bound alternating merges and survive recl test "relational columnar row cursor skips artifact fanout and preserves binary owners" { const alloc = std.testing.allocator; - for ([_]PrimaryBackend{ .lmdb, .{ .lsm = .{ .flush_threshold = 1 } } }) |backend| { - var path_tmp = try TestDirectory.init("db"); - defer path_tmp.cleanup(); - const path = path_tmp.path().ptr; - defer cleanupTempDir(path); - var db = try DB.open(alloc, std.mem.span(path), .{ .start_optional_runtimes = false, .primary_backend = backend }); - defer db.close(); - const columns = [_]schema_mod.RelationalColumn{.{ .name = "n", .path = "n", .column_type = .integer }}; - try db.setSchema(.{ .version = 1, .storage_mode = .relational, .relational_columns = &columns }); - const owners = [_][]const u8{ "", "a", "a\x00", "a\xff", "orphan" }; - for (owners[0..4]) |owner| try db.batch(.{ .writes = &.{.{ .key = owner, .value = "{\"n\":1}" }} }); - var arena = std.heap.ArenaAllocator.init(alloc); - defer arena.deinit(); - const scratch = arena.allocator(); - var batch = try db.core.store.beginWriteBatch(); - var live = true; - defer if (live) batch.abort(); - for (owners) |owner| { - const prefix_key = try internal_keys.artifactRootPrefixAlloc(scratch, owner); - for (0..2048) |i| try batch.asTxn().put(try std.fmt.allocPrint(scratch, "{s}{d:0>4}", .{ prefix_key, i }), "artifact payload is never a row"); + // Exercise both an uninterrupted bootstrap and deterministic partial + // publication. Small published ranges may then be merged by maintenance. + relational_columns.test_disable_deadline = true; + defer relational_columns.test_disable_deadline = false; + defer relational_columns.test_owner_limit = null; + for ([_]?usize{ null, 2 }) |owner_limit| { + for ([_]PrimaryBackend{ .lmdb, .{ .lsm = .{ .flush_threshold = 1 } } }) |backend| { + relational_columns.test_owner_limit = owner_limit; + var path_tmp = try TestDirectory.init("db"); + defer path_tmp.cleanup(); + const path = path_tmp.path().ptr; + defer cleanupTempDir(path); + var db = try DB.open(alloc, std.mem.span(path), .{ .start_optional_runtimes = false, .primary_backend = backend }); + defer db.close(); + const columns = [_]schema_mod.RelationalColumn{.{ .name = "n", .path = "n", .column_type = .integer }}; + try db.setSchema(.{ .version = 1, .storage_mode = .relational, .relational_columns = &columns }); + const owners = [_][]const u8{ "", "a", "a\x00", "a\xff", "orphan" }; + for (owners[0..4]) |owner| try db.batch(.{ .writes = &.{.{ .key = owner, .value = "{\"n\":1}" }} }); + var arena = std.heap.ArenaAllocator.init(alloc); + defer arena.deinit(); + const scratch = arena.allocator(); + var batch = try db.core.store.beginWriteBatch(); + var live = true; + defer if (live) batch.abort(); + for (owners) |owner| { + const prefix_key = try internal_keys.artifactRootPrefixAlloc(scratch, owner); + for (0..2048) |i| try batch.asTxn().put(try std.fmt.allocPrint(scratch, "{s}{d:0>4}", .{ prefix_key, i }), "artifact payload is never a row"); + } + try batch.commit(); + live = false; + var stats: types.ColumnarScanStats = .{}; + var primary = try db.scan(alloc, "", "", .{ .include_documents = true, .include_all_fields = true, .columnar_stats = &stats }); + defer primary.deinit(alloc); + try std.testing.expectEqual(@as(usize, 4), primary.documents.len); + try std.testing.expectEqual(@as(u64, 5), stats.primary_owners_examined); + for (primary.documents, owners[0..4]) |document, owner| try std.testing.expectEqualStrings(owner, document.id); + stats = .{}; + var bounded = try db.scan(alloc, "a", "a\xff", .{ .include_documents = true, .include_all_fields = true, .inclusive_from = false, .exclusive_to = true, .columnar_stats = &stats }); + defer bounded.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), bounded.documents.len); + try std.testing.expectEqualStrings("a\x00", bounded.documents[0].id); + if (owner_limit) |limit| { + try std.testing.expect(try db.rebuildRelationalColumns()); + try std.testing.expectEqual(@as(u64, limit), db.relational_column_maintenance.owners_examined.load(.monotonic)); + // Force a bootstrap split, then restore the normal merge budget. + relational_columns.test_owner_limit = null; + } + try drainTestRelationalMaintenance(&db); + const maintenance = db.relational_column_maintenance.snapshot(); + try std.testing.expectEqual(@as(u64, 4), maintenance.primary_rows_read); + // Owner visits include typed rows revisited by post-bootstrap merges; + // only the five primary owners may be scanned, regardless of fanout. + try std.testing.expectEqual(@as(u64, 5) + maintenance.covered_rows_read, maintenance.owners_examined); + if (owner_limit == null) { + try std.testing.expectEqual(@as(u64, 1), maintenance.bootstrap_quanta); + } else { + try std.testing.expect(maintenance.bootstrap_quanta > 1); + try std.testing.expect(maintenance.covered_rows_read > 0); + } + var covered = try db.scan(alloc, "", "", .{ .include_documents = true, .include_all_fields = false, .fields = &.{"n"} }); + defer covered.deinit(alloc); + try std.testing.expectEqualDeep(primary.documents, covered.documents); } - try batch.commit(); - live = false; - var stats: types.ColumnarScanStats = .{}; - var primary = try db.scan(alloc, "", "", .{ .include_documents = true, .include_all_fields = true, .columnar_stats = &stats }); - defer primary.deinit(alloc); - try std.testing.expectEqual(@as(usize, 4), primary.documents.len); - try std.testing.expectEqual(@as(u64, 5), stats.primary_owners_examined); - for (primary.documents, owners[0..4]) |document, owner| try std.testing.expectEqualStrings(owner, document.id); - stats = .{}; - var bounded = try db.scan(alloc, "a", "a\xff", .{ .include_documents = true, .include_all_fields = true, .inclusive_from = false, .exclusive_to = true, .columnar_stats = &stats }); - defer bounded.deinit(alloc); - try std.testing.expectEqual(@as(usize, 1), bounded.documents.len); - try std.testing.expectEqualStrings("a\x00", bounded.documents[0].id); - try drainTestRelationalMaintenance(&db); - try std.testing.expectEqual(@as(u64, 5), db.relational_column_maintenance.owners_examined.load(.monotonic)); - var covered = try db.scan(alloc, "", "", .{ .include_documents = true, .include_all_fields = false, .fields = &.{"n"} }); - defer covered.deinit(alloc); - try std.testing.expectEqualDeep(primary.documents, covered.documents); } } @@ -89106,7 +89570,7 @@ fn testDenseSourceHashReuse(settings: table_storage_mod.Settings) !void { }); try db.runUntilIdle(); try std.testing.expectEqual(@as(usize, 1), counting.calls); - if (db.source_vectors) |source| { + if (db.source_vectors.load(.acquire)) |source| { const key = try expectedDocumentEmbeddingArtifactKeyAlloc(alloc, "doc:a", "body_dense_v1"); defer alloc.free(key); const before = source.stats.resolved_payloads; @@ -89205,14 +89669,18 @@ test "db dense enrichment republishes unchanged source hash from cached artifact } test "db chunked dense enrichment skips unchanged chunks and deletes stale chunk artifacts" { - try testDenseChunkArtifactLifecycle(.{}); + try testDenseChunkArtifactLifecycle(.{}, false); } test "source vector table deletes stale chunk embeddings" { - try testDenseChunkArtifactLifecycle(.{ .dense_embeddings = .vector_store }); + try testDenseChunkArtifactLifecycle(.{ .dense_embeddings = .vector_store }, false); } -fn testDenseChunkArtifactLifecycle(settings: table_storage_mod.Settings) !void { +test "source vector migration captures enrichment updates and stale chunk deletion" { + try testDenseChunkArtifactLifecycle(.{ .dense_embeddings = .primary_lsm }, true); +} + +fn testDenseChunkArtifactLifecycle(settings: table_storage_mod.Settings, migrate: bool) !void { const alloc = std.testing.allocator; var path_tmp = try TestDirectory.init("db"); @@ -89243,6 +89711,10 @@ fn testDenseChunkArtifactLifecycle(settings: table_storage_mod.Settings) !void { try db.runUntilIdle(); const first_calls = counting.calls; try std.testing.expect(first_calls > 0); + if (migrate) { + try db.startVectorMigration(.{ .job_id = "chunks", .mode = .online }); + try db.advanceVectorMigration("chunks"); + } try db.batch(.{ .writes = &.{.{ .key = "doc:a", .value = "{\"title\":\"changed\",\"body\":\"abcdefghijklmno\"}" }}, @@ -89281,6 +89753,14 @@ fn testDenseChunkArtifactLifecycle(settings: table_storage_mod.Settings) !void { }); try db.runUntilIdle(); try std.testing.expect(counting.calls > first_calls); + if (migrate) { + for (0..256) |_| { + var state = (try vector_migration.load(alloc, db.core.store)).?; + defer state.deinit(); + if (state.value.phase == .complete) break; + if (state.value.phase == .ready) try db.publishVectorMigration("chunks") else try db.advanceVectorMigration("chunks"); + } else return error.VectorMigrationDidNotFinish; + } const chunk_prefix = try internal_keys.artifactNamedPrefixAlloc(alloc, "doc:a", "chunk", "body_chunks_v1"); defer alloc.free(chunk_prefix); @@ -90125,6 +90605,48 @@ test "db runUntilIdle drains lazy dense posting maintenance" { } } +test "db posting refresh status follows verification and invalidates cached observations" { + const alloc = std.testing.allocator; + var path_tmp = try TestDirectory.init("db"); + defer path_tmp.cleanup(); + var db = try DB.open(alloc, path_tmp.path(), .{}); + defer db.close(); + try db.addIndex(.{ + .name = "dv_v1", + .kind = .dense_vector, + .config_json = "{\"field\":\"embedding\",\"dims\":2,\"use_quantization\":false}", + }); + try db.batch(.{ + .writes = &.{.{ .key = "a", .value = "{\"embedding\":[1.0,0.0]}" }}, + .sync_level = .full_index, + }); + const entry = db.core.denseIndex("dv_v1").?; + const manager = entry.index.resource_manager; + entry.index.resource_manager = null; + defer entry.index.resource_manager = manager; + try entry.index.markNodePostingDirtyForTest(entry.index.metadata.root_node); + { + const status = try db.runtimeStatusStatsConsistent(alloc); + defer types.freeDBStats(alloc, status); + try std.testing.expect(status.indexes[0].hbc_posting.refresh_pending); + } + for (0..16) |_| { + if (!(try entry.index.refreshPostingPayloadPage(1, 1)).pending) break; + } + try std.testing.expect(!entry.index.postingRefreshPending()); + var cached = try db.runtimeStatusStatsConsistent(alloc); + defer types.freeDBStats(alloc, cached); + try std.testing.expect(!cached.indexes[0].hbc_posting.refresh_pending); + { + const diagnostic = try db.diagnosticStats(alloc); + defer types.freeDBStats(alloc, diagnostic); + try std.testing.expect(!diagnostic.indexes[0].hbc_posting.refresh_pending); + } + try entry.index.markNodePostingDirtyForTest(entry.index.metadata.root_node); + try db.overlayRuntimeStatusConsistent(alloc, &cached); + try std.testing.expect(cached.indexes[0].hbc_posting.refresh_pending); +} + test "db posting refresh checks clean indexes without exclusive admission" { const alloc = std.testing.allocator; var path_tmp = try TestDirectory.init("db"); @@ -109289,6 +109811,7 @@ test "db last dense catch-up lease finalizes every covered rebuilding generation var db = try DB.open(alloc, std.mem.span(path), .{ .start_index_workers = false, + .start_optional_runtime_workers = false, .ttl_cleanup = .{ .enabled = false }, }); defer db.close(); @@ -128367,6 +128890,262 @@ fn loadStoredSearchDocumentManyCallback( return try loadStoredSearchDocumentsMany(self, alloc, keys, null); } +fn completeVectorMigrationForTest(db: *DB, job_id: []const u8) !void { + const deadline = platform_time.monotonicNs() + 30 * std.time.ns_per_s; + while (true) { + var state = (try vector_migration.load(std.testing.allocator, db.core.store)).?; + defer state.deinit(); + if (state.value.phase == .complete) return; + if (platform_time.monotonicNs() >= deadline) { + std.debug.print("migration deadline: phase={s} scanned={d} rewritten={d} reclamation={}\n", .{ + @tagName(state.value.phase), state.value.scanned_rows, state.value.rewritten_artifacts, state.value.primary_reclamation_requested, + }); + return error.VectorMigrationDidNotFinish; + } + if (state.value.phase == .ready) { + try db.publishVectorMigration(job_id); + } else try db.advanceVectorMigration(job_id); + // Reclamation retains ordinary timed admission retries. A fixed count + // of tight iterations can finish before a retry becomes due on a fast + // filesystem; yielding also lets admitted asynchronous work progress. + if (state.value.phase == .reclaiming) + try db.core.index_manager.checkpointIo().sleep(.fromMilliseconds(1), .awake); + } +} + +test "source vector migration progress pages sync the WAL without flushing tiny runs" { + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("vector-migration-page-wal"); + defer tmp.cleanup(); + var db = try DB.open(alloc, std.mem.span(tmp.path().ptr), .{ + .table_storage = .{ .dense_embeddings = .primary_lsm }, + .start_index_workers = false, + .start_optional_runtimes = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + // Verification and cleanup often change only a small progress record. + // A byte-bounded scan must not turn those records into one SST per page. + for (0..64) |i| { + const key = try std.fmt.allocPrint(alloc, "ordinary-{d:0>4}", .{i}); + defer alloc.free(key); + try db.core.store.put(key, "value"); + } + const request: vector_migration.contract.Request = .{ + .job_id = "page-wal", + .mode = .online, + .budget = .{ .batch_rows = 1, .disk_reserve_bytes = 0 }, + }; + try db.startVectorMigration(request); + const backend = db.core.primary_store_owner.lsmBackend().?; + const before = backend.snapshotWriteStats(); + for (0..32) |_| try db.advanceVectorMigration(request.job_id); + const after = backend.snapshotWriteStats(); + var job = (try vector_migration.load(alloc, db.core.store)).?; + defer job.deinit(); + try std.testing.expectEqual(@as(u64, 32), job.value.scanned_rows); + try std.testing.expectEqual(before.flushes, after.flushes); + try std.testing.expectEqual(before.flush_output_runs, after.flush_output_runs); +} + +test "source vector migration drains late oversized embeddings and skips unrelated payloads" { + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("migration-large-rows"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const options: OpenOptions = .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }; + const key = try internal_keys.embeddingArtifactKeyForDocumentAlloc(alloc, "late", "model"); + defer alloc.free(key); + const values: [2048]f32 = @splat(0.5); + const artifact = try enrichment_artifact_codec.encodeDenseEmbeddingAlloc(alloc, 2, &values); + defer alloc.free(artifact); + const large: [6000]u8 = @splat('x'); + const request: vector_migration.contract.Request = .{ .job_id = "large", .mode = .online, .budget = .{ .batch_rows = 2, .batch_bytes = 4096, .disk_reserve_bytes = 0 } }; + { + var db = try DB.open(alloc, path, options); + defer db.close(); + try db.core.store.put("unrelated-before", &large); + try db.startVectorMigration(request); + for (0..128) |_| { + var job = (try vector_migration.load(alloc, db.core.store)).?; + defer job.deinit(); + if (job.value.phase == .ready) break; + try db.advanceVectorMigration(request.job_id); + } else return error.VectorMigrationDidNotFinish; + // Capture is still active after verification: this valid vector is + // larger than a page but already has a durable candidate reference. + try db.core.store.put(key, artifact); + try db.core.store.put("unrelated-after", &large); + try db.publishVectorMigration(request.job_id); + try db.advanceVectorMigration(request.job_id); + } + var db = try DB.open(alloc, path, options); + defer db.close(); + try completeVectorMigrationForTest(&db, request.job_id); + const actual = try db.core.store.get(alloc, key); + defer alloc.free(actual); + try std.testing.expectEqualSlices(u8, artifact, actual); + inline for (.{ "unrelated-before", "unrelated-after" }) |name| { + const document = try db.core.store.get(alloc, name); + defer alloc.free(document); + try std.testing.expectEqualSlices(u8, &large, document); + } +} + +test "source vector migration recovers each preparation commit and publication boundary" { + const alloc = std.testing.allocator; + const Hook = struct { + var selected: vector_migration.Boundary = .before_prepare; + fn fail(point: vector_migration.Boundary) !void { + if (point == selected) return error.TestVectorMigrationCrash; + } + }; + inline for (std.meta.tags(vector_migration.Boundary)) |point| { + var tmp = try TestDirectory.init("vector-migration-crash"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const options: OpenOptions = .{ + .table_storage = .{ .dense_embeddings = .primary_lsm }, + .start_index_workers = false, + .start_optional_runtimes = false, + .ttl_cleanup = .{ .enabled = false }, + }; + const key = try internal_keys.embeddingArtifactKeyForDocumentAlloc(alloc, "doc", "model"); + defer alloc.free(key); + const value = try enrichment_artifact_codec.encodeDenseEmbeddingAlloc(alloc, 123, &.{ 1, -2, 3 }); + defer alloc.free(value); + const request: vector_migration.contract.Request = .{ .job_id = "crash", .mode = .online }; + { + var db = try DB.open(alloc, path, options); + defer db.close(); + try db.core.store.put(key, value); + try db.startVectorMigration(request); + Hook.selected = point; + vector_migration.test_boundary = Hook.fail; + defer vector_migration.test_boundary = null; + if (point == .reclamation_request or point == .reclamation_receipt) { + var crashed = false; + for (0..512) |_| { + var state = (try vector_migration.load(alloc, db.core.store)).?; + defer state.deinit(); + if (state.value.phase == .ready) { + try db.publishVectorMigration(request.job_id); + } else db.advanceVectorMigration(request.job_id) catch |err| { + try std.testing.expectEqual(error.TestVectorMigrationCrash, err); + crashed = true; + break; + }; + } + try std.testing.expect(crashed); + } else if (point == .publication_commit or point == .publication_sync) { + for (0..256) |_| { + var state = (try vector_migration.load(alloc, db.core.store)).?; + defer state.deinit(); + if (state.value.phase == .ready) break; + try db.advanceVectorMigration(request.job_id); + } else return error.VectorMigrationDidNotFinish; + try std.testing.expectError(error.TestVectorMigrationCrash, db.publishVectorMigration(request.job_id)); + } else { + try std.testing.expectError(error.TestVectorMigrationCrash, db.advanceVectorMigration(request.job_id)); + } + try std.testing.expectError(error.VectorMigrationRecoveryRequired, db.advanceVectorMigration(request.job_id)); + try std.testing.expectError(error.VectorMigrationRecoveryRequired, db.core.store.put(key, value)); + } + // Multiple reopens must agree on both the decision and exact payload. + for (0..3) |_| { + var db = try DB.open(alloc, path, options); + defer db.close(); + try completeVectorMigrationForTest(&db, request.job_id); + const restored = try db.core.store.get(alloc, key); + defer alloc.free(restored); + try std.testing.expectEqualSlices(u8, value, restored); + try std.testing.expectError(error.VectorMigrationAlreadyPublished, db.cancelVectorMigration(request.job_id)); + } + } +} + +test "source vector migration preserves concurrent models deletes and old snapshots through restart" { + const alloc = std.testing.allocator; + const payload = @import("../artifact_payload.zig"); + var tmp = try TestDirectory.init("vector-migration"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const options: OpenOptions = .{ + .table_storage = .{ .dense_embeddings = .primary_lsm }, + .start_index_workers = false, + .start_optional_runtimes = false, + .ttl_cleanup = .{ .enabled = false }, + }; + const key_a = try internal_keys.embeddingArtifactKeyForDocumentAlloc(alloc, "doc", "model-a"); + defer alloc.free(key_a); + const key_b = try internal_keys.embeddingArtifactKeyForDocumentAlloc(alloc, "doc", "model-b"); + defer alloc.free(key_b); + const key_c = try internal_keys.embeddingArtifactKeyForDocumentAlloc(alloc, "earlier", "model-c"); + defer alloc.free(key_c); + const old = try enrichment_artifact_codec.encodeDenseEmbeddingAlloc(alloc, 11, &.{ 1, 2, 3 }); + defer alloc.free(old); + const new = try enrichment_artifact_codec.encodeDenseEmbeddingAlloc(alloc, 12, &.{ 4, 5, 6 }); + defer alloc.free(new); + const request: vector_migration.contract.Request = .{ + .job_id = "online-test", + .mode = .online, + .budget = .{ .batch_rows = 2 }, + }; + { + var db = try DB.open(alloc, path, options); + defer db.close(); + try db.core.store.putBatch(&.{ .{ .key = key_a, .value = old }, .{ .key = key_b, .value = old } }, &.{}); + var old_reader = try db.core.store.beginReadTxn(); + defer old_reader.abort(); + try db.startVectorMigration(request); + try db.startVectorMigration(request); + try db.advanceVectorMigration(request.job_id); + try db.core.store.putBatch(&.{ .{ .key = key_a, .value = new }, .{ .key = key_c, .value = old } }, &.{key_b}); + var raw = try db.core.store.runtime_store.beginRead(); + defer raw.abort(); + try std.testing.expectEqualSlices(u8, new, try raw.get(key_a)); + const candidate_key = try vector_migration.contract.candidateKeyAlloc(alloc, key_a); + defer alloc.free(candidate_key); + try std.testing.expectEqualSlices(u8, &(try payload.Reference.forArtifact(key_a, new)).encode(), try raw.get(candidate_key)); + try std.testing.expectEqualSlices(u8, old, try old_reader.get(key_a)); + try std.testing.expect(!try db.collectSourceVectorGarbage()); + } + { + var db = try DB.open(alloc, path, options); + defer db.close(); + var before_publication = try db.core.store.beginReadTxn(); + defer before_publication.abort(); + for (0..256) |_| { + var state = (try vector_migration.load(alloc, db.core.store)).?; + defer state.deinit(); + if (state.value.phase == .ready) break; + try db.advanceVectorMigration(request.job_id); + } else return error.VectorMigrationDidNotFinish; + try db.publishVectorMigration(request.job_id); + try db.publishVectorMigration(request.job_id); + try db.core.store.put(key_c, new); + for (0..256) |_| { + var state = (try vector_migration.load(alloc, db.core.store)).?; + defer state.deinit(); + if (state.value.phase == .complete) break; + try db.advanceVectorMigration(request.job_id); + } else return error.VectorMigrationDidNotFinish; + try std.testing.expectEqualSlices(u8, old, try before_publication.get(key_c)); + var raw = try db.core.store.runtime_store.beginRead(); + defer raw.abort(); + try std.testing.expect(payload.isReference(try raw.get(key_a))); + try std.testing.expect(payload.isReference(try raw.get(key_c))); + try std.testing.expectError(error.NotFound, raw.get(key_b)); + } + // The durable publication bridges a catalog response lost after DB sync. + var reopened = try DB.open(alloc, path, options); + defer reopened.close(); + try std.testing.expectEqual(.vector_store, reopened.table_storage.dense_embeddings); + const restored = try reopened.core.store.get(alloc, key_a); + defer alloc.free(restored); + try std.testing.expectEqualSlices(u8, new, restored); +} + test "source vector table persists references without an ANN index and reopens" { const alloc = std.testing.allocator; var path_tmp = try TestDirectory.init("db"); @@ -128764,7 +129543,7 @@ test "source vector table defers reopen GC and bounded maintenance preserves rea defer db.close(); try db.core.store.put(key, first); try db.core.store.put(key, second); - const source = db.source_vectors.?; + const source = db.source_vectors.load(.acquire).?; const iface = source.interface(); try iface.vtable.prepare(iface.ptr, &.{.{ .reference = try payload.Reference.forArtifact("uncommitted", first), .artifact = first }}); try db.core.store.sync(true); @@ -128774,7 +129553,7 @@ test "source vector table defers reopen GC and bounded maintenance preserves rea { var db = try DB.open(alloc, std.mem.span(path), opts); defer db.close(); - const source = db.source_vectors.?; + const source = db.source_vectors.load(.acquire).?; try std.testing.expect(source.background_gc and source.mark_outside_lock and source.independent_scan); try std.testing.expectEqual(@as(u64, 0), source.statsSnapshot().collections); try std.testing.expectEqual(@as(u64, 0), source.statsSnapshot().collection_mark_rows); @@ -128800,7 +129579,7 @@ test "source vector table defers reopen GC and bounded maintenance preserves rea } var db = try DB.open(alloc, std.mem.span(path), opts); defer db.close(); - const source = db.source_vectors.?; + const source = db.source_vectors.load(.acquire).?; try std.testing.expectEqual(@as(u64, 0), source.statsSnapshot().collection_mark_rows); const current = try db.core.store.get(alloc, key); defer alloc.free(current); @@ -128835,3 +129614,465 @@ test "source vector table defers reopen GC and bounded maintenance preserves rea try std.testing.expect(turns < 1024); try std.testing.expectEqual(@as(u64, 0), source.statsSnapshot().retained_payloads); } + +test "source vector migration offline resumes a physical shadow preserving every internal namespace" { + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("offline-vector-migration"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const payload = @import("../artifact_payload.zig"); + const offline = @import("../vector_migration_offline.zig"); + const key = try internal_keys.embeddingArtifactKeyForDocumentAlloc(alloc, "doc", "model-a"); + defer alloc.free(key); + const encoded = try enrichment_artifact_codec.encodeDenseEmbeddingAlloc(alloc, 29, &.{ 1, 2, 3 }); + defer alloc.free(encoded); + const options: OpenOptions = .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }; + var identity: DocIdentityNamespace = undefined; + { + var source = try DB.open(alloc, path, options); + defer source.close(); + identity = source.core.identity_namespace; + try source.core.store.putBatch(&.{ .{ .key = key, .value = encoded }, .{ .key = "private-copy-test", .value = "preserve opaque internal state" } }, &.{}); + try source.core.store.runtime_store.sync(true); + } + const request: vector_migration.contract.Request = .{ .job_id = "offline-test", .mode = .offline, .budget = .{ .batch_bytes = 4096, .disk_reserve_bytes = 0 } }; + try std.testing.expectEqual(.pending, try offline.run(alloc, std.testing.io, path, request, .{ .open = options, .max_steps = 1 })); + try std.testing.expectError(error.VectorMigrationOfflineAdmission, DB.open(alloc, path, options)); + try std.testing.expectEqual(.complete, try offline.run(alloc, std.testing.io, path, request, .{ .open = options })); + try std.testing.expectEqual(.complete, try offline.run(alloc, std.testing.io, path, request, .{ .open = options })); + var target = try DB.open(alloc, path, options); + defer target.close(); + try std.testing.expect(identity.eql(target.core.identity_namespace)); + try std.testing.expectEqual(.vector_store, target.table_storage.dense_embeddings); + const restored = try target.core.store.get(alloc, key); + defer alloc.free(restored); + try std.testing.expectEqualSlices(u8, encoded, restored); + const opaque_value = try target.core.store.get(alloc, "private-copy-test"); + defer alloc.free(opaque_value); + try std.testing.expectEqualStrings("preserve opaque internal state", opaque_value); + var raw = try target.core.store.runtime_store.beginRead(); + defer raw.abort(); + try std.testing.expect(payload.isReference(try raw.get(key))); +} + +test "source vector migration consolidates ANN serving while preserving queries and last-index ownership" { + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("migration-ann"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + var db = try DB.open(alloc, path, .{ + .table_storage = .{ .dense_embeddings = .primary_lsm }, + .start_index_workers = false, + .start_optional_runtimes = false, + .ttl_cleanup = .{ .enabled = false }, + }); + defer db.close(); + const cfg: types.IndexConfig = .{ .name = "semantic", .kind = .dense_vector, .config_json = "{\"field\":\"embedding\",\"dims\":3,\"metric\":\"l2_squared\",\"external\":true,\"embedding_name\":\"shared\"}" }; + try db.addIndex(cfg); + try db.batch(.{ .writes = &.{ + .{ .key = "a", .value = "{\"title\":\"alpha\",\"_embeddings\":{\"shared\":[1,0,0]}}" }, + .{ .key = "b", .value = "{\"title\":\"beta\",\"_embeddings\":{\"shared\":[0,1,0]}}" }, + }, .sync_level = .write }); + try db.runUntilIdle(); + const request: vector_migration.contract.Request = .{ .job_id = "ann", .mode = .online, .budget = .{ .batch_rows = 3 } }; + try db.startVectorMigration(request); + try std.testing.expectError(error.VectorMigrationActive, db.deleteIndex(cfg.name)); + for (0..256) |_| { + var status = (try vector_migration.load(alloc, db.core.store)).?; + defer status.deinit(); + var result = try db.search(alloc, .{ .index_name = "semantic", .dense = .{ .vector = &.{ 1, 0, 0 }, .k = 2 }, .limit = 2 }); + defer result.deinit(); + try std.testing.expectEqual(@as(u32, 2), result.total_hits); + try std.testing.expectEqualStrings("a", result.hits[0].id); + if (status.value.phase == .complete) break; + if (status.value.phase == .ready) try db.publishVectorMigration(request.job_id) else try db.advanceVectorMigration(request.job_id); + } else return error.VectorMigrationDidNotFinish; + try std.testing.expect(db.core.index_manager.sourceMigrationServingComplete()); + try std.testing.expect(try db.deleteIndex(cfg.name)); + const key = try expectedDocumentEmbeddingArtifactKeyAlloc(alloc, "a", "shared"); + defer alloc.free(key); + const retained = try db.core.store.get(alloc, key); + defer alloc.free(retained); + try std.testing.expectEqual(@as(usize, 3), try enrichment_artifact_codec.decodeDenseEmbeddingDims(retained)); + _ = try db.admitManagedIndex(.{ .name = "replacement", .kind = cfg.kind, .config_json = cfg.config_json }); + _ = try db.rebuildDenseIndexesFromStoredEmbeddingArtifactsIfNeeded(alloc); + const repair_id = (try db.indexRepairIdForIndex(alloc, "replacement")) orelse return error.TestUnexpectedResult; + _ = try db.advanceIndexRepairIntent(alloc, repair_id, .{}); + try db.runUntilIdle(); + var result = try db.search(alloc, .{ .index_name = "replacement", .dense = .{ .vector = &.{ 1, 0, 0 }, .k = 2 }, .limit = 2 }); + defer result.deinit(); + try std.testing.expectEqual(@as(u32, 2), result.total_hits); +} + +test "source vector migration budget rejection is retryable and cancellation survives restart" { + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("migration-cancel"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const options: OpenOptions = .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }; + const key = try internal_keys.embeddingArtifactKeyForDocumentAlloc(alloc, "doc", "source"); + defer alloc.free(key); + const artifact = try enrichment_artifact_codec.encodeDenseEmbeddingAlloc(alloc, 2, &.{ 1, 2, 3 }); + defer alloc.free(artifact); + const request: vector_migration.contract.Request = .{ .job_id = "too-small", .mode = .online, .budget = .{ .batch_bytes = 4096, .temporary_bytes = 4096, .disk_reserve_bytes = 0 } }; + { + var db = try DB.open(alloc, path, options); + defer db.close(); + try db.core.store.put(key, artifact); + try db.startVectorMigration(request); + try std.testing.expectError(error.VectorMigrationTemporaryBudgetExceeded, db.advanceVectorMigration(request.job_id)); + try std.testing.expect(!db.vector_migration_reopen_required.load(.acquire)); + try std.testing.expectError(error.VectorMigrationTemporaryBudgetExceeded, db.core.store.put(key, artifact)); + try db.cancelVectorMigration(request.job_id); + } + { + var db = try DB.open(alloc, path, options); + defer db.close(); + for (0..128) |_| { + var status = (try vector_migration.load(alloc, db.core.store)).?; + defer status.deinit(); + if (status.value.phase == .cancelled) break; + try db.advanceVectorMigration(request.job_id); + } else return error.VectorMigrationDidNotFinish; + const actual = try db.core.store.get(alloc, key); + defer alloc.free(actual); + try std.testing.expectEqualSlices(u8, artifact, actual); + } + var db = try DB.open(alloc, path, options); + defer db.close(); + try std.testing.expect(db.source_vectors.load(.acquire) == null); + try db.startVectorMigration(.{ .job_id = "retry", .mode = .online }); + var status = (try vector_migration.load(alloc, db.core.store)).?; + defer status.deinit(); + try std.testing.expectEqual(@as(u64, 2), status.value.ownership_epoch); +} + +test "source vector migration cancelled snapshot preserves old readers and rejects a replacement" { + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("migration-cancel-snapshot"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const snapshots = try std.fmt.allocPrint(alloc, "{s}.snapshots", .{path}); + defer alloc.free(snapshots); + defer std.Io.Dir.cwd().deleteTree(std.testing.io, snapshots) catch {}; + var db = try DB.open(alloc, path, .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }); + defer db.close(); + try db.core.store.put("ordinary", "preserved"); + const request: vector_migration.contract.Request = .{ .job_id = "cancel", .mode = .online, .budget = .{ .disk_reserve_bytes = 0 } }; + try db.startVectorMigration(request); + var old = try db.core.store.beginReadTxn(); + defer old.abort(); + try std.testing.expect(old.payload_session != null); + try std.testing.expectError(error.VectorStoreLifecycleUnsupported, db.snapshotNative("active")); + try db.cancelVectorMigration(request.job_id); + try std.testing.expectError(error.VectorStoreLifecycleUnsupported, db.snapshotNative("cancelling")); + try db.advanceVectorMigration(request.job_id); + try std.testing.expect(db.source_vectors.load(.acquire) != null); + try std.testing.expect(try db.snapshotNative("cancelled") > 0); + try std.testing.expectEqualStrings("preserved", try old.get("ordinary")); + try db.startVectorMigration(.{ .job_id = "replacement", .mode = .online, .budget = request.budget }); + try std.testing.expectError(error.VectorStoreLifecycleUnsupported, db.snapshotNative("replacement")); +} + +test "source vector migration cancels rejected admission durably without a source store" { + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("migration-rejected-admission"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const options: OpenOptions = .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }; + const request: vector_migration.contract.Request = .{ .job_id = "disk-rejected", .mode = .online, .budget = .{ .disk_reserve_bytes = std.math.maxInt(u64) } }; + { + var db = try DB.open(alloc, path, options); + defer db.close(); + try db.core.store.put("preserved", "value"); + try std.testing.expectError(error.VectorMigrationDiskReserve, db.startVectorMigration(request)); + try std.testing.expect((try vector_migration.load(alloc, db.core.store)) == null); + } + // Restart with a catalog admission but no DB job, then lose the cancel + // response (the catalog marker is intentionally not reconciled here). + { + var db = try DB.open(alloc, path, options); + defer db.close(); + const raw = try db.vectorMigrationCommand(alloc, .{ .action = .cancel, .request = request }); + defer alloc.free(raw); + var receipt = try std.json.parseFromSlice(vector_migration.contract.Job, alloc, raw, .{}); + defer receipt.deinit(); + try receipt.value.validate(); + try std.testing.expectEqual(.cancelled, receipt.value.phase); + try std.testing.expect(db.source_vectors.load(.acquire) == null); + } + var db = try DB.open(alloc, path, options); + defer db.close(); + const cancelled = try db.vectorMigrationCommand(alloc, .{ .action = .cancel, .request = request }); + defer alloc.free(cancelled); + const retried_start = try db.vectorMigrationCommand(alloc, .{ .action = .start, .request = request }); + defer alloc.free(retried_start); + try std.testing.expectEqualSlices(u8, cancelled, retried_start); + var changed = request; + changed.budget.disk_reserve_bytes = 0; + try std.testing.expectError(error.VectorMigrationIdempotencyConflict, db.vectorMigrationCommand(alloc, .{ .action = .cancel, .request = changed })); + try std.testing.expect(db.source_vectors.load(.acquire) == null); + try std.testing.expect(!db.vector_migration_active.load(.acquire)); + const value = try db.core.store.get(alloc, "preserved"); + defer alloc.free(value); + try std.testing.expectEqualStrings("value", value); + // A later admission can also fail while the previous cancelled receipt + // remains in the DB. Cancellation must create the new receipt in this case. + var second = request; + second.job_id = "second-rejection"; + try std.testing.expectError(error.VectorMigrationDiskReserve, db.startVectorMigration(second)); + const second_cancel = try db.vectorMigrationCommand(alloc, .{ .action = .cancel, .request = second }); + defer alloc.free(second_cancel); + var second_job = try std.json.parseFromSlice(vector_migration.contract.Job, alloc, second_cancel, .{}); + defer second_job.deinit(); + try std.testing.expectEqualStrings(second.job_id, second_job.value.job_id); + try std.testing.expectEqual(@as(u64, 2), second_job.value.ownership_epoch); + try db.startVectorMigration(.{ .job_id = "replacement", .mode = .online, .budget = .{ .disk_reserve_bytes = 0 } }); + try std.testing.expectError(error.VectorMigrationIdempotencyConflict, db.vectorMigrationCommand(alloc, .{ .action = .cancel, .request = second })); + var active = (try vector_migration.load(alloc, db.core.store)).?; + defer active.deinit(); + try std.testing.expectEqual(.backfill, active.value.phase); + try std.testing.expectEqual(@as(u64, 3), active.value.ownership_epoch); +} + +test "source vector migration offline recovers every copy and publication boundary" { + const offline = @import("../vector_migration_offline.zig"); + const Hook = struct { + var selected: offline.Boundary = .fenced; + var fired: bool = false; + fn inject(point: offline.Boundary) !void { + if (!fired and point == selected) { + fired = true; + return error.InjectedMigrationCrash; + } + } + }; + const alloc = std.testing.allocator; + const key = try internal_keys.embeddingArtifactKeyForDocumentAlloc(alloc, "doc", "model"); + defer alloc.free(key); + const artifact = try enrichment_artifact_codec.encodeDenseEmbeddingAlloc(alloc, 99, &.{ 1, 2, 3 }); + defer alloc.free(artifact); + inline for (std.meta.tags(offline.Boundary)) |point| { + var tmp = try TestDirectory.init("offline-crash"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const options: OpenOptions = .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }; + { + var source = try DB.open(alloc, path, options); + defer source.close(); + try source.core.store.put(key, artifact); + } + const request: vector_migration.contract.Request = .{ .job_id = "crash", .mode = .offline, .budget = .{ .batch_bytes = 4096, .disk_reserve_bytes = 0 } }; + Hook.selected = point; + Hook.fired = false; + offline.test_boundary = Hook.inject; + defer offline.test_boundary = null; + try std.testing.expectError(error.InjectedMigrationCrash, offline.run(alloc, std.testing.io, path, request, .{ .open = options })); + try std.testing.expect(Hook.fired); + offline.test_boundary = null; + try std.testing.expectEqual(.complete, try offline.run(alloc, std.testing.io, path, request, .{ .open = options })); + try std.testing.expectEqual(.complete, try offline.run(alloc, std.testing.io, path, request, .{ .open = options })); + try std.testing.expectError(error.VectorMigrationAlreadyPublished, offline.cancel(alloc, std.testing.io, path, request, options)); + var target = try DB.open(alloc, path, options); + defer target.close(); + try std.testing.expectEqual(.vector_store, target.table_storage.dense_embeddings); + const actual = try target.core.store.get(alloc, key); + defer alloc.free(actual); + try std.testing.expectEqualSlices(u8, artifact, actual); + } +} + +test "source vector migration catalog fences configurations topology and stale publication" { + const catalog = @import("../../metadata/table_manager.zig"); + var manager = catalog.TableManager.init(std.testing.allocator); + defer manager.deinit(); + const before: catalog.TableRecord = .{ .table_id = 10, .name = "migrate" }; + const range: catalog.RangeRecord = .{ .group_id = 101, .table_id = 10, .start_key = "", .end_key = null }; + try manager.upsertTable(before); + try manager.upsertRange(range); + var admitted = before; + admitted.storage_migration = .{ .request = .{ .job_id = "online", .mode = .online } }; + try manager.publishVectorMigrationTable(before, admitted); + try std.testing.expect(!std.mem.eql(u8, &catalog.tableDefinitionFingerprint(before), &catalog.tableDefinitionFingerprint(admitted))); + try manager.upsertTable(admitted); + try manager.upsertRange(range); // Normalized range ID is still idempotent. + // Restart/projected-catalog installation restores existing admission, + // while incremental topology changes remain fenced after reload. + try manager.replaceTopology(&.{admitted}, &.{range}); + _ = try manager.replaceProjectedTopology(&.{admitted}, &.{range}); + try manager.upsertRange(range); + var moved = range; + moved.start_key = "m"; + try std.testing.expectError(error.VectorMigrationActive, manager.upsertRange(moved)); + try std.testing.expectError(error.VectorMigrationActive, manager.upsertTable(before)); + var edited = admitted; + edited.schema_json = "{\"version\":2}"; + try std.testing.expectError(error.VectorMigrationActive, manager.upsertTable(edited)); + try std.testing.expectError(error.VectorMigrationConfigurationChanged, manager.publishVectorMigrationTable(admitted, edited)); + try std.testing.expectError(error.VectorMigrationActive, manager.requestSplit(.{ .transition_id = 1, .table_id = 10, .source_group_id = 101, .destination_group_id = 102, .split_key = "m" })); + var published = admitted; + published.storage.dense_embeddings = .vector_store; + try manager.publishVectorMigrationTable(admitted, published); + try std.testing.expectError(error.TableGenerationChanged, manager.publishVectorMigrationTable(admitted, before)); + var complete = published; + complete.storage_migration = null; + try manager.publishVectorMigrationTable(published, complete); + try std.testing.expectError(error.UnsupportedVectorMigrationDirection, manager.publishVectorMigrationTable(complete, before)); +} + +test "source vector migration offline cancellation retains an idempotency receipt" { + const offline = @import("../vector_migration_offline.zig"); + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("offline-cancel"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const options: OpenOptions = .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }; + { + var source = try DB.open(alloc, path, options); + defer source.close(); + try source.core.store.put("preserve", "original"); + } + const request: vector_migration.contract.Request = .{ .job_id = "cancel", .mode = .offline, .budget = .{ .batch_bytes = 4096, .disk_reserve_bytes = 0 } }; + try std.testing.expectEqual(.pending, try offline.run(alloc, std.testing.io, path, request, .{ .open = options, .max_steps = 1 })); + try offline.cancel(alloc, std.testing.io, path, request, options); + try offline.cancel(alloc, std.testing.io, path, request, options); + try std.testing.expectError(error.VectorMigrationCancelled, offline.run(alloc, std.testing.io, path, request, .{ .open = options })); + var source = try DB.open(alloc, path, options); + defer source.close(); + try std.testing.expectEqual(.primary_lsm, source.table_storage.dense_embeddings); + const actual = try source.core.store.get(alloc, "preserve"); + defer alloc.free(actual); + try std.testing.expectEqualStrings("original", actual); +} + +test "source vector migration offline cancellation before the copy fence survives retry" { + const offline = @import("../vector_migration_offline.zig"); + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("offline-cancel-admission"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + const options: OpenOptions = .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }; + { + var source = try DB.open(alloc, path, options); + defer source.close(); + try source.core.store.put("preserve", "original"); + } + const request: vector_migration.contract.Request = .{ .job_id = "cancel-before-fence", .mode = .offline, .budget = .{ .disk_reserve_bytes = std.math.maxInt(u64) } }; + try std.testing.expectError(error.VectorMigrationDiskReserve, offline.run(alloc, std.testing.io, path, request, .{ .open = options })); + const fence = try std.fs.path.join(alloc, &.{ path, vector_migration.contract.offline_fence_file }); + defer alloc.free(fence); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.cwd().access(std.testing.io, fence, .{})); + // Reopening between these calls models losing the successful cancellation + // response before the operator can clear the persisted catalog marker. + try offline.cancel(alloc, std.testing.io, path, request, options); + try offline.cancel(alloc, std.testing.io, path, request, options); + try std.testing.expectError(error.VectorMigrationCancelled, offline.run(alloc, std.testing.io, path, request, .{ .open = options })); + var conflict = request; + conflict.budget.disk_reserve_bytes = 0; + try std.testing.expectError(error.VectorMigrationIdempotencyConflict, offline.cancel(alloc, std.testing.io, path, conflict, options)); + try std.testing.expectError(error.VectorMigrationIdempotencyConflict, offline.run(alloc, std.testing.io, path, conflict, .{ .open = options })); + conflict.job_id = "replacement"; + try std.testing.expectEqual(.pending, try offline.run(alloc, std.testing.io, path, conflict, .{ .open = options, .max_steps = 1 })); + try std.testing.expectError(error.VectorMigrationIdempotencyConflict, offline.cancel(alloc, std.testing.io, path, request, options)); + try offline.cancel(alloc, std.testing.io, path, conflict, options); + var source = try DB.open(alloc, path, options); + defer source.close(); + try std.testing.expectEqual(.primary_lsm, source.table_storage.dense_embeddings); + const actual = try source.core.store.get(alloc, "preserve"); + defer alloc.free(actual); + try std.testing.expectEqualStrings("original", actual); +} + +test "source vector migration fences live probes admitted before activation" { + const alloc = std.testing.allocator; + var tmp = try TestDirectory.init("migration-live-probe"); + defer tmp.cleanup(); + var db = try DB.open(alloc, std.mem.span(tmp.path().ptr), .{ .table_storage = .{ .dense_embeddings = .primary_lsm }, .start_index_workers = false, .start_optional_runtimes = false }); + defer db.close(); + const key = try internal_keys.embeddingArtifactKeyForDocumentAlloc(alloc, "doc", "model"); + defer alloc.free(key); + const artifact = try enrichment_artifact_codec.encodeDenseEmbeddingAlloc(alloc, 4, &.{ 1, 2 }); + defer alloc.free(artifact); + try db.core.store.put(key, artifact); + var probe = try db.core.store.beginProbeTxn(); + defer probe.abort(); + var snapshot = try db.core.store.beginReadTxn(); + defer snapshot.abort(); + const request: vector_migration.contract.Request = .{ .job_id = "probe", .mode = .online }; + try db.startVectorMigration(request); + for (0..128) |_| { + var state = (try vector_migration.load(alloc, db.core.store)).?; + defer state.deinit(); + if (state.value.phase == .reclaiming and state.value.primary_reclamation_requested) break; + if (state.value.phase == .ready) try db.publishVectorMigration(request.job_id) else try db.advanceVectorMigration(request.job_id); + } else return error.VectorMigrationDidNotFinish; + // Reproduce the timed GC retry that a tight Linux test loop can outrun. + const backend = db.core.primary_store_owner.lsmBackend().?; + backend.tombstone_gc_retry_after_ns = backend.nowNs() + 250 * std.time.ns_per_ms; + try completeVectorMigrationForTest(&db, request.job_id); + try std.testing.expectError(error.VectorMigrationReadEpochChanged, probe.get(key)); + try std.testing.expectError(error.VectorMigrationReadEpochChanged, probe.getLeased(key)); + var values: [1]?[]const u8 = undefined; + try std.testing.expectError(error.VectorMigrationReadEpochChanged, probe.getManySorted(&.{key}, &values)); + try std.testing.expectError(error.VectorMigrationReadEpochChanged, probe.getManySortedTransient(&.{key}, &values)); + try std.testing.expectEqualSlices(u8, artifact, try snapshot.get(key)); + var current = try db.core.store.beginProbeTxn(); + defer current.abort(); + try std.testing.expectEqualSlices(u8, artifact, try current.get(key)); +} + +test "source vector migration converts legacy ANN generations in both modes" { + const offline = @import("../vector_migration_offline.zig"); + const Gate = struct { + permitted: bool = false, + fn read(ptr: *const anyopaque) bool { + return (@as(*const @This(), @ptrCast(@alignCast(ptr)))).permitted; + } + }; + const alloc = std.testing.allocator; + inline for (std.meta.tags(vector_migration.contract.Mode)) |mode| { + var tmp = try TestDirectory.init("migration-legacy-ann"); + defer tmp.cleanup(); + const path = std.mem.span(tmp.path().ptr); + var gate = Gate{}; + const options: OpenOptions = .{ + .table_storage = .{ .dense_embeddings = .primary_lsm }, + .start_index_workers = false, + .start_optional_runtimes = false, + .ttl_cleanup = .{ .enabled = false }, + .index_backends = .{ .dense_native_migration_policy_source = .{ .ptr = &gate, .authority_permitted = Gate.read } }, + }; + const request: vector_migration.contract.Request = .{ .job_id = "legacy", .mode = mode, .budget = .{ .disk_reserve_bytes = 0 } }; + { + var source = try DB.open(alloc, path, options); + defer source.close(); + try source.addIndex(.{ .name = "model", .kind = .dense_vector, .config_json = "{\"field\":\"embedding\",\"dims\":3,\"metric\":\"l2_squared\",\"external\":true}" }); + try source.batch(.{ .writes = &.{.{ .key = "a", .value = "{\"title\":\"alpha\",\"_embeddings\":{\"model\":[1,0,0]}}" }}, .sync_level = .full_index }); + try std.testing.expect(!source.core.index_manager.denseIndex("model").?.native_physical_v2); + if (mode == .online) { + try std.testing.expectError(error.VectorStoreLifecycleUnsupported, source.startVectorMigration(request)); + gate.permitted = true; + try source.startVectorMigration(request); + for (0..256) |_| { + var state = (try vector_migration.load(alloc, source.core.store)).?; + defer state.deinit(); + var result = try source.search(alloc, .{ .index_name = "model", .dense = .{ .vector = &.{ 1, 0, 0 }, .k = 1 }, .limit = 1 }); + defer result.deinit(); + try std.testing.expectEqualStrings("a", result.hits[0].id); + if (state.value.phase == .complete) break; + if (state.value.phase == .ready) try source.publishVectorMigration(request.job_id) else try source.advanceVectorMigration(request.job_id); + } else return error.VectorMigrationDidNotFinish; + } + } + gate.permitted = true; + if (mode == .offline) try std.testing.expectEqual(.complete, try offline.run(alloc, std.testing.io, path, request, .{ .open = options })); + var migrated = try DB.open(alloc, path, options); + defer migrated.close(); + try std.testing.expectEqual(.vector_store, migrated.table_storage.dense_embeddings); + try std.testing.expect(migrated.core.index_manager.denseIndex("model").?.native_physical_v2); + try std.testing.expect(migrated.core.index_manager.sourceMigrationServingComplete()); + var result = try migrated.search(alloc, .{ .index_name = "model", .dense = .{ .vector = &.{ 1, 0, 0 }, .k = 1 }, .limit = 1 }); + defer result.deinit(); + try std.testing.expectEqualStrings("a", result.hits[0].id); + } +} diff --git a/zig/pkg/antfly/src/storage/db/generation_lifecycle.zig b/zig/pkg/antfly/src/storage/db/generation_lifecycle.zig index 79396049d7..7d0384c5a7 100644 --- a/zig/pkg/antfly/src/storage/db/generation_lifecycle.zig +++ b/zig/pkg/antfly/src/storage/db/generation_lifecycle.zig @@ -800,6 +800,40 @@ pub const ExclusiveTransition = struct { _ = try reconcilePublishedGenerationExclusive(self.alloc, io_impl.io(), self.path, self.cleanup_scheduler); } + /// Stable, explicitly retained candidate for resumable offline conversion. + /// The caller validates its durable job before opening any existing stage. + pub fn resumeStaging(self: *ExclusiveTransition, job_id: []const u8) !StagedGeneration { + try self.validate(self.path); + try (@import("../../common/vector_migration.zig").Request{ .job_id = job_id, .mode = .offline }).validate(); + const io = self.io orelse return error.MissingBackendRuntimeIo; + try self.reconcilePublished(); + const live = try self.alloc.dupe(u8, self.path); + errdefer self.alloc.free(live); + const live_z = try self.alloc.dupeZ(u8, self.path); + errdefer self.alloc.free(live_z); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(job_id, &digest, .{}); + const stage = try std.fmt.allocPrint(self.alloc, "{s}.restore-stage-{x}-{x}", .{ + self.path, std.mem.readInt(u64, digest[0..8], .little), std.mem.readInt(u64, digest[8..16], .little), + }); + errdefer self.alloc.free(stage); + const stage_z = try self.alloc.dupeZ(u8, stage); + errdefer self.alloc.free(stage_z); + try fs_paths.createDirPathPortable(io, stage); + return .{ + .alloc = self.alloc, + .manager = self.manager, + .transition_id = self.id, + .live_path = live, + .live_path_z = live_z, + .staging_path = stage, + .staging_path_z = stage_z, + .cleanup_scheduler = self.cleanup_scheduler, + .io = io, + .preserve_unpublished = true, + }; + } + pub fn beginStaging(self: *ExclusiveTransition) !StagedGeneration { try self.validate(self.path); return try beginStagingGeneration(self.alloc, self.manager, self.path, self.id, self.cleanup_scheduler, self.io, true); @@ -905,6 +939,7 @@ pub const StagedGeneration = struct { publication_outcome: ?PublicationOutcome = null, sealed: bool = false, preserve_retired: bool = false, + preserve_unpublished: bool = false, cleanup_scheduler: ?CleanupScheduler = null, /// Runtime-owned I/O carried by runtime-backed transitions. Legacy direct /// callers leave this null and retain the historical local-I/O fallback. @@ -1137,7 +1172,7 @@ pub const StagedGeneration = struct { } if (self.published) { if (!self.preserve_retired) std.Io.Dir.cwd().deleteTree(io, self.staging_path) catch {}; - } else { + } else if (!self.preserve_unpublished) { std.Io.Dir.cwd().deleteTree(io, self.staging_path) catch {}; } self.alloc.free(self.staging_path); diff --git a/zig/pkg/antfly/src/storage/db/types.zig b/zig/pkg/antfly/src/storage/db/types.zig index f753ead4db..86b8568681 100644 --- a/zig/pkg/antfly/src/storage/db/types.zig +++ b/zig/pkg/antfly/src/storage/db/types.zig @@ -4127,6 +4127,9 @@ pub fn freeAlgebraicAdaptiveProgress(alloc: Allocator, progress: []AlgebraicAdap } pub const HbcPostingStats = struct { + /// False only after a clean bounded sweep at the current mutation epoch. + /// Defaults conservatively when a runtime observation is unavailable. + refresh_pending: bool = true, scanned_nodes: u64 = 0, scanned_postings: u64 = 0, dirty_postings: u64 = 0, diff --git a/zig/pkg/antfly/src/storage/docstore.zig b/zig/pkg/antfly/src/storage/docstore.zig index d6af55c97c..ce83654ffa 100644 --- a/zig/pkg/antfly/src/storage/docstore.zig +++ b/zig/pkg/antfly/src/storage/docstore.zig @@ -377,6 +377,10 @@ fn columnarMutationToken(txn: anytype, cached: *?internal_keys.ColumnarMutationT pub const DocStore = struct { payload_store: ?artifact_payload.Store = null, + payload_capture_inline: bool = false, + payload_migration_allowance: ?u64 = null, + payload_policy_mutex: std.atomic.Mutex = .unlocked, + payload_recovery_required: std.atomic.Value(bool) = .init(false), alloc: Allocator, /// Process-local wake hint, published only after successful row/schema /// commits. Durable mutation IDs and timers remain the restart authority. @@ -511,7 +515,13 @@ pub const DocStore = struct { pub fn get(self: *Txn, key: []const u8) ![]const u8 { const value = try self.getPhysical(key); - return if (self.payload_session) |session| try session.get(key, value) else value; + if (self.payload_session) |session| return try session.get(key, value); + // A live probe admitted before migration may observe a later + // reference. Retry with a new source lease; never leak its encoding + // or acquire a lease after reading a potentially retired reference. + if (artifact_payload.isEmbeddingKey(key) and artifact_payload.isReference(value)) + return error.VectorMigrationReadEpochChanged; + return value; } pub fn getArtifactMetadata(self: *Txn, key: []const u8) !artifact_payload.Metadata { @@ -541,7 +551,13 @@ pub const DocStore = struct { /// Short-lived value lease, released when this transaction aborts. /// Immutable LSM bytes may be pinned rather than copied. pub fn getLeased(self: *Txn, key: []const u8) ![]const u8 { - if (self.probe) |*probe| return try probe.getLeased(key); + if (self.probe) |*probe| { + const value = try probe.getLeased(key); + if (self.payload_session) |session| return try session.get(key, value); + if (artifact_payload.isEmbeddingKey(key) and artifact_payload.isReference(value)) + return error.VectorMigrationReadEpochChanged; + return value; + } return try self.get(key); } @@ -569,7 +585,10 @@ pub const DocStore = struct { for (keys, values) |key, *value| if (value.*) |raw| { value.* = try session.get(key, raw); }; - } + } else for (keys, values) |key, value| if (value) |raw| { + if (artifact_payload.isEmbeddingKey(key) and artifact_payload.isReference(raw)) + return error.VectorMigrationReadEpochChanged; + }; } pub fn getManySortedPhysical(self: *Txn, keys: []const []const u8, values: []?[]const u8) !void { @@ -612,7 +631,10 @@ pub const DocStore = struct { for (keys, values) |key, *value| if (value.*) |raw| { value.* = try session.get(key, raw); }; - } + } else for (keys, values) |key, value| if (value) |raw| { + if (artifact_payload.isEmbeddingKey(key) and artifact_payload.isReference(raw)) + return error.VectorMigrationReadEpochChanged; + }; return; } return try self.getManySorted(keys, values); @@ -645,7 +667,7 @@ pub const DocStore = struct { } } const stored = if (self.payload_session) |session| try session.put(key, value) else value; - try self.write.?.put(key, stored); + try self.write.?.put(key, if (self.payload_session) |session| session.primaryValue(value, stored) else stored); if (self.payload_session) |session| try session.recordOwnership(&self.write.?, key, stored); } @@ -701,7 +723,7 @@ pub const DocStore = struct { fn openCursorAdapter(self: *Txn) !CursorAdapter { var cursor_adapter = try self.openPhysicalCursorAdapter(); errdefer cursor_adapter.close(); - return try wrapPayloadCursor(self.alloc, cursor_adapter, self.payload_session); + return try wrapPayloadCursor(self.alloc, cursor_adapter, self.payload_session, self.current_scan != null or self.probe != null); } pub fn openPhysicalCursorAdapter(self: *Txn) !CursorAdapter { @@ -761,7 +783,13 @@ pub const DocStore = struct { if (self.raw) |raw| return try raw.get(self.dbi, key); } const value = try self.runtime.?.get(key); - return if (self.payload_session) |session| try session.get(key, value) else value; + if (self.payload_session) |session| return try session.get(key, value); + // A live probe admitted before migration may observe a later + // reference. Retry with a new source lease; never leak its encoding + // or acquire a lease after reading a potentially retired reference. + if (artifact_payload.isEmbeddingKey(key) and artifact_payload.isReference(value)) + return error.VectorMigrationReadEpochChanged; + return value; } pub fn getManySorted(self: @This(), keys: []const []const u8, values: []?[]const u8) !void { @@ -796,7 +824,7 @@ pub const DocStore = struct { } } const stored = if (self.payload_session) |session| try session.put(key, value) else value; - try self.runtime.?.put(key, stored); + try self.runtime.?.put(key, if (self.payload_session) |session| session.primaryValue(value, stored) else stored); if (self.payload_session) |session| try session.recordOwnership(self.runtime.?, key, stored); } @@ -817,7 +845,7 @@ pub const DocStore = struct { if (self.raw != null) return error.Unsupported; } const stored = if (self.payload_session) |session| try session.put(key, value) else value; - try self.runtime.?.appendPut(key, stored); + try self.runtime.?.appendPut(key, if (self.payload_session) |session| session.primaryValue(value, stored) else stored); if (self.payload_session) |session| try session.recordOwnership(self.runtime.?, key, stored); } @@ -862,7 +890,7 @@ pub const DocStore = struct { } var physical = try self.runtime.?.openCursor(); errdefer physical.close(); - return try wrapPayloadCursor(self.alloc, physical, self.payload_session); + return try wrapPayloadCursor(self.alloc, physical, self.payload_session, false); } pub fn setReplayOpaque(self: @This(), sequence: u64, payload: []const u8) !void { @@ -1155,18 +1183,21 @@ pub const DocStore = struct { const PayloadCursor = struct { physical: backend_erased.Cursor, - session: *artifact_payload.Session, + session: ?*artifact_payload.Session, arena: std.heap.ArenaAllocator, pub fn close(self: *@This()) void { self.physical.close(); self.arena.deinit(); - self.session.release(); + if (self.session) |session| session.release(); } fn resolve(self: *@This(), entry: ?backend_erased.Entry) !?backend_erased.Entry { _ = self.arena.reset(.retain_capacity); const value = entry orelse return null; - return .{ .key = value.key, .value = try self.session.getAlloc(self.arena.allocator(), value.key, value.value) }; + if (self.session) |session| return .{ .key = value.key, .value = try session.getAlloc(self.arena.allocator(), value.key, value.value) }; + if (artifact_payload.isEmbeddingKey(value.key) and artifact_payload.isReference(value.value)) + return error.VectorMigrationReadEpochChanged; + return value; } pub fn first(self: *@This()) !?backend_erased.Entry { return self.resolve(try self.physical.first()); @@ -1191,17 +1222,41 @@ pub const DocStore = struct { } }; - fn wrapPayloadCursor(alloc: Allocator, physical: backend_erased.Cursor, session: ?*artifact_payload.Session) !backend_erased.Cursor { - const owner = session orelse return physical; - owner.retain(); - errdefer owner.release(); + fn wrapPayloadCursor(alloc: Allocator, physical: backend_erased.Cursor, session: ?*artifact_payload.Session, live: bool) !backend_erased.Cursor { + if (session == null and !live) return physical; + if (session) |owner| owner.retain(); + errdefer if (session) |owner| owner.release(); return try backend_erased.cursorFrom(alloc, PayloadCursor{ .physical = physical, - .session = owner, + .session = session, .arena = std.heap.ArenaAllocator.init(alloc), }); } + fn lockPayloadPolicy(self: *DocStore) void { + while (!self.payload_policy_mutex.tryLock()) std.atomic.spinLoopHint(); + } + + /// DB apply admission excludes writers. Reader admission holds this mutex + /// until its primary view and source lease have both been captured. + pub fn configurePayloadPolicy(self: *DocStore, store: ?artifact_payload.Store, capture_inline: bool, migration_allowance: ?u64) void { + self.lockPayloadPolicy(); + defer self.payload_policy_mutex.unlock(); + self.payload_store = store; + self.payload_capture_inline = capture_inline; + self.payload_migration_allowance = migration_allowance; + } + + fn createPayloadSession(self: *DocStore) !?*artifact_payload.Session { + if (self.payload_recovery_required.load(.acquire)) return error.VectorMigrationRecoveryRequired; + if (self.kind != .runtime) return null; + const store = self.payload_store orelse return null; + const session = try artifact_payload.Session.create(self.alloc, store); + session.capture_inline = self.payload_capture_inline; + session.migration_allowance = self.payload_migration_allowance; + return session; + } + pub fn beginReadTxn(self: *DocStore) !Txn { return try self.beginReadTxnWithBlockCacheAdmission(.retain); } @@ -1233,7 +1288,9 @@ pub const DocStore = struct { self: *DocStore, admission: backend_types.Namespace.BlockCacheAdmission, ) !Txn { - const payload_session = if (if (self.kind == .runtime) self.payload_store else null) |store| try artifact_payload.Session.create(self.alloc, store) else null; + if (self.kind == .runtime) self.lockPayloadPolicy(); + defer if (self.kind == .runtime) self.payload_policy_mutex.unlock(); + const payload_session = try self.createPayloadSession(); errdefer if (payload_session) |session| session.release(); return switch (self.kind) { .lmdb => if (supports_lmdb) blk: { @@ -1288,7 +1345,9 @@ pub const DocStore = struct { ) !Txn { try self.acquirePortableImportReader(); errdefer self.releasePortableImportReader(); - const payload_session = if (if (self.kind == .runtime) self.payload_store else null) |store| try artifact_payload.Session.create(self.alloc, store) else null; + if (self.kind == .runtime) self.lockPayloadPolicy(); + defer if (self.kind == .runtime) self.payload_policy_mutex.unlock(); + const payload_session = try self.createPayloadSession(); errdefer if (payload_session) |session| session.release(); var txn: Txn = switch (self.kind) { // The outer probe transaction already owns the portable-import @@ -1313,7 +1372,9 @@ pub const DocStore = struct { pub fn beginCurrentScanTxn(self: *DocStore) !Txn { try self.acquirePortableImportReader(); errdefer self.releasePortableImportReader(); - const payload_session = if (if (self.kind == .runtime) self.payload_store else null) |store| try artifact_payload.Session.create(self.alloc, store) else null; + if (self.kind == .runtime) self.lockPayloadPolicy(); + defer if (self.kind == .runtime) self.payload_policy_mutex.unlock(); + const payload_session = try self.createPayloadSession(); errdefer if (payload_session) |session| session.release(); var txn: Txn = switch (self.kind) { .lmdb => try self.beginReadTxnUnchecked(), @@ -1334,7 +1395,9 @@ pub const DocStore = struct { if (!(try self.hasReplayEntries())) return error.ReplayIndexUnavailable; try self.acquirePortableImportReader(); errdefer self.releasePortableImportReader(); - const payload_session = if (if (self.kind == .runtime) self.payload_store else null) |store| try artifact_payload.Session.create(self.alloc, store) else null; + if (self.kind == .runtime) self.lockPayloadPolicy(); + defer if (self.kind == .runtime) self.payload_policy_mutex.unlock(); + const payload_session = try self.createPayloadSession(); errdefer if (payload_session) |session| session.release(); var txn: Txn = switch (self.kind) { .lmdb => try self.beginReadTxnUnchecked(), @@ -1350,7 +1413,9 @@ pub const DocStore = struct { pub fn beginWriteTxn(self: *DocStore) !Txn { try self.ensurePortableImportOperational(); - const payload_session = if (if (self.kind == .runtime) self.payload_store else null) |store| try artifact_payload.Session.create(self.alloc, store) else null; + if (self.kind == .runtime) self.lockPayloadPolicy(); + defer if (self.kind == .runtime) self.payload_policy_mutex.unlock(); + const payload_session = try self.createPayloadSession(); errdefer if (payload_session) |session| session.release(); return switch (self.kind) { .lmdb => if (supports_lmdb) blk: { @@ -1379,7 +1444,9 @@ pub const DocStore = struct { pub fn beginWriteBatchWithOptions(self: *DocStore, options: backend_types.BatchOptions) !Batch { try self.ensurePortableImportOperational(); - const payload_session = if (if (self.kind == .runtime) self.payload_store else null) |store| try artifact_payload.Session.create(self.alloc, store) else null; + if (self.kind == .runtime) self.lockPayloadPolicy(); + defer if (self.kind == .runtime) self.payload_policy_mutex.unlock(); + const payload_session = try self.createPayloadSession(); errdefer if (payload_session) |session| session.release(); return switch (self.kind) { .lmdb => if (supports_lmdb) blk: { diff --git a/zig/pkg/antfly/src/storage/hbc_adapter.zig b/zig/pkg/antfly/src/storage/hbc_adapter.zig index 3b6f20b52b..4c45b74f3f 100644 --- a/zig/pkg/antfly/src/storage/hbc_adapter.zig +++ b/zig/pkg/antfly/src/storage/hbc_adapter.zig @@ -5319,7 +5319,10 @@ pub const HBCIndex = struct { // deliberately volatile: reopen verifies the durable postings again. posting_refresh_next_node: u64 = 1, posting_refresh_observed_epoch: ?u64 = null, - posting_refresh_clean_epoch: ?u64 = null, + // Odd epochs are never clean. This atomic certificate lets operational + // status observe bounded maintenance without traversing the tree or + // racing the mutation owner's scan cursor. Reopen starts uncertified. + posting_refresh_clean_epoch: std.atomic.Value(u64) = .init(std.math.maxInt(u64)), posting_refresh_scan_changed: bool = false, /// Publication commits may include durable I/O. Readers of an odd /// generation retain the active flight and sleep on its runtime event @@ -19443,6 +19446,15 @@ pub const HBCIndex = struct { return true; } + /// A clean sweep certifies one committed mutation epoch. Subsequent + /// writes (including aborts) invalidate it without touching scan state. + /// This is an optimization/convergence signal, not query readiness. + pub fn postingRefreshPending(self: *const HBCIndex) bool { + const clean = self.posting_refresh_clean_epoch.load(.acquire); + const current = self.published_mutation_epoch.load(.acquire); + return current & 1 != 0 or clean != current; + } + pub fn refreshPostingPayloadPage(self: *HBCIndex, max_nodes: usize, max_postings: usize) !PostingRefreshProgress { return try self.refreshPostingPayloadPageWithOptions(max_nodes, max_postings, false, true); } @@ -19453,7 +19465,7 @@ pub const HBCIndex = struct { pub fn refreshPostingPayloadPageWithOptions(self: *HBCIndex, max_nodes: usize, max_postings: usize, allow_query_traffic: bool, allow_mutation: bool) !PostingRefreshProgress { var context: PostingRefreshContext = .{ .index = self, .allow_query_traffic = allow_query_traffic }; const epoch = self.published_mutation_epoch.load(.acquire); - if (self.posting_refresh_clean_epoch == epoch) return .{}; + if (!self.postingRefreshPending()) return .{}; if (!PostingRefreshContext.shouldContinue(&context)) return .{ .pending = true }; if (self.posting_refresh_observed_epoch) |observed| { if (observed != epoch and self.posting_refresh_next_node != 1) self.posting_refresh_scan_changed = true; @@ -19510,7 +19522,7 @@ pub const HBCIndex = struct { var pending = true; if (result.next_node == 0 and !result.limit_reached) { pending = self.posting_refresh_scan_changed; - if (!pending) self.posting_refresh_clean_epoch = self.posting_refresh_observed_epoch; + if (!pending) self.posting_refresh_clean_epoch.store(self.posting_refresh_observed_epoch.?, .release); self.posting_refresh_scan_changed = false; } return .{ @@ -31153,9 +31165,11 @@ test "posting refresh resumes bounded scans and rechecks mutations behind cursor _ = try idx.repairDirtyPostings(); try std.testing.expect(idx.metadata.node_count > 4); + try std.testing.expect(idx.postingRefreshPending()); const first = try idx.refreshPostingPayloadPage(2, 1); try std.testing.expectEqual(@as(usize, 2), first.scanned); try std.testing.expect(first.pending); + try std.testing.expect(idx.postingRefreshPending()); try std.testing.expectEqual(@as(u64, 3), idx.posting_refresh_next_node); // This includes postings already visited. The following sweep must not // certify the old clean prefix after a mutation between pages. @@ -31173,15 +31187,50 @@ test "posting refresh resumes bounded scans and rechecks mutations behind cursor } } try std.testing.expect(settled); + try std.testing.expect(!idx.postingRefreshPending()); + // Even an aborted write invalidates the previously published certificate. + var aborted = try idx.beginWriteTxn(); + try std.testing.expect(idx.postingRefreshPending()); + aborted.abort(); + try std.testing.expect(idx.postingRefreshPending()); + for (0..512) |_| { + if (!(try idx.refreshPostingPayloadPage(2, 1)).pending) break; + } + try std.testing.expect(!idx.postingRefreshPending()); try std.testing.expect(total_repaired > 0); try std.testing.expectEqual(@as(u64, 0), (try idx.postingBacklogStats()).dirty_postings); const idle = try idx.refreshPostingPayloadPage(2, 1); try std.testing.expectEqual(@as(usize, 0), idle.scanned); try std.testing.expect(!idle.pending); _ = try idx.markAllLeafPostingsDirtyForTest(); + try std.testing.expect(idx.postingRefreshPending()); try std.testing.expect((try idx.refreshPostingPayloadPage(2, 1)).pending); } +test "posting refresh certificate must be reverified after reopen" { + const alloc = std.testing.allocator; + var tp: TestPath = .{}; + const path = tp.init(); + defer tp.cleanup(); + const config: HBCConfig = .{ .dims = 2, .use_quantization = false }; + { + var idx = try HBCIndex.open(alloc, path, config); + defer idx.close(); + try idx.insert(1, &.{ 1.0, 0.0 }); + for (0..8) |_| { + if (!(try idx.refreshPostingPayloadPage(1, 1)).pending) break; + } + try std.testing.expect(!idx.postingRefreshPending()); + } + var reopened = try HBCIndex.open(alloc, path, config); + defer reopened.close(); + try std.testing.expect(reopened.postingRefreshPending()); + for (0..8) |_| { + if (!(try reopened.refreshPostingPayloadPage(1, 1)).pending) break; + } + try std.testing.expect(!reopened.postingRefreshPending()); +} + test "posting refresh deferral retains cursor and pending debt" { const alloc = std.testing.allocator; var tp: TestPath = .{}; diff --git a/zig/pkg/antfly/src/storage/hot_standby/mutation_inventory.json b/zig/pkg/antfly/src/storage/hot_standby/mutation_inventory.json index 098e5d61f4..3673d9c532 100644 --- a/zig/pkg/antfly/src/storage/hot_standby/mutation_inventory.json +++ b/zig/pkg/antfly/src/storage/hot_standby/mutation_inventory.json @@ -16,6 +16,8 @@ {"surface":"cluster_restore","disposition":"reject","path_pattern":"/restore","methods":["POST"],"reason":"restore activation replaces local generation state outside the continuous stream"}, {"surface":"table_restore","disposition":"reject","path_pattern":"/tables/{table}/restore","methods":["POST"],"reason":"table restore mutates both catalog and data outside one RemoteApply acknowledgement"}, {"surface":"transaction_session","disposition":"reject","path_pattern":"/transactions[/... mutating operation]","methods":["POST","PUT","DELETE"],"reason":"durable transaction session state and savepoints are primary-local"}, + {"surface":"storage_migration","disposition":"reject","path_pattern":"/tables/{table}/storage/migrations","methods":["POST","DELETE"],"reason":"source ownership migration is qualified only for unreplicated local tables"}, + {"surface":"storage_migration","disposition":"reject","path_pattern":"/tables/{table}/storage/migrations/{job}","methods":["POST","DELETE"],"reason":"source ownership migration is qualified only for unreplicated local tables"}, {"surface":"artifact_repair","disposition":"reject","path_pattern":"/tables/{table}/repair/{run|control-jobs|jobs/...}","methods":["POST","DELETE"],"reason":"repair job checkpoints and direct repair effects do not share one replicated acknowledgement"}, {"surface":"artifact_reprocess","disposition":"reject","path_pattern":"/tables/{table}/.../reprocess[-jobs]","methods":["POST","DELETE"],"reason":"reprocess job checkpoints and derived effects do not share one replicated acknowledgement"}, {"surface":"backup","disposition":"reject","path_pattern":"/backup | /tables/{table}/backup","methods":["POST"],"reason":"backup publication has an external side effect but no final HA authority recheck spanning snapshot and manifest publication"}, diff --git a/zig/pkg/antfly/src/storage/hot_standby/mutation_inventory.zig b/zig/pkg/antfly/src/storage/hot_standby/mutation_inventory.zig index 4d6fe13a3e..15fb66eb15 100644 --- a/zig/pkg/antfly/src/storage/hot_standby/mutation_inventory.zig +++ b/zig/pkg/antfly/src/storage/hot_standby/mutation_inventory.zig @@ -51,6 +51,7 @@ pub const Surface = enum { table_restore, transaction_session, artifact_repair, + storage_migration, artifact_reprocess, backup, read_like_post, @@ -104,6 +105,8 @@ pub const entries = [_]Entry{ .{ .surface = .cluster_restore, .disposition = .reject, .path_pattern = "/restore", .methods = post, .reason = "restore activation replaces local generation state outside the continuous stream" }, .{ .surface = .table_restore, .disposition = .reject, .path_pattern = "/tables/{table}/restore", .methods = post, .reason = "table restore mutates both catalog and data outside one RemoteApply acknowledgement" }, .{ .surface = .transaction_session, .disposition = .reject, .path_pattern = "/transactions[/... mutating operation]", .methods = post_put_delete, .reason = "durable transaction session state and savepoints are primary-local" }, + .{ .surface = .storage_migration, .disposition = .reject, .path_pattern = "/tables/{table}/storage/migrations", .methods = post_delete, .reason = "source ownership migration is qualified only for unreplicated local tables" }, + .{ .surface = .storage_migration, .disposition = .reject, .path_pattern = "/tables/{table}/storage/migrations/{job}", .methods = post_delete, .reason = "source ownership migration is qualified only for unreplicated local tables" }, .{ .surface = .artifact_repair, .disposition = .reject, .path_pattern = "/tables/{table}/repair/{run|control-jobs|jobs/...}", .methods = post_delete, .reason = "repair job checkpoints and direct repair effects do not share one replicated acknowledgement" }, .{ .surface = .artifact_reprocess, .disposition = .reject, .path_pattern = "/tables/{table}/.../reprocess[-jobs]", .methods = post_delete, .reason = "reprocess job checkpoints and derived effects do not share one replicated acknowledgement" }, .{ .surface = .backup, .disposition = .reject, .path_pattern = "/backup | /tables/{table}/backup", .methods = post, .reason = "backup publication has an external side effect but no final HA authority recheck spanning snapshot and manifest publication" }, @@ -189,6 +192,7 @@ pub fn classify(method: http_common.Method, path: []const u8) ?Classification { std.mem.startsWith(u8, path, routes.Routes.transactions_prefix)) return rejected(.transaction_session); + if (routes.Routes.matchTableStorageMigration(path) != null or routes.Routes.matchTableStorageMigrationJob(path) != null) return rejected(.storage_migration); if (routes.Routes.matchTableArtifactRepairRun(path) != null or routes.Routes.matchTableRepairJobs(path) != null or routes.Routes.matchTableRepairControlJobs(path) != null or @@ -265,6 +269,8 @@ test "hot-standby mutation classifier covers acknowledged security catalog and w .{ .method = .POST, .path = "/tables/docs/restore", .surface = .table_restore }, .{ .method = .POST, .path = "/transactions/begin", .surface = .transaction_session }, .{ .method = .POST, .path = "/tables/docs/repair/run", .surface = .artifact_repair }, + .{ .method = .POST, .path = "/tables/docs/storage/migrations", .surface = .storage_migration }, + .{ .method = .POST, .path = "/tables/docs/storage/migrations/job", .surface = .storage_migration }, .{ .method = .POST, .path = "/tables/docs/artifacts/summary/reprocess", .surface = .artifact_reprocess }, }; for (cases) |case| { diff --git a/zig/pkg/antfly/src/storage/kernel_owner_abi.zig b/zig/pkg/antfly/src/storage/kernel_owner_abi.zig index 2c1d20ba36..8be6342144 100644 --- a/zig/pkg/antfly/src/storage/kernel_owner_abi.zig +++ b/zig/pkg/antfly/src/storage/kernel_owner_abi.zig @@ -18,7 +18,7 @@ const failure_abi = @import("runtime_failure_abi"); // Storage layouts evolve independently of the shared failure envelope. -pub const abi_version: u32 = 54; +pub const abi_version: u32 = 55; pub const Status = failure_abi.Status; pub const FailureBoundary = failure_abi.FailureBoundary; pub const FailureIdentity = failure_abi.FailureIdentity; @@ -1854,6 +1854,14 @@ pub extern fn antfly_storage_owner_document_artifact_manifests_json( out_response: *OwnedBytes, ) callconv(.c) Status; +/// Versioned, table-wide source ownership migration control. Kept separate +/// from artifact and ANN repair operations because it changes DB authority. +pub extern fn antfly_storage_owner_vector_migration_json( + owner: ?*anyopaque, + request: *const JsonOperationRequest, + out_response: *OwnedBytes, +) callconv(.c) Status; + pub extern fn antfly_storage_owner_artifact_operation_json( owner: ?*anyopaque, request: *const ArtifactOperationRequest, diff --git a/zig/pkg/antfly/src/storage/kernel_owner_client.zig b/zig/pkg/antfly/src/storage/kernel_owner_client.zig index c769688ade..678a1a96a6 100644 --- a/zig/pkg/antfly/src/storage/kernel_owner_client.zig +++ b/zig/pkg/antfly/src/storage/kernel_owner_client.zig @@ -822,6 +822,13 @@ pub const Owner = struct { return response; } + pub fn vectorMigrationJson(self: *Owner, table_name: []const u8, request_json: []const u8) !Response { + var response: Response = .{}; + const request = operationRequest(table_name, request_json); + try statusToError(abi.antfly_storage_owner_vector_migration_json(self.handle, &request, &response.buffer)); + return response; + } + pub fn artifactOperationJson( self: *Owner, table_name: []const u8, diff --git a/zig/pkg/antfly/src/storage/lsm_backend.zig b/zig/pkg/antfly/src/storage/lsm_backend.zig index 22c124c78a..2ad165cede 100644 --- a/zig/pkg/antfly/src/storage/lsm_backend.zig +++ b/zig/pkg/antfly/src/storage/lsm_backend.zig @@ -2083,6 +2083,67 @@ pub const Backend = struct { try self.finalizeDeferredStorageWorkLocked(); } + /// Request reclamation of superseded values in the current persisted runs. + /// The caller flushes replacement values first. Only metadata is prepared + /// here; the regular admitted, streaming GC jobs rewrite overlap closures. + /// The manifest intent survives restart, partial compaction and old readers. + pub fn requestValueReclamation(self: *Backend) !void { + const locked = runtime_mod.lockBackend(Backend, self); + defer runtime_mod.unlockBackend(Backend, self, locked); + if (self.options.backend.read_only) return error.ReadOnly; + const current = try self.planningDirectory(); + var wire: u64 = 128; + var scratch: u64 = 8192; + const height: u64 = if (current.tree.root) |root| root.height else 1; + var cursor = self.runs.cursor(); + while (cursor.next()) |run| { + if (run.gc_requested) continue; + const names = run.smallest_key.len + run.largest_key.len + + (if (run.path) |path| path.len else 0) + + (if (run.smallest_namespace_name) |name| name.len else 0) + + (if (run.largest_namespace_name) |name| name.len else 0); + wire +|= 192 +| names; + // Bound the administrative metadata clones before allocation. + scratch +|= (height + 1) * 16384 +| names * 2 +| + (if (run.state) |*state| state.estimatedMemoryBytes() else 0); + } + var reservation: ?resource_manager_mod.Reservation = null; + defer if (reservation) |*lease| lease.release(); + if (self.options.resource_manager) |manager| + reservation = try manager.reserve(.lsm_table_builder_working_set, scratch); + var credit = try self.admitCompactionMetadataBytes(wire); + defer credit.release(); + const directory = try current.fork(self.allocator); + var directory_owned = true; + defer if (directory_owned) directory.destroy(self.allocator); + const store = try self.allocator.create(RunStore); + store.* = self.runs.fork(); + defer self.retireRunStore(store); + cursor = self.runs.cursor(); + while (cursor.next()) |run| { + if (run.gc_requested) continue; + var revision = RunStore.revision(run, run.*); + revision.gc_requested = true; + try store.stageRevision(self.allocator, revision); + store.adopt(&revision); + try directory.put(self, revision); + } + std.mem.swap(RunStore, &self.runs, store); + self.invalidateReadVersion(); + self.publishRunDirectory(directory); + directory_owned = false; + credit.commit(); + self.markManifestDirty(); + try self.persistManifestLocked(); + self.notePotentialMaintenanceDebtLocked(); + } + + pub fn hasValueReclamationRequests(self: *Backend) !bool { + const locked = runtime_mod.lockBackend(Backend, self); + defer runtime_mod.unlockBackend(Backend, self, locked); + return (try self.planningDirectory()).hasGcRequest(); + } + pub fn syncReplayState(self: *Backend) !void { _ = try self.syncReplayStateWithStats(); } @@ -3104,13 +3165,22 @@ pub const Backend = struct { pub fn runMaintenanceStep(self: *Backend) !bool { const locked = runtime_mod.lockBackend(Backend, self); defer runtime_mod.unlockBackend(Backend, self, locked); - return try self.runMaintenanceStepLocked(); + return try self.runMaintenanceStepLocked(false); + } + + /// An explicitly requested rewrite must make progress under sustained + /// query traffic. It still uses ordinary I/O/memory admission and yields + /// inside streaming work; only the optional idle-grace deferral is bypassed. + pub fn runValueReclamationStep(self: *Backend) !bool { + const locked = runtime_mod.lockBackend(Backend, self); + defer runtime_mod.unlockBackend(Backend, self, locked); + return try self.runMaintenanceStepLocked(true); } pub fn runMaintenanceStepBestEffort(self: *Backend) !bool { if (!self.mu.tryLock()) return false; defer self.unlockWithReclamation(); - return try self.runMaintenanceStepLocked(); + return try self.runMaintenanceStepLocked(false); } pub fn makeWalCheckpointRetryDueForTest(self: *Backend) void { @@ -3121,7 +3191,7 @@ pub const Backend = struct { self.last_wal_retention_enforce_ns = 0; } - fn runMaintenanceStepLocked(self: *Backend) !bool { + fn runMaintenanceStepLocked(self: *Backend, required_gc: bool) !bool { // Cleanup is safe even after a durability fence or under pressure. // The unlock path executes one bounded FIFO reclamation turn. if (self.retired_ledger_snapshots != null and !self.ledger_reclaim_in_flight) return true; @@ -3226,10 +3296,10 @@ pub const Backend = struct { // pressure selection, so a 3.2x L0 backlog rewrote an 8.5x-overfull // L1 before L1 could be promoted. Use the soft L0 bound as the // pressure denominator while retaining overlap-triggered L0 work. - const defer_soft_compaction = self.optionalMaintenanceDeferredLocked(); + const defer_soft_compaction = !required_gc and self.optionalMaintenanceDeferredLocked(); if (!defer_soft_compaction) { self.gc_maintenance_turn +%= 1; - var compacted = self.pending_directory_closure == null and self.pending_l0_directory_closure == null and self.gc_maintenance_turn % 8 == 0 and (compaction_mod.nextTombstoneGcDelay(self) orelse 1) == 0 and + var compacted = self.pending_directory_closure == null and self.pending_l0_directory_closure == null and (required_gc or self.gc_maintenance_turn % 8 == 0) and (compaction_mod.nextTombstoneGcDelay(self) orelse 1) == 0 and try compaction_mod.compactTombstonesScheduled(Backend, self, score); if (!compacted) { compacted = if (self.l0SoftPressureLocked() and self.options.bulk_ingest_tiered_l0_fan_in >= 2) @@ -21957,6 +22027,83 @@ fn implementationTests() type { try std.testing.expectEqual(deadline, backend.tombstone_gc_retry_after_ns); } + test "lsm value reclamation survives partial progress restart and old readers without tombstones" { + const alloc = std.testing.allocator; + var storage = storage_io.MemoryStorage.init(alloc); + defer storage.deinit(); + const root = "/value-reclamation"; + const options: Options = .{ + .storage = storage.storage(), + .compact_threshold_runs = 1000, + .l0_overlap_compact_threshold_runs = 0, + .level_target_bytes_base = 1024 * 1024, + .tombstone_gc_max_age_ns = 0, + .tombstone_gc_max_input_bytes = 24 * 1024, + .max_compaction_input_bytes = 24 * 1024, + .obsolete_retention_ns = 0, + }; + var backend = try Backend.open(alloc, root, options); + var open = true; + defer if (open) backend.close(); + var bytes: [16 * 1024]u8 = undefined; + var random = std.Random.DefaultPrng.init(42); + random.random().bytes(&bytes); + for (0..3) |i| { + var state: State = .{}; + errdefer state.deinit(alloc); + try state.upsert(alloc, .{}, "key", &bytes, false); + const run = try compaction_mod.makeRunAtLevel(Backend, &backend, state, @intCast(3 - i)); + state = .{}; + try backend.runs.append(alloc, run); + } + try backend.runs.reindexForTest(alloc); + try backend.persistManifest(); + { + var old = try backend.beginRead(); + defer old.abort(); + try std.testing.expectEqualSlices(u8, &bytes, try old.get(.{}, "key")); + var state: State = .{}; + errdefer state.deinit(alloc); + try state.upsert(alloc, .{}, "key", "reference", false); + const run = try compaction_mod.makeRunAtLevel(Backend, &backend, state, 0); + state = .{}; + backend.invalidateReadVersion(); + try backend.runs.append(alloc, run); + try backend.runs.reindexForTest(alloc); + try backend.persistManifest(); + try backend.requestValueReclamation(); + try std.testing.expect(try backend.hasValueReclamationRequests()); + try std.testing.expectEqual(@as(usize, 0), (try backend.planningDirectory()).tombstoneRunCount()); + // Stop after one bounded rewrite, before the closure is done. + for (0..64) |_| { + const before = backend.compaction_stats.input_bytes; + _ = try backend.runMaintenanceStep(); + try std.testing.expect(backend.compaction_stats.input_bytes - before <= options.tombstone_gc_max_input_bytes); + if (backend.compaction_stats.input_bytes != 0) break; + } + try std.testing.expect(backend.compaction_stats.input_bytes != 0); + try std.testing.expect(try backend.hasValueReclamationRequests()); + try std.testing.expectEqualSlices(u8, &bytes, try old.get(.{}, "key")); + } + backend.close(); + open = false; + backend = try Backend.open(alloc, root, options); + open = true; + try std.testing.expect(try backend.hasValueReclamationRequests()); + for (0..256) |_| { + const before = backend.compaction_stats.input_bytes; + _ = try backend.runMaintenanceStep(); + try std.testing.expect(backend.compaction_stats.input_bytes - before <= options.tombstone_gc_max_input_bytes); + if (!try backend.hasValueReclamationRequests()) break; + } + try std.testing.expect(!try backend.hasValueReclamationRequests()); + try std.testing.expectEqualSlices(u8, "reference", try backend.getMergedWithMutable(&backend.mutable, .{}, "key")); + var physical: u64 = 0; + var runs = backend.runs.cursor(); + while (runs.next()) |run| physical += run.size_bytes; + try std.testing.expect(physical < bytes.len); + } + test "lsm GC checkpoints bounded level progress while preserving old readers" { var storage = storage_io.MemoryStorage.init(std.testing.allocator); defer storage.deinit(); diff --git a/zig/pkg/antfly/src/storage/lsm_backend/compaction.zig b/zig/pkg/antfly/src/storage/lsm_backend/compaction.zig index bca57a6858..9b5cc08001 100644 --- a/zig/pkg/antfly/src/storage/lsm_backend/compaction.zig +++ b/zig/pkg/antfly/src/storage/lsm_backend/compaction.zig @@ -113,7 +113,7 @@ pub fn nextTombstoneGcDelay(backend: anytype) ?u64 { } for (0..run_store.count(backend)) |rank| { const run = run_store.at(backend, rank); - if (run.gc_requested and (run.tombstone_count orelse 0) != 0) return 0; + if (run.gc_requested) return 0; } if (comptime !@hasField(@TypeOf(backend.options), "tombstone_gc_max_age_ns")) return null; if (backend.options.tombstone_gc_max_age_ns == 0) return null; @@ -1427,10 +1427,10 @@ fn gcComponentEligible(backend: anytype, indices: []const usize) bool { const deletes = run.tombstone_count orelse 0; tombstones +|= deletes; entries = @max(entries, run.entry_count); - if (deletes != 0) requested = requested or run.gc_requested or tombstoneAgeDue(backend, run); + requested = requested or run.gc_requested or (deletes != 0 and tombstoneAgeDue(backend, run)); } const percent = if (comptime @hasField(@TypeOf(backend.options), "tombstone_gc_min_percent")) @min(@as(u8, 100), backend.options.tombstone_gc_min_percent) else 50; - return tombstones != 0 and (requested or @as(u128, tombstones) * 100 >= @as(u128, entries) * percent); + return requested or (tombstones != 0 and @as(u128, tombstones) * 100 >= @as(u128, entries) * percent); } /// Persist the collection objective on every delete-bearing input, not just @@ -1487,7 +1487,7 @@ fn tombstoneGcCandidate(backend: anytype, index: *const DomainIndex, max_bytes: pub fn hasTombstoneGcDebt(backend: anytype) bool { if (comptime @hasField(@TypeOf(backend.*), "run_directory_dirty")) { if (!backend.run_directory_dirty) if (backend.run_directory) |directory| { - if (directory.tombstoneRunCount() == 0) return false; + if (directory.tombstoneRunCount() == 0 and !directory.hasGcRequest()) return false; }; } if (comptime @hasField(@TypeOf(backend.*), "gc_debt_cache")) { @@ -2046,7 +2046,7 @@ fn selectDirectoryGc(backend: anytype, max_bytes: u64) !?SelectedPlan { const limit = if (max_bytes == 0) configured else if (configured == 0) max_bytes else @min(max_bytes, configured); if (backend.pending_gc == null) { const current_directory = try backend.planningDirectory(); - if (current_directory.tombstoneRunCount() == 0) return null; + if (current_directory.tombstoneRunCount() == 0 and !current_directory.hasGcRequest()) return null; const directory = try current_directory.fork(allocator); errdefer backend.retireCheckpointDirectory(directory); var reservation: ?resource_manager_mod.Reservation = null; @@ -2189,7 +2189,7 @@ fn selectGcProgress(backend: anytype, index: *const DomainIndex, limit: u64) !?S const runs = if (index.mixed) (try run_store.oracleItems(backend)) else index.runs[start..end]; for (runs, 0..) |run, i| { const deletes = run.tombstone_count orelse 0; - if (deletes == 0 or run.level == std.math.maxInt(u32)) continue; + if ((deletes == 0 and !run.gc_requested) or run.level == std.math.maxInt(u32)) continue; const percent = if (comptime @hasField(@TypeOf(backend.options), "tombstone_gc_min_percent")) backend.options.tombstone_gc_min_percent else 50; // The projection may predate the request bit; use the live input. const live_index = if (index.mixed) i else index.order[start + i]; @@ -6051,7 +6051,7 @@ test "bounded GC carries aggregate eligibility across windows and retries indepe outputs[1].tombstone_count = 0; inheritTombstoneAge(&outputs, &.{&runs[0]}); try std.testing.expect(outputs[0].gc_requested); - try std.testing.expect(!outputs[1].gc_requested); + try std.testing.expect(outputs[1].gc_requested); } test "persistent planner ordering matches rebuilt domain and GC indexes after level moves" { @@ -6129,7 +6129,7 @@ test "compaction installation preserves GC requests advanced during its build" { live[1].gc_requested = true; reconcileGcObjective(&live, plan, &outputs); try std.testing.expect(outputs[0].gc_requested); - try std.testing.expect(!outputs[1].gc_requested); + try std.testing.expect(outputs[1].gc_requested); } test "domain compaction maps interleaved inputs and revalidates concurrent publication" { @@ -7812,22 +7812,24 @@ fn reconcileGcObjective(live: anytype, plan: CompactionPlan, outputs: []Run) voi var requested = false; for (0..plan.source_len) |i| requested = requested or run_store.planGet(live, plan, i).gc_requested; for (0..plan.target_len) |i| requested = requested or run_store.planGet(live, plan, plan.source_len + i).gc_requested; - if (requested) for (outputs) |*run| { - if ((run.tombstone_count orelse 0) != 0) run.gc_requested = true; - }; + // Only a validated full overlap closure discharges a rewrite request. + // Partial level jobs and source splits must carry it even without deletes: + // their outputs can still hide obsolete values in deeper runs. + for (outputs) |*run| run.gc_requested = !plan.tombstone_gc and (requested or run.gc_requested); } fn inheritTombstoneAge(outputs: []Run, inputs: anytype) void { var oldest = gcNowNs(); var requested = false; - for (@as([]const @TypeOf(inputs[0]), inputs)) |run| if ((inputRun(run).tombstone_count orelse 0) != 0) { - oldest = @min(oldest, inputRun(run).oldest_tombstone_unix_ns); + for (@as([]const @TypeOf(inputs[0]), inputs)) |run| { + if ((inputRun(run).tombstone_count orelse 0) != 0) + oldest = @min(oldest, inputRun(run).oldest_tombstone_unix_ns); requested = requested or inputRun(run).gc_requested; - }; + } for (outputs) |*run| { const has_deletes = (run.tombstone_count orelse 0) != 0; run.oldest_tombstone_unix_ns = if (has_deletes) oldest else 0; - run.gc_requested = has_deletes and requested; + run.gc_requested = requested; } } diff --git a/zig/pkg/antfly/src/storage/lsm_backend/compaction_publication.zig b/zig/pkg/antfly/src/storage/lsm_backend/compaction_publication.zig index 6c6e12a419..bb62fe670e 100644 --- a/zig/pkg/antfly/src/storage/lsm_backend/compaction_publication.zig +++ b/zig/pkg/antfly/src/storage/lsm_backend/compaction_publication.zig @@ -165,7 +165,9 @@ pub const Job = struct { return; } const output = &outputs[self.index]; - if (self.requested_gc and (output.tombstone_count orelse 0) != 0) output.gc_requested = true; + // A full overlap rewrite discharges both tombstones and + // superseded-value requests. Partial outputs keep the intent. + output.gc_requested = !self.plan.tombstone_gc and (self.requested_gc or output.gc_requested); const names = (if (output.path) |path| path.len else 0) + output.smallest_key.len + output.largest_key.len + (if (output.smallest_namespace_name) |name| name.len else 0) + (if (output.largest_namespace_name) |name| name.len else 0); try self.admitNames(names + (if (output.state) |*state| state.estimatedMemoryBytes() else 0)); diff --git a/zig/pkg/antfly/src/storage/lsm_backend/gc_job.zig b/zig/pkg/antfly/src/storage/lsm_backend/gc_job.zig index 126ae2fb3e..b39ba665d5 100644 --- a/zig/pkg/antfly/src/storage/lsm_backend/gc_job.zig +++ b/zig/pkg/antfly/src/storage/lsm_backend/gc_job.zig @@ -128,7 +128,7 @@ pub const Job = struct { pub fn init(directory: *const Directory, start: usize, age: u64, percent: u8, now: u64, limit: u64) Job { const rank = if (start < directory.count()) start else 0; - return .{ .directory = directory, .cursor = .{ .directory = directory, .rank = rank }, .start_rank = rank, .age = age, .percent = @min(percent, 100), .now = now, .limit = limit }; + return .{ .directory = directory, .cursor = .{ .directory = directory, .rank = rank, .include_requests = true }, .start_rank = rank, .age = age, .percent = @min(percent, 100), .now = now, .limit = limit }; } pub fn step(self: *Job, allocator: std.mem.Allocator, credits_arg: usize, deadline: u64) !bool { var credits = credits_arg; @@ -189,7 +189,7 @@ pub const Job = struct { self.entries = @max(self.entries, run.entry_count); if (run.level == 0 and Directory.readLess({}, self.oldest, handle)) self.oldest = handle; const deletes = run.tombstone_count orelse 0; - if (deletes == 0) continue; + if (deletes == 0 and !run.gc_requested) continue; if (!run.gc_requested) { self.intent_runs += 1; const names = run.smallest_key.len + run.largest_key.len + (if (run.path) |path| path.len else 0) + @@ -201,12 +201,12 @@ pub const Job = struct { self.seen.putPrepared(self.scratch_allocator orelse allocator, .{ .id = run.id }); self.deletes +|= deletes; const due = run.oldest_tombstone_unix_ns +| self.age; - const aged = self.age != 0 and (run.oldest_tombstone_unix_ns == 0 or run.oldest_tombstone_unix_ns > self.now or due <= self.now); + const aged = deletes != 0 and self.age != 0 and (run.oldest_tombstone_unix_ns == 0 or run.oldest_tombstone_unix_ns > self.now or due <= self.now); self.requested = self.requested or run.gc_requested or aged; - if (self.age != 0 and !aged) self.valid_until = @min(self.valid_until, due); + if (deletes != 0 and self.age != 0 and !aged) self.valid_until = @min(self.valid_until, due); continue; } - self.eligible = self.deletes != 0 and (self.requested or @as(u128, self.deletes) * 100 >= @as(u128, self.entries) * self.percent); + self.eligible = self.requested or (self.deletes != 0 and @as(u128, self.deletes) * 100 >= @as(u128, self.entries) * self.percent); if (!self.eligible) { self.phase = .discard; continue; diff --git a/zig/pkg/antfly/src/storage/lsm_backend/run_directory.zig b/zig/pkg/antfly/src/storage/lsm_backend/run_directory.zig index 3880da8cfd..34afdedfec 100644 --- a/zig/pkg/antfly/src/storage/lsm_backend/run_directory.zig +++ b/zig/pkg/antfly/src/storage/lsm_backend/run_directory.zig @@ -131,7 +131,7 @@ const Entry = struct { return .{ .tombstone_runs = left.tombstone_runs + right.tombstone_runs + @intFromBool(deletes), .unknown_tombstone_runs = left.unknown_tombstone_runs + right.unknown_tombstone_runs + @intFromBool(entry.run.tombstone_count == null), - .gc_requested = left.gc_requested or right.gc_requested or (deletes and entry.run.gc_requested), + .gc_requested = left.gc_requested or right.gc_requested or entry.run.gc_requested, .oldest_tombstone = @min(left.oldest_tombstone, right.oldest_tombstone, if (deletes) entry.run.oldest_tombstone_unix_ns else std.math.maxInt(u64)), .newest_tombstone = @max(left.newest_tombstone, right.newest_tombstone, if (deletes) entry.run.oldest_tombstone_unix_ns else 0), }; @@ -836,13 +836,17 @@ pub const Directory = struct { /// retaining only the minimum timestamp would lose that condition. pub fn tombstoneGcDelay(self: *const Directory, age: u64, now: u64) ?u64 { const summary = (self.tree.root orelse return null).summary; - if (summary.tombstone_runs == 0) return null; if (summary.gc_requested) return 0; + if (summary.tombstone_runs == 0) return null; if (age == 0) return null; if (summary.oldest_tombstone == 0 or summary.newest_tombstone > now) return 0; return (summary.oldest_tombstone +| age) -| now; } + pub fn hasGcRequest(self: *const Directory) bool { + return if (self.tree.root) |root| root.summary.gc_requested else false; + } + pub fn tombstoneRunCount(self: *const Directory) usize { return if (self.tree.root) |root| root.summary.tombstone_runs else 0; } @@ -872,20 +876,21 @@ pub const Directory = struct { pub const TombstoneCursor = struct { directory: *const Directory, rank: usize = 0, + include_requests: bool = false, pub fn next(self: *@This()) ?Handle { - const found = nextMarked(self.directory.tree.root, self.rank, 0) orelse return null; + const found = nextMarked(self.directory.tree.root, self.rank, 0, self.include_requests) orelse return null; self.rank = found.rank + 1; return .{ .run = found.node.entry.run, .revision = found.node.entry.payload.? }; } const Found = struct { node: *const Tree.Node, rank: usize }; - fn nextMarked(root: ?*const Tree.Node, after: usize, base: usize) ?Found { + fn nextMarked(root: ?*const Tree.Node, after: usize, base: usize, requests: bool) ?Found { const node = root orelse return null; - if (node.summary.tombstone_runs == 0 or base + node.count <= after) return null; + if ((node.summary.tombstone_runs == 0 and !(requests and node.summary.gc_requested)) or base + node.count <= after) return null; const rank = base + (if (node.left) |left| left.count else 0); - if (nextMarked(node.left, after, base)) |found| return found; - if (rank >= after and (node.entry.run.tombstone_count orelse 0) != 0) return .{ .node = node, .rank = rank }; - return nextMarked(node.right, after, rank + 1); + if (nextMarked(node.left, after, base, requests)) |found| return found; + if (rank >= after and ((node.entry.run.tombstone_count orelse 0) != 0 or (requests and node.entry.run.gc_requested))) return .{ .node = node, .rank = rank }; + return nextMarked(node.right, after, rank + 1, requests); } }; @@ -1031,7 +1036,7 @@ test "run directory path copies preserve pinned epochs through inserts removals old.tombstone_count = 0; old.gc_requested = true; try gc.put(&backend, old); - try std.testing.expectEqual(@as(?u64, null), gc.tombstoneGcDelay(100, 1000)); + try std.testing.expectEqual(@as(?u64, 0), gc.tombstoneGcDelay(100, 1000)); try std.testing.expectEqual(@as(?u64, null), snapshot.tombstoneGcDelay(100, 1000)); var no_deletes = snapshot.tombstoneCursor(); try std.testing.expect(no_deletes.next() == null); diff --git a/zig/pkg/antfly/src/storage/lsm_backend/storage_io.zig b/zig/pkg/antfly/src/storage/lsm_backend/storage_io.zig index 542df2f4b5..eee73cedc8 100644 --- a/zig/pkg/antfly/src/storage/lsm_backend/storage_io.zig +++ b/zig/pkg/antfly/src/storage/lsm_backend/storage_io.zig @@ -1393,6 +1393,20 @@ else fn retain(self: *FdCache, namespace: u64, io: std.Io, path: []const u8) !*Entry { const path_hash = namespacedPathHash(namespace, path); const shard = self.shardForHash(path_hash); + // Cache hits already own a stable filename and descriptor. Avoid + // allocating a transient pathname (mmap/munmap in the process-wide + // page-allocated pool) on every point read. Misses still allocate + // outside the mutex and recheck both the entry and mutation epoch. + { + const locked = lockAtomic(&shard.mutex); + defer if (locked) shard.mutex.unlock(); + if (self.findEntryLocked(shard, namespace, path_hash, path)) |existing| { + existing.ref_count += 1; + existing.last_access = self.nextAccessLocked(); + self.touchEntryLocked(shard, existing); + return existing; + } + } const owned_path = try self.allocator.dupeZ(u8, path); var owned_path_active = true; errdefer if (owned_path_active) self.allocator.free(owned_path); @@ -4552,6 +4566,36 @@ test "native storage retained runtime has a finite worker ceiling" { ); } +test "native fd cache hits reuse their path without allocation" { + if (!supports_posix_fd_cache) return error.SkipZigTest; + var test_tmp = try TestDirectory.init("fd-hit"); + defer test_tmp.cleanup(); + var native = try NativeStorage.init(std.testing.allocator, .threaded); + defer native.deinit(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}-cached-file", .{test_tmp.path()}); + defer native.storage().deleteFileAbsolute(path) catch {}; + try native.storage().writeFileAbsolute(path, "value"); + // The process-wide FD pool uses the page allocator. Exercise that shape, + // and reject any allocation after the descriptor has been populated. + var failing = std.testing.FailingAllocator.init(std.heap.page_allocator, .{}); + var cache = FdCache.init(failing.allocator(), 8); + defer cache.deinit(); + const first = try cache.retain(1, std.testing.io, path); + cache.release(std.testing.io, first); + const before = failing.alloc_index; + const started = @import("antfly_platform").time.monotonicNs(); + for (0..20000) |_| { + const entry = try cache.retain(1, std.testing.io, path); + cache.release(std.testing.io, entry); + } + std.debug.print("fd cache 20000 hits: {d} ns, {d} allocations\n", .{ @import("antfly_platform").time.monotonicNs() - started, failing.alloc_index - before }); + failing.fail_index = failing.alloc_index; + const hit = try cache.retain(1, std.testing.io, path); + cache.release(std.testing.io, hit); + try std.testing.expectEqual(before, failing.alloc_index); +} + test "native fd cache retries an open that straddles a mutation fence" { if (!supports_posix_fd_cache or builtin.single_threaded) return error.SkipZigTest; diff --git a/zig/pkg/antfly/src/storage/test_manifest.zig b/zig/pkg/antfly/src/storage/test_manifest.zig index 0aeff31eba..1a2be52ffb 100644 --- a/zig/pkg/antfly/src/storage/test_manifest.zig +++ b/zig/pkg/antfly/src/storage/test_manifest.zig @@ -28,6 +28,7 @@ comptime { _ = @import("projection_page_cache.zig"); _ = @import("projection_read_trace.zig"); _ = @import("vector_payload_store.zig"); + _ = @import("vector_migration_offline.zig"); _ = @import("vector_wal_view.zig"); _ = @import("backend_adapter.zig"); _ = @import("backend_conformance_test.zig"); diff --git a/zig/pkg/antfly/src/storage/vector_migration.zig b/zig/pkg/antfly/src/storage/vector_migration.zig new file mode 100644 index 0000000000..e261376258 --- /dev/null +++ b/zig/pkg/antfly/src/storage/vector_migration.zig @@ -0,0 +1,285 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Bounded source-vector backfill and verification. The DB owns admission; +//! candidate mappings and progress use the same primary transaction. Online +//! mutation capture maintains one latest reference per artifact in that same +//! transaction, so replay cannot lag or retain a second unbounded payload log. +const std = @import("std"); +pub const contract = @import("../common/vector_migration.zig"); +const payload = @import("artifact_payload.zig"); +const docstore = @import("docstore.zig"); +const erased = @import("backend_erased.zig"); +const internal_keys = @import("internal_keys.zig"); +const codec = @import("db/enrichment/artifact_codec.zig"); +const Allocator = std.mem.Allocator; + +pub const Boundary = enum { before_prepare, after_prepare, after_commit, after_sync, publication_commit, publication_sync, reclamation_request, reclamation_receipt }; +pub var test_boundary: ?*const fn (Boundary) anyerror!void = null; + +pub fn boundary(point: Boundary) !void { + if (@import("builtin").is_test) if (test_boundary) |hook| try hook(point); +} + +pub fn load(alloc: Allocator, primary: *docstore.DocStore) !?std.json.Parsed(contract.Job) { + const raw = primary.get(alloc, contract.job_key) catch |err| switch (err) { + error.NotFound => return null, + else => return err, + }; + defer alloc.free(raw); + var parsed = try std.json.parseFromSlice(contract.Job, alloc, raw, .{ .allocate = .alloc_always }); + errdefer parsed.deinit(); + try parsed.value.validate(); + return parsed; +} + +pub fn save(alloc: Allocator, txn: anytype, job: contract.Job) !void { + try job.validate(); + const raw = try std.json.Stringify.valueAlloc(alloc, job, .{}); + defer alloc.free(raw); + try txn.put(contract.job_key, raw); +} + +const Rows = struct { + const Row = struct { + key: []const u8, + value: []const u8, + dense: bool, + inline_payload: bool, + }; + arena: std.heap.ArenaAllocator, + items: []const Row, + exhausted: bool, + fn deinit(self: *Rows) void { + self.arena.deinit(); + } +}; + +fn readRows(alloc: Allocator, primary: *docstore.DocStore, job: contract.Job) !Rows { + var arena = std.heap.ArenaAllocator.init(alloc); + errdefer arena.deinit(); + const scratch = arena.allocator(); + var rows = std.ArrayListUnmanaged(Rows.Row).empty; + var read = try primary.runtime_store.beginReadWithBlockCacheAdmission(.transient); + defer read.abort(); + var cursor = try read.openCursor(); + defer cursor.close(); + const cleanup = job.phase == .cleanup or job.phase == .cancelling; + const decoded_cursor = try scratch.alloc(u8, job.cursor.len / 2); + if (job.cursor.len % 2 != 0) return error.InvalidVectorMigrationState; + _ = std.fmt.hexToBytes(decoded_cursor, job.cursor) catch return error.InvalidVectorMigrationState; + const lower = if (decoded_cursor.len != 0) decoded_cursor else if (cleanup) contract.candidate_prefix else ""; + var entry = try cursor.seekAtOrAfter(lower); + if (entry) |row| if (decoded_cursor.len != 0 and std.mem.eql(u8, row.key, decoded_cursor)) { + entry = try cursor.next(); + }; + var bytes: u64 = 0; + while (entry) |row| { + if (cleanup and !std.mem.startsWith(u8, row.key, contract.candidate_prefix)) { + entry = null; + break; + } + // Unrelated primary values only contribute a cursor key. In draining, + // hash the borrowed inline value into its compact reference before + // advancing the cursor; a concurrent capture after verification may + // have introduced an embedding larger than the original page budget. + const dense = !cleanup and try denseArtifact(.{ .key = row.key, .value = row.value }); + const inline_payload = dense and !payload.isReference(row.value); + const work_bytes = try std.math.add(u64, row.key.len, if (dense) row.value.len else 0); + if (rows.items.len == job.budget.batch_rows or + (rows.items.len != 0 and bytes + work_bytes > job.budget.batch_bytes)) break; + var reference: [payload.reference_len]u8 = undefined; + const value = if (!dense) "" else if (job.phase == .draining and inline_payload) blk: { + reference = (try payload.Reference.forArtifact(row.key, row.value)).encode(); + break :blk &reference; + } else row.value; + if (job.phase == .final_verification and inline_payload) return error.VectorMigrationInlinePayloadRemains; + const size = try std.math.add(u64, row.key.len, value.len); + if (size > job.budget.batch_bytes) return error.VectorMigrationRowExceedsBudget; + try rows.append(scratch, .{ .key = try scratch.dupe(u8, row.key), .value = try scratch.dupe(u8, value), .dense = dense, .inline_payload = inline_payload }); + // An oversized published inline vector consumes a page by itself. Its + // bytes are borrowed from the cursor; only the reference is retained. + bytes += work_bytes; + entry = try cursor.next(); + } + return .{ .arena = arena, .items = rows.items, .exhausted = entry == null }; +} + +fn denseArtifact(row: docstore.KVPair) !bool { + if (!payload.isEmbeddingKey(row.key)) return false; + if (payload.isReference(row.value)) { + _ = try payload.Reference.decode(row.value); + return true; + } + const header = try codec.decodeHeader(row.value); + return header.kind == .dense_embedding; +} + +fn sameCurrent(txn: *erased.WriteTxn, row: docstore.KVPair) !bool { + const current = txn.get(row.key) catch |err| switch (err) { + error.NotFound => return false, + else => return err, + }; + return std.mem.eql(u8, current, row.value); +} + +/// Caller serializes steps with document mutations. Each call retains at most +/// one budgeted page, prepares source bytes before committing any references, +/// and commits its exclusive cursor together with those references. +pub fn advance(alloc: Allocator, primary: *docstore.DocStore, source: payload.Store, job: contract.Job) !void { + if (job.phase == .ready or job.phase == .serving or !job.active()) return; + var rows = try readRows(alloc, primary, job); + defer rows.deinit(); + var next = job; + next.last_error = null; + const session = try payload.Session.create(alloc, source); + defer session.release(); + session.migration_allowance = job.budget.temporary_bytes; + var txn = try primary.runtime_store.beginWrite(); + var committed = false; + defer if (!committed) txn.abort(); + for (rows.items) |row| { + // Primary identities contain arbitrary bytes. Hex keeps both the + // durable record and HTTP progress valid UTF-8 JSON. + next.cursor = try std.fmt.allocPrint(rows.arena.allocator(), "{x}", .{row.key}); + if (job.phase == .cleanup or job.phase == .cancelling) { + try txn.delete(row.key); + continue; + } + next.scanned_rows +|= 1; + if (!row.dense) continue; + // The DB holds apply-exclusive across page capture and commit. The + // published phases therefore use the reference certified above without + // retaining or hashing another copy of the inline payload. + if (!job.published() and !try sameCurrent(&txn, .{ .key = row.key, .value = row.value })) continue; + const candidate_key = try contract.candidateKeyAlloc(alloc, row.key); + defer alloc.free(candidate_key); + switch (job.phase) { + .backfill => { + if (payload.isReference(row.value)) return error.VectorMigrationCoverageMismatch; + const reference = try session.put(row.key, row.value); + try txn.put(candidate_key, reference); + next.prepared_artifacts +|= 1; + next.prepared_bytes = try std.math.add(u64, next.prepared_bytes, row.value.len); + if (next.prepared_bytes > job.budget.temporary_bytes) return error.VectorMigrationTemporaryBudgetExceeded; + }, + .verifying => { + if (payload.isReference(row.value)) return error.VectorMigrationCoverageMismatch; + const candidate = txn.get(candidate_key) catch |err| switch (err) { + error.NotFound => return error.VectorMigrationCoverageMismatch, + else => return err, + }; + const expected = try payload.Reference.forArtifact(row.key, row.value); + if (!std.mem.eql(u8, candidate, &expected.encode())) return error.VectorMigrationCoverageMismatch; + const restored = try session.getAlloc(alloc, row.key, candidate); + defer alloc.free(restored); + if (!std.mem.eql(u8, restored, row.value)) return error.VectorMigrationCoverageMismatch; + next.verified_artifacts +|= 1; + }, + .draining => { + if (!row.inline_payload) continue; + const reference = row.value; + const candidate = txn.get(candidate_key) catch |err| switch (err) { + error.NotFound => return error.VectorMigrationCoverageMismatch, + else => return err, + }; + if (!std.mem.eql(u8, candidate, reference)) return error.VectorMigrationCoverageMismatch; + // Preparation already committed with the candidate mapping. + // Reuse that proof instead of appending/charging the payload a + // second time while draining the old primary representation. + try txn.put(row.key, reference); + try session.recordOwnership(&txn, row.key, reference); + try txn.delete(candidate_key); + next.rewritten_artifacts +|= 1; + }, + .final_verification => { + if (!payload.isReference(row.value)) return error.VectorMigrationInlinePayloadRemains; + const restored = try session.getAlloc(alloc, row.key, row.value); + defer alloc.free(restored); + const expected = try payload.Reference.forArtifact(row.key, restored); + if (!std.mem.eql(u8, row.value, &expected.encode())) return error.VectorMigrationCoverageMismatch; + }, + else => unreachable, + } + } + if (rows.exhausted) { + next.cursor = ""; + next.phase = switch (job.phase) { + .backfill => .verifying, + .verifying => .ready, + .draining => .final_verification, + .final_verification => .serving, + .cleanup => .reclaiming, + .cancelling => .cancelled, + else => unreachable, + }; + } + try session.stageReferenceEpoch(&txn); + if (txn.get(contract.accounting_key)) |raw| { + if (raw.len != 8) return error.InvalidVectorMigrationState; + next.charged_temporary_bytes = std.mem.readInt(u64, raw[0..8], .little); + } else |err| if (err != error.NotFound) return err; + const epoch = txn.get(payload.reference_epoch_key) catch |err| switch (err) { + error.NotFound => null, + else => return err, + }; + if (epoch) |bytes| { + if (bytes.len != 8) return error.InvalidVectorReferenceEpoch; + next.replay_cursor = std.mem.readInt(u64, bytes[0..8], .little); + } + try save(alloc, &txn, next); + try boundary(.before_prepare); + try session.prepareCommit(); + try boundary(.after_prepare); + session.primary_commit_attempted = true; + try txn.commit(); + committed = true; + try boundary(.after_commit); + // The candidate references and cursor share the committed primary WAL + // record. Make that record durable without forcing an SSTable for every + // bounded page (verification pages often change only the job receipt). + // Publication retains the full storage barrier below. + try primary.runtime_store.syncReplayState(); + try boundary(.after_sync); + session.committed = true; +} + +/// Ownership and the publication decision are one primary commit. A stale +/// catalog may reconcile this decision, but cannot cause a second conversion. +pub fn publish(alloc: Allocator, primary: *docstore.DocStore, job: contract.Job) !void { + if (job.published()) return; + if (job.phase != .ready) return error.VectorMigrationNotReady; + var next = job; + next.phase = .draining; + next.cursor = ""; + var txn = try primary.runtime_store.beginWrite(); + var committed = false; + defer if (!committed) txn.abort(); + const epoch = txn.get(payload.reference_epoch_key) catch |err| switch (err) { + error.NotFound => null, + else => return err, + }; + if (epoch) |bytes| { + if (bytes.len != 8) return error.InvalidVectorReferenceEpoch; + next.replay_cursor = std.mem.readInt(u64, bytes[0..8], .little); + } + next.publication_fence = next.replay_cursor; + try txn.put(&internal_keys.table_storage_settings_key, "{\"dense_embeddings\":\"vector_store\"}"); + try save(alloc, &txn, next); + try txn.commit(); + committed = true; + try boundary(.publication_commit); + try primary.runtime_store.sync(true); + try boundary(.publication_sync); +} diff --git a/zig/pkg/antfly/src/storage/vector_migration_offline.zig b/zig/pkg/antfly/src/storage/vector_migration_offline.zig new file mode 100644 index 0000000000..d29c188fd2 --- /dev/null +++ b/zig/pkg/antfly/src/storage/vector_migration_offline.zig @@ -0,0 +1,520 @@ +// Copyright 2026 Antfly, Inc. +// +// Licensed under the Elastic License 2.0 (ELv2); you may not use this file +// except in compliance with the Elastic License 2.0. You may obtain a copy of +// the Elastic License 2.0 at +// +// https://www.antfly.io/licensing/ELv2-license +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// Elastic License 2.0 for the specific language governing permissions and +// limitations. + +//! Stopped-table migration. The caller also owns catalog admission. Copy every +//! regular file under an exclusive generation lease, recording a durable byte +//! cursor; then use the same source conversion/verifier as online migration. +const std = @import("std"); +const db = @import("db/db.zig"); +const contract = @import("../common/vector_migration.zig"); +const files = @import("../common/migration_files.zig"); +const fs = @import("../common/fs_paths.zig"); +const platform = @import("antfly_platform"); +const Allocator = std.mem.Allocator; +const progress_file = "VECTOR-MIGRATION-COPY.json"; +const cancellation_file = "VECTOR-MIGRATION-CANCELLED.json"; +const Cancellation = struct { version: u32 = 1, request: contract.Request, identity: []const u8 }; + +const Entry = struct { path: []const u8, size: u64 }; +const Fence = struct { + version: u32 = 1, + request: contract.Request, + identity: []const u8, + entries: []const Entry, + source_bytes: u64, +}; +const Progress = struct { + version: u32 = 1, + job_id: []const u8, + file: usize = 0, + offset: u64 = 0, + copied_bytes: u64 = 0, + copy_complete: bool = false, +}; + +pub const Options = struct { + open: db.OpenOptions = .{}, + /// Bounds work for orchestration/tests. Zero runs until complete. A pending + /// return deliberately leaves both durable admission and candidate intact. + max_steps: usize = 0, + progress_ctx: ?*anyopaque = null, + progress_fn: ?*const fn (?*anyopaque, []const u8) anyerror!void = null, +}; +pub const Result = enum { pending, complete }; +pub const Boundary = enum { fenced, chunk_synced, cursor_synced, candidate_complete, before_publication, after_publication }; +pub var test_boundary: ?*const fn (Boundary) anyerror!void = null; +fn boundary(point: Boundary) !void { + if (@import("builtin").is_test) if (test_boundary) |hook| try hook(point); +} + +fn readJson(comptime T: type, alloc: Allocator, io: std.Io, path: []const u8) !std.json.Parsed(T) { + const bytes = try std.Io.Dir.cwd().readFileAlloc(io, path, alloc, .limited(64 * 1024 * 1024)); + defer alloc.free(bytes); + return try std.json.parseFromSlice(T, alloc, bytes, .{ .allocate = .alloc_always }); +} +fn save(alloc: Allocator, io: std.Io, path: []const u8, value: anytype) !void { + const encoded = try std.json.Stringify.valueAlloc(alloc, value, .{}); + defer alloc.free(encoded); + try files.writeAtomic(alloc, io, path, encoded); +} +fn exists(io: std.Io, path: []const u8) !bool { + std.Io.Dir.cwd().access(io, path, .{}) catch |err| switch (err) { + error.FileNotFound => return false, + else => return err, + }; + return true; +} +fn inventory(alloc: Allocator, io: std.Io, root: []const u8) ![]Entry { + var dir = try std.Io.Dir.cwd().openDir(io, root, .{ .iterate = true }); + defer dir.close(io); + var walker = try dir.walk(alloc); + defer walker.deinit(); + var result = std.ArrayListUnmanaged(Entry).empty; + errdefer { + for (result.items) |entry| alloc.free(entry.path); + result.deinit(alloc); + } + while (try walker.next(io)) |entry| { + if (entry.kind == .directory) continue; + if (entry.kind != .file) return error.VectorMigrationUnsupportedFile; + if (std.mem.eql(u8, entry.path, contract.offline_fence_file) or + std.mem.eql(u8, entry.path, progress_file) or std.mem.endsWith(u8, entry.path, ".migration-tmp")) continue; + const stat = try dir.statFile(io, entry.path, .{}); + try result.append(alloc, .{ .path = try alloc.dupe(u8, entry.path), .size = stat.size }); + } + std.mem.sort(Entry, result.items, {}, struct { + fn less(_: void, a: Entry, b: Entry) bool { + return std.mem.lessThan(u8, a.path, b.path); + } + }.less); + return try result.toOwnedSlice(alloc); +} +fn capacity(root: []const u8, budget: contract.Budget, needed: u64) !void { + const available = try platform.filesystem.capacity(root); + if (available.available_bytes < budget.disk_reserve_bytes +| needed) return error.VectorMigrationDiskReserve; +} + +/// Copy and acknowledge one chunk. The durable cursor must never get ahead +/// of either the payload bytes or any directory link used to reach them. +fn copyChunk(alloc: Allocator, io: std.Io, live: []const u8, staging: []const u8, cursor_path: []const u8, entry: Entry, progress: *Progress, buffer: []u8, verify: []u8) !void { + const from = try std.fs.path.join(alloc, &.{ live, entry.path }); + defer alloc.free(from); + const to = try std.fs.path.join(alloc, &.{ staging, entry.path }); + defer alloc.free(to); + const count: usize = @intCast(@min(buffer.len, entry.size - progress.offset)); + var input = try std.Io.Dir.cwd().openFile(io, from, .{}); + defer input.close(io); + if ((try input.stat(io)).size != entry.size) return error.SourceFileChanged; + if (try input.readPositionalAll(io, buffer[0..count], progress.offset) != count) return error.SourceFileChanged; + const parent = std.fs.path.dirname(to).?; + try fs.createDirPathPortable(io, parent); + var output = try std.Io.Dir.cwd().createFile(io, to, .{ .read = true, .truncate = false }); + defer output.close(io); + try output.writePositionalAll(io, buffer[0..count], progress.offset); + if (progress.offset + count == entry.size) try output.setLength(io, entry.size); + try output.sync(io); + try fs.syncDirPortable(io, parent); + if (progress.offset == 0) { + // Retry this even for existing directories: a prior attempt may have + // failed after mkdir, before its parent was synced. Later chunks may + // rely on the directory chain acknowledged by the preceding cursor. + var directory = parent; + while (!std.mem.eql(u8, directory, staging)) { + directory = std.fs.path.dirname(directory) orelse return error.InvalidVectorMigrationState; + try fs.syncDirPortable(io, directory); + } + // resumeStaging can itself have just created the shadow root. + try fs.syncDirPortable(io, std.fs.path.dirname(staging) orelse "."); + } + try boundary(.chunk_synced); + if (try output.readPositionalAll(io, verify[0..count], progress.offset) != count or + !std.mem.eql(u8, buffer[0..count], verify[0..count])) return error.VectorMigrationCopyMismatch; + progress.offset += count; + progress.copied_bytes += count; + if (progress.offset == entry.size) { + progress.file += 1; + progress.offset = 0; + } + try save(alloc, io, cursor_path, progress.*); + try boundary(.cursor_synced); +} + +pub fn run(alloc: Allocator, io: std.Io, root: []const u8, request: contract.Request, options: Options) !Result { + try request.validate(); + if (request.mode != .offline) return error.InvalidVectorMigrationState; + var transition = try @import("db/generation_lifecycle.zig").beginProcessExclusiveWithRuntimeAndIo(root, options.open.backend_runtime, io); + defer transition.deinit(); + try transition.reconcilePublished(); + const live = transition.path; + const fence_path = try std.fs.path.join(alloc, &.{ live, contract.offline_fence_file }); + defer alloc.free(fence_path); + const plan = try db.DB.resolveNativeRestoreOpenPlan(live, options.open); + if (plan.physicalRootMode() != .filesystem_managed) return error.VectorStoreLifecycleUnsupported; + var source_options = try plan.optionsForTarget(live); + source_options.exclusive_generation = &transition; + source_options.staged_generation = null; + source_options.open_mode = .status_only; + source_options.start_index_workers = false; + source_options.start_optional_runtimes = false; + source_options.start_optional_runtime_workers = false; + // A completed candidate may already be the live root after a lost response. + { + var source = try db.DB.open(alloc, live, source_options); + defer source.close(); + if (try source.vectorMigrationStatus(alloc)) |raw| { + defer alloc.free(raw); + var job = try std.json.parseFromSlice(contract.Job, alloc, raw, .{}); + defer job.deinit(); + if (std.mem.eql(u8, job.value.job_id, request.job_id) and job.value.mode == .offline and job.value.phase == .complete) { + if (!std.meta.eql(job.value.budget, request.budget)) return error.VectorMigrationIdempotencyConflict; + return .complete; + } + if (job.value.active() or job.value.published()) return error.VectorMigrationAlreadyExists; + } + if (source.table_storage.dense_embeddings != .primary_lsm) return error.VectorMigrationAlreadyPublished; + if (source.primary_backend != .lsm) return error.VectorStoreRequiresLocalSingleShardTable; + const cancelled_path = try std.fs.path.join(alloc, &.{ live, cancellation_file }); + defer alloc.free(cancelled_path); + if (try exists(io, cancelled_path)) { + var cancelled = try readJson(Cancellation, alloc, io, cancelled_path); + defer cancelled.deinit(); + if (std.mem.eql(u8, cancelled.value.request.job_id, request.job_id)) { + if (cancelled.value.version != 1 or !(contract.Admission{ .request = cancelled.value.request }).eql(.{ .request = request })) + return error.VectorMigrationIdempotencyConflict; + return error.VectorMigrationCancelled; + } + } + if (!try exists(io, fence_path)) { + const identity = try std.json.Stringify.valueAlloc(alloc, source.core.identity_namespace, .{}); + defer alloc.free(identity); + const entries = try inventory(alloc, io, live); + defer { + for (entries) |entry| alloc.free(entry.path); + alloc.free(entries); + } + var total: u64 = 0; + for (entries) |entry| total = try std.math.add(u64, total, entry.size); + // Reserve overlap for the shadow, source payloads and WAL/ANN + // rewriting. Individual copy/preparation steps recheck capacity. + if (total > request.budget.temporary_bytes / 4) return error.VectorMigrationTemporaryBudgetExceeded; + try capacity(live, request.budget, total * 4); + try save(alloc, io, fence_path, Fence{ .request = request, .identity = identity, .entries = entries, .source_bytes = total }); + try boundary(.fenced); + } + } + var fence = try readJson(Fence, alloc, io, fence_path); + defer fence.deinit(); + if (fence.value.version != 1 or !(contract.Admission{ .request = fence.value.request }).eql(.{ .request = request })) + return error.VectorMigrationIdempotencyConflict; + var staged = try transition.resumeStaging(request.job_id); + defer staged.deinit(); + const cursor_path = try std.fs.path.join(alloc, &.{ staged.path(), progress_file }); + defer alloc.free(cursor_path); + var progress = Progress{ .job_id = request.job_id }; + if (try exists(io, cursor_path)) { + var previous = try readJson(Progress, alloc, io, cursor_path); + defer previous.deinit(); + if (previous.value.version != 1 or !std.mem.eql(u8, previous.value.job_id, request.job_id)) return error.InvalidVectorMigrationState; + progress = previous.value; + progress.job_id = request.job_id; + } + var steps: usize = 0; + const buffer = try alloc.alloc(u8, @intCast(request.budget.batch_bytes)); + defer alloc.free(buffer); + const verify = try alloc.alloc(u8, buffer.len); + defer alloc.free(verify); + while (!progress.copy_complete) { + if (options.max_steps != 0 and steps >= options.max_steps) return .pending; + if (progress.file > fence.value.entries.len) return error.InvalidVectorMigrationState; + if (progress.file == fence.value.entries.len) { + progress.copy_complete = true; + try save(alloc, io, cursor_path, progress); + break; + } + const entry = fence.value.entries[progress.file]; + if (std.fs.path.isAbsolute(entry.path) or std.mem.indexOf(u8, entry.path, "..") != null or progress.offset > entry.size) + return error.InvalidVectorMigrationState; + const count: usize = @intCast(@min(buffer.len, entry.size - progress.offset)); + try capacity(live, request.budget, count); + try copyChunk(alloc, io, live, staged.path(), cursor_path, entry, &progress, buffer, verify); + steps += 1; + } + var target_options = try plan.optionsForStagedGeneration(&staged); + target_options.staged_generation = &staged; + target_options.exclusive_generation = null; + target_options.table_storage = null; + target_options.open_mode = .writer_no_replay; + target_options.start_index_workers = false; + target_options.start_optional_runtimes = false; + target_options.start_optional_runtime_workers = false; + // Offline opens do not inherit the server's shared block cache. Verification + // rereads current artifacts and candidate keys in adjacent compressed + // blocks; retain that bounded working set instead of rereading/decompressing + // a block for each point check. Preserve caller-supplied caches and budgets. + const resources = @import("resource_manager.zig"); + const lsm = @import("lsm_backend/mod.zig"); + var owned_manager: ?*resources.ResourceManager = null; + defer if (owned_manager) |manager| { + manager.deinit(alloc); + alloc.destroy(manager); + }; + var owned_cache: ?lsm.Cache = null; + defer if (owned_cache) |*cache| cache.deinit(); + const configured_cache = target_options.lsm_cache orelse switch (target_options.primary_backend) { + .lsm, .lsm_memory => |options_value| options_value.cache, + else => null, + }; + if (configured_cache == null) { + if (target_options.resource_manager == null) { + const manager = try alloc.create(resources.ResourceManager); + manager.* = resources.ResourceManager.init(db.standaloneResourceManagerOptions(alloc)); + owned_manager = manager; + target_options.resource_manager = manager; + } + owned_cache = try lsm.Cache.initFallible(alloc, 64 * 1024 * 1024); + target_options.lsm_cache = &owned_cache.?; + } + { + var target = try db.DB.open(alloc, staged.path(), target_options); + defer target.close(); + const identity = try std.json.Stringify.valueAlloc(alloc, target.core.identity_namespace, .{}); + defer alloc.free(identity); + if (!std.mem.eql(u8, identity, fence.value.identity)) return error.VectorMigrationIdentityMismatch; + try target.authorizeOfflineVectorMigrationCandidate(&staged); + // Replay committed index work from the physical copy without requiring + // an offline operator to instantiate external enrichment providers. + try target.catchUpPendingDerivedReplay(); + try target.startVectorMigration(request); + while (true) { + const raw = (try target.vectorMigrationStatus(alloc)).?; + defer alloc.free(raw); + if (options.progress_fn) |callback| try callback(options.progress_ctx, raw); + var job = try std.json.parseFromSlice(contract.Job, alloc, raw, .{}); + defer job.deinit(); + if (job.value.phase == .complete) break; + if (options.max_steps != 0 and steps >= options.max_steps) return .pending; + try capacity(live, request.budget, request.budget.batch_bytes * 4); + if (job.value.phase == .ready) try target.publishVectorMigration(request.job_id) else try target.advanceVectorMigration(request.job_id); + steps += 1; + } + try target.syncIndexes(true); + try target.core.store.runtime_store.sync(true); + try boundary(.candidate_complete); + } + try staged.seal(); + try boundary(.before_publication); + const outcome = try staged.publish(); + try boundary(.after_publication); + if (outcome == .durability_uncertain) return error.GenerationDurabilityUncertain; + return .complete; +} + +/// Cancel only an unpublished shadow. Reconciliation runs before interpreting +/// the live job, so an ambiguous exchange can never be mistaken for rollback. +pub fn cancel(alloc: Allocator, io: std.Io, root: []const u8, request: contract.Request, options: db.OpenOptions) !void { + try request.validate(); + if (request.mode != .offline) return error.InvalidVectorMigrationState; + var transition = try @import("db/generation_lifecycle.zig").beginProcessExclusiveWithRuntimeAndIo(root, options.backend_runtime, io); + defer transition.deinit(); + try transition.reconcilePublished(); + var open = options; + open.exclusive_generation = &transition; + open.staged_generation = null; + open.open_mode = .status_only; + open.start_index_workers = false; + open.start_optional_runtimes = false; + const identity = blk: { + var source = try db.DB.open(alloc, transition.path, open); + defer source.close(); + if (source.table_storage.dense_embeddings != .primary_lsm) return error.VectorMigrationAlreadyPublished; + if (try source.vectorMigrationStatus(alloc)) |raw| { + defer alloc.free(raw); + var job = try std.json.parseFromSlice(contract.Job, alloc, raw, .{}); + defer job.deinit(); + if (job.value.active() or job.value.published()) return error.VectorMigrationAlreadyExists; + } + break :blk try std.json.Stringify.valueAlloc(alloc, source.core.identity_namespace, .{}); + }; + defer alloc.free(identity); + const fence_path = try std.fs.path.join(alloc, &.{ transition.path, contract.offline_fence_file }); + defer alloc.free(fence_path); + var fence: ?std.json.Parsed(Fence) = if (try exists(io, fence_path)) try readJson(Fence, alloc, io, fence_path) else null; + defer if (fence) |*value| value.deinit(); + if (fence) |value| { + if (value.value.version != 1 or !(contract.Admission{ .request = value.value.request }).eql(.{ .request = request })) + return error.VectorMigrationIdempotencyConflict; + if (!std.mem.eql(u8, value.value.identity, identity)) return error.VectorMigrationIdentityMismatch; + } + const cancelled_path = try std.fs.path.join(alloc, &.{ transition.path, cancellation_file }); + defer alloc.free(cancelled_path); + if (try exists(io, cancelled_path)) { + var cancelled = try readJson(Cancellation, alloc, io, cancelled_path); + defer cancelled.deinit(); + if (std.mem.eql(u8, cancelled.value.request.job_id, request.job_id)) { + if (cancelled.value.version != 1 or !(contract.Admission{ .request = cancelled.value.request }).eql(.{ .request = request })) + return error.VectorMigrationIdempotencyConflict; + if (!std.mem.eql(u8, cancelled.value.identity, identity)) return error.VectorMigrationIdentityMismatch; + if (fence == null) return; + } + } + if (fence != null) { + var staged = try transition.resumeStaging(request.job_id); + defer staged.deinit(); + try std.Io.Dir.cwd().deleteTree(io, staged.path()); + try fs.syncDirPortable(io, std.fs.path.dirname(staged.path()).?); + } + // Keep a receipt before releasing admission. A lost cancellation response + // must not let the same ID silently start a new physical migration, even + // when admission failed before creating the copy fence or shadow root. + try save(alloc, io, cancelled_path, Cancellation{ .request = request, .identity = identity }); + if (fence != null) { + try std.Io.Dir.cwd().deleteFile(io, fence_path); + try fs.syncDirPortable(io, transition.path); + } +} + +// VoprIo normally syncs the entire namespace on any directory fsync. Use a +// stricter adapter here: each fsync persists only that directory's immediate +// entries. File bytes and crash/reopen still use VoprIo's durability model. +const CopyCrashTest = struct { + var fail_directory: ?[]const u8 = null; + var stop_at: ?Boundary = null; + + fn sync(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.SyncError!void { + const sim: *@import("vopr").vopr_io.VoprIo = @ptrCast(@alignCast(userdata.?)); + const handle = sim.files.handles.get(file.handle) orelse return error.AccessDenied; + if (!handle.directory) return sim.files.syncFile(file); + if (fail_directory) |path| if (std.mem.eql(u8, handle.node.path, path)) return error.InputOutput; + for (sim.files.nodes.items) |node| { + const parent = std.fs.path.dirname(node.path) orelse continue; + if (!std.mem.eql(u8, parent, handle.node.path)) continue; + const durable_path = sim.files.allocator.dupe(u8, node.path) catch return error.AccessDenied; + sim.files.allocator.free(node.durable_path); + node.durable_path = durable_path; + node.durable_exists = node.exists; + } + } + + fn stop(point: Boundary) !void { + if (stop_at == point) return error.InjectedMigrationCrash; + } + + fn crash(sim: *@import("vopr").vopr_io.VoprIo) !void { + // A child inode cannot remain reachable through an unpersisted parent + // link. VoprIo's flat path map needs that namespace rule modeled too. + var removed = true; + while (removed) { + removed = false; + for (sim.files.nodes.items) |node| { + if (!node.durable_exists or std.mem.eql(u8, node.durable_path, "/")) continue; + const parent = std.fs.path.dirname(node.durable_path).?; + const reachable = for (sim.files.nodes.items) |ancestor| { + if (ancestor.durable_exists and std.mem.eql(u8, ancestor.durable_path, parent)) break true; + } else false; + if (!reachable) { + node.durable_exists = false; + removed = true; + } + } + } + try sim.crashFileSystem(); + } +}; + +test "source vector migration copy crash model loses unsynced intermediate directories" { + const alloc = std.testing.allocator; + var sim = try @import("vopr").vopr_io.VoprIo.init(.{}); + defer sim.deinit(); + var vtable = sim.io().vtable.*; + vtable.fileSync = CopyCrashTest.sync; + const io = std.Io{ .userdata = &sim, .vtable = &vtable }; + try fs.createDirPathPortable(io, "/stage/indexes/model/runs"); + var file = try std.Io.Dir.cwd().createFile(io, "/stage/indexes/model/runs/data", .{}); + try file.writeStreamingAll(io, "payload"); + try file.sync(io); + file.close(io); + // Reproduce the old ordering, with a durable shadow root and cursor. + try fs.syncDirPortable(io, "/"); + try fs.syncDirPortable(io, "/stage/indexes/model/runs"); + try save(alloc, io, "/stage/" ++ progress_file, Progress{ .job_id = "copy", .file = 1 }); + try CopyCrashTest.crash(&sim); + try std.testing.expect(try exists(io, "/stage/" ++ progress_file)); + try std.testing.expect(!try exists(io, "/stage/indexes/model")); + try std.testing.expect(!try exists(io, "/stage/indexes/model/runs/data")); +} + +test "source vector migration copy resumes durable chunks after directory sync failure and power loss" { + const alloc = std.testing.allocator; + const bytes = "first-second-third"; + inline for (.{ "empty", "data" }) |name| { + inline for (.{ "parent_failure", "parent_retry", "chunk_synced", "cursor_synced" }) |fault| { + var sim = try @import("vopr").vopr_io.VoprIo.init(.{}); + defer sim.deinit(); + var vtable = sim.io().vtable.*; + vtable.fileSync = CopyCrashTest.sync; + const io = std.Io{ .userdata = &sim, .vtable = &vtable }; + try fs.createDirPathPortable(io, "/live/indexes/model/runs"); + const expected = if (std.mem.eql(u8, name, "empty")) "" else bytes; + var source = try std.Io.Dir.cwd().createFile(io, "/live/indexes/model/runs/" ++ name, .{}); + try source.writeStreamingAll(io, expected); + try source.sync(io); + source.close(io); + try sim.files.syncNamespace(); // The fenced source predates copying. + try fs.createDirPathPortable(io, "/stage"); + const entry = Entry{ .path = "indexes/model/runs/" ++ name, .size = expected.len }; + var progress = Progress{ .job_id = "copy" }; + var buffer: [6]u8 = undefined; + var verify: [6]u8 = undefined; + const fail_parent = std.mem.startsWith(u8, fault, "parent_"); + CopyCrashTest.fail_directory = if (fail_parent) "/stage/indexes" else null; + CopyCrashTest.stop_at = if (std.mem.eql(u8, fault, "chunk_synced")) .chunk_synced else if (std.mem.eql(u8, fault, "cursor_synced")) .cursor_synced else null; + test_boundary = CopyCrashTest.stop; + defer { + test_boundary = null; + CopyCrashTest.fail_directory = null; + CopyCrashTest.stop_at = null; + } + try std.testing.expectError(if (fail_parent) error.InputOutput else error.InjectedMigrationCrash, copyChunk(alloc, io, "/live", "/stage", "/stage/" ++ progress_file, entry, &progress, &buffer, &verify)); + CopyCrashTest.fail_directory = null; + CopyCrashTest.stop_at = null; + // Also retry without a crash: already-existing directories from a + // failed attempt must still be synced before publishing progress. + if (std.mem.eql(u8, fault, "parent_retry")) + try copyChunk(alloc, io, "/live", "/stage", "/stage/" ++ progress_file, entry, &progress, &buffer, &verify); + // Discard all volatile file data, directory entries and handles. + // Reload the acknowledged cursor after every subsequent chunk too. + while (true) { + try CopyCrashTest.crash(&sim); + progress = .{ .job_id = "copy" }; + if (try exists(io, "/stage/" ++ progress_file)) { + var saved = try readJson(Progress, alloc, io, "/stage/" ++ progress_file); + defer saved.deinit(); + progress = saved.value; + progress.job_id = "copy"; + inline for (.{ "/stage", "/stage/indexes", "/stage/indexes/model", "/stage/indexes/model/runs" }) |parent| { + try std.testing.expect(try exists(io, parent)); + } + const copied = try std.Io.Dir.cwd().readFileAlloc(io, "/stage/indexes/model/runs/" ++ name, alloc, .limited(1024)); + defer alloc.free(copied); + const acknowledged = if (progress.file == 1) expected.len else progress.offset; + try std.testing.expectEqualSlices(u8, expected[0..@intCast(acknowledged)], copied[0..@intCast(acknowledged)]); + if (progress.file == 1) try std.testing.expectEqualSlices(u8, expected, copied); + } + if (progress.file == 1) break; + try copyChunk(alloc, io, "/live", "/stage", "/stage/" ++ progress_file, entry, &progress, &buffer, &verify); + } + try sim.ensureNoCapabilityViolation(); + } + } +} diff --git a/zig/pkg/antfly/src/storage/vector_payload_store.zig b/zig/pkg/antfly/src/storage/vector_payload_store.zig index 8575358752..c4bf647595 100644 --- a/zig/pkg/antfly/src/storage/vector_payload_store.zig +++ b/zig/pkg/antfly/src/storage/vector_payload_store.zig @@ -73,6 +73,12 @@ pub const SegmentSizing = struct { }; pub const Store = struct { + /// Candidate references are protected by the durable migration job until + /// all primary rows and their final reference coverage have been verified. + migration_retention: std.atomic.Value(bool) = .init(false), + migration_disk_reserve: std.atomic.Value(u64) = .init(0), + migration_temporary_limit: std.atomic.Value(u64) = .init(0), + alloc: Allocator, mutex: std.atomic.Mutex = .unlocked, publication_mutex: std.atomic.Mutex = .unlocked, @@ -261,6 +267,20 @@ pub const Store = struct { self.publication_mutex.unlock(); } + pub fn beginMigrationRetention(self: *Store) !void { + self.lock(); + defer self.mutex.unlock(); + // A collector may be staging outside the mutex. Admit only after its + // owner retires; atomic publication then prevents the next collector. + if (self.marking != null or self.collection != null or self.retiring != null or self.checkpoint_running) + return error.StorageBusy; + self.migration_retention.store(true, .release); + } + + pub fn setMigrationRetention(self: *Store, active: bool) void { + self.migration_retention.store(active, .release); + } + pub fn poison(self: *Store) void { self.lock(); defer self.mutex.unlock(); @@ -1632,6 +1652,13 @@ pub const Store = struct { fn prepareBatch(self: *Store, prepared: []const payload.Prepared) !void { if (self.read_only) return error.ReadOnly; + const reserve = self.migration_disk_reserve.load(.acquire); + if (reserve != 0) { + var bytes: u64 = 0; + for (prepared) |item| bytes +|= @as(u64, item.artifact.len) *| 8; + const capacity = try @import("antfly_platform").filesystem.capacity(self.opened.store.root_dir); + if (capacity.available_bytes < reserve +| bytes) return error.VectorMigrationDiskReserve; + } // Decode independent artifact envelopes before entering source writer // exclusion. Each request has its own allocator reservation. var local_budget: ?resources.BudgetedAllocator = if (self.group_commit and self.preparation_manager != null) @@ -1651,6 +1678,12 @@ pub const Store = struct { defer self.mutex.unlock(); self.waitWriteAdmissionLocked(); if (self.poisoned) return error.VectorPayloadStorePoisoned; + const limit = self.migration_temporary_limit.load(.acquire); + if (limit != 0) { + var retained = self.stats.retained_payload_bytes; + for (prepared) |item| retained +|= item.artifact.len; + if (retained > limit / 8) return error.VectorMigrationTemporaryBudgetExceeded; + } const started = time.monotonicNs(); self.stats.prepare_lock_wait_ns += started -| lock_started; if (self.group_commit) self.stats.decode_outside_lock_ns += lock_started -| decode_started; @@ -2233,6 +2266,7 @@ pub const Store = struct { fn collectStepLocked(self: *Store, primary: *erased.Store, budget_bytes: u64, background: bool) !bool { if (self.read_only) return error.ReadOnly; if (self.poisoned) return error.VectorPayloadStorePoisoned; + if (self.migration_retention.load(.acquire)) return false; if (self.checkpoint_running or self.retiring != null) { self.stats.collection_deferrals += 1; return false; diff --git a/zig/scripts/qualify_vector_migration.py b/zig/scripts/qualify_vector_migration.py new file mode 100644 index 0000000000..04db1df736 --- /dev/null +++ b/zig/scripts/qualify_vector_migration.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python3 +# Copyright 2026 Antfly, Inc. +# +# Licensed under the Elastic License 2.0 (ELv2); you may not use this file +# except in compliance with the Elastic License 2.0. You may obtain a copy of +# the Elastic License 2.0 at +# +# https://www.antfly.io/licensing/ELv2-license +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the Elastic License 2.0 is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# Elastic License 2.0 for the specific language governing permissions and +# limitations. + +"""Compare fresh and migrated source ownership on one host, sequentially. + +Requires numpy and requests. Retains databases, inputs, logs and receipts under +--root; never deletes existing runs. Run 50K before 1M. This is a qualification +screen, not a substitute for alternating repeated VectorDBBench promotion arms. +""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import threading +import time + +import numpy as np +import requests + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2) + "\n") + + +def disk_bytes(root): + return sum(p.stat().st_blocks * 512 for p in root.rglob("*") if p.is_file()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--rows", type=int, default=50_000) + parser.add_argument("--dimensions", type=int, default=768) + parser.add_argument( + "--order", + nargs="+", + choices=("fresh", "online", "offline"), + default=["fresh", "online", "offline"], + ) + parser.add_argument("--query-count", type=int, default=4096) + parser.add_argument("--churn", type=int, default=1000) + args = parser.parse_args() + args.root = args.root.resolve() + args.binary = args.binary.resolve(strict=True) + args.root.mkdir(parents=True, exist_ok=False) + if args.rows < 4 * args.churn or args.dimensions < 3: + parser.error( + "rows must be at least four times churn; dimensions must be at least three" + ) + env = os.environ.copy() + overrides = { + k: v + for k, v in env.items() + if k.startswith(("ANTFLY_SOURCE_VECTOR_", "ANTFLY_DENSE_", "ANTFLY_HBC_")) + } + if overrides: + raise SystemExit( + f"clear experimental overrides before qualification: {sorted(overrides)}" + ) + write_json( + args.root / "configuration.json", + { + **{k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items()}, + "binary_sha256": hashlib.file_digest( + args.binary.open("rb"), "sha256" + ).hexdigest(), + "metric": "cosine", + "seed": 728, + }, + ) + vectors = np.memmap( + args.root / "input.f32", + dtype=np.float32, + mode="w+", + shape=(args.rows, args.dimensions), + ) + rng = np.random.default_rng(728) + for start in range(0, args.rows, 4096): + block = rng.standard_normal( + (min(4096, args.rows - start), args.dimensions), dtype=np.float32 + ) + block /= np.linalg.norm(block, axis=1, keepdims=True) + vectors[start : start + len(block)] = block + vectors.flush() + queries = rng.standard_normal((32, args.dimensions), dtype=np.float32) + queries /= np.linalg.norm(queries, axis=1, keepdims=True) + # Compute exact post-churn neighbors with bounded score scratch. Updates + # negate the first range; the following range is deleted in every arm. + best_scores = np.full((len(queries), 10), -np.inf, dtype=np.float32) + best_ids = np.zeros((len(queries), 10), dtype=np.int64) + for start in range(0, args.rows, 8192): + block = np.array(vectors[start : start + 8192]) + ids = np.arange(start, start + len(block)) + block[ids < args.churn] *= -1 + scores = queries @ block.T + scores[:, (ids >= args.churn) & (ids < 2 * args.churn)] = -np.inf + choices = np.broadcast_to(ids, scores.shape) + scores = np.concatenate((best_scores, scores), axis=1) + choices = np.concatenate((best_ids, choices), axis=1) + top = np.argpartition(scores, -10, axis=1)[:, -10:] + best_scores = np.take_along_axis(scores, top, axis=1) + best_ids = np.take_along_axis(choices, top, axis=1) + truth = [{f"doc:{i:09d}" for i in row} for row in best_ids] + write_json(args.root / "ground_truth.json", [sorted(row) for row in truth]) + results = [] + + for ordinal, mode in enumerate(args.order): + arm = args.root / f"{ordinal}-{mode}" + arm.mkdir() + data = arm / "data" + data.mkdir() + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + url = f"http://127.0.0.1:{port}/db/v1" + table = "migration_qualification" + process = None + log = (arm / "server.log").open("w") + peak = [0] + monitoring = threading.Event() + + def sample(): + while not monitoring.wait(0.5): + proc = process + if proc is not None and proc.poll() is None: + raw = subprocess.run( + ["ps", "-o", "rss=", "-p", str(proc.pid)], + capture_output=True, + text=True, + ) + if raw.returncode == 0 and raw.stdout.strip(): + peak[0] = max(peak[0], int(raw.stdout.strip()) * 1024) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + session = requests.Session() + + def api(method, path, body=None): + response = session.request(method, url + path, json=body, timeout=300) + if response.status_code >= 400: + raise RuntimeError( + f"{method} {path}: {response.status_code} {response.text[:2000]}" + ) + return response.json() if response.content else {} + + def start_server(): + nonlocal process, session + session.close() + session = requests.Session() + process = subprocess.Popen( + [ + str(args.binary), + "standalone", + "--data-dir", + str(data), + "--port", + str(port), + "--health", + "false", + ], + stdout=log, + stderr=subprocess.STDOUT, + env=env, + ) + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"server exited: {arm / 'server.log'}") + try: + api("GET", "/tables") + return + except requests.ConnectionError: + time.sleep(0.1) + raise TimeoutError("server start") + + def stop_server(): + nonlocal process + if process is not None and process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + raise + process = None + + def docs(start, count, changed=False): + block = np.array(vectors[start : start + count]) + if changed: + block *= -1 + return { + f"doc:{i:09d}": { + "text": f"document {i} migration qualification", + "_embeddings": {"model": vector.tolist()}, + } + for i, vector in zip(range(start, start + count), block) + } + + def ready(count): + deadline = time.monotonic() + 1800 + while time.monotonic() < deadline: + state = api("GET", f"/tables/{table}/indexes/model").get("status", {}) + if ( + state.get("total_indexed") == count + and state.get("query_visible_doc_count") == count + ): + return state + time.sleep(0.25) + raise TimeoutError(f"readiness: {state}") + + def query(vector): + return api( + "POST", + f"/tables/{table}/query", + { + "embeddings": {"model": vector.tolist()}, + "indexes": ["model"], + "limit": 10, + }, + ) + + def churn(): + for first in range(0, args.churn, 256): + api( + "POST", + f"/tables/{table}/batch", + { + "inserts": docs(first, min(256, args.churn - first), True), + "sync_level": "write", + }, + ) + for first in range(args.churn, 2 * args.churn, 256): + api( + "POST", + f"/tables/{table}/batch", + { + "deletes": [ + f"doc:{i:09d}" + for i in range(first, min(first + 256, 2 * args.churn)) + ], + "sync_level": "write", + }, + ) + + result = {"mode": mode, "rows": args.rows, "dimensions": args.dimensions} + print(f"starting {mode} rows={args.rows}", flush=True) + try: + start_server() + api( + "POST", + f"/tables/{table}", + { + "num_shards": 1, + "storage": { + "dense_embeddings": "vector_store" + if mode == "fresh" + else "primary_lsm" + }, + }, + ) + api( + "POST", + f"/tables/{table}/indexes/model", + { + "type": "embeddings", + "external": True, + "dimension": args.dimensions, + "distance_metric": "cosine", + }, + ) + started = time.monotonic() + for first in range(0, args.rows, 256): + api( + "POST", + f"/tables/{table}/batch", + { + "inserts": docs(first, min(256, args.rows - first)), + "sync_level": "write", + }, + ) + if first % 25000 < 256: + print(f"{mode} loaded {first}/{args.rows}", flush=True) + result["ingest_seconds"] = time.monotonic() - started + result["initial_readiness"] = ready(args.rows) + result["initial_ready_seconds"] = time.monotonic() - started + result["before"] = api("GET", f"/tables/{table}") + started = time.monotonic() + if mode == "online": + migration_request = { + "job_id": "qualification", + "target": "vector_store", + "budget": {"batch_rows": 1024, "batch_bytes": 4194304}, + } + state = api( + "POST", + f"/tables/{table}/storage/migrations", + migration_request, + ) + # Fixed mutations after capture admission, including deletions. + churn() + ready(args.rows - args.churn) + with (arm / "migration.jsonl").open("w") as progress: + for step in range(100000): + progress.write(json.dumps(state) + "\n") + progress.flush() + if state["phase"] == "complete": + break + if step % 16 == 0: + query(queries[step % len(queries)]) + state = api( + "POST", + f"/tables/{table}/storage/migrations/qualification", + { + "action": "publish" + if state["phase"] == "ready" + else "step", + }, + ) + else: + raise TimeoutError("online migration steps") + result["migration"] = state + else: + churn() + ready(args.rows - args.churn) + if mode == "offline": + stop_server() + with (arm / "migration.log").open("w") as progress: + process = subprocess.Popen( + [ + str(args.binary), + "storage", + "migrate", + "--to", + "vector-store", + "--catalog", + str(data / "metadata/local-metadata.json"), + "--replica-root", + str(data / "data/replicas"), + "--table", + table, + "--job", + "qualification", + ], + stdout=progress, + stderr=subprocess.STDOUT, + ) + code = process.wait(timeout=3600) + if code != 0: + raise RuntimeError( + f"offline migration exited {code}: {arm / 'migration.log'}" + ) + process = None + start_server() + result["migration_and_churn_seconds"] = time.monotonic() - started + ready(args.rows - args.churn) + # Capture file/flush debt at the start of queries, before later + # maintenance or restart can hide migration-induced tiny runs. + result["before_queries"] = api("GET", f"/tables/{table}") + write_json(arm / "before-queries.json", result) + recalls = [] + for vector, expected in zip(queries, truth): + response = query(vector) + actual = { + hit["_id"] for hit in response["responses"][0]["hits"]["hits"] + } + if any( + args.churn <= int(key.removeprefix("doc:")) < 2 * args.churn + for key in actual + ): + raise AssertionError("deleted document returned after migration") + recalls.append(len(actual & expected) / 10) + result["recall_at_10"] = float(np.mean(recalls)) + payloads = [ + json.dumps( + { + "embeddings": {"model": vector.tolist()}, + "indexes": ["model"], + "limit": 10, + } + ) + for vector in queries + ] + local = threading.local() + full_text_payload = json.dumps( + { + "full_text_search": {"field": "text", "match": "qualification"}, + "limit": 10, + } + ) + text_result = api( + "POST", f"/tables/{table}/query", json.loads(full_text_payload) + ) + if not text_result["responses"][0]["hits"]["hits"]: + raise AssertionError("full-text corpus missing after migration") + + def measured(index): + if not hasattr(local, "session"): + local.session = requests.Session() + begin = time.monotonic() + response = local.session.post( + url + f"/tables/{table}/query", + data=active_payloads[index % len(active_payloads)], + headers={"Content-Type": "application/json"}, + timeout=120, + ) + response.raise_for_status() + return time.monotonic() - begin + + def measure_workload(label): + cells = [] + for concurrency in (1, 8, 32): + with ThreadPoolExecutor(max_workers=concurrency) as pool: + list(pool.map(measured, range(128))) + begin = time.monotonic() + latencies = list(pool.map(measured, range(args.query_count))) + seconds = time.monotonic() - begin + cells.append( + { + "concurrency": concurrency, + "qps": args.query_count / seconds, + "p50_ms": float(np.percentile(latencies, 50) * 1000), + "p99_ms": float(np.percentile(latencies, 99) * 1000), + } + ) + write_json(arm / f"{label}.json", cells) + print( + f"{mode} {label} c={concurrency} qps={cells[-1]['qps']:.1f}", + flush=True, + ) + return cells + + for measurement, active_payloads in ( + ("queries", payloads), + ("full_text_queries", [full_text_payload]), + ( + "mixed_queries", + [ + item + for payload in payloads + for item in (payload, full_text_payload) + ], + ), + ): + result[measurement] = measure_workload(measurement) + result["after"] = api("GET", f"/tables/{table}") + stop_server() + started = time.monotonic() + start_server() + ready(args.rows - args.churn) + query(queries[0]) + result["warm_restart_seconds"] = time.monotonic() - started + result["restart"] = api("GET", f"/tables/{table}") + active_payloads = payloads + result["restart_queries"] = measure_workload("restart_queries") + reclaim_started = time.monotonic() + deadline = reclaim_started + 180 + result["source_reclamation_complete"] = False + while time.monotonic() < deadline: + state = api("GET", f"/tables/{table}") + stats = state.get("storage_status", {}).get("source_vectors", {}) + if ( + stats + and stats.get("retained_payloads") == args.rows - args.churn + and stats.get("collection_pending_bytes") == 0 + ): + result["source_reclamation_complete"] = True + break + time.sleep(1) + result["source_reclamation_seconds"] = time.monotonic() - reclaim_started + result["reclamation"] = state + result["peak_rss_bytes"] = peak[0] + stop_server() + result["allocated_disk_bytes"] = disk_bytes(data) + write_json(arm / "result.json", result) + if not result["source_reclamation_complete"]: + raise AssertionError( + "source reclamation did not reach the live payload count" + ) + # This fixed corpus has tiny documents relative to 768-D vectors. + # Catch inline-sized primary retention before qualifying larger arms; + # the exact table/byte measurements remain available in result.json. + if mode != "fresh" and args.dimensions >= 512: + primary_bytes = state["storage_status"]["lsm"]["run_bytes"] + if primary_bytes > args.rows * args.dimensions * 4 // 2: + raise AssertionError( + f"primary SSTables still occupy inline-payload-sized space: {primary_bytes}" + ) + results.append(result) + write_json(args.root / "results.json", results) + print( + f"finished {mode}: recall={result['recall_at_10']:.3f} qps={result['queries']} disk={result['allocated_disk_bytes']}", + flush=True, + ) + except BaseException as error: + result["error"] = repr(error) + write_json(arm / "failure.json", result) + raise + finally: + stop_server() + monitoring.set() + sampler.join() + log.close() + session.close() + + +if __name__ == "__main__": + main()