diff --git a/cmd/cloudctl/README.en.md b/cmd/cloudctl/README.en.md index 2f4442ea4..42e9b65f5 100644 --- a/cmd/cloudctl/README.en.md +++ b/cmd/cloudctl/README.en.md @@ -2,7 +2,7 @@ [中文](README.md) | [English](README.en.md) -`cloudctl` is the restricted operational management tool for Ora Cloud. It runs with database operator credentials outside the public API path to manage database migrations and bootstrap initial tenants and administrators. +`cloudctl` is the restricted operational management tool for Ora Cloud. It runs with database operator credentials outside the public API path to manage database migrations, bootstrap initial tenants and administrators, and explicitly dispose of residual resources. ## Commands and responsibilities @@ -26,9 +26,41 @@ cloudctl -config -command bootstrap -name '' -source ' residual list --tenant-id --project-id --limit 100 +cloudctl -config residual get --tenant-id --workspace-id +cloudctl -config residual get --tenant-id --intent-id +``` + +- `list` intersects optional `--project-id`, `--run-id`, `--workspace-id`, `--operation-id`, and `--reclaim-id` filters, with `--limit` (default 100) and `--after`. It returns `items/nextCursor`; operation filtering can match either the original failed D8 delete or the currently bound deletion operation. +- `get` accepts one root/child `--intent-id`, or an exact `--run-id`, `--workspace-id`, or `--reclaim-id`. Other resource IDs may narrow a resource selector; an intent selector cannot be mixed with resource selectors. Queries return audit history, target evidence, and `openTargets` for external confirmation. + +Termination and data deletion are separate commands, each requiring a reason, idempotency key, and affirmative confirmation. `terminate` requires at least a Run or Workspace; all additional IDs must identify the same original resource. D8 also requires the original failed delete `--operation-id`. Project/tenant selectors cannot authorize batch writes. + +```sh +cloudctl -config residual terminate --tenant-id --run-id --workspace-id --reason 'operator abandoned execution' --key --impact-confirmed +cloudctl -config residual delete-data --tenant-id --intent-id --reason 'delete retained data' --key --data-delete-confirmed +``` + +`terminate` registers termination authorization and retains data by default; acceptance does not mean sandboxes have terminated. The root intent succeeds with `terminated_data_retained` only after every target has genuine termination and late-ensure fencing confirmation. A later `delete-data` accepts only that completed root intent and separately registers a child intent and one fixed deletion operation; `released` requires actual deletion confirmation. It rejects bare Workspace deletion and does not substitute external confirmation for real termination evidence. + +An unfinished intent whose disposal cannot be confirmed by the system may explicitly transfer its remaining responsibility. Read `openTargets` from `get` first and supply one `--external-fact` per unfinished target. Keys must be the exact `sandbox:` or `data:` values; values reference a ticket or another external disposal fact. Duplicate targets, empty references or explanations, and missing responsibility confirmation are rejected; core also checks for omitted targets. + +```sh +cloudctl -config residual confirm-external --tenant-id --intent-id --reason 'external disposal' --key --evidence-note 'host decommissioned; data responsibility accepted' --external-fact 'sandbox:=ticket-123' --external-fact 'data:=ticket-456' --responsibility-transferred +``` + +External confirmation accepts only an existing unfinished intent and records a distinct `externally_confirmed` disposition. It manufactures no `terminated`, `lateEnsureFenced`, or `removed` physical evidence and does not authorize reuse of the same volume. Write keys are tenant-wide across actions: identical input returns current progress for the same responsibility; changing the action, target, reason, or confirmation under the same key returns `idempotency_conflict`. + ## Boundaries and invariants -- **Deployment boundary**: `cloudctl` is not exposed via HTTP and is not invoked by the server daemon. It requires direct network access to PostgreSQL with schema migration permissions. +- **Deployment boundary**: `cloudctl` is not exposed via HTTP and is not invoked by the server daemon. It requires direct PostgreSQL access; only migration commands need schema migration privileges. Residual management adds no public tenant admin/member permissions, and public HTTP routes continue to reject hidden Run Workspaces. - **Structured output**: All command results are output as structured JSON to `stdout`, and errors to `stderr`, enabling integration with CI/CD deployment pipelines. +- **Errors and credentials**: Residual commands preserve stable core Fault codes. Unknown infrastructure errors produce only `internal_error`; configuration or connection failures use safe generic codes without connection strings or passwords. Invalid arguments produce `invalid_cli_arguments`; `--impact-confirmed=false` does not confirm an action. See [cmd overview](../README.en.md), [Database migrations](../../internal/core/migrations/README.en.md), and [Internal credentials](../../docs/identity/authentication.en.md). diff --git a/cmd/cloudctl/README.md b/cmd/cloudctl/README.md index dc225d560..04645b38a 100644 --- a/cmd/cloudctl/README.md +++ b/cmd/cloudctl/README.md @@ -2,7 +2,7 @@ [中文](README.md) | [English](README.en.md) -`cloudctl` 是 Ora Cloud 受限的运维管理工具。它在公共 API 调用路径之外运行，使用数据库操作员凭据管理数据库迁移以及初始租户和管理员的引导配置。 +`cloudctl` 是 Ora Cloud 受限的运维管理工具。它在公共 API 调用路径之外运行，使用数据库操作员凭据管理数据库迁移、初始租户和管理员的引导配置，以及残留资源的显式处置。 ## 命令与职责 @@ -26,9 +26,41 @@ cloudctl -config -command bootstrap -name '' -source ' residual list --tenant-id --project-id --limit 100 +cloudctl -config residual get --tenant-id --workspace-id +cloudctl -config residual get --tenant-id --intent-id +``` + +- `list` 支持 `--project-id`、`--run-id`、`--workspace-id`、`--operation-id`、`--reclaim-id` 相交过滤，以及 `--limit`（默认 100）和 `--after`。返回 `items/nextCursor`；operation 过滤可命中 D8 原 failed delete 或当前绑定的删除操作。 +- `get` 接受一个根/从属 `--intent-id`，或精确的 `--run-id`、`--workspace-id`、`--reclaim-id`。原资源选择器可用其他资源 ID 进一步限制范围；意图选择器不能混用资源选择器。查询返回审计、目标证据和 `openTargets`，供外部确认引用。 + +终止和删数据是两次独立命令，各自需要 reason、幂等 key 和真实确认。`terminate` 至少指定 Run 或 Workspace；提供的其他 ID 必须属于同一个原资源。D8 还必须提供原 failed delete 的 `--operation-id`。Project/tenant 不能作为批量写入范围。 + +```sh +cloudctl -config residual terminate --tenant-id --run-id --workspace-id --reason 'operator abandoned execution' --key --impact-confirmed +cloudctl -config residual delete-data --tenant-id --intent-id --reason 'delete retained data' --key --data-delete-confirmed +``` + +`terminate` 登记终止授权，默认保留数据；命令受理不代表沙盒已经终止。只有每个目标都有真实终止及迟到 ensure 已被封闭的双确认，根意图才可成功并报告 `terminated_data_retained`。后续 `delete-data` 只接受这个已完成的根意图，单独登记从属意图和唯一删除操作；实际删除确认后才能报告 `released`。它不接受裸 Workspace 删除，也不以外部确认代替真实终止证据。 + +无法由系统确认的未完成意图可通过下列命令显式转移剩余责任。先从 `get` 读取 `openTargets`，为每个未完成目标各提供一次 `--external-fact`；键必须是原样的 `sandbox:` 或 `data:`，值是工单或其他外部处置事实引用。重复目标、空引用、空说明及未确认责任转移会被拒绝；core 继续核对是否遗漏目标。 + +```sh +cloudctl -config residual confirm-external --tenant-id --intent-id --reason 'external disposal' --key --evidence-note 'host decommissioned; data responsibility accepted' --external-fact 'sandbox:=ticket-123' --external-fact 'data:=ticket-456' --responsibility-transferred +``` + +外部确认只接受已存在且未完成的意图，保存独立的 `externally_confirmed` 处置，不制造 `terminated`、`lateEnsureFenced` 或 `removed` 物理证据，不授权同卷复用。幂等 key 在租户内跨写动作唯一：相同输入返回同一责任的当前进度；同键改动作、目标、reason 或确认返回 `idempotency_conflict`。 + ## 边界与不变量 -- **部署边界**：`cloudctl` 不通过 HTTP 暴露，也不由 server 守护进程调用。它必须能够直接连接 PostgreSQL，并拥有 Schema 迁移权限。 +- **部署边界**：`cloudctl` 不通过 HTTP 暴露，也不由 server 守护进程调用。它必须能够直接连接 PostgreSQL；只有迁移命令需要 Schema 迁移权限。残留管理不新增 tenant admin/member 的公共权限，隐藏 Run Workspace 的公开 HTTP 入口继续拒绝访问。 - **结构化输出**：所有命令执行结果均以结构化 JSON 格式输出到 `stdout`，错误信息输出到 `stderr`，便于与 CI/CD 自动化部署流水线集成。 +- **错误与凭据**：残留命令保留 core 的稳定 Fault code；未知基础设施错误仅输出 `internal_error`，配置或连接失败输出安全的通用 code，不输出连接串或密码。参数错误为 `invalid_cli_arguments`；`--impact-confirmed=false` 等同于未确认。 参见 [cmd 入口总览](../README.md)、[数据库迁移目录](../../internal/core/migrations/README.md) 与 [内部凭据与服务信任](../../docs/identity/authentication.md)。 diff --git a/cmd/cloudctl/main.go b/cmd/cloudctl/main.go index 4b99b622b..05735a5ab 100644 --- a/cmd/cloudctl/main.go +++ b/cmd/cloudctl/main.go @@ -4,8 +4,10 @@ package main import ( "context" "encoding/json" + "errors" "flag" "fmt" + "io" "os" "github.com/wanglongan587/cloud/internal/config" @@ -21,39 +23,76 @@ func main() { } func run() error { - configFile := flag.String("config", "", "configuration file") - command := flag.String("command", "", "migrate | bootstrap") - name := flag.String("name", "", "tenant name") - source := flag.String("source", "", "stable identity namespace") - subject := flag.String("subject", "", "identity subject") - display := flag.String("display-name", "", "display name") - flag.Parse() - cfg, e := config.Load(*configFile) + return runArgs(context.Background(), os.Args[1:], os.Stdout) +} + +type commandOptions struct { + configFile, command, name, source, subject, display string + residual core.ResidualManagementRequest +} + +func parseCommand(args []string) (commandOptions, error) { + var opts commandOptions + flags := flag.NewFlagSet("cloudctl", flag.ContinueOnError) + flags.SetOutput(io.Discard) + flags.StringVar(&opts.configFile, "config", "", "configuration file") + flags.StringVar(&opts.command, "command", "", "migrate | bootstrap") + flags.StringVar(&opts.name, "name", "", "tenant name") + flags.StringVar(&opts.source, "source", "", "stable identity namespace") + flags.StringVar(&opts.subject, "subject", "", "identity subject") + flags.StringVar(&opts.display, "display-name", "", "display name") + if err := flags.Parse(args); err != nil { + return opts, invalidArguments() + } + if rest := flags.Args(); len(rest) != 0 { + if rest[0] != "residual" || opts.command != "" || opts.name != "" || opts.source != "" || opts.subject != "" || opts.display != "" { + return opts, invalidArguments() + } + request, err := parseResidual(rest[1:]) + if err != nil { + return opts, err + } + opts.command, opts.residual = "residual", request + } + if opts.command != "migrate" && opts.command != "bootstrap" && (opts.command != "residual" || opts.residual.Action == "") { + return opts, invalidArguments() + } + return opts, nil +} + +func runArgs(ctx context.Context, args []string, output io.Writer) error { + opts, e := parseCommand(args) if e != nil { return e } - db, e := repository.InitDB(context.Background(), cfg.Database) + cfg, e := config.Load(opts.configFile) if e != nil { - return e + return errors.New("configuration_error") + } + db, e := repository.InitDB(ctx, cfg.Database) + if e != nil { + return errors.New("database_unavailable") } s, e := core.NewStore(db) if e != nil { - return e + if pool, poolErr := db.DB(); poolErr == nil { + _ = pool.Close() + } + return errors.New("database_unavailable") } defer s.Pool.Close() - ctx := context.Background() var out core.Object - switch *command { + switch opts.command { case "migrate": e = s.Migrate(ctx) out = core.Object{"migrated": e == nil} case "bootstrap": - out, e = s.Bootstrap(ctx, *name, *source, *subject, *display) - default: - return fmt.Errorf("unknown command; choose migrate, bootstrap") + out, e = s.Bootstrap(ctx, opts.name, opts.source, opts.subject, opts.display) + case "residual": + return dispatchResidual(ctx, &opts.residual, output, s.ManageResidual) } if e != nil { - return e + return core.ErrorCode(e) } - return json.NewEncoder(os.Stdout).Encode(out) + return json.NewEncoder(output).Encode(out) } diff --git a/cmd/cloudctl/residual.go b/cmd/cloudctl/residual.go new file mode 100644 index 000000000..cad0bb01d --- /dev/null +++ b/cmd/cloudctl/residual.go @@ -0,0 +1,150 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "io" + "strings" + + "github.com/wanglongan587/cloud/internal/core" +) + +type residualManager func(context.Context, *core.ResidualManagementRequest) (core.Object, error) + +func invalidArguments() error { + return &core.Fault{Code: "invalid_cli_arguments", Status: 400, Params: core.Object{}} +} + +func parseResidual(args []string) (core.ResidualManagementRequest, error) { + var request core.ResidualManagementRequest + if len(args) == 0 { + return request, invalidArguments() + } + allowed := residualFlags(args[0]) + if allowed == nil { + return request, invalidArguments() + } + flags := flag.NewFlagSet("residual "+args[0], flag.ContinueOnError) + flags.SetOutput(io.Discard) + flags.StringVar(&request.TenantID, "tenant-id", "", "tenant ID (required)") + flags.StringVar(&request.ProjectID, "project-id", "", "project filter") + flags.StringVar(&request.RunID, "run-id", "", "original Run ID") + flags.StringVar(&request.WorkspaceID, "workspace-id", "", "original Workspace ID") + flags.StringVar(&request.OperationID, "operation-id", "", "original failed delete operation ID") + flags.StringVar(&request.ReclaimID, "reclaim-id", "", "reclamation responsibility ID") + flags.StringVar(&request.IntentID, "intent-id", "", "existing root or child intent ID") + flags.StringVar(&request.Key, "key", "", "tenant-wide idempotency key") + flags.StringVar(&request.Reason, "reason", "", "operator reason") + flags.StringVar(&request.OperatorLabel, "operator-label", "", "self-reported operator label") + flags.IntVar(&request.Limit, "limit", 100, "page size") + flags.StringVar(&request.After, "after", "", "page cursor") + flags.BoolVar(&request.ImpactConfirmed, "impact-confirmed", false, "confirm termination impact") + flags.BoolVar(&request.DataDeleteConfirmed, "data-delete-confirmed", false, "confirm permanent data deletion") + flags.BoolVar(&request.ResponsibilityTransferred, "responsibility-transferred", false, "accept remaining external responsibility") + flags.StringVar(&request.EvidenceNote, "evidence-note", "", "external disposition explanation") + facts := externalFacts{} + flags.Var(&facts, "external-fact", "repeatable kind:id=external-reference; use openTargets from get") + if err := flags.Parse(args[1:]); err != nil || len(flags.Args()) != 0 { + return request, invalidArguments() + } + validFlags := true + flags.Visit(func(f *flag.Flag) { + if !allowed[f.Name] { + validFlags = false + } + }) + if !validFlags || strings.TrimSpace(request.TenantID) == "" { + return request, invalidArguments() + } + request.ExternalFacts = facts + switch args[0] { + case "list": + request.Action = core.ResidualList + if request.Limit < 1 { + return request, invalidArguments() + } + case "get": + request.Action = core.ResidualGet + if (request.IntentID != "" && hasResourceSelector(&request)) || (request.IntentID == "" && request.RunID == "" && request.WorkspaceID == "" && request.ReclaimID == "") { + return request, invalidArguments() + } + case "terminate": + request.Action = core.ResidualTerminate + if (request.RunID == "" && request.WorkspaceID == "") || !request.ImpactConfirmed { + return request, invalidArguments() + } + case "delete-data": + request.Action = core.ResidualDeleteData + if request.IntentID == "" || !request.DataDeleteConfirmed { + return request, invalidArguments() + } + case "confirm-external": + request.Action = core.ResidualConfirmExternal + if request.IntentID == "" || !request.ResponsibilityTransferred || strings.TrimSpace(request.EvidenceNote) == "" || len(facts) == 0 { + return request, invalidArguments() + } + } + if args[0] != "list" && args[0] != "get" && (strings.TrimSpace(request.Key) == "" || strings.TrimSpace(request.Reason) == "") { + return request, invalidArguments() + } + return request, nil +} + +// Allowed flags keep broad query filters from becoming write selectors. Core +// resolves identities and rechecks authorization; the CLI never chooses an actor. +func residualFlags(action string) map[string]bool { + var names []string + switch action { + case "list": + names = []string{"project-id", "run-id", "workspace-id", "operation-id", "reclaim-id", "limit", "after"} + case "get": + names = []string{"project-id", "run-id", "workspace-id", "operation-id", "reclaim-id", "intent-id"} + case "terminate": + names = []string{"project-id", "run-id", "workspace-id", "operation-id", "reason", "key", "operator-label", "impact-confirmed"} + case "delete-data": + names = []string{"intent-id", "reason", "key", "operator-label", "data-delete-confirmed"} + case "confirm-external": + names = []string{"intent-id", "reason", "key", "operator-label", "evidence-note", "external-fact", "responsibility-transferred"} + default: + return nil + } + allowed := map[string]bool{"tenant-id": true} + for _, name := range names { + allowed[name] = true + } + return allowed +} + +func hasResourceSelector(r *core.ResidualManagementRequest) bool { + return r.ProjectID != "" || r.RunID != "" || r.WorkspaceID != "" || r.OperationID != "" || r.ReclaimID != "" +} + +// External references are opaque evidence, not assertions of physical success. +// Reject duplicates rather than silently replacing evidence supplied earlier. +type externalFacts map[string]string + +func (f *externalFacts) String() string { return "kind:id=external-reference" } + +func (f *externalFacts) Set(value string) error { + target, reference, found := strings.Cut(value, "=") + kind, id, qualified := strings.Cut(target, ":") + if !found || !qualified || (kind != "sandbox" && kind != "data") || strings.TrimSpace(id) == "" || strings.TrimSpace(reference) == "" { + return invalidArguments() + } + if _, exists := (*f)[target]; exists { + return invalidArguments() + } + (*f)[target] = reference + return nil +} + +func dispatchResidual(ctx context.Context, request *core.ResidualManagementRequest, output io.Writer, manage residualManager) error { + out, err := manage(ctx, request) + if err != nil { + // Faults carry the stable safe code. Infrastructure errors may contain a + // connection string, so never print their implementation details. + return core.ErrorCode(err) + } + return json.NewEncoder(output).Encode(out) +} diff --git a/cmd/cloudctl/residual_test.go b/cmd/cloudctl/residual_test.go new file mode 100644 index 000000000..8e25b38b1 --- /dev/null +++ b/cmd/cloudctl/residual_test.go @@ -0,0 +1,166 @@ +package main + +import ( + "bytes" + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/wanglongan587/cloud/internal/core" +) + +func TestCloudctlResidualManagement(t *testing.T) { + cases := []struct { + name string + args []string + want core.ResidualManagementRequest + }{ + { + name: "scoped paginated list", + args: []string{"list", "--tenant-id", "tenant", "--project-id", "project", "--run-id", "run", "--workspace-id", "workspace", "--operation-id", "operation", "--reclaim-id", "reclaim", "--limit", "7", "--after", "cursor"}, + want: core.ResidualManagementRequest{Action: core.ResidualList, TenantID: "tenant", ProjectID: "project", RunID: "run", WorkspaceID: "workspace", OperationID: "operation", ReclaimID: "reclaim", Limit: 7, After: "cursor", ExternalFacts: map[string]string{}}, + }, + { + name: "get intent", + args: []string{"get", "--tenant-id", "tenant", "--intent-id", "intent"}, + want: core.ResidualManagementRequest{Action: core.ResidualGet, TenantID: "tenant", IntentID: "intent", Limit: 100, ExternalFacts: map[string]string{}}, + }, + { + name: "get resource", + args: []string{"get", "--tenant-id", "tenant", "--run-id", "run", "--workspace-id", "workspace"}, + want: core.ResidualManagementRequest{Action: core.ResidualGet, TenantID: "tenant", RunID: "run", WorkspaceID: "workspace", Limit: 100, ExternalFacts: map[string]string{}}, + }, + { + name: "get reclaim", + args: []string{"get", "--tenant-id", "tenant", "--reclaim-id", "reclaim"}, + want: core.ResidualManagementRequest{Action: core.ResidualGet, TenantID: "tenant", ReclaimID: "reclaim", Limit: 100, ExternalFacts: map[string]string{}}, + }, + { + name: "terminate legacy D8 exact scope", + args: []string{"terminate", "--tenant-id", "tenant", "--project-id", "project", "--run-id", "run", "--workspace-id", "workspace", "--operation-id", "failed-delete", "--reason", "operator abandoned execution", "--key", "terminate-key", "--operator-label", "self-reported", "--impact-confirmed"}, + want: core.ResidualManagementRequest{Action: core.ResidualTerminate, TenantID: "tenant", ProjectID: "project", RunID: "run", WorkspaceID: "workspace", OperationID: "failed-delete", Reason: "operator abandoned execution", Key: "terminate-key", OperatorLabel: "self-reported", ImpactConfirmed: true, Limit: 100, ExternalFacts: map[string]string{}}, + }, + { + name: "independent data deletion", + args: []string{"delete-data", "--tenant-id", "tenant", "--intent-id", "root", "--reason", "delete retained data", "--key", "delete-key", "--data-delete-confirmed=true"}, + want: core.ResidualManagementRequest{Action: core.ResidualDeleteData, TenantID: "tenant", IntentID: "root", Reason: "delete retained data", Key: "delete-key", DataDeleteConfirmed: true, Limit: 100, ExternalFacts: map[string]string{}}, + }, + { + name: "external facts preserve exact references", + args: []string{"confirm-external", "--tenant-id", "tenant", "--intent-id", "unfinished", "--reason", "external disposal", "--key", "external-key", "--evidence-note", "host decommissioned", "--external-fact", "sandbox:original=ticket=123", "--external-fact", "data:workspace=ticket-456", "--responsibility-transferred"}, + want: core.ResidualManagementRequest{Action: core.ResidualConfirmExternal, TenantID: "tenant", IntentID: "unfinished", Reason: "external disposal", Key: "external-key", EvidenceNote: "host decommissioned", ResponsibilityTransferred: true, Limit: 100, ExternalFacts: map[string]string{"sandbox:original": "ticket=123", "data:workspace": "ticket-456"}}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + args := append([]string{"--config", "operator.yaml", "residual"}, tc.args...) + opts, err := parseCommand(args) + if err != nil { + t.Fatalf("parse: %v", err) + } + if opts.command != "residual" || opts.configFile != "operator.yaml" || !reflect.DeepEqual(opts.residual, tc.want) { + t.Fatalf("options = %#v; want request %#v with residual command and operator.yaml", opts, tc.want) + } + var output bytes.Buffer + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls := 0 + err = dispatchResidual(ctx, &opts.residual, &output, func(actualCtx context.Context, request *core.ResidualManagementRequest) (core.Object, error) { + calls++ + if actualCtx != ctx || !reflect.DeepEqual(request, &tc.want) { + t.Fatalf("dispatcher changed context or request: %#v", request) + } + return core.Object{"state": "registered"}, nil + }) + if err != nil || calls != 1 || output.String() != "{\"state\":\"registered\"}\n" { + t.Fatalf("dispatch error=%v calls=%d output=%q", err, calls, output.String()) + } + }) + } +} + +func TestCloudctlResidualRejectsUnsafeOrAmbiguousArguments(t *testing.T) { + terminate := []string{"terminate", "--tenant-id", "tenant", "--workspace-id", "workspace", "--reason", "reason", "--key", "key"} + external := []string{"confirm-external", "--tenant-id", "tenant", "--intent-id", "intent", "--reason", "reason", "--key", "key", "--evidence-note", "note", "--responsibility-transferred"} + cases := [][]string{ + {}, + {"unknown"}, + {"list"}, + {"list", "--tenant-id", " "}, + {"list", "--tenant-id", "tenant", "--limit", "0"}, + {"list", "--tenant-id", "tenant", "--limit", "invalid"}, + {"list", "--tenant-id", "tenant", "trailing"}, + {"get", "--tenant-id", "tenant", "--project-id", "project"}, + {"get", "--tenant-id", "tenant", "--intent-id", "intent", "--workspace-id", "workspace"}, + terminate, + appendCopy(terminate, "--impact-confirmed=false"), + appendCopy(terminate, "--impact-confirmed", "--data-delete-confirmed"), + {"terminate", "--tenant-id", "tenant", "--project-id", "project", "--reason", "reason", "--key", "key", "--impact-confirmed"}, + {"terminate", "--tenant-id", "tenant", "--run-id", "run", "--reason", " ", "--key", "key", "--impact-confirmed"}, + {"terminate", "--tenant-id", "tenant", "--run-id", "run", "--reason", "reason", "--key", " ", "--impact-confirmed"}, + {"terminate", "--tenant-id", "tenant", "--run-id", "run", "--reason", "reason", "--impact-confirmed"}, + {"delete-data", "--tenant-id", "tenant", "--workspace-id", "workspace", "--reason", "reason", "--key", "key", "--data-delete-confirmed"}, + {"delete-data", "--tenant-id", "tenant", "--intent-id", "intent", "--reason", "reason", "--key", "key", "--data-delete-confirmed=false"}, + external, + appendCopy(external, "--external-fact", "sandbox:one="), + appendCopy(external, "--external-fact", "sandbox:=ticket"), + appendCopy(external, "--external-fact", "unqualified=ticket"), + appendCopy(external, "--external-fact", "unknown:id=ticket"), + appendCopy(external, "--external-fact", "sandbox:one=a", "--external-fact", "sandbox:one=b"), + {"confirm-external", "--tenant-id", "tenant", "--intent-id", "intent", "--reason", "reason", "--key", "key", "--evidence-note", " ", "--external-fact", "data:workspace=ticket", "--responsibility-transferred"}, + {"confirm-external", "--tenant-id", "tenant", "--intent-id", "intent", "--reason", "reason", "--key", "key", "--evidence-note", "note", "--external-fact", "data:workspace=ticket", "--responsibility-transferred=false"}, + {"list", "--tenant-id", "tenant", "--actor-subject", "spoof"}, + } + for _, args := range cases { + t.Run(strings.Join(args, " "), func(t *testing.T) { + _, err := parseResidual(args) + if err == nil || err.Error() != "invalid_cli_arguments" { + t.Fatalf("error = %v; want invalid_cli_arguments", err) + } + }) + } +} + +func appendCopy(original []string, more ...string) []string { + return append(append([]string(nil), original...), more...) +} + +func TestCloudctlResidualErrorsNeverOutputInfrastructureDetails(t *testing.T) { + for _, tc := range []struct { + err error + code string + }{ + {&core.Fault{Code: "residual_mode_conflict", Status: 409}, "residual_mode_conflict"}, + {errors.New("connection failed: password=do-not-output"), "internal_error"}, + } { + var output bytes.Buffer + err := dispatchResidual(context.Background(), &core.ResidualManagementRequest{}, &output, func(context.Context, *core.ResidualManagementRequest) (core.Object, error) { + return nil, tc.err + }) + if err == nil || err.Error() != tc.code || output.Len() != 0 { + t.Fatalf("error = %v, stdout = %q; want %s and empty stdout", err, output.String(), tc.code) + } + } +} + +func TestCloudctlPreservesLegacyCommandsAndRejectsMixedSyntax(t *testing.T) { + for _, command := range []string{"migrate", "bootstrap"} { + opts, err := parseCommand([]string{"-config", "file", "-command", command}) + if err != nil || opts.command != command || opts.configFile != "file" { + t.Fatalf("legacy command %s: options=%#v error=%v", command, opts, err) + } + } + for _, args := range [][]string{ + {"-command", "residual"}, + {"-command", "migrate", "residual", "list", "--tenant-id", "tenant"}, + {"-name", "tenant", "residual", "list", "--tenant-id", "tenant"}, + {"-command", "bootstrap", "unexpected"}, + {"--unknown", "password=do-not-output"}, + } { + if _, err := parseCommand(args); err == nil || err.Error() != "invalid_cli_arguments" { + t.Fatalf("args=%q error=%v; want invalid_cli_arguments", args, err) + } + } +} diff --git a/docs/agent-runs/issue-runs.en.md b/docs/agent-runs/issue-runs.en.md index 535a05f28..572d630b9 100644 --- a/docs/agent-runs/issue-runs.en.md +++ b/docs/agent-runs/issue-runs.en.md @@ -69,10 +69,10 @@ and it accepts no other run or interactive session. ## Background loops After D4/D5 business settlement, independent ledger reconciliation continues and grants one durable system owned-delete responsibility only -when evidence qualifies. Retries keep the same operation; cleanup does not settle the Run again. The current scope is -[Cloud step 1 of residual reclamation](residual-reclamation.en.md). Controlled operator endpoints, retained-data/external dispositions, -Controller/Sandbox Server physical acceptance, joint acceptance and D8 historical takeover remain incomplete. Neither `done` nor a completed -reconciliation pass proves that resources were reclaimed. +when evidence qualifies. Retries keep the same operation; cleanup does not settle the Run again. Deployment operators can register +controlled reclamation explicitly with `cloudctl residual` (terminate and retain data, delete data separately, or confirm externally) and +take over D8 historical residuals; see [residual reclamation](residual-reclamation.en.md). Neither `done` nor a completed reconciliation pass +proves that resources were reclaimed. Besides the synchronous transitions taken while evidence is taken over, the Cloud server process runs a set of loops every 10 seconds for the progress and settlement that have no external event to follow: diff --git a/docs/agent-runs/issue-runs.md b/docs/agent-runs/issue-runs.md index db48a6e26..6fbee6528 100644 --- a/docs/agent-runs/issue-runs.md +++ b/docs/agent-runs/issue-runs.md @@ -51,8 +51,8 @@ Cloud 在接管上一阶段证据的事务里写入。 ## 后台循环 D4/D5 业务收尾后的残留由独立账本对账继续处理，证据足够才授予一份持久的系统 owned-delete 责任；重试保持同一 operation， -完成回收不重新结算 Run。当前仅覆盖 [残留回收的 Cloud 第 1 步](residual-reclamation.md)：受控运维入口、保留数据/外部确认、 -Controller/Sandbox Server 物理验收、跨仓联合验收和 D8 历史接管仍未完成；`done` 或对账已运行均不证明资源已回收。 +完成回收不重新结算 Run。部署运维可用 `cloudctl residual` 显式登记受控回收（只终止并保留数据、单独删数据或外部确认）， +也可以接管 D8 历史残留，见[残留回收](residual-reclamation.md)。`done` 或对账已运行均不证明资源已回收。 除了接管证据时的同步转换，Cloud server 进程里有一组每 10 秒执行一次的循环，负责那些没有外部事件可跟随的推进和收尾： diff --git a/docs/agent-runs/residual-reclamation.en.md b/docs/agent-runs/residual-reclamation.en.md index 155be8d6d..74b9a1404 100644 --- a/docs/agent-runs/residual-reclamation.en.md +++ b/docs/agent-runs/residual-reclamation.en.md @@ -6,7 +6,7 @@ D4/D5 [bounded startup settlement](startup-settlement.en.md) can finish the busi `agent_run_residuals`. Cloud continues collecting evidence on that original ledger and only grants system cleanup authority when the evidence qualifies. Deletion follows the existing owned-delete path. It neither settles the [IssueRun](issue-runs.en.md) again nor reopens its Workspace. -This page describes the Cloud step 1 implementation boundary. It is not evidence of physical termination by the Controller or Sandbox Server, +This page describes Cloud automatic reclamation and the step 2 controlled-reclamation implementation contract. It is not evidence of physical termination by the Controller or Sandbox Server, deployment acceptance, or joint fault acceptance. The [residual reclamation ADR](https://github.com/ora-space/specs/blob/main/decisions/cloud/issue-run/20261018-residual-agent-run-reclamation.md) remains `approved`; this step does not declare the whole ADR implemented. @@ -19,16 +19,16 @@ remains `approved`; this step does not declare the whole ADR implemented. | `reclaiming` | One durable responsibility has been granted; continue or await evidence | Fixed `reclaim_id`, mode and nonempty authorization; `owned_delete` binds exactly one operation | | `cleared` | Disposition under the original responsibility is complete | Keep that responsibility, with both `cleared_at` and disposition | -Step 1 production authorization uses only `owned_delete` and completes only as `released`. `controlled` reserves the shared internal arbitration -point; it is neither a controlled management endpoint nor operator authority. The schema's `terminated_data_retained` and `externally_confirmed` -dispositions are reserved for later controlled paths. +Automatic reclamation uses `owned_delete` and completes as `released`. Explicit operator commands use `controlled`, allowing +`terminated_data_retained` or external disposition of an existing unfinished intent as `externally_confirmed`. Both modes compete for the +same responsibility and cannot overwrite each other. Failures, timeouts and the existence of operator commands do not escalate automatic authority. Automatic responsibility stays bound to the same `delete_workspace` operation. Restarts, lease changes, retries and quiesce refusals do not create another responsibility or operation, and do not revive a failed historical operation. Ordinary deletion, dispatch and permit paths retain residual guards; an internal system responsibility is not a public deletion flag. The transaction rechecks the ledger, Run, Workspace, Project and operation tenant, original bindings and generation; a UUID foreign key alone does not establish that scope. -The [migration](../../internal/core/migrations/README.en.md) only adds columns. Defaults initialize existing D4/D5 rows as `reconciling`, without +The [migrations](../../internal/core/migrations/README.en.md) retain the ledger and add separate intents, command receipts and per-target termination plans. Defaults initialize existing D4/D5 rows as `reconciling`, without inferring completion, backfilling D8, or changing old `reason`, `responsibility` or business settlement. A legacy row with non-null `cleared_at` causes an explicit error and rollback pending compatibility handling (A2). @@ -49,12 +49,12 @@ runtime control identity, together with original execution terminals. Per-target A no-target proof applies only when original plans and execution relationships truly do not exist; missing rows, NULL results and elapsed time do not establish that proof. -Step 1 accepts delivery eligibility only through a genuinely registered and verified Revision for the original `deliver_revision` execution; +Automatic reclamation accepts delivery eligibility only through a genuinely registered and verified Revision for the original `deliver_revision` execution; every other actual delivery execution needs its own completed verification. There is currently no unified existing durable source for explicit delivery abandonment. `failed`, `skipped`, business timeouts, absent object storage and missing results do not imply explicit abandonment (A1). Entries with insufficient evidence remain open. -Authorized cleanup follows existing quiesce → terminate → cleanup. A quiesce refusal leaves the original operation in `retry_wait`, retaining +Authorized automatic cleanup follows existing quiesce → terminate → cleanup. A quiesce refusal leaves the original operation in `retry_wait`, retaining the original authority and guards for evidence collection and retry (A3). It neither restores new execution eligibility, escalates to force-stop, nor creates another operation. When the existing Controller requests `blocked` for unconfirmed termination, this system responsibility instead persists claimable `retry_wait`; @@ -72,6 +72,50 @@ historical ID without both confirmations remains open, without replacing its ide Data-deletion planning, permits and completion recheck target coverage. One transaction confirms successful data deletion, succeeds the operation, soft-deletes the Workspace and writes `cleared/released`, without changing the Run's phase/status/result or repeating business settlement activity. +## Deployment commands and two-stage disposition + +`cloudctl residual` is a local deployment-operations entrypoint using deployment database credentials. The authenticated subject is the +connection's database `session_user`; an optional operator label is self-reported and cannot replace that identity. Credentials never enter +output or audit. There is no new HTTP route, tenant-admin bypass, Gateway JWT privilege or listening port; public operations on hidden Run +Workspaces continue to return 404. + +| Command | Scope and required confirmation | +|---|---| +| `list` / `get` | Tenant is required; intersect Project, Run, Workspace, operation or reclaim filters; details expose only operational state, targets, audit and evidence | +| `terminate` | One exact Run/Workspace, reason, idempotency key and explicit impact confirmation; D8 also requires the original failed delete operation; retains data by default | +| `delete-data` | One root intent with successful physical termination and retained disposition, separate reason/key and data-deletion confirmation; no bare-Workspace deletion | +| `confirm-external` | One existing unfinished root or dependent intent, reason/key, explanation, external fact references for every unfinished target and explicit acceptance of remaining responsibility | + +Mutations do not accept tenant/Project batch scope. All command keys share one tenant-scoped receipt space. The same key and normalized input +return the original receipt and current progress; a different action, resource, parent intent, reason or confirmation returns +`idempotency_conflict`. Replay creates neither a new responsibility nor repeated external evidence collection. + +Each resource has one `terminate` root intent and at most one dependent `delete_data` intent. Each follows +`registered → terminating → succeeded`. The root owns the sole reclamation responsibility; the dependent does not arbitrate again. +Registration, original-identity checks, admission/execution closure and plan persistence commit in one database transaction, without Controller calls inside it. + +Each original sandbox/ensure target retains a fixed effect ID and request. Cloud projects controlled plans through existing +`ListForceStops`, `ConfirmForceStop` and `GetEffectPermit` fields. It adds no RPC/proto and writes no synthetic ordinary force-stop history. +Current Controller lease, original sandbox/generation and control epoch still fence permits and confirmations. Every target requires both +`terminated=true` and `lateEnsureFenced=true`; a no-target proof must also establish closed entrypoints and no planned or in-flight responsibility. +Ordinary force-stop completion that bulk-finishes operations or changes business state is not used for controlled reclamation. + +After termination, the root remains succeeded and the ledger, when present, becomes `cleared/terminated_data_retained`, retaining data and the +Workspace. A later explicit delete-data command binds the sole `delete_workspace` operation to that same responsibility and starts at cleanup +through existing operation/effect/permit paths. The first root completion, retained history and `cleared_at` never regress or reset. +Only complete physical `removed=true` evidence succeeds the dependent, soft-deletes the Workspace, appends released disposition and updates +the current disposition. Failure resumes the same operation/effect and never revives a historical failed operation. + +External confirmation must cover every unfinished target and its data responsibility; references may be tickets or external fact identifiers. +It transfers responsibility for an existing intent and stops new permits. It does not fabricate `terminated`, `lateEnsureFenced` or `removed`, +permit same-volume reuse or authorize data deletion. Missing intents, succeeded intents, empty explanations and missing targets are rejected. +Late physical facts remain evidence; they cannot silently convert external disposition into system-proven success. + +D8 takeover requires the original done Run, retained Workspace binding and failed/node_unavailable delete operation; it never backfills the D4/D5 +ledger. Without a ledger, the root owns targets, evidence, conflicts and the new delete-operation binding. The old failed operation, Run terminal +and settlement remain unchanged. Ordinary scans do not create controlled D8 responsibility. Missing ledger rows or retained disposition cannot +bypass admission, Project-operation or data-reuse guards. + ## Existing fact intake and conflicts Cloud uses existing `runtime_ack`, creation-fact, execution-result and Revision intake. This step adds no RPC, proto or Controller discovery @@ -95,12 +139,12 @@ If a failed create's original effects have no existing active-discovery path, th Cloud's evidence intake and responsibility scheduling do not establish new Controller discovery capability, or production validation of real containers, late-ensure fencing or physical data deletion. -| Later scope | Step 1 boundary | +| Scope | Cloud boundary | |---|---| -| Step 2: controlled operator endpoints, retained data, external confirmation | Not implemented; internal `controlled` arbitration does not supply authentication, intent or commands | +| Step 2: Cloud deployment CLI, controlled intents, retained data, separate deletion and external confirmation | This page describes the direct Cloud contract; passing tests and continuous Cloud-path evidence require actual execution artifacts | | Step 3: Controller / Sandbox Server physical contracts and per-target evidence | This page states Cloud evidence requirements, not completed physical acceptance | | Step 4: cross-repository joint fault and recovery acceptance | Incomplete; single-repository implementation or database tests are not a substitute | -| Taking over D8 historical deletion abandonment and residuals without ledger rows | Not implemented; no ledger backfill, automatic retry or historical business-result rewrite | +| Taking over D8 historical deletion abandonment and residuals without ledger rows | Explicit operator takeover only; no ledger backfill, automatic retry or historical business-result rewrite; real deployment recovery remains step 4 | See the [core code map](../../internal/core/README.en.md) for implementation entry points. Test results require their execution artifacts; this page does not claim migration, concurrency or end-to-end tests have passed. diff --git a/docs/agent-runs/residual-reclamation.md b/docs/agent-runs/residual-reclamation.md index 92271906f..5daa53d46 100644 --- a/docs/agent-runs/residual-reclamation.md +++ b/docs/agent-runs/residual-reclamation.md @@ -6,7 +6,7 @@ `agent_run_residuals`。Cloud 的残留回收在原账本上继续取证，证据足够后才取得系统清理责任，沿已有 owned delete 执行删除。它不重新结算 [IssueRun](issue-runs.md)，也不重新开放运行 Workspace。 -本文描述 Cloud 第 1 步的实现边界，不作为 Controller、Sandbox Server 的物理终止、部署或联合故障验收证据。 +本文描述 Cloud 自动回收和第 2 步受控回收的实现合同，不作为 Controller、Sandbox Server 的物理终止、部署或联合故障验收证据。 [残留回收 ADR](https://github.com/ora-space/specs/blob/main/decisions/cloud/issue-run/20261018-residual-agent-run-reclamation.md) 仍为 `approved`；本步不是整份 ADR 已实施完成的声明。 @@ -18,14 +18,15 @@ owned delete 执行删除。它不重新结算 [IssueRun](issue-runs.md)，也 | `reclaiming` | 已取得一份持久责任，继续推进或等待证据 | 固定 `reclaim_id`、mode 和非空 authorization；`owned_delete` 必须绑定唯一 operation | | `cleared` | 原责任下的处置已完成 | 保留原责任，同时有 `cleared_at` 和 disposition | -第 1 步的生产授权只使用 `owned_delete`，完成处置只写 `released`。`controlled` 只预留共用内部仲裁位置， -不构成受控管理端点或运维权限。数据库接受的 `terminated_data_retained`、`externally_confirmed` 留给后续受控路径。 +自动回收使用 `owned_delete`，完成处置写 `released`；显式运维命令使用 `controlled`，可先完成为 +`terminated_data_retained`，或对已有未完成意图登记 `externally_confirmed`。两种模式竞争同一责任， +不能相互覆盖；自动层不会因为失败、超时或受控入口存在而自动升级授权。 自动责任始终绑定同一个 `delete_workspace` operation。重启、换 lease、重试和 quiesce 拒绝不会产生第二个责任或 operation， 也不会把失败的历史 operation 恢复为运行态。普通删除、派发和许可入口继续检查残留保护；内部系统责任不能被当成公开删除开关。 账本、Run、Workspace、Project、operation 的租户、原绑定和 generation 在同一事务里重新比对；UUID 外键本身不证明这些范围一致。 -[迁移](../../internal/core/migrations/README.md)只增列。存量 D4/D5 行通过默认值进入 `reconciling`，不推断完成、不回填 D8， +[迁移](../../internal/core/migrations/README.md)保留已有账本，并增加独立意图、命令回执及逐目标终止计划。存量 D4/D5 行通过默认值进入 `reconciling`，不推断完成、不回填 D8， 不修改旧 `reason`、`responsibility` 或业务结算。旧行已有非空 `cleared_at` 时明确报错回滚，等待显式兼容处理（A2）。 ## 原身份取证与删除资格 @@ -41,11 +42,11 @@ execution、clone、work 和数据目标。迟到事实可以补充原目标， 执行侧需要同一 runtime control 身份的关闭输入确认及明确的未完成执行列表，并检查原执行终态；逐目标的终止证据也可证明其 物理执行已封闭。仅在原计划和执行关系确实不存在时才能采用无目标证明，缺行、NULL result 或等待时间均不构成这种证明。 -第 1 步只接受原 `deliver_revision` 执行真实登记、核验完成的 Revision 作为交付资格；其他实际交付执行也必须各自核验完成。 +自动层只接受原 `deliver_revision` 执行真实登记、核验完成的 Revision 作为交付资格；其他实际交付执行也必须各自核验完成。 当前没有可统一引用的既有“明确放弃交付”记录来源。`failed`、`skipped`、业务超时、没有对象存储或无结果都不自动等于明确放弃（A1）， 证据不足的条目继续保持未完成。 -获授权后沿已有 quiesce → terminate → cleanup 推进。quiesce 拒绝使原 operation 留在 `retry_wait`，保留原责任及保护，稍后重新取证再试（A3）。 +自动回收获授权后沿已有 quiesce → terminate → cleanup 推进。quiesce 拒绝使原 operation 留在 `retry_wait`，保留原责任及保护，稍后重新取证再试（A3）。 它不恢复新执行资格，不升级为强停，也不重新创建 operation。 既有 Controller 因终止未确认而请求 `blocked` 时，此系统责任也保存为可领取的 `retry_wait`；实际计划和许可仍重新检查证据。 @@ -58,6 +59,45 @@ execution、clone、work 和数据目标。迟到事实可以补充原目标， 不为它改写目标或另发终止 effect（A5）。数据删除 effect 的计划、许可和完成再次检查目标覆盖；同一事务确认数据删除成功、operation 成功、 Workspace 软删除并写 `cleared/released`，不改 Run 的 phase/status/result 或重复追加业务结算活动。 +## 部署运维命令与两阶段处置 + +`cloudctl residual` 是使用部署数据库凭据的本地运维入口；认证主体来自数据库连接的 `session_user`。 +可附加的 operator 标签仅为调用者自述，不能替代认证主体。凭据不进入输出或审计。 +没有新增 HTTP 路由、tenant admin 旁路、Gateway JWT 权限或监听端口；隐藏 Run Workspace 的公开操作继续返回 404。 + +| 命令 | 范围与必要确认 | +|---|---| +| `list` / `get` | tenant 必填；查询可按 Project、Run、Workspace、operation 或 reclaim 相交过滤，详情只返回运维所需状态、目标、审计与证据 | +| `terminate` | 一个精确 Run/Workspace，reason、幂等键和显式影响确认；D8 还需原 failed delete operation 身份；默认保留数据 | +| `delete-data` | 一个已真实终止成功且 retained 的根意图，独立 reason、幂等键和数据删除确认；不能按裸 Workspace 发起 | +| `confirm-external` | 一个已存在且未完成的根或从属意图，reason、幂等键、说明、每个未完成目标的外部事实引用及承担剩余责任的确认 | + +写命令不接受 tenant/Project 批量处置。所有命令键共用 tenant 内的回执空间；同键同规范输入返回原回执和当前进度， +同键不同动作、资源、父意图、reason 或确认返回 `idempotency_conflict`。重放不创建新责任或重复外部取证。 + +每个资源只有一个 `terminate` 根意图，最多一个依赖它的 `delete_data` 意图；两者各自按 +`registered → terminating → succeeded` 推进。根意图持有唯一回收责任，从属意图不重新仲裁。 +登记、原身份核对、关闭 admission/后续执行资格和计划写入在同一数据库事务提交；不在事务内调用 Controller。 + +每个原 sandbox/ensure 目标保留固定 effect ID 和 request。Cloud 通过既有 `ListForceStops`、`ConfirmForceStop`、 +`GetEffectPermit` 字段投影这些受控计划，不新增 RPC/proto，也不往普通 force-stop 历史写伪记录。 +当前 Controller lease、原 sandbox/generation 和 control epoch 仍约束许可与确认；每个目标必须同时确认 +`terminated=true`、`lateEnsureFenced=true`，无目标时也须证明入口已关闭且没有计划或在途责任。 +普通强停批量结束 operation、改业务状态的收尾逻辑不适用于受控回收。 + +终止完成后，根意图固定为 succeeded，账本（如有）写 `cleared/terminated_data_retained`，保留数据与 Workspace。 +之后的显式 delete-data 沿同一责任绑定唯一 `delete_workspace` operation，从 cleanup 推进已有 operation/effect/permit 路径。 +首次根完成记录、retained 历史和 `cleared_at` 不回退、不重置；只有真实 `removed=true` 齐全后，才完成从属意图、 +软删除 Workspace、追加 released 处置并更新当前 disposition。失败沿同一 operation/effect 恢复，不能复活历史 failed operation。 + +外部确认必须覆盖每个未完成目标及其数据责任，引用可为工单或外部事实标识。它完成的是已有意图的责任转移， +停止该责任的新许可，不伪造 `terminated`、`lateEnsureFenced` 或 `removed`，也不据此开放同卷复用或授权删卷。 +没有意图、意图已 succeeded、说明为空或漏目标均拒绝；迟到物理事实继续留存，不能静默把外部处置改成系统证明。 + +D8 只按原 done Run、保留的 Workspace 绑定及 failed/node_unavailable delete operation 接管，不回填 D4/D5 账本。 +无账本时根意图持有目标、证据、冲突和删除 operation 绑定；原 failed operation、Run 终态与结算保持不变。 +普通扫描不会自动为 D8 创建受控责任；无账本或已 retained 也不能绕过 admission、Project 操作和数据复用保护。 + ## 既有事实入口与冲突 Cloud 使用现有 `runtime_ack`、创建事实、执行结果和 Revision intake；没有新增 RPC、proto 或 Controller 取证机制。 @@ -78,11 +118,11 @@ Cloud 使用现有 `runtime_ack`、创建事实、执行结果和 Revision intak failed create 的原 effects 如果没有现成可主动发现的路径，就继续等待可验证的原身份事实（A6）。Cloud 接收证据和调度责任不证明 Controller 已新增发现能力，更不证明真实容器、迟到 ensure 围栏或物理数据删除已经通过生产验证。 -| 后续范围 | 本步边界 | +| 范围 | Cloud 边界 | |---|---| -| 第 2 步：受控运维入口、保留数据、外部确认 | 未实现；内部 `controlled` 仲裁预留不替代认证、意图或命令 | +| 第 2 步：Cloud 部署 CLI、受控意图、保留数据、独立删除与外部确认 | 本文说明直接 Cloud 合同；测试及连续 Cloud 路径是否通过须看实际执行证据 | | 第 3 步：Controller / Sandbox Server 物理契约与逐目标证据 | 本文仅描述 Cloud 的证据要求，不声明物理验收完成 | | 第 4 步：跨仓联合故障与恢复验收 | 未完成，不以单仓实现或数据库测试代替 | -| D8 历史删除放弃、无账本残留的接管 | 未实现；不补账、不自动重试、不改历史业务结果 | +| D8 历史删除放弃、无账本残留的接管 | 仅显式运维接管；不补账、不自动重试、不改历史业务结果；真实部署恢复留第 4 步 | 核心代码入口见 [core 代码地图](../../internal/core/README.md)。测试结果应以对应执行产物为准，本文不声明迁移、并发或端到端测试已经通过。 diff --git a/integration/agent_run_controlled_clone_receipt_test.go b/integration/agent_run_controlled_clone_receipt_test.go new file mode 100644 index 000000000..e988b556b --- /dev/null +++ b/integration/agent_run_controlled_clone_receipt_test.go @@ -0,0 +1,228 @@ +package integration + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/wanglongan587/cloud/internal/controlpb" + "github.com/wanglongan587/cloud/internal/core" +) + +// Only the preexisting business failure is staged. Both sources acquire controlled +// authority through ManageResidual, and both clone events use the real takeover RPC. +func TestControlledResidualCloneReceiptConflictThroughRegisteredIntentsOverGRPC(t *testing.T) { + newFixture := controlledFixtureFactory(t) + for _, source := range []string{"legacy_d8", "residual_ledger"} { + t.Run(source, func(t *testing.T) { + f, c, run, workspace, operation, original := controlledCloneReceiptScene(t, newFixture(t)) + ctx := context.Background() + legacyOperation := "" + if source == "residual_ledger" { + _, err := c.operations.DeferOperation(c.ctx, &controlpb.DeferOperationRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation.GetId(), Version: operation.GetVersion(), + State: controlpb.DeferState_DEFER_STATE_BLOCKED, Reason: controlpb.DeferReason_DEFER_REASON_NODE_UNAVAILABLE, RetrySeconds: 60, + }) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) + must(t, err) + must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(ctx)) + } else { + // A D8 Workspace has a failed historical delete and no residual ledger. + // Do not create a ledger and then delete it to manufacture this source. + _, err := f.store.Pool.Exec(`UPDATE issue_runs SET phase='done',status='failed',thread_state='ended',idle_since=NULL, + failure_reason='agent_start_failed',completed_at=clock_timestamp(),version=version+1, + result='{"deliveryState":"failed","testBoundary":"preexisting_business_terminal"}'::jsonb WHERE id=$1`, run) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE workspaces SET admission_open=false,admission_epoch=admission_epoch+1, + version=version+1 WHERE id=$1`, workspace) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE operations SET state='failed',error_code='node_unavailable', + version=version+1,updated_at=clock_timestamp() WHERE id=$1`, operation.GetId()) + must(t, err) + legacyOperation = uuid.NewString() + _, err = f.store.Pool.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id, + kind,state,step,request,idempotency_key,request_hash,error_code,controller_epoch) + SELECT $2::uuid,w.tenant_id,w.creator_user_id,w.project_id,w.id,'delete_workspace','failed','terminate', + '{}','controlled-clone-legacy-delete','controlled-clone-legacy-fixture','node_unavailable',$3 + FROM workspaces w WHERE w.id=$1`, workspace, legacyOperation, c.epoch) + must(t, err) + } + + termination := core.ResidualManagementRequest{ + Action: core.ResidualTerminate, TenantID: f.tid, RunID: run, WorkspaceID: workspace, OperationID: legacyOperation, + Key: "registered-clone-terminate", Reason: "Terminate the original residual while retaining its data", ImpactConfirmed: true, + } + accepted, err := f.store.ManageResidual(ctx, &termination) + must(t, err) + root := accepted.O("intent").S("id") + if root == "" { + t.Fatalf("production termination command returned no intent: %v", accepted) + } + var registered bool + must(t, f.store.Pool.QueryRow(`SELECT i.source=$2 AND i.command_kind='terminate' AND i.root_intent_id IS NULL + AND i.run_id=$3 AND i.workspace_id=$4 AND i.impact_confirmed + AND EXISTS(SELECT 1 FROM agent_run_reclamation_command_receipts cr WHERE cr.intent_id=i.id AND cr.idempotency_key=$5) + AND EXISTS(SELECT 1 FROM jsonb_array_elements(i.audit) a WHERE a->>'kind'='registered') + AND CASE WHEN $2='legacy_d8' THEN i.legacy_delete_operation_id::text=$6 + AND NOT EXISTS(SELECT 1 FROM agent_run_residuals r WHERE r.run_id=i.run_id) + ELSE EXISTS(SELECT 1 FROM agent_run_residuals r WHERE r.run_id=i.run_id + AND r.reclaim_mode='controlled' AND r.reclaim_state='reclaiming' AND r.reclaim_id=i.reclaim_id) END + FROM agent_run_reclamation_intents i WHERE i.id=$1`, root, source, run, workspace, termination.Key, + legacyOperation).Scan(®istered)) + if !registered { + t.Fatal("controlled intent did not come from the requested real source and production command") + } + + before := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()) + conflict := proto.Clone(original).(*controlpb.TakeOverNodeEventRequest) + conflict.SubmissionId, conflict.Event = c.sub(), []byte("clone event B") + response, err := f.executions.TakeOverNodeEvent(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("registered intent lost receipt-only conflict: response=%v error=%v", response, err) + } + + // The legacy adapter must persist through legacyIntentId into the root intent; + // ledger-backed authority must persist into its existing residual row instead. + var audited bool + must(t, f.store.Pool.QueryRow(`SELECT + CASE WHEN i.source='legacy_d8' THEN i.conflicted_at IS NOT NULL AND r.run_id IS NULL + ELSE r.reclaim_conflicted_at IS NOT NULL END + AND (SELECT count(*) FROM jsonb_array_elements( + CASE WHEN i.source='legacy_d8' THEN i.evidence ELSE r.reclaim_evidence END) x + WHERE x->>'key'=$2 AND x->'fact'->>'kind'='conflict' + AND x->'fact'->'original'->>'kind'='receipt' AND x->'fact'->'original'->>'event'=$3 + AND x->'fact'->'incoming'->>'kind'='receipt' AND x->'fact'->'incoming'->>'event'=$4 + AND x->>'receivedAt'<>'' AND x->'source'->>'submissionId'=$5)=1 + AND EXISTS(SELECT 1 FROM control_submissions WHERE submission_id=$5 AND response->>'residualFactConflict'='true') + FROM agent_run_reclamation_intents i LEFT JOIN agent_run_residuals r ON r.run_id=i.run_id WHERE i.id=$1`, + root, fmt.Sprintf("clone_receipt:%s:%d", original.GetExecutionId(), original.GetSequence()), + base64.StdEncoding.EncodeToString(original.GetEvent()), base64.StdEncoding.EncodeToString(conflict.GetEvent()), + conflict.GetSubmissionId()).Scan(&audited)) + if !audited { + t.Fatal("conflict response preceded committed receipt evidence, conflict time or submission response on its source") + } + if after := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()); after != before { + t.Fatalf("receipt-only conflict changed original clone result or receipt:\nbefore=%s\nafter=%s", before, after) + } + + // Fresh command keys force the production validation path instead of replaying + // the successful registration receipt. Both actions must fence the conflict first. + termination.Key = "registered-clone-terminate-after-conflict" + for _, request := range []*core.ResidualManagementRequest{ + &termination, + { + Action: core.ResidualDeleteData, TenantID: f.tid, IntentID: root, + Key: "registered-clone-delete-after-conflict", Reason: "Separately authorize retained data deletion", DataDeleteConfirmed: true, + }, + } { + out, commandErr := f.store.ManageResidual(ctx, request) + var fault *core.Fault + if out != nil || !errors.As(commandErr, &fault) || fault.Status != 409 || fault.Code != "residual_fact_conflict" { + t.Fatalf("%s bypassed registered receipt conflict: response=%v error=%v", request.Action, out, commandErr) + } + } + if f.scalar(`SELECT count(*) FROM agent_run_reclamation_intents WHERE root_intent_id=$1`, root) != 0 || + f.scalar(`SELECT count(*) FROM external_effects WHERE workspace_id=$1 AND kind='workspace_data_delete'`, workspace) != 0 { + t.Fatal("blocked controlled commands created a data-delete intent or effect") + } + }) + } +} + +// controlledCloneReceiptScene preserves the original clone receipt setup and assertions while +// accepting this matrix's fresh fixture. The residual test helper remains independently owned. +func controlledCloneReceiptScene(t *testing.T, f *fixture) (*fixture, *grpcController, string, string, *controlpb.Operation, *controlpb.TakeOverNodeEventRequest) { + t.Helper() + f.bindBusinessHooks() + project := seedControlledCloneProject(t, f) + run := f.insertAgentRun(t, project) + provisioned, err := f.store.CreateRunWorkspace(context.Background(), run) + must(t, err) + workspace, operation := provisioned.O("workspace").S("id"), provisioned.O("operation").S("id") + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET phase='provisioning',status='dispatched',workspace_id=$2,version=version+1 WHERE id=$1`, run, workspace) + must(t, err) + c := newGRPCController(t, f) + snapshot := c.claim() + if snapshot.GetOperation().GetId() != operation { + t.Fatalf("claim picked %s, want %s", snapshot.GetOperation().GetId(), operation) + } + op, ensured := c.effect(snapshot.GetOperation(), controlpb.EffectKind_EFFECT_KIND_SANDBOX_ENSURE, workspace) + op, err = c.advance(op) + must(t, err) + nodeID := ensured.GetEvidence().GetSandboxEnsured().GetNodeId() + node, err := c.register(ensured.GetId(), 1, nodeID, "inc-late") + must(t, err) + _, err = c.status(node, controlpb.NodeConnection_NODE_CONNECTION_CONNECTED) + must(t, err) + _, err = c.advance(op) + must(t, err) + snapshot = c.claim() + op = snapshot.GetOperation() + if op.GetStep() != controlpb.OperationStep_OPERATION_STEP_CLONE || snapshot.GetCloneInput() == nil { + t.Fatalf("receipt fixture must dispatch the Workspace's frozen clone step: %v", snapshot) + } + execution := "clone-receipt-execution" + _, err = f.executions.RecordDispatch(c.ctx, &controlpb.RecordDispatchRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, ExecutionId: execution, NodeId: nodeID, + Input: &controlpb.ExecutionInput{Spec: &controlpb.ExecutionInput_Clone{Clone: snapshot.GetCloneInput()}}, + }) + must(t, err) + original := &controlpb.TakeOverNodeEventRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, ExecutionId: execution, Sequence: 1, + Result: &controlpb.ExecutionResult{ + Node: &controlpb.NodeIdentity{NodeId: nodeID, NodeIncarnationId: "inc-late"}, + Outcome: &controlpb.ExecutionResult_CloneFailed{CloneFailed: &controlpb.CloneFailed{Reason: controlpb.CloneFailureReason_CLONE_FAILURE_REASON_SNAPSHOT_UNAVAILABLE}}, + }, + Event: []byte("clone event A"), + } + taken, err := f.executions.TakeOverNodeEvent(c.ctx, original) + must(t, err) + if taken.GetRecord().GetExecutionId() != execution || !proto.Equal(taken.GetRecord().GetResult(), original.GetResult()) { + t.Fatalf("initial clone takeover did not preserve its original result: %v", taken) + } + return f, c, run, workspace, op, original +} + +// seedControlledCloneProject stages only the already available Project and repository snapshot. +// Project/main-Workspace provisioning is outside the receipt-conflict scenario; the run Workspace, +// its sandbox plan, Node registration, frozen clone input, dispatch and receipts remain real below. +func seedControlledCloneProject(t *testing.T, f *fixture) string { + t.Helper() + project, binding, workspace, sync := uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString() + var space string + must(t, f.store.Pool.QueryRow("SELECT id::text FROM collab_workspaces WHERE tenant_id=$1", f.tid).Scan(&space)) + tx, err := f.store.Pool.Begin() + must(t, err) + defer func() { _ = tx.Rollback() }() + exec := func(query string, args ...any) { + t.Helper() + _, err := tx.Exec(query, args...) + must(t, err) + } + exec(`INSERT INTO projects(id,tenant_id,owner_user_id,space_id,name,lifecycle) + VALUES($1,$2,$3,$4,'Controlled clone','active')`, project, f.tid, f.uid, space) + exec(`INSERT INTO repository_bindings(id,tenant_id,project_id,repository_url,branch) + VALUES($1,$2,$3,'https://example.invalid/repo.git','main')`, binding, f.tid, project) + // The Project and its required main Workspace commit together under the deferred constraint. + exec(`INSERT INTO workspaces(id,tenant_id,owner_user_id,creator_user_id,creator_evidence, + project_id,kind,desired_state,observed_state,admission_open,runtime_generation,requested_ref) + VALUES($1,$2,$3,$3,'verified_request',$4,'main','running','ready',true,1,'main')`, workspace, f.tid, f.uid, project) + exec(`INSERT INTO repository_operations(id,type,tenant_id,project_id,binding_id,state,trigger,finished_at) + VALUES($1,'repository_sync',$2,$3,$4,'succeeded','fixture',clock_timestamp())`, sync, f.tid, project, binding) + snapshot := core.GitSnapshotRef(binding, f.commit) + exec(`INSERT INTO repository_snapshots(binding_id,tenant_id,project_id,commit_oid,snapshot_ref,sync_operation_id) + VALUES($1,$2,$3,$4,$5,$6)`, binding, f.tid, project, f.commit, snapshot, sync) + must(t, tx.Commit()) + // This is the existing ref-query fixture for a prepared snapshot, not a physical clone result. + f.refs.pin(f.tid, project, snapshot, f.commit) + return project +} diff --git a/integration/agent_run_controlled_fixture_test.go b/integration/agent_run_controlled_fixture_test.go new file mode 100644 index 000000000..88855937d --- /dev/null +++ b/integration/agent_run_controlled_fixture_test.go @@ -0,0 +1,72 @@ +package integration + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/wanglongan587/cloud/internal/core" +) + +var ( + controlledPreparedFixture *fixture + controlledFixtureCleanups []func() error +) + +// TestMain owns only the controlled suite's reusable schema and immutable source. Other tests keep +// their ordinary per-test fixtures and cleanups; every controlled case still has a fresh service stack. +func TestMain(m *testing.M) { + code := m.Run() + for i := len(controlledFixtureCleanups) - 1; i >= 0; i-- { + if err := controlledFixtureCleanups[i](); err != nil { + fmt.Fprintln(os.Stderr, "controlled fixture cleanup:", err) + code = 1 + } + } + os.Exit(code) +} + +// controlledPreparedResources is called only by serial controlled-reclamation tests. It holds no +// servers, credentials or bootstrap rows, so all mutable fixture state remains local to each case. +func controlledPreparedResources(t *testing.T) *fixture { + t.Helper() + if controlledPreparedFixture != nil { + return controlledPreparedFixture + } + pool, config := testSchemaWithCleanup(t, "test_controlled_", func(cleanup func() error) { + controlledFixtureCleanups = append(controlledFixtureCleanups, cleanup) + }) + store := &core.Store{Pool: pool} + must(t, store.Migrate(context.Background())) + must(t, store.Migrate(context.Background())) + root, err := os.MkdirTemp("", "ora-cloud-controlled-") + must(t, err) + controlledFixtureCleanups = append(controlledFixtureCleanups, func() error { return os.RemoveAll(root) }) + source, commit := prepareFixtureSource(t, root) + controlledPreparedFixture = &fixture{store: store, pgConfig: config, source: source, commit: commit} + return controlledPreparedFixture +} + +// controlledFixtureFactory is deliberately scoped to serial controlled-reclamation matrices. +// Claims, reconciliation scans and the Controller lease are schema-wide, so distinct tenant IDs +// alone do not isolate cases. Clear business rows between cases while retaining migration state +// and the migration-created scan singleton. Each child still gets a fresh Store and service stack. +func controlledFixtureFactory(t *testing.T) func(*testing.T) *fixture { + t.Helper() + prepared := controlledPreparedResources(t) + var tables string + must(t, prepared.store.Pool.QueryRow(`SELECT string_agg(format('%I.%I',schemaname,tablename), ', ' ORDER BY tablename) + FROM pg_tables WHERE schemaname=current_schema() + AND tablename NOT IN ('schema_migrations','task_conversation_scan_state')`).Scan(&tables)) + return func(t *testing.T) *fixture { + t.Helper() + // All related business tables are named in one statement, preserving FK enforcement without + // CASCADE. The immutable migration records and independent scan singleton are not truncated. + _, err := prepared.store.Pool.Exec("TRUNCATE TABLE " + tables) + must(t, err) + _, err = prepared.store.Pool.Exec("UPDATE task_conversation_scan_state SET last_created_at=NULL,last_id=NULL") + must(t, err) + return setupWithPreparedResources(t, prepared) + } +} diff --git a/integration/agent_run_controlled_reclaim_migration_test.go b/integration/agent_run_controlled_reclaim_migration_test.go new file mode 100644 index 000000000..3dd089868 --- /dev/null +++ b/integration/agent_run_controlled_reclaim_migration_test.go @@ -0,0 +1,161 @@ +package integration + +import ( + "context" + "database/sql" + "testing" + + "github.com/google/uuid" +) + +const controlledReclaimMigration = "0058_agent_run_controlled_reclamation.sql" + +func insertControlledMigrationRoot(t *testing.T, pool *sql.DB, seed skeletonSeed, id, reclaim string) { + t.Helper() + _, err := pool.Exec(`INSERT INTO agent_run_reclamation_intents + (id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,reclaim_id, + control_epoch,actor_subject,reason,impact_confirmed) + VALUES($1,'terminate','residual_ledger',$2,$3,$4,$5,27,$6,4,'migration-operator','Retain original data',true)`, + id, seed.tenantID, seed.projectID, seed.runID, seed.runWorkspaceID, reclaim) + must(t, err) +} + +func TestControlledResidualMigrationFreshAndConstraints(t *testing.T) { + pool, _ := testSchema(t, "test_controlled_fresh_") + store := newStoreOnSchema(t, pool) + must(t, store.Migrate(context.Background())) + must(t, store.Migrate(context.Background())) + seed := seedAgentIssueRunSkeleton(t, pool) + root, reclaim := uuid.NewString(), uuid.NewString() + insertControlledMigrationRoot(t, pool, seed, root, reclaim) + for _, table := range []string{"agent_run_reclamation_intents", "agent_run_reclamation_targets", "agent_run_reclamation_command_receipts"} { + if !tableExists(t, pool, table) { + t.Fatalf("fresh migration omitted %s", table) + } + } + for _, tc := range []struct{ name, change string }{ + {"unknown state", "state='unknown'"}, + {"unknown source", "source='unknown'"}, + {"missing D8 source operation", "source='legacy_d8'"}, + {"unknown command", "command_kind='unknown'"}, + {"root without confirmation", "impact_confirmed=false"}, + {"root without responsibility", "reclaim_id=NULL"}, + {"empty reason", "reason=' '"}, + {"targets object", "targets='{}'"}, + {"evidence object", "evidence='{}'"}, + {"audit object", "audit='{}'"}, + {"negative generation", "runtime_generation=-1"}, + {"zero version", "version=0"}, + {"completion without evidence", "state='succeeded'"}, + {"unfinished with disposition", "disposition='released'"}, + {"unfinished with time", "completed_at=clock_timestamp()"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := pool.Exec("UPDATE agent_run_reclamation_intents SET "+tc.change+" WHERE id=$1", root) + wantPGError(t, err, "23514") + }) + } + _, err := pool.Exec(`UPDATE agent_run_reclamation_intents SET delete_operation_id=$2 WHERE id=$1`, root, uuid.NewString()) + wantPGError(t, err, "23503") + other := seed + other.runID, other.runWorkspaceID = newBoundRun(t, pool, seed) + // One Workspace and one reclaim identity each have exactly one root owner. + _, err = pool.Exec(`INSERT INTO agent_run_reclamation_intents + (id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,reclaim_id,control_epoch,actor_subject,reason,impact_confirmed) + VALUES($1,'terminate','residual_ledger',$2,$3,$4,$5,27,$6,4,'operator','Duplicate root',true)`, + uuid.NewString(), seed.tenantID, seed.projectID, seed.runID, seed.runWorkspaceID, uuid.NewString()) + wantPGError(t, err, "23505") + _, err = pool.Exec(`INSERT INTO agent_run_reclamation_intents + (id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,reclaim_id,control_epoch,actor_subject,reason,impact_confirmed) + VALUES($1,'terminate','residual_ledger',$2,$3,$4,$5,27,$6,4,'operator','Duplicate responsibility',true)`, + uuid.NewString(), other.tenantID, other.projectID, other.runID, other.runWorkspaceID, reclaim) + wantPGError(t, err, "23505") + child := uuid.NewString() + insertChild := `INSERT INTO agent_run_reclamation_intents + (id,root_intent_id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,control_epoch,actor_subject,reason,data_delete_confirmed) + VALUES($1,$2,'delete_data','residual_ledger',$3,$4,$5,$6,27,4,'operator','Separately confirmed deletion',true)` + _, err = pool.Exec(insertChild, uuid.NewString(), root, other.tenantID, other.projectID, other.runID, other.runWorkspaceID) + wantPGError(t, err, "23503") + _, err = pool.Exec(insertChild, child, root, seed.tenantID, seed.projectID, seed.runID, seed.runWorkspaceID) + must(t, err) + _, err = pool.Exec(insertChild, uuid.NewString(), root, seed.tenantID, seed.projectID, seed.runID, seed.runWorkspaceID) + wantPGError(t, err, "23505") + // An ensure plan is a durable target even if no sandbox_instances row was ever created. + target, sandbox := uuid.NewString(), uuid.NewString() + _, err = pool.Exec(`INSERT INTO agent_run_reclamation_targets(id,root_intent_id,sandbox_id,generation,request) + VALUES($1,$2,$3,27,'{"kind":"sandbox_terminate"}')`, target, root, sandbox) + must(t, err) + _, err = pool.Exec(`INSERT INTO agent_run_reclamation_targets(id,root_intent_id,sandbox_id,generation,request) + VALUES($1,$2,$3,27,'{"kind":"sandbox_terminate"}')`, uuid.NewString(), root, sandbox) + wantPGError(t, err, "23505") + _, err = pool.Exec(`INSERT INTO agent_run_reclamation_targets(id,root_intent_id,sandbox_id,generation,request) + VALUES($1,$2,$3,27,'{}')`, uuid.NewString(), uuid.NewString(), uuid.NewString()) + wantPGError(t, err, "23503") + _, err = pool.Exec(`UPDATE agent_run_reclamation_targets SET state='succeeded',confirmed_at=clock_timestamp(),confirmed_epoch=1,result='{"terminated":true}' WHERE id=$1`, target) + wantPGError(t, err, "23514") + _, err = pool.Exec(`UPDATE agent_run_reclamation_targets SET state='succeeded',confirmed_at=clock_timestamp(),confirmed_epoch=1,result='{"terminated":true,"lateEnsureFenced":true}' WHERE id=$1`, target) + must(t, err) + _, err = pool.Exec(`INSERT INTO agent_run_reclamation_command_receipts(id,tenant_id,idempotency_key,command_input,intent_id,actor_subject) + VALUES($1,$2,'one-shared-key','{"action":"terminate"}',$3,'operator')`, uuid.NewString(), seed.tenantID, root) + must(t, err) + _, err = pool.Exec(`INSERT INTO agent_run_reclamation_command_receipts(id,tenant_id,idempotency_key,command_input,intent_id,actor_subject) + VALUES($1,$2,'one-shared-key','{"action":"delete-data"}',$3,'operator')`, uuid.NewString(), seed.tenantID, child) + wantPGError(t, err, "23505") + _, err = pool.Exec(`INSERT INTO agent_run_reclamation_command_receipts(id,tenant_id,idempotency_key,command_input,intent_id,actor_subject) + VALUES($1,$2,'missing-intent','{}',$3,'operator')`, uuid.NewString(), seed.tenantID, uuid.NewString()) + wantPGError(t, err, "23503") +} + +func TestControlledResidualMigrationUpgradePreserves0057HistoryAndDoesNotInventD8Ledger(t *testing.T) { + pool, _ := testSchema(t, "test_controlled_upgrade_") + // Include the complete mainline schema through 0055 and the 0057 residual ledger before upgrading. + applyMigrationsUpTo(t, pool, migrationsBefore(t, controlledReclaimMigration)) + base := seedAgentIssueRunSkeleton(t, pool) + for _, state := range []string{"reconciling", "reclaiming", "cleared"} { + seed := base + if state != "reconciling" { + seed.runID, seed.runWorkspaceID = newBoundRun(t, pool, base) + } + seedResidualMigrationRow(t, pool, seed, map[string]string{"reconciling": "agent_start_failed", "reclaiming": "workspace_unavailable", "cleared": "agent_start_failed"}[state]) + if state != "reconciling" { + _, err := pool.Exec(`UPDATE agent_run_residuals SET reclaim_state=$2,reclaim_id=$3,reclaim_mode='controlled', + reclaim_authorization='{"kind":"legacy-controlled","operator":"original"}',reclaim_targets='[{"kind":"sandbox","id":"original"}]', + reclaim_evidence='[{"key":"original-fact","fact":{"state":"succeeded"}}]',reclaim_version=9, + reclaim_checked_at='2026-01-02T04:05:06Z',reclaim_conflicted_at='2026-01-02T04:06:07Z', + cleared_at=CASE WHEN $2='cleared' THEN '2026-01-03T03:04:05Z'::timestamptz END, + disposition=CASE WHEN $2='cleared' THEN 'terminated_data_retained' END WHERE run_id=$1`, seed.runID, state, uuid.NewString()) + must(t, err) + } + } + d8 := base + d8.runID, d8.runWorkspaceID = newBoundRun(t, pool, base) + _, err := pool.Exec(`UPDATE issue_runs SET phase='done',status='completed',completed_at='2026-01-04T03:04:05Z', + failure_reason='workspace_unavailable',result='{"deliveryState":"delivered"}' WHERE id=$1`, d8.runID) + must(t, err) + _, err = pool.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id,kind,state,step,request,idempotency_key,request_hash,error_code) + VALUES($1,$2,$3,$4,$5,'delete_workspace','failed','quiesce','{"original":"D8"}','historical-d8','fixture-request','node_unavailable')`, + uuid.NewString(), d8.tenantID, d8.userID, d8.projectID, d8.runWorkspaceID) + must(t, err) + const snapshot = `SELECT jsonb_build_object( + 'residuals',(SELECT jsonb_agg(to_jsonb(r) ORDER BY run_id) FROM agent_run_residuals r), + 'runs',(SELECT jsonb_agg(to_jsonb(r) ORDER BY id) FROM issue_runs r), + 'workspaces',(SELECT jsonb_agg(to_jsonb(w) ORDER BY id) FROM workspaces w), + 'operations',(SELECT jsonb_agg(to_jsonb(o) ORDER BY id) FROM operations o))::text` + var before, after string + must(t, pool.QueryRow(snapshot).Scan(&before)) + store := newStoreOnSchema(t, pool) + must(t, store.Migrate(context.Background())) + must(t, store.Migrate(context.Background())) + must(t, pool.QueryRow(snapshot).Scan(&after)) + if before != after { + t.Fatal("controlled migration changed 0057 responsibility/state/clear time or historical business rows") + } + var empty bool + must(t, pool.QueryRow(`SELECT NOT EXISTS(SELECT 1 FROM agent_run_reclamation_intents) + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_targets) + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_command_receipts) + AND NOT EXISTS(SELECT 1 FROM agent_run_residuals WHERE run_id=$1)`, d8.runID).Scan(&empty)) + if !empty { + t.Fatal("migration invented a controlled command, effect, receipt or D8 residual ledger") + } +} diff --git a/integration/agent_run_controlled_reclaim_test.go b/integration/agent_run_controlled_reclaim_test.go new file mode 100644 index 000000000..457556f72 --- /dev/null +++ b/integration/agent_run_controlled_reclaim_test.go @@ -0,0 +1,421 @@ +package integration + +import ( + "context" + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/status" + + "github.com/wanglongan587/cloud/internal/controlpb" + "github.com/wanglongan587/cloud/internal/core" +) + +func controlledTerminate(t *testing.T, f *fixture, tenant, run, workspace string) core.Object { + t.Helper() + out, err := f.store.ManageResidual(context.Background(), &core.ResidualManagementRequest{ + Action: core.ResidualTerminate, TenantID: tenant, RunID: run, WorkspaceID: workspace, + Key: "controlled-terminate-" + run, Reason: "Close the original residual runtime and retain its data", ImpactConfirmed: true, + }) + must(t, err) + if out.O("intent").S("id") == "" { + t.Fatalf("accepted command has no durable intent: %v", out) + } + return out +} + +// controlledConfirmInjected supplies Controller termination evidence over the real bufconn +// contract. It proves Cloud's target/permit/transaction handling, not actual Docker termination. +func controlledConfirmInjected(t *testing.T, f *fixture, c *grpcController, intent string) map[string]bool { + t.Helper() + api := controlpb.NewRuntimeControlServiceClient(f.controlConn) + pending, err := api.ListForceStops(c.ctx, &controlpb.ListForceStopsRequest{Epoch: c.epoch}) + must(t, err) + var plan *controlpb.RuntimeForceStopPlan + for _, candidate := range pending.GetPlans() { + if candidate.GetId() == intent { + plan = candidate + } + } + if plan == nil { + t.Fatalf("controlled intent %s was absent from ListForceStops: %v", intent, pending) + } + targets := map[string]bool{} + version := plan.GetVersion() + if len(plan.GetEffects()) == 0 { + _, err = api.ConfirmForceStop(c.ctx, &controlpb.ConfirmForceStopRequest{SubmissionId: c.sub(), Epoch: c.epoch, ForceStopId: intent, Version: version}) + must(t, err) + } + for _, effect := range plan.GetEffects() { + sandbox := effect.GetRequest().GetSandboxTerminate().GetSandboxInstanceId() + if sandbox == "" || targets[sandbox] { + t.Fatalf("force-stop plan lost or duplicated an original target: %v", plan) + } + targets[sandbox] = true + if effect.GetState() == controlpb.EffectState_EFFECT_STATE_SUCCEEDED { + continue + } + permit, permitErr := api.GetEffectPermit(c.ctx, &controlpb.GetEffectPermitRequest{Epoch: c.epoch, ForceStopId: intent, EffectId: effect.GetId()}) + must(t, permitErr) + if permit.GetPermit().GetSandboxId() != sandbox || permit.GetPermit().GetWorkspaceId() != plan.GetWorkspaceId() || permit.GetPermit().GetExpiresAtMs() <= permit.GetPermit().GetIssuedAtMs() { + t.Fatalf("permit did not bind the original target and a live lease: %v", permit) + } + _, err = api.ConfirmForceStop(c.ctx, &controlpb.ConfirmForceStopRequest{SubmissionId: c.sub(), Epoch: c.epoch, ForceStopId: intent, Version: version, EffectId: effect.GetId(), Terminated: true}) + if err == nil { + t.Fatal("termination without durable late-ensure fencing was accepted") + } + confirmed, confirmErr := api.ConfirmForceStop(c.ctx, &controlpb.ConfirmForceStopRequest{SubmissionId: c.sub(), Epoch: c.epoch, ForceStopId: intent, Version: version, EffectId: effect.GetId(), Terminated: true, LateEnsureFenced: true}) + must(t, confirmErr) + version = confirmed.GetVersion() + } + return targets +} + +func controlledDeleteInjected(t *testing.T, f *fixture, c *grpcController, tenant, root, workspace string) core.Object { + t.Helper() + accepted, err := f.store.ManageResidual(context.Background(), &core.ResidualManagementRequest{ + Action: core.ResidualDeleteData, TenantID: tenant, IntentID: root, Key: "controlled-delete-" + root, + Reason: "Delete retained residual data after separate confirmation", DataDeleteConfirmed: true, + }) + must(t, err) + op := c.claim().GetOperation() + if op.GetId() != accepted.O("operation").S("id") || op.GetStep() != controlpb.OperationStep_OPERATION_STEP_CLEANUP { + t.Fatalf("data deletion did not claim its exact cleanup-only operation: %v %v", accepted, op) + } + planned, err := c.operations.PlanEffect(c.ctx, &controlpb.PlanEffectRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: op.GetId(), Version: op.GetVersion(), WorkspaceId: workspace, Kind: controlpb.EffectKind_EFFECT_KIND_WORKSPACE_DATA_DELETE}) + must(t, err) + permit, err := controlpb.NewRuntimeControlServiceClient(f.controlConn).GetEffectPermit(c.ctx, &controlpb.GetEffectPermitRequest{Epoch: c.epoch, EffectId: planned.GetEffect().GetId()}) + must(t, err) + if permit.GetPermit().GetWorkspaceId() != workspace { + t.Fatalf("data-delete permit names another Workspace: %v", permit) + } + // Explicit injected physical evidence; no fixture writes a succeeded operation or disposition. + recorded, err := c.operations.RecordEffectResult(c.ctx, &controlpb.RecordEffectResultRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: op.GetId(), Version: planned.GetOperation().GetVersion(), + EffectId: planned.GetEffect().GetId(), State: controlpb.EffectState_EFFECT_STATE_SUCCEEDED, + ExternalId: "injected-delete-" + planned.GetEffect().GetId(), + Evidence: &controlpb.EffectEvidence{Evidence: &controlpb.EffectEvidence_WorkspaceDataDeleted{WorkspaceDataDeleted: &controlpb.WorkspaceDataDeleted{}}}, + }) + must(t, err) + finished, err := c.advance(recorded.GetOperation()) + must(t, err) + if finished.GetState() != controlpb.OperationState_OPERATION_STATE_SUCCEEDED { + t.Fatalf("real delete-result/advance path did not succeed: %v", finished) + } + return accepted +} + +func TestControlledResidualRetainedThenReleasedOverGRPC(t *testing.T) { + f := controlledFixtureFactory(t)(t) + f.bindBusinessHooks() + created := f.create("controlled-provisioning") + f.drain() + run := f.insertAgentRun(t, created.O("resource").S("id")) + provisioned, err := f.store.CreateRunWorkspace(context.Background(), run) + must(t, err) + workspace, operation := provisioned.O("workspace").S("id"), provisioned.O("operation").S("id") + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET phase='provisioning',status='dispatched',workspace_id=$2,version=version+1 WHERE id=$1`, run, workspace) + must(t, err) + c := newGRPCController(t, f) + op := c.claim().GetOperation() + if op.GetId() != operation { + t.Fatalf("unexpected original provisioning operation: %v", op) + } + ensure, err := c.operations.PlanEffect(c.ctx, &controlpb.PlanEffectRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: op.GetVersion(), WorkspaceId: workspace, Kind: controlpb.EffectKind_EFFECT_KIND_SANDBOX_ENSURE}) + must(t, err) + // A legacy NULL-scope ensure belongs to this original operation but never produced a sandbox + // row. The controlled target set must retain it instead of treating the missing row as absence. + neverEnsured := uuid.NewString() + _, err = f.store.Pool.Exec(`INSERT INTO external_effects(id,operation_id,project_id,workspace_id,kind,state,request,reconciled_epoch) + SELECT $1,id,project_id,NULL,'sandbox_ensure','planned',jsonb_build_object('kind','sandbox_ensure','workspaceId',$2::text),$3 FROM operations WHERE id=$4`, neverEnsured, workspace, c.epoch, operation) + must(t, err) + _, err = c.operations.DeferOperation(c.ctx, &controlpb.DeferOperationRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: ensure.GetOperation().GetVersion(), State: controlpb.DeferState_DEFER_STATE_BLOCKED, Reason: controlpb.DeferReason_DEFER_REASON_NODE_UNAVAILABLE, RetrySeconds: 60}) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) + must(t, err) + must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + var historical string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('run',to_jsonb(r),'reason',a.reason,'responsibility',a.responsibility,'operation',to_jsonb(o))::text + FROM issue_runs r JOIN agent_run_residuals a ON a.run_id=r.id JOIN operations o ON o.id=$2 WHERE r.id=$1`, run, operation).Scan(&historical)) + accepted := controlledTerminate(t, f, f.tid, run, workspace) + root := accepted.O("intent").S("id") + targets := controlledConfirmInjected(t, f, c, root) + if len(targets) != 2 || !targets[ensure.GetEffect().GetId()] || !targets[neverEnsured] { + t.Fatalf("controlled termination omitted an original ensure identity: %v", targets) + } + var clearedAt string + must(t, f.store.Pool.QueryRow(`SELECT cleared_at::text FROM agent_run_residuals WHERE run_id=$1 AND reclaim_state='cleared' AND disposition='terminated_data_retained'`, run).Scan(&clearedAt)) + if f.scalar(`SELECT count(*) FROM workspaces WHERE id=$1 AND deleted_at IS NULL AND NOT admission_open`, workspace) != 1 { + t.Fatal("termination must preserve data and keep admission closed") + } + if f.scalar(`SELECT count(*) FROM external_effects WHERE workspace_id=$1 AND kind='workspace_data_delete'`, workspace) != 0 { + t.Fatal("terminate implicitly planned data deletion") + } + deleted := controlledDeleteInjected(t, f, c, f.tid, root, workspace) + if deleted.O("intent").S("id") == root { + t.Fatal("data deletion did not retain its separate command intent") + } + var retained bool + must(t, f.store.Pool.QueryRow(`SELECT reclaim_state='cleared' AND disposition='released' AND cleared_at::text=$2 + AND (SELECT deleted_at IS NOT NULL AND NOT admission_open FROM workspaces WHERE id=workspace_id) + FROM agent_run_residuals WHERE run_id=$1`, run, clearedAt).Scan(&retained)) + if !retained { + t.Fatal("released completion reset the first clear time or lost actual Workspace deletion") + } + var after string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('run',to_jsonb(r),'reason',a.reason,'responsibility',a.responsibility,'operation',to_jsonb(o))::text + FROM issue_runs r JOIN agent_run_residuals a ON a.run_id=r.id JOIN operations o ON o.id=$2 WHERE r.id=$1`, run, operation).Scan(&after)) + if historical != after { + t.Fatal("controlled cleanup rewrote the original business settlement, reason, responsibility or failed creation") + } + got, err := f.store.ManageResidual(context.Background(), &core.ResidualManagementRequest{Action: core.ResidualGet, TenantID: f.tid, IntentID: root}) + must(t, err) + if got.O("intent").S("state") != "succeeded" || got.O("intent").S("disposition") != "terminated_data_retained" { + t.Fatalf("root lost its original retained completion after the child released data: %v", got) + } +} + +func TestControlledResidualHiddenWorkspaceRemains404ForAdminAndMember(t *testing.T) { + f := controlledFixtureFactory(t)(t) + scene := seedLiveThreadScene(t, f) + seedControlledTerminalIntake(t, f, scene) + workspace := f.runWorkspaceID(scene.runID) + controlledTerminate(t, f, scene.tenantID, scene.runID, workspace) + f.tid = scene.tenantID + admin := f.user + member, _ := f.addUser(t, "controlled-member", "Controlled member") + for _, user := range []core.Claims{admin, member} { + f.user = user + f.call("GET", f.path("/workspaces/"+workspace), nil, "", 404) + f.call("POST", f.path("/workspaces/"+workspace+"/force-stop"), core.Object{"version": 1, "reason": "Hidden run Workspace", "impactConfirmed": true}, uuid.NewString(), 404) + f.call("DELETE", f.path("/workspaces/"+workspace), core.Object{"version": 1, "sessionId": uuid.NewString()}, uuid.NewString(), 404) + } +} + +// The original work and target are production-dispatched; only the already terminal business +// failure is a fixture. All controlled states below are reached through commands and gRPC facts. +func seedControlledTerminalIntake(t *testing.T, f *fixture, scene liveThreadScene) { + t.Helper() + _, err := f.store.Pool.Exec(`UPDATE issue_runs SET phase='done',status='failed',thread_state='ended',idle_since=NULL, + failure_reason='agent_start_failed',completed_at=clock_timestamp(),version=version+1, + result='{"deliveryState":"failed","testBoundary":"preexisting_business_terminal"}'::jsonb WHERE id=$1`, scene.runID) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE workspaces SET admission_open=false,admission_epoch=admission_epoch+1,version=version+1 WHERE issue_run_id=$1`, scene.runID) + must(t, err) + _, err = f.store.Pool.Exec(`INSERT INTO agent_run_residuals(run_id,tenant_id,project_id,workspace_id,runtime_generation,responsibility,reason) + SELECT $1::uuid,w.tenant_id,w.project_id,w.id,w.runtime_generation, + jsonb_build_object('executions',(SELECT jsonb_agg(jsonb_build_object('executionId',e.execution_id)) FROM node_executions e WHERE e.workspace_id=w.id)), + 'agent_start_failed' FROM workspaces w WHERE w.issue_run_id=$1::uuid`, scene.runID) + must(t, err) +} + +func TestControlledResidualTerminalFactsAcrossRealStatesOverGRPC(t *testing.T) { + newFixture := controlledFixtureFactory(t) + for _, delivery := range []bool{false, true} { + for _, state := range []string{"terminating", "retained", "released"} { + name := "session/" + state + if delivery { + name = "delivery/" + state + } + t.Run(name, func(t *testing.T) { + f := newFixture(t) + var scene liveThreadScene + var execution string + sequence := uint64(2) + if delivery { + scene = deliveringScene(t, f) + execution = f.deliverRevision(t, scene) + sequence = 1 + } else { + scene = seedLiveThreadScene(t, f) + scene.start(t, f) + f.runningThread(t, scene) + execution = scene.executionID + } + seedControlledTerminalIntake(t, f, scene) + workspace := f.runWorkspaceID(scene.runID) + accepted := controlledTerminate(t, f, scene.tenantID, scene.runID, workspace) + root := accepted.O("intent").S("id") + // seedLiveThreadScene owns the existing ctrl-a lease used by fact takeover. + c := &grpcController{t: t, f: f, ctx: asController("ctrl-a"), operations: controlpb.NewWorkspaceOperationServiceClient(f.controlConn), epoch: 1} + if state != "terminating" { + controlledConfirmInjected(t, f, c, root) + } + if state == "released" { + controlledDeleteInjected(t, f, c, scene.tenantID, root, workspace) + } + controlledTerminalIntakeCase(t, f, scene, execution, sequence, delivery) + _, err := f.store.ManageResidual(context.Background(), &core.ResidualManagementRequest{Action: core.ResidualDeleteData, TenantID: scene.tenantID, IntentID: root, Key: "after-late-conflict", Reason: "Cannot bypass a committed physical fact conflict", DataDeleteConfirmed: true}) + if err == nil { + t.Fatal("a management command bypassed the durable late-fact conflict") + } + }) + } + } +} + +// A retained/released responsibility already contains physical disposal evidence. Rollback must +// preserve that complete baseline, rather than requiring an empty ledger as the step-1 scene did. +func controlledTerminalIntakeCase(t *testing.T, f *fixture, scene liveThreadScene, execution string, sequence uint64, delivery bool) { + t.Helper() + businessBefore := residualIntakeSnapshot(t, f, scene) + const intakeSnapshot = `SELECT jsonb_build_object('execution',to_jsonb(e),'residual',to_jsonb(r), + 'receipts',(SELECT jsonb_agg(to_jsonb(n) ORDER BY sequence) FROM node_event_receipts n WHERE n.execution_id=e.execution_id), + 'verifications',(SELECT jsonb_agg(to_jsonb(v)) FROM revision_verifications v WHERE v.execution_id=e.execution_id), + 'revisions',(SELECT jsonb_agg(to_jsonb(v)) FROM revisions v WHERE v.execution_id=e.execution_id))::text + FROM node_executions e JOIN agent_run_residuals r ON r.run_id=e.operation_id WHERE e.execution_id=$1` + var before, after string + must(t, f.store.Pool.QueryRow(intakeSnapshot, execution).Scan(&before)) + in := f.deliveryInputOf(execution) + result := sessionEndResult(scene, controlpb.AgentSessionEndReason_AGENT_SESSION_END_REASON_USER_ENDED, "") + conflict := sessionEndResult(scene, controlpb.AgentSessionEndReason_AGENT_SESSION_END_REASON_INTERRUPTED, "") + kind, outcome := "agent_session", "agent_session_ended" + if delivery { + kind, outcome = "deliver_revision", "revision_delivered" + result = deliveredResult(scene, in, revisionFinalCommit) + conflict = deliveredResult(scene, in, strings.Repeat("2", 40)) + f.push(in, revisionFinalCommit) + } + request := func(submission string, fact *controlpb.ExecutionResult) *controlpb.TakeOverNodeEventRequest { + return &controlpb.TakeOverNodeEventRequest{ + SubmissionId: submission, Epoch: 1, OperationId: scene.runID, + ExecutionId: execution, Sequence: sequence, Result: fact, Event: []byte("controlled original residual terminal event"), + } + } + ctx := asController("ctrl-a") + _, err := f.store.Pool.Exec(`CREATE FUNCTION reject_controlled_intake_evidence() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected controlled evidence persistence failure'; END $$; + CREATE TRIGGER reject_controlled_intake_evidence BEFORE UPDATE OF reclaim_evidence ON agent_run_residuals + FOR EACH ROW EXECUTE FUNCTION reject_controlled_intake_evidence()`) + must(t, err) + response, err := f.executions.TakeOverNodeEvent(ctx, request("controlled-terminal", result)) + if err == nil || response != nil { + t.Fatalf("failed evidence persistence was acknowledged: response=%v error=%v", response, err) + } + must(t, f.store.Pool.QueryRow(intakeSnapshot, execution).Scan(&after)) + if before != after { + t.Fatalf("failed intake did not restore its exact execution/receipt/Revision/ledger baseline:\nbefore=%s\nafter=%s", before, after) + } + _, err = f.store.Pool.Exec("DROP TRIGGER reject_controlled_intake_evidence ON agent_run_residuals; DROP FUNCTION reject_controlled_intake_evidence()") + must(t, err) + response, err = f.executions.TakeOverNodeEvent(ctx, request("controlled-terminal", result)) + must(t, err) + if response.GetRecord().GetExecutionId() != execution || response.GetRecord().GetResult() == nil { + t.Fatalf("successful ACK lost the original execution/result: %v", response) + } + var accepted bool + must(t, f.store.Pool.QueryRow(`SELECT e.result->>'outcome'=$3 AND e.last_event_sequence=$2 + AND EXISTS(SELECT 1 FROM node_event_receipts WHERE execution_id=e.execution_id AND sequence=$2) + AND EXISTS(SELECT 1 FROM jsonb_array_elements(r.reclaim_evidence) x WHERE x->>'key'='execution:'||e.execution_id + AND x->'fact'->>'kind'=$4 AND x->'fact'->>'state'='succeeded' AND x->>'receivedAt' IS NOT NULL) + FROM node_executions e JOIN agent_run_residuals r ON r.run_id=e.operation_id WHERE e.execution_id=$1`, execution, sequence, outcome, kind).Scan(&accepted)) + if !accepted || f.receiptEvent(execution, int64(sequence)) != "controlled original residual terminal event" { + t.Fatal("ACK preceded the durable canonical result, original event receipt or residual evidence") + } + if delivery { + row := f.revisionRow(scene.runID) + project, repository := f.runPlacement(scene.runID) + if row == nil || row.RunID != scene.runID || row.TenantID != scene.tenantID || row.WorkspaceID != f.runWorkspaceID(scene.runID) || row.ProjectID != project || row.RepositoryURL != repository || row.FinalCommit != revisionFinalCommit || row.BaseCommit != in.S("baseCommit") || row.RevisionRef != in.S("revisionRef") || row.DeliveryRef != in.S("deliveryRef") || row.HistoryKey != in.S("historyKey") { + t.Fatalf("late verified delivery registered the wrong original Revision: %#v", row) + } + if row.BundleKey != "" || row.BundleSize != nil { + t.Fatalf("late Git delivery registered a retired bundle: %#v", row) + } + if verdict := f.revisionVerification(execution); verdict.S("outcome") != "delivered" { + t.Fatalf("actual Git and history verification is missing: %v", verdict) + } + refs := f.refs.queried() + if len(refs) != 2 || refs[0] != in.S("deliveryRef") || refs[1] != in.S("deliveryRef") { + t.Fatalf("failed intake and successful retry did not verify the declared Git delivery ref: %v", refs) + } + probes := f.objects.probed() + if len(probes) != 2 || probes[0].S("key") != in.S("historyKey") || probes[1].S("key") != in.S("historyKey") { + t.Fatalf("failed intake and successful retry did not verify the declared history: %v", probes) + } + } + var canonical string + var evidenceVersion int64 + must(t, f.store.Pool.QueryRow(`SELECT e.result::text,r.reclaim_version FROM node_executions e + JOIN agent_run_residuals r ON r.run_id=e.operation_id WHERE e.execution_id=$1`, execution).Scan(&canonical, &evidenceVersion)) + for _, submission := range []string{"controlled-terminal", "controlled-terminal-replay"} { + _, err = f.executions.TakeOverNodeEvent(ctx, request(submission, result)) + must(t, err) + } + var replayStable bool + must(t, f.store.Pool.QueryRow(`SELECT reclaim_version=$2 AND + (SELECT count(*) FROM node_event_receipts WHERE execution_id=$1 AND sequence=$3)=1 + FROM agent_run_residuals WHERE run_id=$4`, execution, evidenceVersion, sequence, scene.runID).Scan(&replayStable)) + if !replayStable { + t.Fatal("replayed original terminal duplicated residual evidence or its receipt") + } + response, err = f.executions.TakeOverNodeEvent(ctx, request("controlled-terminal-conflict", conflict)) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("canonical contradiction did not return committed residual conflict: response=%v error=%v", response, err) + } + var audited bool + must(t, f.store.Pool.QueryRow(`SELECT r.reclaim_conflicted_at IS NOT NULL AND e.result=$2::jsonb + AND EXISTS(SELECT 1 FROM jsonb_array_elements(r.reclaim_evidence) x WHERE x->>'key'='execution:'||e.execution_id + AND x->'fact'->>'kind'='conflict' AND x->'fact'->'original'->'result'=$2::jsonb + AND x->'fact'->'incoming'->'result'<>$2::jsonb) + AND (SELECT count(*) FROM node_event_receipts WHERE execution_id=e.execution_id AND sequence=$3)=1 + FROM node_executions e JOIN agent_run_residuals r ON r.run_id=e.operation_id WHERE e.execution_id=$1`, execution, canonical, sequence).Scan(&audited)) + if !audited { + t.Fatal("conflict response rolled back its audit or changed the canonical result/receipt") + } + if delivery && (len(f.refs.queried()) != 2 || len(f.objects.probed()) != 2) { + t.Fatal("known replay or contradiction repeated Git or history verification") + } + if businessAfter := residualIntakeSnapshot(t, f, scene); businessAfter != businessBefore { + t.Fatalf("late controlled facts resurrected business state, admission or dispatch:\nbefore=%s\nafter=%s", businessBefore, businessAfter) + } +} + +func TestControlledResidualProvisioningFactConflictCommitsBeforeGRPCResponse(t *testing.T) { + f := controlledFixtureFactory(t)(t) + f.bindBusinessHooks() + created := f.create("controlled-late-create") + f.drain() + run := f.insertAgentRun(t, created.O("resource").S("id")) + provisioned, err := f.store.CreateRunWorkspace(context.Background(), run) + must(t, err) + workspace, operation := provisioned.O("workspace").S("id"), provisioned.O("operation").S("id") + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET phase='provisioning',status='dispatched',workspace_id=$2,version=version+1 WHERE id=$1`, run, workspace) + must(t, err) + c := newGRPCController(t, f) + op := c.claim().GetOperation() + planned, err := c.operations.PlanEffect(c.ctx, &controlpb.PlanEffectRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: op.GetVersion(), WorkspaceId: workspace, Kind: controlpb.EffectKind_EFFECT_KIND_SANDBOX_ENSURE}) + must(t, err) + _, err = c.operations.DeferOperation(c.ctx, &controlpb.DeferOperationRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: planned.GetOperation().GetVersion(), State: controlpb.DeferState_DEFER_STATE_BLOCKED, Reason: controlpb.DeferReason_DEFER_REASON_NODE_UNAVAILABLE, RetrySeconds: 60}) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) + must(t, err) + must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + accepted := controlledTerminate(t, f, f.tid, run, workspace) + controlledConfirmInjected(t, f, c, accepted.O("intent").S("id")) + fact := &controlpb.RecordProvisioningFactRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_EFFECT, + EffectId: planned.GetEffect().GetId(), State: controlpb.EffectState_EFFECT_STATE_SUCCEEDED, ExternalId: "late-original-ensure", + Evidence: &controlpb.EffectEvidence{Evidence: &controlpb.EffectEvidence_SandboxEnsured{SandboxEnsured: &controlpb.SandboxEnsured{SandboxInstanceId: planned.GetEffect().GetId(), NodeId: "late-node"}}}, + } + _, err = c.operations.RecordProvisioningFact(c.ctx, fact) + must(t, err) + fact.SubmissionId, fact.State, fact.Evidence = c.sub(), controlpb.EffectState_EFFECT_STATE_FAILED, nil + failure := "contradictory original ensure result" + fact.Failure = &failure + _, err = c.operations.RecordProvisioningFact(c.ctx, fact) + if status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("late provisioning contradiction did not return its stable conflict: %v", err) + } + var durable bool + must(t, f.store.Pool.QueryRow(`SELECT a.reclaim_conflicted_at IS NOT NULL AND a.reclaim_state='cleared' + AND a.disposition='terminated_data_retained' AND a.cleared_at IS NOT NULL + AND r.phase='done' AND r.status='failed' AND NOT w.admission_open AND w.deleted_at IS NULL + AND e.state='succeeded' AND e.external_id='late-original-ensure' + AND EXISTS(SELECT 1 FROM jsonb_array_elements(a.reclaim_evidence) x WHERE x->'fact'->>'kind'='conflict') + FROM agent_run_residuals a JOIN issue_runs r ON r.id=a.run_id JOIN workspaces w ON w.id=a.workspace_id + JOIN external_effects e ON e.id=$2 WHERE a.run_id=$1`, run, planned.GetEffect().GetId()).Scan(&durable)) + if !durable { + t.Fatal("conflict response preceded durable evidence or resurrected original state") + } +} diff --git a/integration/cloud_test.go b/integration/cloud_test.go index 9ea356db9..f93a48669 100644 --- a/integration/cloud_test.go +++ b/integration/cloud_test.go @@ -8,6 +8,7 @@ import ( "database/sql" "encoding/hex" "encoding/json" + "errors" "fmt" "net" "net/http" @@ -72,6 +73,19 @@ type fixture struct { // testSchema creates an isolated PostgreSQL schema for one test and returns a pool bound to it. // The schema is dropped on cleanup; REQUIRE_POSTGRES=1 turns a missing database into a failure. func testSchema(t *testing.T, prefix string) (*sql.DB, *pgx.ConnConfig) { + t.Helper() + return testSchemaWithCleanup(t, prefix, func(cleanup func() error) { + t.Cleanup(func() { + if err := cleanup(); err != nil { + t.Error(err) + } + }) + }) +} + +// testSchemaWithCleanup lets the controlled-only suite keep its schema until package shutdown. +// Ordinary callers still register exactly the same resource lifetime through testSchema. +func testSchemaWithCleanup(t *testing.T, prefix string, registerCleanup func(func() error)) (*sql.DB, *pgx.ConnConfig) { t.Helper() dsn := os.Getenv("TEST_DATABASE_URL") if dsn == "" { @@ -89,17 +103,11 @@ func testSchema(t *testing.T, prefix string) (*sql.DB, *pgx.ConnConfig) { must(t, e) config.RuntimeParams["search_path"] = schema pool := stdlib.OpenDB(*config) - t.Cleanup(func() { - if err := pool.Close(); err != nil { - t.Error(err) - } - _, err := admin.Exec("DROP SCHEMA " + schema + " CASCADE") - if err != nil { - t.Error(err) - } - if err = admin.Close(); err != nil { - t.Error(err) - } + registerCleanup(func() error { + poolErr := pool.Close() + _, dropErr := admin.Exec("DROP SCHEMA " + schema + " CASCADE") + adminErr := admin.Close() + return errors.Join(poolErr, dropErr, adminErr) }) return pool, config } @@ -111,7 +119,21 @@ var setGinTestModeOnce sync.Once func setup(t *testing.T) *fixture { t.Helper() - pool, config := testSchema(t, "test_") + return setupWithPreparedResources(t, nil) +} + +// setupWithPreparedResources keeps each case's Store, servers and runtime state independent while +// allowing serial controlled-reclamation cases to reuse their parent's migrated schema and source. +// The parent owns the shared pool and immutable source directory; a child owns everything else. +func setupWithPreparedResources(t *testing.T, prepared *fixture) *fixture { + t.Helper() + var pool *sql.DB + var config *pgx.ConnConfig + if prepared == nil { + pool, config = testSchema(t, "test_") + } else { + pool, config = prepared.store.Pool, prepared.pgConfig + } db, e := gorm.Open(postgres.New(postgres.Config{Conn: pool}), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) must(t, e) store, e := core.NewStore(db) @@ -123,8 +145,10 @@ func setup(t *testing.T) *fixture { // Plugin execution has no real executor yet; the simulation capability is what the fixtures above // stand in for, and it must be opted into explicitly (see plugin_maintenance.go). store.PluginExecution = core.PluginExecutionSimulation - must(t, store.Migrate(context.Background())) - must(t, store.Migrate(context.Background())) + if prepared == nil { + must(t, store.Migrate(context.Background())) + must(t, store.Migrate(context.Background())) + } credentials, e := simulator.NewCredentials() must(t, e) auth, e := core.NewAuthenticator("ora-cloud", credentials.Trust) @@ -141,15 +165,12 @@ func setup(t *testing.T) *fixture { t.Error(err) } }) - repo := filepath.Join(root, "source") - must(t, os.MkdirAll(repo, 0o700)) - runGit(t, "init", "--initial-branch=main", repo) - runGit(t, "-C", repo, "config", "user.name", "Integration") - runGit(t, "-C", repo, "config", "user.email", "integration@example.invalid") - must(t, os.WriteFile(filepath.Join(repo, "README.md"), []byte("durable workspace data\n"), 0o600)) - runGit(t, "-C", repo, "add", ".") - runGit(t, "-C", repo, "commit", "-m", "fixture") - commit := runGit(t, "-C", repo, "rev-parse", "HEAD") + var repo, commit string + if prepared == nil { + repo, commit = prepareFixtureSource(t, root) + } else { + repo, commit = prepared.source, prepared.commit + } git, e := simulator.NewGitServer(filepath.Join(root, "git")) must(t, e) refs := &gitRefs{git: git, pinned: map[string]string{}} @@ -175,6 +196,21 @@ func setup(t *testing.T) *fixture { return f } +// prepareFixtureSource creates the immutable repository used as the external line's initial HEAD. +// Runtime directories and mutable Git Server repositories remain owned by each individual fixture. +func prepareFixtureSource(t *testing.T, root string) (string, string) { + t.Helper() + repo := filepath.Join(root, "source") + must(t, os.MkdirAll(repo, 0o700)) + runGit(t, "init", "--initial-branch=main", repo) + runGit(t, "-C", repo, "config", "user.name", "Integration") + runGit(t, "-C", repo, "config", "user.email", "integration@example.invalid") + must(t, os.WriteFile(filepath.Join(repo, "README.md"), []byte("durable workspace data\n"), 0o600)) + runGit(t, "-C", repo, "add", ".") + runGit(t, "-C", repo, "commit", "-m", "fixture") + return repo, runGit(t, "-C", repo, "rev-parse", "HEAD") +} + // controlConn serves the gRPC control surface for store over bufconn; the simulator Controller // uses it for the clone step, whose executions only that contract carries. func controlConn(t *testing.T, store *core.Store) *grpc.ClientConn { diff --git a/internal/core/README.en.md b/internal/core/README.en.md index 392dcae96..71d5d8cc7 100644 --- a/internal/core/README.en.md +++ b/internal/core/README.en.md @@ -16,6 +16,7 @@ This package owns business state, authorization, transactions and recovery decis | `agent_run_*.go`, `execution*.go`, `thread*.go`, `business_hooks.go` | Run lifecycle, execution registry, Thread and transaction-scoped settlement hooks | [IssueRun](../../docs/agent-runs/issue-runs.en.md), [Controller](../../docs/runtime/controller-integration.en.md) | | `agent_run_residual.go`, `agent_run_residual_reclaim.go` | Original identity/target coverage and evidence eligibility, one reclamation responsibility and one owned-delete operation, effect/permit and completion checks | [Residual reclamation](../../docs/agent-runs/residual-reclamation.en.md) | | `agent_run_residual_fact.go`, `agent_run_residual_loop.go` | Appended facts without rewriting old responsibility, replay and durable conflict blocking; ledger scans independent of Run phase and original-operation recovery | [Residual reclamation](../../docs/agent-runs/residual-reclamation.en.md) | +| `agent_run_controlled_reclaim*.go` | Deployment-operator intents and command receipts, one root responsibility and dependent data deletion, per-target termination and external confirmation, and D8 guards without ledger backfill | [Residual reclamation](../../docs/agent-runs/residual-reclamation.en.md) | | `task_conversation_*.go`, `thread_preview.go` | Ordered task messages, serial same-task Agent handoff and pause barriers, follow-up prompts frozen from Node visible text, configuration-gap hints and a bounded in-process live preview; initial Agent prompts read only the frozen run input | [Thread](../../docs/agent-runs/threads.en.md), [IssueRun](../../docs/agent-runs/issue-runs.en.md) | | `revision*.go`, `session_*.go` | Revision registration, archives, session listing and statistics | [Revision](../../docs/agent-runs/revisions.en.md), [Session reads](../../docs/agent-runs/session-reads.en.md) | | `model_*.go` | Personal configuration, frozen bindings, grant digests and request observations; no API Key decryption | [Model access](../../docs/models/model-access.en.md) | @@ -36,5 +37,11 @@ See [migrations](migrations/README.en.md) for complete migration filenames. Pack Residual facts retain their original intake: `agent_run_provisioning_fact.go` handles creation facts, `node_execution.go` and `revision.go` handle original execution terminals and delivery, and `runtime_control_delivery.go` accepts full runtime evidence through existing `runtime_ack`. `run_workspace.go`, `control.go` and effect/permit paths enforce the original responsibility binding. No new RPC/proto or public system-delete -flag is introduced. This is Cloud step 1 only; see the dedicated page for conservative A1–A6 eligibility and remaining scope, without implying -physical or joint acceptance. +flag is introduced. Step 2 uses `cloudctl residual` with deployment database credentials and authenticated `session_user`; an operator label is +self-reported. No HTTP route or tenant privilege is added. One root terminate and at most one dependent delete_data intent each follow three +states. Root completion never regresses; deletion binds one operation and preserves the first `cleared_at`. External confirmation requires an +existing unfinished intent and per-target fact references, without fabricating physical success. +Controlled targets use existing `ListForceStops` / `ConfirmForceStop` / `GetEffectPermit` fields, independently of ordinary force-stop history +and business completion. D8 takeover is explicit and uses original identity without ledger backfill. See the dedicated page for automatic +A1–A6 eligibility, two-stage disposition and recovery guards. This describes the Cloud implementation contract, not passing tests; +Controller/Sandbox physical proof and cross-repository fault acceptance remain steps 3 and 4 respectively. diff --git a/internal/core/README.md b/internal/core/README.md index 93b74f2a3..0f8ce9c27 100644 --- a/internal/core/README.md +++ b/internal/core/README.md @@ -16,6 +16,7 @@ | `agent_run_*.go`、`execution*.go`、`thread*.go`、`business_hooks.go` | 运行生命周期、执行登记、Thread 与同事务结算钩子 | [IssueRun](../../docs/agent-runs/issue-runs.md)、[Controller](../../docs/runtime/controller-integration.md) | | `agent_run_residual.go`、`agent_run_residual_reclaim.go` | 原身份/目标覆盖与证据资格、单一回收责任和唯一 owned-delete operation、effect/许可及完成检查 | [残留回收](../../docs/agent-runs/residual-reclamation.md) | | `agent_run_residual_fact.go`、`agent_run_residual_loop.go` | 不改旧责任的追加事实、重放与持久冲突阻挡；独立于 Run phase 的账本扫描和原 operation 恢复 | [残留回收](../../docs/agent-runs/residual-reclamation.md) | +| `agent_run_controlled_reclaim*.go` | 部署运维意图与命令回执、单一根责任和从属数据删除、逐目标终止与外部确认、无账本 D8 保护 | [残留回收](../../docs/agent-runs/residual-reclamation.md) | | `task_conversation_*.go`、`thread_preview.go` | 任务级有序消息、同任务 Agent 串行交接与暂停屏障、从 Node 可见正文冻结的后续提示、配置缺口提示和有界进程内实时预览；Agent 初始提示只读冻结的运行输入 | [Thread](../../docs/agent-runs/threads.md)、[IssueRun](../../docs/agent-runs/issue-runs.md) | | `revision*.go`、`session_*.go` | Revision 登记、归档读取、会话列表与统计 | [Revision](../../docs/agent-runs/revisions.md)、[会话读取](../../docs/agent-runs/session-reads.md) | | `model_*.go` | 个人配置、冻结绑定、授权摘要与请求观察；不解密 API Key | [模型访问](../../docs/models/model-access.md) | @@ -35,4 +36,9 @@ 残留事实继续由原入口接管：`agent_run_provisioning_fact.go` 处理创建侧，`node_execution.go` 与 `revision.go` 处理原执行终态和交付， `runtime_control_delivery.go` 接收已有 `runtime_ack` 的完整运行时证据。`run_workspace.go`、`control.go` 及 effect/许可路径检查原责任绑定； -不新增 RPC/proto 或公开的系统删除开关。当前仅为 Cloud 第 1 步实现，A1–A6 的保守资格和后续未实现范围见上述专页，不代表物理或联合验收完成。 +不新增 RPC/proto 或公开的系统删除开关。第 2 步受控入口为 `cloudctl residual`：以部署数据库凭据及 `session_user` 识别认证主体， +人工 operator 标签仅为自述，不新增 HTTP 或租户权限。根 terminate 与最多一个从属 delete_data 各自三态推进；根完成不回退， +删除只绑定一个 operation，保留首次 `cleared_at`。外部确认仅面向已有未完成意图，并要求逐目标事实引用，不替代物理成功证据。 +受控目标借既有 `ListForceStops` / `ConfirmForceStop` / `GetEffectPermit` 字段交付，普通 force-stop 历史与业务收尾保持独立。 +D8 按原身份显式接管，不回填账本。自动层 A1–A6、两阶段处置和恢复保护见上述专页；本文说明 Cloud 实现合同， +不声明测试通过，Controller/Sandbox 物理证明及跨仓故障验收仍分别属于第 3/4 步。 diff --git a/internal/core/agent_run_controlled_reclaim.go b/internal/core/agent_run_controlled_reclaim.go new file mode 100644 index 000000000..2f88d644b --- /dev/null +++ b/internal/core/agent_run_controlled_reclaim.go @@ -0,0 +1,168 @@ +package core + +import "context" + +// ResidualManagementAction selects a deployment-only command, never a tenant HTTP action. +type ResidualManagementAction string + +const ( + // ResidualList queries residual responsibilities in a tenant and optional resource scope. + ResidualList ResidualManagementAction = "list" + // ResidualGet returns a responsibility's plans, evidence and audit. + ResidualGet ResidualManagementAction = "get" + // ResidualTerminate explicitly abandons unconfirmed output and terminates original targets. + ResidualTerminate ResidualManagementAction = "terminate" + // ResidualDeleteData separately authorizes deletion after confirmed termination. + ResidualDeleteData ResidualManagementAction = "delete-data" + // ResidualConfirmExternal transfers an unfinished intent's responsibility to an operator. + ResidualConfirmExternal ResidualManagementAction = "confirm-external" +) + +// ResidualManagementRequest is accepted only by the local deployment command. Database +// session_user supplies the authenticated actor; OperatorLabel is self-reported, not authority. +type ResidualManagementRequest struct { + Action ResidualManagementAction + TenantID, ProjectID, RunID, WorkspaceID, OperationID, ReclaimID, IntentID string + Key, Reason, OperatorLabel, After string + Limit int + ImpactConfirmed, DataDeleteConfirmed, ResponsibilityTransferred bool + EvidenceNote string + ExternalFacts map[string]string +} + +// ManageResidual uses the deployment's database credential, independently of tenant identities. +// A conflict response is returned only after its audit has committed. +func (s *Store) ManageResidual(ctx context.Context, r *ResidualManagementRequest) (Object, error) { + out, err := s.transact(ctx, func(t *transaction) Object { return manageResidualCommand(t, r) }) + if err == nil && out.S("managementConflict") != "" { + return nil, &Fault{Status: 409, Code: out.S("managementConflict"), Params: Object{}} + } + return out, err +} + +func manageResidualCommand(t *transaction, r *ResidualManagementRequest) Object { + request := *r + r = &request + require(validID(r.TenantID), 400, "invalid_tenant") + for _, id := range []string{r.ProjectID, r.RunID, r.WorkspaceID, r.OperationID, r.ReclaimID, r.IntentID, r.After} { + require(id == "" || validID(id), 400, "invalid_residual_scope") + } + if r.Action == ResidualList { + return listManagedResiduals(t, r) + } + if r.Action == ResidualGet { + return getManagedResidual(t, r) + } + require(r.Action == ResidualTerminate || r.Action == ResidualDeleteData || r.Action == ResidualConfirmExternal, 400, "invalid_residual_action") + require(r.Key != "" && len(r.Key) <= 200, 400, "idempotency_key_required") + r.Reason = validText(r.Reason, 2000) + require(len(r.OperatorLabel) <= 200, 400, "invalid_operator_label") + actor := t.one("SELECT session_user::text AS actor").S("actor") + input := Object{"action": string(r.Action), "tenantId": r.TenantID, "projectId": r.ProjectID, "runId": r.RunID, "workspaceId": r.WorkspaceID, "operationId": r.OperationID, "reclaimId": r.ReclaimID, "intentId": r.IntentID, "reason": r.Reason, "impactConfirmed": r.ImpactConfirmed, "dataDeleteConfirmed": r.DataDeleteConfirmed, "responsibilityTransferred": r.ResponsibilityTransferred, "evidenceNote": r.EvidenceNote, "externalFacts": r.ExternalFacts, "operatorLabel": r.OperatorLabel} + if old := t.one("SELECT * FROM agent_run_reclamation_command_receipts WHERE tenant_id=$1 AND idempotency_key=$2", r.TenantID, r.Key); old != nil { + if jsonText(old.O("commandInput")) != jsonText(input) { + event := Object{"kind": "idempotency_conflict", "actorSubject": actor, "incoming": input, "at": controlledNow(t)} + t.exec("UPDATE agent_run_reclamation_command_receipts SET audit=audit||jsonb_build_array($2::jsonb) WHERE id=$1", old.S("id"), jsonText(event)) + return Object{"managementConflict": "idempotency_conflict"} + } + return managedIntentView(t, old.S("intentId")) + } + var intent Object + switch r.Action { + case ResidualTerminate: + intent = registerControlledTermination(t, r, actor) + case ResidualDeleteData: + intent = registerControlledDataDelete(t, r, actor) + case ResidualConfirmExternal: + intent = confirmControlledExternal(t, r, actor) + } + t.exec(`INSERT INTO agent_run_reclamation_command_receipts(id,tenant_id,idempotency_key,command_input,intent_id,actor_subject,result) + VALUES($1,$2,$3,$4,$5,$6,$7)`, newID(), r.TenantID, r.Key, jsonText(input), intent.S("id"), actor, jsonText(Object{"intentId": intent.S("id")})) + return managedIntentView(t, intent.S("id")) +} + +func controlledNow(t *transaction) string { + return t.one("SELECT clock_timestamp()::text AS at").S("at") +} + +func controlledAudit(t *transaction, id, kind, actor string, detail Object) { + event := Object{"kind": kind, "actorSubject": actor, "at": controlledNow(t), "detail": detail} + t.exec("UPDATE agent_run_reclamation_intents SET audit=audit||jsonb_build_array($2::jsonb),version=version+1 WHERE id=$1", id, jsonText(event)) +} + +func registerControlledTermination(t *transaction, r *ResidualManagementRequest, actor string) Object { + require(r.ImpactConfirmed, 400, "force_stop_impact_confirmation_required") + require(r.RunID != "" || r.WorkspaceID != "", 400, "residual_target_required") + w := t.one(`SELECT * FROM workspaces WHERE tenant_id=$1 AND ($2::uuid IS NULL OR issue_run_id=$2) + AND ($3::uuid IS NULL OR id=$3) AND ($4::uuid IS NULL OR project_id=$4)`, r.TenantID, nullable(r.RunID), nullable(r.WorkspaceID), nullable(r.ProjectID)) + require(w != nil && w.S("issueRunId") != "" && w.S("kind") == "isolated" && w["deletedAt"] == nil, 404, "residual_not_found") + if root := controlledRootForWorkspace(t, w.S("id")); root != nil { + require(controlledRootRow(t, root)["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + require(r.OperationID == "" || r.OperationID == root.S("legacyDeleteOperationId"), 409, "residual_identity_conflict") + return root + } + row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", w.S("issueRunId")) + source := "residual_ledger" + if row == nil { + source = "legacy_d8" + old := t.one(`SELECT o.* FROM operations o JOIN issue_runs i ON i.id=$2 WHERE o.id=$1 AND o.tenant_id=$3 + AND o.workspace_id=$4 AND o.kind='delete_workspace' AND o.state='failed' AND o.error_code='node_unavailable' AND i.phase='done'`, nullable(r.OperationID), w.S("issueRunId"), r.TenantID, w.S("id")) + require(old != nil, 409, "legacy_residual_identity_required") + row = Object{"runId": w.S("issueRunId"), "tenantId": r.TenantID, "projectId": w.S("projectId"), "workspaceId": w.S("id"), "runtimeGeneration": w.N("runtimeGeneration"), "responsibility": Object{}, "reclaimTargets": []Object{}} + } else { + require(r.OperationID == "", 400, "invalid_residual_scope") + require(row.S("reclaimState") == "reconciling", 409, "residual_mode_conflict") + require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + } + _, _, p := residualIdentity(t, row) + require(p.S("lifecycle") == "active" && !w.B("admissionOpen"), 409, "resource_unavailable") + require(t.one("SELECT id FROM operations WHERE project_id=$1 AND state IN ('queued','running','retry_wait','blocked')", p.S("id")) == nil, 409, "resource_in_use") + requireNoForceStop(t, w.S("id")) + id, reclaim := newID(), newID() + c := runtimeControl(t, w.S("id")) + t.exec(`INSERT INTO agent_run_reclamation_intents(id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,reclaim_id,legacy_delete_operation_id,control_epoch,actor_subject,operator_label,reason,impact_confirmed) + VALUES($1,'terminate',$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,true)`, id, source, r.TenantID, p.S("id"), row.S("runId"), w.S("id"), row.N("runtimeGeneration"), reclaim, nullable(r.OperationID), c.N("controlEpoch"), actor, r.OperatorLabel, r.Reason) + if source == "residual_ledger" { + claimed := claimResidualReclamation(t, row.S("runId"), "controlled", Object{"kind": "controlled_intent", "intentId": id, "source": "cloudctl"}) + require(claimed.B("claimed"), 409, "resource_in_use") + row = claimed.O("residual") + t.exec("UPDATE agent_run_reclamation_intents SET reclaim_id=$2 WHERE id=$1", id, row.S("reclaimId")) + } + closeAdmission(t, w, "stopped") + t.exec(`UPDATE runtime_controls SET state='reconciling',control_epoch=control_epoch+1,session_id=NULL,holder_user_id=NULL, + expires_at=NULL,binding_confirmed=false,input_closed=false,version=version+1,updated_at=clock_timestamp() WHERE workspace_id=$1`, w.S("id")) + t.exec("UPDATE agent_run_reclamation_intents SET control_epoch=(SELECT control_epoch FROM runtime_controls WHERE workspace_id=$2) WHERE id=$1", id, w.S("id")) + root := t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", id) + refreshControlledTargets(t, root) + controlledAudit(t, id, "registered", actor, Object{"impactConfirmed": true, "operatorLabel": r.OperatorLabel, "source": source}) + return t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", id) +} + +func controlledRootForWorkspace(t *transaction, wid string) Object { + if !validID(wid) { + return nil + } + return t.one("SELECT * FROM agent_run_reclamation_intents WHERE workspace_id=$1 AND root_intent_id IS NULL", wid) +} + +func controlledIntentRoot(t *transaction, intent Object) Object { + if intent.S("rootIntentId") == "" { + return intent + } + return t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", intent.S("rootIntentId")) +} + +func controlledRootRow(t *transaction, root Object) Object { + if root.S("source") == "residual_ledger" { + return t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", root.S("runId")) + } + state := "reclaiming" + if root.S("state") == "succeeded" { + state = "cleared" + } + disposition := root.S("disposition") + if child := t.one("SELECT disposition FROM agent_run_reclamation_intents WHERE root_intent_id=$1 AND state='succeeded'", root.S("id")); child != nil { + disposition = child.S("disposition") + } + return Object{"runId": root.S("runId"), "tenantId": root.S("tenantId"), "projectId": root.S("projectId"), "workspaceId": root.S("workspaceId"), "runtimeGeneration": root.N("runtimeGeneration"), "responsibility": Object{}, "reclaimId": root.S("reclaimId"), "reclaimMode": "controlled", "reclaimState": state, "reclaimOperationId": root.S("deleteOperationId"), "reclaimTargets": root["targets"], "reclaimEvidence": root["evidence"], "reclaimConflictedAt": root["conflictedAt"], "clearedAt": root["completedAt"], "disposition": disposition, "legacyIntentId": root.S("id")} +} diff --git a/internal/core/agent_run_controlled_reclaim_db_test.go b/internal/core/agent_run_controlled_reclaim_db_test.go new file mode 100644 index 000000000..fa1b7123f --- /dev/null +++ b/internal/core/agent_run_controlled_reclaim_db_test.go @@ -0,0 +1,498 @@ +package core + +import ( + "errors" + "fmt" + "testing" + "time" +) + +func seedControlledRequest(scene residualReclaimScene, key string) ResidualManagementRequest { + r := ResidualManagementRequest{Action: ResidualTerminate, TenantID: scene.tenant, ProjectID: scene.project, RunID: scene.run, WorkspaceID: scene.workspace, Key: key, Reason: "deployment operator accepted residual impact", OperatorLabel: "test operator", ImpactConfirmed: true} + if scene.responsibility == "" { + r.OperationID = scene.operation + } + return r +} + +func seedControlledManage(t *testing.T, s *Store, r ResidualManagementRequest) Object { + t.Helper() + out, err := s.ManageResidual(t.Context(), &r) + if err != nil { + t.Fatalf("manage residual %s: %v", r.Action, err) + } + return out +} + +func seedControlledRow(t *testing.T, s *Store, id string) Object { + t.Helper() + out, err := s.transact(t.Context(), func(tx *transaction) Object { + return tx.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", id) + }) + if err != nil { + t.Fatal(err) + } + return out +} + +func seedControlledRoot(t *testing.T, scene residualReclaimScene) Object { + t.Helper() + out := seedControlledManage(t, scene.store, seedControlledRequest(scene, "terminate-"+newID())) + intent := out.O("intent") + if intent.S("id") == "" || intent.S("commandKind") != "terminate" || intent.S("state") != "registered" { + t.Fatalf("terminate did not register an explicit root: %v", out) + } + return intent +} + +// List and confirm through the same Controller API as the deployed force-stop transport. The +// test never writes succeeded/cleared or target proof directly to manufacture completion. +func seedControlledPlan(t *testing.T, scene residualReclaimScene, id string) Object { + t.Helper() + out, err := scene.store.Control(t.Context(), &ControlRequest{Action: "force_pending", Body: Object{"epoch": 1}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + for _, plan := range residualObjects(out["plans"]) { + if plan.O("forceStop").S("id") == id { + return plan + } + } + t.Fatalf("controlled root %s missing from force_pending: %v", id, out) + return nil +} + +func seedControlledConfirm(t *testing.T, scene residualReclaimScene, id, effect string, proof Object) (Object, error) { + t.Helper() + intent := seedControlledRow(t, scene.store, id) + body := Object{"epoch": 1, "forceStopId": id, "version": intent.N("version"), "effectId": effect} + for key, value := range proof { + body[key] = value + } + return scene.store.Control(t.Context(), &ControlRequest{Action: "force_confirm", Body: body, Service: scene.claims}) +} + +func seedControlledRetained(t *testing.T, scene residualReclaimScene) Object { + t.Helper() + root := seedControlledRoot(t, scene) + plan := seedControlledPlan(t, scene, root.S("id")) + for _, target := range residualObjects(plan["effects"]) { + if _, err := seedControlledConfirm(t, scene, root.S("id"), target.S("id"), Object{"terminated": true, "lateEnsureFenced": true}); err != nil { + t.Fatal(err) + } + } + if len(residualObjects(plan["effects"])) == 0 { + if _, err := seedControlledConfirm(t, scene, root.S("id"), "", nil); err != nil { + t.Fatal(err) + } + } + root = seedControlledRow(t, scene.store, root.S("id")) + if root.S("state") != "succeeded" || root.S("disposition") != "terminated_data_retained" { + t.Fatalf("physical dual confirmations did not retain data: %v", root) + } + return root +} + +func seedControlledDeleteRequest(scene residualReclaimScene, root Object, key string) ResidualManagementRequest { + return ResidualManagementRequest{Action: ResidualDeleteData, TenantID: scene.tenant, IntentID: root.S("id"), Key: key, Reason: "independent approval to delete retained workspace data", DataDeleteConfirmed: true} +} + +func seedControlledDeletePlan(t *testing.T, scene residualReclaimScene, root Object) (Object, Object, Object) { + t.Helper() + out := seedControlledManage(t, scene.store, seedControlledDeleteRequest(scene, root, "delete-"+newID())) + child := out.O("intent") + if child.S("id") == root.S("id") || child.S("rootIntentId") != root.S("id") || child.S("commandKind") != "delete_data" { + t.Fatalf("data deletion did not create a separate child intent: %v", out) + } + claimed, err := scene.store.Control(t.Context(), &ControlRequest{Action: "claim", Body: Object{"epoch": 1}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + op := claimed.O("operation") + if op.S("id") != out.O("operation").S("id") || op.S("step") != "cleanup" { + t.Fatalf("controlled delete did not start its exact operation at cleanup: %v / %v", out, claimed) + } + planned := seedResidualAction(t, scene, op, "plan", Object{"kind": "workspace_data_delete", "workspaceId": scene.workspace}, "") + return child, planned.O("operation"), planned.O("effect") +} + +func seedControlledExternalRequest(scene residualReclaimScene, intent Object, key string) ResidualManagementRequest { + facts := map[string]string{"data:" + scene.workspace: "incident-251/data-responsibility-transferred"} + for _, target := range residualObjects(intent["targets"]) { + if target.S("kind") == "sandbox" { + facts["sandbox:"+target.S("id")] = "incident-251/host-disposal/" + target.S("id") + } + } + if scene.effect != "" && intent.S("commandKind") == "terminate" { + facts["sandbox:"+scene.effect] = "incident-251/host-disposal/" + scene.effect + } + return ResidualManagementRequest{Action: ResidualConfirmExternal, TenantID: scene.tenant, IntentID: intent.S("id"), Key: key, Reason: "operator accepts unresolved physical responsibility", EvidenceNote: "incident-251 records each disposal responsibility", ResponsibilityTransferred: true, ExternalFacts: facts} +} + +func assertControlledRejected(t *testing.T, err error) { + t.Helper() + var fault *Fault + if !errors.As(err, &fault) || fault.Status < 400 || fault.Status >= 500 { + t.Fatalf("expected a stable command rejection, got %v", err) + } +} + +func TestControlledResidualCommands(t *testing.T) { + // Every invalid command rolls back; the assertions below keep this same pristine scene. + scene := seedControlledScene(t, controlledStores(t)(t), false) + for _, invalid := range []string{"impact", "reason", "tenant", "workspace", "key"} { + t.Run(invalid, func(t *testing.T) { + r := seedControlledRequest(scene, "invalid-command") + switch invalid { + case "impact": + r.ImpactConfirmed = false + case "reason": + r.Reason = " " + case "tenant": + r.TenantID = newID() + case "workspace": + r.WorkspaceID = newID() + case "key": + r.Key = "" + } + _, err := scene.store.ManageResidual(t.Context(), &r) + assertControlledRejected(t, err) + seedResidualAssert(t, scene, `SELECT reclaim_id IS NULL AND reclaim_state='reconciling' + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_intents WHERE workspace_id=$2) + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_targets x JOIN agent_run_reclamation_intents i ON i.id=x.root_intent_id WHERE i.workspace_id=$2) + FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.workspace) + }) + } + r := seedControlledRequest(scene, "one-command") + first := seedControlledManage(t, scene.store, r).O("intent") + replay := seedControlledManage(t, seedResidualRebuildStore(scene.store), r).O("intent") + if replay.S("id") != first.S("id") { + t.Fatal("same command replay substituted a new intent") + } + r.Reason = "different authorized input" + _, err := scene.store.ManageResidual(t.Context(), &r) + assertControlledRejected(t, err) + var actor string + if err := scene.store.Pool.QueryRow("SELECT session_user::text").Scan(&actor); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT i.actor_subject=$2 AND i.operator_label='test operator' AND i.reason=$3 AND i.impact_confirmed + AND r.reclaim_mode='controlled' AND r.reclaim_authorization->>'intentId'=i.id::text + AND r.responsibility=$4::jsonb AND NOT w.admission_open AND w.deleted_at IS NULL + AND (SELECT count(*) FROM agent_run_reclamation_intents WHERE workspace_id=w.id AND command_kind='terminate')=1 + AND NOT EXISTS(SELECT 1 FROM operations WHERE workspace_id=w.id AND kind='delete_workspace') + FROM agent_run_residuals r JOIN agent_run_reclamation_intents i ON i.reclaim_id=r.reclaim_id JOIN workspaces w ON w.id=r.workspace_id WHERE r.run_id=$1`, scene.run, actor, first.S("reason"), scene.responsibility) +} + +func TestControlledResidualEveryPlannedSandbox(t *testing.T) { + scene := seedControlledScene(t, controlledStores(t)(t), false) + // A historical ensure can have no sandbox row. Its stable original identity is still a + // termination target; the NULL workspace scope is resolved via its original operation. + neverEnsured := newID() + seedResidualExec(t, scene.store, `INSERT INTO external_effects(id,operation_id,project_id,workspace_id,kind,state,request,reconciled_epoch) + VALUES($1,$2,$3,NULL,'sandbox_ensure','planned',$4,1)`, neverEnsured, scene.operation, scene.project, jsonText(Object{"kind": "sandbox_ensure", "workspaceId": scene.workspace})) + root := seedControlledRoot(t, scene) + plan := seedControlledPlan(t, scene, root.S("id")) + targets := residualObjects(plan["effects"]) + seen := map[string]bool{} + for _, target := range targets { + sid := target.O("request").S("sandboxInstanceId") + seen[sid] = true + permit, err := scene.store.Control(t.Context(), &ControlRequest{Action: "effect_permit", EffectID: target.S("id"), Body: Object{"epoch": 1, "forceStopId": root.S("id")}, Service: scene.claims}) + if err != nil || permit.S("sandboxId") != sid || permit.S("workspaceId") != scene.workspace || permit.N("runtimeGeneration") != 1 || permit.N("expiresAtMs") <= permit.N("issuedAtMs") { + t.Fatalf("target permit did not bind stable identity: %v / %v", permit, err) + } + for _, proof := range []Object{{"terminated": true}, {"lateEnsureFenced": true}, {"terminated": false, "lateEnsureFenced": true}} { + _, err := seedControlledConfirm(t, scene, root.S("id"), target.S("id"), proof) + assertControlledRejected(t, err) + } + } + if !seen[scene.effect] || !seen[neverEnsured] || len(targets) != 2 { + t.Fatalf("planned sandbox union omitted an allocating or never ensured target: %v", plan) + } + if _, err := seedControlledConfirm(t, scene, root.S("id"), targets[0].S("id"), Object{"terminated": true, "lateEnsureFenced": true}); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT r.cleared_at IS NULL AND i.state<>'succeeded' AND w.deleted_at IS NULL + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=w.id AND kind='workspace_data_delete') + FROM agent_run_residuals r JOIN agent_run_reclamation_intents i ON i.reclaim_id=r.reclaim_id JOIN workspaces w ON w.id=r.workspace_id WHERE r.run_id=$1`, scene.run) + if _, err := seedControlledConfirm(t, scene, root.S("id"), targets[1].S("id"), Object{"terminated": true, "lateEnsureFenced": true}); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT r.disposition='terminated_data_retained' AND r.cleared_at IS NOT NULL + AND (SELECT count(*) FROM agent_run_reclamation_targets WHERE root_intent_id=i.id AND state='succeeded' AND result @> '{"terminated":true,"lateEnsureFenced":true}')=2 + AND NOT EXISTS(SELECT 1 FROM runtime_force_stops WHERE workspace_id=r.workspace_id) + FROM agent_run_residuals r JOIN agent_run_reclamation_intents i ON i.reclaim_id=r.reclaim_id WHERE r.run_id=$1`, scene.run) +} + +func TestControlledResidualRetainedThenReleased(t *testing.T) { + scene := seedControlledScene(t, controlledStores(t)(t), false) + root := seedControlledRetained(t, scene) + var clearedAt string + if err := scene.store.Pool.QueryRow(`SELECT cleared_at::text FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&clearedAt); err != nil { + t.Fatal(err) + } + r := seedControlledDeleteRequest(scene, root, "explicit-delete") + r.DataDeleteConfirmed = false + _, err := scene.store.ManageResidual(t.Context(), &r) + assertControlledRejected(t, err) + child, op, effect := seedControlledDeletePlan(t, scene, root) + seedResidualAssert(t, scene, `SELECT r.disposition='terminated_data_retained' AND r.cleared_at::text=$2 AND w.deleted_at IS NULL + AND i.state='succeeded' AND i.completed_at IS NOT NULL AND c.state<>'succeeded' AND c.root_intent_id=i.id + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN agent_run_reclamation_intents i ON i.reclaim_id=r.reclaim_id + JOIN agent_run_reclamation_intents c ON c.id=$3 WHERE r.run_id=$1`, scene.run, clearedAt, child.S("id")) + op = seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "controlled-data-delete", "result": Object{"removed": true}}, effect.S("id")).O("operation") + finished := seedResidualAction(t, scene, op, "advance", nil, "").O("operation") + if finished.S("state") != "succeeded" { + t.Fatalf("physical data result did not finish the bound operation: %v", finished) + } + seedResidualAssert(t, scene, `SELECT r.disposition='released' AND r.cleared_at::text=$2 AND r.responsibility=$3::jsonb + AND i.state='succeeded' AND i.disposition='terminated_data_retained' AND i.completed_at=$4::timestamptz + AND c.state='succeeded' AND c.disposition='released' AND c.delete_operation_id=$5 AND w.deleted_at IS NOT NULL + AND run.phase='done' AND run.status='failed' AND run.failure_reason='workspace_unavailable' + AND original.state='failed' + AND (SELECT count(*) FROM operations WHERE workspace_id=w.id AND kind='delete_workspace')=1 + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN issue_runs run ON run.id=r.run_id + JOIN operations original ON original.id=$6 JOIN agent_run_reclamation_intents i ON i.reclaim_id=r.reclaim_id + JOIN agent_run_reclamation_intents c ON c.id=$7 WHERE r.run_id=$1`, scene.run, clearedAt, scene.responsibility, root.S("completedAt"), finished.S("id"), scene.operation, child.S("id")) +} + +func TestControlledResidualExternalRequiresUnfinishedIntent(t *testing.T) { + scene := seedControlledScene(t, controlledStores(t)(t), false) + initialRoot := seedControlledRoot(t, scene) + for _, invalid := range []string{"no_intent", "reason", "note", "transfer", "missing_sandbox", "missing_data", "succeeded"} { + t.Run(invalid, func(t *testing.T) { + // Rejections leave the unfinished intent intact; the final case completes it first. + root := initialRoot + if invalid == "succeeded" { + root = seedControlledRetained(t, scene) + } + r := seedControlledExternalRequest(scene, root, "external-invalid") + switch invalid { + case "no_intent": + r.IntentID = newID() + case "reason": + r.Reason = " " + case "note": + r.EvidenceNote = " " + case "transfer": + r.ResponsibilityTransferred = false + case "missing_sandbox": + delete(r.ExternalFacts, "sandbox:"+scene.effect) + case "missing_data": + delete(r.ExternalFacts, "data:"+scene.workspace) + } + _, err := scene.store.ManageResidual(t.Context(), &r) + assertControlledRejected(t, err) + seedResidualAssert(t, scene, `SELECT w.deleted_at IS NULL AND NOT EXISTS(SELECT 1 FROM operations WHERE workspace_id=w.id AND kind='delete_workspace') + AND ($3 OR NOT EXISTS(SELECT 1 FROM agent_run_reclamation_targets WHERE root_intent_id=$2 AND result @> '{"terminated":true}')) + FROM workspaces w JOIN issue_runs run ON run.workspace_id=w.id WHERE run.id=$1`, scene.run, root.S("id"), invalid == "succeeded") + }) + } +} + +func TestControlledResidualExternalDisposition(t *testing.T) { + scene := seedControlledScene(t, controlledStores(t)(t), false) + root := seedControlledRoot(t, scene) + r := seedControlledExternalRequest(scene, root, "external-close") + out := seedControlledManage(t, scene.store, r) + if out.O("intent").S("disposition") != "externally_confirmed" { + t.Fatalf("external materials did not produce distinct disposition: %v", out) + } + if replay := seedControlledManage(t, scene.store, r); replay.O("intent").S("id") != root.S("id") { + t.Fatal("external command replay changed intent identity") + } + seedResidualAssert(t, scene, `SELECT r.reclaim_state='cleared' AND r.disposition='externally_confirmed' AND r.cleared_at IS NOT NULL + AND r.responsibility=$2::jsonb AND w.deleted_at IS NULL AND NOT w.admission_open AND sb.terminated_at IS NULL + AND i.state='succeeded' AND i.disposition='externally_confirmed' + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_targets WHERE root_intent_id=i.id AND state='succeeded') + AND NOT EXISTS(SELECT 1 FROM operations WHERE workspace_id=w.id AND kind='delete_workspace') + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN sandbox_instances sb ON sb.id=$3 + JOIN agent_run_reclamation_intents i ON i.reclaim_id=r.reclaim_id WHERE r.run_id=$1`, scene.run, scene.responsibility, scene.effect) + deleteRequest := seedControlledDeleteRequest(scene, root, "external-cannot-authorize-delete") + _, err := scene.store.ManageResidual(t.Context(), &deleteRequest) + assertControlledRejected(t, err) +} + +func TestControlledResidualGuardsAndRollback(t *testing.T) { + scene := seedControlledScene(t, controlledStores(t)(t), false) + r := seedControlledRequest(scene, "rolled-back-command") + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + manageResidualCommand(tx, &r) + reject(409, "controlled_fixture_rollback") + return nil + }) + assertControlledRejected(t, err) + seedResidualAssert(t, scene, `SELECT reclaim_id IS NULL AND reclaim_state='reconciling' + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_intents WHERE workspace_id=$2) + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_command_receipts WHERE tenant_id=$3) + FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.workspace, scene.tenant) + root := seedControlledRetained(t, scene) + for _, stage := range []string{"retained", "delete_pending"} { + if stage == "delete_pending" { + seedControlledManage(t, scene.store, seedControlledDeleteRequest(scene, root, "guard-delete")) + } + if _, err := scene.store.DeleteRunWorkspace(t.Context(), scene.run); err == nil { + t.Fatalf("%s ordinary deletion bypassed controlled responsibility", stage) + } + for _, projectWide := range []bool{false, true} { + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + if projectWide { + for _, w := range tx.list("SELECT * FROM workspaces WHERE project_id=$1", scene.project) { + checkActivities(tx, w) + } + } else { + checkActivities(tx, tx.one("SELECT * FROM workspaces WHERE id=$1", scene.workspace)) + } + return nil + }) + if err == nil { + t.Fatalf("%s project=%v activity guard ignored controlled data protection", stage, projectWide) + } + } + } +} + +func TestControlledResidualQueryAndResume(t *testing.T) { + scene := seedControlledScene(t, controlledStores(t)(t), false) + root := seedControlledRoot(t, scene) + restarted := seedResidualRebuildStore(scene.store) + if err := restarted.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + out := seedControlledManage(t, restarted, ResidualManagementRequest{Action: ResidualGet, TenantID: scene.tenant, IntentID: root.S("id")}) + if out.O("intent").S("id") != root.S("id") { + t.Fatalf("get lost durable root identity: %v", out) + } + list := seedControlledManage(t, restarted, ResidualManagementRequest{Action: ResidualList, TenantID: scene.tenant, Limit: 1}) + items := residualObjects(list["items"]) + if len(items) != 1 { + t.Fatalf("list did not return bounded durable management state: %v", list) + } + _, err := restarted.ManageResidual(t.Context(), &ResidualManagementRequest{Action: ResidualGet, TenantID: newID(), IntentID: root.S("id")}) + assertControlledRejected(t, err) + seedResidualAssert(t, scene, `SELECT i.state<>'succeeded' AND i.id=$2 AND r.reclaim_mode='controlled' AND r.cleared_at IS NULL + AND (SELECT count(*) FROM agent_run_reclamation_intents WHERE workspace_id=r.workspace_id)=1 + AND NOT EXISTS(SELECT 1 FROM operations WHERE workspace_id=r.workspace_id AND kind='delete_workspace') + FROM agent_run_residuals r JOIN agent_run_reclamation_intents i ON i.reclaim_id=r.reclaim_id WHERE r.run_id=$1`, scene.run, root.S("id")) +} + +// This is a genuine D8 terminal: production declares the original delete and the give-up pass +// fails it. Only the predecessor scene (a releasing run with an unreachable sandbox) is staged. +func seedControlledLegacyD8(t *testing.T, prepared ...*Store) residualReclaimScene { + t.Helper() + var s *Store + if len(prepared) == 0 { + s = controlledStores(t)(t) + } else { + s = prepared[0] + } + legacy := seedReleaseScene(t, s, "releasing", "failed") + sandbox := newID() + seedResidualExec(t, s, `INSERT INTO sandbox_instances(id,workspace_id,generation,observed_state) VALUES($1,$2,1,'running')`, sandbox, legacy.runws) + if _, err := s.DeleteRunWorkspace(t.Context(), legacy.seed.run); err != nil { + t.Fatal(err) + } + s.DeliveryUnreachableAfter = time.Second + if err := s.GiveUpStaleWorkspaceReleasesOnce(t.Context()); err != nil { + t.Fatal(err) + } + seedControllerLease(t, s) + var operation string + if err := s.Pool.QueryRow(`SELECT id::text FROM operations WHERE workspace_id=$1 AND kind='delete_workspace'`, legacy.runws).Scan(&operation); err != nil { + t.Fatal(err) + } + return residualReclaimScene{store: s, abandonedCreation: abandonedCreation{provisioningSeed: provisioningSeed{dispSeed: legacy.seed, workspace: legacy.runws, operation: operation}, claims: controllerClaims(), effect: sandbox}} +} + +func TestControlledResidualLegacyD8(t *testing.T) { + scene := seedControlledLegacyD8(t) + root := seedControlledRoot(t, scene) + if root.S("source") != "legacy_d8" || root.S("legacyDeleteOperationId") != scene.operation { + t.Fatalf("D8 root did not preserve its original delete binding: %v", root) + } + if err := scene.store.RedeclareRunWorkspaceDeletesOnce(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := scene.store.DeleteRunWorkspace(t.Context(), scene.run); err == nil { + t.Fatal("D8 root without a ledger failed to protect ordinary deletion") + } + seedControlledManage(t, scene.store, seedControlledExternalRequest(scene, root, "external-d8")) + var valid bool + if err := scene.store.Pool.QueryRow(`SELECT run.phase='done' AND run.status='failed' AND run.failure_reason='workspace_unavailable' + AND op.state='failed' AND op.error_code='node_unavailable' AND w.deleted_at IS NULL + AND NOT EXISTS(SELECT 1 FROM agent_run_residuals WHERE run_id=run.id) + AND (SELECT count(*) FROM operations WHERE workspace_id=w.id AND kind='delete_workspace')=1 + FROM issue_runs run JOIN workspaces w ON w.id=run.workspace_id JOIN operations op ON op.id=$2 WHERE run.id=$1`, scene.run, scene.operation).Scan(&valid); err != nil || !valid { + t.Fatalf("D8 management rewrote history or invented a ledger: %v / %v", valid, err) + } +} + +func TestControlledResidualLegacyD8RetainedThenReleased(t *testing.T) { + scene := seedControlledLegacyD8(t) + var originalRun, originalOperation string + if err := scene.store.Pool.QueryRow(`SELECT to_jsonb(run)::text,to_jsonb(op)::text + FROM issue_runs run JOIN operations op ON op.id=$2 WHERE run.id=$1`, scene.run, scene.operation).Scan(&originalRun, &originalOperation); err != nil { + t.Fatal(err) + } + root := seedControlledRetained(t, scene) + var retained bool + if err := scene.store.Pool.QueryRow(`SELECT i.source='legacy_d8' AND i.legacy_delete_operation_id=$2 + AND i.state='succeeded' AND i.disposition='terminated_data_retained' AND i.completed_at IS NOT NULL + AND w.deleted_at IS NULL AND NOT w.admission_open + AND NOT EXISTS(SELECT 1 FROM agent_run_residuals WHERE run_id=i.run_id) + AND (SELECT count(*) FROM operations WHERE workspace_id=w.id AND kind='delete_workspace')=1 + FROM agent_run_reclamation_intents i JOIN workspaces w ON w.id=i.workspace_id WHERE i.id=$1`, root.S("id"), scene.operation).Scan(&retained); err != nil || !retained { + t.Fatalf("D8 physical termination did not retain data without a ledger: %v / %v", retained, err) + } + child, op, effect := seedControlledDeletePlan(t, scene, root) + op = seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "legacy-controlled-data-delete", "result": Object{"removed": true}}, effect.S("id")).O("operation") + finished := seedResidualAction(t, scene, op, "advance", nil, "").O("operation") + if finished.S("state") != "succeeded" || finished.S("id") == scene.operation { + t.Fatalf("D8 controlled data deletion did not finish its new bound operation: %v", finished) + } + var released bool + if err := scene.store.Pool.QueryRow(`SELECT to_jsonb(run)=$2::jsonb AND to_jsonb(original)=$3::jsonb + AND run.phase='done' AND run.status='failed' AND run.failure_reason='workspace_unavailable' + AND original.state='failed' AND original.error_code='node_unavailable' + AND root.state='succeeded' AND root.disposition='terminated_data_retained' AND root.completed_at=$4::timestamptz + AND root.legacy_delete_operation_id=original.id AND root.delete_operation_id=$5 + AND child.root_intent_id=root.id AND child.command_kind='delete_data' AND child.state='succeeded' + AND child.disposition='released' AND child.delete_operation_id=$5 AND child.completed_at IS NOT NULL + AND disposal.id=$5 AND disposal.state='succeeded' AND disposal.step='done' + AND w.deleted_at IS NOT NULL AND w.issue_run_id=run.id + AND NOT EXISTS(SELECT 1 FROM agent_run_residuals WHERE run_id=run.id) + AND (SELECT count(*) FROM operations WHERE workspace_id=w.id AND kind='delete_workspace')=2 + AND (SELECT count(*) FROM external_effects WHERE operation_id=disposal.id AND kind='workspace_data_delete' + AND state='succeeded' AND result @> '{"removed":true}')=1 + FROM issue_runs run JOIN workspaces w ON w.id=run.workspace_id + JOIN operations original ON original.id=$6 JOIN agent_run_reclamation_intents root ON root.id=$7 + JOIN agent_run_reclamation_intents child ON child.id=$8 JOIN operations disposal ON disposal.id=child.delete_operation_id + WHERE run.id=$1`, scene.run, originalRun, originalOperation, root.S("completedAt"), finished.S("id"), scene.operation, root.S("id"), child.S("id")).Scan(&released); err != nil || !released { + t.Fatalf("D8 retained-to-released path lost history, binding, or physical evidence: %v / %v", released, err) + } +} + +func TestControlledResidualDoesNotAuthorizeAutomaticEscalation(t *testing.T) { + scene := seedControlledScene(t, controlledStores(t)(t), false) + root := seedControlledRoot(t, scene) + if err := scene.store.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT reclaim_mode='controlled' AND reclaim_authorization->>'intentId'=$2 AND cleared_at IS NULL + AND NOT EXISTS(SELECT 1 FROM operations WHERE workspace_id=$3 AND kind='delete_workspace') + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=$3 AND kind='workspace_data_delete') + FROM agent_run_residuals WHERE run_id=$1`, scene.run, root.S("id"), scene.workspace) +} + +func controlledRaceError(result residualRaceResult) error { + if result.err != nil { + return result.err + } + if code := result.out.S("managementConflict"); code != "" { + return &Fault{Status: 409, Code: code} + } + return nil +} + +func controlledRaceName(first bool) string { return fmt.Sprint("controlled_first=", first) } diff --git a/internal/core/agent_run_controlled_reclaim_delete.go b/internal/core/agent_run_controlled_reclaim_delete.go new file mode 100644 index 000000000..c9b771c1d --- /dev/null +++ b/internal/core/agent_run_controlled_reclaim_delete.go @@ -0,0 +1,128 @@ +package core + +func registerControlledDataDelete(t *transaction, r *ResidualManagementRequest, actor string) Object { + require(r.DataDeleteConfirmed, 400, "data_delete_confirmation_required") + root := t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1 AND tenant_id=$2 AND command_kind='terminate'", nullable(r.IntentID), r.TenantID) + require(root != nil, 404, "residual_not_found") + require(controlledRootRow(t, root)["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + if child := t.one("SELECT * FROM agent_run_reclamation_intents WHERE root_intent_id=$1", root.S("id")); child != nil { + return child + } + require(root.S("state") == "succeeded" && root.S("disposition") == "terminated_data_retained", 409, "termination_unconfirmed") + row := refreshControlledTargets(t, root) + require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + require(controlledTerminationComplete(t, root, row), 409, "termination_unconfirmed") + _, w, p := residualIdentity(t, row) + require(p.S("lifecycle") == "active" && w["deletedAt"] == nil && !w.B("admissionOpen"), 409, "resource_unavailable") + require(t.one("SELECT id FROM operations WHERE project_id=$1 AND state IN ('queued','running','retry_wait','blocked')", p.S("id")) == nil, 409, "resource_in_use") + id := newID() + t.exec(`INSERT INTO agent_run_reclamation_intents(id,root_intent_id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,legacy_delete_operation_id,control_epoch,actor_subject,operator_label,reason,data_delete_confirmed) + VALUES($1,$2,'delete_data',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,true)`, id, root.S("id"), root.S("source"), root.S("tenantId"), root.S("projectId"), root.S("runId"), root.S("workspaceId"), root.N("runtimeGeneration"), nullable(root.S("legacyDeleteOperationId")), root.N("controlEpoch"), actor, r.OperatorLabel, r.Reason) + snapshot := Object{} + for key, value := range w { + if key != "issueRunId" { + snapshot[key] = value + } + } + closeAdmission(t, w, "deleted") + request := Object{"previous": Object{w.S("id"): snapshot}, "reclaimId": root.S("reclaimId"), "controlledIntentId": root.S("id"), "dataDeleteIntentId": id} + operationActor := w.S("creatorUserId") + if operationActor == "" && root.S("legacyDeleteOperationId") != "" { + // Legacy creator evidence may be absent. Keep the original deletion's tenant actor + // reference; the deployment operator's authenticated identity stays on the intent. + operationActor = t.one("SELECT actor_user_id FROM operations WHERE id=$1", root.S("legacyDeleteOperationId")).S("actorUserId") + } + require(validID(operationActor), 409, "residual_actor_unknown") + op := newOperation(t, &PublicRequest{TenantID: root.S("tenantId"), Key: "controlled-" + root.S("reclaimId")}, operationActor, p.S("id"), w.S("id"), "delete_workspace", "cleanup", jsonText(Object{"intentId": id}), request) + t.exec("UPDATE agent_run_reclamation_intents SET delete_operation_id=$2 WHERE id=$1", id, op.S("id")) + if root.S("source") == "residual_ledger" { + t.exec("UPDATE agent_run_residuals SET reclaim_operation_id=$2,reclaim_version=reclaim_version+1 WHERE run_id=$1 AND reclaim_operation_id IS NULL", root.S("runId"), op.S("id")) + } else { + t.exec("UPDATE agent_run_reclamation_intents SET delete_operation_id=$2,version=version+1 WHERE id=$1 AND delete_operation_id IS NULL", root.S("id"), op.S("id")) + } + reserveRuntimeMaintenance(t, w.S("id"), op.S("id")) + controlledAudit(t, id, "data_delete_registered", actor, Object{"dataDeleteConfirmed": true, "operationId": op.S("id")}) + return t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", id) +} + +func controlledOperationRoot(t *transaction, op Object) Object { + if op == nil { + return nil + } + return t.one(`SELECT i.* FROM agent_run_reclamation_intents i LEFT JOIN agent_run_residuals ar ON ar.run_id=i.run_id + WHERE i.root_intent_id IS NULL AND (i.delete_operation_id=$1 OR (i.source='residual_ledger' AND ar.reclaim_operation_id=$1))`, op.S("id")) +} + +func checkControlledOperation(t *transaction, op, root Object) Object { + row := refreshControlledTargets(t, root) + _, w, p := residualIdentity(t, row) + child := t.one("SELECT * FROM agent_run_reclamation_intents WHERE root_intent_id=$1 AND command_kind='delete_data'", root.S("id")) + require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + require(root.S("state") == "succeeded" && root.S("disposition") == "terminated_data_retained" && child != nil && child.S("state") != "succeeded" && + child.B("dataDeleteConfirmed") && op.S("id") == row.S("reclaimOperationId") && op.S("kind") == "delete_workspace" && op.S("step") == "cleanup" && + op.S("tenantId") == root.S("tenantId") && op.S("projectId") == p.S("id") && op.S("workspaceId") == w.S("id") && + op.O("request").S("controlledIntentId") == root.S("id") && op.O("request").S("dataDeleteIntentId") == child.S("id") && + op.O("request").S("reclaimId") == root.S("reclaimId") && !w.B("admissionOpen"), 409, "residual_operation_conflict") + require(controlledTerminationComplete(t, root, row), 409, "termination_unconfirmed") + require(t.one("SELECT id FROM operations WHERE project_id=$1 AND id<>$2 AND state IN ('queued','running','retry_wait','blocked')", p.S("id"), op.S("id")) == nil, 409, "resource_in_use") + t.exec("UPDATE agent_run_reclamation_intents SET state='terminating',started_at=COALESCE(started_at,clock_timestamp()),version=version+1 WHERE id=$1 AND state='registered'", child.S("id")) + return row +} + +func completeControlledDataDelete(t *transaction, op, root Object) bool { + row := checkControlledOperation(t, op, root) + effect := t.one("SELECT * FROM external_effects WHERE operation_id=$1 AND workspace_id=$2 AND kind='workspace_data_delete' AND state='succeeded' AND reconciled_epoch=$3", op.S("id"), root.S("workspaceId"), op.N("controllerEpoch")) + require(effect != nil && effect.O("result").B("removed"), 409, "residual_cleanup_incomplete") + child := t.one("SELECT * FROM agent_run_reclamation_intents WHERE root_intent_id=$1", root.S("id")) + if !appendResidualFact(t, row.S("runId"), "controlled:"+child.S("id")+":released", Object{"kind": "reclaim_complete", "state": "succeeded", "intentId": child.S("id"), "operationId": op.S("id"), "effectId": effect.S("id"), "disposition": "released"}) { + return false + } + t.exec("UPDATE agent_run_reclamation_intents SET state='succeeded',disposition='released',completed_at=clock_timestamp(),version=version+1 WHERE id=$1", child.S("id")) + if root.S("source") == "residual_ledger" { + t.exec("UPDATE agent_run_residuals SET disposition='released',reclaim_version=reclaim_version+1 WHERE run_id=$1", root.S("runId")) + } + controlledAudit(t, child.S("id"), "released", "controller", Object{"effectId": effect.S("id")}) + return true +} + +func confirmControlledExternal(t *transaction, r *ResidualManagementRequest, actor string) Object { + require(r.ResponsibilityTransferred, 400, "external_responsibility_confirmation_required") + r.EvidenceNote = validText(r.EvidenceNote, 2000) + intent := t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1 AND tenant_id=$2", nullable(r.IntentID), r.TenantID) + require(intent != nil, 404, "residual_not_found") + require(intent.S("state") != "succeeded", 409, "reclamation_intent_completed") + root := controlledIntentRoot(t, intent) + row := refreshControlledTargets(t, root) + require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + open := controlledOpenTargets(t, root, row, intent) + require(len(r.ExternalFacts) == len(open), 400, "external_target_confirmation_required") + for _, target := range open { + validText(r.ExternalFacts[target.S("key")], 2000) + } + fact := Object{"kind": "external_disposition", "state": "succeeded", "intentId": intent.S("id"), "disposition": "externally_confirmed", "reason": r.Reason, "evidenceNote": r.EvidenceNote, "externalFacts": r.ExternalFacts, "actorSubject": actor, "responsibilityTransferred": true} + if !appendResidualFact(t, row.S("runId"), "controlled:"+intent.S("id")+":external", fact) { + return intent + } + t.exec("UPDATE agent_run_reclamation_intents SET state='succeeded',disposition='externally_confirmed',started_at=COALESCE(started_at,clock_timestamp()),completed_at=clock_timestamp(),version=version+1 WHERE id=$1", intent.S("id")) + if root.S("source") == "residual_ledger" { + t.exec("UPDATE agent_run_residuals SET reclaim_state='cleared',disposition='externally_confirmed',cleared_at=COALESCE(cleared_at,clock_timestamp()),reclaim_version=reclaim_version+1 WHERE run_id=$1", row.S("runId")) + } + if intent.S("commandKind") == "delete_data" { + t.exec("UPDATE operations SET state='failed',error_code='external_disposition',version=version+1,updated_at=clock_timestamp() WHERE id=$1 AND state IN ('queued','running','retry_wait','blocked')", nullable(row.S("reclaimOperationId"))) + } + controlledAudit(t, intent.S("id"), "externally_confirmed", actor, fact) + return t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", intent.S("id")) +} + +func controlledOpenTargets(t *transaction, root, row, intent Object) []Object { + open := []Object{} + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") == "sandbox" && !residualSandboxTerminated(t, row, target.S("id")) { + open = append(open, Object{"key": "sandbox:" + target.S("id"), "kind": "sandbox", "id": target.S("id")}) + } + } + // Terminating only retains data by an operator decision too. External disposition must + // explicitly account for the volume responsibility instead of silently writing it off. + open = append(open, Object{"key": "data:" + root.S("workspaceId"), "kind": "data", "id": root.S("workspaceId")}) + return open +} diff --git a/internal/core/agent_run_controlled_reclaim_dispatch_db_test.go b/internal/core/agent_run_controlled_reclaim_dispatch_db_test.go new file mode 100644 index 000000000..496679147 --- /dev/null +++ b/internal/core/agent_run_controlled_reclaim_dispatch_db_test.go @@ -0,0 +1,42 @@ +package core + +import ( + "testing" + "time" +) + +// A durable delivery skip must remain idempotent even when a retry carries a legacy non-UUID target. +func TestControlledResidualSkippedDeliveryWithInvalidWorkspaceID(t *testing.T) { + store := controlledStores(t)(t) + seed := seedDispatchScene(t, store.Pool, false) + createdAt := time.Date(2026, time.October, 20, 12, 0, 0, 0, time.UTC) + if _, err := store.Pool.Exec(`INSERT INTO revision_delivery_skips(run_id,created_at) VALUES($1,$2)`, seed.run, createdAt); err != nil { + t.Fatalf("seed durable delivery skip: %v", err) + } + + work, err := store.EnqueueExecutionWork(t.Context(), seed.run, "deliver_revision", + Object{"kind": "deliver_revision"}, + Object{"workspaceId": "w", "sandboxInstanceId": "s", "nodeId": "n"}, time.Time{}) + if err != nil { + t.Fatalf("retry skipped delivery with a non-UUID workspace target: %v", err) + } + if !work.B("skipped") { + t.Fatalf("retry must return the durable skip, got %v", work) + } + + var workCount int + if err := store.Pool.QueryRow(`SELECT count(*) FROM execution_work WHERE run_id=$1`, seed.run).Scan(&workCount); err != nil { + t.Fatalf("count execution work after skipped retry: %v", err) + } + if workCount != 0 { + t.Fatalf("skipped retry created %d execution work items; want none", workCount) + } + var skipRunID string + var skipCreatedAt time.Time + if err := store.Pool.QueryRow(`SELECT run_id,created_at FROM revision_delivery_skips WHERE run_id=$1`, seed.run).Scan(&skipRunID, &skipCreatedAt); err != nil { + t.Fatalf("read original durable delivery skip: %v", err) + } + if skipRunID != seed.run || !skipCreatedAt.Equal(createdAt) { + t.Fatalf("durable skip changed: run=%q created_at=%v; want run=%q created_at=%v", skipRunID, skipCreatedAt, seed.run, createdAt) + } +} diff --git a/internal/core/agent_run_controlled_reclaim_fixture_test.go b/internal/core/agent_run_controlled_reclaim_fixture_test.go new file mode 100644 index 000000000..486d21521 --- /dev/null +++ b/internal/core/agent_run_controlled_reclaim_fixture_test.go @@ -0,0 +1,83 @@ +package core + +import ( + "os" + "strings" + "testing" +) + +var ( + controlledPrepared *Store + controlledCleanup func() +) + +func TestMain(m *testing.M) { + code := m.Run() + if controlledCleanup != nil { + controlledCleanup() + } + os.Exit(code) +} + +// Controlled tests are serial and share only a migrated schema. Cleanup belongs to TestMain, +// not the first test. Every leaf starts empty so schema-wide claims/scans remain isolated. +func controlledStores(t *testing.T) func(*testing.T) *Store { + t.Helper() + if controlledPrepared == nil { + controlledPrepared = dispatcherDBWithCleanup(t, func(cleanup func()) { controlledCleanup = cleanup }) + } + prepared := controlledPrepared + var tables string + if err := prepared.Pool.QueryRow(`SELECT string_agg(format('%I.%I',schemaname,tablename), ',' ORDER BY tablename) + FROM pg_tables WHERE schemaname=current_schema() + AND tablename NOT IN ('schema_migrations','task_conversation_scan_state')`).Scan(&tables); err != nil { + t.Fatal(err) + } + reset := "TRUNCATE " + tables + return func(t *testing.T) *Store { + t.Helper() + if _, err := prepared.Pool.Exec(reset); err != nil { + t.Fatal(err) + } + // 0046's scan cursor is the only non-migration row in an empty migrated schema. + if _, err := prepared.Pool.Exec("UPDATE task_conversation_scan_state SET last_created_at=NULL,last_id=NULL"); err != nil { + t.Fatal(err) + } + store := &Store{Pool: prepared.Pool, Signals: NewControlHub()} + BindBusinessHooks(store) + return store + } +} + +// Keep #112's scene helper unchanged; only this controlled adapter supplies an already migrated +// Store. Its original provisioning, delivery and runtime evidence still enters through the same helpers. +func seedControlledScene(t *testing.T, s *Store, ready bool) residualReclaimScene { + t.Helper() + scene := residualReclaimScene{store: s, abandonedCreation: seedAbandonedCreation(t, s, true), node: newID(), delivery: "residual-delivery-" + newID(), work: newID()} + seedResidualExec(t, s, `INSERT INTO node_instances(id,workspace_id,sandbox_instance_id,service_subject,connection_state,protocol_version,initialized,node_id,node_incarnation_id) + VALUES($1,$2,$3,'residual-node','connected',1,true,'residual-node','residual-incarnation')`, scene.node, scene.workspace, scene.effect) + seedResidualExec(t, s, `UPDATE sandbox_instances SET observed_state='running' WHERE id=$1`, scene.effect) + if ready { + _, err := submitFact(t, s, scene.abandonedCreation, "residual-ensure", Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "succeeded", "externalId": "residual-substrate", "result": Object{"sandboxInstanceId": scene.effect, "nodeId": "residual-node"}}) + if err != nil { + t.Fatal(err) + } + _, err = submitFact(t, s, scene.abandonedCreation, "residual-creation", Object{"epoch": 1, "kind": "creation", "outcome": "ready"}) + if err != nil { + t.Fatal(err) + } + } + scene.deliveryInput = Object{"kind": "deliver_revision", "baseCommit": baseCommit, "revisionRef": "refs/ora/revisions/" + scene.run, "deliveryRef": GitDeliveryRefPrefix + scene.work, "historyKey": "revisions/" + scene.run + "/history"} + scene.deliveryResult = Object{"outcome": "revision_unchanged", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}, "baseCommit": baseCommit, "finalCommit": baseCommit, "revisionRef": scene.deliveryInput.S("revisionRef"), "history": Object{"key": scene.deliveryInput.S("historyKey"), "size": 0, "sha256": strings.Repeat("a", 64)}} + seedResidualExec(t, s, `INSERT INTO execution_work(id,run_id,kind,input,target,execution_id) VALUES($1,$2,'deliver_revision',$3,$4,$5)`, scene.work, scene.run, jsonText(scene.deliveryInput), jsonText(Object{"workspaceId": scene.workspace, "sandboxInstanceId": scene.effect, "runtimeGeneration": 1, "nodeId": "residual-node"}), scene.delivery) + seedResidualExec(t, s, `INSERT INTO node_executions(execution_id,kind,operation_id,work_id,workspace_id,node_id,node_operation_id,input,dispatched_epoch) + VALUES($1,'deliver_revision',$2,$3,$4,'residual-node',$1,$5,1)`, scene.delivery, scene.run, scene.work, scene.workspace, jsonText(scene.deliveryInput)) + if ready { + _, err := s.transact(t.Context(), func(tx *transaction) Object { return seedResidualDelivery(tx, scene, scene.deliveryResult) }) + if err != nil { + t.Fatal(err) + } + seedResidualRuntimeAck(t, scene, true) + } + return scene +} diff --git a/internal/core/agent_run_controlled_reclaim_loop.go b/internal/core/agent_run_controlled_reclaim_loop.go new file mode 100644 index 000000000..bf051f135 --- /dev/null +++ b/internal/core/agent_run_controlled_reclaim_loop.go @@ -0,0 +1,26 @@ +package core + +import "context" + +// reconcileControlledResiduals only revisits explicitly registered operator intents. It does +// not discover/authorize old D8 deletions or escalate an automatic owned-delete responsibility. +func (s *Store) reconcileControlledResiduals(ctx context.Context) error { + _, err := s.transact(ctx, func(t *transaction) Object { + for _, intent := range t.list("SELECT * FROM agent_run_reclamation_intents WHERE state<>'succeeded' ORDER BY last_checked_at NULLS FIRST,registered_at,id LIMIT $1", agentDeliveryBatchSize) { + root := controlledIntentRoot(t, intent) + row := refreshControlledTargets(t, root) + t.exec("UPDATE agent_run_reclamation_intents SET last_checked_at=clock_timestamp() WHERE id=$1", intent.S("id")) + if row["reclaimConflictedAt"] != nil { + continue + } + if intent.S("commandKind") == "delete_data" && row.S("reclaimOperationId") != "" { + op := t.one("SELECT * FROM operations WHERE id=$1", row.S("reclaimOperationId")) + if op != nil && (op.S("state") == "queued" || op.S("state") == "retry_wait" || op.S("state") == "running") { + t.queued = append(t.queued, op.S("id")) + } + } + } + return nil + }) + return err +} diff --git a/internal/core/agent_run_controlled_reclaim_query.go b/internal/core/agent_run_controlled_reclaim_query.go new file mode 100644 index 000000000..29338cc82 --- /dev/null +++ b/internal/core/agent_run_controlled_reclaim_query.go @@ -0,0 +1,89 @@ +package core + +func managedIntentView(t *transaction, id string) Object { + intent := t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", id) + require(intent != nil, 404, "residual_not_found") + root := controlledIntentRoot(t, intent) + row := controlledRootRow(t, root) + out := Object{"intent": intent, "rootIntent": root, "residual": row, "targets": t.list("SELECT * FROM agent_run_reclamation_targets WHERE root_intent_id=$1 ORDER BY id", root.S("id")), "openTargets": controlledOpenTargets(t, root, row, intent), "audit": intent["audit"]} + if intent.S("state") == "succeeded" { + out["openTargets"] = []Object{} + } + if row.S("reclaimOperationId") != "" { + out["operation"] = t.one("SELECT id,kind,state,step,error_code,version FROM operations WHERE id=$1", row.S("reclaimOperationId")) + } + return out +} + +func getManagedResidual(t *transaction, r *ResidualManagementRequest) Object { + if r.IntentID != "" { + intent := t.one("SELECT id FROM agent_run_reclamation_intents WHERE id=$1 AND tenant_id=$2", r.IntentID, r.TenantID) + require(intent != nil, 404, "residual_not_found") + return managedIntentView(t, intent.S("id")) + } + require(r.RunID != "" || r.WorkspaceID != "" || r.ReclaimID != "", 400, "residual_target_required") + items := listManagedResiduals(t, r) + rows := residualObjects(items["items"]) + require(len(rows) == 1, 404, "residual_not_found") + item := rows[0] + if item.S("intentId") != "" { + return managedIntentView(t, item.S("intentId")) + } + return Object{"resource": item, "residual": residualForWorkspace(t, item.S("workspaceId"))} +} + +func listManagedResiduals(t *transaction, r *ResidualManagementRequest) Object { + limit := r.Limit + if limit == 0 { + limit = 100 + } + require(limit > 0 && limit <= 100, 400, "invalid_limit") + items := t.list(`WITH resources AS ( + SELECT ar.tenant_id,ar.project_id,ar.run_id,ar.workspace_id,ar.reclaim_id,ar.reclaim_state AS state, + ar.disposition,ar.reclaim_operation_id AS operation_id,NULL::uuid AS legacy_operation_id,'residual_ledger'::text AS source, + ar.reclaim_conflicted_at AS conflicted_at,ar.reclaim_checked_at AS checked_at + FROM agent_run_residuals ar + UNION ALL + SELECT w.tenant_id,w.project_id,i.id,w.id,ri.reclaim_id,COALESCE(ri.state,'reconciling'),COALESCE(child.disposition,ri.disposition), + ri.delete_operation_id,o.id,'legacy_d8',ri.conflicted_at,ri.last_checked_at + FROM workspaces w JOIN issue_runs i ON i.id=w.issue_run_id + JOIN LATERAL (SELECT op.id FROM operations op WHERE op.workspace_id=w.id AND op.kind='delete_workspace' + AND op.state='failed' AND op.error_code='node_unavailable' ORDER BY op.created_at,op.id LIMIT 1) o ON true + LEFT JOIN agent_run_reclamation_intents ri ON ri.workspace_id=w.id AND ri.root_intent_id IS NULL + LEFT JOIN agent_run_reclamation_intents child ON child.root_intent_id=ri.id AND child.state='succeeded' + WHERE i.phase='done' AND i.executor_type='agent' AND w.kind='isolated' + AND NOT EXISTS(SELECT 1 FROM agent_run_residuals ar WHERE ar.workspace_id=w.id) + ) SELECT r.*,ri.id AS intent_id FROM resources r + LEFT JOIN agent_run_reclamation_intents ri ON ri.workspace_id=r.workspace_id AND ri.root_intent_id IS NULL + WHERE r.tenant_id=$1 AND ($2::uuid IS NULL OR r.project_id=$2) AND ($3::uuid IS NULL OR r.run_id=$3) + AND ($4::uuid IS NULL OR r.workspace_id=$4) AND ($5::uuid IS NULL OR r.operation_id=$5 OR r.legacy_operation_id=$5) + AND ($6::uuid IS NULL OR r.reclaim_id=$6) AND ($7::uuid IS NULL OR r.run_id>$7) + ORDER BY r.run_id LIMIT $8`, r.TenantID, nullable(r.ProjectID), nullable(r.RunID), nullable(r.WorkspaceID), nullable(r.OperationID), nullable(r.ReclaimID), nullable(r.After), limit+1) + next := "" + if len(items) > limit { + items = items[:limit] + next = items[len(items)-1].S("runId") + } + return Object{"items": items, "nextCursor": next} +} + +// legacyResidualForWorkspace projects the D8 intent into the shared evidence functions; it does +// not insert a ledger row or change the source or business history. +func legacyResidualForWorkspace(t *transaction, wid string) Object { + root := controlledRootForWorkspace(t, wid) + if root == nil || root.S("source") != "legacy_d8" { + return nil + } + return controlledRootRow(t, root) +} + +func residualForRun(t *transaction, run string) Object { + if row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", run); row != nil { + return row + } + root := t.one("SELECT * FROM agent_run_reclamation_intents WHERE run_id=$1 AND source='legacy_d8' AND root_intent_id IS NULL", run) + if root == nil { + return nil + } + return controlledRootRow(t, root) +} diff --git a/internal/core/agent_run_controlled_reclaim_race_db_test.go b/internal/core/agent_run_controlled_reclaim_race_db_test.go new file mode 100644 index 000000000..226750fcc --- /dev/null +++ b/internal/core/agent_run_controlled_reclaim_race_db_test.go @@ -0,0 +1,310 @@ +package core + +import ( + "fmt" + "testing" + "time" +) + +func TestControlledResidualRaceAutomaticClaimBothCommitOrders(t *testing.T) { + stores := controlledStores(t) + for _, controlledFirst := range []bool{false, true} { + t.Run(controlledRaceName(controlledFirst), func(t *testing.T) { + scene := seedControlledScene(t, stores(t), true) + r := seedControlledRequest(scene, "race-controlled") + automatic := func(tx *transaction) Object { return authorizeResidualOwnedDelete(tx, scene.run) } + controlled := func(tx *transaction) Object { return manageResidualCommand(tx, &r) } + first, second := automatic, controlled + if controlledFirst { + first, second = controlled, automatic + } + a, b := seedResidualRaceStores(t, scene, seedResidualRebuildStore(scene.store), seedResidualRebuildStore(scene.store), first, second) + if err := controlledRaceError(a); err != nil { + t.Fatal(err) + } + if err := controlledRaceError(b); err == nil { + t.Fatal("second claimant acquired a second mode instead of respecting committed authority") + } else if fault := ErrorCode(err); fault.Code != "residual_mode_conflict" { + t.Fatalf("losing claimant returned %v", err) + } + mode, intents, operations := "owned_delete", 0, 1 + if controlledFirst { + mode, intents, operations = "controlled", 1, 0 + } + seedResidualAssert(t, scene, `SELECT r.reclaim_mode=$2 AND r.reclaim_id IS NOT NULL AND r.reclaim_state='reclaiming' + AND r.responsibility=$3::jsonb AND r.cleared_at IS NULL + AND (SELECT count(*) FROM agent_run_reclamation_intents WHERE workspace_id=r.workspace_id)=$4 + AND (SELECT count(*) FROM operations WHERE workspace_id=r.workspace_id AND kind='delete_workspace')=$5 + AND (r.reclaim_mode<>'controlled' OR EXISTS(SELECT 1 FROM agent_run_reclamation_intents i WHERE i.reclaim_id=r.reclaim_id AND r.reclaim_authorization->>'intentId'=i.id::text)) + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=r.workspace_id AND kind='workspace_data_delete') + FROM agent_run_residuals r WHERE r.run_id=$1`, scene.run, mode, scene.responsibility, intents, operations) + }) + } +} + +func TestControlledResidualRaceIdempotencyBothCommitOrders(t *testing.T) { + stores := controlledStores(t) + for _, action := range []ResidualManagementAction{ResidualTerminate, ResidualDeleteData, ResidualConfirmExternal} { + for _, conflict := range []bool{false, true} { + for _, reverse := range []bool{false, true} { + t.Run(fmt.Sprint(action, "/conflict=", conflict, "/reverse=", reverse), func(t *testing.T) { + scene := seedControlledScene(t, stores(t), false) + r := seedControlledRequest(scene, "racing-command") + switch action { + case ResidualDeleteData: + r = seedControlledDeleteRequest(scene, seedControlledRetained(t, scene), r.Key) + case ResidualConfirmExternal: + r = seedControlledExternalRequest(scene, seedControlledRoot(t, scene), r.Key) + } + other := r + if conflict { + other.Reason = "a different operator command with the same key" + } + if reverse { + r, other = other, r + } + first := func(tx *transaction) Object { return manageResidualCommand(tx, &r) } + second := func(tx *transaction) Object { return manageResidualCommand(tx, &other) } + a, b := seedResidualRaceStores(t, scene, seedResidualRebuildStore(scene.store), seedResidualRebuildStore(scene.store), first, second) + if err := controlledRaceError(a); err != nil { + t.Fatal(err) + } + if conflict { + if err := controlledRaceError(b); err == nil || ErrorCode(err).Code != "idempotency_conflict" { + t.Fatalf("same-key conflicting input did not commit a conflict response: %v / %v", b.out, err) + } + } else if err := controlledRaceError(b); err != nil || b.out.O("intent").S("id") != a.out.O("intent").S("id") { + t.Fatalf("same command did not replay original intent: %v / %v", b.out, err) + } + seedResidualAssert(t, scene, `SELECT (SELECT count(*) FROM agent_run_reclamation_command_receipts WHERE tenant_id=r.tenant_id AND idempotency_key=$2)=1 + AND EXISTS(SELECT 1 FROM agent_run_reclamation_command_receipts x WHERE x.tenant_id=r.tenant_id AND x.idempotency_key=$2 + AND x.command_input->>'reason'=$3 AND jsonb_array_length(x.audit)=$4) + AND (SELECT count(*) FROM agent_run_reclamation_intents WHERE workspace_id=r.workspace_id AND command_kind='terminate')=1 + AND (SELECT count(*) FROM agent_run_reclamation_intents WHERE workspace_id=r.workspace_id AND command_kind='delete_data')=$5 + AND (SELECT count(*) FROM operations WHERE workspace_id=r.workspace_id AND kind='delete_workspace')=$5 + FROM agent_run_residuals r WHERE r.run_id=$1`, scene.run, r.Key, r.Reason, map[bool]int{false: 0, true: 1}[conflict], map[bool]int{false: 0, true: 1}[action == ResidualDeleteData]) + }) + } + } + } +} + +func TestControlledResidualRaceDifferentKeysKeepOneRoot(t *testing.T) { + stores := controlledStores(t) + for _, reverse := range []bool{false, true} { + t.Run(fmt.Sprint("reverse=", reverse), func(t *testing.T) { + scene := seedControlledScene(t, stores(t), false) + r, other := seedControlledRequest(scene, "first-key"), seedControlledRequest(scene, "second-key") + if reverse { + r, other = other, r + } + a, b := seedResidualRace(t, scene, func(tx *transaction) Object { return manageResidualCommand(tx, &r) }, func(tx *transaction) Object { return manageResidualCommand(tx, &other) }) + if controlledRaceError(a) != nil || controlledRaceError(b) != nil || a.out.O("intent").S("id") != b.out.O("intent").S("id") { + t.Fatalf("different keys obtained a second root: %v / %v", a, b) + } + seedResidualAssert(t, scene, `SELECT (SELECT count(*) FROM agent_run_reclamation_intents WHERE workspace_id=r.workspace_id)=1 + AND (SELECT count(*) FROM agent_run_reclamation_targets x JOIN agent_run_reclamation_intents i ON i.id=x.root_intent_id WHERE i.workspace_id=r.workspace_id)=1 + FROM agent_run_residuals r WHERE r.run_id=$1`, scene.run) + }) + } +} + +func seedControlledCompletionRace(t *testing.T, store *Store, released bool) (residualReclaimScene, Object, func(*transaction) Object) { + t.Helper() + scene := seedControlledScene(t, store, true) + if released { + root := seedControlledRetained(t, scene) + _, op, effect := seedControlledDeletePlan(t, scene, root) + op = seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "race-controlled-data", "result": Object{"removed": true}}, effect.S("id")).O("operation") + return scene, root, func(tx *transaction) Object { + return advance(tx, &ControlRequest{}, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id"))) + } + } + root := seedControlledRoot(t, scene) + plan := seedControlledPlan(t, scene, root.S("id")) + targets := residualObjects(plan["effects"]) + if len(targets) != 1 { + t.Fatalf("race fixture has %d termination targets", len(targets)) + } + return scene, root, func(tx *transaction) Object { + intent := tx.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", root.S("id")) + return forceStopCommand(tx, &ControlRequest{Action: "force_confirm", Body: Object{"epoch": 1, "forceStopId": root.S("id"), "version": intent.N("version"), "effectId": targets[0].S("id"), "terminated": true, "lateEnsureFenced": true}}) + } +} + +func TestControlledResidualRaceCompletionAndLateFactsBothCommitOrders(t *testing.T) { + stores := controlledStores(t) + for _, released := range []bool{false, true} { + for _, kind := range []string{"creation", "effect", "delivery", "session"} { + for _, factFirst := range []bool{false, true} { + t.Run(fmt.Sprint("released=", released, "/", kind, "/fact_first=", factFirst), func(t *testing.T) { + scene, root, complete := seedControlledCompletionRace(t, stores(t), released) + fact := func(tx *transaction) Object { + switch kind { + case "creation": + return provisioningCreationFact(tx, &ControlRequest{OperationID: scene.operation, Body: Object{"outcome": "failed"}}, scene.run) + case "effect": + return provisioningFact(tx, &ControlRequest{OperationID: scene.operation, Body: Object{"kind": "effect", "effectId": scene.effect, "state": "failed", "externalId": "residual-substrate", "result": Object{"error": "late-conflicting-failure"}}}) + case "delivery": + return seedResidualDelivery(tx, scene, Object{"outcome": "revision_failed", "reason": "upload_failed", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}}) + default: + return seedResidualLateExecution(tx, scene, "agent_session") + } + } + first, second := complete, fact + if factFirst { + first, second = fact, complete + } + a, b := seedResidualRace(t, scene, first, second) + conflict := kind != "session" + if !conflict { + if a.err != nil || b.err != nil { + t.Fatalf("same fenced sandbox's late business terminal blocked physical completion: %v / %v", a.err, b.err) + } + } else if factFirst { + assertResidualFactConflict(t, a.err) + if b.err == nil { + t.Fatal("completion ignored the other transaction's durable conflict") + } + } else { + if a.err != nil { + t.Fatal(a.err) + } + assertResidualFactConflict(t, b.err) + } + completed := !conflict || !factFirst + seedResidualAssert(t, scene, `SELECT r.responsibility=$2::jsonb AND (r.reclaim_conflicted_at IS NOT NULL)=$3 + AND (w.deleted_at IS NOT NULL)=$4 AND run.phase='done' AND run.status='failed' AND run.failure_reason='workspace_unavailable' + AND (i.state='succeeded')=$5 AND (SELECT count(*) FROM revisions WHERE run_id=r.run_id)=1 + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN issue_runs run ON run.id=r.run_id + JOIN agent_run_reclamation_intents i ON i.id=$6 WHERE r.run_id=$1`, scene.run, scene.responsibility, conflict, released && completed, released || completed, root.S("id")) + }) + } + } + } +} + +func TestControlledResidualRaceExternalAndPhysicalCompletionBothCommitOrders(t *testing.T) { + stores := controlledStores(t) + for _, released := range []bool{false, true} { + for _, externalFirst := range []bool{false, true} { + t.Run(fmt.Sprint("released=", released, "/external_first=", externalFirst), func(t *testing.T) { + scene, root, complete := seedControlledCompletionRace(t, stores(t), released) + intent := root + if released { + var id string + if err := scene.store.Pool.QueryRow(`SELECT id::text FROM agent_run_reclamation_intents WHERE root_intent_id=$1`, root.S("id")).Scan(&id); err != nil { + t.Fatal(err) + } + intent = seedControlledRow(t, scene.store, id) + } + r := seedControlledExternalRequest(scene, intent, "external-race") + external := func(tx *transaction) Object { return manageResidualCommand(tx, &r) } + first, second := complete, external + if externalFirst { + first, second = external, complete + } + a, b := seedResidualRace(t, scene, first, second) + if err := controlledRaceError(a); err != nil { + t.Fatal(err) + } + if !externalFirst && controlledRaceError(b) == nil { + t.Fatal("new external command overwrote an already physically completed intent") + } + disposition := "terminated_data_retained" + if released { + disposition = "released" + } + if externalFirst { + disposition = "externally_confirmed" + } + seedResidualAssert(t, scene, `SELECT r.disposition=$2 AND r.cleared_at IS NOT NULL AND r.responsibility=$3::jsonb + AND i.state='succeeded' AND i.disposition=$2 AND (w.deleted_at IS NOT NULL)=$4 + AND (SELECT count(*) FROM agent_run_reclamation_intents WHERE workspace_id=w.id AND command_kind='terminate')=1 + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN agent_run_reclamation_intents i ON i.id=$5 WHERE r.run_id=$1`, scene.run, disposition, scene.responsibility, released && !externalFirst, intent.S("id")) + }) + } + } +} + +func TestControlledResidualRaceNoTargetProofAndDispatchBothCommitOrders(t *testing.T) { + stores := controlledStores(t) + for _, proofFirst := range []bool{false, true} { + t.Run(fmt.Sprint("proof_first=", proofFirst), func(t *testing.T) { + s := stores(t) + abandoned := seedAbandonedCreation(t, s, false) + scene := residualReclaimScene{store: s, abandonedCreation: abandoned} + root := seedControlledRoot(t, scene) + proof := func(tx *transaction) Object { + current := tx.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", root.S("id")) + return controlledForceConfirm(tx, &ControlRequest{Action: "force_confirm", Body: Object{"epoch": 1, "version": current.N("version")}}, current) + } + producer := func(tx *transaction) Object { + return enqueueExecutionWork(tx, scene.run, "agent_session", Object{"kind": "agent_session"}, Object{"workspaceId": scene.workspace, "sandboxInstanceId": newID(), "nodeId": "late-producer"}, "", time.Time{}) + } + first, second := producer, proof + if proofFirst { + first, second = proof, producer + } + a, b := seedResidualRace(t, scene, first, second) + proved, rejected := b, a + if proofFirst { + proved, rejected = a, b + } + if proved.err != nil { + t.Fatalf("closed, never-planned responsibility could not complete: %v", proved.err) + } + if rejected.err == nil { + t.Fatal("old execution producer registered work against a closed residual") + } + // This scene is already terminal and ledger-protected before the controlled command. + // It proves the real producer cannot invalidate the no-target proof in either order; + // it does not claim that controlled closure is its only reason for rejection. + seedResidualAssert(t, scene, `SELECT r.reclaim_state='cleared' AND r.disposition='terminated_data_retained' + AND r.cleared_at IS NOT NULL AND NOT w.admission_open AND w.deleted_at IS NULL + AND NOT EXISTS(SELECT 1 FROM execution_work WHERE run_id=r.run_id) + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=r.workspace_id) + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_targets WHERE root_intent_id=$2) + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id WHERE r.run_id=$1`, scene.run, root.S("id")) + }) + } + for _, controlledFirst := range []bool{false, true} { + t.Run("legacy_producer/"+controlledRaceName(controlledFirst), func(t *testing.T) { + scene := seedControlledLegacyD8(t, stores(t)) + r := seedControlledRequest(scene, "legacy-dispatch-race") + controlled := func(tx *transaction) Object { return manageResidualCommand(tx, &r) } + producer := func(tx *transaction) Object { + return enqueueExecutionWork(tx, scene.run, "agent_session", Object{"kind": "agent_session"}, Object{"workspaceId": scene.workspace, "sandboxInstanceId": scene.effect, "runtimeGeneration": 1, "nodeId": "legacy-node"}, "", time.Time{}) + } + first, second := producer, controlled + if controlledFirst { + first, second = controlled, producer + } + a, b := seedResidualRace(t, scene, first, second) + registration, dispatch := b, a + if controlledFirst { + registration, dispatch = a, b + } + if err := controlledRaceError(registration); err != nil { + t.Fatal(err) + } + if controlledFirst && dispatch.err == nil { + t.Fatal("old D8 producer bypassed the committed controlled admission closure") + } + root := registration.out.O("intent") + out, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + return Object{"complete": completeControlledTermination(tx, tx.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", root.S("id")))} + }) + if err != nil || out.B("complete") { + t.Fatalf("legacy allocating identity was treated as a no-target proof: %v / %v", out, err) + } + var valid bool + if err := scene.store.Pool.QueryRow(`SELECT i.state<>'succeeded' AND i.completed_at IS NULL + AND EXISTS(SELECT 1 FROM agent_run_reclamation_targets WHERE root_intent_id=i.id AND sandbox_id=$2 AND state='planned') + AND (SELECT count(*) FROM execution_work WHERE run_id=i.run_id)=$3 + FROM agent_run_reclamation_intents i WHERE i.id=$1`, root.S("id"), scene.effect, map[bool]int{false: 0, true: 1}[dispatch.err == nil]).Scan(&valid); err != nil || !valid { + t.Fatalf("D8 dispatch race lost original identity or accepted unproved completion: %v / %v", valid, err) + } + }) + } +} diff --git a/internal/core/agent_run_controlled_reclaim_targets.go b/internal/core/agent_run_controlled_reclaim_targets.go new file mode 100644 index 000000000..d15b8eafa --- /dev/null +++ b/internal/core/agent_run_controlled_reclaim_targets.go @@ -0,0 +1,146 @@ +package core + +// refreshControlledTargets extends the original responsibility under the same transaction lock +// as planning and dispatch. It never removes an ensure identity merely because it has no sandbox row. +func refreshControlledTargets(t *transaction, root Object) Object { + row := refreshResidualTargets(t, controlledRootRow(t, root)) + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") != "sandbox" { + continue + } + generation := target.N("generation") + if generation == 0 { + if sb := t.one("SELECT generation FROM sandbox_instances WHERE id=$1 AND workspace_id=$2", target.S("id"), root.S("workspaceId")); sb != nil { + generation = sb.N("generation") + } else { + generation = root.N("runtimeGeneration") + } + } + request := Object{"kind": "sandbox_terminate", "projectId": root.S("projectId"), "workspaceId": root.S("workspaceId"), "sandboxInstanceId": target.S("id")} + t.exec(`INSERT INTO agent_run_reclamation_targets(id,root_intent_id,sandbox_id,generation,request) + VALUES($1,$2,$3,$4,$5) ON CONFLICT(root_intent_id,sandbox_id) DO NOTHING`, newID(), root.S("id"), target.S("id"), generation, jsonText(request)) + } + return row +} + +func controlledTerminationComplete(t *transaction, root, row Object) bool { + if row["reclaimConflictedAt"] != nil || !residualTargetsLocated(t, row) { + return false + } + found := false + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") == "sandbox" { + found = true + if !residualSandboxTerminated(t, row, target.S("id")) { + return false + } + } + } + if !found && !residualNeverPlanned(t, row) { + return false + } + return t.one("SELECT id FROM agent_run_reclamation_targets WHERE root_intent_id=$1 AND state<>'succeeded'", root.S("id")) == nil +} + +func controlledPendingPlans(t *transaction, r *ControlRequest) []Object { + plans := []Object{} + for _, root := range t.list("SELECT * FROM agent_run_reclamation_intents WHERE command_kind='terminate' AND state<>'succeeded' ORDER BY registered_at,id") { + row := refreshControlledTargets(t, root) + if row["reclaimConflictedAt"] != nil || !residualTargetsLocated(t, row) { + continue + } + t.exec("UPDATE agent_run_reclamation_intents SET state='terminating',started_at=COALESCE(started_at,clock_timestamp()),last_checked_at=clock_timestamp(),version=version+1 WHERE id=$1 AND state='registered'", root.S("id")) + root = t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", root.S("id")) + effects := []Object{} + for _, target := range t.list("SELECT * FROM agent_run_reclamation_targets WHERE root_intent_id=$1 ORDER BY id", root.S("id")) { + effects = append(effects, Object{"id": target.S("id"), "workspaceId": root.S("workspaceId"), "kind": "sandbox_terminate", "state": target.S("state"), "reconciledEpoch": r.Body.N("epoch"), "request": target.O("request"), "result": target.O("result")}) + } + plans = append(plans, Object{"forceStop": root, "effects": effects}) + } + return plans +} + +func controlledForceConfirm(t *transaction, r *ControlRequest, root Object) Object { + // Completed roots remain evidence intake identities, not reusable execution authority. + row := refreshControlledTargets(t, root) + id := r.Body.S("effectId") + if id != "" { + target := t.one("SELECT * FROM agent_run_reclamation_targets WHERE id=$1 AND root_intent_id=$2", id, root.S("id")) + require(target != nil, 404, "not_found") + fact := Object{"kind": "effect", "effectKind": "sandbox_terminate", "state": "succeeded", "result": Object{"terminated": r.Body.B("terminated"), "lateEnsureFenced": r.Body.B("lateEnsureFenced")}} + if root.S("state") == "succeeded" { + if target.S("state") == "succeeded" && jsonText(target.O("result")) != jsonText(fact.O("result")) { + appendResidualConflict(t, row, "controlled_effect:"+id, Object{"kind": "effect", "state": "succeeded", "result": target.O("result")}, fact) + } else { + appendResidualFact(t, row.S("runId"), "controlled_late:"+id, fact) + } + return root + } + version(root, r.Body.N("version")) + require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + require(r.Body.B("terminated") && r.Body.B("lateEnsureFenced"), 409, "termination_unconfirmed") + if !appendResidualFact(t, row.S("runId"), "controlled_effect:"+id, fact) { + return root + } + t.exec("UPDATE agent_run_reclamation_targets SET state='succeeded',result=$2,confirmed_epoch=$3,confirmed_at=COALESCE(confirmed_at,clock_timestamp()) WHERE id=$1", id, jsonText(fact.O("result")), r.Body.N("epoch")) + // Only confirmed physical targets change resource observations; business execution results + // and the historical failed operation are never synthesized or rewritten here. + t.exec("UPDATE sandbox_instances SET observed_state='terminated',terminated_at=COALESCE(terminated_at,clock_timestamp()) WHERE id=$1 AND workspace_id=$2", target.S("sandboxId"), root.S("workspaceId")) + t.exec("UPDATE node_instances SET connection_state='ended',ended_at=COALESCE(ended_at,clock_timestamp()),version=version+1 WHERE sandbox_instance_id=$1 AND ended_at IS NULL", target.S("sandboxId")) + } else { + require(root.S("state") != "succeeded", 409, "force_stop_completed") + version(root, r.Body.N("version")) + } + t.exec("UPDATE agent_run_reclamation_intents SET state='terminating',started_at=COALESCE(started_at,clock_timestamp()),last_checked_at=clock_timestamp(),version=version+1 WHERE id=$1", root.S("id")) + completeControlledTermination(t, root) + return t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", root.S("id")) +} + +func completeControlledTermination(t *transaction, root Object) bool { + root = t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", root.S("id")) + if root.S("state") == "succeeded" { + return root.S("disposition") == "terminated_data_retained" + } + row := refreshControlledTargets(t, root) + if !controlledTerminationComplete(t, root, row) { + return false + } + if !appendResidualFact(t, row.S("runId"), "controlled:"+root.S("id")+":retained", Object{"kind": "reclaim_complete", "state": "succeeded", "intentId": root.S("id"), "disposition": "terminated_data_retained"}) { + return false + } + t.exec("UPDATE agent_run_reclamation_intents SET state='succeeded',disposition='terminated_data_retained',started_at=COALESCE(started_at,clock_timestamp()),completed_at=clock_timestamp(),version=version+1 WHERE id=$1 AND state<>'succeeded'", root.S("id")) + if root.S("source") == "residual_ledger" { + t.exec("UPDATE agent_run_residuals SET reclaim_state='cleared',disposition='terminated_data_retained',cleared_at=COALESCE(cleared_at,clock_timestamp()),reclaim_version=reclaim_version+1 WHERE run_id=$1", root.S("runId")) + } + t.exec("UPDATE workspaces SET desired_state='stopped',observed_state='stopped',admission_open=false,version=version+1 WHERE id=$1", root.S("workspaceId")) + controlledAudit(t, root.S("id"), "terminated_data_retained", "controller", Object{}) + return true +} + +func controlledEffectPermit(t *transaction, r *ControlRequest, root Object) Object { + require(root.S("state") != "succeeded", 409, "force_stop_completed") + row := refreshControlledTargets(t, root) + require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + require(residualTargetsLocated(t, row), 409, "residual_target_unknown") + target := t.one("SELECT * FROM agent_run_reclamation_targets WHERE id=$1 AND root_intent_id=$2 AND state='planned'", r.EffectID, root.S("id")) + require(target != nil, 409, "stale_effect") + c := runtimeControl(t, root.S("workspaceId")) + require(c.N("controlEpoch") == root.N("controlEpoch"), 409, "stale_control") + clock := t.one("SELECT floor(extract(epoch FROM clock_timestamp())*1000)::bigint AS issued_at_ms,floor(extract(epoch FROM LEAST(clock_timestamp()+interval '10 seconds',(SELECT expires_at FROM controller_leases WHERE name='global')))*1000)::bigint AS expires_at_ms") + return Object{"tenantId": root.S("tenantId"), "workspaceId": root.S("workspaceId"), "sandboxId": target.S("sandboxId"), "runtimeGeneration": target.N("generation"), "controlEpoch": root.N("controlEpoch"), "controllerEpoch": r.Body.N("epoch"), "issuedAtMs": clock.N("issuedAtMs"), "expiresAtMs": clock.N("expiresAtMs")} +} + +// requireNoControlledReclamation protects ledgerless D8 and retained/external resources too. +// The only exception is the exact dependent delete operation bound to that root responsibility. +func requireNoControlledReclamation(t *transaction, wid, operationID string) { + root := controlledRootForWorkspace(t, wid) + if root == nil { + return + } + row := controlledRootRow(t, root) + require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + if operationID != "" && row.S("reclaimOperationId") == operationID { + return + } + require(root.S("state") == "succeeded" && row.S("disposition") == "released", 409, "resource_in_use") +} diff --git a/internal/core/agent_run_release_db_test.go b/internal/core/agent_run_release_db_test.go index f192cd400..a33810c31 100644 --- a/internal/core/agent_run_release_db_test.go +++ b/internal/core/agent_run_release_db_test.go @@ -47,8 +47,10 @@ func seedReleaseScene(t *testing.T, store *Store, phase, status string) releaseS } // The run Workspace a release deletes: isolated, bound to this exact run by the unique // workspaces.issue_run_id the operation identity is defined on. - exec("run-workspace", `INSERT INTO workspaces(id,tenant_id,owner_user_id,project_id,kind,desired_state,observed_state,runtime_generation,requested_ref,issue_run_id) - VALUES($1,$2,$3,$4,'isolated','deleted','deleting',1,'HEAD',$5)`, runws, seed.tenant, owner, seed.project, seed.run) + // Match insertWorkspace's verified_request identity at insertion: migration 0018 requires + // evidence whenever creator_user_id is non-NULL, and the creator tuple is immutable afterward. + exec("run-workspace", `INSERT INTO workspaces(id,tenant_id,owner_user_id,creator_user_id,creator_evidence,project_id,kind,desired_state,observed_state,runtime_generation,requested_ref,issue_run_id) + VALUES($1,$2,$3,$3,'verified_request',$4,'isolated','deleted','deleting',1,'HEAD',$5)`, runws, seed.tenant, owner, seed.project, seed.run) exec("task", `INSERT INTO tasks(id,workspace_id,title) VALUES($1,$2,'release task')`, newID(), runws) exec("stage", `UPDATE issue_runs SET phase=$2, status=$3, workspace_id=$4, version=version+1, updated_at=now() WHERE id=$1`, seed.run, phase, status, runws) diff --git a/internal/core/agent_run_residual.go b/internal/core/agent_run_residual.go index cab7db3f0..854e507ab 100644 --- a/internal/core/agent_run_residual.go +++ b/internal/core/agent_run_residual.go @@ -125,6 +125,12 @@ func collectResidualTargets(t *transaction, row Object) []Object { func refreshResidualTargets(t *transaction, row Object) Object { targets := collectResidualTargets(t, row) + if row.S("legacyIntentId") != "" { + if jsonText(targets) != jsonText(row["reclaimTargets"]) { + t.exec("UPDATE agent_run_reclamation_intents SET targets=$2,version=version+1 WHERE id=$1", row.S("legacyIntentId"), jsonText(targets)) + } + return residualForRun(t, row.S("runId")) + } if jsonText(targets) != jsonText(row["reclaimTargets"]) { t.exec("UPDATE agent_run_residuals SET reclaim_targets=$2,reclaim_version=reclaim_version+1 WHERE run_id=$1", row.S("runId"), jsonText(targets)) } @@ -134,6 +140,11 @@ func refreshResidualTargets(t *transaction, row Object) Object { // residualSandboxTerminated requires the two distinct physical assertions for the exact stable // sandbox ID. terminated_at, failed ensures, elapsed time and a missing sandbox row prove neither. func residualSandboxTerminated(t *transaction, row Object, sid string) bool { + if target := t.one(`SELECT rt.id FROM agent_run_reclamation_targets rt JOIN agent_run_reclamation_intents i ON i.id=rt.root_intent_id + WHERE i.tenant_id=$1 AND i.workspace_id=$2 AND rt.sandbox_id=$3 AND rt.state='succeeded' + AND rt.result->>'terminated'='true' AND rt.result->>'lateEnsureFenced'='true'`, row.S("tenantId"), row.S("workspaceId"), sid); target != nil { + return true + } for _, e := range t.list(`SELECT * FROM external_effects WHERE project_id=$1 AND workspace_id=$2 AND kind='sandbox_terminate' AND request->>'sandboxInstanceId'=$3 AND state='succeeded'`, row.S("projectId"), row.S("workspaceId"), sid) { if e.O("result").B("terminated") && e.O("result").B("lateEnsureFenced") { diff --git a/internal/core/agent_run_residual_fact.go b/internal/core/agent_run_residual_fact.go index d103825eb..247d4dfa6 100644 --- a/internal/core/agent_run_residual_fact.go +++ b/internal/core/agent_run_residual_fact.go @@ -11,7 +11,10 @@ func residualForWorkspace(t *transaction, workspaceID string) Object { if workspaceID == "" { return nil } - return t.one("SELECT * FROM agent_run_residuals WHERE workspace_id=$1", workspaceID) + if row := t.one("SELECT * FROM agent_run_residuals WHERE workspace_id=$1", workspaceID); row != nil { + return row + } + return legacyResidualForWorkspace(t, workspaceID) } func residualObjects(value any) []Object { @@ -36,7 +39,7 @@ func residualObjects(value any) []Object { // False means a contradictory terminal was durably audited; callers must not apply it to the // canonical execution/effect. Control returns the stable error only after this transaction commits. func appendResidualFact(t *transaction, run, key string, fact Object) bool { - row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", run) + row := residualForRun(t, run) require(row != nil, 409, "operation_not_abandoned") var previous Object entries := residualObjects(row["reclaimEvidence"]) @@ -74,7 +77,7 @@ func appendResidualFact(t *transaction, run, key string, fact Object) bool { // later success as an A4 contradiction. Historical provisioning failures and every successful // conclusion remain immutable, and external binding conflicts are checked before this path. func residualCleanupFailureRetryable(t *transaction, row Object, key string, fact Object) bool { - if fact.S("state") != "failed" || fact.S("kind") != "effect" || row.S("reclaimMode") != "owned_delete" || + if fact.S("state") != "failed" || fact.S("kind") != "effect" || (row.S("reclaimMode") != "owned_delete" && row.S("reclaimMode") != "controlled") || row.S("reclaimOperationId") == "" || !strings.HasPrefix(key, "effect:") { return false } @@ -107,6 +110,11 @@ func appendResidualEntry(t *transaction, row Object, key string, fact Object, co "key": key, "fact": fact, "source": source, "receivedAt": t.one("SELECT clock_timestamp()::text AS received_at").S("receivedAt"), } + if row.S("legacyIntentId") != "" { + t.exec(`UPDATE agent_run_reclamation_intents SET evidence=evidence||jsonb_build_array($2::jsonb),version=version+1, + conflicted_at=CASE WHEN $3 THEN COALESCE(conflicted_at,clock_timestamp()) ELSE conflicted_at END WHERE id=$1`, row.S("legacyIntentId"), jsonText(entry), conflict) + return + } t.exec(`UPDATE agent_run_residuals SET reclaim_evidence=reclaim_evidence||jsonb_build_array($2::jsonb), reclaim_version=reclaim_version+1, reclaim_conflicted_at=CASE WHEN $3 THEN COALESCE(reclaim_conflicted_at,clock_timestamp()) ELSE reclaim_conflicted_at END @@ -118,8 +126,12 @@ func residualEffectRow(t *transaction, e Object) Object { return row } // Some legacy preallocated ensure plans have no Workspace binding on the effect row. - return t.one(`SELECT ar.* FROM agent_run_residuals ar JOIN operations o ON o.workspace_id=ar.workspace_id - WHERE o.id=$1 AND o.tenant_id=ar.tenant_id AND o.project_id=ar.project_id`, e.S("operationId")) + if row := t.one(`SELECT ar.* FROM agent_run_residuals ar JOIN operations o ON o.workspace_id=ar.workspace_id + WHERE o.id=$1 AND o.tenant_id=ar.tenant_id AND o.project_id=ar.project_id`, e.S("operationId")); row != nil { + return row + } + op := t.one("SELECT workspace_id FROM operations WHERE id=$1", e.S("operationId")) + return legacyResidualForWorkspace(t, op.S("workspaceId")) } func acceptResidualEffectFact(t *transaction, row, e, incoming Object) bool { diff --git a/internal/core/agent_run_residual_loop.go b/internal/core/agent_run_residual_loop.go index 342c8749e..537a8ff37 100644 --- a/internal/core/agent_run_residual_loop.go +++ b/internal/core/agent_run_residual_loop.go @@ -51,5 +51,5 @@ func (s *Store) ReconcileAgentRunResidualsOnce(ctx context.Context) error { return resumeResidualReclamation(t, row) }) } - return nil + return s.reconcileControlledResiduals(ctx) } diff --git a/internal/core/agent_run_residual_reclaim.go b/internal/core/agent_run_residual_reclaim.go index 3aad66409..a37461dba 100644 --- a/internal/core/agent_run_residual_reclaim.go +++ b/internal/core/agent_run_residual_reclaim.go @@ -11,6 +11,9 @@ func claimResidualReclamation(t *transaction, runID, mode string, authorization require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") if row.S("reclaimState") != "reconciling" { require(row.S("reclaimMode") == mode, 409, "residual_mode_conflict") + if mode == "controlled" { + require(row.O("reclaimAuthorization").S("intentId") == authorization.S("intentId"), 409, "residual_mode_conflict") + } return Object{"residual": row, "operation": residualBoundOperation(t, row), "claimed": false} } row = refreshResidualTargets(t, row) @@ -69,16 +72,26 @@ func residualOperation(t *transaction, op Object) Object { if op == nil { return nil } - return t.one("SELECT * FROM agent_run_residuals WHERE reclaim_operation_id=$1", op.S("id")) + if row := t.one("SELECT * FROM agent_run_residuals WHERE reclaim_operation_id=$1", op.S("id")); row != nil { + return row + } + if root := controlledOperationRoot(t, op); root != nil { + return controlledRootRow(t, root) + } + return nil } // checkResidualOperation is called before plan, permit and advance, even when returning an existing // effect. Ordinary operations return nil only when their Workspace has no residual responsibility. // A conflicting cleared record remains a durable block, rather than reopening ordinary cleanup. func checkResidualOperation(t *transaction, op Object) Object { + if root := controlledOperationRoot(t, op); root != nil { + return checkControlledOperation(t, op, root) + } row := residualOperation(t, op) if row == nil { if op.S("workspaceId") != "" { + requireNoControlledReclamation(t, op.S("workspaceId"), "") other := residualForWorkspace(t, op.S("workspaceId")) require(other == nil || (other.S("reclaimState") == "cleared" && other["reclaimConflictedAt"] == nil), 409, "residual_operation_required") } @@ -100,6 +113,9 @@ func checkResidualOperation(t *transaction, op Object) Object { // sandbox/ensure identity is dual-confirmed, never by result=NULL or a ticket timeout. func checkResidualCleanupActivities(t *transaction, w, op Object) { row := checkResidualOperation(t, op) + if row != nil && row.S("reclaimMode") == "controlled" { + return + } if row == nil { checkActivities(t, w) return @@ -136,6 +152,10 @@ func checkResidualEffect(t *transaction, op Object, kind, workspaceID string, ef return } require(workspaceID == row.S("workspaceId"), 409, "residual_identity_conflict") + if row.S("reclaimMode") == "controlled" { + require(kind == "workspace_data_delete", 409, "residual_effect_forbidden") + return + } w := t.one("SELECT * FROM workspaces WHERE id=$1", workspaceID) checkResidualCleanupActivities(t, w, op) require(kind == "sandbox_terminate" || kind == "workspace_data_delete", 409, "residual_effect_forbidden") @@ -169,6 +189,9 @@ func checkResidualEffect(t *transaction, op Object, kind, workspaceID string, ef // completeResidualReclamation is called from cleanup->done in the SAME transaction that marks the // operation succeeded and soft-deletes the Workspace. Failure rolls those writes back together. func completeResidualReclamation(t *transaction, op Object) bool { + if root := controlledOperationRoot(t, op); root != nil { + return completeControlledDataDelete(t, op, root) + } row := residualOperation(t, op) if row == nil { return true diff --git a/internal/core/agent_run_session_settle.go b/internal/core/agent_run_session_settle.go index fa12e4ca4..4bc79bdc8 100644 --- a/internal/core/agent_run_session_settle.go +++ b/internal/core/agent_run_session_settle.go @@ -89,7 +89,7 @@ func (s *Store) settleSessionEnded(t *transaction, runID, executionID string, en if !sessionEndReasons[ended.S("reason")] { return fmt.Errorf("sessionEnded: run %s execution %s carries end reason %q, outside the closed set", runID, executionID, ended.S("reason")) } - if o.S("phase") == "done" && t.one("SELECT run_id FROM agent_run_residuals WHERE run_id=$1", runID) != nil { + if o.S("phase") == "done" && residualForRun(t, runID) != nil { t.exec("UPDATE agent_run_residuals SET session_ended_at=COALESCE(session_ended_at,clock_timestamp()) WHERE run_id=$1", runID) return nil } diff --git a/internal/core/agent_run_test_support_test.go b/internal/core/agent_run_test_support_test.go index d4bbe89bf..60f734939 100644 --- a/internal/core/agent_run_test_support_test.go +++ b/internal/core/agent_run_test_support_test.go @@ -39,6 +39,11 @@ import ( // cmd/server wires them at process start. A Store returned here is therefore the production Store as // far as the A/B seam is concerned; only the sleepers and background loops are the caller's business. func dispatcherDB(t *testing.T) *Store { + t.Helper() + return dispatcherDBWithCleanup(t, t.Cleanup) +} + +func dispatcherDBWithCleanup(t *testing.T, cleanup func(func())) *Store { t.Helper() dsn := os.Getenv("TEST_DATABASE_URL") if dsn == "" { @@ -61,7 +66,7 @@ func dispatcherDB(t *testing.T) *Store { } config.RuntimeParams["search_path"] = schema pool := stdlib.OpenDB(*config) - t.Cleanup(func() { + cleanup(func() { _ = pool.Close() _, _ = admin.Exec("DROP SCHEMA " + schema + " CASCADE") _ = admin.Close() diff --git a/internal/core/control.go b/internal/core/control.go index a470f792c..520b3a300 100644 --- a/internal/core/control.go +++ b/internal/core/control.go @@ -105,6 +105,9 @@ func (s *Store) Control(ctx context.Context, r *ControlRequest) (Object, error) // operationCommand runs one Effect-level action on the operation the caller claimed. func operationCommand(t *transaction, r *ControlRequest) Object { if r.Action == "effect_result" && validID(r.OperationID) { + if completed := t.one("SELECT * FROM operations WHERE id=$1 AND state IN ('succeeded','failed')", r.OperationID); completed != nil && controlledOperationRoot(t, completed) != nil { + return effectResult(t, r, completed) + } // A completed residual operation has no execution authority left, but its original // effect receipts still arrive through this existing intake. Do not require a running // operation or its old version merely to preserve an identical fact or audit a conflict. @@ -132,7 +135,7 @@ func operationCommand(t *transaction, r *ControlRequest) Object { require(deferCodes[code], 400, "invalid_error_code") delay := r.Body.N("retrySeconds") require(delay >= 1 && delay <= 3600, 400, "invalid_retry_delay") - if row := residualOperation(t, o); row != nil && row.S("reclaimMode") == "owned_delete" && row.S("reclaimState") == "reclaiming" && row["reclaimConflictedAt"] == nil { + if row := residualOperation(t, o); row != nil && (row.S("reclaimMode") == "owned_delete" || row.S("reclaimMode") == "controlled") && row["reclaimConflictedAt"] == nil { // A3 has no public/operator retry path for this hidden Workspace. Preserve the // same retryable operation even when the existing Controller requests blocked // for an unconfirmed termination; plan/permit still require fresh real evidence. @@ -296,6 +299,7 @@ func planEffect(t *transaction, r *ControlRequest, o Object) Object { kind, wid := r.Body.S("kind"), r.Body.S("workspaceId") require(validID(wid), 400, "invalid_effect_scope") requireNoResidualConflict(t, wid) + requireNoControlledReclamation(t, wid, o.S("id")) checkResidualEffect(t, o, kind, wid, nil) found := false for _, w := range operationWorkspaces(t, o) { @@ -384,7 +388,7 @@ func effectResult(t *transaction, r *ControlRequest, o Object) Object { if !acceptResidualEffectFact(t, residual, e, fact) { return Object{"effect": e, "operation": t.one("SELECT * FROM operations WHERE id=$1", o.S("id"))} } - if residual.S("reclaimState") == "cleared" && residual.S("reclaimOperationId") == o.S("id") { + if residual.S("reclaimState") == "cleared" && residual.S("reclaimOperationId") == o.S("id") && (o.S("state") == "succeeded" || o.S("state") == "failed") { // Receipt only: neither canonical effects, operation versions nor resource rows may // advance again after atomic completion. Contradictions already committed their block. return Object{"effect": e, "operation": o} @@ -539,7 +543,9 @@ func advance(t *transaction, r *ControlRequest, o Object) Object { runWorkspaceSettled(t, runID, "ready") } case "delete_workspace": - runWorkspaceDeleted(t, runID) + if controlledOperationRoot(t, o) == nil { + runWorkspaceDeleted(t, runID) + } } } } diff --git a/internal/core/execution_work.go b/internal/core/execution_work.go index d1b218c35..ab5fc4110 100644 --- a/internal/core/execution_work.go +++ b/internal/core/execution_work.go @@ -29,6 +29,7 @@ func enqueueExecutionWork(t *transaction, runID, kind string, input, target Obje return enqueueWorkflowExecutionWork(t, runID, kind, input, target, graphNode, availableAt) } require(t.one("SELECT run_id FROM agent_run_residuals WHERE run_id=$1", runID) == nil, 409, "dispatch_conflict") + requireNoControlledReclamation(t, target.S("workspaceId"), "") if kind == "deliver_revision" { if t.one("SELECT run_id FROM revision_delivery_skips WHERE run_id=$1", runID) != nil { return Object{"skipped": true} diff --git a/internal/core/migrations/0058_agent_run_controlled_reclamation.sql b/internal/core/migrations/0058_agent_run_controlled_reclamation.sql new file mode 100644 index 000000000..1ac2946c5 --- /dev/null +++ b/internal/core/migrations/0058_agent_run_controlled_reclamation.sql @@ -0,0 +1,69 @@ +-- The ledger remains authoritative for D4/D5. A root intent owns a D8 responsibility only +-- when no ledger exists; a dependent delete intent never acquires a second responsibility. +CREATE TABLE agent_run_reclamation_intents ( + id uuid PRIMARY KEY, + root_intent_id uuid REFERENCES agent_run_reclamation_intents(id), + command_kind text NOT NULL CHECK(command_kind IN ('terminate','delete_data')), + source text NOT NULL CHECK(source IN ('residual_ledger','legacy_d8')), + tenant_id uuid NOT NULL REFERENCES tenants(id), project_id uuid NOT NULL REFERENCES projects(id), + run_id uuid NOT NULL REFERENCES issue_runs(id), workspace_id uuid NOT NULL REFERENCES workspaces(id), + runtime_generation bigint NOT NULL CHECK(runtime_generation>=0), + reclaim_id uuid, legacy_delete_operation_id uuid REFERENCES operations(id), + delete_operation_id uuid REFERENCES operations(id), + state text NOT NULL DEFAULT 'registered' CHECK(state IN ('registered','terminating','succeeded')), + disposition text CHECK(disposition IN ('terminated_data_retained','released','externally_confirmed')), + control_epoch bigint NOT NULL, actor_subject text NOT NULL CHECK(length(actor_subject)>0), + operator_label text NOT NULL DEFAULT '' CHECK(length(operator_label)<=200), + reason text NOT NULL CHECK(length(btrim(reason)) BETWEEN 1 AND 2000), + impact_confirmed boolean NOT NULL DEFAULT false, data_delete_confirmed boolean NOT NULL DEFAULT false, + targets jsonb NOT NULL DEFAULT '[]' CHECK(jsonb_typeof(targets)='array'), + evidence jsonb NOT NULL DEFAULT '[]' CHECK(jsonb_typeof(evidence)='array'), + audit jsonb NOT NULL DEFAULT '[]' CHECK(jsonb_typeof(audit)='array'), + conflicted_at timestamptz, + registered_at timestamptz NOT NULL DEFAULT clock_timestamp(), started_at timestamptz, + completed_at timestamptz, last_checked_at timestamptz, version bigint NOT NULL DEFAULT 1 CHECK(version>0), + UNIQUE(id,tenant_id,project_id,run_id,workspace_id), + FOREIGN KEY(root_intent_id,tenant_id,project_id,run_id,workspace_id) + REFERENCES agent_run_reclamation_intents(id,tenant_id,project_id,run_id,workspace_id), + CHECK((command_kind='terminate' AND root_intent_id IS NULL AND reclaim_id IS NOT NULL AND impact_confirmed) + OR (command_kind='delete_data' AND root_intent_id IS NOT NULL AND reclaim_id IS NULL AND data_delete_confirmed)), + CHECK((source='legacy_d8' AND legacy_delete_operation_id IS NOT NULL) + OR (source='residual_ledger' AND legacy_delete_operation_id IS NULL)), + CHECK((state='succeeded' AND completed_at IS NOT NULL AND disposition IS NOT NULL) + OR (state<>'succeeded' AND completed_at IS NULL AND disposition IS NULL)) +); +CREATE UNIQUE INDEX agent_run_reclamation_root_workspace ON agent_run_reclamation_intents(workspace_id) + WHERE root_intent_id IS NULL; +CREATE UNIQUE INDEX agent_run_reclamation_root_responsibility ON agent_run_reclamation_intents(reclaim_id) + WHERE reclaim_id IS NOT NULL; +CREATE UNIQUE INDEX agent_run_reclamation_one_delete ON agent_run_reclamation_intents(root_intent_id) + WHERE command_kind='delete_data'; +CREATE UNIQUE INDEX agent_run_reclamation_delete_operation ON agent_run_reclamation_intents(delete_operation_id) + WHERE command_kind='delete_data' AND delete_operation_id IS NOT NULL; +CREATE INDEX agent_run_reclamation_scope ON agent_run_reclamation_intents(tenant_id,project_id,run_id,workspace_id); +CREATE INDEX agent_run_reclamation_pending ON agent_run_reclamation_intents(last_checked_at NULLS FIRST,registered_at,id) + WHERE state<>'succeeded'; + +-- sandbox_id deliberately has no sandbox_instances FK: an ensure plan is already a target +-- before an allocating sandbox row has arrived. Its original identity is verified in the transaction. +CREATE TABLE agent_run_reclamation_targets ( + id uuid PRIMARY KEY, root_intent_id uuid NOT NULL REFERENCES agent_run_reclamation_intents(id), + sandbox_id uuid NOT NULL, generation bigint NOT NULL CHECK(generation>=0), + state text NOT NULL DEFAULT 'planned' CHECK(state IN ('planned','succeeded')), + request jsonb NOT NULL CHECK(jsonb_typeof(request)='object'), + result jsonb NOT NULL DEFAULT '{}' CHECK(jsonb_typeof(result)='object'), + confirmed_epoch bigint, confirmed_at timestamptz, + UNIQUE(root_intent_id,sandbox_id), + CHECK((state='planned' AND confirmed_at IS NULL) + OR (state='succeeded' AND confirmed_at IS NOT NULL AND confirmed_epoch IS NOT NULL + AND result @> '{"terminated":true,"lateEnsureFenced":true}'::jsonb)) +); +CREATE TABLE agent_run_reclamation_command_receipts ( + id uuid PRIMARY KEY, tenant_id uuid NOT NULL REFERENCES tenants(id), + idempotency_key text NOT NULL CHECK(length(idempotency_key) BETWEEN 1 AND 200), + command_input jsonb NOT NULL CHECK(jsonb_typeof(command_input)='object'), + intent_id uuid NOT NULL REFERENCES agent_run_reclamation_intents(id), + actor_subject text NOT NULL, result jsonb NOT NULL DEFAULT '{}' CHECK(jsonb_typeof(result)='object'), + audit jsonb NOT NULL DEFAULT '[]' CHECK(jsonb_typeof(audit)='array'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), UNIQUE(tenant_id,idempotency_key) +); diff --git a/internal/core/migrations/README.en.md b/internal/core/migrations/README.en.md index 06fd662f6..9a437d5a3 100644 --- a/internal/core/migrations/README.en.md +++ b/internal/core/migrations/README.en.md @@ -6,7 +6,7 @@ SQL is embedded in the binary through `embed.FS` in `core/store.go`. `Store.Migr ## File index -There are currently 59 SQL files. +There are currently 60 SQL files. | SQL file | Schema change | |---|---| @@ -69,6 +69,7 @@ There are currently 59 SQL files. | [0055_workflow_run_workspace_release.sql](0055_workflow_run_workspace_release.sql) | Workspace release facts for WorkflowRuns | | [0056_workflow_invocation_reference.sql](0056_workflow_invocation_reference.sql) | Invocation reference from workflow IssueRuns to WorkflowRuns | | [0057_agent_run_residual_reclamation.sql](0057_agent_run_residual_reclamation.sql) | Residual reclamation responsibility, appended evidence and disposition | +| [0058_agent_run_controlled_reclamation.sql](0058_agent_run_controlled_reclamation.sql) | Deployment-controlled reclamation intents, dependent data deletion and per-target termination | 0050 copies no personal connection or key and backfills no historical run. Clearing configuration retains its revision; run options are stored separately from shared run input. @@ -89,6 +90,7 @@ There are currently 59 SQL files. - 0055 adds the forward-only `workspace_release_state` (`pending`/`releasing`/`released`/`abandoned`/`not_required`), `workspace_release_error` and `workspace_released_at` to `workflow_runs`, and backfills by upgrade-time state: non-terminal rows stay `pending`; terminal rows with no Workspace become `not_required`, those whose Workspace is already soft-deleted become `released` (using that row's `deleted_at`), and the rest become `releasing`. Two CHECK constraints require a release instant for `released` and a reason for `abandoned`. Only `released` means Cloud observed the Workspace deleted; `abandoned` keeps the Workspace row and writes no release instant. Semantics are in [workflows](../../../docs/workflows/workflows.en.md#scheduling); coverage is `TestMigration0055WorkflowRunWorkspaceReleaseAppliesFreshAndUpgrades` and `internal/core/workflow_run_release_db_test.go`. - 0056 first settles every unlinked non-terminal historical workflow IssueRun (including running and deferred) as failed with `workflow_execution_unavailable_before_invocation_linkage`, preserving its input and outcome without inventing an execution. It then adds the `(workflow_invocation_ref, tenant_id)` → `workflow_runs(id, tenant_id)` composite foreign key (targeting the `workflow_run_tenant_identity` unique constraint 0044 already added, without duplicating it), a unique reference, and a CHECK that non-terminal workflow rows carry a reference while other executors carry none, all unconditionally validated. Historical terminal workflow rows may keep a NULL reference. Fresh, seeded upgrade, repeated application and constraint validation coverage is in `integration/workflow_invocation_migration_test.go`. - 0057 only adds residual-ledger columns: defaults initialize existing D4/D5 rows as `reconciling`, without backfilling targets or evidence, rewriting old `reason`, `responsibility` or business settlement, or backfilling D8. Legacy rows with non-null `cleared_at` cause an explicit migration error and rollback pending compatibility handling (A2). Reentrant DDL preserves progressed rows. The schema accepts `released`, `terminated_data_retained` and `externally_confirmed`; this step's automatic path implements only `released`, leaving the other dispositions to later controlled entry points. The authorization transaction checks the operation's tenant and Workspace relationships; a UUID foreign key alone does not establish resource scope. See [agent_run_residual_reclaim_migration_test.go](../../../integration/agent_run_residual_reclaim_migration_test.go) for fresh, upgrade, DDL replay and constraint coverage. +- 0058 preserves 0057 history, responsibility, evidence, disposition and first `cleared_at`, and never backfills D8. Only an explicit root intent owns protection and evidence for ledgerless D8. Root termination and dependent data deletion each have three states; each resource has one root, at most one deletion intent and one fixed deletion operation. Command keys are unique throughout a tenant. Successful targets require both boolean confirmations, so a missing JSON field cannot pass through SQL NULL. Fresh, 0057-upgrade and constraint coverage is in [agent_run_controlled_reclaim_migration_test.go](../../../integration/agent_run_controlled_reclaim_migration_test.go). - New migrations must verify fresh databases, upgrades from the preceding schema, repeated application and key constraints. See [migration_upgrade_path_test.go](../../../integration/migration_upgrade_path_test.go) and domain upgrade tests in integration. See [Persistence](../../../docs/core/persistence.en.md) for ownership/transactions, and [Workflows](../../../docs/workflows/workflows.en.md) / [External Git line](../../../docs/repositories/external-git.en.md) for runtime semantics. diff --git a/internal/core/migrations/README.md b/internal/core/migrations/README.md index 873dc1c08..0caad7ba2 100644 --- a/internal/core/migrations/README.md +++ b/internal/core/migrations/README.md @@ -6,7 +6,7 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 ## 文件索引 -当前共 59 个 SQL 文件。 +当前共 60 个 SQL 文件。 | SQL 文件 | schema 变化 | |---|---| @@ -69,6 +69,7 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 | [0055_workflow_run_workspace_release.sql](0055_workflow_run_workspace_release.sql) | WorkflowRun 的 Workspace 释放事实 | | [0056_workflow_invocation_reference.sql](0056_workflow_invocation_reference.sql) | workflow IssueRun 到 WorkflowRun 的调用引用 | | [0057_agent_run_residual_reclamation.sql](0057_agent_run_residual_reclamation.sql) | 残留回收责任、追加证据和处置状态 | +| [0058_agent_run_controlled_reclamation.sql](0058_agent_run_controlled_reclamation.sql) | 部署受控回收意图、从属数据删除与逐目标终止 | 0050 不复制个人连接或 Key，不回填历史运行；清除配置保留 revision，运行选项独立于共享运行输入保存。 @@ -89,6 +90,7 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 - 0055 给 `workflow_runs` 增加单向的 `workspace_release_state`（`pending`/`releasing`/`released`/`abandoned`/`not_required`）、`workspace_release_error` 和 `workspace_released_at`，并按升级时的状态回填：非终态行保持 `pending`；终态行没有 Workspace 的记 `not_required`，Workspace 已软删除的记 `released`（释放时刻取该行的 `deleted_at`），其余记 `releasing`。两条 CHECK 保证 `released` 必有释放时刻、`abandoned` 必有原因。只有 `released` 表示 Cloud 观察到 Workspace 已被删除；`abandoned` 保留 Workspace 行、不写释放时刻。语义见[工作流](../../../docs/workflows/workflows.md#调度)，覆盖见 `TestMigration0055WorkflowRunWorkspaceReleaseAppliesFreshAndUpgrades` 与 `internal/core/workflow_run_release_db_test.go`。 - 0056 先把没有引用的历史非终态 workflow IssueRun（含 running、deferred）结算为 failed（`workflow_execution_unavailable_before_invocation_linkage`），保留其输入和结果、不虚构执行；再加 `(workflow_invocation_ref, tenant_id)` → `workflow_runs(id, tenant_id)` 复合外键（引用 0044 已建的 `workflow_run_tenant_identity` 唯一约束，不重复建）、唯一引用，以及「非终态 workflow 行必须有引用、其他执行器不得有引用」的 CHECK，无条件 VALIDATE。历史终态 workflow 行可保留 NULL 引用。空库、带历史数据升级、重复应用与约束验证状态见 `integration/workflow_invocation_migration_test.go`。 - 0057 只给残留账本增列：存量 D4/D5 行通过默认值初始化为 `reconciling`，不回填目标或证据，不改旧 `reason`、`responsibility` 或业务结算，也不回填 D8。历史行若已有非空 `cleared_at`，迁移明确报错并回滚，等待显式兼容处理（A2）。DDL 可重入，不会重置已推进条目。数据库接受 `released`、`terminated_data_retained`、`externally_confirmed` 三种处置；本步自动路径只实现 `released`，其余处置留给后续受控入口。operation 的租户和 Workspace 关联由授权事务比对，UUID 外键本身不证明资源范围。空库、升级、DDL 重放和约束覆盖见 [agent_run_residual_reclaim_migration_test.go](../../../integration/agent_run_residual_reclaim_migration_test.go)。 +- 0058 不改 0057 的历史列、责任、证据、处置或首次 `cleared_at`，也不回填 D8；无账本的 D8 只由显式根意图承载保护与证据。根终止与从属删卷各自三态；每资源一个根、最多一个从属删除意图及固定删除 operation，命令键在 tenant 范围统一唯一。目标的成功约束要求两个布尔确认均为 true，缺字段不能以 SQL NULL 通过。新库、0057 升级与约束覆盖见 [agent_run_controlled_reclaim_migration_test.go](../../../integration/agent_run_controlled_reclaim_migration_test.go)。 - 新增迁移须验证空库、前一 schema 升级、重复应用与关键约束。覆盖见 [migration_upgrade_path_test.go](../../../integration/migration_upgrade_path_test.go) 和 integration 的领域升级测试。 数据归属与事务规则见 [持久化](../../../docs/core/persistence.md)，运行语义见 [工作流](../../../docs/workflows/workflows.md) 和 [外部 Git 线](../../../docs/repositories/external-git.md)。 diff --git a/internal/core/node.go b/internal/core/node.go index 882eb8031..fa4f15c53 100644 --- a/internal/core/node.go +++ b/internal/core/node.go @@ -117,7 +117,8 @@ func nodeStatus(t *transaction, r *ControlRequest, n, w Object) { // A business-abandoned execution still owns residual resources after it reconnects; // connection evidence must not reopen their admission or imply physical cleanup. t.exec(`UPDATE workspaces SET observed_state='ready',admission_open=NOT EXISTS( - SELECT 1 FROM agent_run_residuals WHERE workspace_id=$1 AND (cleared_at IS NULL OR reclaim_conflicted_at IS NOT NULL)), + SELECT 1 FROM agent_run_residuals WHERE workspace_id=$1 AND (cleared_at IS NULL OR reclaim_conflicted_at IS NOT NULL)) + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_intents WHERE workspace_id=$1 AND root_intent_id IS NULL), version=version+1 WHERE id=$1`, w.S("id")) } } diff --git a/internal/core/node_execution.go b/internal/core/node_execution.go index 09c95af4f..b8c664e4b 100644 --- a/internal/core/node_execution.go +++ b/internal/core/node_execution.go @@ -53,6 +53,9 @@ func dispatchPluginExecution(t *transaction, r *ControlRequest, operation, execu } func dispatchRunExecution(t *transaction, r *ControlRequest, runID, execution, node string, input Object) Object { + if root := t.one("SELECT workspace_id FROM agent_run_reclamation_intents WHERE run_id=$1 AND root_intent_id IS NULL", runID); root != nil { + requireNoControlledReclamation(t, root.S("workspaceId"), "") + } require(t.one("SELECT run_id FROM agent_run_residuals WHERE run_id=$1", runID) == nil, 409, "dispatch_conflict") // The work item names its owner — IssueRun or WorkflowRun (D8); operation_id is that run's id // either way, so the ownership check re-reads whichever column the row carries. diff --git a/internal/core/public.go b/internal/core/public.go index 38f94fd72..a6f5e649c 100644 --- a/internal/core/public.go +++ b/internal/core/public.go @@ -117,7 +117,9 @@ func (s *Store) Public(ctx context.Context, r *PublicRequest) (Object, int, erro out, status = createTenant(t, r, uid) return out } - isAdmin := r.SpaceID == "" && (strings.HasSuffix(r.Path, "/resource-status") || strings.HasSuffix(r.Path, "/force-stop") || (r.UserID != "" && r.Method == "PUT") || strings.Contains(r.Path, "/invitations") || strings.Contains(r.Path, "/join-links") || strings.Contains(r.Path, "/join-requests") || strings.HasSuffix(r.Path, "/members/huawei")) + // Force-stop resolves the ordinary Workspace before checking administrator status. Hidden + // run Workspaces must remain 404 for every tenant role, including ordinary members. + isAdmin := r.SpaceID == "" && (strings.HasSuffix(r.Path, "/resource-status") || (r.UserID != "" && r.Method == "PUT") || strings.Contains(r.Path, "/invitations") || strings.Contains(r.Path, "/join-links") || strings.Contains(r.Path, "/join-requests") || strings.HasSuffix(r.Path, "/members/huawei")) membership(t, r.TenantID, uid, isAdmin) refreshRuntimeControls(t) if r.Method == "GET" { @@ -717,6 +719,7 @@ func newOperation(t *transaction, r *PublicRequest, uid, pid, wid, kind, step, h } func checkActivities(t *transaction, w Object) { + requireNoControlledReclamation(t, w.S("id"), "") requireNoResidualConflict(t, w.S("id")) require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL", w.S("id")) == nil, 409, "resource_in_use") require(t.one("SELECT id FROM execution_tickets WHERE workspace_id=$1 AND state='active' AND terminated_by_force_stop_id IS NULL", w.S("id")) == nil, 409, "resource_in_use") diff --git a/internal/core/run_workspace.go b/internal/core/run_workspace.go index 943ed1db1..d4d1bdc15 100644 --- a/internal/core/run_workspace.go +++ b/internal/core/run_workspace.go @@ -115,6 +115,7 @@ func deleteWorkflowRunWorkspace(t *transaction, runID string) Object { func deleteRunWorkspace(t *transaction, runID string) Object { w := t.one("SELECT * FROM workspaces WHERE issue_run_id=$1", runID) require(w != nil, 404, "not_found") + requireNoControlledReclamation(t, w.S("id"), "") requireNoResidualConflict(t, w.S("id")) require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL", w.S("id")) == nil, 409, "resource_in_use") if w["deletedAt"] != nil { diff --git a/internal/core/runtime_control.go b/internal/core/runtime_control.go index 0cc0473cf..b3ce419bb 100644 --- a/internal/core/runtime_control.go +++ b/internal/core/runtime_control.go @@ -35,6 +35,7 @@ func reconcileRuntimeControls(t *transaction) { WHERE c.state IN ('draining','reconciling') AND NOT EXISTS(SELECT 1 FROM runtime_force_stops f WHERE f.workspace_id=w.id AND f.state<>'succeeded') AND NOT EXISTS(SELECT 1 FROM agent_run_residuals ar WHERE ar.workspace_id=w.id AND (ar.cleared_at IS NULL OR ar.reclaim_conflicted_at IS NOT NULL)) + AND NOT EXISTS(SELECT 1 FROM agent_run_reclamation_intents ri WHERE ri.workspace_id=w.id AND ri.root_intent_id IS NULL) AND (c.input_closed OR (w.observed_state IN ('stopped','deleted') AND NOT EXISTS(SELECT 1 FROM sandbox_instances s WHERE s.workspace_id=w.id AND s.terminated_at IS NULL))) AND NOT EXISTS(SELECT 1 FROM execution_tickets a WHERE a.workspace_id=w.id AND a.state='active' AND a.terminated_by_force_stop_id IS NULL) diff --git a/internal/core/runtime_force_stop.go b/internal/core/runtime_force_stop.go index 7627d765c..933d246ba 100644 --- a/internal/core/runtime_force_stop.go +++ b/internal/core/runtime_force_stop.go @@ -46,8 +46,12 @@ func forceStopCommand(t *transaction, r *ControlRequest) Object { } plans = append(plans, Object{"forceStop": f, "effects": targets}) } + plans = append(plans, controlledPendingPlans(t, r)...) return Object{"plans": plans} case "force_confirm": + if root := t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1 AND command_kind='terminate'", r.Body.S("forceStopId")); root != nil { + return controlledForceConfirm(t, r, root) + } f := t.one("SELECT * FROM runtime_force_stops WHERE id=$1", r.Body.S("forceStopId")) require(f != nil, 404, "not_found") version(f, r.Body.N("version")) @@ -94,6 +98,9 @@ func effectPermit(t *transaction, r *ControlRequest) Object { var wid, sid, tid string var generation, controlEpoch int64 if fid := r.Body.S("forceStopId"); fid != "" { + if root := t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1 AND command_kind='terminate'", fid); root != nil { + return controlledEffectPermit(t, r, root) + } f := t.one("SELECT f.*,w.project_id FROM runtime_force_stops f JOIN workspaces w ON w.id=f.workspace_id WHERE f.id=$1 AND f.state<>'succeeded'", fid) require(f != nil, 409, "force_stop_completed") target := t.one("SELECT * FROM runtime_force_stop_targets WHERE id=$1 AND force_stop_id=$2", r.EffectID, fid) @@ -103,6 +110,7 @@ func effectPermit(t *transaction, r *ControlRequest) Object { e := t.one("SELECT e.*,o.state AS operation_state,o.controller_epoch,w.tenant_id,w.runtime_generation FROM external_effects e JOIN operations o ON o.id=e.operation_id JOIN workspaces w ON w.id=e.workspace_id WHERE e.id=$1", r.EffectID) require(e != nil && e.S("operationState") == "running" && e.N("controllerEpoch") == r.Body.N("epoch"), 409, "stale_effect") op := t.one("SELECT * FROM operations WHERE id=$1", e.S("operationId")) + requireNoControlledReclamation(t, e.S("workspaceId"), op.S("id")) requireNoResidualConflict(t, e.S("workspaceId")) checkResidualEffect(t, op, e.S("kind"), e.S("workspaceId"), e) wid, tid, generation = e.S("workspaceId"), e.S("tenantId"), e.N("runtimeGeneration")