From 8946a480542683cb85bf1b6ebc7c4134a9b27630 Mon Sep 17 00:00:00 2001 From: Wen Date: Sat, 3 Oct 2026 15:36:27 +0800 Subject: [PATCH 1/2] feat(db): persist globally keyed trash cleanup receipts --- .../migrations/0051_global_trash_cleanup.sql | 7 +++++++ packages/db/migrations/meta/_journal.json | 7 +++++++ packages/db/src/schema.ts | 20 +++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 packages/db/migrations/0051_global_trash_cleanup.sql diff --git a/packages/db/migrations/0051_global_trash_cleanup.sql b/packages/db/migrations/0051_global_trash_cleanup.sql new file mode 100644 index 00000000..514dd110 --- /dev/null +++ b/packages/db/migrations/0051_global_trash_cleanup.sql @@ -0,0 +1,7 @@ +-- Global Owner cleanup receipts have no channel lifetime or channel-scoped key. +-- Existing C21 per-channel receipts and deletion/recovery records remain unchanged. +CREATE TABLE storage_cleanup_receipts ( + request_key text PRIMARY KEY CHECK (request_key ~ '^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$'), + response jsonb NOT NULL CHECK (jsonb_typeof(response) = 'object' AND octet_length(response::text) <= 262144), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index 0934d9e7..4dcda53e 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -358,6 +358,13 @@ "when": 1790975448593, "tag": "0050_channel_storage_purge", "breakpoints": true + }, + { + "idx": 51, + "version": "7", + "when": 1791011308235, + "tag": "0051_global_trash_cleanup", + "breakpoints": true } ] } diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 1ffee815..0f0fcc90 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -816,6 +816,26 @@ export const attachmentCleanupReceipts = pgTable( ], ); +/** Global cleanup replays across channel deletion; its request key alone is the identity. */ +export const storageCleanupReceipts = pgTable( + "storage_cleanup_receipts", + { + requestKey: text("request_key").primaryKey(), + response: jsonb("response").notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + check( + "storage_cleanup_receipts_request_key_check", + sql`${table.requestKey} ~ '^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$'`, + ), + check( + "storage_cleanup_receipts_response_check", + sql`jsonb_typeof(${table.response}) = 'object' AND octet_length(${table.response}::text) <= 262144`, + ), + ], +); + export const ownerStorageSettings = pgTable( "owner_storage_settings", { From 010439bb9002fabb7e1facd8450ee85edbafa309 Mon Sep 17 00:00:00 2001 From: Wen Date: Sat, 3 Oct 2026 16:03:49 +0800 Subject: [PATCH 2/2] feat(storage): add global cleanup and bounded reference and profile contracts --- .../openbot_server/attachment_references.py | 52 +++- .../src/openbot_server/browser_protocol.py | 1 + .../product_attachment_routes.py | 14 +- .../src/openbot_server/storage_routes.py | 5 + .../src/openbot_server/storage_service.py | 25 ++ .../src/openbot_server/storage_usage.py | 5 +- .../tests/test_attachment_references.py | 84 +++++- .../tests/test_browser_sessions.py | 29 +- .../server-python/tests/test_storage_purge.py | 76 ++++++ deploy/server/product-preflight.ts | 2 +- deploy/server/smoke-product.py | 4 +- docs/API.md | 86 +++++- docs/API.zh-CN.md | 74 +++++- docs/OPEN_SOURCE_REUSE.md | 18 ++ docs/OPEN_SOURCE_REUSE.zh-CN.md | 16 ++ docs/research/storage-cleanup-follow-ups.md | 95 ++++++- .../storage-cleanup-follow-ups.zh-CN.md | 77 +++++- experiments/s7-migration/README.md | 11 +- experiments/s7-migration/README.zh-CN.md | 11 +- .../evidence/global-trash-result.json | 251 ++++++++++++++++++ experiments/s7-migration/target-history.json | 16 +- packages/protocol/src/browser.test.ts | 19 ++ packages/protocol/src/browser.ts | 9 +- .../docker/src/browser-maintenance.test.ts | 71 ++++- providers/docker/src/browser.ts | 17 +- 25 files changed, 1025 insertions(+), 43 deletions(-) create mode 100644 experiments/s7-migration/evidence/global-trash-result.json diff --git a/apps/server-python/src/openbot_server/attachment_references.py b/apps/server-python/src/openbot_server/attachment_references.py index d4d1e66d..fb6e11a1 100644 --- a/apps/server-python/src/openbot_server/attachment_references.py +++ b/apps/server-python/src/openbot_server/attachment_references.py @@ -1,7 +1,12 @@ -"""Owner-only metadata projection over retained channel Message and task identities.""" +"""Owner-only bounded counts and previews over retained channel Message and task identities.""" +from datetime import datetime + from .control_errors import ControlError +from .models import iso_timestamp REFERENCE_LIMIT = 10000 +LIST_LIMIT = 100 +_MARKER_PATTERN = r'\[openbot attachment: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}\]' # Counts come from one SQL snapshot. Match the same canonical marker as task_store.attachment_ids; # existence counts a record once even if its text repeats the marker. No content leaves PostgreSQL. @@ -37,3 +42,48 @@ async def with_reference_counts(db, channel_id, attachments): if len(counts) != len(attachments): raise ControlError(503, 'attachment_references_unavailable') return [{**item, 'referenceCount': counts[item['id']]} for item in attachments] + + +_LIST_QUERY = """ +WITH message_refs AS MATERIALIZED ( + SELECT id,created_at FROM messages + WHERE channel_id=%s AND position(%s in lower(content COLLATE "C"))>0 LIMIT %s +), task_refs AS MATERIALIZED ( + SELECT id,created_at FROM runs + WHERE channel_id=%s AND position(%s in lower(instruction COLLATE "C"))>0 LIMIT %s +), message_page AS ( + SELECT * FROM message_refs ORDER BY created_at DESC,id COLLATE "C" DESC LIMIT %s +), task_page AS ( + SELECT * FROM task_refs ORDER BY created_at DESC,id COLLATE "C" DESC LIMIT %s +) +SELECT (SELECT count(*) FROM message_refs) AS message_count, + (SELECT count(*) FROM task_refs) AS task_count, + COALESCE((SELECT jsonb_agg(jsonb_build_object('id',m.id,'createdAt',m.created_at, + 'authorType',m.author_type,'authorId',m.author_id, + 'preview',left(btrim(regexp_replace(m.content COLLATE "C",%s,'','gi')),120)) + ORDER BY m.created_at DESC,m.id COLLATE "C" DESC) + FROM message_page p JOIN messages m ON m.id=p.id),'[]'::jsonb) AS messages, + COALESCE((SELECT jsonb_agg(jsonb_build_object('runId',r.id,'title',left(r.title,160), + 'status',r.status,'createdAt',r.created_at) ORDER BY r.created_at DESC,r.id COLLATE "C" DESC) + FROM task_page p JOIN runs r ON r.id=p.id),'[]'::jsonb) AS tasks +""" + + +async def reference_list(db, channel_id, attachment_id, limit): + # Counts and both bounded pages share one statement/snapshot; Work mappings add no identity. + marker = '[openbot attachment: ' + attachment_id.lower() + ']' + row = await (await db.execute(_LIST_QUERY, (channel_id, marker, REFERENCE_LIMIT + 1, + channel_id, marker, REFERENCE_LIMIT + 1, limit, limit, _MARKER_PATTERN))).fetchone() + if row is None: raise ControlError(503, 'attachment_references_unavailable') + if any(type(row[key]) is not int or not 0 <= row[key] <= REFERENCE_LIMIT for key in ('message_count', 'task_count')): + raise ControlError(503, 'attachment_reference_limit') + messages = [] + for item in row['messages']: + kind = {'human': 'owner', 'bot': 'bot', 'system': 'system'}[item['authorType']] + author = dict(kind=kind) + if kind == 'bot' and item['authorId'] is not None: author['botId'] = item['authorId'] + messages.append(dict(id=item['id'], createdAt=iso_timestamp(datetime.fromisoformat(item['createdAt'])), + author=author, preview=item['preview'])) + tasks = [{**item, 'createdAt': iso_timestamp(datetime.fromisoformat(item['createdAt']))} for item in row['tasks']] + return dict(messages=messages, tasks=tasks, messageCount=row['message_count'], taskCount=row['task_count'], + hasMore=row['message_count'] > len(messages) or row['task_count'] > len(tasks)) diff --git a/apps/server-python/src/openbot_server/browser_protocol.py b/apps/server-python/src/openbot_server/browser_protocol.py index bb949d12..48df7909 100644 --- a/apps/server-python/src/openbot_server/browser_protocol.py +++ b/apps/server-python/src/openbot_server/browser_protocol.py @@ -169,6 +169,7 @@ class Maintenance(Strict): class RuntimeState(Strict): running: bool + profileBytes: Annotated[int, BeforeValidator(integer), Field(ge=0, le=9007199254740991)] | None = None class MaintenanceInput(Strict): diff --git a/apps/server-python/src/openbot_server/product_attachment_routes.py b/apps/server-python/src/openbot_server/product_attachment_routes.py index 8b67566a..e62687bc 100644 --- a/apps/server-python/src/openbot_server/product_attachment_routes.py +++ b/apps/server-python/src/openbot_server/product_attachment_routes.py @@ -13,7 +13,7 @@ from fastapi.responses import Response -from .attachment_references import with_reference_counts +from .attachment_references import LIST_LIMIT, reference_list, with_reference_counts from .control_errors import ControlError from .http_input import read_attachment_upload, request_signal from .owner_files import MAX_BYTES @@ -94,6 +94,18 @@ async def metadata(token,path,*_): return {'attachment':product.files.metadata(path['channel_id'],path['attachment_id'])} route(CHANNEL_BASE+'/{attachment_id}','GET',metadata) + async def references(token,path,_body,request): + async with product.files.lock(),product.transactions.transaction(token) as db: + await product.channel(db,path['channel_id']) + item = product.files.metadata(path['channel_id'],path['attachment_id']) + query = list(request.query_params.multi_items()) + limit = request.query_params.get('limit', '20') + if (len(query) > 1 or any(key != 'limit' for key, _ in query) or not limit.isascii() + or not limit.isdecimal() or len(limit) > 3 or not 1 <= int(limit) <= LIST_LIMIT): + raise ControlError(422, 'invalid_attachment_reference_query') + return await reference_list(db,path['channel_id'],item['id'],int(limit)) + route(CHANNEL_BASE+'/{attachment_id}/references','GET',references) + async def content(token,path,*_): async with product.files.lock(),product.transactions.transaction(token) as db: await product.channel(db,path['channel_id']) diff --git a/apps/server-python/src/openbot_server/storage_routes.py b/apps/server-python/src/openbot_server/storage_routes.py index 44091dad..40f88532 100644 --- a/apps/server-python/src/openbot_server/storage_routes.py +++ b/apps/server-python/src/openbot_server/storage_routes.py @@ -8,6 +8,11 @@ async def usage(token, _path, _body, request): return await product.storage.usage(token) route('/api/v1/storage', 'GET', usage) + async def cleanup_all(token, _path, body, request): + if request.query_params: raise ControlError(422, 'invalid_storage_query') + return await product.storage.cleanup_all(token, body) + route('/api/v1/storage/trash/cleanup', 'POST', cleanup_all, limit=4096) + async def settings(token, *_): return await product.storage.settings(token) async def save(token, _path, body, _request): return await product.storage.save_settings(token, body) route('/api/v1/settings/storage', 'GET', settings) diff --git a/apps/server-python/src/openbot_server/storage_service.py b/apps/server-python/src/openbot_server/storage_service.py index 077c2d48..550cfcef 100644 --- a/apps/server-python/src/openbot_server/storage_service.py +++ b/apps/server-python/src/openbot_server/storage_service.py @@ -172,6 +172,31 @@ async def settings(self, token): async with self.transactions.transaction(token) as db: return settings_projection(await (await db.execute("SELECT * FROM owner_storage_settings WHERE owner_id='owner'")).fetchone()) + async def cleanup_all(self, token, command): + async with self.files.lock(): + try: + async with self.transactions.transaction(token) as db: + if (type(command) is not dict or set(command) != {'requestKey'} or type(command['requestKey']) is not str + or not UUID.fullmatch(command['requestKey'])): raise ControlError(422, 'invalid_cleanup_request') + key = command['requestKey'].lower() + prior = await (await db.execute('SELECT response FROM storage_cleanup_receipts WHERE request_key=%s', (key,))).fetchone() + if prior: result = prior['response'] + else: + items, _ = self.catalog() + trash = [item for item in items if item.get('deletedAt')] + active = await (await db.execute('SELECT id FROM channels WHERE id=ANY(%s) AND deleted_at IS NULL ORDER BY id FOR SHARE', + (sorted({item['channelId'] for item in trash}),))).fetchall() + allowed = {row['id'] for row in active} + result = await self._remove(db, [item for item in trash if item['channelId'] in allowed], + actor='owner', reason='empty_trash_all') + result['channelCount'] = len(allowed) + await db.execute('INSERT INTO storage_cleanup_receipts(request_key,response) VALUES(%s,%s)', (key, Jsonb(result))) + await self.recover_locked() + return result + except BaseException: + await self.recover_locked() + raise + async def save_settings(self, token, command): async with self.transactions.transaction(token) as db: if (type(command) is not dict or set(command) != {'expectedRevision', 'trashAutoPurgeDays'} diff --git a/apps/server-python/src/openbot_server/storage_usage.py b/apps/server-python/src/openbot_server/storage_usage.py index 361a03b1..86ce07fa 100644 --- a/apps/server-python/src/openbot_server/storage_usage.py +++ b/apps/server-python/src/openbot_server/storage_usage.py @@ -45,6 +45,8 @@ async def measured_usage(db, files, object_root, artifact_root, channel, owner, attachment_prefix = files.root.relative_to(object_root).parts claimed = set() channel_active = channel_trash = owner_bytes = 0 + referenced_ids = {item['id'] for item in trash if any(item['referenceCount'].values())} + referenced_bytes = 0 channels = defaultdict(lambda: dict(sizeBytes=0, fileCount=0)) for item in [*channel, *owner]: total = 0 @@ -60,6 +62,7 @@ async def measured_usage(db, files, object_root, artifact_root, channel, owner, channels[item['channelId']]['fileCount'] += 1 if item.get('deletedAt'): channel_trash += total else: channel_active += total + if item['id'] in referenced_ids: referenced_bytes += total task_bytes = task_count = 0 for key, size in objects.items(): if key[0] == 'runs': @@ -102,6 +105,6 @@ async def measured_usage(db, files, object_root, artifact_root, channel, owner, other=dict(sizeBytes=other, fileCount=sum(key not in claimed for key in objects)+extra_other_count), database=dict(sizeBytes=observed['bytes']), workingComputerBrowserData=None), trash=dict(fileCount=len(trash), sizeBytes=channel_trash, - referencedFileCount=sum(any(item['referenceCount'].values()) for item in trash)), + referencedFileCount=len(referenced_ids), referencedSizeBytes=referenced_bytes), topChannels=[dict(id=identity, name=names[identity]['name'], deleted=names[identity]['deleted_at'] is not None, **channels[identity]) for identity in top], topChannelsLimit=TOP_CHANNEL_LIMIT) diff --git a/apps/server-python/tests/test_attachment_references.py b/apps/server-python/tests/test_attachment_references.py index d9077179..0a846408 100644 --- a/apps/server-python/tests/test_attachment_references.py +++ b/apps/server-python/tests/test_attachment_references.py @@ -1,4 +1,4 @@ -"""Real disposable SQL/HTTP counts expose metadata, never the referencing content.""" +"""Real disposable SQL/HTTP checks for scoped counts and bounded Owner reference previews.""" from uuid import uuid4 import psycopg @@ -21,6 +21,7 @@ def seed(synthetic_db): value={**synthetic_db,"bot":bot,"peer":peer,"channel":channel} yield value with psycopg.connect(synthetic_db["dsn"]) as db: + db.execute("DELETE FROM run_events WHERE channel_id=%s",(channel,)) db.execute("DELETE FROM work_sources WHERE channel_id=%s",(channel,)) db.execute("DELETE FROM work_tasks WHERE bot_id=ANY(%s)",([bot,peer],)) db.execute("DELETE FROM runs WHERE channel_id=%s",(channel,)) @@ -55,7 +56,8 @@ def setup(seed, tmp_path): db.execute("INSERT INTO work_tasks(id,owner_id,bot_id,request_key,request_digest,objective,token_limit) VALUES (%s,'owner',%s,%s,%s,%s,0)", (task,seed['bot'],task,'a'*64,marker)) db.execute('INSERT INTO work_sources(task_id,legacy_run_id,channel_id,source_message_id) VALUES (%s,%s,%s,%s)',(task,run1,seed['channel'],first)) - app=create_app(PostgresReadStore(seed['dsn']),owner_name='Owner',secure_cookies=False,product=product) + app=create_app(PostgresReadStore(seed['dsn']),owner_name='Owner',secure_cookies=False, + allowed_origins=('http://testserver',),product=product) return product,item,unused,other,TestClient(app) @@ -106,3 +108,81 @@ async def revoke_after_read(db,channel,items): api.cookies.set('openbot_session',seed['token']) response=api.get(f"/api/v1/channels/{seed['channel']}/attachments") assert response.status_code==401 and 'attachments' not in response.json() + + +def test_reference_lists_keep_c19_matching_order_identity_and_short_previews(seed,tmp_path): + product,item,unused,other,api=setup(seed,tmp_path) + route=f"/api/v1/channels/{seed['channel']}/attachments/{item['id']}/references" + bot_message,system_message = str(uuid4()),str(uuid4()) + marker=f"[OpenBot attachment: {item['id']}]" + with psycopg.connect(seed['dsn']) as db: + db.execute("INSERT INTO messages(id,channel_id,author_type,author_id,content) VALUES(%s,%s,'bot',%s,%s),(%s,%s,'system',NULL,%s)", + (bot_message,seed['channel'],seed['bot'],marker+'😀'*130+f"[OPENBOT ATTACHMENT: {unused['id']}]", + system_message,seed['channel'],'Scheduled '+marker.upper())) + db.execute("UPDATE messages SET created_at='2026-10-03T00:00:00Z' WHERE channel_id=%s",(seed['channel'],)) + db.execute("UPDATE runs SET created_at='2026-10-03T00:00:00Z' WHERE channel_id=%s",(seed['channel'],)) + with api: + assert api.get(route).status_code==401 + api.cookies.set('openbot_session',seed['token']) + response=api.get(route) + assert response.status_code==200,response.text + result=response.json() + assert set(result)=={'messages','tasks','messageCount','taskCount','hasMore'} + assert result['messageCount']==4 and result['taskCount']==3 and result['hasMore'] is False + assert [m['id'] for m in result['messages']]==sorted((m['id'] for m in result['messages']),reverse=True) + assert [r['runId'] for r in result['tasks']]==sorted((r['runId'] for r in result['tasks']),reverse=True) + by_id={m['id']:m for m in result['messages']} + assert by_id[bot_message]['author']==dict(kind='bot',botId=seed['bot']) + assert by_id[bot_message]['preview']=='😀'*120 + assert by_id[system_message]['author']==dict(kind='system') + assert by_id[system_message]['preview']=='Scheduled' + for value in result['messages']: + assert set(value)=={'id','createdAt','author','preview'} + assert len(value['preview'])<=120 and '[openbot attachment:' not in value['preview'].lower() + for value in result['tasks']: assert set(value)=={'runId','title','status','createdAt'} + assert {value['status'] for value in result['tasks']}=={'queued','completed'} + limited=api.get(route+'?limit=1').json() + assert limited=={**result,'messages':result['messages'][:1],'tasks':result['tasks'][:1],'hasMore':True} + product.files.set_deleted(seed['channel'],item['id'],True) + assert api.get(route).json()==result + assert api.get(route.replace(seed['channel'],other)).status_code==404 + + +def test_reference_list_query_limits_gone_files_and_unknown_counts_fail_closed(seed,tmp_path,monkeypatch): + from openbot_server import attachment_references + product,item,unused,_,api=setup(seed,tmp_path) + base=f"/api/v1/channels/{seed['channel']}/attachments/" + route=base+item['id']+'/references' + with api: + api.cookies.set('openbot_session',seed['token']) + for query in ('limit=0','limit=101','limit=-1','limit=1.5','limit=','limit=20&limit=20','offset=1','limit=20'): + response=api.get(route+'?'+query) + assert response.status_code==422,response.text + assert 'messages' not in response.json() + assert api.get(route+'?limit=100').status_code==200 + assert api.get(base+str(uuid4())+'/references').status_code==404 + assert api.get(base+unused['id']+'/references').json()==dict(messages=[],tasks=[],messageCount=0,taskCount=0,hasMore=False) + product.files.set_deleted(seed['channel'],unused['id'],True) + assert api.delete(base+unused['id']+'/purge',headers={'Origin':'http://testserver'}).status_code==200 + gone=api.get(base+unused['id']+'/references') + assert gone.status_code==410 and 'messages' not in gone.json() + monkeypatch.setattr(attachment_references,'REFERENCE_LIMIT',1) + refused=api.get(route) + assert refused.status_code==503 and refused.json()=={'error':'attachment_reference_limit'} + + +def test_reference_list_rechecks_owner_before_releasing_content(seed,tmp_path,monkeypatch): + import hashlib + from openbot_server import product_attachment_routes + _,item,_,_,api=setup(seed,tmp_path) + original=product_attachment_routes.reference_list + async def revoke_after_read(db,*args): + result=await original(db,*args) + await db.execute('UPDATE auth_sessions SET revoked_at=now() WHERE token_digest=%s', + (hashlib.sha256(seed['token'].encode()).hexdigest(),)) + return result + monkeypatch.setattr(product_attachment_routes,'reference_list',revoke_after_read) + with api: + api.cookies.set('openbot_session',seed['token']) + response=api.get(f"/api/v1/channels/{seed['channel']}/attachments/{item['id']}/references") + assert response.status_code==401 and 'PRIVATE' not in response.text and 'messages' not in response.json() diff --git a/apps/server-python/tests/test_browser_sessions.py b/apps/server-python/tests/test_browser_sessions.py index 541d05ea..8df1ed69 100644 --- a/apps/server-python/tests/test_browser_sessions.py +++ b/apps/server-python/tests/test_browser_sessions.py @@ -103,7 +103,7 @@ async def enroll_worker(seed, http): @asynccontextmanager -async def worker(seed, http, url, *, capability=True, hook=None, credential=None, page_hook=None, maintenance=False): +async def worker(seed, http, url, *, capability=True, hook=None, credential=None, page_hook=None, maintenance=False, profile_bytes=None): credential = credential or await enroll_worker(seed, http) async with connect(url, proxy=None, compression=None, open_timeout=3, close_timeout=1) as ws: await ws.send(json.dumps(dict(type="node.hello", protocolVersion="0.9.0", nodeId=seed["node"], @@ -126,7 +126,7 @@ async def reply(): if isinstance(override, dict): frame = override extra = await page_hook(command) if page_hook is not None and command['action']['kind']=='agent' else {} await ws.send(json.dumps(dict(type="browser.result", protocolVersion="0.9.0", nodeId=seed["node"], - sessionId=command["sessionId"], requestId=command["requestId"], ok=True, **({"runtime":{"running":command["action"]["operation"]!="clear"}} if command["action"]["kind"]=="maintenance" else {"frame":frame}|extra)))) + sessionId=command["sessionId"], requestId=command["requestId"], ok=True, **({"runtime":{"running":command["action"]["operation"]!="clear", **({'profileBytes':profile_bytes} if profile_bytes is not None else {})}} if command["action"]["kind"]=="maintenance" else {"frame":frame}|extra)))) task = asyncio.create_task(reply()) try: yield calls, ws @@ -513,7 +513,7 @@ async def run(): assert not calls assert (await http.post(path,json={"operation":"status"},headers={"Origin":"https://wrong.invalid"})).status_code==403 status=await http.post(path,json={"operation":"status"});assert status.status_code==200,status.text - assert status.json()==dict(botId=seed['botId'],nodeId=seed['node'],running=True,paused=False) + assert status.json()==dict(botId=seed['botId'],nodeId=seed['node'],running=True,paused=False,profileBytes=None) restarted=await http.post(path,json={"operation":"restart"});assert restarted.status_code==200,restarted.text assert restarted.json()['paused'] is True assert (await command(http,old,'observe')).status_code==404 @@ -539,3 +539,26 @@ async def run(): response=await http.post(f'/api/v1/bots/{seed["botId"]}/browser/maintenance',json={"operation":"restart"}) assert response.status_code==503 and not calls asyncio.run(run()) + + +def test_maintenance_projects_measured_profile_bytes_from_original_worker(seed): + async def run(): + async with server(seed) as (service,registry,http,url): + async with worker(seed,http,url,maintenance=True,profile_bytes=12345) as (calls,ws): + await opened(http,seed) + response=await http.post(f'/api/v1/bots/{seed["botId"]}/browser/maintenance',json={'operation':'status'}) + assert response.status_code==200,response.text + assert response.json()==dict(botId=seed['botId'],nodeId=seed['node'],running=True,paused=False,profileBytes=12345) + assert len(calls)==1 + asyncio.run(run()) + + +def test_profile_size_wire_matches_typescript_bounds_and_legacy_unknown(): + from pydantic import ValidationError + from openbot_server.browser_protocol import RuntimeState + assert RuntimeState.model_validate({'running':True}).model_dump()=={'running':True,'profileBytes':None} + for value in (None,0,123,9007199254740991): + assert RuntimeState.model_validate({'running':True,'profileBytes':value}).profileBytes==value + for value in (True,'123',-1,1.5,9007199254740992,float('nan'),float('inf')): + with pytest.raises(ValidationError): RuntimeState.model_validate({'running':True,'profileBytes':value}) + with pytest.raises(ValidationError): RuntimeState.model_validate({'running':True,'profileBytes':1,'path':'/private'}) diff --git a/apps/server-python/tests/test_storage_purge.py b/apps/server-python/tests/test_storage_purge.py index 3abafd1b..e0fab2cd 100644 --- a/apps/server-python/tests/test_storage_purge.py +++ b/apps/server-python/tests/test_storage_purge.py @@ -188,6 +188,80 @@ def test_empty_trash_idempotent_receipt_and_new_files_require_new_key(seed, tmp_ assert api.get(base + '/' + doomed['id']).status_code == 410 +def test_global_cleanup_covers_active_channel_trash_with_one_replayable_receipt(seed, tmp_path, monkeypatch): + service, api, base = setup(seed, tmp_path, monkeypatch) + doomed = file(service, seed) + second = file(service, seed, 'second.txt', channel=seed['direct']) + blocked = file(service, seed, 'kept.txt', channel=seed['direct']) + reference(seed, blocked, task=True) + active = file(service, seed, 'active.txt', trash=False) + owner = service.files.owner_persist('owner.txt', b'Owner file') + service.files.owner_set_deleted(owner['id'], True) + route = '/api/v1/storage/trash/cleanup' + key = str(uuid4()) + with api: + usage = api.get('/api/v1/storage').json()['trash'] + expected_bytes = sum(path.stat().st_size for path in service.files.root.glob(blocked['id'] + '.*')) + assert usage['referencedSizeBytes'] == expected_bytes > blocked['sizeBytes'] + assert usage['referencedFileCount'] == 1 + first = api.post(route, headers=HEADERS, json={'requestKey': key}) + assert first.status_code == 200, first.text + result = first.json() + assert result['removed'] == 2 and result['channelCount'] == 2 + assert result['retained'] == [{'id': blocked['id'], 'name': blocked['name'], 'referenceCount': {'messages': 1, 'tasks': 1}}] + assert result['retainedCount'] == 1 and not result['retainedHasMore'] and result['freedBytes'] > 0 + later = file(service, seed, 'later.txt', channel=seed['direct']) + assert api.post(route, headers=HEADERS, json={'requestKey': key.upper()}).json() == result + for item in (blocked, active, later): assert_original(service, item) + assert service.files.owner_metadata(owner['id'])['deletedAt'] + with psycopg.connect(seed['dsn']) as db: + rows = db.execute("SELECT channel_id,payload FROM run_events WHERE type='CHANNEL_ATTACHMENT_PURGED' AND channel_id=ANY(%s)", + (seed['channels'],)).fetchall() + assert {(row[0], row[1]['attachmentId'], row[1]['reason']) for row in rows} == { + (seed['channel'], doomed['id'], 'empty_trash_all'), (seed['direct'], second['id'], 'empty_trash_all')} + assert db.execute('SELECT response FROM storage_cleanup_receipts WHERE request_key=%s', (key,)).fetchone()[0] == result + # Receipt identity/lifetime cannot depend on any surviving channel. + db.execute('UPDATE channels SET deleted_at=now() WHERE id=ANY(%s)', (seed['channels'],)) + assert api.post(route, headers=HEADERS, json={'requestKey': key}).json() == result + empty = api.post(route, headers=HEADERS, json={'requestKey': str(uuid4())}) + assert empty.json() == dict(removed=0, retained=[], retainedCount=0, retainedHasMore=False, freedBytes=0, channelCount=0) + assert_original(service, later) + + +def test_global_cleanup_refuses_unknown_references_without_partial_deletion(seed, tmp_path, monkeypatch): + from openbot_server import attachment_references + service, api, _ = setup(seed, tmp_path, monkeypatch) + first = file(service, seed) + unknown = file(service, seed, 'unknown.txt', channel=seed['direct']) + reference(seed, unknown) + monkeypatch.setattr(attachment_references, 'REFERENCE_LIMIT', 0) + key = str(uuid4()) + with api: + response = api.post('/api/v1/storage/trash/cleanup', headers=HEADERS, json={'requestKey': key}) + assert response.status_code == 503 and response.json() == {'error': 'attachment_reference_limit'} + for item in (first, unknown): assert_original(service, item) + assert events(seed) == [] + with psycopg.connect(seed['dsn']) as db: + assert db.execute('SELECT 1 FROM storage_cleanup_receipts WHERE request_key=%s', (key,)).fetchone() is None + + +def test_global_cleanup_owner_origin_and_strict_request_bounds(seed, tmp_path, monkeypatch): + service, api, _ = setup(seed, tmp_path, monkeypatch) + item = file(service, seed) + route = '/api/v1/storage/trash/cleanup' + valid = {'requestKey': str(uuid4())} + with api: + for value in ({}, {'requestKey': 'bad'}, {'requestKey': 1}, {**valid, 'channelId': seed['channel']}, []): + assert api.post(route, headers=HEADERS, json=value).status_code == 422 + assert api.post(route + '?channelId=' + seed['channel'], headers=HEADERS, json=valid).status_code == 422 + assert api.post(route, headers={'Origin': 'https://wrong.invalid'}, json=valid).status_code == 403 + api.cookies.set('openbot_session', 'stale') + assert api.post(route, headers=HEADERS, json=valid).status_code == 401 + api.cookies.clear() + assert api.post(route, headers=HEADERS, json=valid).status_code == 401 + assert_original(service, item) + + def test_clear_retained_list_and_usage_top_channels_have_real_bounds(seed, tmp_path, monkeypatch): service, api, base = setup(seed, tmp_path, monkeypatch) items = [file(service, seed, f'{index}.txt') for index in range(101)] @@ -207,6 +281,8 @@ def test_clear_retained_list_and_usage_top_channels_have_real_bounds(seed, tmp_p assert usage.status_code == 200, usage.text assert len(usage.json()['topChannels']) == 20 and usage.json()['topChannelsLimit'] == 20 assert usage.json()['trash']['referencedFileCount'] == 101 + global_result = api.post('/api/v1/storage/trash/cleanup', headers=HEADERS, json={'requestKey': str(uuid4())}).json() + assert len(global_result['retained']) == 100 and global_result['retainedCount'] == 101 and global_result['retainedHasMore'] assert events(seed) == [] for item in items: assert_original(service, item) diff --git a/deploy/server/product-preflight.ts b/deploy/server/product-preflight.ts index f2e88f2c..940c6b83 100644 --- a/deploy/server/product-preflight.ts +++ b/deploy/server/product-preflight.ts @@ -9,7 +9,7 @@ const RUNTIME_MODULES = [ "@tesseract.js-data/eng", "@tesseract.js-data/chi_sim", ] as const; -const MIGRATION_COUNT = 51; +const MIGRATION_COUNT = 52; if (process.versions.node !== PRODUCT_NODE_VERSION) throw new Error("Product Node version mismatch."); const require = createRequire(new URL("../../package.json", import.meta.url)); diff --git a/deploy/server/smoke-product.py b/deploy/server/smoke-product.py index 860c43b5..dd36393d 100644 --- a/deploy/server/smoke-product.py +++ b/deploy/server/smoke-product.py @@ -74,7 +74,7 @@ def sql(statement):return docker('exec',pg,'psql','-U','openbot','-d','openbot', created.append(server) docker('run','-d','--name',server,*common,'-e','OPENBOT_CONTROL_OWNER_PASSWORD=openbot-product-smoke-synthetic-owner',image) ready(server) - assert sql('select count(*) from drizzle.__drizzle_migrations')=='51' + assert sql('select count(*) from drizzle.__drizzle_migrations')=='52' before=sql('select hash,created_at from drizzle.__drizzle_migrations order by id') client=(HERE/'product-smoke-client.py').read_bytes() first=json.loads(docker('exec','-i',server,PYTHON,'-I','-B','-',f'http://{server}:3001','create',input=client,timeout=150)) @@ -88,7 +88,7 @@ def sql(statement):return docker('exec',pg,'psql','-U','openbot','-d','openbot', second=json.loads(docker('exec','-i',server,PYTHON,'-I','-B','-',f'http://{server}:3001','restart',input=client,timeout=40)) assert sql('select hash,created_at from drizzle.__drizzle_migrations order by id')==before docker('stop','--time','20',server);assert inspect(server)['State']['ExitCode'] in (0,143) - result={'ok':True,'architecture':metadata['Architecture'],'migrations':51,'first':first,'restart':second, + result={'ok':True,'architecture':metadata['Architecture'],'migrations':52,'first':first,'restart':second, 'preflightBeforeSchema':True,'sigtermExitWithoutKill':True,'temporalConfigured':False,'realModelCalled':False} finally: for name in reversed(created): diff --git a/docs/API.md b/docs/API.md index 9e16bdec..63cdb20e 100644 --- a/docs/API.md +++ b/docs/API.md @@ -553,7 +553,8 @@ network. See [Node enrollment](NODE_ENROLLMENT.md). `POST /api/v1/bots/{botId}/browser/maintenance` requires Owner cookie and exact allowed Origin, and a strict JSON body up to 1 KiB: `{operation:"status"|"restart"|"clear",confirmation?:"clear-browser-data"}`. Only `clear` requires the confirmation, and other operations reject it. Unknown fields and any path, -URL or command are rejected. Response: `{botId,nodeId,running:boolean,paused:boolean}`. +URL or command are rejected. Response: `{botId,nodeId,running:boolean,paused:boolean,profileBytes:number|null}`. +See C23 below for measured profile size and compatibility with older Workers/upstreams. The Server requires an already bound original browser identity or an explicit operator Bot-to-Node route; it never selects a replacement Host for maintenance. The original Worker must advertise @@ -865,8 +866,9 @@ backups, WAL or physical disk free space. Content-addressed output counts reflec files, not logical task references. `freedBytes` measures removed attachment file bytes minus the minimal gone marker, excluding PostgreSQL receipts/audit growth and physical disk allocation. -`trash` is `{fileCount, sizeBytes, referencedFileCount}`; the last count is files with any retained -message/task reference. `topChannels` contains at most 20 `{id, name, deleted, sizeBytes, fileCount}`, +`trash` is `{fileCount, sizeBytes, referencedFileCount, referencedSizeBytes}`; the reference fields +count files with any retained message/task reference and their measured original/metadata/derived +bytes. `topChannels` contains at most 20 `{id, name, deleted, sizeBytes, fileCount}`, including active and recycled attachments, sorted by size descending then ID. Traversal refuses symlinks, hardlinks, foreign ownership, more than 10,000 entries, depth above 8, unavailable/corrupt catalogs, or unknown references; no partial/estimated success is returned. No paths or file contents @@ -889,3 +891,81 @@ a started-only run indicates interrupted/unknown completion. Per-file audits com A completed zero-file pass is audited too; disabled/not-yet-due checks make no purge audit. This policy replaces a fixed seven-day-cleanup claim: automatic permanent deletion is disabled until an Owner explicitly selects 30 days. Existing whole-channel deletion remains a separate lifecycle operation. + +### C22: global trash cleanup + +`POST /api/v1/storage/trash/cleanup`, `{requestKey: ""}` → +`{removed, retained, retainedCount, retainedHasMore, freedBytes, channelCount}`. +The current Owner session, allowed Origin, final authority recheck and C21 deletion rules apply. +No query fields are accepted; the body is limited to 4 KiB and accepts only `requestKey`. + +This command considers recycled attachments of every active channel in one transaction. It reuses +C21 staging/recovery, the initial and locked final reference checks, late-reference SQL guards and +per-file audit, with `reason: "empty_trash_all"`. Active files, deleted channels and Owner-native +Task files are excluded. Unknown references refuse the whole request. C21's 1,024-file catalog, +10,000-reference bounds and first-100 retained list apply. Retained items keep the C21 shape and +ID order. `channelCount` counts active channels with candidate trash, including channels whose +files were all retained; it is zero for an empty candidate set. + +The persisted receipt is keyed by **requestKey alone**, independent of any channel's lifetime. +Retrying the same UUID returns the saved response, without new deletion or duplicate audit; later +trash requires a fresh key. `GET /api/v1/storage` adds `trash.referencedSizeBytes`: measured logical +bytes of referenced trash, including originals, metadata and derived text. It uses the same measured +file tree as `trash.sizeBytes`; their difference describes unreferenced trash at measurement +time, not a promise of eventual `freedBytes` after concurrent changes and retained gone markers. + +### C23: browser profile bytes + +`POST /api/v1/bots/{botId}/browser/maintenance` with `{operation: "status"}` now returns +`{botId, nodeId, running, paused, profileBytes: number | null}`. The `browser.maintenance@1` +Worker result carries the same `profileBytes`. A value is a nonnegative safe integer of logical +file bytes for this Bot on its original working computer. Missing legacy fields normalize to +`null`; unsupported routes, rejected or malformed measurements and transport failures yield +`null`, never an estimated zero. Owner/Host binding, capabilities and cancellation are unchanged. +Restart and clear results include `profileBytes: null`; request status afterward to measure. + +The Docker Provider requests authenticated `GET /computers/profile-usage` with the selected +`Bot-Id`. The [upstream contribution](https://github.com/CopilotKit/OpenBot/pull/730) reads only +that Bot's profile, without starting Chromium or exposing paths/content. Its Linux traversal skips +symlinks, rejects linked roots, hardlinks/special files and foreign devices, and bounds entries +(10,000), depth (16), elapsed work (2 seconds checked around filesystem operations) and concurrent +measurements (one). Refusal or an unavailable root returns `null`; a confirmed absent Bot directory +returns zero. This is a live logical-byte observation, not an atomic snapshot or disk allocation. + +The deployed upstream pin remains `257c1280d684089be9adb0b35cce262efc7064bf` while the contribution +is pending, so that version reports `null`. A numeric production result requires deploying the +reviewed route after upstream acceptance (or a separately recorded narrow fork after rejection). +Remote profile bytes are not included in the Server's `/storage.totalBytes`; +`categories.workingComputerBrowserData` stays `null`. + +### C24: attachment reference lists + +`GET /api/v1/channels/:channelId/attachments/:id/references?limit=20` requires a current Owner +session and an active channel. Response: + +```json +{ + "messages": [{"id": "", "createdAt": "", "author": {"kind": "bot", "botId": ""}, "preview": "Short text"}], + "tasks": [{"runId": "", "title": "Task title", "status": "completed", "createdAt": ""}], + "messageCount": 1, + "taskCount": 1, + "hasMore": false +} +``` + +`author.kind` is `owner`, `bot` or `system`; only a Bot author can include `botId`, when retained. +System messages keep their actual identity. Previews remove canonical attachment markers, trim +surrounding spaces and contain at most 120 Unicode code points; task titles contain at most 160. +No full message content or task instruction is returned. + +`limit` defaults to 20, is an ASCII decimal integer 1–100, and caps **each list separately**. +Empty, malformed, repeated or unknown query fields return 422 `invalid_attachment_reference_query`. +Each list is newest first by stored timestamp, then ID descending with C collation. Matching, +record identity, channel scope and the 10,000-per-type count bound are exactly C19's. The counts +and both pages share one SQL snapshot; `hasMore` is true if either count exceeds its returned list. +Overflow returns 503 without partial content. There is no cursor or offset in this route. + +Recycled files remain readable; purged files return 410, missing/wrong-channel files and deleted +channels return 404. Missing or revoked Owner authority returns 401, including at the final session +recheck. The private file lock and bounded Owner transaction remain in force. No model or mutation +is invoked. diff --git a/docs/API.zh-CN.md b/docs/API.zh-CN.md index a0dbfc16..3d81db1f 100644 --- a/docs/API.zh-CN.md +++ b/docs/API.zh-CN.md @@ -506,7 +506,7 @@ Artifact 与临时画面内容接口使用同一个 Owner Session,响应为 `p `POST /api/v1/bots/{botId}/browser/maintenance` 要求 Owner cookie、严格允许的 Origin 和最多 1 KiB 严格 JSON:`{operation:"status"|"restart"|"clear",confirmation?:"clear-browser-data"}`。 只有 `clear` 必须附确认字段,其余操作拒绝该字段;拒绝未知字段、路径、URL 或命令。 -返回 `{botId,nodeId,running:boolean,paused:boolean}`。 +返回 `{botId,nodeId,running:boolean,paused:boolean,profileBytes:number|null}`(详见下文 C23)。 Server 要求已绑定原浏览器身份或运维明确配置的 Bot→Node 路由,禁止选择替代主机。 原 Worker 必须声明 Docker Provider 的 `browser.maintenance@1` 与 `browser.session@1`。 @@ -767,7 +767,7 @@ Owner 原生 Task 的文件命名空间独立,不加入频道计数。引用 WAL 或物理磁盘剩余空间。内容寻址产出统计实际存储的去重文件,不按逻辑任务引用重复计算。 `freedBytes` 为删除的附件文件字节数减去最小删除标记大小,不含 SQL 凭证与审计增长,也不是物理磁盘分配量。 -`trash` 为 `{fileCount, sizeBytes, referencedFileCount}`,最后一项是仍有消息或任务引用的文件个数。 +`trash` 为 `{fileCount, sizeBytes, referencedFileCount, referencedSizeBytes}`;后两项分别表示仍有消息或任务引用的文件个数,以及它们的实测原件、元数据和派生文本字节数。 `topChannels` 最多 20 个 `{id, name, deleted, sizeBytes, fileCount}`,包含正常及回收站附件, 按大小降序、ID 升序。遍历拒绝符号链接、硬链接、非当前用户所有、超过 10,000 项或深度超过 8 的目录, 目录损坏、不可用或引用不明也拒绝,不返回部分或估算成功。不会泄露路径或文件内容。 @@ -787,3 +787,73 @@ Server 维护任务在启动时、设置变更时及每小时检查;启用后 逐文件审计与删除一同提交,零删除的成功轮次也审计;关闭或尚未到期的检查不产生清理审计。 该策略取代“固定七天清理”的承诺:只有 Owner 明确选择 30 天才启用自动永久删除。 删除整个频道仍是独立的生命周期操作。 + +### C22:全局清空回收站 + +`POST /api/v1/storage/trash/cleanup`,`{requestKey: ""}` → +`{removed, retained, retainedCount, retainedHasMore, freedBytes, channelCount}`。 +沿用有效 Owner 会话、允许的 Origin、返回前权限复核及 C21 删除规则。不接受查询参数, +请求体最多 4 KiB,只接受 `requestKey`。 + +一次事务处理所有有效频道的回收站附件,复用 C21 暂存与恢复、初始及加锁后的最终引用检查、 +防止迟到引用的 SQL 守卫和逐文件审计,原因为 `empty_trash_all`。正常文件、已删除频道和 +Owner 原生任务文件不在范围内。引用不明时整次拒绝。沿用目录最多 1,024 个附件、每类引用 +最多 10,000 条、保留列表最多 100 项的限制;保留项格式和 ID 排序同 C21。 +`channelCount` 是有候选回收站文件的有效频道数,包括所有文件都被保留的频道;无候选时为零。 + +持久凭证**只按 requestKey 记**,不依赖频道生命周期。同一 UUID 重试原样返回已保存结果, +不删除后来进入回收站的文件,也不重复审计;新的清空操作须使用新 UUID。 +`GET /api/v1/storage` 的 `trash.referencedSizeBytes` 统计被引用回收站文件的实测逻辑字节, +包含原件、元数据及派生文本,与 `trash.sizeBytes` 共用实测文件树。两者之差表示测量时的 +未引用回收站字节,不保证并发变化及最小删除标记保留后的最终 `freedBytes`。 + +### C23:浏览器资料字节数 + +`POST /api/v1/bots/{botId}/browser/maintenance`,`{operation: "status"}`,现在返回 +`{botId, nodeId, running, paused, profileBytes: number | null}`。 +`browser.maintenance@1` 的 Worker 结果携带同一字段。数字是原工作电脑上该 Bot 的资料文件 +逻辑字节数,必须是非负安全整数。旧 Worker 缺字段时归一为 `null`;接口不支持、测量拒绝、 +格式异常或传输失败也返回 `null`,不能估算为零。Owner/Host 绑定、能力要求及取消行为保持不变。 +重启、清除操作的结果包含 `profileBytes: null`,需随后查询状态才能测量。 + +Docker Provider 携带选定的 `Bot-Id` 调用已认证的 `GET /computers/profile-usage`。 +[上游贡献](https://github.com/CopilotKit/OpenBot/pull/730)只读取这个 Bot 的资料,不启动 Chromium, +不返回路径或内容。Linux 遍历跳过符号链接,拒绝链接根目录、硬链接、特殊文件及跨设备目录; +限制为 10,000 项、16 层、2 秒工作期限(在文件系统操作前后检查),同一进程同时最多一次测量。 +拒绝测量或根目录不可用时返回 `null`;确认该 Bot 目录不存在时返回零。 +这是实时逻辑字节观察,不是原子快照或磁盘分配量。 + +上游贡献待处理期间,生产固定版本仍为 `257c1280d684089be9adb0b35cce262efc7064bf`,因此该版本 +返回 `null`。生产环境要得到数字,须在上游接受后部署已审阅接口;只有上游拒绝才另行记录窄分支。 +远端资料大小不加入服务电脑的 `/storage.totalBytes`, +`categories.workingComputerBrowserData` 仍为 `null`。 + +### C24:附件引用列表 + +`GET /api/v1/channels/:channelId/attachments/:id/references?limit=20` 要求有效 Owner 会话和有效频道。 +响应为: + +```json +{ + "messages": [{"id": "<消息 ID>", "createdAt": "", "author": {"kind": "bot", "botId": ""}, "preview": "简短文本"}], + "tasks": [{"runId": "", "title": "任务标题", "status": "completed", "createdAt": ""}], + "messageCount": 1, + "taskCount": 1, + "hasMore": false +} +``` + +`author.kind` 为 `owner`、`bot` 或 `system`;只有 Bot 作者可在保留了身份时带 `botId`。 +系统消息保留其真实类型。预览去除规范附件标记,裁掉首尾空格,最多 120 个 Unicode 码点; +任务标题最多 160 个码点。不返回完整消息正文或任务指令。 + +`limit` 默认 20,必须是 1–100 的 ASCII 十进制整数,**分别限制两个列表**。 +空值、格式错误、重复或未知查询参数返回 422 `invalid_attachment_reference_query`。 +各列表按数据库时间降序,再按 ID(C 排序规则)降序。匹配规则、记录身份、频道范围与 +每类最多 10,000 的计数界限完全同 C19。两个列表和计数来自同一次 SQL 快照; +任一计数大于返回列表长度时 `hasMore` 为 true。超限返回 503,不返回部分内容。 +该接口不提供游标或 offset。 + +回收站文件仍可查询;已永久删除文件返回 410,不存在、频道不符或频道已删除返回 404。 +缺失或撤销 Owner 权限返回 401,返回前仍复核会话。沿用私有文件锁与有界 Owner 事务。 +不调用模型,不产生写操作。 diff --git a/docs/OPEN_SOURCE_REUSE.md b/docs/OPEN_SOURCE_REUSE.md index a83be8b1..a90365c1 100644 --- a/docs/OPEN_SOURCE_REUSE.md +++ b/docs/OPEN_SOURCE_REUSE.md @@ -1051,3 +1051,21 @@ extends the existing Zod4.6.2 / Pydantic2.13.5 closed appearance enum by four de The existing v1/v2 portable package carries them without another registry or format version. Old values, strict rejection, checksums and Owner import review remain intact. No new dependency, schema migration or source incorporation is added; geometry and UI selection remain separate. + +## C22–C24 storage and browser measurements (2026-10-03) + +The [follow-up decision](research/storage-cleanup-follow-ups.md) reuses C21's private file lock, +`_remove`, staging/recovery, PostgreSQL reference guards and per-file audit for global cleanup; +C19's canonical marker and retained Run identity for bounded Owner-only reference lists. Additive +migration 0051 supplies global request-key receipts without changing channel-scoped C21 receipts. +No external source was copied or substantially adapted for these extensions. + +C23 is an original MIT upstream contribution, [CopilotKit/OpenBot #730](https://github.com/CopilotKit/OpenBot/pull/730), +reviewed base `cb5dc32a44517622c6db4e527e61d3abb389b43c`, contribution head +`46eb7af817027c5de4202846c73c43bbb2fa67b7`. It adds only a bounded, no-follow, authenticated per-Bot +profile-byte route and tests/docs. Linux directory descriptors anchor traversal; unknown/refused +measurements are null. Provider and `browser.maintenance@1` carry safe-integer/null results. +The deployed pin remains `257c1280d684089be9adb0b35cce262efc7064bf`, so existing images report null. +The personal fork used to submit the PR is not a runtime fork adoption. Upstream review is pending: +review/pin an accepted version before rollout; only rejection triggers a separately recorded narrow +fork pin and qualification. Existing upstream attribution and MIT notices are retained. diff --git a/docs/OPEN_SOURCE_REUSE.zh-CN.md b/docs/OPEN_SOURCE_REUSE.zh-CN.md index adbe9096..02bf4470 100644 --- a/docs/OPEN_SOURCE_REUSE.zh-CN.md +++ b/docs/OPEN_SOURCE_REUSE.zh-CN.md @@ -580,3 +580,19 @@ pip-audit 2.10.1(Apache-2.0)通过隔离、精确锁定的工具闭包审计 Zod4.6.2 / Pydantic2.13.5 外观封闭枚举中增加四个设计指定颜色。已有 v1/v2 模板直接携带 新颜色,不增加另一套注册表或格式版本。旧取值、严格拒绝、摘要和 Owner 导入审核保持 有效;没有新增依赖、数据库迁移或第三方源码,头像绘制与界面选择属于独立范围。 + +## C22–C24 存储与浏览器测量(2026-10-03) + +[后续决定](research/storage-cleanup-follow-ups.zh-CN.md)复用 C21 私有文件锁、`_remove`、 +暂存与恢复、PostgreSQL 引用守卫及逐文件审计实现全局清理;复用 C19 规范标记及保留 Run 身份 +实现有界 Owner 引用列表。新增迁移 0051 提供全局请求标识凭证,不改 C21 频道凭证。 +这些扩展没有复制或实质改编外部源码。 + +C23 是原创 MIT 上游贡献 [CopilotKit/OpenBot #730](https://github.com/CopilotKit/OpenBot/pull/730), +审阅基线 `cb5dc32a44517622c6db4e527e61d3abb389b43c`,贡献提交 +`46eb7af817027c5de4202846c73c43bbb2fa67b7`。只新增有上限、不跟随链接、已认证的单 Bot 资料大小 +接口及测试、文档;Linux 使用目录描述符锚定遍历,未知或拒绝测量返回 null。 +Provider 与 `browser.maintenance@1` 携带安全整数或 null。 +生产固定版本仍为 `257c1280d684089be9adb0b35cce262efc7064bf`,现有镜像返回 null。 +用于贡献 PR 的个人 fork 不代表采用运行时分支。上游仍待审阅:接受后须审阅、固定并验证新版本再部署; +只有拒绝才另外记录窄分支固定版本和验证。保留原有上游归属及 MIT 声明。 diff --git a/docs/research/storage-cleanup-follow-ups.md b/docs/research/storage-cleanup-follow-ups.md index 1608ee00..c3ea29ff 100644 --- a/docs/research/storage-cleanup-follow-ups.md +++ b/docs/research/storage-cleanup-follow-ups.md @@ -2,7 +2,7 @@ English · [简体中文](storage-cleanup-follow-ups.zh-CN.md) -- Status: Proposed +- Status: Contracts implemented; C23 upstream contribution pending - Date: 2026-10-03 - Owner: @yxflc11 - Related issue: follow-up to C21 (#164) and its UI (#169) @@ -80,8 +80,8 @@ English · [简体中文](storage-cleanup-follow-ups.zh-CN.md) - The receipt is keyed by `requestKey` alone: a retry with the same key replays the result and touches nothing new. The same limits as C21 apply, and an unknown reference refuses the whole request. - - `/storage` `trash` adds `referencedSizeBytes`, so the confirmation can state the exact size to - be removed. + - `/storage` `trash` adds `referencedSizeBytes`, so the confirmation can distinguish measured referenced + and unreferenced trash bytes; concurrent changes and gone markers affect eventual freed bytes. - **C23 — browser profile size.** - Upstream `agent-computer` reports one Bot's profile bytes, measured with a bounded no-follow traversal. There is no path in the reply, and the size is `null` when the measurement is @@ -90,7 +90,7 @@ English · [简体中文](storage-cleanup-follow-ups.zh-CN.md) - Record the upstream PR, or the narrow fork pin, in `OPEN_SOURCE_REUSE.md`. - **C24 — attachment references.** - `GET /api/v1/channels/:channelId/attachments/:id/references?limit=20`, Owner-only, returning: - - `messages: [{id, createdAt, author: {kind: "owner" | "bot", botId?}, preview}]`, where + - `messages: [{id, createdAt, author: {kind: "owner" | "bot" | "system", botId?}, preview}]`, where `preview` is at most 120 characters with attachment markers removed; - `tasks: [{runId, title, status, createdAt}]`; - `messageCount`, `taskCount` and `hasMore`. @@ -135,3 +135,90 @@ English · [简体中文](storage-cleanup-follow-ups.zh-CN.md) - Whether `agent-computer` upstream accepts a usage route, which decides between an upstream pin and a narrow fork for C23. + +## Implementation checkpoint (2026-10-03) + +C22 and C24 backend contracts are implemented on `codex/c22-c24-storage-follow-ups` from +`dafc2e13`. Migration `0051_global_trash_cleanup` adds a separate global `storage_cleanup_receipts` +table keyed only by UUID, with a bounded JSON response and no channel foreign key; existing C21 +channel receipts keep their namespace/lifetime. `channelCount` counts active channels with candidate +trash, including retained-only channels. Referenced trash bytes include metadata and derived text. +The existing `_remove` owns all staging, reference rechecks, SQL guards, receipts and per-file audit. +No cleanup authority is inferred from the measured size. + +C24 uses one SQL statement for bounded counts and both newest-first lists. The optional ASCII +`limit` is 1–100 (default 20) per list; timestamp ties use descending C-collated ID. Task titles are +bounded to 160 code points. The Owner approved `author.kind: "system"` on 2026-10-03 because C19 +already includes automatic system messages; labelling them as Owner would misrepresent authorship. +This is an accepted addition to the proposed author union above. + +C23 is submitted as [CopilotKit/OpenBot #730](https://github.com/CopilotKit/OpenBot/pull/730), +against reviewed upstream `cb5dc32a44517622c6db4e527e61d3abb389b43c`, contribution commit +`46eb7af817027c5de4202846c73c43bbb2fa67b7`. The MIT contribution +adds authenticated `GET /computers/profile-usage` before session creation, selecting only the +validated `Bot-Id`. Linux descriptor-relative traversal anchors each directory via `/proc/self/fd`, +opens directories with `O_NOFOLLOW`, skips symlinks, refuses hardlinks/special files/device changes, +and detects directory replacement during observation. Bounds are 10,000 entries, depth 16, a +2-second cooperative deadline around filesystem operations, and one concurrent measurement. +Confirmed absent Bot directories return 0; errors, refusal, unsupported platforms or absent roots +return null. Regular files may change during a live observation; no atomic snapshot is promised. + +Primary boundary evidence: [Node filesystem APIs](https://nodejs.org/api/fs.html) and +[Linux open(2)](https://man7.org/linux/man-pages/man2/open.2.html). `O_NOFOLLOW` only protects the +last path component, hence descriptor-anchored traversal instead of recursively joining untrusted +paths. The finite traversal limits and Provider request deadline bound ordinary work; stalled +kernel filesystem operations are not claimed to be preemptible by the cooperative deadline. + +Production remains pinned to `257c1280d684089be9adb0b35cce262efc7064bf`. Provider/status protocol +wiring is ready, with safe-integer validation and legacy/missing/error measurements normalized to +null. Upstream review is outstanding; an open PR is neither acceptance nor rejection, so no runtime +fork is adopted. After acceptance, review and pin the upstream version and qualify its image; only +explicit rejection permits the separately recorded narrow-fork path. UI work remains the separate +Claude slice above, and the DESIGN deviations remain accurate until that slice lands. + +Focused acceptance evidence: 52 real PostgreSQL/HTTP Python tests across storage purge, attachment +references and browser sessions; 3 protocol tests and 35 Docker browser/maintenance tests. Upstream +Linux tests passed 46 cases (one non-Linux test skipped), including the real authenticated HTTP route +without Chromium startup, using the existing `openbot-browser:257c1280` image with Bun 1.4.2. +macOS Bun 1.3.14 passed the unsupported-platform case with four Linux-only cases skipped. +No user database, paid model or production mutation was used. Integration and migration +qualification results, including limitations, follow below. + +Migration qualification: the committed 52-entry SQL source `8946a480542683cb85bf1b6ebc7c4134a9b27630` +passed all 40 retained upgrade/restore cases and 8 cleanup cases. The +[checked-in result](../../experiments/s7-migration/evidence/global-trash-result.json) records exact +SQL/journal hashes; sealed histories and test logic are unchanged. + +The actual Linux arm64 product image +`sha256:059fef02e0a3ea0a9b599953c6df20d1612ea34f74358e91a86181914c8e04b8` +passed `deploy/server/smoke-product.py`: all 52 migrations, real Owner HTTP, built Web, DOCX/PDF +and blank OCR initialization, invalid startup before schema creation, SIGTERM and restart with +original files/key retained. All owned containers/network/volume were removed. This was local +Docker VM evidence, with no configured Temporal engine or real model call, not hosted qualification. + +Integration outcome: `npm run check` ran all preliminary gates through lint/typecheck, then failed +on an unchanged publisher CLI 30-second timeout. Its three tests passed alone. A serial Turbo test +retry passed Web (644), Docker Provider (67) and the other completed packages, but Desktop had +four timeouts in three unchanged files (530 passed, three skipped). Those three files passed all +15 tests with `--maxWorkers=1`. The interrupted Node suite was run separately with one Worker: +129 Vitest tests passed, three skipped, plus all 54 bounded-transport Node tests passed. Final +`npm run build` succeeded for all 18 tasks (five executed, 13 cached). Existing passing preliminary +gates were reused; the original aggregate check is recorded as failed, not relabelled green. + +`npm run test:control:python` exceeded its existing 300-second suite deadline twice. The first run +also hit four SDK failures under concurrent load; all four passed on the second run. The second +completed 789 tests without a failed test before timing out in employee portability (1,039 collected, +two skipped). The interrupted module and all following modules were checked separately in owned +PostgreSQL: 301 tests completed, initially 288 passed and 13 failed because the temporary diagnostic +fixture omitted the original base Bot/Channel. Supplying that original fixture shape made all +13 reruns pass. These are supplemental results, not success of the default full command or its +post-pytest TS read-back assertions. Dedicated Worker/Temporal and hosted CI qualification remain +unrun. Documentation checks passed (12 tests and 576 Markdown files); research-check unit tests +passed (27), while the PR-event check correctly skipped outside a pull-request event. + +Handoff: local backend/contracts, translations and tests are ready for review on the branch above; +the migration source commit is fixed in the qualification manifest. No product PR or deployment +was created in this C22–C24 slice; the explicitly requested upstream PR is open without reviews. +The unrelated pre-existing `output/` directory is untouched. No test or implementation writer is +left running. Remaining external dependency is acceptance/rejection of upstream #730; UI remains +the separate planned slice. Default full-command timeouts remain visible acceptance limitations. diff --git a/docs/research/storage-cleanup-follow-ups.zh-CN.md b/docs/research/storage-cleanup-follow-ups.zh-CN.md index 6498fba5..d1004c6c 100644 --- a/docs/research/storage-cleanup-follow-ups.zh-CN.md +++ b/docs/research/storage-cleanup-follow-ups.zh-CN.md @@ -2,7 +2,7 @@ [English](storage-cleanup-follow-ups.md) · 简体中文 -- 状态:提议 +- 状态:接口已实现;C23 上游贡献待处理 - 日期:2026-10-03 - 负责人:@yxflc11 - 相关:C21(#164)及其界面(#169)的后续 @@ -59,7 +59,8 @@ `empty_trash_all`。 - 幂等凭证只以 `requestKey` 为键:用同一个标识重试,会返回同样的结果,不会多删。上限沿用 C21;只要有引用情况不明,就整次拒绝。 - - `/storage` 的 `trash` 增加 `referencedSizeBytes`,让确认框能写出准确的删除大小。 + - `/storage` 的 `trash` 增加 `referencedSizeBytes`,让确认框区分实测的被引用与未引用字节; + 并发变化和最小删除标记会影响最终释放量。 - **C23——浏览器资料大小。** - 上游 `agent-computer` 报告单个 Bot 的资料大小:遍历有上限、不跟随链接,返回内容里不出现 路径;量不出来时返回 `null`。 @@ -67,7 +68,7 @@ - 在 `OPEN_SOURCE_REUSE.md` 记录上游 PR 或窄分支的固定版本。 - **C24——附件引用列表。** - `GET /api/v1/channels/:channelId/attachments/:id/references?limit=20`,只有 Owner 能读,返回: - - `messages: [{id, createdAt, author: {kind: "owner" | "bot", botId?}, preview}]`,其中 + - `messages: [{id, createdAt, author: {kind: "owner" | "bot" | "system", botId?}, preview}]`,其中 `preview` 最多 120 个字符,并去掉附件标记; - `tasks: [{runId, title, status, createdAt}]`; - `messageCount`、`taskCount`、`hasMore`。 @@ -108,3 +109,73 @@ ## 未决问题 - 上游 `agent-computer` 是否接受新增大小接口,这决定 C23 用上游版本还是窄分支。 + +## 实现检查点(2026-10-03) + +C22、C24 后端接口基于 `dafc2e13` 实现在 `codex/c22-c24-storage-follow-ups`。 +迁移 `0051_global_trash_cleanup` 新增独立的 `storage_cleanup_receipts` 表,只按 UUID 为键, +限制 JSON 响应大小,不设频道外键;C21 原凭证的命名空间和生命周期保持不变。 +`channelCount` 统计有候选回收站文件的有效频道,包括全部保留的频道。 +引用文件大小包含元数据及派生文本;所有暂存、引用复核、SQL 守卫、删除凭证和逐文件审计复用 `_remove`。 +测得大小不授予清理权限。 + +C24 在同一 SQL 语句内获取有界计数和两个新到旧的列表。ASCII `limit` 默认 20、范围 1–100, +分别限制各列表;同时间按 ID 的 C 排序规则降序。任务标题最多 160 个码点。 +Owner 已于 2026-10-03 同意增加 `author.kind: "system"`:C19 已经统计自动系统消息,不能冒充 Owner。 +这是上文作者类型的已批准补充。 + +C23 已提交 [CopilotKit/OpenBot #730](https://github.com/CopilotKit/OpenBot/pull/730), +审阅上游版本 `cb5dc32a44517622c6db4e527e61d3abb389b43c`,贡献提交 +`46eb7af817027c5de4202846c73c43bbb2fa67b7`。MIT 贡献在创建会话前提供已认证的 `GET /computers/profile-usage`,只选择校验后的 `Bot-Id`。 +Linux 遍历使用 `/proc/self/fd` 锚定目录及 `O_NOFOLLOW`,跳过符号链接,拒绝硬链接、特殊文件、 +跨设备目录,并检查观察时目录被替换的情况。最多 10,000 项、16 层、2 秒协作式期限 +(在文件系统操作前后检查)、同时一次测量。确认 Bot 目录不存在返回零;错误、拒绝、不支持的平台 +或根目录缺失返回 null。普通文件在观察期间可能变化,不承诺原子快照。 + +依据:[Node 文件系统接口](https://nodejs.org/api/fs.html)及 +[Linux open(2)](https://man7.org/linux/man-pages/man2/open.2.html)。`O_NOFOLLOW` 只保护最后一个 +路径分量,因此使用目录描述符锚定,不能直接递归拼接不可信路径。有限遍历和 Provider 请求期限 +限制通常的工作量;协作式期限不保证抢占阻塞中的内核文件系统操作。 + +生产仍固定 `257c1280d684089be9adb0b35cce262efc7064bf`。Provider 和状态协议已接好,校验安全整数, +旧版本、字段缺失或错误统一为 null。上游仍待审阅;PR 开放不代表接受或拒绝,因此没有采用运行时分支。 +接受后须审阅并固定上游版本、验证镜像;明确拒绝后才按约定记录窄分支。界面仍由上述 Claude 切片完成, +在该切片落地前,DESIGN 中的差异仍有效。 + +已执行证据:真实 PostgreSQL/HTTP 的存储清理、附件引用和浏览器会话共 52 项 Python 测试; +3 项协议测试及 35 项 Docker 浏览器/维护测试通过。上游 Linux 测试 46 项通过、1 项非 Linux 测试跳过, +包括不启动 Chromium 的真实认证 HTTP 接口,使用现有 `openbot-browser:257c1280` 镜像及 Bun 1.4.2。 +macOS Bun 1.3.14 的不支持平台用例通过,4 项 Linux 专用用例跳过。 +没有访问用户数据库、调用付费模型或修改生产数据。完整集成、迁移验证及限制记录在下方。 + +迁移验证:已提交的 52 条 SQL 来源 `8946a480542683cb85bf1b6ebc7c4134a9b27630` 通过全部 +40 项历史升级/恢复及 8 项清理检查。[入库结果](../../experiments/s7-migration/evidence/global-trash-result.json) +保存精确 SQL/journal 哈希,封存历史和测试逻辑未修改。 + +实际 Linux arm64 产品镜像 +`sha256:059fef02e0a3ea0a9b599953c6df20d1612ea34f74358e91a86181914c8e04b8` +通过 `deploy/server/smoke-product.py`:52 条迁移、真实 Owner HTTP、构建后的 Web、DOCX/PDF 与 +空白 OCR 初始化、错误启动在创建 schema 前拒绝、SIGTERM 和重启后原文件/密钥保留。 +所有专属容器、网络及卷已清理。这是本地 Docker VM 证据,不是托管平台资格验证;未配置 Temporal, +未调用真实模型。 + +集成结果:`npm run check` 完成此前各门槛及 lint/typecheck,随后在未修改的发布密钥 CLI +30 秒超时处失败;该文件单独运行 3 项全过。串行 Turbo 补跑通过 Web 644 项、Docker Provider +67 项及其他已完成包;Desktop 的三份未修改文件出现 4 项超时(530 项通过、3 项跳过)。 +这三份文件以 `--maxWorkers=1` 单独重跑,15 项全过。被中断的 Node 测试另以单 Worker 执行: +Vitest 129 项通过、3 项跳过,另有 54 项有界传输 Node 测试全过。最终 `npm run build` +18 个任务成功(5 个实际执行、13 个缓存)。复用已通过的前置门槛,原整套命令仍记录为失败, +不改称一次全绿。 + +`npm run test:control:python` 两次超过既定 300 秒套件期限。第一次并发负载下还出现 4 项 SDK +失败,第二次这 4 项均通过;第二次完成 789 项且无测试失败,在员工导入导出模块超时 +(收集 1,039 项、另跳过 2 项)。被中断模块及之后所有模块另在专属 PostgreSQL 补测 301 项: +初次 288 项通过,13 项因临时诊断夹具漏了原有基础 Bot/Channel 而失败;补齐原夹具格式后, +这 13 项重跑全过。这些是补充证据,不代表默认完整命令及 pytest 之后的 TS 回读断言通过。 +独立 Worker/Temporal 和托管 CI 未执行。文档检查通过(12 项测试、576 份 Markdown); +研究检查的 27 项单元测试通过,PR 事件检查因不在 pull_request 事件内而跳过。 + +交付状态:上述本地分支的后端、协议、中英文文档及测试可供审阅,迁移提交固定在验证清单中。 +本轮 C22–C24 未创建产品 PR 或部署;明确要求的上游 PR 已开放,尚无评审。 +原有无关 `output/` 目录未改动,没有仍在运行的测试或实现写入者。 +剩余外部依赖为上游 #730 接受/拒绝,界面保持为另一个计划切片;默认整套命令超时仍是明确的验收限制。 diff --git a/experiments/s7-migration/README.md b/experiments/s7-migration/README.md index d6101ac0..5ce45bd4 100644 --- a/experiments/s7-migration/README.md +++ b/experiments/s7-migration/README.md @@ -9,13 +9,14 @@ preparation for S7, not a production migration utility or evidence that S7 is co | --- | --- | --- | | Architecture, 27 migrations | `c33e03f1a14de739196113769c59fdaace9029e7` | Restore old data, then apply current migrations with the existing production startup guard. | | Feature, 19 migrations | `9cc73c9e78451e572f57d142d6b9caf62ccb78e2` | Direct upgrade fails at index 17. A separate fixture-only transfer copies a bounded compatible record set into a freshly migrated target. | -| Qualified target, 51 migrations | Committed SQL source `38c10b39ffa91ab4746da6d35ddf77505106ac33`, with exact current SQL/journal hashes | SQL bytes and journal entries must match `target-history.json`; changes require an explicit requalification. | +| Qualified target, 52 migrations | Committed SQL source `8946a480542683cb85bf1b6ebc7c4134a9b27630`, with exact current SQL/journal hashes | SQL bytes and journal entries must match `target-history.json`; changes require an explicit requalification. | -The 51-entry C21 target passed all 40 retained migration/restore cases on 2026-10-03; the eight +The 52-entry C22 target passed all 40 retained migration/restore cases on 2026-10-03; the eight cleanup cases passed too. Exact current hashes and actual results are in -[the current evidence](evidence/channel-storage-result.json). Migration 0050 adds purge/retry -receipts, default-off storage policy and late-reference guards; C21 product tests separately cover -file staging/recovery and policy behavior. The sealed historical SQL/fixtures/assertions are unchanged. +[the current evidence](evidence/global-trash-result.json). Migration 0051 adds globally keyed +cleanup receipts; C22 product tests separately cover replay, scope, references and measured bytes. +The sealed historical SQL/fixtures/assertions are unchanged. The +[51-entry C21 evidence](evidence/channel-storage-result.json) remains historical. Earlier [46-entry identity evidence](evidence/identity-lifecycle-result.json), [45-entry evidence](evidence/browser-pages-result.json) and [44-entry evidence](evidence/browser-profiles-result.json) remain dated evidence. diff --git a/experiments/s7-migration/README.zh-CN.md b/experiments/s7-migration/README.zh-CN.md index e1915dde..5c138221 100644 --- a/experiments/s7-migration/README.zh-CN.md +++ b/experiments/s7-migration/README.zh-CN.md @@ -9,12 +9,13 @@ | --- | --- | --- | | 架构历史,27 条迁移 | `c33e03f1a14de739196113769c59fdaace9029e7` | 恢复旧数据,再通过现有生产启动守卫执行增量迁移。 | | 功能历史,19 条迁移 | `9cc73c9e78451e572f57d142d6b9caf62ccb78e2` | 直接升级必须在索引 17 失败;专用实验随后将有限兼容记录转入新建目标库。 | -| 已验证目标,51 条迁移 | 已提交 SQL 来源 `38c10b39ffa91ab4746da6d35ddf77505106ac33`,附当前 SQL/journal 准确哈希 | SQL 与 journal 必须匹配 `target-history.json`;变化后重新验证。 | +| 已验证目标,52 条迁移 | 已提交 SQL 来源 `8946a480542683cb85bf1b6ebc7c4134a9b27630`,附当前 SQL/journal 准确哈希 | SQL 与 journal 必须匹配 `target-history.json`;变化后重新验证。 | -51 条迁移的 C21 目标在 2026-10-03 通过全部 40 项保留数据迁移/恢复检查,以及 8 项清理检查。 -[当前证据](evidence/channel-storage-result.json)保存精确哈希和实际结果。迁移 0050 增加最小删除/重试 -凭证、默认关闭的存储设置及迟到引用守卫;C21 产品测试另行覆盖文件暂存恢复与策略行为。 -历史 SQL、夹具和断言未改动。[46 条身份生命周期证据](evidence/identity-lifecycle-result.json)、 +52 条迁移的 C22 目标在 2026-10-03 通过全部 40 项保留数据迁移/恢复检查,以及 8 项清理检查。 +[当前证据](evidence/global-trash-result.json)保存精确哈希和实际结果。迁移 0051 增加全局请求标识 +凭证;C22 产品测试另行覆盖重放、清理范围、引用及实测字节数。历史 SQL、夹具和断言未改动。 +[51 条 C21 证据](evidence/channel-storage-result.json)仍作为历史记录。 +[46 条身份生命周期证据](evidence/identity-lifecycle-result.json)、 [45 条证据](evidence/browser-pages-result.json)和[44 条证据](evidence/browser-profiles-result.json) 仍作为有日期的历史证据。本轮不证明生产附件暂存日志恢复,也不表示 S7 完成。 diff --git a/experiments/s7-migration/evidence/global-trash-result.json b/experiments/s7-migration/evidence/global-trash-result.json new file mode 100644 index 00000000..43e37918 --- /dev/null +++ b/experiments/s7-migration/evidence/global-trash-result.json @@ -0,0 +1,251 @@ +{ + "schemaVersion": 1, + "syntheticOnly": true, + "result": "passed", + "generatedAt": "2026-10-03T07:37:45.144Z", + "platform": "darwin", + "architecture": "arm64", + "node": "v22.23.2", + "image": "postgres:17.11-bookworm@sha256:051f7b7b3abdd564d5d1bd1e8c4b9c1b6e77087d1dd22020ede611c096a272e0", + "pgDumpVersion": "pg_dump (PostgreSQL) 17.11 (Debian 17.11-1.pgdg12+2)", + "histories": { + "feature": { + "commit": "9cc73c9e78451e572f57d142d6b9caf62ccb78e2", + "migrationCount": 19, + "digest": "f1b1dd00e5903d1fa592db1430fbc62ba3a57f358034af5b515673d28298764c" + }, + "architecture": { + "commit": "c33e03f1a14de739196113769c59fdaace9029e7", + "migrationCount": 27, + "digest": "adfe5fa4650c778bddade614c8473d2fdbdff16cce03c62339b045bc9d6332bf" + }, + "target": { + "commit": "8946a480542683cb85bf1b6ebc7c4134a9b27630", + "commitRole": "committed-migration-source", + "qualificationInput": { + "state": "committed-source-requalification", + "previousQualifiedTarget": "38c10b39ffa91ab4746da6d35ddf77505106ac33", + "journal": { + "path": "packages/db/migrations/meta/_journal.json", + "sha256": "f9b0ea7bf64201e9ceccfbd5fb24e036175dbe14c4451d8523ce7f99a295eeac" + }, + "changedSql": [ + { + "path": "packages/db/migrations/0051_global_trash_cleanup.sql", + "sha256": "f00bc68a8e1c0a552fe51b2b736c76c36f48e4c1a678acde52d9bbf4831a950c" + } + ] + }, + "migrationCount": 52, + "digest": "02fb9d57094d834e9df9632788ca7e870ac1ef7e702f6c53a60097eabc3547c7" + } + }, + "cases": [ + { + "name": "architecture: historical bootstrap and linked fixture", + "result": "passed", + "durationMs": 1345 + }, + { + "name": "architecture: nonempty restore destination rejected", + "result": "passed", + "durationMs": 67 + }, + { + "name": "architecture: damaged backup rejected before restore", + "result": "passed", + "durationMs": 32 + }, + { + "name": "architecture: full old database and file restore", + "result": "passed", + "durationMs": 402 + }, + { + "name": "architecture: guarded additive upgrade preserves records", + "result": "passed", + "durationMs": 962 + }, + { + "name": "architecture: repeat guarded startup is idempotent", + "result": "passed", + "durationMs": 172 + }, + { + "name": "architecture: target backup restores database and file references", + "result": "passed", + "durationMs": 1097 + }, + { + "name": "architecture: same-size corrupted file rejected", + "result": "passed", + "durationMs": 4 + }, + { + "name": "architecture: missing file rejected", + "result": "passed", + "durationMs": 2 + }, + { + "name": "architecture: hash history failure leaves database unchanged", + "result": "passed", + "durationMs": 948 + }, + { + "name": "architecture: timestamp history failure leaves database unchanged", + "result": "passed", + "durationMs": 1175 + }, + { + "name": "architecture: missing history failure leaves database unchanged", + "result": "passed", + "durationMs": 1491 + }, + { + "name": "architecture: ahead history failure leaves database unchanged", + "result": "passed", + "durationMs": 1743 + }, + { + "name": "architecture: native restore failure rolls back all DDL", + "result": "passed", + "durationMs": 953 + }, + { + "name": "feature: historical bootstrap and linked fixture", + "result": "passed", + "durationMs": 540 + }, + { + "name": "feature: nonempty restore destination rejected", + "result": "passed", + "durationMs": 143 + }, + { + "name": "feature: damaged backup rejected before restore", + "result": "passed", + "durationMs": 46 + }, + { + "name": "feature: full old database and file restore", + "result": "passed", + "durationMs": 295 + }, + { + "name": "feature: direct upgrade rejects divergence without mutation", + "result": "passed", + "durationMs": 225 + }, + { + "name": "feature: unknown nonempty table rejected", + "result": "passed", + "durationMs": 233 + }, + { + "name": "feature: unsupported s7_extra schema table rejected", + "result": "passed", + "durationMs": 234 + }, + { + "name": "feature: unsupported drizzle schema table rejected", + "result": "passed", + "durationMs": 240 + }, + { + "name": "feature: model connection rejected", + "result": "passed", + "durationMs": 459 + }, + { + "name": "feature: unmapped source column rejected", + "result": "passed", + "durationMs": 349 + }, + { + "name": "feature: model Run with null selection rejected", + "result": "passed", + "durationMs": 183 + }, + { + "name": "feature: active work rejected", + "result": "passed", + "durationMs": 173 + }, + { + "name": "feature: unprotected message reference orphan rejected", + "result": "passed", + "durationMs": 281 + }, + { + "name": "feature: drifted source ledger rejected", + "result": "passed", + "durationMs": 178 + }, + { + "name": "feature: late insert failure rolls back the whole transfer", + "result": "passed", + "durationMs": 317 + }, + { + "name": "feature: bounded transfer preserves identities and references", + "result": "passed", + "durationMs": 601 + }, + { + "name": "feature: duplicate transfer rejected without mutation", + "result": "passed", + "durationMs": 199 + }, + { + "name": "feature: repeat guarded startup is idempotent", + "result": "passed", + "durationMs": 414 + }, + { + "name": "feature: target backup restores database and file references", + "result": "passed", + "durationMs": 1085 + }, + { + "name": "feature: same-size corrupted file rejected", + "result": "passed", + "durationMs": 5 + }, + { + "name": "feature: missing file rejected", + "result": "passed", + "durationMs": 3 + }, + { + "name": "feature: hash history failure leaves database unchanged", + "result": "passed", + "durationMs": 1456 + }, + { + "name": "feature: timestamp history failure leaves database unchanged", + "result": "passed", + "durationMs": 1192 + }, + { + "name": "feature: missing history failure leaves database unchanged", + "result": "passed", + "durationMs": 1279 + }, + { + "name": "feature: ahead history failure leaves database unchanged", + "result": "passed", + "durationMs": 1127 + }, + { + "name": "feature: native restore failure rolls back all DDL", + "result": "passed", + "durationMs": 950 + } + ], + "limitations": [ + "Fixture-only feature transfer; no general divergent-history upgrade", + "Legacy runs retained; no work_tasks conversion", + "No settings/key/plugin/attachment/Temporal restore", + "No production migration, default switch, release, or S7 completion" + ] +} diff --git a/experiments/s7-migration/target-history.json b/experiments/s7-migration/target-history.json index 94b79831..22bd0d95 100644 --- a/experiments/s7-migration/target-history.json +++ b/experiments/s7-migration/target-history.json @@ -1,5 +1,5 @@ { - "commit": "38c10b39ffa91ab4746da6d35ddf77505106ac33", + "commit": "8946a480542683cb85bf1b6ebc7c4134a9b27630", "commitRole": "committed-migration-source", "migrations": [ { @@ -307,19 +307,25 @@ "tag": "0050_channel_storage_purge", "timestamp": 1790975448593, "sha256": "ccfe9031645f9228ab1a5ca5c102df61983f61fc679ef4d4e07c7e7be9557704" + }, + { + "index": 51, + "tag": "0051_global_trash_cleanup", + "timestamp": 1791011308235, + "sha256": "f00bc68a8e1c0a552fe51b2b736c76c36f48e4c1a678acde52d9bbf4831a950c" } ], "qualificationInput": { "state": "committed-source-requalification", - "previousQualifiedTarget": "ce46574be1ff8189d12c027df78a6534783c8704", + "previousQualifiedTarget": "38c10b39ffa91ab4746da6d35ddf77505106ac33", "journal": { "path": "packages/db/migrations/meta/_journal.json", - "sha256": "3708bf44582d0c060823e7426f72a0d00aa9cc728f89d4b0035321443e0e4e46" + "sha256": "f9b0ea7bf64201e9ceccfbd5fb24e036175dbe14c4451d8523ce7f99a295eeac" }, "changedSql": [ { - "path": "packages/db/migrations/0050_channel_storage_purge.sql", - "sha256": "ccfe9031645f9228ab1a5ca5c102df61983f61fc679ef4d4e07c7e7be9557704" + "path": "packages/db/migrations/0051_global_trash_cleanup.sql", + "sha256": "f00bc68a8e1c0a552fe51b2b736c76c36f48e4c1a678acde52d9bbf4831a950c" } ] } diff --git a/packages/protocol/src/browser.test.ts b/packages/protocol/src/browser.test.ts index 063cc032..2d3c9c00 100644 --- a/packages/protocol/src/browser.test.ts +++ b/packages/protocol/src/browser.test.ts @@ -64,4 +64,23 @@ it("keeps lifecycle actions off the public input command and requires separate c }).success, ).toBe(false); expect(browserRuntimeStateSchema.safeParse({ running: "true" }).success).toBe(false); + expect(browserRuntimeStateSchema.parse({ running: true })).toEqual({ + running: true, + profileBytes: null, + }); + for (const profileBytes of [null, 0, 123, Number.MAX_SAFE_INTEGER]) { + expect(browserRuntimeStateSchema.parse({ running: true, profileBytes })).toEqual({ + running: true, + profileBytes, + }); + } + for (const profileBytes of [true, "123", -1, 1.5, Number.MAX_SAFE_INTEGER + 1, NaN, Infinity]) { + expect(browserRuntimeStateSchema.safeParse({ running: true, profileBytes }).success).toBe( + false, + ); + } + expect( + browserRuntimeStateSchema.safeParse({ running: true, profileBytes: 1, path: "/private" }) + .success, + ).toBe(false); }); diff --git a/packages/protocol/src/browser.ts b/packages/protocol/src/browser.ts index 7c1c5e52..d3fb8df2 100644 --- a/packages/protocol/src/browser.ts +++ b/packages/protocol/src/browser.ts @@ -130,7 +130,12 @@ export const browserMaintenanceInputSchema = z .strict() .refine((value) => (value.operation === "clear") === (value.confirmation !== undefined)); export type BrowserMaintenanceInput = z.infer; -export const browserRuntimeStateSchema = z.object({ running: z.boolean() }).strict(); +export const browserRuntimeStateSchema = z + .object({ + running: z.boolean(), + profileBytes: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable().default(null), + }) + .strict(); export type BrowserRuntimeState = z.infer; export const browserMaintenanceActionSchema = z .object({ kind: z.literal("maintenance"), operation: z.enum(["status", "restart", "clear"]) }) @@ -195,7 +200,7 @@ export const browserMaintenanceResultSchema = z .object({ botId: z.string().uuid(), nodeId: z.string().min(1).max(128), - running: z.boolean(), + ...browserRuntimeStateSchema.shape, paused: z.boolean(), }) .strict(); diff --git a/providers/docker/src/browser-maintenance.test.ts b/providers/docker/src/browser-maintenance.test.ts index 69aca5a1..7dd4f69f 100644 --- a/providers/docker/src/browser-maintenance.test.ts +++ b/providers/docker/src/browser-maintenance.test.ts @@ -65,6 +65,7 @@ describe("original browser lifecycle adapter", () => { const c = command("restart"); expect(await coordinator.maintenance(c, new AbortController().signal)).toEqual({ running: true, + profileBytes: null, }); expect(paths).toEqual(["/computers/stop", "/screenshot", "/health"]); await expect( @@ -83,7 +84,7 @@ describe("original browser lifecycle adapter", () => { coordinator.run(c.botId, new AbortController().signal, async () => true), ).rejects.toThrow("paused"); }); - it("projects only a boolean and observes health without starting a browser", async () => { + it("projects only health and bounded usage without starting a browser", async () => { const request = vi.fn(async () => ({ status: "ok", browser: false, @@ -93,12 +94,78 @@ describe("original browser lifecycle adapter", () => { const coordinator = new BrowserCoordinator(request, async () => {}); expect(await coordinator.maintenance(command("status"), new AbortController().signal)).toEqual({ running: false, + profileBytes: null, }); - expect(request).toHaveBeenCalledExactlyOnceWith( + expect(request).toHaveBeenNthCalledWith( + 1, expect.any(String), "/health", expect.any(AbortSignal), undefined, ); + expect(request).toHaveBeenNthCalledWith( + 2, + expect.any(String), + "/computers/profile-usage", + expect.any(AbortSignal), + undefined, + ); + expect(request).toHaveBeenCalledTimes(2); + }); + + it.each([ + 0, + 12345, + Number.MAX_SAFE_INTEGER, + null, + -1, + 1.5, + true, + "123", + Number.NaN, + Infinity, + Number.MAX_SAFE_INTEGER + 1, + undefined, + ])("reports only a safe measured byte count: %s", async (profileBytes) => { + const request = vi.fn(async (_bot: string, path: string) => + path === "/health" + ? { status: "ok", browser: true } + : { profileBytes, path: "/private/profile", cookies: "private" }, + ); + const c = command("status"); + const coordinator = new BrowserCoordinator(request, async () => {}); + expect(await coordinator.maintenance(c, new AbortController().signal)).toEqual({ + running: true, + profileBytes: + typeof profileBytes === "number" && Number.isSafeInteger(profileBytes) && profileBytes >= 0 + ? profileBytes + : null, + }); + expect(request).toHaveBeenNthCalledWith( + 2, + c.botId, + "/computers/profile-usage", + expect.any(AbortSignal), + undefined, + ); + }); + + it("keeps unavailable usage unknown while preserving cancellation", async () => { + const abort = new AbortController(); + const request = vi.fn(async (_bot: string, path: string) => { + if (path === "/health") return { status: "ok", browser: false }; + throw new Error("Older upstream returned 404"); + }); + const coordinator = new BrowserCoordinator(request, async () => {}); + expect(await coordinator.maintenance(command("status"), abort.signal)).toEqual({ + running: false, + profileBytes: null, + }); + request.mockImplementation(async (_bot: string, path: string) => { + if (path === "/health") return { status: "ok", browser: false }; + abort.abort(); + throw new Error("Cancelled measurement"); + }); + await expect(coordinator.maintenance(command("status"), abort.signal)).rejects.toThrow(); }); }); diff --git a/providers/docker/src/browser.ts b/providers/docker/src/browser.ts index 1b1c60bf..9f49973c 100644 --- a/providers/docker/src/browser.ts +++ b/providers/docker/src/browser.ts @@ -165,7 +165,22 @@ export class BrowserCoordinator { const running = (value as { browser: boolean }).browser; if ((operation === "restart" && !running) || (operation === "clear" && running)) throw new Error("Unexpected browser state."); - return { running }; + let profileBytes: number | null = null; + if (operation === "status") { + try { + const usage = await call("/computers/profile-usage"); + const measured = + usage && typeof usage === "object" + ? (usage as { profileBytes?: unknown }).profileBytes + : undefined; + if (typeof measured === "number" && Number.isSafeInteger(measured) && measured >= 0) + profileBytes = measured; + } catch { + // Older upstreams and refused measurements are unknown, never a zero-byte profile. + } + signal.throwIfAborted(); + } + return { running, profileBytes }; }); }