From 233871bf3f6dc6807c046cb3411d1424c9d06c98 Mon Sep 17 00:00:00 2001 From: Kevin Hong Date: Mon, 16 Nov 2026 10:05:10 +0800 Subject: [PATCH 1/6] [CU-1k5fpda7hax] Add omics workbench storage reindex Re-scanning a connected storage bucket was only possible from the Reindex button in the Workbench Data tab, so node setup and clean-up (for example removing test samples from a customer node) could not be scripted from the omics CLI. The command starts the same background reindex; --wait polls the account until indexing is no longer RUNNING so a script can continue once the samples reflect the bucket. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011GSYqNdqNKNGKkpH2TRtPG --- .../commands/workbench/storage/commands.py | 63 +++++++++++ dnastack/client/workbench/storage/client.py | 11 +- dnastack/client/workbench/storage/models.py | 2 + .../commands/workbench/storage/__init__.py | 0 .../workbench/storage/test_commands.py | 102 ++++++++++++++++++ .../unit/client/workbench/storage/__init__.py | 0 .../workbench/storage/test_storage_client.py | 33 ++++++ 7 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 tests/unit/cli/commands/workbench/storage/__init__.py create mode 100644 tests/unit/cli/commands/workbench/storage/test_commands.py create mode 100644 tests/unit/client/workbench/storage/__init__.py create mode 100644 tests/unit/client/workbench/storage/test_storage_client.py diff --git a/dnastack/cli/commands/workbench/storage/commands.py b/dnastack/cli/commands/workbench/storage/commands.py index 7d4fcd97..52c8ac7d 100644 --- a/dnastack/cli/commands/workbench/storage/commands.py +++ b/dnastack/cli/commands/workbench/storage/commands.py @@ -1,3 +1,4 @@ +from time import monotonic, sleep from typing import Optional, List import click @@ -11,6 +12,8 @@ from dnastack.cli.helpers.iterator_printer import OutputFormat, show_iterator from dnastack.client.workbench.storage.models import StorageListOptions, Provider +REINDEX_POLL_INTERVAL_SECONDS = 10 + def init_storage_commands(group: Group): @formatted_command( @@ -123,3 +126,63 @@ def get_storage_accounts(context: Optional[str], + + + @formatted_command( + group=group, + name='reindex', + specs=[ + ArgumentSpec( + name='storage_id', + arg_type=ArgumentType.POSITIONAL, + help='The storage account id', + required=True, + ), + ArgumentSpec( + name='wait', + arg_names=['--wait'], + help='Wait for indexing to finish, then print the account.', + type=bool, + ), + ArgumentSpec( + name='timeout', + arg_names=['--timeout'], + help='With --wait, how many seconds to wait before giving up. Indexing keeps running.', + type=int, + default=3600, + ), + NAMESPACE_ARG, + CONTEXT_ARG, + SINGLE_ENDPOINT_ID_ARG, + ] + ) + def reindex_storage_account(context: Optional[str], + endpoint_id: Optional[str], + namespace: Optional[str], + storage_id: str, + wait: bool = False, + timeout: int = 3600): + """ + Re-scan a storage account for sample files + + Indexing runs in the background: without --wait the command returns once indexing has started. Indexing + also removes sample files whose objects are no longer in the bucket. Requires the workbench.storage.update + permission (namespace admin). + """ + client = get_storage_client(context, endpoint_id, namespace) + account = client.reindex_storage_account(storage_id) + + if wait: + if account.indexing_status is None: + raise click.ClickException('This Workbench does not report indexing status, so --wait cannot ' + 'tell when indexing finishes.') + deadline = monotonic() + timeout + while account.indexing_status == 'RUNNING': + remaining = deadline - monotonic() + if remaining <= 0: + raise click.ClickException(f'Indexing of storage account {storage_id} is still running after ' + f'{timeout} seconds.') + sleep(min(REINDEX_POLL_INTERVAL_SECONDS, remaining)) + account = client.get_storage_account(storage_id) + + click.echo(to_json(normalize(account))) diff --git a/dnastack/client/workbench/storage/client.py b/dnastack/client/workbench/storage/client.py index e2f50b3e..803d6288 100644 --- a/dnastack/client/workbench/storage/client.py +++ b/dnastack/client/workbench/storage/client.py @@ -4,7 +4,7 @@ from dnastack import ServiceEndpoint from dnastack.client.result_iterator import ResultIterator from dnastack.client.service_registry.models import ServiceType -from dnastack.client.workbench.base_client import BaseWorkbenchClient, WorkbenchResultLoader +from dnastack.client.workbench.base_client import BaseWorkbenchClient, WorkbenchResultLoader, path_segment from dnastack.client.workbench.storage.models import StorageAccount, StorageListOptions, StorageListResponse from dnastack.common.tracing import Span from dnastack.http.session import HttpSession @@ -80,6 +80,15 @@ def get_storage_account(self, storage_account_id: str) -> StorageAccount: response = session.get(urljoin(self.endpoint.url, f'{self.namespace}/storage/{storage_account_id}')) return StorageAccount(**response.json()) + def reindex_storage_account(self, storage_account_id: str) -> StorageAccount: + """Start re-scanning a storage account's bucket. Indexing runs in the background; the returned + account reports it as RUNNING until it finishes.""" + with self.create_http_session() as session: + response = session.post( + urljoin(self.endpoint.url, f'{self.namespace}/storage/{path_segment(storage_account_id)}/reindex') + ) + return StorageAccount(**response.json()) + def list_storage_accounts(self, list_options: Optional[StorageListOptions], max_results: int) -> Iterator[StorageAccount]: """List storage accounts.""" return ResultIterator(StorageAccountListResultLoader( diff --git a/dnastack/client/workbench/storage/models.py b/dnastack/client/workbench/storage/models.py index d31189e0..f8b078f7 100644 --- a/dnastack/client/workbench/storage/models.py +++ b/dnastack/client/workbench/storage/models.py @@ -58,6 +58,8 @@ class StorageAccount(BaseModel): created_at: Optional[str] = None last_updated_at: Optional[str] = None bucket: Optional[str] = None + indexing_status: Optional[str] = None + last_indexed_at: Optional[str] = None credentials: Optional[Union[AwsStorageAccountCredentials, GcpStorageAccountCredentials, AzureStorageAccountCredentials]] = None diff --git a/tests/unit/cli/commands/workbench/storage/__init__.py b/tests/unit/cli/commands/workbench/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/cli/commands/workbench/storage/test_commands.py b/tests/unit/cli/commands/workbench/storage/test_commands.py new file mode 100644 index 00000000..6abf7ba9 --- /dev/null +++ b/tests/unit/cli/commands/workbench/storage/test_commands.py @@ -0,0 +1,102 @@ +import json +from unittest.mock import patch + +import pytest +import responses +from click import Group +from click.testing import CliRunner + +from dnastack.cli.commands.workbench.storage.commands import init_storage_commands +from dnastack.client.models import ServiceEndpoint +from dnastack.client.workbench.storage.client import StorageClient + +_SAMPLE_SERVICE_URL = 'https://sample-service.example.com/' +_ACCOUNT_URL = f'{_SAMPLE_SERVICE_URL}ns-1/storage/storage-1' + + +def _account(indexing_status: str, last_indexed_at: str = '2026-11-15T12:00:00Z') -> dict: + return {"id": "storage-1", "name": "instruments", "indexing_status": indexing_status, + "last_indexed_at": last_indexed_at} + + +class TestReindexCommand: + """Tests for the storage reindex CLI command, through the real HTTP session.""" + + def setup_method(self): + self.runner = CliRunner(mix_stderr=False) + self.group = Group() + init_storage_commands(self.group) + self._client_patch = patch( + 'dnastack.cli.commands.workbench.storage.commands.get_storage_client', + return_value=StorageClient.make(ServiceEndpoint(id='test', url=_SAMPLE_SERVICE_URL), 'ns-1')) + self.mock_get_client = self._client_patch.start() + + def teardown_method(self): + self._client_patch.stop() + + @responses.activate + def test_starts_reindex_and_prints_the_running_account(self): + responses.post(f'{_ACCOUNT_URL}/reindex', json=_account('RUNNING')) + + result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--namespace', 'ns-1']) + + assert result.exit_code == 0, result.stderr + self.mock_get_client.assert_called_once_with(None, None, 'ns-1') + assert json.loads(result.stdout)["indexing_status"] == "RUNNING" + assert [c.request.method for c in responses.calls] == ['POST'] + + @pytest.mark.parametrize('status', [400, 403, 404]) + @responses.activate + def test_reports_http_status_and_exits_non_zero(self, status): + responses.post(f'{_ACCOUNT_URL}/reindex', status=status, body='rejected') + + result = self.runner.invoke(self.group, ['reindex', 'storage-1']) + + assert result.exit_code == 1 + assert f"HTTP {status}: rejected" in result.stderr + assert result.stdout == "" + + def test_requires_storage_id(self): + result = self.runner.invoke(self.group, ['reindex']) + + assert result.exit_code == 2 + self.mock_get_client.assert_not_called() + + @responses.activate + @patch('dnastack.cli.commands.workbench.storage.commands.sleep') + def test_wait_polls_until_indexing_finishes_and_prints_the_final_account(self, mock_sleep): + responses.post(f'{_ACCOUNT_URL}/reindex', json=_account('RUNNING')) + responses.get(_ACCOUNT_URL, json=_account('RUNNING')) + responses.get(_ACCOUNT_URL, json=_account('IDLE', last_indexed_at='2026-11-16T09:30:00Z')) + + result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait']) + + assert result.exit_code == 0, result.stderr + assert [c.request.method for c in responses.calls] == ['POST', 'GET', 'GET'] + assert mock_sleep.call_count == 2 + output = json.loads(result.stdout) + assert output["indexing_status"] == "IDLE" + assert output["last_indexed_at"] == "2026-11-16T09:30:00Z" + + @responses.activate + @patch('dnastack.cli.commands.workbench.storage.commands.sleep') + def test_wait_gives_up_after_the_timeout_and_exits_non_zero(self, mock_sleep): + responses.post(f'{_ACCOUNT_URL}/reindex', json=_account('RUNNING')) + + result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait', '--timeout', '0']) + + assert result.exit_code == 1 + assert "still running" in result.stderr + assert result.stdout == "" + mock_sleep.assert_not_called() + + @responses.activate + @patch('dnastack.cli.commands.workbench.storage.commands.sleep') + def test_wait_fails_when_workbench_does_not_report_indexing_status(self, mock_sleep): + responses.post(f'{_ACCOUNT_URL}/reindex', json={"id": "storage-1"}) + + result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait']) + + assert result.exit_code == 1 + assert "indexing status" in result.stderr + assert result.stdout == "" diff --git a/tests/unit/client/workbench/storage/__init__.py b/tests/unit/client/workbench/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/client/workbench/storage/test_storage_client.py b/tests/unit/client/workbench/storage/test_storage_client.py new file mode 100644 index 00000000..89c149ec --- /dev/null +++ b/tests/unit/client/workbench/storage/test_storage_client.py @@ -0,0 +1,33 @@ +import pytest +import responses + +from dnastack.client.models import ServiceEndpoint +from dnastack.client.workbench.storage.client import StorageClient + +_SAMPLE_SERVICE_URL = 'https://sample-service.example.com/' + + +def _make_client(): + return StorageClient.make(ServiceEndpoint(id='test', url=_SAMPLE_SERVICE_URL), 'ns-1') + + +class TestReindexStorageAccount: + """Tests for StorageClient.reindex_storage_account""" + + @responses.activate + def test_posts_reindex_without_a_body_and_returns_the_running_account(self): + responses.post(f'{_SAMPLE_SERVICE_URL}ns-1/storage/storage-1/reindex', + json={"id": "storage-1", "name": "instruments", "indexing_status": "RUNNING", + "last_indexed_at": "2026-11-15T12:00:00Z"}) + + account = _make_client().reindex_storage_account('storage-1') + + assert responses.calls[0].request.body is None + assert account.id == "storage-1" + assert account.indexing_status == "RUNNING" + assert account.last_indexed_at == "2026-11-15T12:00:00Z" + + @pytest.mark.parametrize('storage_account_id', ['', '.', '..']) + def test_rejects_ids_that_cannot_be_one_path_segment(self, storage_account_id): + with pytest.raises(ValueError): + _make_client().reindex_storage_account(storage_account_id) From 14de65a7afa551b3bc8d191eb7a914a89992b3ae Mon Sep 17 00:00:00 2001 From: Kevin Hong Date: Mon, 16 Nov 2026 10:07:37 +0800 Subject: [PATCH 2/6] [CU-1k5fpda7hax] Make reindex --wait fail when indexing did not complete Workbench returns a failed indexing run to IDLE just like a finished one, so --wait would exit 0 and let a setup script carry on with samples that do not reflect the bucket. Workbench only advances last_indexed_at when a run completes, so --wait now reads it before triggering and fails unless it moved. An indexing status it does not recognise also fails instead of counting as finished, and a negative --timeout is rejected before any request is sent. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011GSYqNdqNKNGKkpH2TRtPG --- .../commands/workbench/storage/commands.py | 55 ++++++++++++------- .../workbench/storage/test_commands.py | 38 ++++++++++++- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/dnastack/cli/commands/workbench/storage/commands.py b/dnastack/cli/commands/workbench/storage/commands.py index 52c8ac7d..b764cd47 100644 --- a/dnastack/cli/commands/workbench/storage/commands.py +++ b/dnastack/cli/commands/workbench/storage/commands.py @@ -10,11 +10,40 @@ from dnastack.cli.core.command_spec import ArgumentSpec, ArgumentType, CONTEXT_ARG, SINGLE_ENDPOINT_ID_ARG from dnastack.cli.helpers.exporter import to_json, normalize from dnastack.cli.helpers.iterator_printer import OutputFormat, show_iterator -from dnastack.client.workbench.storage.models import StorageListOptions, Provider +from dnastack.client.workbench.storage.client import StorageClient +from dnastack.client.workbench.storage.models import StorageAccount, StorageListOptions, Provider REINDEX_POLL_INTERVAL_SECONDS = 10 +def _wait_for_indexing(client: StorageClient, + storage_id: str, + account: StorageAccount, + timeout: int, + last_completed_at: Optional[str]) -> StorageAccount: + """Poll until indexing stops, and fail unless it recorded a completed run after `last_completed_at`.""" + if account.indexing_status is None: + raise click.ClickException('This Workbench does not report indexing status, so --wait cannot ' + 'tell when indexing finishes.') + deadline = monotonic() + timeout + while account.indexing_status == 'RUNNING': + remaining = deadline - monotonic() + if remaining <= 0: + raise click.ClickException(f'Indexing of storage account {storage_id} is still running after ' + f'{timeout} seconds.') + sleep(min(REINDEX_POLL_INTERVAL_SECONDS, remaining)) + account = client.get_storage_account(storage_id) + + if account.indexing_status != 'IDLE': + raise click.ClickException(f'Storage account {storage_id} reports indexing status ' + f'{account.indexing_status}, which --wait does not recognise.') + # Workbench records last_indexed_at only when a run completes; a failed run returns to IDLE without it. + if account.last_indexed_at == last_completed_at: + raise click.ClickException(f'Indexing of storage account {storage_id} stopped but did not complete. ' + f'The last completed indexing is still {last_completed_at or "none"}.') + return account + + def init_storage_commands(group: Group): @formatted_command( group=group, @@ -123,11 +152,6 @@ def get_storage_accounts(context: Optional[str], storage_accounts = [client.get_storage_account(storage_account_id) for storage_account_id in storage_id] click.echo(to_json(normalize(storage_accounts))) - - - - - @formatted_command( group=group, name='reindex', @@ -141,7 +165,7 @@ def get_storage_accounts(context: Optional[str], ArgumentSpec( name='wait', arg_names=['--wait'], - help='Wait for indexing to finish, then print the account.', + help='Wait for indexing to finish, then print the account. Exits non-zero if indexing did not complete.', type=bool, ), ArgumentSpec( @@ -169,20 +193,11 @@ def reindex_storage_account(context: Optional[str], also removes sample files whose objects are no longer in the bucket. Requires the workbench.storage.update permission (namespace admin). """ + if timeout < 0: + raise click.BadParameter('must be zero or more seconds.', param_hint="'--timeout'") client = get_storage_client(context, endpoint_id, namespace) + last_completed_at = client.get_storage_account(storage_id).last_indexed_at if wait else None account = client.reindex_storage_account(storage_id) - if wait: - if account.indexing_status is None: - raise click.ClickException('This Workbench does not report indexing status, so --wait cannot ' - 'tell when indexing finishes.') - deadline = monotonic() + timeout - while account.indexing_status == 'RUNNING': - remaining = deadline - monotonic() - if remaining <= 0: - raise click.ClickException(f'Indexing of storage account {storage_id} is still running after ' - f'{timeout} seconds.') - sleep(min(REINDEX_POLL_INTERVAL_SECONDS, remaining)) - account = client.get_storage_account(storage_id) - + account = _wait_for_indexing(client, storage_id, account, timeout, last_completed_at) click.echo(to_json(normalize(account))) diff --git a/tests/unit/cli/commands/workbench/storage/test_commands.py b/tests/unit/cli/commands/workbench/storage/test_commands.py index 6abf7ba9..c1b0a9e6 100644 --- a/tests/unit/cli/commands/workbench/storage/test_commands.py +++ b/tests/unit/cli/commands/workbench/storage/test_commands.py @@ -65,6 +65,7 @@ def test_requires_storage_id(self): @responses.activate @patch('dnastack.cli.commands.workbench.storage.commands.sleep') def test_wait_polls_until_indexing_finishes_and_prints_the_final_account(self, mock_sleep): + responses.get(_ACCOUNT_URL, json=_account('IDLE')) responses.post(f'{_ACCOUNT_URL}/reindex', json=_account('RUNNING')) responses.get(_ACCOUNT_URL, json=_account('RUNNING')) responses.get(_ACCOUNT_URL, json=_account('IDLE', last_indexed_at='2026-11-16T09:30:00Z')) @@ -72,7 +73,7 @@ def test_wait_polls_until_indexing_finishes_and_prints_the_final_account(self, m result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait']) assert result.exit_code == 0, result.stderr - assert [c.request.method for c in responses.calls] == ['POST', 'GET', 'GET'] + assert [c.request.method for c in responses.calls] == ['GET', 'POST', 'GET', 'GET'] assert mock_sleep.call_count == 2 output = json.loads(result.stdout) assert output["indexing_status"] == "IDLE" @@ -81,6 +82,7 @@ def test_wait_polls_until_indexing_finishes_and_prints_the_final_account(self, m @responses.activate @patch('dnastack.cli.commands.workbench.storage.commands.sleep') def test_wait_gives_up_after_the_timeout_and_exits_non_zero(self, mock_sleep): + responses.get(_ACCOUNT_URL, json=_account('IDLE')) responses.post(f'{_ACCOUNT_URL}/reindex', json=_account('RUNNING')) result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait', '--timeout', '0']) @@ -93,6 +95,7 @@ def test_wait_gives_up_after_the_timeout_and_exits_non_zero(self, mock_sleep): @responses.activate @patch('dnastack.cli.commands.workbench.storage.commands.sleep') def test_wait_fails_when_workbench_does_not_report_indexing_status(self, mock_sleep): + responses.get(_ACCOUNT_URL, json={"id": "storage-1"}) responses.post(f'{_ACCOUNT_URL}/reindex', json={"id": "storage-1"}) result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait']) @@ -100,3 +103,36 @@ def test_wait_fails_when_workbench_does_not_report_indexing_status(self, mock_sl assert result.exit_code == 1 assert "indexing status" in result.stderr assert result.stdout == "" + + @responses.activate + @patch('dnastack.cli.commands.workbench.storage.commands.sleep') + def test_wait_fails_when_indexing_stops_without_recording_a_completed_run(self, mock_sleep): + responses.get(_ACCOUNT_URL, json=_account('IDLE')) + responses.post(f'{_ACCOUNT_URL}/reindex', json=_account('RUNNING')) + responses.get(_ACCOUNT_URL, json=_account('IDLE')) + + result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait']) + + assert result.exit_code == 1 + assert "did not complete" in result.stderr + assert result.stdout == "" + + @responses.activate + @patch('dnastack.cli.commands.workbench.storage.commands.sleep') + def test_wait_fails_on_an_indexing_status_it_does_not_know(self, mock_sleep): + responses.get(_ACCOUNT_URL, json=_account('IDLE')) + responses.post(f'{_ACCOUNT_URL}/reindex', json=_account('QUEUED', last_indexed_at='2026-11-16T09:30:00Z')) + + result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait']) + + assert result.exit_code == 1 + assert "QUEUED" in result.stderr + assert result.stdout == "" + + @responses.activate + def test_wait_rejects_a_negative_timeout(self): + result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait', '--timeout', '-1']) + + assert result.exit_code == 2 + assert len(responses.calls) == 0 + assert "--timeout" in result.stderr From 1d4108fea570050d68a99b11e74de57e66558dcd Mon Sep 17 00:00:00 2001 From: Kevin Hong Date: Mon, 16 Nov 2026 10:09:08 +0800 Subject: [PATCH 3/6] [CU-1k5fpda7hax] Drop sleep patches from reindex tests that never poll The unknown-status and missing-status tests fail before the poll loop, so patching sleep there suggested they exercised it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011GSYqNdqNKNGKkpH2TRtPG --- tests/unit/cli/commands/workbench/storage/test_commands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/cli/commands/workbench/storage/test_commands.py b/tests/unit/cli/commands/workbench/storage/test_commands.py index c1b0a9e6..9cfddd8c 100644 --- a/tests/unit/cli/commands/workbench/storage/test_commands.py +++ b/tests/unit/cli/commands/workbench/storage/test_commands.py @@ -93,8 +93,7 @@ def test_wait_gives_up_after_the_timeout_and_exits_non_zero(self, mock_sleep): mock_sleep.assert_not_called() @responses.activate - @patch('dnastack.cli.commands.workbench.storage.commands.sleep') - def test_wait_fails_when_workbench_does_not_report_indexing_status(self, mock_sleep): + def test_wait_fails_when_workbench_does_not_report_indexing_status(self): responses.get(_ACCOUNT_URL, json={"id": "storage-1"}) responses.post(f'{_ACCOUNT_URL}/reindex', json={"id": "storage-1"}) @@ -118,8 +117,7 @@ def test_wait_fails_when_indexing_stops_without_recording_a_completed_run(self, assert result.stdout == "" @responses.activate - @patch('dnastack.cli.commands.workbench.storage.commands.sleep') - def test_wait_fails_on_an_indexing_status_it_does_not_know(self, mock_sleep): + def test_wait_fails_on_an_indexing_status_it_does_not_know(self): responses.get(_ACCOUNT_URL, json=_account('IDLE')) responses.post(f'{_ACCOUNT_URL}/reindex', json=_account('QUEUED', last_indexed_at='2026-11-16T09:30:00Z')) From 7c217c44a139601fbc800974fd573b53c83deb79 Mon Sep 17 00:00:00 2001 From: Kevin Hong Date: Mon, 16 Nov 2026 21:26:09 +0800 Subject: [PATCH 4/6] [CU-1k5fpda7hax] Escape the storage id in every storage account request reindex --wait reads the account before and while reindexing, so with only the reindex POST escaped an id that needs escaping failed on the first GET. Escaping it for get, update and delete too keeps one request chain consistent and stops a crafted id from addressing a path outside the account, as the engine client already does. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011GSYqHzB6MG7utV8vYMnZX --- dnastack/client/workbench/storage/client.py | 17 +++++----- .../workbench/storage/test_storage_client.py | 34 +++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/dnastack/client/workbench/storage/client.py b/dnastack/client/workbench/storage/client.py index 803d6288..afdd096f 100644 --- a/dnastack/client/workbench/storage/client.py +++ b/dnastack/client/workbench/storage/client.py @@ -48,6 +48,9 @@ def make(cls, endpoint: ServiceEndpoint, namespace: str): endpoint.type = cls.get_default_service_type() return cls(endpoint, namespace) + def _account_url(self, storage_account_id: str, suffix: str = '') -> str: + return urljoin(self.endpoint.url, f'{self.namespace}/storage/{path_segment(storage_account_id)}{suffix}') + def add_storage_account(self, storage_account: StorageAccount) -> StorageAccount: """Create a new storage account.""" with self.create_http_session() as session: @@ -61,7 +64,7 @@ def update_storage_account(self, storage_account_id: str, storage_account: Stora """Update a storage account.""" with self.create_http_session() as session: response = session.submit("PUT", - urljoin(self.endpoint.url, f'{self.namespace}/storage/{storage_account_id}'), + self._account_url(storage_account_id), json=storage_account.model_dump(), headers={"If-Match": if_match} ) @@ -71,22 +74,20 @@ def update_storage_account(self, storage_account_id: str, storage_account: Stora def delete_storage_account(self, storage_account_id: str) -> None: """Delete a storage account.""" with self.create_http_session() as session: - session.delete(urljoin(self.endpoint.url, f'{self.namespace}/storage/{storage_account_id}')) + session.delete(self._account_url(storage_account_id)) return None def get_storage_account(self, storage_account_id: str) -> StorageAccount: """Get a storage account by its ID.""" with self.create_http_session() as session: - response = session.get(urljoin(self.endpoint.url, f'{self.namespace}/storage/{storage_account_id}')) + response = session.get(self._account_url(storage_account_id)) return StorageAccount(**response.json()) def reindex_storage_account(self, storage_account_id: str) -> StorageAccount: - """Start re-scanning a storage account's bucket. Indexing runs in the background; the returned - account reports it as RUNNING until it finishes.""" + """Start re-scanning a storage account's bucket. Indexing runs in the background: the returned account + reports RUNNING until it finishes, or IDLE if indexing already finished before the server responded.""" with self.create_http_session() as session: - response = session.post( - urljoin(self.endpoint.url, f'{self.namespace}/storage/{path_segment(storage_account_id)}/reindex') - ) + response = session.post(self._account_url(storage_account_id, '/reindex')) return StorageAccount(**response.json()) def list_storage_accounts(self, list_options: Optional[StorageListOptions], max_results: int) -> Iterator[StorageAccount]: diff --git a/tests/unit/client/workbench/storage/test_storage_client.py b/tests/unit/client/workbench/storage/test_storage_client.py index 89c149ec..196f6f8d 100644 --- a/tests/unit/client/workbench/storage/test_storage_client.py +++ b/tests/unit/client/workbench/storage/test_storage_client.py @@ -3,6 +3,7 @@ from dnastack.client.models import ServiceEndpoint from dnastack.client.workbench.storage.client import StorageClient +from dnastack.client.workbench.storage.models import StorageAccount _SAMPLE_SERVICE_URL = 'https://sample-service.example.com/' @@ -31,3 +32,36 @@ def test_posts_reindex_without_a_body_and_returns_the_running_account(self): def test_rejects_ids_that_cannot_be_one_path_segment(self, storage_account_id): with pytest.raises(ValueError): _make_client().reindex_storage_account(storage_account_id) + + +_ODD_ID = 'a/../b#c?d' +_ODD_SEGMENT = 'a%2F..%2Fb%23c%3Fd' +_ACCOUNT_JSON = {"id": _ODD_ID, "name": "instruments"} + + +class TestStorageAccountIdIsOnePathSegment: + """Every StorageClient method that addresses one account sends its id as a single escaped path segment.""" + + @pytest.mark.parametrize('method, call, suffix', [ + ('GET', lambda client: client.get_storage_account(_ODD_ID), ''), + ('PUT', lambda client: client.update_storage_account(_ODD_ID, StorageAccount(), 'etag-1'), ''), + ('DELETE', lambda client: client.delete_storage_account(_ODD_ID), ''), + ('POST', lambda client: client.reindex_storage_account(_ODD_ID), '/reindex'), + ]) + @responses.activate + def test_escapes_the_id(self, method, call, suffix): + expected_url = f'{_SAMPLE_SERVICE_URL}ns-1/storage/{_ODD_SEGMENT}{suffix}' + responses.add(method, expected_url, json=_ACCOUNT_JSON) + + call(_make_client()) + + assert responses.calls[0].request.url == expected_url + + @pytest.mark.parametrize('call', [ + lambda client: client.get_storage_account('..'), + lambda client: client.update_storage_account('', StorageAccount(), 'etag-1'), + lambda client: client.delete_storage_account('.'), + ]) + def test_rejects_ids_that_cannot_be_one_path_segment(self, call): + with pytest.raises(ValueError): + call(_make_client()) From 1ca71dae4f944e1983e22c0c549c69d164a8241b Mon Sep 17 00:00:00 2001 From: Kevin Hong Date: Mon, 16 Nov 2026 21:26:53 +0800 Subject: [PATCH 5/6] [CU-1k5fpda7hax] Fail reindex --wait misuse before starting a reindex A Workbench that does not report indexing status could only be detected after the POST had already started indexing, and --timeout without --wait was silently ignored. Both now fail before any reindex is sent. The --wait help also says indexing keeps running when the wait ends early (timeout, poll error or Ctrl-C), and fits the 120-character line limit. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011GSYqHzB6MG7utV8vYMnZX --- .../commands/workbench/storage/commands.py | 31 ++++++++++++------- .../workbench/storage/test_commands.py | 22 ++++++++++++- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/dnastack/cli/commands/workbench/storage/commands.py b/dnastack/cli/commands/workbench/storage/commands.py index b764cd47..71cbbee9 100644 --- a/dnastack/cli/commands/workbench/storage/commands.py +++ b/dnastack/cli/commands/workbench/storage/commands.py @@ -14,6 +14,7 @@ from dnastack.client.workbench.storage.models import StorageAccount, StorageListOptions, Provider REINDEX_POLL_INTERVAL_SECONDS = 10 +REINDEX_DEFAULT_TIMEOUT_SECONDS = 3600 def _wait_for_indexing(client: StorageClient, @@ -22,9 +23,6 @@ def _wait_for_indexing(client: StorageClient, timeout: int, last_completed_at: Optional[str]) -> StorageAccount: """Poll until indexing stops, and fail unless it recorded a completed run after `last_completed_at`.""" - if account.indexing_status is None: - raise click.ClickException('This Workbench does not report indexing status, so --wait cannot ' - 'tell when indexing finishes.') deadline = monotonic() + timeout while account.indexing_status == 'RUNNING': remaining = deadline - monotonic() @@ -165,15 +163,16 @@ def get_storage_accounts(context: Optional[str], ArgumentSpec( name='wait', arg_names=['--wait'], - help='Wait for indexing to finish, then print the account. Exits non-zero if indexing did not complete.', + help='Wait for indexing to finish, then print the account. Exits non-zero if indexing did not ' + 'complete. Indexing keeps running if the wait ends early.', type=bool, ), ArgumentSpec( name='timeout', arg_names=['--timeout'], - help='With --wait, how many seconds to wait before giving up. Indexing keeps running.', + help=f'With --wait, how many seconds to wait before giving up ' + f'(default {REINDEX_DEFAULT_TIMEOUT_SECONDS}).', type=int, - default=3600, ), NAMESPACE_ARG, CONTEXT_ARG, @@ -185,7 +184,7 @@ def reindex_storage_account(context: Optional[str], namespace: Optional[str], storage_id: str, wait: bool = False, - timeout: int = 3600): + timeout: Optional[int] = None): """ Re-scan a storage account for sample files @@ -193,11 +192,21 @@ def reindex_storage_account(context: Optional[str], also removes sample files whose objects are no longer in the bucket. Requires the workbench.storage.update permission (namespace admin). """ - if timeout < 0: + if timeout is not None and not wait: + raise click.BadParameter('only applies with --wait.', param_hint="'--timeout'") + if timeout is not None and timeout < 0: raise click.BadParameter('must be zero or more seconds.', param_hint="'--timeout'") client = get_storage_client(context, endpoint_id, namespace) - last_completed_at = client.get_storage_account(storage_id).last_indexed_at if wait else None + if not wait: + click.echo(to_json(normalize(client.reindex_storage_account(storage_id)))) + return + + before = client.get_storage_account(storage_id) + if before.indexing_status is None: + raise click.ClickException('This Workbench does not report indexing status, so --wait cannot ' + 'tell when indexing finishes.') account = client.reindex_storage_account(storage_id) - if wait: - account = _wait_for_indexing(client, storage_id, account, timeout, last_completed_at) + account = _wait_for_indexing(client, storage_id, account, + REINDEX_DEFAULT_TIMEOUT_SECONDS if timeout is None else timeout, + before.last_indexed_at) click.echo(to_json(normalize(account))) diff --git a/tests/unit/cli/commands/workbench/storage/test_commands.py b/tests/unit/cli/commands/workbench/storage/test_commands.py index 9cfddd8c..d3a5884a 100644 --- a/tests/unit/cli/commands/workbench/storage/test_commands.py +++ b/tests/unit/cli/commands/workbench/storage/test_commands.py @@ -93,7 +93,7 @@ def test_wait_gives_up_after_the_timeout_and_exits_non_zero(self, mock_sleep): mock_sleep.assert_not_called() @responses.activate - def test_wait_fails_when_workbench_does_not_report_indexing_status(self): + def test_wait_fails_before_reindexing_when_workbench_does_not_report_indexing_status(self): responses.get(_ACCOUNT_URL, json={"id": "storage-1"}) responses.post(f'{_ACCOUNT_URL}/reindex', json={"id": "storage-1"}) @@ -102,6 +102,26 @@ def test_wait_fails_when_workbench_does_not_report_indexing_status(self): assert result.exit_code == 1 assert "indexing status" in result.stderr assert result.stdout == "" + assert [c.request.method for c in responses.calls] == ['GET'] + + @responses.activate + def test_wait_succeeds_without_polling_when_indexing_finished_before_the_response(self): + responses.get(_ACCOUNT_URL, json=_account('IDLE')) + responses.post(f'{_ACCOUNT_URL}/reindex', json=_account('IDLE', last_indexed_at='2026-11-16T09:30:00Z')) + + result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--wait']) + + assert result.exit_code == 0, result.stderr + assert [c.request.method for c in responses.calls] == ['GET', 'POST'] + assert json.loads(result.stdout)["last_indexed_at"] == "2026-11-16T09:30:00Z" + + @responses.activate + def test_rejects_timeout_without_wait(self): + result = self.runner.invoke(self.group, ['reindex', 'storage-1', '--timeout', '60']) + + assert result.exit_code == 2 + assert len(responses.calls) == 0 + assert "--wait" in result.stderr @responses.activate @patch('dnastack.cli.commands.workbench.storage.commands.sleep') From 9d7315a904e26201110dc05d82f422112e3559f7 Mon Sep 17 00:00:00 2001 From: Kevin Hong Date: Mon, 16 Nov 2026 21:28:31 +0800 Subject: [PATCH 6/6] [CU-1k5fpda7hax] Check every storage id method rejects unescapable ids in one test The reindex and get/update/delete rejection cases lived in two tests that each covered part of the method and id combinations; one parametrized test covers all of them. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_011GSYqHzB6MG7utV8vYMnZX --- .../workbench/storage/test_storage_client.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/unit/client/workbench/storage/test_storage_client.py b/tests/unit/client/workbench/storage/test_storage_client.py index 196f6f8d..7c480acf 100644 --- a/tests/unit/client/workbench/storage/test_storage_client.py +++ b/tests/unit/client/workbench/storage/test_storage_client.py @@ -28,11 +28,6 @@ def test_posts_reindex_without_a_body_and_returns_the_running_account(self): assert account.indexing_status == "RUNNING" assert account.last_indexed_at == "2026-11-15T12:00:00Z" - @pytest.mark.parametrize('storage_account_id', ['', '.', '..']) - def test_rejects_ids_that_cannot_be_one_path_segment(self, storage_account_id): - with pytest.raises(ValueError): - _make_client().reindex_storage_account(storage_account_id) - _ODD_ID = 'a/../b#c?d' _ODD_SEGMENT = 'a%2F..%2Fb%23c%3Fd' @@ -57,11 +52,13 @@ def test_escapes_the_id(self, method, call, suffix): assert responses.calls[0].request.url == expected_url + @pytest.mark.parametrize('storage_account_id', ['', '.', '..']) @pytest.mark.parametrize('call', [ - lambda client: client.get_storage_account('..'), - lambda client: client.update_storage_account('', StorageAccount(), 'etag-1'), - lambda client: client.delete_storage_account('.'), + lambda client, account_id: client.get_storage_account(account_id), + lambda client, account_id: client.update_storage_account(account_id, StorageAccount(), 'etag-1'), + lambda client, account_id: client.delete_storage_account(account_id), + lambda client, account_id: client.reindex_storage_account(account_id), ]) - def test_rejects_ids_that_cannot_be_one_path_segment(self, call): + def test_rejects_ids_that_cannot_be_one_path_segment(self, call, storage_account_id): with pytest.raises(ValueError): - call(_make_client()) + call(_make_client(), storage_account_id)