diff --git a/fastapi_startkit/pyproject.toml b/fastapi_startkit/pyproject.toml index 6358cf7b..b886b9ef 100644 --- a/fastapi_startkit/pyproject.toml +++ b/fastapi_startkit/pyproject.toml @@ -108,6 +108,7 @@ dev = [ "langchain>=1.0.0", "langchain-core>=1.0.0", "pyright>=1.1.411", + "boto3>=1.43.93", ] diff --git a/fastapi_startkit/src/fastapi_startkit/storage/drivers/local.py b/fastapi_startkit/src/fastapi_startkit/storage/drivers/local.py index b932a0f0..091f7523 100644 --- a/fastapi_startkit/src/fastapi_startkit/storage/drivers/local.py +++ b/fastapi_startkit/src/fastapi_startkit/storage/drivers/local.py @@ -115,13 +115,22 @@ def make_file_path_if_not_exists(self, file_path): return False def get_files(self, directory=""): + """List the files directly under ``directory`` (non-recursive). + + Directory entries are excluded and an empty or nonexistent directory + yields ``[]``. Each ``File`` is named by its bare filename and carries + the file's content. + """ file_path = self.get_path(directory) + if not os.path.isdir(file_path): + return [] + files = [] for f in os.listdir(file_path): if not isfile(join(file_path, f)): continue - files.append(File(self.get(f), f)) + files.append(File(self.get(join(directory, f)), f)) return files diff --git a/fastapi_startkit/src/fastapi_startkit/storage/drivers/s3.py b/fastapi_startkit/src/fastapi_startkit/storage/drivers/s3.py index a5ca66fe..d76a1e21 100644 --- a/fastapi_startkit/src/fastapi_startkit/storage/drivers/s3.py +++ b/fastapi_startkit/src/fastapi_startkit/storage/drivers/s3.py @@ -148,17 +148,25 @@ def make_file_path_if_not_exists(self, file_path): return False def get_files(self, directory=None): + """List the files directly under ``directory`` (non-recursive). + + Directory entries are excluded and an empty or nonexistent directory + yields ``[]``. Each ``File`` is named by its bare filename, matching + the local driver's contract. + """ bucket = self.get_resource().Bucket(self.get_bucket()) - if directory: - objects = bucket.objects.all().filter(Prefix=directory) + prefix = f"{directory.rstrip('/')}/" if directory else "" + if prefix: + objects = bucket.objects.filter(Prefix=prefix) else: objects = bucket.objects.all() files = [] - for my_bucket_object in objects.all(): - if "/" not in my_bucket_object.key: - files.append(File(my_bucket_object, my_bucket_object.key)) + for my_bucket_object in objects: + relative = my_bucket_object.key[len(prefix):] + if relative and "/" not in relative: + files.append(File(my_bucket_object, relative)) return files diff --git a/fastapi_startkit/tests/storage/test_local_driver.py b/fastapi_startkit/tests/storage/test_local_driver.py index de2b9bcb..351f3709 100644 --- a/fastapi_startkit/tests/storage/test_local_driver.py +++ b/fastapi_startkit/tests/storage/test_local_driver.py @@ -112,6 +112,50 @@ def test_get_files_skips_subdirectories(self, driver, tmp_path): files = driver.get_files("") assert [f.name() for f in files] == ["file.txt"] + def test_get_files_root_returns_real_content(self, driver, tmp_path): + storage = tmp_path / "storage" + storage.mkdir(parents=True, exist_ok=True) + (storage / "root.txt").write_text("root content") + files = driver.get_files("") + assert [f.stream() for f in files] == ["root content"] + + def test_get_files_in_subdirectory_returns_real_content(self, driver, tmp_path): + audio = tmp_path / "storage" / "audio" + audio.mkdir(parents=True) + (audio / "tts-1.txt").write_text("speech one") + (audio / "tts-2.txt").write_text("speech two") + + files = driver.get_files("audio") + + by_name = {f.name(): f.stream() for f in files} + assert by_name == {"tts-1.txt": "speech one", "tts-2.txt": "speech two"} + + def test_get_files_directory_with_trailing_slash(self, driver, tmp_path): + audio = tmp_path / "storage" / "audio" + audio.mkdir(parents=True) + (audio / "a.txt").write_text("a") + + files = driver.get_files("audio/") + + assert [(f.name(), f.stream()) for f in files] == [("a.txt", "a")] + + def test_get_files_nonexistent_directory_returns_empty_list(self, driver): + assert driver.get_files("does-not-exist") == [] + + def test_get_files_empty_directory_returns_empty_list(self, driver, tmp_path): + (tmp_path / "storage" / "empty").mkdir(parents=True) + assert driver.get_files("empty") == [] + + def test_get_files_in_subdirectory_skips_nested_directories(self, driver, tmp_path): + audio = tmp_path / "storage" / "audio" + (audio / "nested").mkdir(parents=True) + (audio / "keep.txt").write_text("kept") + (audio / "nested" / "skip.txt").write_text("skipped") + + files = driver.get_files("audio") + + assert [f.name() for f in files] == ["keep.txt"] + class TestLocalDriverPathResolution: def test_get_path_joins_relative_root_to_base_path(self, tmp_path): @@ -254,6 +298,31 @@ def test_fake_driver_context_manager(self): root = fake._root assert not os.path.exists(root) + def test_fake_driver_get_files_in_subdirectory(self, fake): + fake.put("audio/one.txt", "first") + fake.put("audio/two.txt", "second") + + files = fake.get_files("audio") + + by_name = {f.name(): f.stream() for f in files} + assert by_name == {"one.txt": "first", "two.txt": "second"} + + def test_fake_driver_get_files_nonexistent_directory(self, fake): + assert fake.get_files("missing") == [] + + def test_fake_driver_get_files_skips_nested_directories(self, fake): + fake.put("docs/keep.txt", "kept") + fake.put("docs/nested/skip.txt", "skipped") + + files = fake.get_files("docs") + + assert [f.name() for f in files] == ["keep.txt"] + + def test_fake_driver_assert_count_in_subdirectory(self, fake): + fake.put("uploads/a.txt", "a") + fake.put("uploads/b.txt", "b") + fake.assert_count(2, "uploads") + def test_fake_driver_isolated_between_instances(self): app = MagicMock() app.base_path = "/fake" diff --git a/fastapi_startkit/tests/storage/test_s3_driver.py b/fastapi_startkit/tests/storage/test_s3_driver.py index 90c10ed4..1570d278 100644 --- a/fastapi_startkit/tests/storage/test_s3_driver.py +++ b/fastapi_startkit/tests/storage/test_s3_driver.py @@ -261,6 +261,69 @@ def test_move_does_not_delete_source_when_copy_fails(self, driver): mock_delete.assert_not_called() +# --------------------------------------------------------------------------- +# get_files +# --------------------------------------------------------------------------- + + +class _Obj: + def __init__(self, key): + self.key = key + + +BUCKET_KEYS = ["backups/a.dump", "backups/b.dump", "backups/nested/deep.dump", "root.txt", "audio/x/y.mp3"] + + +def _stub_listing(mock_resource): + """Make the mocked bucket answer .objects.all() / .filter(Prefix=...) like S3 would.""" + objects = [_Obj(key) for key in BUCKET_KEYS] + bucket = mock_resource.Bucket.return_value + bucket.objects.all.return_value = objects + bucket.objects.filter.side_effect = lambda Prefix: [o for o in objects if o.key.startswith(Prefix)] + return bucket + + +class TestS3DriverGetFiles: + def test_root_listing_keeps_only_root_level_keys(self, driver, mock_resource): + _stub_listing(mock_resource) + files = driver.get_files() + assert [f.name() for f in files] == ["root.txt"] + + def test_directory_listing_returns_files_directly_under_prefix(self, driver, mock_resource): + _stub_listing(mock_resource) + files = driver.get_files("backups") + assert sorted(f.name() for f in files) == ["a.dump", "b.dump"] + + def test_directory_listing_filters_with_slash_terminated_prefix(self, driver, mock_resource): + bucket = _stub_listing(mock_resource) + driver.get_files("backups") + bucket.objects.filter.assert_called_once_with(Prefix="backups/") + + def test_trailing_slash_directory_is_equivalent(self, driver, mock_resource): + _stub_listing(mock_resource) + files = driver.get_files("backups/") + assert sorted(f.name() for f in files) == ["a.dump", "b.dump"] + + def test_partial_name_prefix_is_not_a_directory_match(self, driver, mock_resource): + _stub_listing(mock_resource) + assert driver.get_files("back") == [] + + def test_directory_with_only_nested_subdirectories_is_empty(self, driver, mock_resource): + _stub_listing(mock_resource) + assert driver.get_files("audio") == [] + + def test_nonexistent_directory_returns_empty_list(self, driver, mock_resource): + _stub_listing(mock_resource) + assert driver.get_files("nope") == [] + + def test_directory_marker_key_is_skipped(self, driver, mock_resource): + objects = [_Obj("backups/"), _Obj("backups/a.dump")] + bucket = mock_resource.Bucket.return_value + bucket.objects.filter.side_effect = lambda Prefix: [o for o in objects if o.key.startswith(Prefix)] + files = driver.get_files("backups") + assert [f.name() for f in files] == ["a.dump"] + + # --------------------------------------------------------------------------- # Connection caching # --------------------------------------------------------------------------- diff --git a/fastapi_startkit/tests/storage/test_s3_minio_integration.py b/fastapi_startkit/tests/storage/test_s3_minio_integration.py new file mode 100644 index 00000000..8a9e215d --- /dev/null +++ b/fastapi_startkit/tests/storage/test_s3_minio_integration.py @@ -0,0 +1,123 @@ +"""Integration tests for S3Driver.get_files against a real S3 API (MinIO). + +These run against a live MinIO server (issue #218). They are skipped when +boto3 is not installed or no server is reachable, so the suite stays green +in environments without MinIO. + +Override the target with MINIO_ENDPOINT / MINIO_ACCESS_KEY / MINIO_SECRET_KEY. +""" + +import os +import uuid +from unittest.mock import MagicMock + +import pytest + +boto3 = pytest.importorskip("boto3") +import botocore.config # noqa: E402 + +from fastapi_startkit.storage.drivers.s3 import S3Driver # noqa: E402 + +ENDPOINT = os.environ.get("MINIO_ENDPOINT", "http://localhost:9002") +ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY", "minio") +SECRET_KEY = os.environ.get("MINIO_SECRET_KEY", "minio123") + +KEYS = { + "root.txt": b"root content", + "backups/a.dump": b"dump a", + "backups/b.dump": b"dump b", + "backups/nested/deep.dump": b"deep", + "audio/x/y.mp3": b"audio bytes", +} + + +def _client(): + session = boto3.Session( + aws_access_key_id=ACCESS_KEY, + aws_secret_access_key=SECRET_KEY, + region_name="us-east-1", + ) + return session.client( + "s3", + endpoint_url=ENDPOINT, + config=botocore.config.Config( + s3={"addressing_style": "path"}, + connect_timeout=2, + retries={"max_attempts": 1}, + ), + ) + + +def _minio_reachable(): + try: + _client().list_buckets() + return True + except Exception: + return False + + +pytestmark = pytest.mark.skipif(not _minio_reachable(), reason=f"MinIO not reachable at {ENDPOINT}") + + +@pytest.fixture(scope="module") +def bucket_name(): + client = _client() + name = f"fsk-get-files-{uuid.uuid4().hex[:12]}" + client.create_bucket(Bucket=name) + for key, body in KEYS.items(): + client.put_object(Bucket=name, Key=key, Body=body) + + yield name + + listing = client.list_objects_v2(Bucket=name).get("Contents", []) + for obj in listing: + client.delete_object(Bucket=name, Key=obj["Key"]) + client.delete_bucket(Bucket=name) + + +@pytest.fixture +def driver(bucket_name): + d = S3Driver(MagicMock()) + d.set_options( + { + "bucket": bucket_name, + "key": ACCESS_KEY, + "secret": SECRET_KEY, + "region": "us-east-1", + "endpoint": ENDPOINT, + "use_path_style_endpoint": True, + } + ) + return d + + +class TestGetFilesAgainstMinio: + def test_root_listing_returns_only_root_level_files(self, driver): + files = driver.get_files() + assert [f.name() for f in files] == ["root.txt"] + + def test_one_level_deep_directory(self, driver): + files = driver.get_files("backups") + assert sorted(f.name() for f in files) == ["a.dump", "b.dump"] + + def test_trailing_slash_is_equivalent(self, driver): + files = driver.get_files("backups/") + assert sorted(f.name() for f in files) == ["a.dump", "b.dump"] + + def test_two_levels_deep_directory(self, driver): + files = driver.get_files("backups/nested") + assert [f.name() for f in files] == ["deep.dump"] + + def test_directory_containing_only_subdirectories_is_empty(self, driver): + assert driver.get_files("audio") == [] + + def test_nonexistent_directory_returns_empty_list(self, driver): + assert driver.get_files("does-not-exist") == [] + + def test_partial_name_prefix_is_not_a_directory_match(self, driver): + assert driver.get_files("back") == [] + + def test_listed_objects_point_at_the_real_keys(self, driver): + files = driver.get_files("backups") + keys = sorted(f.stream().key for f in files) + assert keys == ["backups/a.dump", "backups/b.dump"] diff --git a/fastapi_startkit/uv.lock b/fastapi_startkit/uv.lock index c47982b9..a8f1ce48 100644 --- a/fastapi_startkit/uv.lock +++ b/fastapi_startkit/uv.lock @@ -99,6 +99,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/d7/8fb3044eaef08a310acfe23dae9a8e2e07d305edc29a53497e52bc76eca7/asyncpg-0.31.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3", size = 706062, upload-time = "2025-11-24T23:26:44.086Z" }, ] +[[package]] +name = "boto3" +version = "1.43.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/37/7a09d8320685b3b8c8a014e392e7595025d00965c46c5f410797905fab7a/boto3-1.43.93.tar.gz", hash = "sha256:196bfc8b4c9cd5505f9f7b963e30956db3a00fd47e20dd0ee3574a243c1fb212", size = 112752, upload-time = "2026-09-11T19:23:03.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/68/f8f661b9e68daba4f775bfa1e750732ec52fc89b32d55f300aef530e1d62/boto3-1.43.93-py3-none-any.whl", hash = "sha256:3c948fe231490d446bf90bf3322d1452632107329d3683b37d88b7399bf481a0", size = 140022, upload-time = "2026-09-11T19:23:01.995Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/a0/2ce10897323d67dd85de6190fdee159013a75741d1dc48b74d4815ec0592/botocore-1.43.93.tar.gz", hash = "sha256:82da355d18a7f784347b00444be33942834651f31b6c5ffef49999cd47364c5e", size = 16103276, upload-time = "2026-09-11T19:22:58.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/7e/f858c401f32d980f924c8f8328b62fab463313834a9ad2326f44613b97ec/botocore-1.43.93-py3-none-any.whl", hash = "sha256:3ca57bb5d26d88b554a74de708a5c991f45306436c91aacca931252d1d4d54ff", size = 15794156, upload-time = "2026-09-11T19:22:55.964Z" }, +] + [[package]] name = "certifi" version = "2026.4.22" @@ -527,7 +555,7 @@ wheels = [ [[package]] name = "fastapi-startkit" -version = "0.51.0" +version = "0.56.0" source = { editable = "." } dependencies = [ { name = "cleo" }, @@ -575,6 +603,7 @@ dev = [ { name = "aiomysql" }, { name = "aiosqlite" }, { name = "asyncpg" }, + { name = "boto3" }, { name = "dumpdie" }, { name = "faker" }, { name = "fastapi", extra = ["standard"] }, @@ -620,6 +649,7 @@ dev = [ { name = "aiomysql", specifier = ">=0.2.0" }, { name = "aiosqlite", specifier = ">=0.22.1" }, { name = "asyncpg", specifier = ">=0.29.0" }, + { name = "boto3", specifier = ">=1.43.93" }, { name = "dumpdie", specifier = ">=1.5.0" }, { name = "faker", specifier = ">=40.13.0" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.124.4" }, @@ -922,6 +952,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -1871,6 +1910,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, +] + [[package]] name = "secretstorage" version = "3.5.0"