diff --git a/README.md b/README.md index da7afee..507071c 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Define connection ─▶ Author SQL (with :bind params) ─▶ Attach auth ─ ``` 1. **Connect** to your Oracle database with securely stored, encrypted credentials. -2. **Author** a `SELECT` query using named bind parameters (`:param_name`) in a rich SQL editor. The wizard detects parameters as you type and requests temporary sample values before previewing the results. +2. **Author** a `SELECT` query using named bind parameters (`:param_name`) in a rich SQL editor. The wizard detects parameters as you type, requests temporary sample values before previewing, and supports fixed or dynamic date defaults (`today` and `yesterday`). Snapshot endpoints require a default for every bind so scheduled refreshes can execute without request inputs. Scheduler defaults do not make required parameters optional for live HTTP requests; callers must supply required dates as either `YYYY-MM-DD` or `DD-MM-YYYY`. 3. **Secure** the endpoint by attaching a Bearer token, Basic Auth, or API key policy. 4. **Choose** a data strategy: serve results **live** on each request, or from a **scheduled snapshot** cache. 5. **Publish** a versioned endpoint under `/api/v1/data/*` that resolves dynamically — no service restart needed. @@ -73,16 +73,16 @@ QueryGateway is organized into five admin modules, all driven from the React adm | Module | What it does | |--------|--------------| | **Connections** | Create, edit, test, and delete Oracle database connections. Credentials are encrypted at rest; pool sizing and timeouts are configurable. Uses `python-oracledb`; thin mode needs no native client, while the backend Docker image includes Oracle Instant Client 19.32 for thick mode. | -| **API Creation Wizard** | A multi-step wizard that turns a parameterized SQL query into a deployable GET endpoint: pick a connection, author SQL with a rich editor, supply preview-only sample values for detected bind parameters, preview sample rows and inferred schema, map/rename output columns, attach an auth method, and select a data strategy. | -| **Authentication** | Manage per-endpoint auth methods — Bearer token (JWT), Basic Auth, and API key. Tokens are issued/verified with `PyJWT`; credentials are hashed with `bcrypt`. When an auth method is attached to an endpoint, middleware enforces it on every request; endpoints with no auth method attached are served publicly. | -| **Scheduling & Snapshots** | Schedule query refreshes with the in-process APScheduler (cron or interval). Run now, pause/resume, and enable/disable jobs. Results are cached as PostgreSQL JSONB snapshots and served with freshness metadata. Schedule definitions are persisted in the app database; the active APScheduler jobs run in-memory and are (re)registered when a schedule is created, updated, or resumed. | +| **API Creation Wizard** | A multi-step wizard that turns a parameterized SQL query into a deployable GET endpoint: pick a connection, author SQL with a rich editor, supply preview-only sample values for detected bind parameters, configure fixed values, explicit SQL `NULL` values for optional binds, or dynamic date defaults, preview sample rows and inferred schema, map/rename output columns, attach an auth method, and select a data strategy. | +| **Authentication** | Manage per-endpoint auth methods — Bearer token (JWT), Basic Auth, and API key. Tokens are issued/verified with `PyJWT`; credentials are hashed with `bcrypt`. Every `/api/v1/data/*` request is authenticated; endpoints without a dedicated method require the platform admin Bearer token. | +| **Scheduling & Snapshots** | Schedule query refreshes with friendly hourly, daily, weekly, or monthly calendar controls, an advanced custom-cron option, or a fixed interval. Run now, pause/resume, and enable/disable jobs. Results are cached as PostgreSQL JSONB snapshots and served with freshness metadata. Schedule definitions are persisted in the app database, active APScheduler jobs are restored on API startup, and deleting a schedule preserves its job-run and snapshot history. Deleting an endpoint removes its schedule and snapshots while retaining orphaned job-run audit records. | | **Settings & Health** | Configure runtime settings (base URL/port, logging level, query timeouts, CORS/rate-limit inputs) and view a health dashboard covering API, PostgreSQL, Oracle connectivity, scheduler status, and recent job outcomes. | ### Security by Default - **SQL injection resistant** — user-defined SQL runs only through SQLAlchemy `text()` with named bind parameters. Request values are never concatenated into SQL strings, and bind values are validated through typed schemas before execution. - **Encrypted credentials** — Oracle connection secrets are encrypted at rest using an environment-provided key. -- **Per-endpoint authentication** — attach a Bearer token, Basic Auth, or API key policy to an endpoint and it is enforced on every request. Endpoints published without an auth method are public, so assign one to any endpoint that should be protected. +- **Mandatory data authentication** — attach a Bearer token, Basic Auth, or API key policy to an endpoint. If no dedicated method is attached, the endpoint requires the platform admin Bearer token; anonymous data access is never allowed. - **Structured, redacted logging** — `structlog` emits JSON logs with correlation fields (`request_id`, `user`, `endpoint`, `status`, `duration_ms`); credentials and tokens are redacted before emission. ### Two API Surfaces diff --git a/backend/alembic/versions/b2d18f4a6c73_preserve_job_runs_when_deleting_schedules.py b/backend/alembic/versions/b2d18f4a6c73_preserve_job_runs_when_deleting_schedules.py new file mode 100644 index 0000000..a6f0f4f --- /dev/null +++ b/backend/alembic/versions/b2d18f4a6c73_preserve_job_runs_when_deleting_schedules.py @@ -0,0 +1,60 @@ +"""Preserve job runs when deleting schedules. + +Revision ID: b2d18f4a6c73 +Revises: a8307fb20816 +Create Date: 2026-08-30 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "b2d18f4a6c73" +down_revision: str | None = "a8307fb20816" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.drop_constraint( + "job_runs_schedule_id_fkey", "job_runs", type_="foreignkey" + ) + op.alter_column( + "job_runs", + "schedule_id", + existing_type=sa.UUID(), + nullable=True, + ) + op.create_foreign_key( + "job_runs_schedule_id_fkey", + "job_runs", + "schedules", + ["schedule_id"], + ["id"], + ondelete="SET NULL", + ) + + +def downgrade() -> None: + op.drop_constraint( + "job_runs_schedule_id_fkey", "job_runs", type_="foreignkey" + ) + # Schedules deleted after this migration cannot be reconstructed. Remove + # only their orphaned audit rows so the original NOT NULL contract can be + # restored; snapshots remain and their job_run_id becomes NULL. + op.execute("DELETE FROM job_runs WHERE schedule_id IS NULL") + op.alter_column( + "job_runs", + "schedule_id", + existing_type=sa.UUID(), + nullable=False, + ) + op.create_foreign_key( + "job_runs_schedule_id_fkey", + "job_runs", + "schedules", + ["schedule_id"], + ["id"], + ondelete="RESTRICT", + ) diff --git a/backend/alembic/versions/c7e91a4f2d60_preserve_job_runs_when_deleting_endpoints.py b/backend/alembic/versions/c7e91a4f2d60_preserve_job_runs_when_deleting_endpoints.py new file mode 100644 index 0000000..0ad0b3b --- /dev/null +++ b/backend/alembic/versions/c7e91a4f2d60_preserve_job_runs_when_deleting_endpoints.py @@ -0,0 +1,56 @@ +"""Preserve job runs when deleting endpoints. + +Revision ID: c7e91a4f2d60 +Revises: b2d18f4a6c73 +Create Date: 2026-08-30 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "c7e91a4f2d60" +down_revision: str | None = "b2d18f4a6c73" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.drop_constraint("job_runs_endpoint_id_fkey", "job_runs", type_="foreignkey") + op.alter_column( + "job_runs", + "endpoint_id", + existing_type=sa.UUID(), + nullable=True, + ) + op.create_foreign_key( + "job_runs_endpoint_id_fkey", + "job_runs", + "endpoints", + ["endpoint_id"], + ["id"], + ondelete="SET NULL", + ) + + +def downgrade() -> None: + op.drop_constraint("job_runs_endpoint_id_fkey", "job_runs", type_="foreignkey") + # Endpoints deleted after this migration cannot be reconstructed. Remove + # only their orphaned audit rows so the original NOT NULL contract can be + # restored; any surviving snapshots already reference other job runs. + op.execute("DELETE FROM job_runs WHERE endpoint_id IS NULL") + op.alter_column( + "job_runs", + "endpoint_id", + existing_type=sa.UUID(), + nullable=False, + ) + op.create_foreign_key( + "job_runs_endpoint_id_fkey", + "job_runs", + "endpoints", + ["endpoint_id"], + ["id"], + ondelete="RESTRICT", + ) diff --git a/backend/app/main.py b/backend/app/main.py index a30e5c7..5ac12cf 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -101,7 +101,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None]: debug=settings.debug, ) _init_oracle_client() - start_scheduler() + await start_scheduler() yield stop_scheduler() log.info("application_shutdown") diff --git a/backend/app/models/endpoint.py b/backend/app/models/endpoint.py index e9427c1..cb5a4c2 100644 --- a/backend/app/models/endpoint.py +++ b/backend/app/models/endpoint.py @@ -51,12 +51,9 @@ class ApiEndpoint(UUIDPrimaryKeyMixin, TimestampMixin, Base): auth_method_id: Mapped[uuid.UUID | None] = mapped_column( ForeignKey("auth_methods.id", ondelete="SET NULL"), nullable=True ) - # When no auth method is attached, an endpoint is served unauthenticated - # (public). This flag forces that to be an explicit, deliberate choice: - # the admin API rejects an endpoint that has no auth method unless this - # is set to True (see app.schemas.endpoint). Existing rows default to - # False so they keep being served (the data plane logs a warning) but - # any future edit must opt in explicitly. + # Legacy-named compatibility flag. When no endpoint-specific auth method is + # attached, True opts into platform-admin Bearer authentication. The data + # plane never serves anonymous requests. allow_unauthenticated: Mapped[bool] = mapped_column( Boolean, default=False, nullable=False, server_default=text("false") ) diff --git a/backend/app/models/job_run.py b/backend/app/models/job_run.py index 63c0ebe..27d9f4f 100644 --- a/backend/app/models/job_run.py +++ b/backend/app/models/job_run.py @@ -30,17 +30,15 @@ class JobRun(UUIDPrimaryKeyMixin, Base): id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) - schedule_id: Mapped[uuid.UUID] = mapped_column( - ForeignKey("schedules.id", ondelete="RESTRICT"), nullable=False + schedule_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("schedules.id", ondelete="SET NULL"), nullable=True ) - endpoint_id: Mapped[uuid.UUID] = mapped_column( - ForeignKey("endpoints.id", ondelete="RESTRICT"), nullable=False + endpoint_id: Mapped[uuid.UUID | None] = mapped_column( + ForeignKey("endpoints.id", ondelete="SET NULL"), nullable=True ) started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - finished_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True - ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) status: Mapped[JobRunStatus] = mapped_column( SAEnum(JobRunStatus, name="job_run_status"), nullable=False, diff --git a/backend/app/routers/endpoints.py b/backend/app/routers/endpoints.py index 51e2537..1a26dad 100644 --- a/backend/app/routers/endpoints.py +++ b/backend/app/routers/endpoints.py @@ -21,15 +21,18 @@ from app.dependencies import get_db from app.repositories.connection import ConnectionRepository from app.repositories.endpoint import EndpointRepository +from app.repositories.schedule import ScheduleRepository from app.schemas.endpoint import ( EndpointCreate, EndpointResponse, EndpointUpdate, PublicEndpointError, + SnapshotConfigurationError, SqlPreviewRequest, SqlPreviewResponse, ) from app.services.endpoint import EndpointService +from app.services.scheduler import remove_schedule_job log = structlog.get_logger() @@ -72,14 +75,12 @@ async def create_endpoint( ) -> EndpointResponse: try: result = await svc.create_endpoint(payload) - except PublicEndpointError as exc: + except (PublicEndpointError, SnapshotConfigurationError) as exc: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc) ) from exc except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, detail=str(exc) - ) from exc + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc await db.commit() return result @@ -95,9 +96,7 @@ async def get_endpoint( ) -> EndpointResponse: result = await svc.get_endpoint(endpoint_id) if result is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Endpoint not found." - ) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Endpoint not found.") return result @@ -114,18 +113,14 @@ async def update_endpoint( ) -> EndpointResponse: try: result = await svc.update_endpoint(endpoint_id, payload) - except PublicEndpointError as exc: + except (PublicEndpointError, SnapshotConfigurationError) as exc: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc) ) from exc except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, detail=str(exc) - ) from exc + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc if result is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Endpoint not found." - ) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Endpoint not found.") await db.commit() return result @@ -140,12 +135,14 @@ async def delete_endpoint( db: AsyncSession = Depends(get_db), svc: EndpointService = Depends(_service), ) -> None: + schedule = await ScheduleRepository(db).get_by_endpoint_id(endpoint_id) + schedule_id = schedule.id if schedule is not None else None deleted = await svc.delete_endpoint(endpoint_id) if not deleted: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Endpoint not found." - ) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Endpoint not found.") await db.commit() + if schedule_id is not None: + remove_schedule_job(schedule_id) @router.post( @@ -164,6 +161,4 @@ async def preview_sql( try: return await svc.preview_sql(payload) except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc) - ) from exc + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc diff --git a/backend/app/routers/schedules.py b/backend/app/routers/schedules.py index 07f2bc7..90ce0dd 100644 --- a/backend/app/routers/schedules.py +++ b/backend/app/routers/schedules.py @@ -29,6 +29,7 @@ from app.repositories.job_run import JobRunRepository from app.repositories.schedule import ScheduleRepository from app.repositories.snapshot import SnapshotRepository +from app.schemas.endpoint import SnapshotConfigurationError from app.schemas.schedule import ( JobRunResponse, ScheduleCreate, @@ -38,6 +39,7 @@ SnapshotResponse, ) from app.services.schedule import ScheduleService +from app.services.scheduler import remove_schedule_job log = structlog.get_logger() @@ -85,6 +87,10 @@ async def create_schedule( ) -> ScheduleResponse: try: result = await svc.create_schedule(payload) + except SnapshotConfigurationError as exc: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc) + ) from exc except ValueError as exc: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=str(exc) @@ -151,6 +157,7 @@ async def delete_schedule( status_code=status.HTTP_404_NOT_FOUND, detail="Schedule not found." ) await db.commit() + remove_schedule_job(schedule_id) # ── Control actions ────────────────────────────────────────────────────────── diff --git a/backend/app/schemas/endpoint.py b/backend/app/schemas/endpoint.py index c4e5b96..9c14081 100644 --- a/backend/app/schemas/endpoint.py +++ b/backend/app/schemas/endpoint.py @@ -9,10 +9,11 @@ import re import uuid +from collections.abc import Mapping from datetime import datetime -from typing import Self +from typing import Literal, Self -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator from app.models.endpoint import DataStrategy @@ -33,18 +34,21 @@ # Valid path segment: lowercase alphanumeric, hyphens, underscores, slashes. _PATH_RE = re.compile(r"^[a-z0-9][a-z0-9\-_/]*$") -# Message shown when an endpoint would be served with no auth method and -# without an explicit opt-in to public access (M1 — silent public endpoints). +# Message shown when an endpoint has no dedicated auth method and has not opted +# into the platform-admin Bearer fallback. Anonymous data access is forbidden. PUBLIC_OPT_IN_MESSAGE = ( - "Endpoint has no auth_method_id. Attach an auth method to protect it, or " - "set allow_unauthenticated=true to deliberately publish it as a PUBLIC " - "(unauthenticated) endpoint." + "Endpoint has no auth_method_id. Attach an endpoint auth method, or set " + "allow_unauthenticated=true to use platform-admin Bearer authentication. " + "Anonymous data access is not supported." ) class PublicEndpointError(ValueError): - """Raised when a write would leave an endpoint unauthenticated without an - explicit ``allow_unauthenticated`` opt-in. Routers surface this as 422.""" + """Raised when an endpoint has no configured authentication path.""" + + +class SnapshotConfigurationError(ValueError): + """Raised when a snapshot endpoint cannot execute without request inputs.""" def extract_bind_params(sql: str) -> list[str]: @@ -75,6 +79,20 @@ class ParamDescriptor(BaseModel): ) required: bool = True default: str | int | float | bool | None = None + default_is_null: bool = Field( + False, + description=( + "Use an explicit SQL NULL when this optional parameter is omitted. " + "This is distinct from having no configured default." + ), + ) + default_expression: Literal["today", "yesterday"] | None = Field( + None, + description=( + "Dynamic date default evaluated from the application server date " + "when the query executes." + ), + ) description: str | None = None max_length: int | None = Field( None, @@ -84,11 +102,77 @@ class ParamDescriptor(BaseModel): @model_validator(mode="after") def optional_must_have_default(self) -> Self: - if not self.required and self.default is None: - raise ValueError("Optional parameters must declare a default value.") + configured_defaults = sum( + ( + self.default is not None, + self.default_is_null, + self.default_expression is not None, + ) + ) + if configured_defaults > 1: + raise ValueError( + "Declare only one of default, default_is_null, or default_expression." + ) + if self.default_is_null and self.required: + raise ValueError("A NULL default is supported only for optional parameters.") + if self.default_expression is not None and self.type != "date": + raise ValueError("default_expression is supported only for date parameters.") + if configured_defaults: + from app.sql.param_models import build_param_model # noqa: PLC0415 + + try: + model = build_param_model({"value": self.model_dump()}) + model.model_validate({}) + except (TypeError, ValueError) as exc: + raise ValueError( + f"Invalid default for parameter type '{self.type}'." + ) from exc return self +def missing_snapshot_defaults( + param_schema: Mapping[str, object], +) -> list[str]: + """Return snapshot bind names that cannot be resolved without a request.""" + missing: list[str] = [] + for name, raw_descriptor in param_schema.items(): + if isinstance(raw_descriptor, ParamDescriptor): + descriptor = raw_descriptor + elif isinstance(raw_descriptor, dict): + descriptor = ParamDescriptor.model_validate(raw_descriptor) + else: + missing.append(name) + continue + + if ( + descriptor.default is None + and not descriptor.default_is_null + and descriptor.default_expression is None + ): + missing.append(name) + return sorted(missing) + + +def require_snapshot_defaults( + data_strategy: DataStrategy, + param_schema: Mapping[str, object], +) -> None: + """Reject snapshot endpoints whose binds require caller-supplied values.""" + if data_strategy != DataStrategy.snapshot: + return + try: + missing = missing_snapshot_defaults(param_schema) + except ValidationError as exc: + raise SnapshotConfigurationError( + "Snapshot endpoint has an invalid parameter schema." + ) from exc + if missing: + names = ", ".join(f":{name}" for name in missing) + raise SnapshotConfigurationError( + f"Snapshot endpoints require a default for every parameter. Missing: {names}." + ) + + class EndpointCreate(BaseModel): """Payload for POST /api/v1/admin/endpoints.""" @@ -116,8 +200,8 @@ class EndpointCreate(BaseModel): allow_unauthenticated: bool = Field( False, description=( - "Explicit opt-in to serve this endpoint with NO authentication. " - "Required (must be true) when auth_method_id is omitted." + "Legacy-named opt-in to platform-admin Bearer authentication when " + "auth_method_id is omitted. It never permits anonymous access." ), ) data_strategy: DataStrategy = DataStrategy.live @@ -134,9 +218,9 @@ def validate_path(cls, v: str) -> str: return v @model_validator(mode="after") - def require_auth_or_explicit_public(self) -> Self: - # M1: never allow an endpoint to be created with no auth method - # unless the admin explicitly opts into public access. + def require_auth_or_explicit_fallback(self) -> Self: + # Without a dedicated method, require explicit use of the platform + # Bearer fallback. The legacy field name does not permit anonymity. if self.auth_method_id is None and not self.allow_unauthenticated: raise ValueError(PUBLIC_OPT_IN_MESSAGE) return self @@ -163,6 +247,7 @@ def bind_params_match_schema(self) -> Self: raise ValueError( f"Schema declares params not referenced in SQL: {sorted(unused)}" ) + require_snapshot_defaults(self.data_strategy, self.param_schema) return self @@ -183,8 +268,8 @@ class EndpointUpdate(BaseModel): allow_unauthenticated: bool | None = Field( None, description=( - "Explicit opt-in to serve this endpoint with NO authentication. " - "Set true when detaching the auth method to keep it public." + "Legacy-named opt-in to platform-admin Bearer authentication when " + "detaching the endpoint-specific auth method." ), ) data_strategy: DataStrategy | None = None @@ -233,11 +318,9 @@ def bind_params_match_schema(self) -> Self: return self @model_validator(mode="after") - def require_auth_or_explicit_public(self) -> Self: - # M1: when this request explicitly sets BOTH fields, reject the unsafe - # combination here (422). The merged-state case — e.g. detaching the - # auth method without touching allow_unauthenticated — is enforced - # against the stored row in EndpointService.update_endpoint. + def require_auth_or_explicit_fallback(self) -> Self: + # When this request explicitly sets both fields, reject a configuration + # with neither a dedicated method nor the platform Bearer fallback. fields_set = self.model_fields_set if "auth_method_id" in fields_set and "allow_unauthenticated" in fields_set: if self.auth_method_id is None and not self.allow_unauthenticated: diff --git a/backend/app/schemas/schedule.py b/backend/app/schemas/schedule.py index 3a110ac..c0f33bd 100644 --- a/backend/app/schemas/schedule.py +++ b/backend/app/schemas/schedule.py @@ -8,6 +8,23 @@ # ── Schedule schemas ───────────────────────────────────────────────────────── +def _validate_cron_expression(value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + parts = normalized.split() + if len(parts) != 5: + raise ValueError("Cron expression must have exactly 5 fields.") + + from apscheduler.triggers.cron import CronTrigger # noqa: PLC0415 + + try: + CronTrigger.from_crontab(normalized) + except ValueError as exc: + raise ValueError(f"Invalid cron expression: {exc}") from exc + return normalized + + class ScheduleCreate(BaseModel): endpoint_id: uuid.UUID schedule_type: str = Field(..., pattern=r"^(cron|interval)$") @@ -17,14 +34,9 @@ class ScheduleCreate(BaseModel): @field_validator("cron_expression") @classmethod - def validate_cron(cls, v: str | None, info: object) -> str | None: - """Basic cron expression validation (5-field format).""" - if v is None: - return v - parts = v.strip().split() - if len(parts) != 5: - raise ValueError("Cron expression must have exactly 5 fields.") - return v.strip() + def validate_cron(cls, v: str | None) -> str | None: + """Validate cron syntax and ranges with APScheduler's parser.""" + return _validate_cron_expression(v) @field_validator("interval_seconds") @classmethod @@ -37,9 +49,7 @@ def model_post_init(self, __context: object) -> None: if self.schedule_type == "cron" and not self.cron_expression: raise ValueError("cron_expression is required when schedule_type is 'cron'.") if self.schedule_type == "interval" and not self.interval_seconds: - raise ValueError( - "interval_seconds is required when schedule_type is 'interval'." - ) + raise ValueError("interval_seconds is required when schedule_type is 'interval'.") class ScheduleUpdate(BaseModel): @@ -48,6 +58,11 @@ class ScheduleUpdate(BaseModel): interval_seconds: int | None = Field(None, ge=10) is_active: bool | None = None + @field_validator("cron_expression") + @classmethod + def validate_cron(cls, v: str | None) -> str | None: + return _validate_cron_expression(v) + class ScheduleResponse(BaseModel): id: uuid.UUID @@ -69,8 +84,8 @@ class ScheduleResponse(BaseModel): class JobRunResponse(BaseModel): id: uuid.UUID - schedule_id: uuid.UUID - endpoint_id: uuid.UUID + schedule_id: uuid.UUID | None + endpoint_id: uuid.UUID | None started_at: datetime finished_at: datetime | None status: str diff --git a/backend/app/services/data.py b/backend/app/services/data.py index c75da10..bbe70a4 100644 --- a/backend/app/services/data.py +++ b/backend/app/services/data.py @@ -26,9 +26,11 @@ import structlog from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse +from fastapi.security import HTTPAuthorizationCredentials from pydantic import ValidationError from sqlalchemy.ext.asyncio import AsyncSession +from app.auth.admin import get_current_admin from app.middleware import resolve_request_id from app.models.endpoint import ApiEndpoint from app.repositories.auth_method import AuthMethodRepository @@ -111,56 +113,67 @@ async def serve(self, path: str, request: Request) -> DataServiceResult: if endpoint.auth_method_id is not None: principal = await self._enforce_auth(request, endpoint.auth_method_id) elif not endpoint.allow_unauthenticated: - # No auth method AND no explicit opt-in. EndpointCreate/Update both - # reject this combination, but it can still arise out-of-band — most - # importantly when an auth method an endpoint references is deleted - # (the FK is ondelete=SET NULL), which would otherwise silently turn - # a previously protected endpoint public. Default-deny rather than - # serve it unauthenticated (closes the M1 deletion side-channel). - log.warning( - "unauthenticated_endpoint_denied", - endpoint_id=str(endpoint.id), - endpoint=path, - user="anonymous", - status=status.HTTP_401_UNAUTHORIZED, - method=request.method, - client_ip=request.client.host if request.client else None, - request_id=resolve_request_id(request), - duration_ms=round((time.perf_counter() - started_at) * 1000, 2), - ) + self._log_platform_auth_denied(request, endpoint, started_at, path) raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Authentication configuration is unavailable.", ) + else: + principal = await self._enforce_platform_auth(request, endpoint, started_at, path) if endpoint.data_strategy.value == "snapshot": response = await self._serve_snapshot(endpoint, path, principal) else: response = await self._serve_live(endpoint, request, path, principal) - if endpoint.auth_method_id is None: - # Reached only for an explicitly public endpoint (the default-deny - # branch above already returned). Audit every public hit with the - # full structured-log field set (§3.5); ``allow_unauthenticated`` is - # always True here. - log.warning( - "public_endpoint_served", - endpoint_id=str(endpoint.id), - endpoint=path, - user=principal or "anonymous", - status=response.status_code, - method=request.method, - client_ip=request.client.host if request.client else None, - request_id=resolve_request_id(request), - duration_ms=round((time.perf_counter() - started_at) * 1000, 2), - ) - return DataServiceResult( response=response, principal=principal, endpoint_id=endpoint.id, ) + async def _enforce_platform_auth( + self, + request: Request, + endpoint: ApiEndpoint, + started_at: float, + path: str, + ) -> str: + """Require the platform admin bearer token when no endpoint auth is set.""" + authorization = request.headers.get("Authorization", "") + credentials = None + if authorization.lower().startswith("bearer "): + credentials = HTTPAuthorizationCredentials( + scheme="Bearer", + credentials=authorization[7:].strip(), + ) + + try: + principal = await get_current_admin(credentials) + except HTTPException: + self._log_platform_auth_denied(request, endpoint, started_at, path) + raise + return principal.username + + @staticmethod + def _log_platform_auth_denied( + request: Request, + endpoint: ApiEndpoint, + started_at: float, + path: str, + ) -> None: + log.warning( + "unauthenticated_endpoint_denied", + endpoint_id=str(endpoint.id), + endpoint=path, + user="anonymous", + status=status.HTTP_401_UNAUTHORIZED, + method=request.method, + client_ip=request.client.host if request.client else None, + request_id=resolve_request_id(request), + duration_ms=round((time.perf_counter() - started_at) * 1000, 2), + ) + # ── Endpoint lookup ───────────────────────────────────────────────────── async def _resolve_endpoint(self, path: str) -> ApiEndpoint: @@ -367,7 +380,11 @@ async def _serve_live( @staticmethod def _coerce_params(endpoint: ApiEndpoint, request: Request) -> dict[str, Any]: param_schema = endpoint.param_schema_json or {} - Model = build_param_model(param_schema) + # Scheduler defaults make snapshot refreshes autonomous, but they do + # not make required HTTP query parameters optional. Live callers must + # supply every descriptor marked required; configured defaults remain + # available to the scheduler and to omitted optional request fields. + Model = build_param_model(param_schema, enforce_required=True) # Pull only declared params from the query string; ignore unknowns # so the legacy loop's behavior is preserved. Filter on # ``isinstance(descriptor, dict)`` so a corrupted non-dict diff --git a/backend/app/services/endpoint.py b/backend/app/services/endpoint.py index 4d93896..60279c8 100644 --- a/backend/app/services/endpoint.py +++ b/backend/app/services/endpoint.py @@ -24,9 +24,11 @@ EndpointUpdate, ParamDescriptor, PublicEndpointError, + SnapshotConfigurationError, SqlPreviewRequest, SqlPreviewResponse, extract_bind_params, + require_snapshot_defaults, ) from app.sql.executor import SqlExecutionError, execute_query @@ -163,24 +165,31 @@ async def update_endpoint( for field in payload.model_fields_set & _updatable } - # M1: never persist an endpoint that ends up unauthenticated without an - # explicit opt-in. Evaluate the MERGED state (payload over the stored - # row) on EVERY update — not just when an auth field is in the payload — - # so a legacy/orphaned (auth_method_id=None, allow_unauthenticated=False) - # row can't stay silently public via an unrelated edit (rename, SQL, …). + # Always preserve an authentication path in the merged state: either a + # dedicated endpoint method or the platform-admin Bearer fallback. effective_auth = ( payload.auth_method_id if "auth_method_id" in payload.model_fields_set else obj.auth_method_id ) - effective_public = ( + effective_fallback = ( payload.allow_unauthenticated if "allow_unauthenticated" in payload.model_fields_set else obj.allow_unauthenticated ) - if effective_auth is None and not effective_public: + if effective_auth is None and not effective_fallback: raise PublicEndpointError(PUBLIC_OPT_IN_MESSAGE) + if "data_strategy" in payload.model_fields_set and payload.data_strategy is None: + raise SnapshotConfigurationError("data_strategy cannot be null.") + effective_strategy = payload.data_strategy or obj.data_strategy + effective_param_schema: dict[str, ParamDescriptor] | dict[str, object] + if "param_schema" in payload.model_fields_set and payload.param_schema is not None: + effective_param_schema = payload.param_schema + else: + effective_param_schema = obj.param_schema_json or {} + require_snapshot_defaults(effective_strategy, effective_param_schema) + # Uniqueness check on name change if payload.name is not None and payload.name != obj.name: conflict = await self._repo.get_by_name(payload.name) diff --git a/backend/app/services/schedule.py b/backend/app/services/schedule.py index b7b835e..43d273d 100644 --- a/backend/app/services/schedule.py +++ b/backend/app/services/schedule.py @@ -18,6 +18,7 @@ from app.repositories.job_run import JobRunRepository from app.repositories.schedule import ScheduleRepository from app.repositories.snapshot import SnapshotRepository +from app.schemas.endpoint import require_snapshot_defaults from app.schemas.schedule import ( JobRunResponse, ScheduleCreate, @@ -85,6 +86,7 @@ async def create_schedule( ep = await self._ep_repo.get_by_id(payload.endpoint_id) if ep is None: raise ValueError(f"Endpoint '{payload.endpoint_id}' not found.") + require_snapshot_defaults(ep.data_strategy, ep.param_schema_json or {}) # Check uniqueness — one schedule per endpoint existing = await self._repo.get_by_endpoint_id(payload.endpoint_id) @@ -172,9 +174,6 @@ async def delete_schedule( if obj is None: return False - # Remove from APScheduler - remove_schedule_job(obj.id) - await self._repo.delete(obj) log.info( diff --git a/backend/app/services/scheduler.py b/backend/app/services/scheduler.py index 595b350..20da67e 100644 --- a/backend/app/services/scheduler.py +++ b/backend/app/services/scheduler.py @@ -6,8 +6,8 @@ - Persist job runs and snapshots. - Update schedule metadata (last_run_at, next_run_at). -The scheduler currently relies on APScheduler's in-memory job store; -scheduled jobs themselves do not persist across process restarts. +The scheduler uses APScheduler's in-memory job store. Active schedules are +rehydrated from the application database whenever the process starts. """ import uuid @@ -26,8 +26,17 @@ from app.repositories.schedule import ScheduleRepository from app.repositories.snapshot import SnapshotRepository from app.sql.executor import SqlExecutionError, execute_query - -log = structlog.get_logger() +from app.sql.param_models import build_param_model + +log = structlog.get_logger().bind( + request_id=None, + user="scheduler", + endpoint=None, + status=None, + duration_ms=None, + method="SCHEDULE", + client_ip=None, +) # Module-level scheduler instance — initialized in start_scheduler(). _scheduler: Any = None @@ -46,6 +55,12 @@ def get_scheduler() -> Any: return _scheduler +def resolve_scheduled_params(param_schema: dict[str, Any]) -> dict[str, Any]: + """Resolve every scheduled bind default, retaining explicit SQL NULLs.""" + ParamModel = build_param_model(param_schema) + return ParamModel.model_validate({}).model_dump() + + async def execute_scheduled_job(schedule_id: str, endpoint_id: str) -> None: """Execute a single scheduled job — query Oracle, save snapshot. @@ -95,13 +110,14 @@ async def execute_scheduled_job(schedule_id: str, endpoint_id: str) -> None: if connection is None or not connection.is_active: raise ValueError("Data source connection is unavailable.") - # Execute SQL (no parameters for scheduled jobs — snapshot queries - # should be parameterless or use defaults) - params: dict[str, object] = {} + # Scheduled snapshots have no request inputs, so validate and + # resolve every configured static or dynamic default through the + # same typed model used by live data requests. param_schema = endpoint.param_schema_json or {} - for param_name, descriptor in param_schema.items(): - if isinstance(descriptor, dict) and descriptor.get("default") is not None: - params[param_name] = descriptor["default"] + # Keep explicit NULL defaults in the bind dictionary. Omitting a + # None-valued entry makes Oracle see a missing bind variable rather + # than a bind whose value is SQL NULL. + params = resolve_scheduled_params(param_schema) columns, rows, duration_ms = await execute_query( connection=connection, @@ -139,28 +155,30 @@ async def execute_scheduled_job(schedule_id: str, endpoint_id: str) -> None: from app.repositories.settings import SettingsRepository # noqa: PLC0415 settings_repo = SettingsRepository(db) - retention_setting = await settings_repo.get_by_key( - "snapshot_retention_count" - ) - retention_count = ( - int(retention_setting.value) if retention_setting else 5 - ) + retention_setting = await settings_repo.get_by_key("snapshot_retention_count") + retention_count = int(retention_setting.value) if retention_setting else 5 await snap_repo.delete_old(eid, keep=retention_count) # Mark job success finished_at = datetime.now(UTC) - await job_repo.update(job_run, { - "finished_at": finished_at, - "status": JobRunStatus.success, - "row_count": len(rows), - }) + await job_repo.update( + job_run, + { + "finished_at": finished_at, + "status": JobRunStatus.success, + "row_count": len(rows), + }, + ) # Update schedule last_run_at schedule = await sched_repo.get_by_id(sid) if schedule: - await sched_repo.update(schedule, { - "last_run_at": finished_at, - }) + await sched_repo.update( + schedule, + { + "last_run_at": finished_at, + }, + ) await db.commit() @@ -182,18 +200,24 @@ async def execute_scheduled_job(schedule_id: str, endpoint_id: str) -> None: if "timeout" in error_detail.lower(): status = JobRunStatus.timeout - await job_repo.update(job_run, { - "finished_at": finished_at, - "status": status, - "error_detail": error_detail, - }) + await job_repo.update( + job_run, + { + "finished_at": finished_at, + "status": status, + "error_detail": error_detail, + }, + ) # Update schedule last_run_at even on failure schedule = await sched_repo.get_by_id(sid) if schedule: - await sched_repo.update(schedule, { - "last_run_at": finished_at, - }) + await sched_repo.update( + schedule, + { + "last_run_at": finished_at, + }, + ) await db.commit() @@ -207,10 +231,43 @@ async def execute_scheduled_job(schedule_id: str, endpoint_id: str) -> None: ) -def start_scheduler() -> None: +async def restore_active_schedules() -> int: + """Register all active database schedules in the in-memory scheduler.""" + restored = 0 + failed_schedule_ids: list[str] = [] + async with AsyncSessionLocal() as db: + schedules = await ScheduleRepository(db).get_all(active_only=True) + for schedule in schedules: + try: + registered = add_schedule_job( + schedule_id=schedule.id, + endpoint_id=schedule.endpoint_id, + schedule_type=schedule.schedule_type, + cron_expression=schedule.cron_expression, + interval_seconds=schedule.interval_seconds, + ) + if not registered: + raise ValueError("Schedule configuration could not be registered.") + restored += 1 + except Exception as exc: # noqa: BLE001 + failed_schedule_ids.append(str(schedule.id)) + log.error( + "scheduler_job_restore_failed", + schedule_id=str(schedule.id), + error=str(exc), + ) + if failed_schedule_ids: + failed = ", ".join(failed_schedule_ids) + raise RuntimeError(f"Failed to restore active schedules: {failed}") + log.info("scheduler_jobs_restored", restored_count=restored) + return restored + + +async def start_scheduler() -> None: """Initialize and start the APScheduler instance.""" global _scheduler # noqa: PLW0603 + scheduler = None try: from apscheduler.schedulers.asyncio import AsyncIOScheduler # noqa: PLC0415 @@ -224,8 +281,17 @@ def start_scheduler() -> None: scheduler.start() _scheduler = scheduler log.info("scheduler_started") + await restore_active_schedules() except Exception as exc: # noqa: BLE001 + if _scheduler is not None: + stop_scheduler() + elif scheduler is not None: + try: + scheduler.shutdown(wait=False) + except Exception as shutdown_exc: # noqa: BLE001 + log.warning("scheduler_cleanup_failed", error=str(shutdown_exc)) log.error("scheduler_start_failed", error=str(exc)) + raise def stop_scheduler() -> None: @@ -246,11 +312,11 @@ def add_schedule_job( schedule_type: str, cron_expression: str | None = None, interval_seconds: int | None = None, -) -> None: +) -> bool: """Register a job in APScheduler.""" if _scheduler is None: log.warning("scheduler_not_running", action="add_job") - return + return False from apscheduler.schedulers.asyncio import AsyncIOScheduler # noqa: PLC0415 @@ -284,7 +350,7 @@ def add_schedule_job( kwargs["seconds"] = interval_seconds else: log.warning("invalid_schedule_config", schedule_id=str(schedule_id)) - return + return False scheduler.add_job(**kwargs) log.info( @@ -292,6 +358,7 @@ def add_schedule_job( job_id=job_id, schedule_type=schedule_type, ) + return True def remove_schedule_job(schedule_id: uuid.UUID) -> None: diff --git a/backend/app/sql/param_models.py b/backend/app/sql/param_models.py index bbc6614..3461a63 100644 --- a/backend/app/sql/param_models.py +++ b/backend/app/sql/param_models.py @@ -9,10 +9,12 @@ The descriptor format mirrors ``app.schemas.endpoint.ParamDescriptor`` — ``{"type": "string|integer|float|boolean|date", "required": bool, -"default": , "max_length": int | None}``. +"default": , "default_is_null": bool, +"default_expression": "today|yesterday", +"max_length": int | None}``. """ -from datetime import date +from datetime import date, datetime, timedelta from typing import Annotated, Any, Literal, get_args import structlog @@ -49,7 +51,38 @@ def _coerce_bool(value: object) -> object: return value -def _build_field(descriptor: dict[str, Any]) -> tuple[type, Any]: +def _coerce_date(value: object) -> object: + """Accept the documented HTTP date formats and return a date value.""" + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + if isinstance(value, str): + for date_format in ("%Y-%m-%d", "%d-%m-%Y"): + try: + return datetime.strptime(value, date_format).date() + except ValueError: + continue + raise ValueError("expected YYYY-MM-DD or DD-MM-YYYY") + return value + + +def resolve_default_expression(expression: str, *, current_date: date | None = None) -> date: + """Resolve a supported dynamic date default using the server's local date.""" + resolved_date = current_date or date.today() + if expression == "today": + return resolved_date + if expression == "yesterday": + return resolved_date - timedelta(days=1) + raise ValueError(f"Unsupported default expression: {expression}") + + +def _build_field( + descriptor: dict[str, Any], + *, + current_date: date | None = None, + enforce_required: bool = False, +) -> tuple[type, Any]: """Map one descriptor to a ``(annotation, default)`` pair for ``create_model``.""" raw_type = descriptor.get("type", "string") if raw_type not in _VALID_PARAM_TYPES: @@ -59,6 +92,14 @@ def _build_field(descriptor: dict[str, Any]) -> tuple[type, Any]: required = bool(descriptor.get("required", True)) default = descriptor.get("default") + default_is_null = bool(descriptor.get("default_is_null", False)) + default_expression = descriptor.get("default_expression") + if default_expression is not None: + if raw_type != "date": + raise ValueError("Dynamic defaults are supported only for date parameters.") + if default is not None: + raise ValueError("Declare either default or default_expression, not both.") + default = resolve_default_expression(str(default_expression), current_date=current_date) max_length = descriptor.get("max_length") annotation: type @@ -69,20 +110,22 @@ def _build_field(descriptor: dict[str, Any]) -> tuple[type, Any]: elif raw_type == "boolean": annotation = Annotated[bool, BeforeValidator(_coerce_bool)] # type: ignore[assignment] elif raw_type == "date": - annotation = date + annotation = Annotated[date, BeforeValidator(_coerce_date)] # type: ignore[assignment] else: # "string" annotation = str if isinstance(max_length, int) and max_length >= 1: annotation = Annotated[str, Field(max_length=max_length)] # type: ignore[assignment] - # Legacy ``_coerce_param`` semantics: if a default is configured, - # apply it whenever the query param is missing — regardless of the - # ``required`` flag. The previous Pydantic-based draft made every - # required field unconditionally mandatory, which would 422 endpoints - # whose stored schema combined ``required=true`` with a non-null - # ``default``. ``ParamDescriptor`` still allows that combination, so - # honor it here. - if default is not None: + # Scheduled execution must resolve every configured default without + # request input. Live requests use ``enforce_required=True`` so a + # scheduler default never weakens their public required-parameter + # contract. Optional request fields continue to use their defaults. + if enforce_required and required: + field_default = ... + elif default_is_null: + annotation = annotation | None # type: ignore[assignment] + field_default = None + elif default is not None: field_default = default elif required: field_default = ... @@ -100,7 +143,12 @@ def _build_field(descriptor: dict[str, Any]) -> tuple[type, Any]: return annotation, field_default -def build_param_model(param_schema: dict[str, Any]) -> type[BaseModel]: +def build_param_model( + param_schema: dict[str, Any], + *, + current_date: date | None = None, + enforce_required: bool = False, +) -> type[BaseModel]: """Construct a Pydantic model that validates ``param_schema`` payloads. Non-dict values in ``param_schema`` are ignored to match the legacy @@ -121,9 +169,13 @@ def build_param_model(param_schema: dict[str, Any]) -> type[BaseModel]: descriptor_type=type(descriptor).__name__, ) continue - fields[name] = _build_field(descriptor) + fields[name] = _build_field( + descriptor, + current_date=current_date, + enforce_required=enforce_required, + ) - model = create_model( + model: type[BaseModel] = create_model( "EndpointParams", # ``validate_default=True`` makes Pydantic coerce/validate the # ``default`` we feed each field. Without it, a corrupted stored diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 186c4be..ebc2e26 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -49,6 +49,7 @@ from app.main import app from app.models.base import Base from httpx import ASGITransport, AsyncClient +from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import ( AsyncEngine, AsyncSession, @@ -61,6 +62,16 @@ ADMIN_TEST_PASSWORD = "admin-password-do-not-use-in-prod" +def _assert_safe_test_database(database_url: str, app_env: str) -> None: + """Require both an explicit test environment and a test-named database.""" + database_name = make_url(database_url).database or "" + if app_env != "test" or "test" not in database_name.lower(): + raise RuntimeError( + "Refusing to run destructive test schema setup: APP_ENV must be " + "'test' and the parsed database name must contain 'test'." + ) + + def _mint_admin_token() -> str: """Mint a valid admin JWT using the test-time settings. @@ -98,16 +109,9 @@ async def engine() -> AsyncGenerator[AsyncEngine]: so the database is left empty between tests. """ database_url = settings.database_url - # Hard safety guard: never run destructive schema ops against a DB that - # isn't clearly marked as a test database. Without this, a developer - # invoking pytest with the default DATABASE_URL would have their app - # database wiped on every test run. - if settings.app_env != "test" and "test" not in database_url.lower(): - raise RuntimeError( - "Refusing to run drop_all/create_all: APP_ENV is not 'test' and " - "DATABASE_URL does not contain 'test'. Set APP_ENV=test or point " - "DATABASE_URL at a test database." - ) + # Hard safety guard: both signals are mandatory. Requiring only one means + # APP_ENV=test can accidentally target and wipe the development database. + _assert_safe_test_database(database_url, settings.app_env) test_engine = create_async_engine(database_url, echo=False) async with test_engine.begin() as conn: diff --git a/backend/tests/test_endpoints.py b/backend/tests/test_endpoints.py index 7553348..11ce992 100644 --- a/backend/tests/test_endpoints.py +++ b/backend/tests/test_endpoints.py @@ -9,12 +9,16 @@ import uuid import pytest +from app.models.endpoint import DataStrategy from app.schemas.endpoint import ( EndpointCreate, EndpointResponse, EndpointUpdate, + ParamDescriptor, + SnapshotConfigurationError, SqlPreviewRequest, extract_bind_params, + require_snapshot_defaults, validate_sql_safety, ) @@ -87,6 +91,150 @@ def test_endpoint_create_valid() -> None: assert payload.connection_id == conn_id +def test_date_parameter_accepts_dynamic_default() -> None: + descriptor = ParamDescriptor(type="date", required=True, default_expression="today") + assert descriptor.default_expression == "today" + + +def test_dynamic_default_rejected_for_non_date_parameter() -> None: + with pytest.raises(ValueError, match="only for date"): + ParamDescriptor(type="string", required=True, default_expression="today") + + +def test_static_and_dynamic_defaults_are_mutually_exclusive() -> None: + with pytest.raises(ValueError, match="only one of"): + ParamDescriptor( + type="date", + required=True, + default="2026-08-30", + default_expression="today", + ) + + +def test_optional_parameter_accepts_explicit_null_default() -> None: + descriptor = ParamDescriptor(type="string", required=False, default_is_null=True) + assert descriptor.default_is_null is True + assert descriptor.default is None + + +def test_required_parameter_rejects_explicit_null_default() -> None: + with pytest.raises(ValueError, match="only for optional parameters"): + ParamDescriptor(type="string", required=True, default_is_null=True) + + +def test_optional_parameter_accepts_no_default() -> None: + descriptor = ParamDescriptor(type="integer", required=False) + assert descriptor.default is None + assert descriptor.default_is_null is False + + +def test_parameter_rejects_incompatible_static_default() -> None: + with pytest.raises(ValueError, match="Invalid default"): + ParamDescriptor(type="integer", required=True, default="abc") + + +def test_endpoint_create_rejects_incompatible_static_default() -> None: + with pytest.raises(ValueError, match="Invalid default"): + EndpointCreate( + name="invalid-default", + path="invalid-default", + connection_id=uuid.uuid4(), + sql_text="SELECT * FROM stores WHERE id = :store_id", + param_schema={ + "store_id": {"type": "integer", "required": True, "default": "abc"} + }, + allow_unauthenticated=True, + ) + + +def test_endpoint_update_rejects_incompatible_static_default() -> None: + with pytest.raises(ValueError, match="Invalid default"): + EndpointUpdate( + param_schema={ + "store_id": {"type": "integer", "required": True, "default": "abc"} + } + ) + + +def test_snapshot_default_validation_rejects_incompatible_stored_default() -> None: + with pytest.raises(SnapshotConfigurationError, match="invalid parameter schema"): + require_snapshot_defaults( + DataStrategy.snapshot, + {"store_id": {"type": "integer", "required": True, "default": "abc"}}, + ) + + +def test_snapshot_endpoint_requires_defaults_for_all_parameters() -> None: + with pytest.raises(ValueError, match=r"Missing: :end_date, :start_date"): + EndpointCreate( + name="snapshot-without-defaults", + path="snapshot-without-defaults", + connection_id=uuid.uuid4(), + sql_text=("SELECT * FROM orders WHERE business_date BETWEEN :start_date AND :end_date"), + param_schema={ + "start_date": {"type": "date", "required": True}, + "end_date": {"type": "date", "required": True}, + }, + allow_unauthenticated=True, + data_strategy="snapshot", + ) + + +def test_snapshot_endpoint_accepts_dynamic_defaults() -> None: + payload = EndpointCreate( + name="snapshot-with-dynamic-defaults", + path="snapshot-with-dynamic-defaults", + connection_id=uuid.uuid4(), + sql_text=("SELECT * FROM orders WHERE business_date BETWEEN :start_date AND :end_date"), + param_schema={ + "start_date": { + "type": "date", + "required": True, + "default_expression": "yesterday", + }, + "end_date": { + "type": "date", + "required": True, + "default_expression": "today", + }, + }, + allow_unauthenticated=True, + data_strategy="snapshot", + ) + assert payload.param_schema["start_date"].default_expression == "yesterday" + + +def test_snapshot_endpoint_accepts_explicit_null_default() -> None: + payload = EndpointCreate( + name="snapshot-with-null-default", + path="snapshot-with-null-default", + connection_id=uuid.uuid4(), + sql_text="SELECT * FROM stores WHERE :str_id IS NULL OR id = :str_id", + param_schema={ + "str_id": { + "type": "string", + "required": False, + "default_is_null": True, + } + }, + allow_unauthenticated=True, + data_strategy="snapshot", + ) + assert payload.param_schema["str_id"].default_is_null is True + + +def test_parameterless_snapshot_endpoint_is_valid() -> None: + payload = EndpointCreate( + name="parameterless-snapshot", + path="parameterless-snapshot", + connection_id=uuid.uuid4(), + sql_text="SELECT 1 FROM dual", + allow_unauthenticated=True, + data_strategy="snapshot", + ) + assert payload.param_schema == {} + + def test_endpoint_create_normalizes_path() -> None: conn_id = uuid.uuid4() payload = EndpointCreate( @@ -341,6 +489,119 @@ async def test_update_endpoint(async_client: object) -> None: assert data["is_deprecated"] is True +@pytest.mark.integration +async def test_update_live_endpoint_to_snapshot_requires_merged_defaults( + async_client: object, +) -> None: + from httpx import AsyncClient + + client: AsyncClient = async_client # type: ignore[assignment] + conn_payload = { + "name": f"test-conn-snapshot-update-{uuid.uuid4().hex[:8]}", + "host": "oracle.example.com", + "service_name": "SVC", + "username": "scott", + "password": "tiger", + } + connection = await client.post("/api/v1/admin/connections/", json=conn_payload) + endpoint = await client.post( + "/api/v1/admin/endpoints/", + json={ + "name": f"snapshot-update-{uuid.uuid4().hex[:8]}", + "path": f"snapshot-update-{uuid.uuid4().hex[:8]}", + "connection_id": connection.json()["id"], + "sql_text": "SELECT * FROM orders WHERE business_date = :business_date", + "param_schema": {"business_date": {"type": "date", "required": True}}, + "allow_unauthenticated": True, + "data_strategy": "live", + }, + ) + assert endpoint.status_code == 201 + + invalid = await client.put( + f"/api/v1/admin/endpoints/{endpoint.json()['id']}", + json={"data_strategy": "snapshot"}, + ) + assert invalid.status_code == 422 + assert ":business_date" in invalid.json()["detail"] + + valid = await client.put( + f"/api/v1/admin/endpoints/{endpoint.json()['id']}", + json={ + "data_strategy": "snapshot", + "param_schema": { + "business_date": { + "type": "date", + "required": True, + "default_expression": "today", + } + }, + }, + ) + assert valid.status_code == 200 + assert valid.json()["param_schema"]["business_date"]["default_expression"] == "today" + + +@pytest.mark.integration +async def test_update_snapshot_with_invalid_stored_schema_returns_422( + async_client: object, + db_session: object, +) -> None: + from app.models.endpoint import ApiEndpoint + from httpx import AsyncClient + from sqlalchemy import update + from sqlalchemy.ext.asyncio import AsyncSession + + client: AsyncClient = async_client # type: ignore[assignment] + session: AsyncSession = db_session # type: ignore[assignment] + connection = await client.post( + "/api/v1/admin/connections/", + json={ + "name": f"test-conn-invalid-schema-update-{uuid.uuid4().hex[:8]}", + "host": "oracle.example.com", + "service_name": "SVC", + "username": "scott", + "password": "tiger", + }, + ) + endpoint = await client.post( + "/api/v1/admin/endpoints/", + json={ + "name": f"invalid-schema-update-{uuid.uuid4().hex[:8]}", + "path": f"invalid-schema-update-{uuid.uuid4().hex[:8]}", + "connection_id": connection.json()["id"], + "sql_text": "SELECT * FROM stores WHERE id = :store_id", + "param_schema": { + "store_id": {"type": "integer", "required": True, "default": 1} + }, + "allow_unauthenticated": True, + "data_strategy": "snapshot", + }, + ) + assert endpoint.status_code == 201 + endpoint_id = uuid.UUID(endpoint.json()["id"]) + + await session.execute( + update(ApiEndpoint) + .where(ApiEndpoint.id == endpoint_id) + .values( + param_schema_json={ + "store_id": {"type": "integer", "required": True, "default": "abc"} + } + ) + ) + await session.flush() + session.expire_all() + + response = await client.put( + f"/api/v1/admin/endpoints/{endpoint_id}", + json={"description": "still invalid"}, + ) + + assert response.status_code == 422 + assert "invalid parameter schema" in response.json()["detail"] + + @pytest.mark.integration async def test_delete_endpoint(async_client: object) -> None: from httpx import AsyncClient @@ -375,6 +636,94 @@ async def test_delete_endpoint(async_client: object) -> None: assert r_get.status_code == 404 +@pytest.mark.integration +async def test_delete_endpoint_preserves_job_history_and_removes_scheduler_job( + async_client: object, + db_session: object, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from datetime import UTC, datetime + + from app.models.job_run import JobRun, JobRunStatus + from httpx import AsyncClient + from sqlalchemy import select + from sqlalchemy.ext.asyncio import AsyncSession + + client: AsyncClient = async_client # type: ignore[assignment] + session: AsyncSession = db_session # type: ignore[assignment] + + connection = await client.post( + "/api/v1/admin/connections/", + json={ + "name": f"test-conn-del-history-{uuid.uuid4().hex[:8]}", + "host": "oracle.example.com", + "service_name": "SVC", + "username": "scott", + "password": "tiger", + }, + ) + assert connection.status_code == 201 + + endpoint = await client.post( + "/api/v1/admin/endpoints/", + json={ + "name": f"del-history-{uuid.uuid4().hex[:8]}", + "path": f"del-history-path-{uuid.uuid4().hex[:8]}", + "connection_id": connection.json()["id"], + "sql_text": "SELECT 1 FROM dual", + "allow_unauthenticated": True, + "data_strategy": "snapshot", + }, + ) + assert endpoint.status_code == 201 + endpoint_id = endpoint.json()["id"] + + schedule = await client.post( + "/api/v1/admin/schedules/", + json={ + "endpoint_id": endpoint_id, + "schedule_type": "interval", + "interval_seconds": 300, + }, + ) + assert schedule.status_code == 201 + schedule_id = schedule.json()["id"] + + run_id = uuid.uuid4() + session.add( + JobRun( + id=run_id, + schedule_id=uuid.UUID(schedule_id), + endpoint_id=uuid.UUID(endpoint_id), + started_at=datetime.now(UTC), + finished_at=datetime.now(UTC), + status=JobRunStatus.success, + row_count=1, + ) + ) + await session.commit() + + removed_jobs: list[uuid.UUID] = [] + monkeypatch.setattr("app.routers.endpoints.remove_schedule_job", removed_jobs.append) + + deleted = await client.delete(f"/api/v1/admin/endpoints/{endpoint_id}") + assert deleted.status_code == 204 + assert removed_jobs == [uuid.UUID(schedule_id)] + + ids = await session.execute( + select(JobRun.endpoint_id, JobRun.schedule_id).where(JobRun.id == run_id) + ) + assert ids.one() == (None, None) + + historical_run = await client.get(f"/api/v1/admin/schedules/jobs/{run_id}") + assert historical_run.status_code == 200 + assert historical_run.json()["endpoint_id"] is None + assert historical_run.json()["schedule_id"] is None + + deleted_schedule = await client.get(f"/api/v1/admin/schedules/{schedule_id}") + assert deleted_schedule.status_code == 404 + + @pytest.mark.integration async def test_data_endpoint_not_found(async_client: object) -> None: from httpx import AsyncClient diff --git a/backend/tests/test_migration.py b/backend/tests/test_migration.py index 53e72af..3d77481 100644 --- a/backend/tests/test_migration.py +++ b/backend/tests/test_migration.py @@ -132,6 +132,22 @@ def test_allow_unauthenticated_migration(self) -> None: assert "drop_column" in src, f"{f.name} must drop the column (downgrade)" assert "endpoints" in src, f"{f.name} must target the endpoints table" + def test_schedule_delete_preserves_job_runs_migration(self) -> None: + migration = MIGRATION_DIR / "b2d18f4a6c73_preserve_job_runs_when_deleting_schedules.py" + source = migration.read_text() + assert migration.is_file() + assert 'ondelete="SET NULL"' in source + assert '"schedule_id"' in source + assert "nullable=True" in source + + def test_endpoint_delete_preserves_job_runs_migration(self) -> None: + migration = MIGRATION_DIR / "c7e91a4f2d60_preserve_job_runs_when_deleting_endpoints.py" + source = migration.read_text() + assert migration.is_file() + assert 'ondelete="SET NULL"' in source + assert '"endpoint_id"' in source + assert "nullable=True" in source + def test_migration_chain_is_linear(self) -> None: """The revision graph must be a single linear chain: exactly one base (down_revision=None), exactly one head, and every down_revision known.""" @@ -233,9 +249,7 @@ async def test_tables_created(self, async_client: object) -> None: r = await client.get("/api/v1/admin/health/ready") assert r.status_code == 200 - async def test_connection_crud_after_fresh_schema( - self, async_client: object - ) -> None: + async def test_connection_crud_after_fresh_schema(self, async_client: object) -> None: """Verify CRUD operations work on fresh schema.""" from httpx import AsyncClient diff --git a/backend/tests/test_param_models.py b/backend/tests/test_param_models.py index 7fedfcb..b247e01 100644 --- a/backend/tests/test_param_models.py +++ b/backend/tests/test_param_models.py @@ -42,8 +42,9 @@ ({"type": "boolean", "required": True}, "false", False), ({"type": "boolean", "required": True}, "0", False), ({"type": "boolean", "required": True}, "NO", False), - # date — must be ISO format + # date — public API accepts ISO and the Oracle-oriented DD-MM-YYYY format ({"type": "date", "required": True}, "2024-01-15", date(2024, 1, 15)), + ({"type": "date", "required": True}, "15-01-2024", date(2024, 1, 15)), ] _GOLDEN_FAILURES: list[tuple[dict[str, object], str]] = [ @@ -69,9 +70,7 @@ def test_build_param_model_coerces_value( @pytest.mark.parametrize(("descriptor", "raw"), _GOLDEN_FAILURES) -def test_build_param_model_rejects_value( - descriptor: dict[str, object], raw: str -) -> None: +def test_build_param_model_rejects_value(descriptor: dict[str, object], raw: str) -> None: Model = build_param_model({"p": descriptor}) with pytest.raises(ValidationError): Model.model_validate({"p": raw}) @@ -140,10 +139,7 @@ def test_build_param_model_boolean_default_round_trips() -> None: def test_build_param_model_default_applies_when_required_and_missing() -> None: - """Legacy code applied a configured ``default`` whenever the param was - missing — regardless of the ``required`` flag. Pin that contract here - so the Pydantic-based path doesn't 422 endpoints whose stored schema - combined ``required=true`` with a non-null default.""" + """Default-resolution mode lets scheduled jobs resolve required binds.""" Model = build_param_model( {"p": {"type": "integer", "required": True, "default": 42}}, ) @@ -181,3 +177,87 @@ def test_build_param_model_optional_without_default_accepts_value() -> None: Model = build_param_model({"p": {"type": "integer", "required": False}}) instance = Model.model_validate({"p": "42"}) assert instance.model_dump()["p"] == 42 + + +def test_build_param_model_enforces_required_despite_static_default() -> None: + Model = build_param_model( + {"p": {"type": "integer", "required": True, "default": 42}}, + enforce_required=True, + ) + with pytest.raises(ValidationError) as exc_info: + Model.model_validate({}) + assert any(err["loc"] == ("p",) for err in exc_info.value.errors()) + + +def test_build_param_model_enforces_required_despite_dynamic_default() -> None: + Model = build_param_model( + { + "p": { + "type": "date", + "required": True, + "default_expression": "today", + } + }, + current_date=date(2026, 8, 30), + enforce_required=True, + ) + with pytest.raises(ValidationError) as exc_info: + Model.model_validate({}) + assert any(err["loc"] == ("p",) for err in exc_info.value.errors()) + + +def test_build_param_model_required_value_overrides_scheduler_default() -> None: + Model = build_param_model( + { + "p": { + "type": "date", + "required": True, + "default_expression": "today", + } + }, + current_date=date(2026, 8, 30), + enforce_required=True, + ) + assert Model.model_validate({"p": "15-01-2026"}).model_dump()["p"] == date(2026, 1, 15) + + +def test_build_param_model_explicit_null_default_binds_none() -> None: + Model = build_param_model({"p": {"type": "string", "required": False, "default_is_null": True}}) + instance = Model.model_validate({}) + assert instance.model_dump() == {"p": None} + + +def test_build_param_model_explicit_value_overrides_null_default() -> None: + Model = build_param_model({"p": {"type": "string", "required": False, "default_is_null": True}}) + instance = Model.model_validate({"p": "10105"}) + assert instance.model_dump() == {"p": "10105"} + + +def test_build_param_model_resolves_today_expression() -> None: + Model = build_param_model( + {"p": {"type": "date", "required": True, "default_expression": "today"}}, + current_date=date(2026, 8, 30), + ) + assert Model.model_validate({}).model_dump()["p"] == date(2026, 8, 30) + + +def test_build_param_model_resolves_yesterday_expression() -> None: + Model = build_param_model( + { + "p": { + "type": "date", + "required": True, + "default_expression": "yesterday", + } + }, + current_date=date(2026, 8, 30), + ) + assert Model.model_validate({}).model_dump()["p"] == date(2026, 8, 29) + + +def test_build_param_model_explicit_value_overrides_dynamic_default() -> None: + Model = build_param_model( + {"p": {"type": "date", "required": True, "default_expression": "today"}}, + current_date=date(2026, 8, 30), + ) + assert Model.model_validate({"p": "2026-01-15"}).model_dump()["p"] == date(2026, 1, 15) diff --git a/backend/tests/test_public_endpoint.py b/backend/tests/test_public_endpoint.py index e80801b..0dc5efc 100644 --- a/backend/tests/test_public_endpoint.py +++ b/backend/tests/test_public_endpoint.py @@ -1,16 +1,22 @@ -"""M1 — silent-public-endpoint prevention. +"""Mandatory authentication for every dynamic data endpoint. -An endpoint with no auth method is served unauthenticated by design, but -that must now be a *deliberate* choice: the admin API rejects a create or -update that would leave an endpoint unauthenticated unless -``allow_unauthenticated`` is explicitly set, and the data plane emits a -``public_endpoint_served`` warning on every public hit. +The legacy ``allow_unauthenticated`` field is retained for stored/API contract +compatibility, but now opts into platform-admin Bearer fallback. It never +permits anonymous data access. """ import uuid +from types import SimpleNamespace +from typing import cast +from unittest.mock import AsyncMock import pytest +from app.auth.jwt_utils import create_access_token +from app.config import settings from app.schemas.endpoint import EndpointCreate +from app.services.data import DataService +from fastapi import HTTPException, Request +from fastapi.responses import JSONResponse from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession from structlog.testing import capture_logs @@ -34,8 +40,8 @@ def test_create_without_auth_or_optin_rejected() -> None: ) -def test_create_explicit_public_allowed() -> None: - """No auth method + explicit opt-in is a valid, deliberate public endpoint.""" +def test_create_platform_auth_fallback_allowed() -> None: + """No endpoint method plus explicit platform fallback remains valid.""" ep = EndpointCreate( name="t", path="p", @@ -58,6 +64,78 @@ def test_create_with_auth_method_allowed() -> None: assert ep.allow_unauthenticated is False +@pytest.mark.asyncio +async def test_data_service_rejects_anonymous_legacy_public_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = DataService(cast(AsyncSession, object())) + endpoint = SimpleNamespace( + id=uuid.uuid4(), + auth_method_id=None, + allow_unauthenticated=True, + data_strategy=SimpleNamespace(value="snapshot"), + ) + monkeypatch.setattr(service, "_resolve_endpoint", AsyncMock(return_value=endpoint)) + request = Request( + { + "type": "http", + "method": "GET", + "path": "/api/v1/data/legacy-public", + "headers": [], + "query_string": b"", + "client": ("127.0.0.1", 1234), + "server": ("testserver", 80), + "scheme": "http", + "root_path": "", + } + ) + + with pytest.raises(HTTPException) as exc_info: + await service.serve("legacy-public", request) + + assert exc_info.value.status_code == 401 + + +@pytest.mark.asyncio +async def test_data_service_accepts_platform_auth_for_legacy_public_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = DataService(cast(AsyncSession, object())) + endpoint = SimpleNamespace( + id=uuid.uuid4(), + auth_method_id=None, + allow_unauthenticated=True, + data_strategy=SimpleNamespace(value="snapshot"), + ) + monkeypatch.setattr(service, "_resolve_endpoint", AsyncMock(return_value=endpoint)) + serve_snapshot = AsyncMock(return_value=JSONResponse(status_code=503, content={})) + monkeypatch.setattr(service, "_serve_snapshot", serve_snapshot) + token, _ = create_access_token( + subject=settings.admin_username, + secret=settings.jwt_secret_key, + algorithm=settings.jwt_algorithm, + expire_minutes=5, + ) + request = Request( + { + "type": "http", + "method": "GET", + "path": "/api/v1/data/legacy-public", + "headers": [(b"authorization", f"Bearer {token}".encode())], + "query_string": b"", + "client": ("127.0.0.1", 1234), + "server": ("testserver", 80), + "scheme": "http", + "root_path": "", + } + ) + + result = await service.serve("legacy-public", request) + + assert result.principal == settings.admin_username + serve_snapshot.assert_awaited_once() + + # ── API integration ────────────────────────────────────────────────────────── @@ -159,11 +237,10 @@ async def test_update_detaching_auth_without_optin_returns_422( @pytest.mark.integration -async def test_public_endpoint_serves_and_logs_warning( +async def test_legacy_public_endpoint_requires_platform_authentication( async_client: object, unauth_client: AsyncClient ) -> None: - """An explicitly public endpoint is served with NO credentials and emits a - fully-populated public_endpoint_served audit event.""" + """The legacy public flag never permits anonymous data access.""" admin: AsyncClient = async_client # type: ignore[assignment] conn_id = await _make_connection(admin) @@ -182,17 +259,25 @@ async def test_public_endpoint_serves_and_logs_warning( assert r.status_code == 201 with capture_logs() as logs: - # unauth_client carries no Authorization header — proves true public access. resp = await unauth_client.get(f"/api/v1/data/{ep_path}") - # Served (reached the snapshot path → 503 "no snapshot yet"), not auth-blocked. - assert resp.status_code == 503 - warnings = [e for e in logs if e.get("event") == "public_endpoint_served"] - assert warnings, f"expected a public_endpoint_served warning, got {logs}" - assert warnings[0]["endpoint"] == ep_path - assert warnings[0]["status"] == 503 - assert warnings[0]["user"] == "anonymous" - assert warnings[0]["log_level"] == "warning" + assert resp.status_code == 401 + denials = [e for e in logs if e.get("event") == "unauthenticated_endpoint_denied"] + assert denials, f"expected unauthenticated_endpoint_denied, got {logs}" + assert not any(e.get("event") == "public_endpoint_served" for e in logs) + + # The shared admin client carries a valid platform bearer token. It reaches + # the snapshot path and returns 503 only because no snapshot exists yet. + authenticated = await admin.get(f"/api/v1/data/{ep_path}") + assert authenticated.status_code == 503 + + authorization = admin.headers["Authorization"] + token = authorization.removeprefix("Bearer ") + whitespace_authenticated = await unauth_client.get( + f"/api/v1/data/{ep_path}", + headers={"Authorization": f"Bearer {token} "}, + ) + assert whitespace_authenticated.status_code == 503 @pytest.mark.integration @@ -249,3 +334,9 @@ async def test_deleting_auth_method_default_denies_endpoint( assert "client_ip" in denials[0] assert "duration_ms" in denials[0] assert not any(e.get("event") == "public_endpoint_served" for e in logs) + + # A valid platform-admin token must not bypass an orphaned endpoint's + # explicit fallback setting. The administrator must repair the endpoint + # configuration before data access resumes. + authenticated = await admin.get(f"/api/v1/data/{ep_path}") + assert authenticated.status_code == 401 diff --git a/backend/tests/test_scheduler_restore.py b/backend/tests/test_scheduler_restore.py new file mode 100644 index 0000000..2abb05c --- /dev/null +++ b/backend/tests/test_scheduler_restore.py @@ -0,0 +1,143 @@ +"""Unit coverage for restoring in-memory scheduler jobs after restart.""" + +import uuid +from types import SimpleNamespace +from typing import cast +from unittest.mock import AsyncMock, MagicMock + +import pytest +from structlog.testing import capture_logs + + +def test_scheduled_params_retain_explicit_null_bind() -> None: + from app.services.scheduler import resolve_scheduled_params + + params = resolve_scheduled_params( + { + "str_id": { + "type": "string", + "required": False, + "default_is_null": True, + } + } + ) + + assert params == {"str_id": None} + + +@pytest.mark.asyncio +async def test_restore_active_schedules_registers_each_row( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from app.services import scheduler as scheduler_service + + schedules = [ + SimpleNamespace( + id=uuid.uuid4(), + endpoint_id=uuid.uuid4(), + schedule_type="cron", + cron_expression="0 6 * * *", + interval_seconds=None, + ), + SimpleNamespace( + id=uuid.uuid4(), + endpoint_id=uuid.uuid4(), + schedule_type="interval", + cron_expression=None, + interval_seconds=300, + ), + ] + + class FakeSessionContext: + async def __aenter__(self) -> object: + return object() + + async def __aexit__(self, *args: object) -> None: + return None + + class FakeScheduleRepository: + def __init__(self, db: object) -> None: + self.db = db + + async def get_all(self, *, active_only: bool = False) -> list[object]: + assert active_only is True + return cast(list[object], schedules) + + add_job = MagicMock() + monkeypatch.setattr(scheduler_service, "AsyncSessionLocal", FakeSessionContext) + monkeypatch.setattr(scheduler_service, "ScheduleRepository", FakeScheduleRepository) + monkeypatch.setattr(scheduler_service, "add_schedule_job", add_job) + + restored = await scheduler_service.restore_active_schedules() + + assert restored == 2 + assert add_job.call_count == 2 + assert add_job.call_args_list[0].kwargs["schedule_id"] == schedules[0].id + + +@pytest.mark.asyncio +async def test_restore_active_schedules_fails_when_a_job_is_not_registered( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from app.services import scheduler as scheduler_service + + schedule = SimpleNamespace( + id=uuid.uuid4(), + endpoint_id=uuid.uuid4(), + schedule_type="cron", + cron_expression=None, + interval_seconds=None, + ) + + class FakeSessionContext: + async def __aenter__(self) -> object: + return object() + + async def __aexit__(self, *args: object) -> None: + return None + + class FakeScheduleRepository: + def __init__(self, db: object) -> None: + self.db = db + + async def get_all(self, *, active_only: bool = False) -> list[object]: + assert active_only is True + return [schedule] + + monkeypatch.setattr(scheduler_service, "AsyncSessionLocal", FakeSessionContext) + monkeypatch.setattr(scheduler_service, "ScheduleRepository", FakeScheduleRepository) + + with pytest.raises(RuntimeError, match=str(schedule.id)): + await scheduler_service.restore_active_schedules() + + +@pytest.mark.asyncio +async def test_start_scheduler_cleans_up_and_propagates_restore_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from app.services import scheduler as scheduler_service + from apscheduler.schedulers import asyncio as apscheduler_asyncio + + scheduler = MagicMock() + monkeypatch.setattr(apscheduler_asyncio, "AsyncIOScheduler", MagicMock(return_value=scheduler)) + monkeypatch.setattr( + scheduler_service, + "restore_active_schedules", + AsyncMock(side_effect=RuntimeError("database unavailable")), + ) + monkeypatch.setattr(scheduler_service, "_scheduler", None) + + with capture_logs() as logs: + with pytest.raises(RuntimeError, match="database unavailable"): + await scheduler_service.start_scheduler() + + scheduler.shutdown.assert_called_once_with(wait=False) + assert scheduler_service._scheduler is None + failure = next(log for log in logs if log["event"] == "scheduler_start_failed") + assert failure["request_id"] is None + assert failure["user"] == "scheduler" + assert failure["endpoint"] is None + assert failure["status"] is None + assert failure["duration_ms"] is None + assert failure["method"] == "SCHEDULE" + assert failure["client_ip"] is None diff --git a/backend/tests/test_schedules.py b/backend/tests/test_schedules.py index 8c60ef9..fd01272 100644 --- a/backend/tests/test_schedules.py +++ b/backend/tests/test_schedules.py @@ -7,6 +7,7 @@ """ import uuid +from datetime import UTC, datetime import pytest from app.schemas.schedule import ( @@ -66,6 +67,32 @@ def test_schedule_create_invalid_cron_fields() -> None: ) +@pytest.mark.parametrize( + "cron_expression", + [ + "invalid invalid invalid invalid invalid", + "60 * * * *", + "0 24 * * *", + "0 0 32 * *", + "0 0 * 13 *", + "0 0 * * 7", + "*/0 * * * *", + ], +) +def test_schedule_create_rejects_invalid_cron_syntax(cron_expression: str) -> None: + with pytest.raises(ValueError, match="Invalid cron expression"): + ScheduleCreate( + endpoint_id=uuid.uuid4(), + schedule_type="cron", + cron_expression=cron_expression, + ) + + +def test_schedule_update_rejects_invalid_cron_syntax() -> None: + with pytest.raises(ValueError, match="Invalid cron expression"): + ScheduleUpdate(cron_expression="60 * * * *") + + def test_schedule_create_interval_minimum() -> None: with pytest.raises(ValueError): ScheduleCreate( @@ -115,6 +142,38 @@ def test_job_run_response_fields() -> None: assert "error_detail" in fields +def test_job_run_response_allows_deleted_schedule_history() -> None: + now = datetime.now(UTC) + response = JobRunResponse( + id=uuid.uuid4(), + schedule_id=None, + endpoint_id=uuid.uuid4(), + started_at=now, + finished_at=now, + status="success", + row_count=1, + error_detail=None, + created_at=now, + ) + assert response.schedule_id is None + + +def test_job_run_response_allows_deleted_endpoint_history() -> None: + now = datetime.now(UTC) + response = JobRunResponse( + id=uuid.uuid4(), + schedule_id=uuid.uuid4(), + endpoint_id=None, + started_at=now, + finished_at=now, + status="success", + row_count=1, + error_detail=None, + created_at=now, + ) + assert response.endpoint_id is None + + def test_snapshot_response_fields() -> None: fields = SnapshotResponse.model_fields assert "id" in fields @@ -179,6 +238,70 @@ async def test_create_schedule(async_client: object) -> None: assert uuid.UUID(data["id"]) +@pytest.mark.integration +async def test_create_schedule_with_invalid_stored_schema_returns_422( + async_client: object, + db_session: object, +) -> None: + from app.models.endpoint import ApiEndpoint + from httpx import AsyncClient + from sqlalchemy import update + from sqlalchemy.ext.asyncio import AsyncSession + + client: AsyncClient = async_client # type: ignore[assignment] + session: AsyncSession = db_session # type: ignore[assignment] + connection = await client.post( + "/api/v1/admin/connections/", + json={ + "name": f"test-conn-invalid-schema-schedule-{uuid.uuid4().hex[:8]}", + "host": "oracle.example.com", + "service_name": "SVC", + "username": "scott", + "password": "tiger", + }, + ) + endpoint = await client.post( + "/api/v1/admin/endpoints/", + json={ + "name": f"invalid-schema-schedule-{uuid.uuid4().hex[:8]}", + "path": f"invalid-schema-schedule-{uuid.uuid4().hex[:8]}", + "connection_id": connection.json()["id"], + "sql_text": "SELECT * FROM stores WHERE id = :store_id", + "param_schema": { + "store_id": {"type": "integer", "required": True, "default": 1} + }, + "allow_unauthenticated": True, + "data_strategy": "snapshot", + }, + ) + assert endpoint.status_code == 201 + endpoint_id = uuid.UUID(endpoint.json()["id"]) + + await session.execute( + update(ApiEndpoint) + .where(ApiEndpoint.id == endpoint_id) + .values( + param_schema_json={ + "store_id": {"type": "integer", "required": True, "default": "abc"} + } + ) + ) + await session.flush() + session.expire_all() + + response = await client.post( + "/api/v1/admin/schedules/", + json={ + "endpoint_id": str(endpoint_id), + "schedule_type": "interval", + "interval_seconds": 60, + }, + ) + + assert response.status_code == 422 + assert "invalid parameter schema" in response.json()["detail"] + + @pytest.mark.integration async def test_create_duplicate_schedule_returns_409(async_client: object) -> None: from httpx import AsyncClient @@ -332,6 +455,68 @@ async def test_delete_schedule(async_client: object) -> None: assert r_get.status_code == 404 +@pytest.mark.integration +async def test_delete_schedule_preserves_job_run_history( + async_client: object, db_session: object +) -> None: + from app.models.job_run import JobRun, JobRunStatus + from httpx import AsyncClient + from sqlalchemy import select + from sqlalchemy.ext.asyncio import AsyncSession + + client: AsyncClient = async_client # type: ignore[assignment] + db: AsyncSession = db_session # type: ignore[assignment] + + connection = await client.post( + "/api/v1/admin/connections/", + json={ + "name": f"test-conn-history-{uuid.uuid4().hex[:8]}", + "host": "oracle.example.com", + "service_name": "SVC", + "username": "scott", + "password": "test-password", + }, + ) + endpoint = await client.post( + "/api/v1/admin/endpoints/", + json={ + "name": f"history-ep-{uuid.uuid4().hex[:8]}", + "path": f"history-path-{uuid.uuid4().hex[:8]}", + "connection_id": connection.json()["id"], + "allow_unauthenticated": True, + "sql_text": "SELECT 1 FROM dual", + "data_strategy": "snapshot", + }, + ) + schedule = await client.post( + "/api/v1/admin/schedules/", + json={ + "endpoint_id": endpoint.json()["id"], + "schedule_type": "interval", + "interval_seconds": 300, + }, + ) + schedule_id = uuid.UUID(schedule.json()["id"]) + job_run = JobRun( + schedule_id=schedule_id, + endpoint_id=uuid.UUID(endpoint.json()["id"]), + started_at=datetime.now(UTC), + finished_at=datetime.now(UTC), + status=JobRunStatus.success, + row_count=1, + ) + db.add(job_run) + await db.commit() + + deleted = await client.delete(f"/api/v1/admin/schedules/{schedule_id}") + assert deleted.status_code == 204 + + preserved = await db.scalar(select(JobRun).where(JobRun.id == job_run.id)) + assert preserved is not None + await db.refresh(preserved) + assert preserved.schedule_id is None + + @pytest.mark.integration async def test_list_job_runs(async_client: object) -> None: from httpx import AsyncClient diff --git a/backend/tests/test_security.py b/backend/tests/test_security.py index b800fe7..927080e 100644 --- a/backend/tests/test_security.py +++ b/backend/tests/test_security.py @@ -11,6 +11,7 @@ """ import uuid +from datetime import date import pytest from app.schemas.endpoint import EndpointCreate, validate_sql_safety @@ -468,6 +469,114 @@ async def test_missing_required_param(self, async_client: object) -> None: assert r.status_code == 422 assert "id" in r.json()["detail"] + async def test_required_live_param_does_not_fall_back_to_scheduler_default( + self, async_client: object + ) -> None: + client: AsyncClient = async_client # type: ignore[assignment] + + connection = await client.post( + "/api/v1/admin/connections/", + json={ + "name": _unique("required-default-conn"), + "host": "oracle.example.com", + "service_name": "SVC", + "username": "hr", + "password": "secret", + }, + ) + assert connection.status_code == 201 + + ep_path = _unique("required-default-data") + endpoint = await client.post( + "/api/v1/admin/endpoints/", + json={ + "name": _unique("required-default-ep"), + "path": ep_path, + "connection_id": connection.json()["id"], + "allow_unauthenticated": True, + "sql_text": "SELECT * FROM t WHERE business_date = :business_date", + "param_schema": { + "business_date": { + "type": "date", + "required": True, + "default_expression": "yesterday", + } + }, + }, + ) + assert endpoint.status_code == 201 + + response = await client.get(f"/api/v1/data/{ep_path}") + assert response.status_code == 422 + assert "business_date" in response.json()["detail"] + assert "Field required" in response.json()["detail"] + + async def test_live_date_params_accept_dd_mm_yyyy( + self, + async_client: object, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + client: AsyncClient = async_client # type: ignore[assignment] + captured_params: dict[str, object] = {} + + async def fake_execute_query( + **kwargs: object, + ) -> tuple[list[str], list[dict[str, object]], float]: + params = kwargs["params"] + assert isinstance(params, dict) + captured_params.update(params) + return ["ok"], [{"ok": True}], 1.0 + + monkeypatch.setattr("app.services.data.execute_query", fake_execute_query) + + connection = await client.post( + "/api/v1/admin/connections/", + json={ + "name": _unique("date-format-conn"), + "host": "oracle.example.com", + "service_name": "SVC", + "username": "hr", + "password": "secret", + }, + ) + assert connection.status_code == 201 + + ep_path = _unique("date-format-data") + endpoint = await client.post( + "/api/v1/admin/endpoints/", + json={ + "name": _unique("date-format-ep"), + "path": ep_path, + "connection_id": connection.json()["id"], + "allow_unauthenticated": True, + "sql_text": ( + "SELECT * FROM t WHERE business_date BETWEEN :start_date AND :end_date" + ), + "param_schema": { + "start_date": { + "type": "date", + "required": True, + "default_expression": "yesterday", + }, + "end_date": { + "type": "date", + "required": True, + "default_expression": "yesterday", + }, + }, + }, + ) + assert endpoint.status_code == 201 + + response = await client.get( + f"/api/v1/data/{ep_path}?start_date=08-08-2026&end_date=15-08-2026" + ) + assert response.status_code == 200 + assert captured_params == { + "start_date": date(2026, 8, 8), + "end_date": date(2026, 8, 15), + } + async def test_invalid_param_type(self, async_client: object) -> None: client: AsyncClient = async_client # type: ignore[assignment] diff --git a/backend/tests/test_test_database_guard.py b/backend/tests/test_test_database_guard.py new file mode 100644 index 0000000..e1b7d62 --- /dev/null +++ b/backend/tests/test_test_database_guard.py @@ -0,0 +1,41 @@ +"""Regression coverage for the destructive test-database safety guard.""" + +import pytest + +from tests.conftest import _assert_safe_test_database + + +def test_database_guard_accepts_explicit_test_database() -> None: + _assert_safe_test_database( + "postgresql+asyncpg://user:pass@db:5432/db2api_test", "test" + ) + + +@pytest.mark.parametrize( + ("database_url", "app_env"), + [ + ("postgresql+asyncpg://user:pass@db:5432/db2api", "test"), + ("postgresql+asyncpg://user:pass@db:5432/db2api_test", "development"), + ("postgresql+asyncpg://user:pass@db:5432/db2api", "development"), + ], +) +def test_database_guard_rejects_any_non_test_signal( + database_url: str, app_env: str +) -> None: + with pytest.raises(RuntimeError, match="APP_ENV must be 'test'"): + _assert_safe_test_database(database_url, app_env) + + +@pytest.mark.parametrize( + "database_url", + [ + "postgresql+asyncpg://test_runner:pass@db:5432/customer_data", + "postgresql+asyncpg://user:pass@test-db:5432/customer_data", + "postgresql+asyncpg://user:pass@db:5432/customer_data?application_name=test", + ], +) +def test_database_guard_ignores_test_marker_outside_database_name( + database_url: str, +) -> None: + with pytest.raises(RuntimeError, match="database name must contain 'test'"): + _assert_safe_test_database(database_url, "test") diff --git a/docs/architecture.md b/docs/architecture.md index 7a01749..b618e1c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -23,8 +23,9 @@ │ ┌───────────▼───────┐ ┌─────────────▼──────┐ │ │ │ PostgreSQL │ │ Oracle Database │ │ │ │ (app metadata, │ │ (user data source)│ │ -│ │ job store, │ │ │ │ -│ │ snapshots) │ │ python-oracledb │ │ +│ │ schedule defs, │ │ │ │ +│ │ job history, │ │ python-oracledb │ │ +│ │ snapshots) │ │ │ │ │ └───────────────────┘ └────────────────────┘ │ └───────────────────────────────────────────────────────────────────┘ ``` @@ -39,23 +40,33 @@ ## Key Design Decisions ### SQL Safety + All user-defined SQL is executed via SQLAlchemy `text()` with named bind parameters (`:param_name`). String interpolation of user input is prohibited at all layers. Bind values are validated through typed Pydantic schemas before reaching the query executor. +Live data requests enforce every parameter marked `required`, even when that parameter also has a scheduler default. Defaults exist so scheduled execution can run without request input and so optional live parameters can be omitted. Date query parameters accept `YYYY-MM-DD` and `DD-MM-YYYY`; both formats are normalized to Python `date` values before database binding. + ### Authentication + - Admin API: session-based or JWT Bearer (TBD per Phase 1). -- Data endpoints: per-endpoint configurable auth — Bearer token, Basic Auth, or API key. Middleware resolves the policy from endpoint metadata at request time and enforces it when an auth method is attached. Endpoints published without an auth method are served publicly, so one must be assigned to any endpoint that should be protected. +- Data endpoints: every `/api/v1/data/*` request is authenticated. Middleware enforces the endpoint's Bearer token, Basic Auth, or API key method when configured; otherwise it requires the platform admin Bearer token. Anonymous data access is never allowed. - Credentials are hashed with `bcrypt`; tokens are issued/verified with `PyJWT`. ### Scheduler -APScheduler 3.x runs in-process with an in-memory job store. Schedule definitions are persisted in the app database (PostgreSQL), and the corresponding APScheduler jobs are (re)registered when a schedule is created, updated, or resumed. The in-memory jobs are not automatically reloaded on process restart — active schedule rows remain in the database, but their jobs are re-registered on the next create/update/resume. Execution telemetry (start time, duration, row count, status, errors) is written to the app DB and exposed through admin APIs. + +APScheduler 3.x runs in-process with an in-memory job store. Schedule definitions are persisted in PostgreSQL, and every active schedule is registered when it is created, updated, resumed, or restored during API startup. Execution telemetry (start time, duration, row count, status, errors) is written to the app DB and exposed through admin APIs. Deleting a schedule sets the historical job run's `schedule_id` to `NULL`, preserving both audit history and snapshots. Deleting an endpoint removes its schedule and cached snapshots, unregisters its in-memory job after the database commit, and sets the historical job run's `endpoint_id` (and cascaded `schedule_id`) to `NULL` so the audit record remains available. + +The scheduler is intentionally single-process. Run one API process/replica unless distributed scheduler coordination is added; otherwise each process would register and execute the same persisted schedules. ### Snapshot Cache -Scheduled endpoints can serve results from a PostgreSQL JSONB snapshot rather than executing live queries. Freshness metadata is returned in responses. Stale snapshot fallback behavior is configurable per endpoint. + +Scheduled endpoints can serve results from a PostgreSQL JSONB snapshot rather than executing live queries. Every bind parameter on a snapshot endpoint must define a fixed default, an explicit SQL `NULL` default for an optional bind, or a dynamic date expression (`today` or `yesterday`) so refreshes never depend on request inputs. Dynamic defaults are resolved from the application server date at execution time, while explicit null defaults remain present in the bind dictionary with a `None` value so the database receives SQL `NULL`. Freshness metadata is returned in responses. Stale snapshot fallback behavior is configurable per endpoint. ### Configuration + Pydantic Settings v2 loads all configuration from environment variables (`.env` in development, injected secrets in production). No hardcoded configuration values in source code. ### Logging + `structlog` emits structured JSON with mandatory correlation fields (`request_id`, `user`, `endpoint`, `status`, `duration_ms`, `event`). Sensitive fields are redacted at middleware level before emission. ## Directory Layout diff --git a/docs/deployment.md b/docs/deployment.md index fab2ffb..4bf8b97 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -86,7 +86,7 @@ pip install -r requirements.txt alembic upgrade head # Start the server -uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers 4 +uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers 1 ``` For production, use a process manager: @@ -96,12 +96,14 @@ For production, use a process manager: pip install gunicorn gunicorn app.main:app \ --worker-class uvicorn.workers.UvicornWorker \ - --workers 4 \ + --workers 1 \ --bind 0.0.0.0:8000 \ --access-logfile - \ --error-logfile - ``` +Keep exactly one API worker while APScheduler uses its in-process job store. Each worker restores active schedules during startup, so multiple workers or replicas would execute duplicate snapshot refreshes until distributed scheduler coordination is implemented. + #### Frontend Setup ```bash @@ -156,26 +158,50 @@ psql -c "CREATE DATABASE db2api_db OWNER db2api_user;" Use the Docker images as a starting point. Key considerations: -- Deploy the API as a `Deployment` with `replicas: 2+` +- Deploy the API as a `Deployment` with `replicas: 1` and `strategy.type: Recreate` + while the scheduler is in-process. A rolling update can overlap the old and new + Pods even with one replica, causing both schedulers to execute the same jobs. +- Add distributed scheduler coordination before using rolling updates or multiple + API replicas. - Use a `Service` for internal routing and an `Ingress` for external access - PostgreSQL: use a managed service (e.g., AWS RDS, GCP Cloud SQL) or a StatefulSet - Store `ENCRYPTION_KEY` and `DATABASE_URL` as Kubernetes Secrets - Configure liveness and readiness probes: ```yaml -livenessProbe: - httpGet: - path: /api/v1/admin/health/live - port: 8000 - initialDelaySeconds: 10 - periodSeconds: 30 - -readinessProbe: - httpGet: - path: /api/v1/admin/health/ready - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 10 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: querygateway-api +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: querygateway-api + template: + metadata: + labels: + app: querygateway-api + spec: + containers: + - name: api + image: your-registry/querygateway-api:latest + ports: + - containerPort: 8000 + livenessProbe: + httpGet: + path: /api/v1/admin/health/live + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /api/v1/admin/health/ready + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 ``` ## Database Migrations @@ -221,7 +247,7 @@ alembic downgrade -1 4. **Create first connection**: Use the Connections page to add an Oracle data source 5. **Test connection**: Click "Test" to verify Oracle connectivity 6. **Create first endpoint**: Use the API Endpoints page wizard -7. **Verify data endpoint**: `curl http://localhost:8000/api/v1/data/` +7. **Verify data endpoint**: `curl -H "Authorization: Bearer " http://localhost:8000/api/v1/data/` ## Security Hardening for Production diff --git a/docs/security_checklist.md b/docs/security_checklist.md index 11aaa2c..5e9fb1c 100644 --- a/docs/security_checklist.md +++ b/docs/security_checklist.md @@ -96,7 +96,7 @@ Comprehensive security validation for QueryGateway production deployments. All i | # | Check | Status | Notes | |---|-------|--------|-------| -| 51 | Public (no-auth) endpoints require explicit opt-in; orphaned ones default-deny | Verified | M1: create/update reject `auth_method_id=None` without `allow_unauthenticated=true` (422). Data plane **default-denies with 401** (`unauthenticated_endpoint_denied`) when an endpoint has no auth method and no opt-in — e.g. after its auth method is deleted (FK SET NULL); genuinely public hits log `public_endpoint_served` | +| 51 | Every dynamic data endpoint requires authentication | Verified | A configured endpoint method is enforced when present. Otherwise the legacy `allow_unauthenticated=true` value opts into platform-admin Bearer fallback; it never permits anonymous access. Missing or invalid credentials return 401 and log `unauthenticated_endpoint_denied`. | | 52 | App refuses to boot with wildcard CORS under credentials | Verified | M3: `cors_origins` validator rejects `*` at startup | | 53 | API keys accepted only via header, not query string | Verified | L1: `X-Api-Key` only; `?api_key=` fallback removed | | 54 | Sensitive keys redacted at the logging boundary | Verified | L2: structlog processor masks password/secret/token/api_key/key/authorization/signing_secret | diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 78f3f78..a32a298 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -121,6 +121,7 @@ "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -481,6 +482,7 @@ "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.39.16.tgz", "integrity": "sha512-m6S22fFpKtOWhq8HuhzsI1WzUP/hB9THbDj0Tl5KX4gbO6Y91hwBl7Yky33NdvB6IffuRFiBxf1R8kJMyXmA4Q==", "license": "MIT", + "peer": true, "dependencies": { "@codemirror/state": "^6.5.0", "crelt": "^1.0.6", @@ -576,6 +578,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -599,6 +602,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -1903,8 +1907,7 @@ "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/babel__core": { "version": "7.20.5", @@ -1989,6 +1992,7 @@ "integrity": "sha512-akNQMv0wW5uyRpD2v2IEyRSZiR+BeGuoB6L310EgGObO44HSMNT8z1xzio28V8qOrgYaopIDNA18YgdXd+qTiw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -2006,6 +2010,7 @@ "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -2017,6 +2022,7 @@ "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", "dev": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^18.0.0" } @@ -2066,6 +2072,7 @@ "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -2220,16 +2227,16 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", - "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { @@ -2509,6 +2516,7 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2559,7 +2567,6 @@ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -2763,9 +2770,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.15", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", - "integrity": "sha512-EwOCDEex4quD37XhqM3omwtMoJjr//isUZz1JopUNWms+4Z2ViyM/k1YIRePpoVNnQhENnxtFjLaxNHrT7xIUg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -2806,6 +2813,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -3196,8 +3204,7 @@ "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/dunder-proto": { "version": "1.0.1", @@ -3356,6 +3363,7 @@ "integrity": "sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -4087,6 +4095,7 @@ "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", "dev": true, "license": "MIT", + "peer": true, "bin": { "jiti": "bin/jiti.js" } @@ -4098,9 +4107,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", - "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { @@ -4126,6 +4135,7 @@ "integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "cssstyle": "^4.1.0", "data-urls": "^5.0.0", @@ -4312,7 +4322,6 @@ "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "lz-string": "bin/bin.js" } @@ -4423,9 +4432,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.15", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", - "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -4657,9 +4666,9 @@ } }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", "dev": true, "funding": [ { @@ -4676,8 +4685,9 @@ } ], "license": "MIT", + "peer": true, "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -4849,6 +4859,7 @@ "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -4952,7 +4963,6 @@ "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -4968,7 +4978,6 @@ "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -5021,6 +5030,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -5033,6 +5043,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -5046,8 +5057,7 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/react-refresh": { "version": "0.17.0", @@ -5060,9 +5070,9 @@ } }, "node_modules/react-router": { - "version": "7.18.0", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.0.tgz", - "integrity": "sha512-pTTGt8J+ji1NOmYnjzT+bAJy/1zD+Jp4ziO6cL7T3ZLvXKtusO7BpFqlRXitqpcPVqllsIXFHRMt+2/k3Xn6HQ==", + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.3.tgz", + "integrity": "sha512-gyXgtdr5uACJ5b1Q4udzjVV+tb/rlHIMJKuJ0e89R4Kzgz47z/rgP0dIKxktqIEUhDHluGTPJJH/wRha7CyqsA==", "license": "MIT", "dependencies": { "cookie": "^1.0.1", @@ -5082,12 +5092,12 @@ } }, "node_modules/react-router-dom": { - "version": "7.18.0", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.18.0.tgz", - "integrity": "sha512-Fi0yY6kgtKae/Th2xibdWK0KSdYZ4B53Gyf6wRtomOKWgpNm7H7+DyfDhncdz9FKbpS+1jmDhg3F4WoGJ+yFOA==", + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.18.3.tgz", + "integrity": "sha512-ytVbyBBM7vMfRCam25r0WMhSVSom909A8p+8m0/f1w853dz/xfFu6etAT2SEbVoSnI+ZoPRDqIsQXVT89gp7kg==", "license": "MIT", "dependencies": { - "react-router": "7.18.0" + "react-router": "7.18.3" }, "engines": { "node": ">=20.0.0" @@ -5455,6 +5465,7 @@ "integrity": "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -5582,6 +5593,7 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -5697,6 +5709,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -5790,6 +5803,7 @@ "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", @@ -5883,6 +5897,7 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, diff --git a/frontend/src/components/endpoints/EndpointWizard.test.tsx b/frontend/src/components/endpoints/EndpointWizard.test.tsx new file mode 100644 index 0000000..1a51a58 --- /dev/null +++ b/frontend/src/components/endpoints/EndpointWizard.test.tsx @@ -0,0 +1,127 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const listConnectionsMock = vi.fn(); +const listAuthMethodsMock = vi.fn(); +const previewMock = vi.fn(); +const createMock = vi.fn(); + +vi.mock("@/components/endpoints/SqlEditor", () => ({ + SqlEditor: ({ value, onChange }: { value: string; onChange: (value: string) => void }) => ( +