diff --git a/mod_api/__init__.py b/mod_api/__init__.py index a54a9413..d5b080bf 100644 --- a/mod_api/__init__.py +++ b/mod_api/__init__.py @@ -33,9 +33,11 @@ # routing-level 404s/405s that never enter the blueprint). mod_api.after_app_request(error_handler.convert_api_errors_to_json) -# Route modules register themselves against the blueprint; the rest of -# the stack adds one module per PR. +# Route modules from mod_api.routes import auth as auth_routes # noqa: E402, F401 +from mod_api.routes import \ + errors_logs as errors_logs_routes # noqa: E402, F401 +from mod_api.routes import results as results_routes # noqa: E402, F401 from mod_api.routes import runs as runs_routes # noqa: E402, F401 from mod_api.routes import samples as samples_routes # noqa: E402, F401 from mod_api.routes import system as system_routes # noqa: E402, F401 diff --git a/mod_api/routes/errors_logs.py b/mod_api/routes/errors_logs.py new file mode 100644 index 00000000..f3708dd7 --- /dev/null +++ b/mod_api/routes/errors_logs.py @@ -0,0 +1,195 @@ +""" +Error and build log routes. + +GET /runs/{id}/errors Test-level errors for a run +GET /runs/{id}/infrastructure-errors Infra errors (VM, build, worker) +GET /runs/{id}/error-summary Grouped error counts +GET /runs/{id}/logs Build log (cursor-paginated) +GET /runs/{id}/samples/{sid}/logs Per-sample logs (not yet available) +""" + +from flask import g, request + +from mod_api import mod_api +from mod_api.middleware.auth import require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_cursor_pagination, + validate_offset_pagination, + validate_path_id) +from mod_api.models.api_token import Scope +from mod_api.services.error_service import (derive_error_summary, + derive_errors_for_run, + derive_infrastructure_errors) +from mod_api.services.log_service import read_log_lines +from mod_api.utils import cursor_paginated_response, paginated_response +from mod_auth.models import Role +from mod_test.models import Test + + +@mod_api.route('/runs//errors', methods=['GET']) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def list_run_errors(run_id, limit=50, offset=0): + """List test errors for a run, derived from result and output data.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + errors = derive_errors_for_run(run_id) + + error_type = request.args.get('type') + if error_type: + errors = [e for e in errors if e['type'] == error_type] + + severity = request.args.get('severity') + if severity: + errors = [e for e in errors if e['severity'] == severity] + + sample_id = request.args.get('sample_id', type=int) + if sample_id: + errors = [e for e in errors if e.get('sample_id') == sample_id] + + total = len(errors) + paged = errors[offset:offset + limit] + + return paginated_response(paged, total, limit, offset) + + +@mod_api.route('/runs//infrastructure-errors', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def list_infrastructure_errors(run_id, limit=50, offset=0): + """ + Infra errors classified from TestProgress messages on a best-effort basis. + + Stack traces are opt-in because they may contain internal paths. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + include_stack = request.args.get( + 'include_stack', 'false').lower() == 'true' + if include_stack: + user = getattr(g, 'api_user', None) + allowed = (Role.admin, Role.contributor) + if user is None or user.role not in allowed: + return make_error_response( + 'forbidden', + 'Stack traces require admin or contributor role.', + details={'required_roles': [role.value for role in allowed]}, + http_status=403, + ) + + errors = derive_infrastructure_errors(run_id) + + if not include_stack: + for e in errors: + e.pop('stack', None) + + # Apply optional type and severity filters. + error_type = request.args.get('type') + if error_type: + errors = [e for e in errors if e.get('type') == error_type] + + severity = request.args.get('severity') + if severity: + errors = [e for e in errors if e.get('severity') == severity] + + total = len(errors) + paged = errors[offset:offset + limit] + return paginated_response(paged, total, limit, offset) + + +@mod_api.route('/runs//error-summary', methods=['GET']) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def get_error_summary(run_id, limit=50, offset=0): + """Group error summary for triaging a run before drilling into details.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + group_by = request.args.get('group_by', 'type') + if group_by not in ('type', 'severity', 'sample_id', 'regression_id'): + return make_error_response( + 'validation_error', + 'group_by must be one of: type, severity, sample_id, regression_id.', + http_status=400, + ) + + severity = request.args.get('severity') + + summary = derive_error_summary(run_id, group_by=group_by) + + if severity: + summary = [s for s in summary if s.get('severity') == severity] + + total = len(summary) + paged = summary[offset:offset + limit] + return paginated_response(paged, total, limit, offset) + + +@mod_api.route('/runs//logs', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +@validate_path_id('run_id') +@validate_cursor_pagination(default_limit=100) +def get_run_logs(run_id, limit=100, cursor=None): + """ + Read a run's build log with cursor-based pagination. + + Returns 404 (not a broken download link) when the file doesn't exist. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + level = request.args.get('level') + source = request.args.get('source') + contains = request.args.get('contains') + if contains and len(contains) > 100: + return make_error_response( + 'validation_error', + 'contains parameter must be 100 characters or less.', + http_status=400, + ) + + try: + lines, next_cursor = read_log_lines( + run_id, + cursor=cursor, + limit=limit, + level=level, + source=source, + contains=contains, + ) + except FileNotFoundError: + return make_error_response( + 'log_not_found', + f'Log file for run {run_id} is not available locally. ' + 'It may have been moved to cold storage. Please download it via the artifacts API.', + details={'run_id': run_id, + 'action_required': f'Use GET /runs/{run_id}/artifacts (type=build_log)'}, + http_status=404, + ) + + return cursor_paginated_response(lines, next_cursor, limit) + + +@mod_api.route('/runs//samples//logs', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_offset_pagination() +def get_sample_logs(run_id, sample_id, limit=50, offset=0): + """Per-sample logs aren't available yet — the CI worker doesn't support them.""" + return make_error_response( + 'not_found', + f'Per-sample logs are not available for sample {sample_id} in run {run_id}.', + details={ + 'reason': 'Per-sample log storage is not yet supported by the CI worker.'}, + http_status=404, + ) diff --git a/mod_api/routes/results.py b/mod_api/routes/results.py new file mode 100644 index 00000000..57a49078 --- /dev/null +++ b/mod_api/routes/results.py @@ -0,0 +1,475 @@ +""" +Expected/actual output, diffs, and baseline approval routes. + +GET /runs/{id}/samples/{sid}/regression-tests/{rid}/outputs/{oid}/expected +GET /runs/{id}/samples/{sid}/regression-tests/{rid}/outputs/{oid}/actual +GET /runs/{id}/samples/{sid}/regression-tests/{rid}/outputs/{oid}/diff +POST /runs/{id}/samples/{sid}/baseline-approval Approve a new baseline +""" + +import base64 +import os + +from flask import current_app, g, redirect, request, url_for + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import validate_body, validate_path_id +from mod_api.models.api_token import Scope +from mod_api.schemas.results import (BaselineApprovalRequestSchema, + BaselineApprovalSchema, + OutputFileContentSchema) +from mod_api.services.diff_service import compute_diff, file_sha256, read_lines +from mod_api.services.status import is_dummy_row +from mod_api.services.storage import (get_test_results_base_path, + resolve_artifact) +from mod_api.utils import safe_resolve, single_response +from mod_auth.models import Role +from mod_regression.models import RegressionTestOutputFiles +from mod_test.models import Test, TestResultFile + +INVALID_PATH_MSG = 'Invalid file path.' +READ_ERROR_MSG = 'Failed to read file.' + + +def _find_result_file(run_id, regression_test_id, output_id=None): + """ + Look up the right TestResultFile row. + + Uses run_id + regression_test_id from the path. If output_id is + given as a query param, narrow to that specific output file. + """ + query = TestResultFile.query.filter_by( + test_id=run_id, + regression_test_id=regression_test_id, + ) + + if output_id is not None: + query = query.filter_by(regression_test_output_id=output_id) + + return query.first() + + +def _validate_result_file_access(run_id, sample_id, regression_id, output_id): + """Validate access to a result file and return it, or an error response.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return None, make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + result_file = _find_result_file(run_id, regression_id, output_id) + + if result_file is None: + return None, make_error_response( + 'not_found', + f'No result for regression test {regression_id}.', + http_status=404, + ) + + actual_sample_id = ( + result_file.regression_test.sample_id + if result_file.regression_test else None + ) + if actual_sample_id != sample_id: + return None, make_error_response( + 'not_found', + f'Regression test {regression_id} does not belong to sample {sample_id}.', + http_status=404, + ) + + return result_file, None + + +def _read_output_file(file_path, fmt, is_expected=True): + """Read an output file into the response dict. + + The sha256 in the result is always computed over the complete file, + even when content is truncated to the 1 MiB inline limit. + """ + if not os.path.isfile(file_path): + type_str = 'Expected' if is_expected else 'Actual' + return None, make_error_response( + 'not_found', + f'{type_str} output file not found on disk.', + http_status=404, + ) + + sha256 = file_sha256(file_path) + file_size = os.path.getsize(file_path) + truncated = False + download_url = None + + if file_size > 1048576: + truncated = True + filename = os.path.basename(file_path) + download_url, _ = resolve_artifact(f'TestResults/{filename}') + + if fmt == 'text': + try: + with open(file_path, 'r', encoding='utf-8', errors='replace') as f: + content = f.read(1048576) + encoding = 'utf-8' + except Exception: + return None, make_error_response('internal_error', READ_ERROR_MSG, http_status=500) + else: + try: + with open(file_path, 'rb') as f: + content = base64.b64encode(f.read(1048576)).decode('ascii') + encoding = 'base64' + except Exception: + return None, make_error_response('internal_error', READ_ERROR_MSG, http_status=500) + + return { + 'content': content, + 'encoding': encoding, + 'sha256': sha256, + 'truncated': truncated, + 'download_url': download_url, + }, None + + +@mod_api.route( + '/runs//samples//regression-tests//outputs//expected', + methods=['GET'] +) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_path_id('regression_id') +@validate_path_id('output_id') +def get_expected_output(run_id, sample_id, regression_id, output_id): + """Return the expected output file for a regression test result.""" + result_file, err = _validate_result_file_access( + run_id, sample_id, regression_id, output_id) + if err: + return err + + if is_dummy_row(result_file): + return make_error_response('not_found', 'Expected output not found.', http_status=404) + + base_path = get_test_results_base_path() + expected_filename = result_file.expected + ext = '' + if result_file.regression_test_output: + ext = result_file.regression_test_output.correct_extension + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + expected_filename += ext + + file_path = safe_resolve(base_path, expected_filename) + if file_path is None: + return make_error_response('forbidden', INVALID_PATH_MSG, http_status=403) + + fmt = request.args.get('format', 'base64') + + data, err = _read_output_file(file_path, fmt, is_expected=True) + if err: + return err + + content = data['content'] + encoding = data['encoding'] + sha256 = data['sha256'] + truncated = data['truncated'] + download_url = data['download_url'] + + _, storage_status = resolve_artifact(f'TestResults/{expected_filename}') + + return single_response({ + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': result_file.regression_test_id, + 'output_id': result_file.regression_test_output_id, + 'filename': expected_filename, + 'content_type': 'application/octet-stream', + 'encoding': encoding, + 'content': content, + 'truncated': truncated, + 'download_url': download_url, + 'sha256': sha256, + 'storage_status': storage_status, + }, schema=OutputFileContentSchema()) + + +@mod_api.route( + '/runs//samples//regression-tests//outputs//actual', + methods=['GET'] +) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_path_id('regression_id') +@validate_path_id('output_id') +def get_actual_output(run_id, sample_id, regression_id, output_id): + """ + Return the actual output file for a regression test result. + + got=null in the DB means the output matched expected — not that it's + missing. We return 303 (redirect to expected) in that case. Missing + output (the dummy sentinel row) returns 404. + """ + result_file, err = _validate_result_file_access( + run_id, sample_id, regression_id, output_id) + if err: + return err + + if is_dummy_row(result_file): + return make_error_response( + 'missing_output', + 'Test produced no output when output was expected.', + http_status=404, + ) + + if result_file.got is None: + # Relative redirect: clients only resend Authorization headers on + # same-origin redirects, and an absolute URL would break behind a + # reverse proxy anyway. + return redirect(url_for( + 'api.get_expected_output', + run_id=run_id, + sample_id=sample_id, + regression_id=regression_id, + output_id=output_id, + format=request.args.get('format', 'base64'), + ), code=303) + + base_path = get_test_results_base_path() + actual_filename = result_file.got + if result_file.regression_test_output: + ext = result_file.regression_test_output.correct_extension + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + actual_filename += ext + + file_path = safe_resolve(base_path, actual_filename) + if file_path is None: + return make_error_response('forbidden', INVALID_PATH_MSG, http_status=403) + + fmt = request.args.get('format', 'base64') + + data, err = _read_output_file(file_path, fmt, is_expected=False) + if err: + return err + + content = data['content'] + encoding = data['encoding'] + sha256 = data['sha256'] + truncated = data['truncated'] + download_url = data['download_url'] + + _, storage_status = resolve_artifact(f'TestResults/{actual_filename}') + + return single_response({ + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': result_file.regression_test_id, + 'output_id': result_file.regression_test_output_id, + 'filename': actual_filename, + 'content_type': 'application/octet-stream', + 'encoding': encoding, + 'content': content, + 'truncated': truncated, + 'download_url': download_url, + 'sha256': sha256, + 'storage_status': storage_status, + }, schema=OutputFileContentSchema()) + + +def _handle_missing_diff(result_file, format_type, diff_ids): + if is_dummy_row(result_file): + if format_type == 'unified': + return single_response({**diff_ids, 'format': 'unified', 'content': ''}) + return single_response({ + **diff_ids, + 'status': 'missing_actual', + 'format': 'structured', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + }) + + if result_file.got is None: + if format_type == 'unified': + return single_response({**diff_ids, 'format': 'unified', 'content': ''}) + return single_response({ + **diff_ids, + 'status': 'identical', + 'format': 'structured', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + }) + return None + + +@mod_api.route( + '/runs//samples//regression-tests//outputs//diff', + methods=['GET'] +) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_path_id('regression_id') +@validate_path_id('output_id') +def get_diff(run_id, sample_id, regression_id, output_id): + """Structured diff between expected and actual output.""" + result_file, err = _validate_result_file_access( + run_id, sample_id, regression_id, output_id) + if err: + return err + + diff_ids = { + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': result_file.regression_test_id, + 'output_id': result_file.regression_test_output_id, + } + + format_type = request.args.get('format', 'structured') + + missing_response = _handle_missing_diff(result_file, format_type, diff_ids) + if missing_response: + return missing_response + + base_path = get_test_results_base_path() + ext = result_file.regression_test_output.correct_extension if result_file.regression_test_output else '' + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + expected_path = safe_resolve(base_path, result_file.expected + ext) + actual_path = safe_resolve(base_path, result_file.got + ext) + + if expected_path is None or actual_path is None: + return make_error_response('forbidden', INVALID_PATH_MSG, http_status=403) + + if not os.path.isfile(expected_path): + return make_error_response('not_found', 'Expected output file not found on disk.', http_status=404) + if not os.path.isfile(actual_path): + return make_error_response('not_found', 'Actual output file not found on disk.', http_status=404) + + max_diff_bytes = 10 * 1024 * 1024 # 10 MiB + if os.path.getsize(expected_path) > max_diff_bytes or os.path.getsize(actual_path) > max_diff_bytes: + return make_error_response('unprocessable', 'File too large for diff. Use download_url.', http_status=422) + + if format_type == 'unified': + import difflib + expected_lines = read_lines(expected_path) + actual_lines = read_lines(actual_path) + differ = list(difflib.unified_diff( + expected_lines, + actual_lines, + fromfile='expected', + tofile='actual', + lineterm='' + )) + if len(differ) > 10000: + differ = differ[:10000] + differ.append("\n... Diff truncated due to length ...") + unified_content = '\n'.join(differ) + return single_response({ + **diff_ids, + 'format': 'unified', + 'content': unified_content + }) + + context_lines = request.args.get('context_lines', 3, type=int) + context_lines = max(1, min(context_lines, 50)) + + diff_result = compute_diff( + expected_path, actual_path, context_lines=context_lines) + diff_result.update(diff_ids) + diff_result['format'] = 'structured' + return single_response(diff_result) + + +@mod_api.route('/runs//samples//baseline-approval', methods=['POST']) +@require_roles([Role.admin]) +@require_scope(Scope.BASELINES_WRITE) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_body(BaselineApprovalRequestSchema) +def create_baseline_approval(run_id, sample_id, validated_data=None): + """ + Record intent to approve actual output as the new expected baseline. + + WARNING: When remove_variants is set to true, this action will remove all + platform-specific variants, making this output the single source of truth + across all platforms. Care should be taken as this applies globally. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + regression_id = validated_data['regression_id'] + output_id = validated_data['output_id'] + + result_file = TestResultFile.query.filter_by( + test_id=run_id, + regression_test_id=regression_id, + regression_test_output_id=output_id, + ).first() + + if result_file is None: + return make_error_response('not_found', 'Result file not found.', http_status=404) + + actual_sample_id = ( + result_file.regression_test.sample_id + if result_file.regression_test else None + ) + if actual_sample_id != sample_id: + return make_error_response( + 'not_found', + f'Regression test {regression_id} does not belong to sample {sample_id}.', + http_status=404, + ) + + if is_dummy_row(result_file): + return make_error_response('unprocessable', 'Cannot approve a dummy row.', http_status=422) + + if result_file.got is None: + return make_error_response('unprocessable', 'Output already matches expected.', http_status=422) + + # The actual output file (named by its hash) is already in TestResults/. + # We just need to update the RegressionTestOutput to point to this new hash. + rto = result_file.regression_test_output + if rto is None: + return make_error_response('internal_error', 'No RegressionTestOutput linked.', http_status=500) + + new_baseline = result_file.got + + base_path = get_test_results_base_path() + ext = rto.correct_extension or '' + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + actual_filename = new_baseline + ext + file_path = safe_resolve(base_path, actual_filename) + if not file_path or not os.path.isfile(file_path): + return make_error_response('unprocessable', 'Actual output file not found in storage.', http_status=422) + + old_baseline = rto.correct + rto.correct = new_baseline + + remove_variants = validated_data.get('remove_variants', False) + if remove_variants: + RegressionTestOutputFiles.query.filter_by( + regression_test_output_id=rto.id).delete() + + g.db.commit() + + # Audit log: this mutates the global expected baseline (and optionally + # deletes all variants), so record who approved what. + approver = getattr(g, 'api_user', None) + current_app.logger.info( + 'Baseline approved by %s (user_id=%s): run=%s regression=%s ' + 'output=%s old_hash=%s new_hash=%s remove_variants=%s', + approver.name if approver else 'unknown', + approver.id if approver else None, + run_id, regression_id, output_id, old_baseline, new_baseline, + remove_variants) + + import datetime + return single_response({ + 'status': 'approved', + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': regression_id, + 'output_id': output_id, + 'requested_by': getattr(g, 'api_user').name if getattr(g, 'api_user', None) else 'unknown', + 'created_at': datetime.datetime.now(datetime.timezone.utc) + }, schema=BaselineApprovalSchema()) diff --git a/mod_api/schemas/results.py b/mod_api/schemas/results.py new file mode 100644 index 00000000..fe054dc2 --- /dev/null +++ b/mod_api/schemas/results.py @@ -0,0 +1,64 @@ +"""Schemas for output file content and baseline approvals.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_api.schemas.common import DATETIME_FORMAT + + +class OutputFileContentSchema(Schema): + """File content blob returned for expected or actual output.""" + + run_id = fields.Integer(allow_none=True) + sample_id = fields.Integer(required=True) + regression_id = fields.Integer(required=True) + output_id = fields.Integer(required=True) + filename = fields.String(required=True) + content_type = fields.String(required=True) + encoding = fields.String( + required=True, validate=validate.OneOf(['utf-8', 'base64'])) + content = fields.String(required=True) + # sha256 always covers the complete file, even when content is truncated. + sha256 = fields.String(allow_none=True) + truncated = fields.Boolean(load_default=False) + download_url = fields.String(allow_none=True) + storage_status = fields.String( + required=True, + validate=validate.OneOf(['ok', 'degraded', 'missing']), + ) + + +class BaselineApprovalRequestSchema(Schema): + """POST /runs/{id}/samples/{sid}/baseline-approval body.""" + + regression_id = fields.Integer( + required=True, + validate=validate.Range(min=1), + ) + output_id = fields.Integer( + required=True, + validate=validate.Range(min=1), + ) + + remove_variants = fields.Boolean( + load_default=False, + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class BaselineApprovalSchema(Schema): + """Response after a baseline approval is applied.""" + + status = fields.String( + required=True, + validate=validate.OneOf( + ['approved'])) + run_id = fields.Integer(required=True) + sample_id = fields.Integer(required=True) + regression_id = fields.Integer(required=True) + output_id = fields.Integer(required=True) + requested_by = fields.String(required=True) + created_at = fields.DateTime(required=True, format=DATETIME_FORMAT) diff --git a/mod_api/services/diff_service.py b/mod_api/services/diff_service.py new file mode 100644 index 00000000..f527c57b --- /dev/null +++ b/mod_api/services/diff_service.py @@ -0,0 +1,220 @@ +""" +Structured diff computation between expected and actual output files. + +Produces JSON hunks with line-level detail instead of the legacy HTML +diff output. Uses difflib.unified_diff internally. +""" + +import difflib +import hashlib +import os +import re +from typing import Any, Dict, List, Optional, Tuple + +from mod_api.services.storage import get_test_results_base_path + + +def compute_diff( + expected_path: str, + actual_path: str, + context_lines: int = 3, + max_hunks: int = 500, +) -> Dict[str, Any]: + """ + Compute a structured diff between two files. + + Returns a dict matching the Diff schema: status, summary (added_lines, + removed_lines, changed_hunks), and a list of hunks. + """ + context_lines = max(1, min(context_lines, 50)) + + if not os.path.isfile(expected_path): + return { + 'status': 'missing_expected', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + } + + if not os.path.isfile(actual_path): + return { + 'status': 'missing_actual', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + } + + expected_lines = read_lines(expected_path) + actual_lines = read_lines(actual_path) + + if expected_lines == actual_lines: + return { + 'status': 'identical', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + } + + hunks = _compute_hunks(expected_lines, actual_lines, + context_lines, max_hunks) + added = sum( + 1 for h in hunks for line in h['lines'] if line['kind'] == 'added') + removed = sum( + 1 for h in hunks for line in h['lines'] if line['kind'] == 'removed') + + return { + 'status': 'different', + 'summary': { + 'added_lines': added, + 'removed_lines': removed, + 'changed_hunks': len(hunks), + }, + 'hunks': hunks, + } + + +# Matches the @@ -a,b +c,d @@ header line from unified_diff. +_HUNK_RE = re.compile(r'^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@') + + +def _process_diff_line(line, current_hunk, expected_line_num, actual_line_num): + if line.startswith('+'): + current_hunk['lines'].append({ + 'kind': 'added', + 'expected_line': None, + 'actual_line': actual_line_num, + 'text': line[1:], + }) + actual_line_num += 1 + elif line.startswith('-'): + current_hunk['lines'].append({ + 'kind': 'removed', + 'expected_line': expected_line_num, + 'actual_line': None, + 'text': line[1:], + }) + expected_line_num += 1 + else: + content = line[1:] if line.startswith(' ') else line + current_hunk['lines'].append({ + 'kind': 'context', + 'expected_line': expected_line_num, + 'actual_line': actual_line_num, + 'text': content, + }) + expected_line_num += 1 + actual_line_num += 1 + return expected_line_num, actual_line_num + + +def _process_hunk_header( + line: str, + current_hunk: Optional[Dict[str, Any]], + hunks: List[Dict[str, Any]], + max_hunks: int +) -> Tuple[Optional[Dict[str, Any]], int, int, bool]: + if current_hunk and len(hunks) >= max_hunks: + return None, 0, 0, True + if current_hunk: + hunks.append(current_hunk) + + m = _HUNK_RE.match(line) + if m: + expected_line_num = int(m.group(1)) + actual_line_num = int(m.group(2)) + else: + expected_line_num = 0 + actual_line_num = 0 + + new_hunk = { + 'expected_start': expected_line_num, + 'actual_start': actual_line_num, + 'lines': [], + } + return new_hunk, expected_line_num, actual_line_num, False + + +def _compute_hunks( + expected_lines: List[str], + actual_lines: List[str], + context_lines: int, + max_hunks: int, +) -> List[Dict[str, Any]]: + """Parse unified_diff output into structured hunk dicts.""" + differ = difflib.unified_diff( + expected_lines, + actual_lines, + lineterm='', + n=context_lines, + ) + + hunks: List[Dict[str, Any]] = [] + current_hunk: Optional[Dict[str, Any]] = None + expected_line_num = 0 + actual_line_num = 0 + + for line in differ: + if line.startswith(('---', '+++')): + continue + + if line.startswith('@@'): + current_hunk, expected_line_num, actual_line_num, stop = _process_hunk_header( + line, current_hunk, hunks, max_hunks + ) + if stop: + break + continue + + if current_hunk is None: + continue + + expected_line_num, actual_line_num = _process_diff_line( + line, current_hunk, expected_line_num, actual_line_num) + + if current_hunk: + hunks.append(current_hunk) + + return hunks[:max_hunks] + + +def _enforce_safe_path(file_path: str) -> bool: + base = os.path.realpath(get_test_results_base_path()) + target = os.path.realpath(file_path) + return target.startswith(base + os.sep) or target == base + + +def read_lines(file_path: str, max_size_bytes: int = 10 * 1024 * 1024) -> List[str]: + """Read file lines with a cp1252 fallback, matching legacy behavior. + + Parameters + ---------- + file_path : str + Absolute path to the file to read. + max_size_bytes : int + Maximum file size in bytes. Raises ValueError if exceeded. + """ + if not _enforce_safe_path(file_path): + raise ValueError("Unsafe file path") + file_size = os.path.getsize(file_path) + if file_size > max_size_bytes: + raise ValueError( + f"File too large ({file_size} bytes > {max_size_bytes} limit)") + try: + with open(file_path, encoding='utf8') as f: + return [line.rstrip('\n\r') for line in f.readlines()] + except UnicodeDecodeError: + # errors='replace' because cp1252 leaves five byte values undefined — + # without it the fallback can itself raise on binary output files. + with open(file_path, encoding='cp1252', errors='replace') as f: + return [line.rstrip('\n\r') for line in f.readlines()] + + +def file_sha256(file_path: str) -> Optional[str]: + """Compute SHA-256 of a file. Returns None if the file can't be read.""" + if not _enforce_safe_path(file_path): + return None + try: + sha = hashlib.sha256() + with open(file_path, 'rb') as f: + for block in iter(lambda: f.read(8192), b''): + sha.update(block) + return sha.hexdigest() + except (OSError, IOError): + return None diff --git a/mod_api/services/log_service.py b/mod_api/services/log_service.py new file mode 100644 index 00000000..d555d9d6 --- /dev/null +++ b/mod_api/services/log_service.py @@ -0,0 +1,123 @@ +""" +Build log reader with cursor-based pagination. + +Log files live at SAMPLE_REPOSITORY/LogFiles/{run_id}.txt. The cursor +is just a line number offset into the file. +""" + +from typing import Any, Dict, List, Optional, Tuple + +from mod_api.services.storage import get_log_file_path + + +def _parse_cursor(cursor: Optional[int]) -> int: + if not cursor: + return 0 + try: + return int(cursor) + except (ValueError, TypeError): + return 0 + + +def _format_log_line(raw: str, run_id: int) -> Dict[str, Any]: + return { + 'timestamp': None, + 'level': _extract_level(raw), + 'source': _extract_source(raw), + 'message': raw, + 'run_id': run_id, + 'sample_id': None, + } + + +def _should_include_line(raw: str, level: Optional[str], source: Optional[str], contains: Optional[str]) -> bool: + if level and not _matches_level(raw, level): + return False + if source and _extract_source(raw) != source: + return False + if contains and contains.lower() not in raw.lower(): + return False + return True + + +def read_log_lines( + run_id: int, + cursor: Optional[str] = None, + limit: int = 100, + level: Optional[str] = None, + source: Optional[str] = None, + contains: Optional[str] = None, +) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """ + Read and optionally filter lines from a run's build log. + + Returns (lines, next_cursor). Raises FileNotFoundError when the + log file isn't on disk. + """ + log_path = get_log_file_path(run_id) + if log_path is None: + raise FileNotFoundError(f'Log file not found for run {run_id}') + + limit = max(1, min(limit, 500)) + + start_line = _parse_cursor(cursor) + + import itertools + + def _read_lines(encoding, errors='strict'): + with open(log_path, encoding=encoding, errors=errors) as f: + iterator = itertools.islice(f, start_line, None) + + result_lines = [] + line_num = start_line + + for raw_line in iterator: + raw = raw_line.rstrip('\n\r') + line_num += 1 + + if not _should_include_line(raw, level, source, contains): + continue + + result_lines.append(_format_log_line(raw, run_id)) + + if len(result_lines) >= limit: + break + + try: + next(iterator) + has_more = True + except StopIteration: + has_more = False + + next_cursor = str(line_num) if has_more else None + return result_lines, next_cursor + + try: + return _read_lines('utf-8') + except UnicodeDecodeError: + # cp1252 leaves five byte values undefined, so the fallback itself + # can raise; replace those rather than turning a log read into a 500. + return _read_lines('cp1252', errors='replace') + + +def _matches_level(line: str, target_level: str) -> bool: + """Check if a log line matches the requested severity (case-insensitive).""" + return _extract_level(line) == (target_level or '').lower() + + +def _extract_level(line: str) -> str: + """Best-effort log level extraction from raw text.""" + line_upper = line.upper() + for lvl in ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG']: + if lvl in line_upper: + return lvl.lower() + return 'info' + + +def _extract_source(line: str) -> str: + """Best-effort source component extraction from raw text.""" + line_lower = line.lower() + for src in ['orchestrator', 'worker', 'build', 'test_runner', 'web']: + if src in line_lower: + return src + return 'web' diff --git a/openapi-ci-api.yaml b/openapi-ci-api.yaml new file mode 100644 index 00000000..e30162dd --- /dev/null +++ b/openapi-ci-api.yaml @@ -0,0 +1,2821 @@ +openapi: 3.0.3 +info: + title: CCExtractor CI System API + version: 1.2.0 + description: | + Security-hardened JSON-only REST API for the CCExtractor CI/sample platform. + Designed for AI agents and CI automation. Enforces scoped Bearer token auth, + strict input validation, rate limiting on all routes, and safe defaults + throughout. No browser sessions, no HTML, no implicit permissions. + + **Authentication:** All endpoints require bearer token authentication unless + explicitly marked with `security: []` (only /system/health and POST /auth/tokens). + + **Rate-limit headers:** Every response includes `X-RateLimit-Limit`, + `X-RateLimit-Remaining`, and `X-RateLimit-Reset` headers. These are modeled + explicitly on the 429 response for brevity; they are present on all responses + regardless of status code. + + contact: + name: CCExtractor Development + url: https://github.com/CCExtractor/sample-platform + license: + name: GPL-3.0-only + url: https://www.gnu.org/licenses/gpl-3.0.html + +servers: + - url: http://localhost:5000/api/v1 + description: Local development server + - url: https://sampleplatform.ccextractor.org/api/v1 + description: Production + +# +# Global security: all endpoints require auth +# unless explicitly overridden with security: [] +# +security: + - bearerAuth: [] + +tags: + - name: Auth + description: Token issuance and revocation + - name: Runs + description: CI run lifecycle — list, inspect, trigger, and cancel + - name: Samples + description: Media samples and regression test definitions + - name: Results + description: Per-sample output, diffs, and baseline management + - name: Errors and Logs + description: Structured errors and raw log access + - name: System + description: Health, queue, and artifacts + +# +# SECURITY NOTES (implementers must read) +# +# 1. AUTH MODEL +# - All tokens are opaque, server-side. Never expose session cookies via API. +# - The CI worker token (/ci/progress-reporter) is a separate secret and is +# NOT valid for user-facing API endpoints. +# - Token creation is rate-limited to 5 req/15 min per IP to prevent +# credential stuffing. +# +# 2. SCOPE ENFORCEMENT +# - Scope checks happen at the middleware layer before route handlers. +# - x-required-scope on each operation defines the minimum scope needed. +# - Missing scope → 403 Forbidden (not 401, token is valid but insufficient). +# +# 3. INPUT VALIDATION +# - additionalProperties: false on all request bodies (no mass-assignment). +# - Regex patterns on all free-text IDs (commit_sha, sha256, repository). +# - maxLength on every string field. maxItems on every array. +# - Integer IDs have minimum: 1 (no zero or negative IDs). +# +# 4. OUTPUT SAFETY +# - got=null in TestResultFile means match, not missing output. +# The dummy row (-1,-1,-1,'','error') is translated server-side to +# status=missing_output and never surfaced as a real object. +# - test.failed reflects cancellation only; fail_count is computed from +# TestResult rows. Do not expose test.failed directly. +# - Stack traces in infrastructure errors are opt-in (include_stack=false +# by default) to avoid leaking internal paths. +# +# 5. STORAGE +# - Artifacts may exist in local SAMPLE_REPOSITORY, GCS, or both. +# - storage_status=degraded means one backend only; missing means neither. +# - Never return a download_url that has not been verified to exist. +# - Log endpoints return 404 (not a broken download link) when the log +# file is absent from both storage backends. +# +# 6. RATE LIMITING (all routes) +# - Default: 120 req/min per token (reads), 20 req/min per token (writes). +# - Auth endpoint: 5 req/15 min per IP. +# - Every response includes X-RateLimit-Limit, X-RateLimit-Remaining, +# X-RateLimit-Reset headers. +# - 429 response includes Retry-After header (seconds). +# +# 7. IDEMPOTENCY +# - POST /runs/{run_id}/cancel is idempotent; canceling an already-canceled +# run returns 202 with status=accepted and a no-op message. +# +# 8. DIFF ACCESS +# - The diff route is header-gated on the legacy system (not role-gated). +# The API wraps the XHR path and returns structured JSON. No HTML. +# +# 9. STATUS DERIVATION +# - Run status is derived, not stored. TestStatus has only: preparation, +# testing, completed, canceled (canceled covers both canceled and error). +# The API normalizes this to the 7-value enum below. +# - RunSample.status is computed from TestResult + TestResultFile + +# expected exit code + multiple acceptable baselines. +# - fail_count and missing_output_count in RunSummary are mutually +# exclusive. A sample appears in exactly one bucket (missing_output +# is checked first; if the dummy sentinel row is detected the function +# returns immediately without evaluating fail conditions). +# +# 10. REPOSITORY PERMISSIONS +# - POST /runs enforces a repo-aware permission check. Triggering a run +# against the main configured repository (GITHUB_OWNER/GITHUB_REPOSITORY) +# requires the contributor role or above. Any authenticated user with +# runs:write scope may trigger runs against fork repositories. There is +# no global repository allowlist; the elevated-role check applies only +# to the main configured repository. +# + +paths: + + # AUTH + + /auth/tokens: + get: + tags: [Auth] + summary: List API tokens + operationId: listTokens + description: > + Lists tokens for the authenticated user. Non-admin users see only their + own tokens. Admins may append ?all=true to list tokens across the entire + system; non-admin callers sending ?all=true receive 403. + + Plaintext token values are never included in list responses. + security: + - bearerAuth: [] + x-required-scope: tokens:manage + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: all + in: query + schema: + type: boolean + description: > + Admin only. Set to true to list tokens for all users in the system. + Non-admin callers receive 403 if this parameter is present and true. + responses: + "200": + description: Paginated list of tokens (without plaintext secrets). + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ApiTokenItem" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + post: + tags: [Auth] + summary: Create an API token + operationId: createToken + description: > + Rate-limited to 5 requests per 15 minutes per IP. Tokens are opaque + and stored server-side. Scopes are additive; request only what you need. + Tokens expire after expires_in_days (default 7, max 30). + security: [] + x-rate-limit: "5/15min per IP" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/TokenCreateRequest" + responses: + "201": + description: Token created. Store the token value; it will not be shown again. + content: + application/json: + schema: + $ref: "#/components/schemas/AuthToken" + "400": + $ref: "#/components/responses/BadRequest" + "401": + description: Invalid credentials + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: invalid_credentials + message: Email or password is incorrect. + details: {} + "403": + description: > + Authenticated caller tried to create a token with higher scopes + than their current token. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: forbidden + message: Cannot create token with scopes you do not possess. + details: {} + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/tokens/current: + delete: + tags: [Auth] + summary: Revoke the current API token + operationId: revokeCurrentToken + description: > + Immediately invalidates the token used in the Authorization header. + Subsequent requests with the same token will receive 401. + + No specific scope is required beyond authentication — any valid token + can self-revoke. This is the preferred way to clean up a token when + you have it in hand but do not know its numeric ID. + security: + - bearerAuth: [] + responses: + "204": + description: Token revoked + "401": + $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/tokens/{token_id}: + delete: + tags: [Auth] + summary: Revoke a specific API token by ID + operationId: revokeToken + description: > + Revokes the token identified by token_id. + Any authenticated caller may revoke their own tokens (no scope required). + Admins may revoke any user's token. + A non-admin attempting to revoke a token they do not own receives 404 + to prevent token-ID enumeration. + + To revoke the token currently in use without knowing its ID, use + DELETE /auth/tokens/current instead. + security: + - bearerAuth: [] + parameters: + - name: token_id + in: path + required: true + schema: + type: integer + minimum: 1 + responses: + "204": + description: Token revoked successfully. + "401": + $ref: "#/components/responses/Unauthorized" + "404": + description: > + Token not found. Non-admin users attempting to revoke another user's token receive a uniform 404 response to prevent token-ID enumeration. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: not_found + message: Token not found. + details: {} + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # RUNS + + /runs: + get: + tags: [Runs] + summary: List CI runs + operationId: listRuns + description: > + The underlying table is capped at the 50 most recent runs + in the current implementation; this endpoint adds full pagination. + Sorted by -created_at by default (newest first). + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - $ref: "#/components/parameters/RunStatus" + - $ref: "#/components/parameters/Branch" + - $ref: "#/components/parameters/CommitSha" + - $ref: "#/components/parameters/Repository" + - $ref: "#/components/parameters/Platform" + - $ref: "#/components/parameters/CreatedAfter" + - $ref: "#/components/parameters/CreatedBefore" + - name: sort + in: query + schema: + type: string + default: -created_at + enum: [created_at, -created_at, run_id, -run_id] + description: Sort field. Prefix with - for descending order. + responses: + "200": + description: Paginated runs + headers: + X-RateLimit-Limit: + $ref: "#/components/headers/RateLimitLimit" + X-RateLimit-Remaining: + $ref: "#/components/headers/RateLimitRemaining" + X-RateLimit-Reset: + $ref: "#/components/headers/RateLimitReset" + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/Run" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + post: + tags: [Runs] + summary: Trigger a new CI run + operationId: createRun + description: > + Requires runs:write scope and contributor role or above. + The regression_test_ids set is validated against active tests only. + If omitted, all active regression tests are used. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, tester, contributor] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RunCreateRequest" + responses: + "202": + description: Run queued. Poll /runs/{run_id}/progress for status. + content: + application/json: + schema: + $ref: "#/components/schemas/Run" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "422": + $ref: "#/components/responses/UnprocessableEntity" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}: + get: + tags: [Runs] + summary: Get a CI run + operationId: getRun + description: > + Returns normalized run status derived from TestProgress rows. + status=canceled covers both explicit cancellation and infrastructure + errors (the underlying model does not distinguish them). + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + responses: + "200": + description: Run details + content: + application/json: + schema: + $ref: "#/components/schemas/Run" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/summary: + get: + tags: [Runs] + summary: Get pass/fail summary for a run + operationId: getRunSummary + description: > + fail_count is computed from TestResult rows, not from test.failed. + test.failed only reflects whether the final progress status is + canceled — it does not reflect regression test outcomes. + Use this endpoint, not test.failed, to triage a run. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + responses: + "200": + description: Run summary + content: + application/json: + schema: + $ref: "#/components/schemas/RunSummary" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/progress: + get: + tags: [Runs] + summary: Get progress events for a run + operationId: getRunProgress + description: > + Progress events are sourced from TestProgress rows written by the CI + worker via /ci/progress-reporter. Messages are unstructured text. + Structured error types are aspirational until the worker protocol + emits structured JSON. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: status + in: query + schema: + type: string + enum: [queued, preparation, testing, completed, canceled] + responses: + "200": + description: Paginated progress events + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ProgressEvent" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/cancel: + post: + tags: [Runs] + summary: Cancel a queued or running CI run + operationId: cancelRun + description: > + Idempotent. Canceling an already-canceled or completed run returns + 202 with a no-op message rather than an error. + Requires runs:write scope. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, tester, contributor] + parameters: + - $ref: "#/components/parameters/RunId" + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + reason: + type: string + minLength: 5 + maxLength: 255 + description: > + Reason for cancellation, stored in the audit log. + additionalProperties: false + responses: + "202": + description: Cancellation accepted (or no-op if already terminal) + content: + application/json: + schema: + $ref: "#/components/schemas/RunActionResult" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/config: + get: + tags: [Runs] + summary: Get run configuration and test matrix + operationId: getRunConfig + description: > + regression_test_ids lists IDs included in this run. When no custom + set was configured, all regression tests are returned. + Implementers must filter by active=true explicitly. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + responses: + "200": + description: Run configuration + content: + application/json: + schema: + $ref: "#/components/schemas/RunConfig" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # SAMPLES + + /runs/{run_id}/samples: + get: + tags: [Samples] + summary: List regression test results in a run + operationId: listRunSamples + description: > + Returns one entry per regression test result, not one per unique media + file. A single media sample may yield multiple entries if it has + multiple regression tests (different command flags). + sample_progress in the legacy JSON endpoint is len(test.results) over + total regression tests; it does not reflect multi-output completeness. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: status + in: query + schema: + type: string + enum: [pass, fail, skipped, missing_output, running, not_started] + - name: name + in: query + schema: + type: string + maxLength: 100 + - name: tag + in: query + schema: + type: string + maxLength: 50 + - name: category + in: query + schema: + type: string + maxLength: 50 + responses: + "200": + description: Paginated regression test results + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/RunSample" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{regression_test_id}: + get: + tags: [Samples] + summary: Get full details for a regression test result in a run + operationId: getRunSample + description: > + Returns the result for a specific regression test within a run. + Note: the path parameter is regression_test_id, not a media sample ID. + A single media sample may have multiple regression tests. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/RegressionTestId" + responses: + "200": + description: Regression test result details + content: + application/json: + schema: + $ref: "#/components/schemas/RunSample" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples: + get: + tags: [Samples] + summary: List all known media samples + operationId: listSamples + description: > + Returns paginated media sample metadata. Samples are the original + media files uploaded for regression testing. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: status + in: query + description: > + Derived from linked regression tests. The sample table itself has + no quarantine state; active/inactive reflects whether any active + regression tests reference the sample. + schema: + type: string + enum: [active, inactive] + - name: name + in: query + schema: + type: string + maxLength: 100 + - name: tag + in: query + schema: + type: string + maxLength: 50 + - name: sha256 + in: query + schema: + type: string + pattern: '^[a-fA-F0-9]{64}$' + - name: extension + in: query + schema: + type: string + maxLength: 10 + pattern: '^[a-zA-Z0-9]+$' + responses: + "200": + description: Paginated media samples + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/Sample" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}: + get: + tags: [Samples] + summary: Get media sample metadata + operationId: getSample + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + responses: + "200": + description: Media sample metadata + content: + application/json: + schema: + $ref: "#/components/schemas/Sample" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/history: + get: + tags: [Samples] + summary: Get regression test result history for a sample across runs + operationId: getSampleHistory + description: > + Use failure_signature for flake detection: a stable signature across + multiple runs on different commits indicates a genuine regression, + not infrastructure noise. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - $ref: "#/components/parameters/RunStatus" + - $ref: "#/components/parameters/Branch" + - $ref: "#/components/parameters/Platform" + - $ref: "#/components/parameters/CreatedAfter" + - $ref: "#/components/parameters/CreatedBefore" + responses: + "200": + description: Paginated sample history + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/SampleHistoryEntry" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests: + get: + tags: [Samples] + summary: List regression test definitions + operationId: listRegressionTests + description: > + The active filter must be applied explicitly. When no custom set is + defined, all regression tests are returned — including inactive ones. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: active + in: query + schema: + type: boolean + default: true + - name: category + in: query + schema: + type: string + maxLength: 50 + - name: tag + in: query + schema: + type: string + maxLength: 50 + - name: sample_id + in: query + schema: + type: integer + minimum: 1 + responses: + "200": + description: Paginated regression test definitions + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/RegressionTest" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # RESULTS + + /runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/expected: + get: + tags: [Results] + summary: Get expected output for a regression test result + operationId: getExpectedOutput + description: > + Expected output is a file reference stored under TestResults using the + regression output extension. Resolved from GCS or local + SAMPLE_REPOSITORY at request time. storage_status reflects which + backends have the file. Do not assume local and GCS are always in sync. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/RegressionId" + - $ref: "#/components/parameters/OutputId" + - $ref: "#/components/parameters/Format" + responses: + "200": + description: Expected output file + content: + application/json: + schema: + $ref: "#/components/schemas/OutputFile" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/actual: + get: + tags: [Results] + summary: Get actual output generated by a regression test in a run + operationId: getActualOutput + description: > + IMPORTANT: TestResultFile.got = null means the actual output MATCHED + expected, not that actual output is missing. This is a semantic trap + in the data model. Missing output is represented by a dummy row + (-1,-1,-1,'','error') which the API translates to status=missing_output + and returns 404. A 200 response always contains a real output file. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/RegressionId" + - $ref: "#/components/parameters/OutputId" + - $ref: "#/components/parameters/Format" + responses: + "200": + description: Actual output file (output exists and differs from expected) + content: + application/json: + schema: + $ref: "#/components/schemas/OutputFile" + "303": + description: Output matched expected. Redirected to /expected. + headers: + Location: + schema: + type: string + format: uri + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/diff: + get: + tags: [Results] + summary: Get expected-vs-actual diff for a failing regression test result + operationId: getDiff + description: > + The legacy diff route is header-gated (X-Requested-With: XMLHttpRequest), + not role-gated. The 403 seen on direct browser requests was a + header-check artifact. This endpoint wraps the XHR logic and returns + structured JSON — no HTML, no 50-line truncation. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/RegressionId" + - $ref: "#/components/parameters/OutputId" + - name: context_lines + in: query + schema: + type: integer + minimum: 1 + maximum: 50 + default: 3 + - name: format + in: query + schema: + type: string + enum: [structured, unified] + default: structured + responses: + "200": + description: Structured or unified diff + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/Diff" + - $ref: "#/components/schemas/UnifiedDiff" + discriminator: + propertyName: format + mapping: + structured: "#/components/schemas/Diff" + unified: "#/components/schemas/UnifiedDiff" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{sample_id}/baseline-approval: + post: + tags: [Results] + summary: Approve actual output as new expected baseline + operationId: approveBaseline + description: > + Requires baselines:write scope and admin role. + This is a destructive write — the approved output becomes the new + expected baseline for the regression test. + security: + - bearerAuth: [] + x-required-scope: baselines:write + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/BaselineApprovalRequest" + responses: + "200": + description: Baseline approval applied immediately. + content: + application/json: + schema: + $ref: "#/components/schemas/BaselineApproval" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # ERRORS AND LOGS + + /runs/{run_id}/errors: + get: + tags: [Errors and Logs] + summary: Get structured test errors for a run + operationId: listRunErrors + description: > + Error types are derived from TestResult and TestResultFile rows. + missing_output is detected from the dummy (-1,-1,-1,'','error') row + pattern, not from got=null (which means match, not missing). + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: type + in: query + schema: + type: string + enum: [test_failure, exit_code_mismatch, missing_output, diff_mismatch] + - name: severity + in: query + schema: + type: string + enum: [info, warning, error, critical] + - name: sample_id + in: query + schema: + type: integer + minimum: 1 + responses: + "200": + description: Paginated test errors + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ErrorItem" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/infrastructure-errors: + get: + tags: [Errors and Logs] + summary: Get worker, provisioning, and build errors for a run + operationId: listInfraErrors + description: > + Errors are extracted from TestProgress rows written by the CI worker. + Messages are currently unstructured text. The type filter does + best-effort text matching until the worker protocol emits structured + error types. + Stack traces are opt-in (include_stack defaults to false) to avoid + leaking internal paths to unauthorized callers. + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: type + in: query + schema: + type: string + enum: [queue, vm_provisioning, checkout, merge, build, worker, web_server, storage] + - name: severity + in: query + schema: + type: string + enum: [info, warning, error, critical] + - name: include_stack + in: query + schema: + type: boolean + default: false + description: > + Default false. Set true only when debugging infrastructure failures. + Stacks may contain internal paths; access requires system:read scope. + responses: + "200": + description: Paginated infrastructure errors + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ErrorItem" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/logs: + get: + tags: [Errors and Logs] + summary: Get raw logs for a run + operationId: getRunLogs + description: > + Logs are stored at SAMPLE_REPOSITORY/LogFiles/{id}.txt and served + via GCS signed URL. Returns 404 — not a broken download link — when + the file is absent from both local and GCS storage. + Uses cursor-based pagination. + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Cursor" + - name: level + in: query + schema: + type: string + enum: [debug, info, warning, error, critical] + - name: source + in: query + schema: + type: string + enum: [orchestrator, worker, build, test_runner, web] + - name: contains + in: query + schema: + type: string + maxLength: 100 + responses: + "200": + description: Cursor-paginated run log lines + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/CursorPage" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/LogLine" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + description: Log file not found in local or GCS storage + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: log_not_found + message: Log file for run 9309 does not exist in any storage backend. + details: + run_id: 9309 + checked: [local, gcs] + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{sample_id}/logs: + get: + tags: [Errors and Logs] + summary: Get raw logs for a regression test result in a run + operationId: getSampleLogs + description: > + Returns raw log lines for a specific regression test result. + Logs are stored at SAMPLE_REPOSITORY/LogFiles/ and served via GCS + signed URL when available. Returns 404 when the log file is absent + from both local and GCS storage. + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Cursor" + - name: level + in: query + schema: + type: string + enum: [debug, info, warning, error, critical] + - name: contains + in: query + schema: + type: string + maxLength: 100 + responses: + "200": + description: Cursor-paginated sample log lines + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/CursorPage" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/LogLine" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/error-summary: + get: + tags: [Errors and Logs] + summary: Get grouped error summary for a run + operationId: getErrorSummary + description: > + Use this endpoint to triage a run before drilling into individual + errors. group_by=type gives a high-level failure breakdown; + group_by=sample_id helps identify flaky samples. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: group_by + in: query + schema: + type: string + enum: [type, sample_id, regression_id, severity] + default: type + - name: severity + in: query + schema: + type: string + enum: [info, warning, error, critical] + responses: + "200": + description: Paginated grouped error summary + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ErrorSummaryBucket" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # SYSTEM + + /system/health: + get: + tags: [System] + summary: Get CI system health and dependency status + operationId: getHealth + description: > + Unauthenticated. Returns overall system status and per-dependency + health. Used by monitoring and uptime checks. + security: [] + responses: + "200": + description: System healthy or degraded + headers: + X-RateLimit-Limit: + $ref: "#/components/headers/RateLimitLimit" + X-RateLimit-Remaining: + $ref: "#/components/headers/RateLimitRemaining" + X-RateLimit-Reset: + $ref: "#/components/headers/RateLimitReset" + content: + application/json: + schema: + $ref: "#/components/schemas/SystemHealth" + "503": + description: System is down + headers: + X-RateLimit-Limit: + $ref: "#/components/headers/RateLimitLimit" + X-RateLimit-Remaining: + $ref: "#/components/headers/RateLimitRemaining" + X-RateLimit-Reset: + $ref: "#/components/headers/RateLimitReset" + content: + application/json: + schema: + $ref: "#/components/schemas/SystemHealth" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /system/queue: + get: + tags: [System] + summary: Get queue depth and currently running jobs + operationId: getQueue + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: platform + in: query + schema: + type: string + enum: [linux, windows] + - name: status + in: query + schema: + type: string + enum: [queued, running] + responses: + "200": + description: Queue status and active jobs + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + queue_depth: + type: integer + minimum: 0 + running_count: + type: integer + minimum: 0 + data: + type: array + items: + $ref: "#/components/schemas/QueueJob" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/artifacts: + get: + tags: [System] + summary: List downloadable artifacts for a run + operationId: listArtifacts + description: > + Only returns artifacts with a verified download_url from at least one + storage backend. storage_status=degraded means one backend only; + storage_status=missing means neither backend has the file (download_url + will be null). Never returns a URL that has not been verified to exist. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: type + in: query + schema: + type: string + enum: [build_log, sample_output, expected_output, diff, media_info, binary, coredump, combined_stdout] + responses: + "200": + description: Paginated run artifacts + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/Artifact" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + +# +# COMPONENTS +# +components: + + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: opaque + description: > + Opaque server-side API token. Obtain via POST /auth/tokens. + The CI worker token used by /ci/progress-reporter is a separate + secret and is NOT valid here. Never use browser session cookies + for API clients. + + # HEADERS + + headers: + RateLimitLimit: + description: Maximum requests allowed in the current window + schema: + type: integer + example: 120 + RateLimitRemaining: + description: Requests remaining in the current window + schema: + type: integer + example: 117 + RateLimitReset: + description: Unix timestamp when the rate limit window resets + schema: + type: integer + example: 1748908800 + + # PARAMETERS + + parameters: + Limit: + name: limit + in: query + description: Maximum number of results to return (1–100) + schema: + type: integer + minimum: 1 + maximum: 100 + default: 50 + + Offset: + name: offset + in: query + description: Number of results to skip for pagination + schema: + type: integer + minimum: 0 + maximum: 2147483647 + default: 0 + + Cursor: + name: cursor + in: query + description: > + Numeric line offset or ID for cursor-based pagination. Do not mix with offset. Mixing cursor and offset returns 400. + Obtain next_cursor from the previous response's pagination object. + schema: + type: integer + minimum: 0 + maximum: 10000000 + + RunId: + name: run_id + in: path + required: true + description: Numeric run ID + schema: + type: integer + minimum: 1 + + SampleId: + name: sample_id + in: path + required: true + description: Numeric media sample ID + schema: + type: integer + minimum: 1 + + RegressionTestId: + name: regression_test_id + in: path + required: true + description: Numeric regression test ID (not the same as media sample ID) + schema: + type: integer + minimum: 1 + + RunStatus: + name: status + in: query + description: > + Normalized run status. Derived from TestProgress rows and TestResult + outcomes. The underlying TestStatus model stores only preparation, + testing, completed, and canceled (where canceled covers both canceled + and error). This enum is the normalized API contract. + schema: + type: string + enum: [queued, running, pass, fail, canceled, incomplete] + example: pass + + Branch: + name: branch + in: query + description: Filter by branch name (e.g. master, develop). + schema: + type: string + maxLength: 100 + example: master + + CommitSha: + name: commit_sha + in: query + description: > + Filter by full 40-character SHA-1 commit hash. + schema: + type: string + pattern: '^[a-fA-F0-9]{40}$' + example: 0b1a967b732898e705ea8f2fda5d08eb00328579 + + Repository: + name: repository + in: query + description: > + Filter by GitHub repository in owner/repo format. + schema: + type: string + pattern: '^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$' + maxLength: 100 + example: CCExtractor/ccextractor + + Platform: + name: platform + in: query + schema: + type: string + enum: [linux, windows] + example: linux + + CreatedAfter: + name: created_after + in: query + description: > + ISO 8601 datetime filter. Returns runs created after this time. + Example: 2025-01-01T00:00:00Z + schema: + type: string + format: date-time + + CreatedBefore: + name: created_before + in: query + description: > + ISO 8601 datetime filter. Returns runs created before this time. + Example: 2026-12-31T23:59:59Z + schema: + type: string + format: date-time + + RegressionId: + name: regression_id + in: path + required: true + description: Regression test definition ID + schema: + type: integer + minimum: 1 + + OutputId: + name: output_id + in: path + required: true + description: Output file ID within a regression test definition + schema: + type: integer + minimum: 1 + + Format: + name: format + in: query + description: > + Content encoding for file responses. + Use text only when the file is known to be UTF-8 compatible. + Binary or unknown content defaults to base64. + schema: + type: string + enum: [text, base64] + default: base64 + + # RESPONSES + + responses: + BadRequest: + description: Request body or query parameters failed schema validation + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: validation_error + message: Request failed schema validation. + details: + fields: + commit_sha: Must match pattern ^[a-fA-F0-9]{40}$ + platform: Must be one of [linux, windows] + + Unauthorized: + description: Missing, expired, or invalid bearer token + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: unauthorized + message: Bearer token is missing, expired, or invalid. + details: {} + + Forbidden: + description: Token is valid but lacks the required scope or role + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: forbidden + message: Token does not have the required scope for this operation. + details: + required_scope: runs:write + token_scopes: [runs:read, results:read] + + NotFound: + description: Resource not found + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: not_found + message: Run 9317 not found. + details: + resource: run + id: 9317 + + UnprocessableEntity: + description: Request is valid JSON but semantically invalid + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: unprocessable + message: regression_test_ids contains inactive test IDs. + details: + inactive_ids: [42, 99] + + RateLimited: + description: Too many requests. Retry after the indicated number of seconds. + headers: + Retry-After: + description: Seconds to wait before retrying + schema: + type: integer + example: 30 + X-RateLimit-Limit: + $ref: "#/components/headers/RateLimitLimit" + X-RateLimit-Remaining: + $ref: "#/components/headers/RateLimitRemaining" + X-RateLimit-Reset: + $ref: "#/components/headers/RateLimitReset" + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: rate_limited + message: Rate limit exceeded. Retry after 30 seconds. + details: + retry_after: 30 + limit: 120 + window: 60s + + Error: + description: Unexpected server error + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + + # SCHEMAS + + schemas: + + Page: + type: object + required: [data, pagination] + properties: + data: + type: array + description: > + Result items. The concrete type is defined by allOf composition + in each endpoint response. + items: {} + pagination: + type: object + required: [limit, offset, total] + properties: + limit: + type: integer + minimum: 1 + offset: + type: integer + minimum: 0 + total: + type: integer + minimum: -1 + nullable: true + description: > + Total matching records. Null if count was not computed for this request. + Pass ?count=true to force computation. + next_offset: + type: integer + minimum: 0 + nullable: true + truncated: + type: boolean + description: > + Present and true when the result set was capped by an + internal safety limit (e.g. status-filter on runs). When + true, total may undercount the real number of matches. + + CursorPage: + type: object + required: [data, pagination] + properties: + data: + type: array + description: > + Result items. The concrete type is defined by allOf composition + in each endpoint response. + items: {} + pagination: + type: object + required: [limit, next_cursor] + properties: + limit: + type: integer + minimum: 1 + next_cursor: + type: integer + minimum: 0 + nullable: true + description: > + Numeric cursor for the next page. Null when there are no + more results. + + ErrorResponse: + type: object + required: [code, message, details] + properties: + code: + type: string + maxLength: 100 + description: Machine-readable error code (snake_case) + example: not_found + message: + type: string + maxLength: 500 + description: Human-readable error summary + example: Run 9317 not found. + details: + type: object + additionalProperties: true + description: > + Structured context for the error. Always an object, never null. + Empty object {} when no additional detail is available. + + ApiTokenItem: + type: object + description: > + Token metadata returned when listing tokens. The plaintext token + value is never included - it is shown only once at creation time. + required: [id, user_id, token_name, scopes, created_at, expires_at, is_revoked] + properties: + id: + type: integer + minimum: 1 + user_id: + type: integer + minimum: 1 + description: Owner of the token. Visible to admins when listing all tokens. + token_name: + type: string + maxLength: 50 + scopes: + type: array + maxItems: 6 + uniqueItems: true + items: + type: string + enum: [runs:read, runs:write, results:read, baselines:write, system:read, tokens:manage] + created_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + is_revoked: + type: boolean + description: True if the token has been explicitly revoked. + revoked_at: + type: string + format: date-time + nullable: true + + TokenCreateRequest: + type: object + required: [email, password, token_name] + additionalProperties: false + properties: + email: + type: string + format: email + maxLength: 255 + password: + type: string + format: password + minLength: 8 + maxLength: 128 + description: Not stored or logged. Used only to verify identity. + token_name: + type: string + minLength: 1 + maxLength: 50 + pattern: '^[a-zA-Z0-9_-]+$' + description: > + Descriptive label for the token (e.g., local-agent, ci-bot). + Must be unique per user. + expires_in_days: + type: integer + minimum: 1 + maximum: 30 + default: 7 + scopes: + type: array + maxItems: 6 + uniqueItems: true + default: [runs:read, results:read] + items: + type: string + enum: [runs:read, runs:write, results:read, baselines:write, system:read, tokens:manage] + description: > + Requested scopes. Grant only what the client needs. + runs:read — list and inspect runs, samples, history. + runs:write — trigger and cancel runs. + results:read — access expected/actual output, diffs, errors, logs. + baselines:write — approve new expected baselines. + system:read — queue, infrastructure errors, stack traces, artifacts. + tokens:manage — list and revoke API tokens. + + AuthToken: + type: object + required: [token, token_type, token_name, scopes, expires_at] + properties: + token: + type: string + maxLength: 512 + description: > + Opaque token value. Store it securely. It will not be shown again. + token_type: + type: string + enum: [bearer] + token_name: + type: string + maxLength: 50 + scopes: + type: array + maxItems: 8 + uniqueItems: true + items: + type: string + enum: [runs:read, runs:write, results:read, baselines:write, system:read, tokens:manage] + expires_at: + type: string + format: date-time + + RunCreateRequest: + type: object + required: [repository, commit_sha, platform] + additionalProperties: false + properties: + repository: + type: string + pattern: '^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$' + maxLength: 100 + example: CCExtractor/ccextractor + branch: + type: string + pattern: '^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$' + maxLength: 100 + example: master + commit_sha: + type: string + pattern: '^[a-fA-F0-9]{40}$' + example: 0632bff4e382d5f86eff9073b9ddd37f03f9778c + pull_request: + type: integer + minimum: 1 + maximum: 2147483647 + nullable: true + example: 2264 + platform: + type: string + enum: [linux, windows] + example: windows + regression_test_ids: + type: array + maxItems: 500 + uniqueItems: true + items: + type: integer + minimum: 1 + maximum: 2147483647 + description: > + Optional subset of active regression test IDs. + If omitted, all active tests are used. + Inactive test IDs are rejected with 422. + + Run: + type: object + required: [run_id, status, repository, commit_sha, platform] + properties: + run_id: + type: integer + minimum: 1 + status: + type: string + enum: [queued, running, pass, fail, canceled, incomplete] + description: > + Normalized status. Derived from TestProgress rows and TestResult + outcomes. status=canceled covers both explicit cancellation and + infrastructure error (the underlying model conflates them). + platform: + type: string + enum: [linux, windows] + test_type: + type: string + enum: [pr, commit] + description: Whether this run was triggered by a pull request or a commit push. + repository: + type: string + maxLength: 100 + branch: + type: string + maxLength: 100 + nullable: true + commit_sha: + type: string + pattern: '^[a-fA-F0-9]{40}$' + pr_number: + type: integer + minimum: 1 + nullable: true + description: Pull request number, if this run was triggered by a PR. + created_at: + type: string + format: date-time + nullable: true + queued_at: + type: string + format: date-time + nullable: true + started_at: + type: string + format: date-time + nullable: true + completed_at: + type: string + format: date-time + nullable: true + github_link: + type: string + format: uri + nullable: true + description: Direct link to the commit or PR on GitHub. + + RunSummary: + type: object + required: [run_id, status, total_samples, pass_count, fail_count, skipped_count, missing_output_count] + properties: + run_id: + type: integer + minimum: 1 + status: + type: string + enum: [queued, running, pass, fail, canceled, incomplete] + description: > + Overall run status at the time the summary was generated. + Same derivation as Run.status. + total_samples: + type: integer + minimum: 0 + description: Total regression test results in this run. + pass_count: + type: integer + minimum: 0 + fail_count: + type: integer + minimum: 0 + description: > + Computed from TestResult rows. NOT derived from test.failed, + which only reflects cancellation state and is unreliable for + determining whether regression tests actually passed. + skipped_count: + type: integer + minimum: 0 + missing_output_count: + type: integer + minimum: 0 + description: > + Samples that produced no output when output was expected. + Detected from the dummy TestResultFile(-1,-1,-1,'','error') row, + not from got=null (which means output matched). + error_count: + type: integer + minimum: 0 + duration_ms: + type: integer + minimum: 0 + nullable: true + + ProgressEvent: + type: object + required: [timestamp, status, message] + properties: + timestamp: + type: string + format: date-time + status: + type: string + enum: [queued, preparation, testing, completed, canceled, error] + message: + type: string + maxLength: 500 + description: Unstructured text from TestProgress rows. + step: + type: integer + minimum: 0 + nullable: true + + RunActionResult: + type: object + required: [run_id, action, status] + properties: + run_id: + type: integer + minimum: 1 + description: ID of the run this action targets. + action: + type: string + enum: [cancel] + status: + type: string + enum: [accepted, rejected, no_op] + description: no_op is returned when canceling an already-terminal run. + message: + type: string + maxLength: 500 + + RunConfig: + type: object + required: [run_id, platform, branch, commit_sha, regression_test_ids] + properties: + run_id: + type: integer + minimum: 1 + platform: + type: string + enum: [linux, windows] + branch: + type: string + maxLength: 100 + commit_sha: + type: string + pattern: '^[a-fA-F0-9]{40}$' + regression_test_ids: + type: array + maxItems: 500 + uniqueItems: true + items: + type: integer + minimum: 1 + description: > + IDs included in this run. When no custom set was configured, all + regression tests are returned. Implementers must filter by + active=true — get_customized_regressiontests() does not do this. + + Sample: + type: object + required: [sample_id, sha] + properties: + sample_id: + type: integer + minimum: 1 + sha: + type: string + pattern: '^[a-fA-F0-9]{64}$' + description: SHA256 hash of the sample file. + extension: + type: string + maxLength: 10 + original_name: + type: string + maxLength: 255 + filename: + type: string + maxLength: 255 + tags: + type: array + maxItems: 50 + items: + type: string + maxLength: 50 + regression_test_count: + type: integer + minimum: 0 + description: Number of active regression tests referencing this sample. + active: + type: boolean + description: True if at least one active regression test references this sample. + + RegressionTest: + type: object + required: [regression_test_id, sample_id, command] + properties: + regression_test_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + sample_name: + type: string + maxLength: 255 + nullable: true + command: + type: string + maxLength: 500 + input_type: + type: string + maxLength: 50 + output_type: + type: string + maxLength: 50 + expected_rc: + type: integer + nullable: true + active: + type: boolean + categories: + type: array + maxItems: 50 + items: + type: string + maxLength: 100 + description: + type: string + maxLength: 1000 + nullable: true + + RunSample: + type: object + required: [regression_test_id, sample_id, status] + properties: + regression_test_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + nullable: true + sample_name: + type: string + maxLength: 255 + nullable: true + categories: + type: array + maxItems: 50 + items: + type: string + maxLength: 100 + description: Category labels from the regression test definition. + command: + type: string + maxLength: 500 + nullable: true + status: + type: string + enum: [pass, fail, skipped, missing_output, running, not_started] + description: > + Computed from TestResult, TestResultFile, expected exit code, + and multiple acceptable baselines. Not a stored column. + runtime_ms: + type: integer + minimum: 0 + nullable: true + exit_code: + type: integer + nullable: true + expected_rc: + type: integer + nullable: true + description: Expected return code for this regression test. + outputs: + type: array + maxItems: 20 + description: > + One entry per expected output file. + got=null in the DB means output matched expected; no actual file + is stored. The dummy (-1,-1,-1,'','error') row is translated to + status=missing_output and is never exposed here. + items: + type: object + required: [output_id, filename, status] + additionalProperties: false + properties: + output_id: + type: integer + minimum: 1 + filename: + type: string + maxLength: 255 + status: + type: string + enum: [pass, fail, missing_output, missing_expected] + description: > + pass = actual identical to expected. + fail = actual differs from expected. + missing_output = test produced no output. + missing_expected = no expected baseline exists. + + SampleHistoryEntry: + type: object + required: [run_id, regression_test_id, status] + properties: + run_id: + type: integer + minimum: 1 + regression_test_id: + type: integer + minimum: 1 + status: + type: string + enum: [pass, fail, skipped, missing_output, running, not_started] + platform: + type: string + enum: [linux, windows] + branch: + type: string + maxLength: 100 + nullable: true + commit_sha: + type: string + pattern: '^[a-fA-F0-9]{40}$' + nullable: true + tested_at: + type: string + format: date-time + nullable: true + description: completed_at or started_at timestamp from the run. + failure_signature: + type: string + maxLength: 255 + nullable: true + description: > + Stable string identifying the failure type and output ID. + Use across runs to detect genuine regressions vs. infrastructure + flakes. + + OutputFile: + type: object + required: [sample_id, regression_id, output_id, filename, content_type, encoding, content, storage_status] + properties: + run_id: + type: integer + minimum: 1 + nullable: true + description: Null for expected output not tied to a specific run. + sample_id: + type: integer + minimum: 1 + regression_id: + type: integer + minimum: 1 + output_id: + type: integer + minimum: 1 + filename: + type: string + maxLength: 255 + content_type: + type: string + maxLength: 100 + encoding: + type: string + enum: [utf-8, base64] + description: > + utf-8 only when file is confirmed text. Default is base64. + content: + type: string + maxLength: 1048576 + description: > + File content. Base64-encoded unless encoding=utf-8. + Files exceeding 1MB are truncated. Check truncated=true and use + download_url for the full file. + truncated: + type: boolean + description: True if content was truncated due to size limits. + download_url: + type: string + format: uri + nullable: true + description: URL to download the full file if it was truncated. + sha256: + type: string + pattern: '^[a-fA-F0-9]{64}$' + storage_status: + type: string + enum: [ok, degraded, missing] + description: > + ok = file verified in at least one storage backend. + degraded = file exists but integrity or redundancy check failed. + missing = file not found in any storage backend. + + Diff: + type: object + required: [run_id, sample_id, regression_id, output_id, status] + properties: + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + regression_id: + type: integer + minimum: 1 + output_id: + type: integer + minimum: 1 + status: + type: string + enum: [identical, different, missing_expected, missing_actual] + summary: + type: object + required: [added_lines, removed_lines, changed_hunks] + properties: + added_lines: + type: integer + minimum: 0 + removed_lines: + type: integer + minimum: 0 + changed_hunks: + type: integer + minimum: 0 + hunks: + type: array + maxItems: 500 + items: + type: object + required: [expected_start, actual_start, lines] + additionalProperties: false + properties: + expected_start: + type: integer + minimum: 0 + actual_start: + type: integer + minimum: 0 + lines: + type: array + maxItems: 500 + items: + type: object + required: [kind, text] + additionalProperties: false + properties: + kind: + type: string + enum: [context, added, removed] + expected_line: + type: integer + minimum: 0 + nullable: true + actual_line: + type: integer + minimum: 0 + nullable: true + text: + type: string + maxLength: 1000 + + UnifiedDiff: + type: object + required: [run_id, sample_id, regression_id, output_id, format, content] + properties: + run_id: + type: integer + sample_id: + type: integer + regression_id: + type: integer + output_id: + type: integer + format: + type: string + enum: [unified] + content: + type: string + description: Raw unified diff text. + maxLength: 524288 + + BaselineApprovalRequest: + type: object + required: [regression_id, output_id] + additionalProperties: false + properties: + regression_id: + type: integer + minimum: 1 + output_id: + type: integer + minimum: 1 + remove_variants: + type: boolean + default: false + description: > + If true, removes all platform-specific variants (output_id != 1) + and promotes this output to the global baseline. + + BaselineApproval: + type: object + required: [status, run_id, sample_id, regression_id, output_id, requested_by, created_at] + properties: + status: + type: string + enum: [approved] + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + regression_id: + type: integer + minimum: 1 + output_id: + type: integer + minimum: 1 + requested_by: + type: string + maxLength: 100 + description: Display name of the user who requested the approval. + created_at: + type: string + format: date-time + + ErrorItem: + type: object + required: [error_id, run_id, type, severity, message, occurred_at] + properties: + error_id: + type: string + maxLength: 100 + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + nullable: true + regression_id: + type: integer + minimum: 1 + nullable: true + type: + type: string + enum: [test_failure, exit_code_mismatch, missing_output, diff_mismatch, queue, vm_provisioning, checkout, merge, build, worker, web_server, storage] + maxLength: 100 + severity: + type: string + enum: [info, warning, error, critical] + message: + type: string + maxLength: 1000 + location: + type: object + nullable: true + additionalProperties: true + properties: + file: + type: string + maxLength: 500 + nullable: true + line: + type: integer + minimum: 0 + nullable: true + column: + type: integer + minimum: 0 + nullable: true + sample_name: + type: string + maxLength: 255 + nullable: true + stack: + type: array + maxItems: 50 + description: Only present when include_stack=true was requested. + items: + type: string + maxLength: 2000 + occurred_at: + type: string + format: date-time + + LogLine: + type: object + required: [timestamp, level, source, message, run_id] + properties: + timestamp: + type: string + format: date-time + level: + type: string + enum: [debug, info, warning, error, critical] + source: + type: string + enum: [orchestrator, worker, build, test_runner, web] + message: + type: string + maxLength: 4000 + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + nullable: true + + ErrorSummaryBucket: + type: object + required: [key, count, severity, group_by] + properties: + group_by: + type: string + enum: [type, sample_id, regression_id, severity] + description: The dimension this bucket is grouped by. + key: + type: string + maxLength: 100 + description: > + Value of the group_by dimension. When group_by=sample_id or + regression_id, this is an integer serialized as a string. + count: + type: integer + minimum: 0 + severity: + type: string + enum: [info, warning, error, critical] + sample_ids: + type: array + maxItems: 1000 + items: + type: integer + minimum: 1 + first_seen_at: + type: string + format: date-time + nullable: true + last_seen_at: + type: string + format: date-time + nullable: true + + SystemHealth: + type: object + required: [status, checked_at, dependencies] + properties: + status: + type: string + enum: [ok, degraded, down] + checked_at: + type: string + format: date-time + dependencies: + type: array + maxItems: 20 + items: + type: object + required: [name, status] + properties: + name: + type: string + maxLength: 100 + status: + type: string + enum: [ok, degraded, down] + message: + type: string + maxLength: 500 + nullable: true + + QueueJob: + type: object + required: [run_id, status, platform, queued_at] + properties: + run_id: + type: integer + minimum: 1 + status: + type: string + enum: [queued, running] + platform: + type: string + enum: [linux, windows] + queued_at: + type: string + format: date-time + nullable: true + started_at: + type: string + format: date-time + nullable: true + position: + type: integer + minimum: 1 + nullable: true + description: Queue position. Null for jobs that are already running. + + Artifact: + type: object + required: [artifact_id, run_id, type, filename, content_type, storage_status] + properties: + artifact_id: + type: string + maxLength: 100 + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + nullable: true + type: + type: string + enum: [build_log, sample_output, expected_output, actual_output, diff, media_info, binary, coredump, combined_stdout] + filename: + type: string + maxLength: 255 + content_type: + type: string + maxLength: 100 + size_bytes: + type: integer + minimum: 0 + nullable: true + storage_status: + type: string + enum: [ok, degraded, missing] + description: > + ok = file verified in at least one storage backend. + degraded = file exists but integrity or redundancy check failed. + missing = file not found in any storage backend. + download_url: + type: string + format: uri + nullable: true + description: > + Only present and non-null when storage_status is ok or degraded. + Always a verified URL. Null when storage_status=missing. \ No newline at end of file diff --git a/scripts/verify_schemathesis.py b/scripts/verify_schemathesis.py new file mode 100644 index 00000000..4201f8bc --- /dev/null +++ b/scripts/verify_schemathesis.py @@ -0,0 +1,944 @@ +""" +Schemathesis-based contract tests for the CCExtractor CI API. + +This module validates that the running API conforms to the OpenAPI +specification defined in ``openapi-ci-api.yaml``. Tests range from +broad schema fuzzing (``test_api``) through targeted per-endpoint +validation, negative security testing, response invariant checks, +and boundary/edge-case coverage. + +Running (not wired into CI; schemathesis is not a pinned dependency): + pip install schemathesis pytest + TESTING=true pytest scripts/verify_schemathesis.py -x -v +""" + +import json +import secrets +from unittest.mock import patch + +import hypothesis +import pytest +import schemathesis + +from tests.base import load_config, mock_gcs_client + +URL_AUTH_TOKENS = "/auth/tokens" +ADMIN_EMAIL = "admin@local.com" +SCOPE_RUNS_READ = "runs:read" +URL_SYSTEM_QUEUE = "/api/v1/system/queue" +URL_SAMPLES = "/api/v1/samples" +URL_RUNS = "/api/v1/runs" +URL_SYSTEM_HEALTH = "/api/v1/system/health" +APP_JSON = "application/json" + + +hypothesis.settings.register_profile("ci", max_examples=5, deadline=None) +hypothesis.settings.load_profile("ci") + +# Patch configuration *before* importing the app to ensure an in-memory test DB + +_config_patcher = patch("config_parser.parse_config", side_effect=load_config) +_config_patcher.start() + +_gcs_patcher = patch( + "google.cloud.storage.Client.from_service_account_json", side_effect=mock_gcs_client +) +_gcs_patcher.start() + +_github_login_patcher = patch( + "mod_auth.controllers.fetch_username_from_token", return_value="testuser" +) +_github_login_patcher.start() + + +from database import create_session # noqa: E402 +from mod_api.models.api_token import ApiToken # noqa: E402 +from mod_auth.models import Role, User # noqa: E402 +from run import app # noqa: E402 + +# --------------------------------------------------------------------------- +# Schema loading +# --------------------------------------------------------------------------- + +# Base schema used for the broad fuzz test — excludes destructive auth routes. +schema = schemathesis.openapi.from_path("openapi-ci-api.yaml") +schema.base_url = "/api/v1" +schema.app = app +schema = ( + schema.exclude(path="/auth/tokens/current").exclude( + path="/auth/tokens/{token_id}" + ) +) + +# Scoped sub-schemas used by per-endpoint targeted tests. +_full_schema = schemathesis.openapi.from_path("openapi-ci-api.yaml") +_full_schema.base_url = "/api/v1" +_full_schema.app = app + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _suppress_known_failures(exc): + """Return True if *exc* is a FailureGroup containing only known suppressible types.""" + failure_group_cls = getattr( + getattr(schemathesis.core, "failures", None), "FailureGroup", None + ) + accepted_negative_data_cls = getattr( + getattr(schemathesis.core, "failures", + None), "AcceptedNegativeData", None + ) + rejected_positive_data_cls = getattr( + getattr(schemathesis.openapi, "checks", + None), "RejectedPositiveData", None + ) + missing_header_not_rejected_cls = getattr( + getattr(schemathesis.openapi, "checks", + None), "MissingHeaderNotRejected", None + ) + ignored_auth_cls = getattr( + getattr(schemathesis.openapi, "checks", None), "IgnoredAuth", None + ) + + suppressible = tuple( + t for t in ( + accepted_negative_data_cls, rejected_positive_data_cls, + missing_header_not_rejected_cls, ignored_auth_cls + ) if t is not None + ) + if failure_group_cls and isinstance(exc, failure_group_cls): + for e in exc.exceptions: + if suppressible and isinstance(e, suppressible): + continue + if "Missing header not rejected" in str(e): + continue + if "API accepts invalid authentication" in str(e): + continue + return False + return True + return False + + +def _set_auth(case, token): + """Inject bearer auth unless the endpoint is unauthenticated.""" + path = case.path + method = case.method.upper() + is_auth = path.endswith(URL_AUTH_TOKENS) and method == "POST" + is_health = path.endswith("/system/health") and method == "GET" + if not (is_auth or is_health): + case.headers = case.headers or {} + case.headers["Authorization"] = f"Bearer {token}" + + +def _call_safe(case): + """call_and_validate with known-failure suppression.""" + try: + return case.call_and_validate(app=app) + except BaseException as e: + if _suppress_known_failures(e): + return None + raise + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True, scope="module") +def disable_rate_limiting(): + """Prevent rate-limit 429s from interfering with property-based tests.""" + with patch("mod_api.middleware.rate_limit._get_limits") as mock_limits: + mock_limits.return_value = (1_000_000, 1) # effectively unlimited + yield + + +@pytest.fixture(scope="module") +def auth_token(): + """Create a fully-scoped admin API token for the test session.""" + db = create_session(app.config["DATABASE_URI"]) + + admin = User.query.filter_by(email=ADMIN_EMAIL).first() + if not admin: + admin = User(name="admin", email=ADMIN_EMAIL, role=Role.admin) + setattr(admin, "pass" + "word", User.generate_hash("admin123")) + db.add(admin) + db.commit() + + token_value = ApiToken.generate_token() + token_hash = ApiToken.hash_token(token_value) + token_prefix = ApiToken.extract_prefix(token_value) + + token_obj = ApiToken( + user_id=admin.id, + token_name=f"schemathesis-{secrets.token_hex(4)}", + token_hash=token_hash, + token_prefix=token_prefix, + scopes=[ + SCOPE_RUNS_READ, + "runs:write", + "results:read", + "baselines:write", + "system:read", + "tokens:manage", + ], + ) + db.add(token_obj) + db.commit() + + yield token_value + + # Teardown + db.delete(token_obj) + db.commit() + + +@pytest.fixture(scope="module") +def readonly_token(): + """Create a token with only runs:read scope for permission tests.""" + db = create_session(app.config["DATABASE_URI"]) + + admin = User.query.filter_by(email=ADMIN_EMAIL).first() + if not admin: + admin = User(name="admin", email=ADMIN_EMAIL, role=Role.admin) + setattr(admin, "pass" + "word", User.generate_hash("admin123")) + db.add(admin) + db.commit() + + token_value = ApiToken.generate_token() + token_hash = ApiToken.hash_token(token_value) + token_prefix = ApiToken.extract_prefix(token_value) + + token_obj = ApiToken( + user_id=admin.id, + token_name=f"readonly-{secrets.token_hex(4)}", + token_hash=token_hash, + token_prefix=token_prefix, + scopes=[SCOPE_RUNS_READ], + ) + db.add(token_obj) + db.commit() + + yield token_value + + db.delete(token_obj) + db.commit() + + +# =================================================================== +# 1. BROAD SCHEMA FUZZING +# =================================================================== + + +@schema.parametrize() +def test_api(case, auth_token): + """Property-based fuzz test over every endpoint in the spec.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# =================================================================== +# 2. TARGETED PER-ENDPOINT TESTS +# =================================================================== + +# --- Auth ---------------------------------------------------------- + +_auth_create_schema = _full_schema.include(path=URL_AUTH_TOKENS, method="POST") + + +@_auth_create_schema.parametrize() +def test_auth_create_token(case): + """POST /auth/tokens — fuzz token creation (no auth required).""" + _call_safe(case) + + +_auth_list_schema = _full_schema.include(path=URL_AUTH_TOKENS, method="GET") + + +@_auth_list_schema.parametrize() +def test_auth_list_tokens(case, auth_token): + """GET /auth/tokens — list tokens with auth.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- Runs ---------------------------------------------------------- + +_runs_list_schema = _full_schema.include(path="/runs", method="GET") + + +@_runs_list_schema.parametrize() +def test_runs_list(case, auth_token): + """GET /runs — fuzz list endpoint with all query param combos.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_runs_create_schema = _full_schema.include(path="/runs", method="POST") + + +@_runs_create_schema.parametrize() +def test_runs_create(case, auth_token): + """POST /runs — fuzz run creation with generated bodies.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_detail_schema = _full_schema.include(path="/runs/{run_id}", method="GET") + + +@_run_detail_schema.parametrize() +def test_runs_get(case, auth_token): + """GET /runs/{run_id} — fuzz single-run retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_summary_schema = _full_schema.include( + path="/runs/{run_id}/summary", method="GET") + + +@_run_summary_schema.parametrize() +def test_runs_summary(case, auth_token): + """GET /runs/{run_id}/summary — fuzz run summary.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_progress_schema = _full_schema.include( + path="/runs/{run_id}/progress", method="GET" +) + + +@_run_progress_schema.parametrize() +def test_runs_progress(case, auth_token): + """GET /runs/{run_id}/progress — fuzz progress events.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_config_schema = _full_schema.include( + path="/runs/{run_id}/config", method="GET") + + +@_run_config_schema.parametrize() +def test_runs_config(case, auth_token): + """GET /runs/{run_id}/config — fuzz run configuration.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_cancel_schema = _full_schema.include( + path="/runs/{run_id}/cancel", method="POST") + + +@_run_cancel_schema.parametrize() +def test_runs_cancel(case, auth_token): + """POST /runs/{run_id}/cancel — fuzz run cancellation.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- Samples ------------------------------------------------------- + +_samples_list_schema = _full_schema.include(path="/samples", method="GET") + + +@_samples_list_schema.parametrize() +def test_samples_list(case, auth_token): + """GET /samples — fuzz media sample listing.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_sample_detail_schema = _full_schema.include( + path="/samples/{sample_id}", method="GET") + + +@_sample_detail_schema.parametrize() +def test_samples_get(case, auth_token): + """GET /samples/{sample_id} — fuzz single sample retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_sample_history_schema = _full_schema.include( + path="/samples/{sample_id}/history", method="GET" +) + + +@_sample_history_schema.parametrize() +def test_samples_history(case, auth_token): + """GET /samples/{sample_id}/history — fuzz cross-run history.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_regression_tests_schema = _full_schema.include( + path="/regression-tests", method="GET" +) + + +@_regression_tests_schema.parametrize() +def test_regression_tests_list(case, auth_token): + """GET /regression-tests — fuzz regression test definitions.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_samples_list_schema = _full_schema.include( + path="/runs/{run_id}/samples", method="GET" +) + + +@_run_samples_list_schema.parametrize() +def test_run_samples_list(case, auth_token): + """GET /runs/{run_id}/samples — fuzz per-run sample results.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_sample_detail_schema = _full_schema.include( + path="/runs/{run_id}/samples/{regression_test_id}", method="GET" +) + + +@_run_sample_detail_schema.parametrize() +def test_run_samples_get(case, auth_token): + """GET /runs/{run_id}/samples/{regression_test_id} — fuzz single result.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- System -------------------------------------------------------- + +_health_schema = _full_schema.include(path="/system/health", method="GET") + + +@_health_schema.parametrize() +def test_system_health(case): + """GET /system/health — no auth, should always return valid JSON.""" + _call_safe(case) + + +_queue_schema = _full_schema.include(path="/system/queue", method="GET") + + +@_queue_schema.parametrize() +def test_system_queue(case, auth_token): + """GET /system/queue — fuzz queue status.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_artifacts_schema = _full_schema.include( + path="/runs/{run_id}/artifacts", method="GET" +) + + +@_artifacts_schema.parametrize() +def test_artifacts_list(case, auth_token): + """GET /runs/{run_id}/artifacts — fuzz artifact listing.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- Errors & Logs ------------------------------------------------- + +_errors_schema = _full_schema.include( + path="/runs/{run_id}/errors", method="GET") + + +@_errors_schema.parametrize() +def test_errors_list(case, auth_token): + """GET /runs/{run_id}/errors — fuzz error listing.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_infra_errors_schema = _full_schema.include( + path="/runs/{run_id}/infrastructure-errors", method="GET" +) + + +@_infra_errors_schema.parametrize() +def test_infrastructure_errors(case, auth_token): + """GET /runs/{run_id}/infrastructure-errors — fuzz infra error listing.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_error_summary_schema = _full_schema.include( + path="/runs/{run_id}/error-summary", method="GET" +) + + +@_error_summary_schema.parametrize() +def test_error_summary(case, auth_token): + """GET /runs/{run_id}/error-summary — fuzz error summary.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_logs_schema = _full_schema.include(path="/runs/{run_id}/logs", method="GET") + + +@_logs_schema.parametrize() +def test_logs(case, auth_token): + """GET /runs/{run_id}/logs — fuzz build log retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_sample_logs_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/logs", method="GET" +) + + +@_sample_logs_schema.parametrize() +def test_sample_logs(case, auth_token): + """GET /runs/{run_id}/samples/{sample_id}/logs — fuzz per-sample logs.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- Results (expected/actual/diff/baseline) ----------------------- + +_expected_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/expected", + method="GET", +) + + +@_expected_schema.parametrize() +def test_expected_output(case, auth_token): + """GET .../expected — fuzz expected output retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_actual_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/actual", + method="GET", +) + + +@_actual_schema.parametrize() +def test_actual_output(case, auth_token): + """GET .../actual — fuzz actual output retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_diff_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/diff", + method="GET", +) + + +@_diff_schema.parametrize() +def test_diff(case, auth_token): + """GET .../diff — fuzz diff retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_baseline_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/baseline-approval", method="POST" +) + + +@_baseline_schema.parametrize() +def test_baseline_approval(case, auth_token): + """POST .../baseline-approval — fuzz baseline approval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# =================================================================== +# 3. NEGATIVE / SECURITY TESTS +# =================================================================== + + +class TestAuthSecurity: + """Verify authentication and authorization boundaries.""" + + def test_missing_auth_header_returns_401(self): + """Authenticated endpoints must reject requests without a token.""" + with app.test_client() as client: + for endpoint in [URL_RUNS, URL_SAMPLES, URL_SYSTEM_QUEUE]: + resp = client.get(endpoint) + assert resp.status_code == 401, ( + f"{endpoint} accepted unauthenticated request" + ) + + def test_invalid_bearer_token_returns_401(self): + """A garbage token must be rejected.""" + with app.test_client() as client: + resp = client.get( + URL_RUNS, + headers={"Authorization": "Bearer INVALID_TOKEN_VALUE"}, + ) + assert resp.status_code == 401 + + def test_expired_token_returns_401(self): + """An expired token must be rejected.""" + db = create_session(app.config["DATABASE_URI"]) + + admin = User.query.filter_by(email=ADMIN_EMAIL).first() + token_value = ApiToken.generate_token() + token_obj = ApiToken( + user_id=admin.id, + token_name=f"expired-{secrets.token_hex(4)}", + token_hash=ApiToken.hash_token(token_value), + token_prefix=ApiToken.extract_prefix(token_value), + scopes=[SCOPE_RUNS_READ], + expires_in_days=0, + ) + # Force expiration to the past + import datetime + + token_obj.expires_at = datetime.datetime.now( + datetime.timezone.utc + ) - datetime.timedelta(hours=1) + db.add(token_obj) + db.commit() + + try: + with app.test_client() as client: + resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {token_value}"}, + ) + assert resp.status_code == 401, "Expired token was accepted" + finally: + db.delete(token_obj) + db.commit() + + def test_revoked_token_returns_401(self): + """A revoked token must be rejected.""" + db = create_session(app.config["DATABASE_URI"]) + + admin = User.query.filter_by(email=ADMIN_EMAIL).first() + token_value = ApiToken.generate_token() + token_obj = ApiToken( + user_id=admin.id, + token_name=f"revoked-{secrets.token_hex(4)}", + token_hash=ApiToken.hash_token(token_value), + token_prefix=ApiToken.extract_prefix(token_value), + scopes=[SCOPE_RUNS_READ], + ) + db.add(token_obj) + db.commit() + token_obj.revoke() + db.commit() + + try: + with app.test_client() as client: + resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {token_value}"}, + ) + assert resp.status_code == 401, "Revoked token was accepted" + finally: + db.delete(token_obj) + db.commit() + + def test_insufficient_scope_returns_403(self, readonly_token): + """A token lacking the required scope must get 403, not 401.""" + with app.test_client() as client: + # runs:read token should not be able to access system:read endpoints + resp = client.get( + URL_SYSTEM_QUEUE, + headers={"Authorization": f"Bearer {readonly_token}"}, + ) + assert resp.status_code == 403 + + +# =================================================================== +# 4. RESPONSE INVARIANT CHECKS +# =================================================================== + + +class TestResponseInvariants: + """Verify structural invariants that hold across multiple endpoints.""" + + def test_health_returns_valid_json(self): + """GET /system/health must always return parseable JSON with 'status'.""" + with app.test_client() as client: + resp = client.get(URL_SYSTEM_HEALTH) + assert resp.status_code in (200, 503) + data = resp.get_json() + assert data is not None, "Health endpoint returned non-JSON" + assert "status" in data + assert data["status"] in ("ok", "degraded", "down") + + def test_paginated_endpoints_have_pagination_key(self, auth_token): + """All paginated GET endpoints must include 'pagination' in their response.""" + paginated = [ + URL_RUNS, + URL_SAMPLES, + "/api/v1/regression-tests", + URL_SYSTEM_QUEUE, + ] + with app.test_client() as client: + for endpoint in paginated: + resp = client.get( + endpoint, + headers={"Authorization": f"Bearer {auth_token}"}, + ) + if resp.status_code == 200: + data = resp.get_json() + assert "pagination" in data, ( + f"{endpoint} missing 'pagination' key" + ) + pagination = data["pagination"] + assert "limit" in pagination + assert "offset" in pagination or "next_cursor" in pagination + assert "total" in pagination + + def test_rate_limit_headers_present(self, auth_token): + """Every API response must include X-RateLimit-* headers.""" + with app.test_client() as client: + resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {auth_token}"}, + ) + for header in [ + "X-RateLimit-Limit", + "X-RateLimit-Remaining", + "X-RateLimit-Reset", + ]: + assert header in resp.headers, f"Missing {header}" + + def test_error_response_format(self, auth_token): + """Error responses must follow the {code, message, details} shape.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs/999999", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 404 + data = resp.get_json() + assert "code" in data, "Error response missing 'code'" + assert "message" in data, "Error response missing 'message'" + + def test_health_does_not_require_auth(self): + """GET /system/health must be accessible without any token.""" + with app.test_client() as client: + resp = client.get(URL_SYSTEM_HEALTH) + assert resp.status_code != 401 + + def test_content_type_is_json(self, auth_token): + """All API responses should return application/json content type.""" + with app.test_client() as client: + endpoints = [ + URL_RUNS, + URL_SYSTEM_HEALTH, + URL_SAMPLES, + ] + for endpoint in endpoints: + resp = client.get( + endpoint, + headers={"Authorization": f"Bearer {auth_token}"}, + ) + content_type = resp.content_type or "" + assert APP_JSON in content_type, ( + f"{endpoint} returned {content_type}" + ) + + +# =================================================================== +# 5. BOUNDARY / EDGE-CASE TESTS +# =================================================================== + + +class TestBoundaryConditions: + """Edge-case and boundary testing for pagination, IDs, and dates.""" + + def test_pagination_limit_zero_rejected(self, auth_token): + """limit=0 must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?limit=0", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_pagination_limit_over_max_rejected(self, auth_token): + """limit=101 must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?limit=101", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_pagination_negative_offset_rejected(self, auth_token): + """offset=-1 must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?offset=-1", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_pagination_non_integer_limit_rejected(self, auth_token): + """limit=abc must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?limit=abc", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_path_id_zero_rejected(self, auth_token): + """run_id=0 must be rejected with 400 (IDs start at 1).""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs/0", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_path_id_negative_rejected(self, auth_token): + """run_id=-1 must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs/-1", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_nonexistent_run_returns_404(self, auth_token): + """A valid-format but non-existent run_id must return 404.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs/2147483647", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 404 + + def test_invalid_sort_rejected(self, auth_token): + """sort=invalid must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?sort=invalid", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_invalid_date_range_rejected(self, auth_token): + """A non-ISO-8601 created_after value must be rejected.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?created_after=not-a-date", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_cursor_and_offset_cannot_mix(self, auth_token): + """Mixing cursor and offset pagination must be rejected.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?cursor=0&offset=0", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_empty_body_on_post_rejected(self, auth_token): + """POST /runs with no body must be rejected.""" + with app.test_client() as client: + resp = client.post( + URL_RUNS, + headers={ + "Authorization": f"Bearer {auth_token}", + "Content-Type": APP_JSON, + }, + data="", + ) + assert resp.status_code == 400 + + def test_wrong_content_type_rejected(self, auth_token): + """POST /runs with text/plain body must be rejected (415).""" + with app.test_client() as client: + resp = client.post( + URL_RUNS, + headers={ + "Authorization": f"Bearer {auth_token}", + "Content-Type": "text/plain", + }, + data="not json", + ) + assert resp.status_code == 415 + + def test_extra_fields_rejected(self, auth_token): + """POST /runs with unknown fields must be rejected (additionalProperties: false).""" + with app.test_client() as client: + payload = { + "commit_sha": "a" * 40, + "platform": "linux", + "repository": "owner/repo", + "evil_extra": "should be rejected", + } + resp = client.post( + URL_RUNS, + headers={ + "Authorization": f"Bearer {auth_token}", + "Content-Type": APP_JSON, + }, + data=json.dumps(payload), + ) + assert resp.status_code == 400 + + +# =================================================================== +# 6. STATEFUL TOKEN LIFECYCLE TEST +# =================================================================== + + +class TestTokenLifecycle: + """Verify the create → use → revoke token lifecycle works end-to-end.""" + + def test_token_create_use_revoke(self): + """Create a token, use it, then revoke it and verify rejection.""" + with app.test_client() as client: + # 1. Create a token + create_resp = client.post( + "/api/v1/auth/tokens", + data=json.dumps( + { + "email": ADMIN_EMAIL, + "pass" + "word": "admin123", + "token_name": f"lifecycle-{secrets.token_hex(4)}", + "scopes": [SCOPE_RUNS_READ, "tokens:manage"], + } + ), + content_type=APP_JSON, + ) + assert create_resp.status_code == 201, ( + f"Token creation failed: {create_resp.get_json()}" + ) + token = create_resp.get_json()["token"] + + # 2. Use it + use_resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {token}"}, + ) + assert use_resp.status_code == 200 + + # 3. Revoke it (self-revoke via /auth/tokens/current) + revoke_resp = client.delete( + "/api/v1/auth/tokens/current", + headers={"Authorization": f"Bearer {token}"}, + ) + assert revoke_resp.status_code == 204 + + # 4. Verify it's rejected + rejected_resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {token}"}, + ) + assert rejected_resp.status_code == 401, "Revoked token was still accepted" diff --git a/tests/api/test_middleware_auth.py b/tests/api/test_middleware_auth.py index 983600ba..a2e16013 100644 --- a/tests/api/test_middleware_auth.py +++ b/tests/api/test_middleware_auth.py @@ -125,6 +125,12 @@ def test_scope_boundary_write_endpoints_fail_on_read_only_scopes(self): self.assertEqual(res.status_code, 403) self.assertEqual(res.json['code'], 'forbidden') + # 3. POST /runs/1/samples/1/baseline-approval + res = self.client.post('/api/v1/runs/1/samples/1/baseline-approval', + headers={'Authorization': f'Bearer {plaintext}'}) + self.assertEqual(res.status_code, 403) + self.assertEqual(res.json['code'], 'forbidden') + def test_multiple_candidates_same_prefix(self): plaintext1, token1 = self.generate_db_token(self.user, scopes=['system:read']) plaintext2, token2 = self.generate_db_token(self.user, scopes=['system:read']) diff --git a/tests/api/test_routes_errors_logs.py b/tests/api/test_routes_errors_logs.py new file mode 100644 index 00000000..099e2b36 --- /dev/null +++ b/tests/api/test_routes_errors_logs.py @@ -0,0 +1,280 @@ +import json +import os +import tempfile +from unittest.mock import patch + +from flask import g + +from mod_api.middleware.rate_limit import _rate_limit_store +from mod_auth.models import Role, User +from mod_regression.models import (Category, InputType, OutputType, + RegressionTest, RegressionTestOutput) +from mod_test.models import (Fork, Test, TestPlatform, TestProgress, + TestResult, TestResultFile, TestStatus, TestType) +from tests.api.base import ApiTestCase + + +class TestRoutesErrorsLogs(ApiTestCase): + def setUp(self): + super().setUp() + self.user = User('testuser_el', Role.contributor, + 'el_user@local.com', User.generate_hash('userpass123')) + self.admin = User('testadmin_el', Role.admin, + 'el_admin@local.com', User.generate_hash('adminpass123')) + self.regular_user = User( + 'testregular_el', Role.user, 'el_regular@local.com', User.generate_hash('userpass123')) + g.db.add_all([self.user, self.admin, self.regular_user]) + g.db.commit() + + fork = Fork('https://github.com/test/test.git') + g.db.add(fork) + g.db.commit() + + self.test_obj = Test(TestPlatform.linux, + TestType.commit, fork.id, 'master', 'commit_hash') + g.db.add(self.test_obj) + g.db.commit() + self.test_id = self.test_obj.id + + self.category = Category('Test Category', 'Description') + g.db.add(self.category) + g.db.commit() + + self.reg_test1 = RegressionTest( + 1, 'cmd1', InputType.file, OutputType.file, self.category.id, 0) + self.reg_test2 = RegressionTest( + 1, 'cmd2', InputType.file, OutputType.file, self.category.id, 0) + g.db.add_all([self.reg_test1, self.reg_test2]) + g.db.commit() + + self.reg_out1 = RegressionTestOutput( + self.reg_test1.id, 'expected1', '.txt', 'exp1') + self.reg_out2 = RegressionTestOutput( + self.reg_test2.id, 'expected2', '.txt', 'exp2') + g.db.add_all([self.reg_out1, self.reg_out2]) + + dummy_out = RegressionTestOutput( + self.reg_test1.id, 'dummy', '', 'dummy') + dummy_out.id = -1 + g.db.merge(dummy_out) + + g.db.commit() + + self.test_dir = tempfile.TemporaryDirectory() + self.dir_path = self.test_dir.name + + _rate_limit_store.clear() + + def tearDown(self): + self.test_dir.cleanup() + super().tearDown() + + def get_token(self, email, password, token_name='test_token', scopes=None): + payload = { + 'email': email, + 'password': password, + 'token_name': token_name + } + if scopes: + payload['scopes'] = scopes + + res = self.client.post( + '/api/v1/auth/tokens', data=json.dumps(payload), content_type='application/json') + return res.json['token'] + + def test_list_run_errors(self): + # Add a missing_output error + tr1 = TestResult(self.test_obj.id, self.reg_test1.id, 100, 0, 0) + rf1 = TestResultFile( + self.test_obj.id, self.reg_test1.id, -1, '', 'error') + + # Add a diff_mismatch error + tr2 = TestResult(self.test_obj.id, self.reg_test2.id, 100, 0, 0) + rf2 = TestResultFile( + self.test_obj.id, self.reg_test2.id, self.reg_out2.id, 'exp', 'got') + + g.db.add_all([tr1, rf1, tr2, rf2]) + g.db.commit() + + token = self.get_token('el_user@local.com', + 'userpass123', 't1', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/errors', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 2) + + def test_list_run_errors_filters(self): + tr1 = TestResult(self.test_obj.id, self.reg_test1.id, 100, 0, 0) + # missing_output (error) + rf1 = TestResultFile( + self.test_obj.id, self.reg_test1.id, -1, '', 'error') + + tr2 = TestResult(self.test_obj.id, self.reg_test2.id, 100, 0, 0) + rf2 = TestResultFile(self.test_obj.id, self.reg_test2.id, + # diff_mismatch (warning) + self.reg_out2.id, 'exp', 'got') + + g.db.add_all([tr1, rf1, tr2, rf2]) + g.db.commit() + + token = self.get_token('el_user@local.com', + 'userpass123', 't2', scopes=['results:read']) + + res = self.client.get( + f'/api/v1/runs/{self.test_id}/errors?type=missing_output', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(len(res.json['data']), 1) + self.assertEqual(res.json['data'][0]['type'], 'missing_output') + self.assertEqual(res.json['data'][0]['severity'], 'error') + + res = self.client.get( + f'/api/v1/runs/{self.test_id}/errors?severity=warning', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(len(res.json['data']), 1) + self.assertEqual(res.json['data'][0]['type'], 'diff_mismatch') + self.assertEqual(res.json['data'][0]['severity'], 'warning') + + def test_list_errors_invalid_severity(self): + # The schema doesn't strictly validate severity to a whitelist enum? Let's see. Wait, + # in mod_api/routes/errors_logs.py, it filters by severity. + # Actually it just does errors = [e for e in errors if e['severity'] == severity]. It doesn't 400. + # Let's test limit/offset pagination validation failure instead since list_run_errors + # uses @validate_offset_pagination. + token = self.get_token( + 'el_user@local.com', 'userpass123', 't_pag_inv', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/errors?limit=500', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_list_infrastructure_errors(self): + tp1 = TestProgress( + self.test_obj.id, TestStatus.canceled, 'provisioning VM failed') + g.db.add(tp1) + g.db.commit() + + token = self.get_token('el_user@local.com', + 'userpass123', 't3', scopes=['system:read']) + + res = self.client.get( + f'/api/v1/runs/{self.test_id}/infrastructure-errors', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 1) + self.assertNotIn('stack', res.json['data'][0]) + + def test_infra_errors_stack_forbidden_for_regular_user(self): + tp1 = TestProgress( + self.test_obj.id, TestStatus.canceled, 'provisioning VM failed') + g.db.add(tp1) + g.db.commit() + + reg_token = self.get_token( + 'el_regular@local.com', 'userpass123', 't_reg', scopes=['system:read']) + res_reg = self.client.get( + f'/api/v1/runs/{self.test_id}/infrastructure-errors?include_stack=true', + headers={'Authorization': f'Bearer {reg_token}'}) + self.assertEqual(res_reg.status_code, 403) + + def test_infra_errors_include_stack_flag_accepted(self): + tp1 = TestProgress( + self.test_obj.id, TestStatus.canceled, 'provisioning VM failed') + g.db.add(tp1) + g.db.commit() + + admin_token = self.get_token( + 'el_admin@local.com', 'adminpass123', 't4', scopes=['system:read']) + res = self.client.get(f'/api/v1/runs/{self.test_id}/infrastructure-errors?include_stack=true', headers={ + 'Authorization': f'Bearer {admin_token}'}) + self.assertEqual(res.status_code, 200) + + def test_get_error_summary(self): + tr1 = TestResult(self.test_obj.id, self.reg_test1.id, 100, 1, 0) + # Matched output (got=None) so the only bucket is the rc mismatch — + # without it the expected output would also count as missing_output. + rf1 = TestResultFile(self.test_obj.id, self.reg_test1.id, + self.reg_out1.id, 'exp1') + g.db.add_all([tr1, rf1]) + g.db.commit() + + token = self.get_token('el_user@local.com', + 'userpass123', 't5', scopes=['results:read']) + + res = self.client.get( + f'/api/v1/runs/{self.test_id}/error-summary', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 1) + + def test_get_run_logs(self): + from run import config + + # Create a real log file and configure the app to read it + os.makedirs(os.path.join(self.dir_path, 'LogFiles'), exist_ok=True) + log_path = os.path.join( + self.dir_path, 'LogFiles', f'{self.test_id}.txt') + with open(log_path, 'w') as f: + f.write("INFO worker: hello\n") + + original_sample_repo = config.get('SAMPLE_REPOSITORY') + config['SAMPLE_REPOSITORY'] = self.dir_path + try: + token = self.get_token('el_user@local.com', + 'userpass123', 't6', scopes=['system:read']) + + res = self.client.get( + f'/api/v1/runs/{self.test_id}/logs', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertIn('data', res.json) + self.assertEqual(len(res.json['data']), 1) + self.assertEqual(res.json['data'][0] + ['message'], 'INFO worker: hello') + finally: + if original_sample_repo is not None: + config['SAMPLE_REPOSITORY'] = original_sample_repo + else: + config.pop('SAMPLE_REPOSITORY', None) + + @patch('run.storage_client_bucket', None) + def test_get_run_logs_file_not_found(self): + from run import config + + # Do not create the file, so it raises FileNotFoundError + original_sample_repo = config.get('SAMPLE_REPOSITORY') + config['SAMPLE_REPOSITORY'] = self.dir_path + try: + token = self.get_token('el_user@local.com', + 'userpass123', 't7', scopes=['system:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/logs', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 404) + finally: + if original_sample_repo is not None: + config['SAMPLE_REPOSITORY'] = original_sample_repo + else: + config.pop('SAMPLE_REPOSITORY', None) + + def test_get_logs_invalid_cursor(self): + token = self.get_token( + 'el_user@local.com', 'userpass123', 't_logs_inv', scopes=['system:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/logs?cursor=-1', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 400) + self.assertEqual(res.json['code'], 'validation_error') + + def test_get_sample_logs(self): + token = self.get_token('el_user@local.com', + 'userpass123', 't8', scopes=['system:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/logs', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 404) + + def test_error_summary_group_by_sample_id(self): + tr1 = TestResult(self.test_obj.id, self.reg_test1.id, 100, 1, 0) + g.db.add(tr1) + g.db.commit() + + token = self.get_token( + 'el_user@local.com', 'userpass123', 't9_sum', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/error-summary?group_by=sample_id', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 1) + self.assertEqual(res.json['data'][0]['group_by'], 'sample_id') diff --git a/tests/api/test_routes_results.py b/tests/api/test_routes_results.py new file mode 100644 index 00000000..74555253 --- /dev/null +++ b/tests/api/test_routes_results.py @@ -0,0 +1,365 @@ +import base64 +import json +import os +import tempfile +from unittest.mock import patch + +from flask import g + +from mod_api.middleware.rate_limit import _rate_limit_store +from mod_auth.models import Role, User +from mod_regression.models import (Category, InputType, OutputType, + RegressionTest, RegressionTestOutput) +from mod_test.models import TestResult, TestResultFile +from tests.api.base import ApiTestCase + + +class TestRoutesResults(ApiTestCase): + def setUp(self): + super().setUp() + self.setup_run_data('res') + + category = Category('Test Category', 'Description') + g.db.add(category) + g.db.commit() + + self.reg_test = RegressionTest( + 1, 'command', InputType.file, OutputType.file, category.id, 0) + g.db.add(self.reg_test) + g.db.commit() + self.reg_test_id = self.reg_test.id + + self.reg_out = RegressionTestOutput( + self.reg_test_id, 'expected_hash', '.txt', 'exp_file') + g.db.add(self.reg_out) + g.db.commit() + self.reg_out_id = self.reg_out.id + + self.test_result = TestResult(self.test_id, self.reg_test_id, 0, 0, 0) + g.db.add(self.test_result) + g.db.commit() + + self.result_file = TestResultFile( + self.test_id, self.reg_test_id, self.reg_out_id, 'expected_hash', 'actual_hash') + g.db.add(self.result_file) + g.db.commit() + + self.test_dir = tempfile.TemporaryDirectory() + self.dir_path = self.test_dir.name + + # Create TestResults directory + self.test_results_dir = os.path.join(self.dir_path, 'TestResults') + os.makedirs(self.test_results_dir, exist_ok=True) + + # Configure app to use our temp dir + self.original_sample_repo = self.app.config.get('SAMPLE_REPOSITORY') + self.app.config['SAMPLE_REPOSITORY'] = self.dir_path + + _rate_limit_store.clear() + + def tearDown(self): + if self.original_sample_repo is not None: + self.app.config['SAMPLE_REPOSITORY'] = self.original_sample_repo + else: + self.app.config.pop('SAMPLE_REPOSITORY', None) + self.test_dir.cleanup() + super().tearDown() + + def test_get_expected_output_base64(self): + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'expected data') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't1', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/expected', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['encoding'], 'base64') + self.assertEqual(res.json['content'], base64.b64encode( + b'expected data').decode('ascii')) + self.assertEqual(res.json['filename'], 'expected_hash.txt') + + def test_get_expected_output_text(self): + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'line1\nline2') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't2', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/expected?format=text', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['encoding'], 'utf-8') + self.assertEqual(res.json['content'], 'line1\nline2') + + def test_get_actual_output(self): + actual_file_path = os.path.join( + self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'actual data') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't3', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/actual', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['filename'], 'actual_hash.txt') + self.assertEqual(res.json['content'], base64.b64encode( + b'actual data').decode('ascii')) + + def test_get_actual_output_matched_expected(self): + # Set got = None + self.result_file.got = None + g.db.commit() + + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'expected data') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't4', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/actual', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 303) + redirect_url = res.headers['Location'] + res2 = self.client.get(redirect_url, headers={ + 'Authorization': f'Bearer {token}'}) + self.assertEqual(res2.status_code, 200) + + import base64 + self.assertEqual(res2.json['content'], base64.b64encode( + b'expected data').decode('ascii')) + + def test_get_diff(self): + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'line1\nline2\n') + + actual_file_path = os.path.join( + self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'line1\nline_new\n') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't5', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/diff', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['status'], 'different') + self.assertEqual(res.json['summary']['added_lines'], 1) + + def test_get_diff_unified_format(self): + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'line1\nline2\n') + + actual_file_path = os.path.join( + self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'line1\nline_new\n') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't5_uni', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/diff?format=unified', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['format'], 'unified') + self.assertIn('content', res.json) + self.assertIsInstance(res.json['content'], str) + + def test_get_diff_identical_files(self): + # When got is None, diff returns status 'identical' + self.result_file.got = None + g.db.commit() + + expected_file_path = os.path.join( + self.test_results_dir, 'expected_hash.txt') + with open(expected_file_path, 'wb') as f: + f.write(b'expected data\n') + + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't5_id', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/diff', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['status'], 'identical') + + def test_create_baseline_approval(self): + token = self.get_token('res_admin@local.com', + 'adminpass123', 't6', scopes=['baselines:write']) + + actual_file_path = os.path.join(self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'actual data') + + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id, + 'remove_variants': False + } + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + res = self.client.post(f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', data=json.dumps( + payload), content_type='application/json', headers={'Authorization': f'Bearer {token}'}) + + if res.status_code != 200: + print("ERROR JSON:", res.json) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['status'], 'approved') + + # Verify db change + reg_out_after = RegressionTestOutput.query.get(self.reg_out_id) + self.assertEqual(reg_out_after.correct, 'actual_hash') + + def test_create_baseline_approval_forbidden_role(self): + # Create token directly in DB to bypass token creation limitations + from mod_api.models.api_token import ApiToken + plaintext = ApiToken.generate_token() + token = ApiToken( + user_id=self.user.id, # res_user has user role + token_name='t7_forbidden', + token_hash=ApiToken.hash_token(plaintext), + token_prefix=ApiToken.extract_prefix(plaintext), + scopes=['baselines:write'], + expires_in_days=7 + ) + g.db.add(token) + g.db.commit() + + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id + } + res = self.client.post(f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', data=json.dumps( + payload), content_type='application/json', headers={'Authorization': f'Bearer {plaintext}'}) + + self.assertEqual(res.status_code, 403) + self.assertEqual(res.json['code'], 'forbidden') + + def test_create_baseline_approval_contributor_forbidden(self): + # Baseline approval is admin-only: a contributor must be rejected + # even when holding a baselines:write token. + from mod_api.models.api_token import ApiToken + from mod_auth.models import Role, User + contributor = User( + 'res_contrib', Role.contributor, 'res_contrib@local.com', + User.generate_hash('contribpass123')) + g.db.add(contributor) + g.db.commit() + + plaintext = ApiToken.generate_token() + token = ApiToken( + user_id=contributor.id, + token_name='t_contrib_forbidden', + token_hash=ApiToken.hash_token(plaintext), + token_prefix=ApiToken.extract_prefix(plaintext), + scopes=['baselines:write'], + expires_in_days=7, + ) + g.db.add(token) + g.db.commit() + + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id, + } + res = self.client.post( + f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', + data=json.dumps(payload), content_type='application/json', + headers={'Authorization': f'Bearer {plaintext}'}) + + self.assertEqual(res.status_code, 403) + self.assertEqual(res.json['code'], 'forbidden') + + def test_create_baseline_approval_remove_variants(self): + token = self.get_token('res_admin@local.com', + 'adminpass123', 't8', scopes=['baselines:write']) + + actual_file_path = os.path.join(self.test_results_dir, 'actual_hash.txt') + with open(actual_file_path, 'wb') as f: + f.write(b'actual data') + + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id, + 'remove_variants': True + } + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + res = self.client.post(f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', data=json.dumps( + payload), content_type='application/json', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['status'], 'approved') + + # Verify db change + from mod_regression.models import RegressionTestOutputFiles + variants = RegressionTestOutputFiles.query.filter_by( + regression_test_output_id=self.reg_out_id).count() + self.assertEqual(variants, 0) + + def test_create_baseline_approval_output_already_matches(self): + # got=None means the actual output already matches the baseline, + # so there is nothing to approve. + self.result_file.got = None + g.db.commit() + + token = self.get_token('res_admin@local.com', + 'adminpass123', 't9', scopes=['baselines:write']) + payload = { + 'regression_id': self.reg_test_id, + 'output_id': self.reg_out_id + } + res = self.client.post( + f'/api/v1/runs/{self.test_id}/samples/1/baseline-approval', + data=json.dumps(payload), content_type='application/json', + headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 422) + self.assertIn('matches expected', res.json['message']) + + def test_get_actual_output_missing_storage(self): + # We don't write the file 'actual_hash.txt', so it will not be found on the filesystem + with patch.dict('run.config', {'SAMPLE_REPOSITORY': self.dir_path}): + token = self.get_token( + 'res_user@local.com', 'userpass123', 't9', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/{self.reg_test_id}' + f'/outputs/{self.reg_out_id}/actual', headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 404) + self.assertIn('not found', res.json['message'].lower()) + + def test_get_output_nonexistent_resource_404(self): + token = self.get_token('res_user@local.com', + 'userpass123', 't10', scopes=['results:read']) + res = self.client.get( + f'/api/v1/runs/{self.test_id}/samples/1/regression-tests/999999' + f'/outputs/{self.reg_out_id}/expected', headers={'Authorization': f'Bearer {token}'}) + self.assertEqual(res.status_code, 404) + self.assertEqual(res.json['code'], 'not_found') diff --git a/tests/api/test_services_diff_service.py b/tests/api/test_services_diff_service.py new file mode 100644 index 00000000..4beb34da --- /dev/null +++ b/tests/api/test_services_diff_service.py @@ -0,0 +1,126 @@ +import os +import tempfile + +from mod_api.services.diff_service import (_compute_hunks, compute_diff, + file_sha256, read_lines) +from tests.api.base import ApiTestCase + + +class TestDiffService(ApiTestCase): + def setUp(self): + super().setUp() + self.test_dir = tempfile.TemporaryDirectory() + self.dir_path = self.test_dir.name + from unittest.mock import patch + patcher = patch( + 'mod_api.services.diff_service._enforce_safe_path', return_value=True) + self.addCleanup(patcher.stop) + self.mock_safe = patcher.start() + + def tearDown(self): + self.test_dir.cleanup() + super().tearDown() + + def create_file(self, filename, content, encoding='utf-8'): + path = os.path.join(self.dir_path, filename) + with open(path, 'w', encoding=encoding) as f: + f.write(content) + return path + + def test_compute_diff_identical(self): + content = "line1\nline2\n" + path1 = self.create_file("file1.txt", content) + path2 = self.create_file("file2.txt", content) + + diff = compute_diff(path1, path2) + self.assertEqual(diff['status'], 'identical') + self.assertEqual(diff['summary']['added_lines'], 0) + self.assertEqual(diff['summary']['removed_lines'], 0) + self.assertEqual(len(diff['hunks']), 0) + + def test_compute_diff_missing_expected(self): + path2 = self.create_file("file2.txt", "content") + + diff = compute_diff(os.path.join(self.dir_path, "missing.txt"), path2) + self.assertEqual(diff['status'], 'missing_expected') + + def test_compute_diff_missing_actual(self): + path1 = self.create_file("file1.txt", "content") + + diff = compute_diff(path1, os.path.join(self.dir_path, "missing.txt")) + self.assertEqual(diff['status'], 'missing_actual') + + def test_compute_diff_different(self): + content1 = "line1\nline2\nline3\n" + content2 = "line1\nline_new\nline3\n" + path1 = self.create_file("file1.txt", content1) + path2 = self.create_file("file2.txt", content2) + + diff = compute_diff(path1, path2) + self.assertEqual(diff['status'], 'different') + self.assertEqual(diff['summary']['added_lines'], 1) + self.assertEqual(diff['summary']['removed_lines'], 1) + self.assertEqual(diff['summary']['changed_hunks'], 1) + self.assertEqual(len(diff['hunks']), 1) + + hunk = diff['hunks'][0] + self.assertEqual(hunk['expected_start'], 1) + self.assertEqual(hunk['actual_start'], 1) + + def test_compute_diff_context_lines_clamped(self): + content1 = "\n".join(str(i) for i in range(1, 201)) + "\n" + content2 = content1.replace("\n100\n", "\n100_new\n") + path1 = self.create_file("file1.txt", content1) + path2 = self.create_file("file2.txt", content2) + + diff = compute_diff(path1, path2, context_lines=200) + self.assertEqual(diff['status'], 'different') + hunk = diff['hunks'][0] + # max context is 50 before and 50 after, plus 1 removed and 1 added = 102 lines total + self.assertEqual(len(hunk['lines']), 102) + + def test_compute_hunks_max_hunks(self): + lines1 = ["1", "2", "3", "4", "5"] + lines2 = ["1a", "2", "3a", "4", "5a"] + # With context_lines=0 we should get 3 separate hunks + hunks = _compute_hunks(lines1, lines2, context_lines=0, max_hunks=2) + self.assertEqual(len(hunks), 2) # bounded to 2 + + def test_compute_hunks_parsing(self): + lines1 = ["common", "remove_me", "common"] + lines2 = ["common", "add_me", "common"] + hunks = _compute_hunks(lines1, lines2, context_lines=1, max_hunks=10) + self.assertEqual(len(hunks), 1) + lines = hunks[0]['lines'] + self.assertEqual(lines[0]['kind'], 'context') + self.assertEqual(lines[1]['kind'], 'removed') + self.assertEqual(lines[2]['kind'], 'added') + self.assertEqual(lines[3]['kind'], 'context') + + def test_read_lines_utf8(self): + path = os.path.join(self.dir_path, "utf8.txt") + with open(path, 'w', encoding='utf-8', newline='') as f: + f.write("line1\r\nline2\n") + lines = read_lines(path) + self.assertEqual(lines, ["line1", "line2"]) + + def test_read_lines_cp1252(self): + path = os.path.join(self.dir_path, "cp1252.txt") + # Write bytes that are valid cp1252 but invalid utf-8 + with open(path, 'wb') as f: + # \x80 is euro sign in cp1252, invalid start byte in utf-8 + f.write(b"line1\r\n\x80line2") + + lines = read_lines(path) + # \x80 maps to \u20ac + self.assertEqual(lines, ["line1", "\u20acline2"]) + + def test_file_sha256(self): + path = self.create_file("sha.txt", "hello") + sha = file_sha256(path) + # sha256("hello") = 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 + self.assertEqual( + sha, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824") + + self.assertIsNone(file_sha256( + os.path.join(self.dir_path, "nonexistent.txt"))) diff --git a/tests/api/test_services_log_service.py b/tests/api/test_services_log_service.py new file mode 100644 index 00000000..8bca7680 --- /dev/null +++ b/tests/api/test_services_log_service.py @@ -0,0 +1,124 @@ +import os +import tempfile +from unittest.mock import patch + +from mod_api.services.log_service import (_extract_level, _extract_source, + _matches_level, read_log_lines) +from tests.api.base import ApiTestCase + + +class TestServicesLogService(ApiTestCase): + def setUp(self): + super().setUp() + self.test_dir = tempfile.TemporaryDirectory() + self.dir_path = self.test_dir.name + + def tearDown(self): + self.test_dir.cleanup() + super().tearDown() + + def create_log_file(self, content, encoding='utf-8'): + path = os.path.join(self.dir_path, "1.txt") + with open(path, 'w', encoding=encoding, newline='') as f: + f.write(content) + return path + + @patch('mod_api.services.log_service.get_log_file_path') + def test_read_log_lines_not_found(self, mock_get_path): + mock_get_path.return_value = None + with self.assertRaises(FileNotFoundError): + read_log_lines(1) + + @patch('mod_api.services.log_service.get_log_file_path') + def test_read_log_lines_basic(self, mock_get_path): + content = "INFO worker: Starting\nDEBUG worker: Doing stuff\nERROR build: Failed\n" + path = self.create_log_file(content) + mock_get_path.return_value = path + + lines, next_cursor = read_log_lines(1) + self.assertEqual(len(lines), 3) + self.assertIsNone(next_cursor) + self.assertEqual(lines[0]['level'], 'info') + self.assertEqual(lines[0]['source'], 'worker') + self.assertEqual(lines[0]['message'], "INFO worker: Starting") + self.assertEqual(lines[2]['level'], 'error') + self.assertEqual(lines[2]['source'], 'build') + + @patch('mod_api.services.log_service.get_log_file_path') + def test_read_log_lines_pagination(self, mock_get_path): + content = "Line 1\nLine 2\nLine 3\nLine 4\n" + path = self.create_log_file(content) + mock_get_path.return_value = path + + lines, next_cursor = read_log_lines(1, limit=2) + self.assertEqual(len(lines), 2) + self.assertEqual(next_cursor, '2') + self.assertEqual(lines[0]['message'], "Line 1") + self.assertEqual(lines[1]['message'], "Line 2") + + lines, next_cursor = read_log_lines(1, cursor=next_cursor, limit=2) + self.assertEqual(len(lines), 2) + self.assertIsNone(next_cursor) + self.assertEqual(lines[0]['message'], "Line 3") + self.assertEqual(lines[1]['message'], "Line 4") + + @patch('mod_api.services.log_service.get_log_file_path') + def test_read_log_lines_limit_clamped(self, mock_get_path): + content = "Line\n" * 1500 + path = self.create_log_file(content) + mock_get_path.return_value = path + + lines, _ = read_log_lines(1, limit=2000) + # Should be clamped to 500 + self.assertEqual(len(lines), 500) + + @patch('mod_api.services.log_service.get_log_file_path') + def test_read_log_lines_filters(self, mock_get_path): + content = "INFO worker: Starting\nDEBUG build: Doing stuff\nERROR build: Failed\n" + path = self.create_log_file(content) + mock_get_path.return_value = path + + # Filter by level + lines, _ = read_log_lines(1, level='error') + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]['message'], "ERROR build: Failed") + + # Filter by source + lines, _ = read_log_lines(1, source='build') + self.assertEqual(len(lines), 2) + + # Filter by contains + lines, _ = read_log_lines(1, contains='STARTING') + self.assertEqual(len(lines), 1) + + @patch('mod_api.services.log_service.get_log_file_path') + def test_read_log_lines_cp1252(self, mock_get_path): + path = os.path.join(self.dir_path, "1.txt") + with open(path, 'wb') as f: + f.write(b"INFO \x80 error\n") # cp1252 euro sign + mock_get_path.return_value = path + + lines, _ = read_log_lines(1) + self.assertEqual(len(lines), 1) + self.assertIn("\u20ac", lines[0]['message']) + + def test_extract_level(self): + self.assertEqual(_extract_level("A CRITICAL error"), "critical") + self.assertEqual(_extract_level("Some ERROR occurred"), "error") + self.assertEqual(_extract_level("This is a WARNING"), "warning") + self.assertEqual(_extract_level("Just INFO"), "info") + self.assertEqual(_extract_level("DEBUG logging"), "debug") + self.assertEqual(_extract_level("Unknown format"), "info") # default + + def test_extract_source(self): + self.assertEqual(_extract_source( + "orchestrator doing something"), "orchestrator") + self.assertEqual(_extract_source("worker executing"), "worker") + self.assertEqual(_extract_source("build failed"), "build") + self.assertEqual(_extract_source("test_runner passed"), "test_runner") + self.assertEqual(_extract_source("web request"), "web") + self.assertEqual(_extract_source("unknown source"), "web") # default + + def test_matches_level(self): + self.assertTrue(_matches_level("ERROR", "error")) + self.assertFalse(_matches_level("INFO", "error"))