[Feature] : API ENDPOINTS PR 7: Test Artifacts

.github/workflows/ci-vm-scripts.yml

@@ -0,0 +1,37 @@
+name: CI-VM scripts
+
+# Static analysis for the CI-VM runner scripts (the scripts that run on the
+# throwaway test VMs). Runs only when those scripts change. shellcheck lints the
+# bash runners; PSScriptAnalyzer lints the PowerShell startup script.
+
+on:
+  pull_request:
+    paths:
+      - 'install/ci-vm/**'
+      - '.github/workflows/ci-vm-scripts.yml'
+  push:
+    paths:
+      - 'install/ci-vm/**'
+      - '.github/workflows/ci-vm-scripts.yml'
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install shellcheck
+        run: sudo apt-get update && sudo apt-get install -y shellcheck
+      - name: shellcheck (bash runner scripts)
+        run: shellcheck --severity=error install/ci-vm/ci-linux/ci/runCI install/ci-vm/ci-linux/startup-script.sh
+
+  psscriptanalyzer:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: PSScriptAnalyzer (startup-script.ps1)
+        shell: pwsh
+        run: |
+          Set-PSRepository PSGallery -InstallationPolicy Trusted
+          Install-Module -Name PSScriptAnalyzer -Force -Scope CurrentUser
+          $issues = Invoke-ScriptAnalyzer -Path install/ci-vm/ci-windows/startup-script.ps1 -Severity Error
+          if ($issues) { $issues | Format-Table -AutoSize; exit 1 }

install/ci-vm/ci-linux/ci/runCI

@@ -103,6 +103,27 @@ function executeCommand {
         fi
 }
 
+# Send a build artifact to the server (best-effort: a failed upload must never
+# abort the run, so this always returns 0). Reuses the reportURL/curl pattern.
+function sendArtifact {
+    local artifactType="$1"
+    local filePath="$2"
+    if [ ! -f "${filePath}" ]; then
+        echo "Artifact ${artifactType}: no file at '${filePath}', skipping" >> "${logFile}"
+        return 0
+    fi
+    echo "Uploading ${artifactType} artifact (${filePath})" >> "${logFile}"
+    # --fail surfaces HTTP >= 400 (e.g. 413 on an oversized coredump) as a
+    # curl error so failed uploads are visible in the log; still best-effort.
+    curl -s --fail -A "${userAgent}" --form "type=artifactupload" --form "artifact_type=${artifactType}" \
+        --form "file=@${filePath}" "${reportURL}" >> "${logFile}" 2>&1
+    local curlStatus=$?
+    if [ ${curlStatus} -ne 0 ]; then
+        echo "Artifact ${artifactType}: upload failed (curl exit ${curlStatus})" >> "${logFile}"
+    fi
+    return 0
+}
+
 # Source variables
 . "$DIR/variables"
 
@@ -125,7 +146,25 @@ if [ -e "${dstDir}/ccextractor" ]; then
         echo "=== End Version Info ===" >> "${logFile}"
         postStatus "testing" "Running tests"
         executeCommand cd ${suiteDstDir}
-        executeCommand ${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}"
+
+        # Enable core dumps and capture the test run's combined stdout/stderr so
+        # both can be uploaded as artifacts (the API serves them per run).
+        ulimit -c unlimited 2>/dev/null
+        echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern >/dev/null 2>&1
+        combinedLog="${reportFolder}/combined_stdout.log"
+        ${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}" > "${combinedLog}" 2>&1
+        testerStatus=$?
+        cat "${combinedLog}" >> "${logFile}"
+
+        # Upload artifacts before any failure-halt so crash data survives.
+        sendArtifact "binary" "${dstDir}/ccextractor"
+        sendArtifact "combined_stdout" "${combinedLog}"
+        sendArtifact "coredump" "$(ls -1t core.* 2>/dev/null | head -n1)"
+
+        if [ ${testerStatus} -ne 0 ]; then
+                haltAndCatchFire ""
+        fi
+
         sendLogFile
         postStatus "completed" "Ran all tests"
 

install/ci-vm/ci-linux/startup-script.sh

@@ -4,7 +4,7 @@ curl -L -O https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v3.2
 dpkg --install gcsfuse_3.2.0_amd64.deb
 rm gcsfuse_3.2.0_amd64.deb
 
-apt install gnupg ca-certificates
+apt install -y gnupg ca-certificates
 gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
 echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list
 sudo apt update
@@ -14,7 +14,8 @@ mkdir repository
 cd repository
 
 # Use gcsfuse and import required files
-mkdir temp TestFiles TestResults vm_data reports
+# TempFiles is used by the tester (--tempfolder) and must exist
+mkdir temp TestFiles TestResults TempFiles vm_data reports
 
 gcs_bucket=$(curl http://metadata/computeMetadata/v1/instance/attributes/bucket -H "Metadata-Flavor: Google")
 
@@ -31,6 +32,9 @@ mount vm_data
 mount TestFiles
 mount TestResults
 
+# Give gcsfuse mounts time to become ready
+sleep 10
+
 cp temp/* ./
 
 chmod +x bootstrap

install/ci-vm/ci-windows/ci/runCI.bat

@@ -15,7 +15,10 @@ for /F %%R in ('curl http://metadata/computeMetadata/v1/instance/attributes/repo
 SET userAgent="CCX/CI_BOT"
 SET logFile="%reportFolder%/log.html"
 
-call :postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact" >> "%logFile%"
+rem NB: no outer ">> %logFile%" here. postStatus already appends to %logFile%
+rem internally; an outer redirect to the same file self-locks on Windows
+rem (the inner append cannot open a file the outer redirect holds open).
+call :postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact"
 
 echo Checking for CCExtractor build artifact
 if EXIST "%dstDir%\ccextractorwinfull.exe" (
@@ -27,7 +30,21 @@ if EXIST "%dstDir%\ccextractorwinfull.exe" (
     echo === End Version Info === >> "%logFile%"
     call :postStatus "testing" "Running tests"
     call :executeCommand cd %suiteDstDir%
-    call :executeCommand "%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%"
+
+    rem Capture the test run's combined stdout/stderr for upload as an artifact.
+    rem NB: "||" instead of "if errorlevel 1" - it fires at runtime on ANY
+    rem nonzero exit code, including the negative NTSTATUS codes a crash
+    rem returns, e.g. 0xC0000005, and works inside this parenthesized block
+    rem where %ERRORLEVEL% would expand too early.
+    set "testerFailed="
+    "%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%" > "%reportFolder%/combined_stdout.log" 2>&1 || set "testerFailed=1"
+    type "%reportFolder%/combined_stdout.log" >> "%logFile%"
+
+    rem Upload artifacts (best-effort; never aborts the run). Windows skips coredump for v1.
+    call :sendArtifact "binary" "%dstDir%\ccextractorwinfull.exe"
+    call :sendArtifact "combined_stdout" "%reportFolder%/combined_stdout.log"
+
+    if defined testerFailed call :haltAndCatchFire ""
 
     call :sendLogFile
 
@@ -144,3 +161,21 @@ if !sl_attempt! LEQ %sl_max_retries% (
 echo ERROR: Failed to upload log after %sl_max_retries% attempts >> "%logFile%"
 endlocal
 EXIT /B 1
+
+rem Send a build artifact to the server (best-effort; never aborts the run)
+:sendArtifact
+setlocal
+set "sa_type=%~1"
+set "sa_file=%~2"
+if NOT EXIST "%sa_file%" (
+    echo Artifact %sa_type%: no file at "%sa_file%", skipping >> "%logFile%"
+    endlocal
+    EXIT /B 0
+)
+echo Uploading %sa_type% artifact (%sa_file%) >> "%logFile%"
+rem --fail makes HTTP >= 400 (e.g. 413 on an oversized file) a curl error so
+rem the failure is at least visible in the log; the upload stays best-effort.
+curl -s --fail -A "%userAgent%" --form "type=artifactupload" --form "artifact_type=%sa_type%" --form "file=@%sa_file%" "%reportURL%" >> "%logFile%" 2>&1
+if errorlevel 1 echo Artifact %sa_type%: upload failed with curl exit code %ERRORLEVEL% >> "%logFile%"
+endlocal
+EXIT /B 0

mod_api/__init__.py

@@ -33,9 +33,11 @@
 # routing-level 404s/405s that never enter the blueprint).
 mod_api.after_app_request(error_handler.convert_api_errors_to_json)
 
-# Route modules register themselves against the blueprint; the rest of
-# the stack adds one module per PR.
+# Route modules
 from mod_api.routes import auth as auth_routes  # noqa: E402, F401
+from mod_api.routes import \
+    errors_logs as errors_logs_routes  # noqa: E402, F401
+from mod_api.routes import results as results_routes  # noqa: E402, F401
 from mod_api.routes import runs as runs_routes  # noqa: E402, F401
 from mod_api.routes import samples as samples_routes  # noqa: E402, F401
 from mod_api.routes import system as system_routes  # noqa: E402, F401

mod_api/routes/errors_logs.py

@@ -0,0 +1,195 @@
+"""
+Error and build log routes.
+
+GET /runs/{id}/errors                  Test-level errors for a run
+GET /runs/{id}/infrastructure-errors   Infra errors (VM, build, worker)
+GET /runs/{id}/error-summary           Grouped error counts
+GET /runs/{id}/logs                    Build log (cursor-paginated)
+GET /runs/{id}/samples/{sid}/logs      Per-sample logs (not yet available)
+"""
+
+from flask import g, request
+
+from mod_api import mod_api
+from mod_api.middleware.auth import require_scope
+from mod_api.middleware.error_handler import make_error_response
+from mod_api.middleware.validation import (validate_cursor_pagination,
+                                           validate_offset_pagination,
+                                           validate_path_id)
+from mod_api.models.api_token import Scope
+from mod_api.services.error_service import (derive_error_summary,
+                                            derive_errors_for_run,
+                                            derive_infrastructure_errors)
+from mod_api.services.log_service import read_log_lines
+from mod_api.utils import cursor_paginated_response, paginated_response
+from mod_auth.models import Role
+from mod_test.models import Test
+
+
+@mod_api.route('/runs/<run_id>/errors', methods=['GET'])
+@require_scope(Scope.RESULTS_READ)
+@validate_path_id('run_id')
+@validate_offset_pagination()
+def list_run_errors(run_id, limit=50, offset=0):
+    """List test errors for a run, derived from result and output data."""
+    test = Test.query.filter(Test.id == run_id).first()
+    if test is None:
+        return make_error_response('not_found', f'Run {run_id} not found.', http_status=404)
+
+    errors = derive_errors_for_run(run_id)
+
+    error_type = request.args.get('type')
+    if error_type:
+        errors = [e for e in errors if e['type'] == error_type]
+
+    severity = request.args.get('severity')
+    if severity:
+        errors = [e for e in errors if e['severity'] == severity]
+
+    sample_id = request.args.get('sample_id', type=int)
+    if sample_id:
+        errors = [e for e in errors if e.get('sample_id') == sample_id]
+
+    total = len(errors)
+    paged = errors[offset:offset + limit]
+
+    return paginated_response(paged, total, limit, offset)
+
+
+@mod_api.route('/runs/<run_id>/infrastructure-errors', methods=['GET'])
+@require_scope(Scope.SYSTEM_READ)
+@validate_path_id('run_id')
+@validate_offset_pagination()
+def list_infrastructure_errors(run_id, limit=50, offset=0):
+    """
+    Infra errors classified from TestProgress messages on a best-effort basis.
+
+    Stack traces are opt-in because they may contain internal paths.
+    """
+    test = Test.query.filter(Test.id == run_id).first()
+    if test is None:
+        return make_error_response('not_found', f'Run {run_id} not found.', http_status=404)
+
+    include_stack = request.args.get(
+        'include_stack', 'false').lower() == 'true'
+    if include_stack:
+        user = getattr(g, 'api_user', None)
+        allowed = (Role.admin, Role.contributor)
+        if user is None or user.role not in allowed:
+            return make_error_response(
+                'forbidden',
+                'Stack traces require admin or contributor role.',
+                details={'required_roles': [role.value for role in allowed]},
+                http_status=403,
+            )
+
+    errors = derive_infrastructure_errors(run_id)
+
+    if not include_stack:
+        for e in errors:
+            e.pop('stack', None)
+
+    # Apply optional type and severity filters.
+    error_type = request.args.get('type')
+    if error_type:
+        errors = [e for e in errors if e.get('type') == error_type]
+
+    severity = request.args.get('severity')
+    if severity:
+        errors = [e for e in errors if e.get('severity') == severity]
+
+    total = len(errors)
+    paged = errors[offset:offset + limit]
+    return paginated_response(paged, total, limit, offset)
+
+
+@mod_api.route('/runs/<run_id>/error-summary', methods=['GET'])
+@require_scope(Scope.RESULTS_READ)
+@validate_path_id('run_id')
+@validate_offset_pagination()
+def get_error_summary(run_id, limit=50, offset=0):
+    """Group error summary for triaging a run before drilling into details."""
+    test = Test.query.filter(Test.id == run_id).first()
+    if test is None:
+        return make_error_response('not_found', f'Run {run_id} not found.', http_status=404)
+
+    group_by = request.args.get('group_by', 'type')
+    if group_by not in ('type', 'severity', 'sample_id', 'regression_id'):
+        return make_error_response(
+            'validation_error',
+            'group_by must be one of: type, severity, sample_id, regression_id.',
+            http_status=400,
+        )
+
+    severity = request.args.get('severity')
+
+    summary = derive_error_summary(run_id, group_by=group_by)
+
+    if severity:
+        summary = [s for s in summary if s.get('severity') == severity]
+
+    total = len(summary)
+    paged = summary[offset:offset + limit]
+    return paginated_response(paged, total, limit, offset)
+
+
+@mod_api.route('/runs/<run_id>/logs', methods=['GET'])
+@require_scope(Scope.SYSTEM_READ)
+@validate_path_id('run_id')
+@validate_cursor_pagination(default_limit=100)
+def get_run_logs(run_id, limit=100, cursor=None):
+    """
+    Read a run's build log with cursor-based pagination.
+
+    Returns 404 (not a broken download link) when the file doesn't exist.
+    """
+    test = Test.query.filter(Test.id == run_id).first()
+    if test is None:
+        return make_error_response('not_found', f'Run {run_id} not found.', http_status=404)
+
+    level = request.args.get('level')
+    source = request.args.get('source')
+    contains = request.args.get('contains')
+    if contains and len(contains) > 100:
+        return make_error_response(
+            'validation_error',
+            'contains parameter must be 100 characters or less.',
+            http_status=400,
+        )
+
+    try:
+        lines, next_cursor = read_log_lines(
+            run_id,
+            cursor=cursor,
+            limit=limit,
+            level=level,
+            source=source,
+            contains=contains,
+        )
+    except FileNotFoundError:
+        return make_error_response(
+            'log_not_found',
+            f'Log file for run {run_id} is not available locally. '
+            'It may have been moved to cold storage. Please download it via the artifacts API.',
+            details={'run_id': run_id,
+                     'action_required': f'Use GET /runs/{run_id}/artifacts (type=build_log)'},
+            http_status=404,
+        )
+
+    return cursor_paginated_response(lines, next_cursor, limit)
+
+
+@mod_api.route('/runs/<run_id>/samples/<sample_id>/logs', methods=['GET'])
+@require_scope(Scope.SYSTEM_READ)
+@validate_path_id('run_id')
+@validate_path_id('sample_id')
+@validate_offset_pagination()
+def get_sample_logs(run_id, sample_id, limit=50, offset=0):
+    """Per-sample logs aren't available yet — the CI worker doesn't support them."""
+    return make_error_response(
+        'not_found',
+        f'Per-sample logs are not available for sample {sample_id} in run {run_id}.',
+        details={
+            'reason': 'Per-sample log storage is not yet supported by the CI worker.'},
+        http_status=404,
+    )