Add hosted consumer UAT fixture

.github/workflows/model-eol-consumer-live-uat.yml

@@ -0,0 +1,129 @@
+name: model-eol consumer live UAT
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: model-eol-consumer-live-uat
+  cancel-in-progress: false
+
+env:
+  MODEL_EOL_PACKAGE: model-eol@0.4.0
+  MODEL_EOL_CONFIG: model-eol-uat.json
+
+jobs:
+  plan:
+    name: build trusted plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Build the versioned plan
+        run: |
+          mkdir -p "$RUNNER_TEMP/model-eol-plan"
+          MODEL_EOL_VERSION="$(npm view "$MODEL_EOL_PACKAGE" version --json | node -e 'let s="";process.stdin.on("data",c=>s+=c).on("end",()=>{const v=JSON.parse(s);const r=Array.isArray(v)?v.at(-1):v;if(typeof r!=="string")process.exit(1);process.stdout.write(r)})')"
+          if [[ ! "$MODEL_EOL_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "refusing invalid resolved model-eol version: $MODEL_EOL_VERSION" >&2
+            exit 1
+          fi
+          printf '%s\n' "$MODEL_EOL_VERSION" > "$RUNNER_TEMP/model-eol-plan/model-eol-version"
+          npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol plan . \
+            --config "$MODEL_EOL_CONFIG" \
+            > "$RUNNER_TEMP/model-eol-plan/model-eol-plan.json"
+      - name: Upload plan
+        uses: actions/upload-artifact@v4
+        with:
+          name: model-eol-plan
+          path: |
+            ${{ runner.temp }}/model-eol-plan/model-eol-plan.json
+            ${{ runner.temp }}/model-eol-plan/model-eol-version
+          if-no-files-found: error
+
+  evaluate:
+    name: evaluate isolated migration
+    needs: plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Download trusted plan
+        uses: actions/download-artifact@v4
+        with:
+          name: model-eol-plan
+          path: ${{ runner.temp }}/model-eol-plan
+      - name: Apply and evaluate the migration
+        run: |
+          mkdir -p "$RUNNER_TEMP/model-eol-eval"
+          MODEL_EOL_VERSION="$(cat "$RUNNER_TEMP/model-eol-plan/model-eol-version")"
+          if [[ ! "$MODEL_EOL_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "refusing invalid resolved model-eol version: $MODEL_EOL_VERSION" >&2
+            exit 1
+          fi
+          npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol-bot evaluate \
+            --target-dir . \
+            --config "$MODEL_EOL_CONFIG" \
+            --plan-file "$RUNNER_TEMP/model-eol-plan/model-eol-plan.json" \
+            --output-file "$RUNNER_TEMP/model-eol-eval/model-eol-eval-results.json"
+      - name: Upload bounded eval results
+        uses: actions/upload-artifact@v4
+        with:
+          name: model-eol-eval-results
+          path: ${{ runner.temp }}/model-eol-eval/model-eol-eval-results.json
+          if-no-files-found: error
+
+  publish:
+    name: reconcile migration work
+    needs: [plan, evaluate]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Download trusted plan
+        uses: actions/download-artifact@v4
+        with:
+          name: model-eol-plan
+          path: ${{ runner.temp }}/model-eol-plan
+      - name: Download eval results
+        uses: actions/download-artifact@v4
+        with:
+          name: model-eol-eval-results
+          path: ${{ runner.temp }}/model-eol-eval
+      - name: Publish and reconcile model-eol work
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MODEL_EOL_PLAN_FILE: ${{ runner.temp }}/model-eol-plan/model-eol-plan.json
+          MODEL_EOL_EVAL_RESULTS_FILE: ${{ runner.temp }}/model-eol-eval/model-eol-eval-results.json
+          MODEL_EOL_TOKEN_KIND: github-token
+        run: |
+          MODEL_EOL_VERSION="$(cat "$RUNNER_TEMP/model-eol-plan/model-eol-version")"
+          if [[ ! "$MODEL_EOL_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "refusing invalid resolved model-eol version: $MODEL_EOL_VERSION" >&2
+            exit 1
+          fi
+          npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol-bot \
+            --repo "${{ github.repository }}" \
+            --target-dir . \
+            --config "$MODEL_EOL_CONFIG"

model-eol-uat.json

@@ -0,0 +1,31 @@
+{
+  "days": 90,
+  "scope": "direct",
+  "ignore": {
+    "paths": [
+      ".claude/**",
+      ".github/**",
+      "bot/**",
+      "docs/**",
+      "examples/**",
+      "feeds/**",
+      "lib/**",
+      "refresh/**",
+      "schema/**",
+      "scripts/**",
+      "test/**",
+      "*.example",
+      "*.json",
+      "*.md",
+      "*.mjs",
+      "*.yaml",
+      "*.yml",
+      "LICENSE"
+    ]
+  },
+  "eval": {
+    "command": "node uat/live-consumer/eval.mjs",
+    "timeout_ms": 30000,
+    "pass_env": []
+  }
+}

uat/live-consumer/client.mjs

@@ -0,0 +1,4 @@
+export const anthropicClient = {
+  apiKey: process.env.ANTHROPIC_API_KEY,
+  model: 'claude-3-5-sonnet-20241022',
+}

uat/live-consumer/eval.mjs

@@ -0,0 +1,32 @@
+import fs from 'node:fs'
+
+const expectedOldModel = ['claude', '3', '5', 'sonnet', '20241022'].join('-')
+const expectedNewModel = ['claude', 'sonnet', '4', '6'].join('-')
+const expectedFile = 'uat/live-consumer/client.mjs'
+
+if (process.env.MODEL_EOL_OLD_ID !== expectedOldModel || process.env.MODEL_EOL_NEW_ID !== expectedNewModel) {
+  throw new Error('migration identity environment did not match the selected model')
+}
+
+if (!process.env.MODEL_EOL_PLAN || !process.env.MODEL_EOL_REPORT) {
+  throw new Error('migration plan and report paths are required')
+}
+
+const selectedPlan = JSON.parse(fs.readFileSync(process.env.MODEL_EOL_PLAN, 'utf8'))
+if (
+  selectedPlan.items?.length !== 1 ||
+  selectedPlan.items[0].file !== expectedFile ||
+  selectedPlan.items[0].replacement !== expectedNewModel
+) {
+  throw new Error('expected one isolated migration item')
+}
+
+const clientSource = fs.readFileSync(new URL('./client.mjs', import.meta.url), 'utf8')
+if (clientSource.includes(expectedOldModel) || !clientSource.includes(expectedNewModel)) {
+  throw new Error('isolated checkout does not contain the expected migration')
+}
+
+fs.writeFileSync(
+  process.env.MODEL_EOL_REPORT,
+  `Verified isolated migration from ${expectedOldModel} to ${expectedNewModel}.\n`,
+)