diff --git a/.github/workflows/model-eol-consumer-live-uat.yml b/.github/workflows/model-eol-consumer-live-uat.yml new file mode 100644 index 0000000..dcad222 --- /dev/null +++ b/.github/workflows/model-eol-consumer-live-uat.yml @@ -0,0 +1,129 @@ +name: model-eol consumer live UAT + +on: + workflow_dispatch: + +concurrency: + group: model-eol-consumer-live-uat + cancel-in-progress: false + +env: + MODEL_EOL_PACKAGE: model-eol@0.4.0 + MODEL_EOL_CONFIG: model-eol-uat.json + +jobs: + plan: + name: build trusted plan + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Build the versioned plan + run: | + mkdir -p "$RUNNER_TEMP/model-eol-plan" + MODEL_EOL_VERSION="$(npm view "$MODEL_EOL_PACKAGE" version --json | node -e 'let s="";process.stdin.on("data",c=>s+=c).on("end",()=>{const v=JSON.parse(s);const r=Array.isArray(v)?v.at(-1):v;if(typeof r!=="string")process.exit(1);process.stdout.write(r)})')" + if [[ ! "$MODEL_EOL_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "refusing invalid resolved model-eol version: $MODEL_EOL_VERSION" >&2 + exit 1 + fi + printf '%s\n' "$MODEL_EOL_VERSION" > "$RUNNER_TEMP/model-eol-plan/model-eol-version" + npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol plan . \ + --config "$MODEL_EOL_CONFIG" \ + > "$RUNNER_TEMP/model-eol-plan/model-eol-plan.json" + - name: Upload plan + uses: actions/upload-artifact@v4 + with: + name: model-eol-plan + path: | + ${{ runner.temp }}/model-eol-plan/model-eol-plan.json + ${{ runner.temp }}/model-eol-plan/model-eol-version + if-no-files-found: error + + evaluate: + name: evaluate isolated migration + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Download trusted plan + uses: actions/download-artifact@v4 + with: + name: model-eol-plan + path: ${{ runner.temp }}/model-eol-plan + - name: Apply and evaluate the migration + run: | + mkdir -p "$RUNNER_TEMP/model-eol-eval" + MODEL_EOL_VERSION="$(cat "$RUNNER_TEMP/model-eol-plan/model-eol-version")" + if [[ ! "$MODEL_EOL_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "refusing invalid resolved model-eol version: $MODEL_EOL_VERSION" >&2 + exit 1 + fi + npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol-bot evaluate \ + --target-dir . \ + --config "$MODEL_EOL_CONFIG" \ + --plan-file "$RUNNER_TEMP/model-eol-plan/model-eol-plan.json" \ + --output-file "$RUNNER_TEMP/model-eol-eval/model-eol-eval-results.json" + - name: Upload bounded eval results + uses: actions/upload-artifact@v4 + with: + name: model-eol-eval-results + path: ${{ runner.temp }}/model-eol-eval/model-eol-eval-results.json + if-no-files-found: error + + publish: + name: reconcile migration work + needs: [plan, evaluate] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + issues: write + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Download trusted plan + uses: actions/download-artifact@v4 + with: + name: model-eol-plan + path: ${{ runner.temp }}/model-eol-plan + - name: Download eval results + uses: actions/download-artifact@v4 + with: + name: model-eol-eval-results + path: ${{ runner.temp }}/model-eol-eval + - name: Publish and reconcile model-eol work + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MODEL_EOL_PLAN_FILE: ${{ runner.temp }}/model-eol-plan/model-eol-plan.json + MODEL_EOL_EVAL_RESULTS_FILE: ${{ runner.temp }}/model-eol-eval/model-eol-eval-results.json + MODEL_EOL_TOKEN_KIND: github-token + run: | + MODEL_EOL_VERSION="$(cat "$RUNNER_TEMP/model-eol-plan/model-eol-version")" + if [[ ! "$MODEL_EOL_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "refusing invalid resolved model-eol version: $MODEL_EOL_VERSION" >&2 + exit 1 + fi + npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol-bot \ + --repo "${{ github.repository }}" \ + --target-dir . \ + --config "$MODEL_EOL_CONFIG" diff --git a/model-eol-uat.json b/model-eol-uat.json new file mode 100644 index 0000000..898d345 --- /dev/null +++ b/model-eol-uat.json @@ -0,0 +1,31 @@ +{ + "days": 90, + "scope": "direct", + "ignore": { + "paths": [ + ".claude/**", + ".github/**", + "bot/**", + "docs/**", + "examples/**", + "feeds/**", + "lib/**", + "refresh/**", + "schema/**", + "scripts/**", + "test/**", + "*.example", + "*.json", + "*.md", + "*.mjs", + "*.yaml", + "*.yml", + "LICENSE" + ] + }, + "eval": { + "command": "node uat/live-consumer/eval.mjs", + "timeout_ms": 30000, + "pass_env": [] + } +} diff --git a/uat/live-consumer/client.mjs b/uat/live-consumer/client.mjs new file mode 100644 index 0000000..473a3d1 --- /dev/null +++ b/uat/live-consumer/client.mjs @@ -0,0 +1,4 @@ +export const anthropicClient = { + apiKey: process.env.ANTHROPIC_API_KEY, + model: 'claude-3-5-sonnet-20241022', +} diff --git a/uat/live-consumer/eval.mjs b/uat/live-consumer/eval.mjs new file mode 100644 index 0000000..0652a60 --- /dev/null +++ b/uat/live-consumer/eval.mjs @@ -0,0 +1,32 @@ +import fs from 'node:fs' + +const expectedOldModel = ['claude', '3', '5', 'sonnet', '20241022'].join('-') +const expectedNewModel = ['claude', 'sonnet', '4', '6'].join('-') +const expectedFile = 'uat/live-consumer/client.mjs' + +if (process.env.MODEL_EOL_OLD_ID !== expectedOldModel || process.env.MODEL_EOL_NEW_ID !== expectedNewModel) { + throw new Error('migration identity environment did not match the selected model') +} + +if (!process.env.MODEL_EOL_PLAN || !process.env.MODEL_EOL_REPORT) { + throw new Error('migration plan and report paths are required') +} + +const selectedPlan = JSON.parse(fs.readFileSync(process.env.MODEL_EOL_PLAN, 'utf8')) +if ( + selectedPlan.items?.length !== 1 || + selectedPlan.items[0].file !== expectedFile || + selectedPlan.items[0].replacement !== expectedNewModel +) { + throw new Error('expected one isolated migration item') +} + +const clientSource = fs.readFileSync(new URL('./client.mjs', import.meta.url), 'utf8') +if (clientSource.includes(expectedOldModel) || !clientSource.includes(expectedNewModel)) { + throw new Error('isolated checkout does not contain the expected migration') +} + +fs.writeFileSync( + process.env.MODEL_EOL_REPORT, + `Verified isolated migration from ${expectedOldModel} to ${expectedNewModel}.\n`, +)