Merge test into main

.github/workflows/client-test.yml

@@ -0,0 +1,47 @@
+# Test datapipe-client on its own.
+#
+# Separate from node.js.yml because that workflow builds the Next.js app and
+# the Cloud Functions, and carries a page of NEXT_PUBLIC_* configuration that
+# this package has no use for. packages/client has no Firebase project, no
+# emulators, and no secrets -- it is a browser library with a mocked fetch.
+#
+# This exists so that a broken client cannot reach main and first fail at
+# release time, when npm's refusal to republish a version makes a bad publish
+# awkward to undo.
+
+name: Test datapipe-client
+
+on:
+  pull_request:
+    branches: ["main", "test"]
+    paths:
+      - "packages/client/**"
+      - ".github/workflows/client-test.yml"
+  push:
+    branches: ["test"]
+    paths:
+      - "packages/client/**"
+      - ".github/workflows/client-test.yml"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: packages/client
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Use Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 22.x
+      - run: npm ci
+      - run: npm run typecheck
+      - run: npm test
+      - run: npm run build
+      # The browser bundle carries the Firebase SDK, so its size is the whole
+      # reason streaming lives here rather than in the jsPsych plugin. Print it
+      # on every run, so a jump is visible in the log rather than discovered by
+      # a researcher on a slow connection.
+      - run: npm run size

.github/workflows/firebase-deploy-test.yml

@@ -8,6 +8,20 @@ on:
     branches: ["test"]
   workflow_dispatch:
 
+# Deploys to one Firebase project must not overlap. Two merges landing within
+# a few minutes of each other used to start two deploys at once; the second
+# one's update of the SSR function was rejected with a 409 ("unable to queue
+# the operation") because the first was still mid-flight, and the site kept
+# serving the previous build. Note that firebase deploy reports that failure
+# as a warning and still exits 0, so the collision does not show up as a red
+# check -- keeping deploys serialized is what prevents it, not the exit code.
+# cancel-in-progress stays false on purpose --
+# a deploy that is already updating functions should be allowed to finish,
+# not killed halfway through. Later runs queue behind it instead.
+concurrency:
+  group: deploy-test
+  cancel-in-progress: false
+
 env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
   #FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
@@ -16,10 +30,23 @@ env:
   NEXT_PUBLIC_REDIRECT_URI: "https://datapipe-test.web.app/oauth2/callback"
   NEXT_PUBLIC_OAUTH_CALLBACK: "https://datapipe-test.web.app/api/oauth2callback"
   NEXT_PUBLIC_OAUTH_FINAL: "/admin"
-  NEXT_PUBLIC_OAUTH_REGENERATE: "https://datapipe-test.web.app/api/oauth2regenerate"
   NEXT_PUBLIC_GENERATE_STATE: "https://datapipe-test.web.app/api/generateoauthstate"
   NEXT_PUBLIC_BASE_URL: "https://datapipe-test.web.app"
   NEXT_PUBLIC_OSF_ENV: ""
+  # Frontend-only: drives the TestEnvironmentWarning banner gate (see
+  # components/TestEnvironmentWarning.js). Distinct from NEXT_PUBLIC_OSF_ENV
+  # above, which is being retired as OSF is deprecated and was left "" here
+  # -- that meant the banner stopped rendering on this site once OSF_ENV
+  # went empty. Not echoed into functions/.env; functions has no use for it.
+  NEXT_PUBLIC_DEPLOY_ENV: "test"
+  # sandbox.zenodo.org. Without this the TEST site creates real
+  # depositions on the live Zenodo using the researcher's real account.
+  NEXT_PUBLIC_ZENODO_ENV: "sandbox."
+  # Google Picker (folder selection for gdrive experiments). The API key is
+  # restricted to the Picker API + our domains, so it is safe in the browser
+  # bundle -- not a secret. The project number is public.
+  NEXT_PUBLIC_GOOGLE_PICKER_API_KEY: "AIzaSyDLg6uprrY5BPjY4ClVZGSvy7sd_ug0t9M"
+  NEXT_PUBLIC_GDRIVE_PROJECT_NUMBER: "699904257039"
 
 jobs:
   deploy:
@@ -41,16 +68,13 @@ jobs:
         with:
           credentials_json: '${{ secrets.GOOGLE_TEST_CREDENTIALS }}' # Replace with the name of your GitHub Actions secret
       - name: Install firebase tools
-        run: npm install -g firebase-tools@15.8.0
+        run: npm install -g firebase-tools@15.28.1
       - name: Enable firebase webframeworks
         run: firebase experiments:enable webframeworks
       - name: Install dependencies 
         run: npm ci
-      - name: Install dependencies and build metadata
-        working-directory: functions/metadata
-        run: |
-          npm ci
-          npm run build
+      # functions/metadata is a pre-built vendored dist (see functions/metadata/README.md);
+      # it is installed as a file: dependency by the functions npm ci below.
       - name: Create functions environment file
         working-directory: functions
         run: |
@@ -59,12 +83,46 @@ jobs:
           echo "REDIRECT_URI=https://datapipe-test.web.app/oauth2/callback" >> .env
           echo "TOKEN_ENCRYPTION_KEY=${{ secrets.FIRESTORE_KEY_TEST }}" >> .env
           echo "NEXT_PUBLIC_OSF_ENV=" >> .env
+          # Google Drive client secret for the deployed test site. Client id and
+          # redirect uri are non-secret and live in functions/.env.datapipe-test.
+          echo "GDRIVE_CLIENT_SECRET=${{ secrets.TEST_GDRIVE_CLIENT_SECRET }}" >> .env
+          # Zenodo OAuth client secret for the deployed test site. Client id,
+          # redirect uri and ZENODO_ENV are non-secret and live in
+          # functions/.env.datapipe-test.
+          echo "ZENODO_CLIENT_SECRET=${{ secrets.TEST_ZENODO_CLIENT_SECRET }}" >> .env
+          # Resend for the deployed test site (functions/src/mail-delivery.ts).
+          # SET this one. The test site is the only place mail DELIVERY is
+          # exercised before production -- the emulator short-circuits before
+          # sending and the unit suites mock the transport -- so it is where a
+          # mail change is proved to pass DKIM/SPF and reach an inbox. Use a
+          # SEPARATE sending-only key from prod, on the same Resend account
+          # (the From domain is the same, so it has to be the same account).
+          # Unset is still safe rather than silent: the onmailcreated trigger
+          # records a terminal MailConfigMissingError on each mail document and
+          # sends nothing. Reputation and daily quota are shared with prod --
+          # see docs/deploy-contact-email.md §2(d).
+          echo "RESEND_API_KEY=${{ secrets.TEST_RESEND_API_KEY }}" >> .env
+          # Not secret, so literals here, the same way REDIRECT_URI above is.
+          # Same sender as production: Resend verifies the DOMAIN
+          # (jspsych.org), not the deployment, and datapipe-test.web.app is not
+          # a verified domain -- sending from it is a 403 validation_error,
+          # which is why contact-email codes never arrived on the test site.
+          # Only the display name differs, so test mail is recognisable in an
+          # inbox.
+          echo "MAIL_FROM=DataPipe (test) <datapipe-notifications@jspsych.org>" >> .env
+          echo "MAIL_REPLY_TO=datapipe@jspsych.org" >> .env
       - name: Install dependencies and build functions
         working-directory: functions
         run: |
           npm ci
           npm run build
       - name: Use test firebase project
         run: firebase use test
+      # storage deploys storage.rules to the project's default bucket
+      # (datapipe-test.appspot.com, which already exists). It is listed
+      # before functions because firebase-tools deploys --only targets in
+      # the order given, and the deny-all rules should be in place before
+      # functions keep writing participant data under pending-data/,
+      # upload-queue/ and finalization/ in that bucket.
       - name: Deploy firebase project
-        run: firebase deploy --only firestore,functions,hosting --force
+        run: firebase deploy --only firestore,database,storage,functions,hosting --force

.github/workflows/firebase-deploy.yml

@@ -8,6 +8,14 @@ on:
     branches: ["main"]
   workflow_dispatch:
 
+# See the matching block in firebase-deploy-test.yml: concurrent deploys to the
+# same Firebase project collide on the SSR function update and leave the site
+# on the old build. Separate group from the test deploy -- they target
+# different projects and have no reason to queue behind each other.
+concurrency:
+  group: deploy-production
+  cancel-in-progress: false
+
 env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
   #FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
@@ -16,10 +24,19 @@ env:
   NEXT_PUBLIC_REDIRECT_URI: "https://pipe.jspsych.org/oauth2/callback"
   NEXT_PUBLIC_OAUTH_CALLBACK: "https://pipe.jspsych.org/api/oauth2callback"
   NEXT_PUBLIC_OAUTH_FINAL: "/admin"
-  NEXT_PUBLIC_OAUTH_REGENERATE: "https://pipe.jspsych.org/api/oauth2regenerate"
   NEXT_PUBLIC_GENERATE_STATE: "https://pipe.jspsych.org/api/generateoauthstate"
   NEXT_PUBLIC_BASE_URL: "https://pipe.jspsych.org"
   NEXT_PUBLIC_OSF_ENV: ""
+  # Empty = the real zenodo.org. Set explicitly rather than relying on the
+  # default, so production never inherits a sandbox value by accident.
+  NEXT_PUBLIC_ZENODO_ENV: ""
+  # Google Picker (folder selection for gdrive experiments). The key is
+  # restricted to the Picker API and to the https://pipe.jspsych.org/* referrer
+  # (API key "DataPipe Picker – production" in the osf-relay GCP project), so
+  # it is safe in the browser bundle -- not a secret. The project number is
+  # osf-relay's; the Picker's setAppId wants the number, not the id.
+  NEXT_PUBLIC_GOOGLE_PICKER_API_KEY: "AIzaSyDNGwbzk4qwjhq_jJZVIg8ajxXtqi2A3wQ"
+  NEXT_PUBLIC_GDRIVE_PROJECT_NUMBER: "54492763028"
 
 jobs:
   build:
@@ -41,16 +58,13 @@ jobs:
         with:
           credentials_json: '${{ secrets.GOOGLE_PRODUCTION_CREDENTIALS }}'
       - name: Install firebase tools
-        run: npm install -g firebase-tools@15.8.0
+        run: npm install -g firebase-tools@15.28.1
       - name: Enable firebase webframeworks
         run: firebase experiments:enable webframeworks
       - name: Install dependencies 
         run: npm ci
-      - name: Install dependencies and build metadata
-        working-directory: functions/metadata
-        run: |
-          npm ci
-          npm run build
+      # functions/metadata is a pre-built vendored dist (see functions/metadata/README.md);
+      # it is installed as a file: dependency by the functions npm ci below.
       - name: Create functions environment file
         working-directory: functions
         run: |
@@ -59,12 +73,36 @@ jobs:
           echo "REDIRECT_URI=https://pipe.jspsych.org/oauth2/callback" >> .env
           echo "TOKEN_ENCRYPTION_KEY=${{ secrets.FIRESTORE_KEY_PRODUCTION }}" >> .env
           echo "NEXT_PUBLIC_OSF_ENV=" >> .env
+          # Google Drive client secret for production. Client id and redirect
+          # uri are non-secret and live in functions/.env.osf-relay.
+          echo "GDRIVE_CLIENT_SECRET=${{ secrets.PROD_GDRIVE_CLIENT_SECRET }}" >> .env
+          # Zenodo OAuth client secret for production. Client id and redirect
+          # uri are non-secret and live in functions/.env.osf-relay; ZENODO_ENV
+          # stays unset there so the flow runs against the real zenodo.org.
+          echo "ZENODO_CLIENT_SECRET=${{ secrets.PROD_ZENODO_CLIENT_SECRET }}" >> .env
+          # Resend. functions/src/mail-delivery.ts delivers everything mail.ts
+          # queues into the `mail` collection; without this key the
+          # onmailcreated trigger writes a terminal MailConfigMissingError on
+          # every notification and sends nothing. Scope the key to Sending
+          # access only -- see docs/deploy-contact-email.md §2, which also
+          # covers domain verification.
+          echo "RESEND_API_KEY=${{ secrets.PROD_RESEND_API_KEY }}" >> .env
+          # Not secret, so literals here, the same way REDIRECT_URI above is.
+          # MAIL_FROM must be an address on a domain verified in Resend.
+          echo "MAIL_FROM=DataPipe <datapipe-notifications@jspsych.org>" >> .env
+          echo "MAIL_REPLY_TO=datapipe@jspsych.org" >> .env
       - name: Install dependencies and build functions
         working-directory: functions
         run: |
           npm ci
           npm run build
       - name: Use default firebase project
         run: firebase use default
+      # storage deploys storage.rules to the project's default bucket
+      # (osf-relay.appspot.com, which already exists). It is listed before
+      # functions because firebase-tools deploys --only targets in the order
+      # given, and the deny-all rules should be in place before functions
+      # keep writing participant data under pending-data/, upload-queue/ and
+      # finalization/ in that bucket.
       - name: Deploy firebase project
-        run: firebase deploy --only firestore,functions,hosting --force
+        run: firebase deploy --only firestore,database,storage,functions,hosting --force

.github/workflows/metadata-drift-check.yml

@@ -0,0 +1,74 @@
+# Nudge (never an auto-merge): flags when upstream @jspsych/metadata main has moved past
+# the commit DataPipe currently vendors. The vendored copy is pinned + built by
+# functions/scripts/sync-metadata.mjs; this job just tells a human it's time to re-sync.
+name: Metadata vendor drift check
+
+on:
+  schedule:
+    - cron: "0 12 * * 1" # Mondays 12:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  drift-check:
+    runs-on: ubuntu-latest
+    env:
+      UPSTREAM: jspsych/metadata
+      ISSUE_TITLE: "[metadata-sync] Vendored @jspsych/metadata is behind upstream main"
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Compare pinned commit against upstream main
+        id: compare
+        run: |
+          PINNED=$(jq -r .commit functions/metadata/VENDORED_FROM.json)
+          echo "Pinned commit: $PINNED"
+          if [ -z "$PINNED" ] || [ "$PINNED" = "null" ]; then
+            echo "::error::Could not read pinned commit from functions/metadata/VENDORED_FROM.json"
+            exit 1
+          fi
+          UPSTREAM_MAIN=$(gh api "repos/$UPSTREAM/commits/main" --jq .sha)
+          echo "Upstream main:  $UPSTREAM_MAIN"
+          if [ "$PINNED" = "$UPSTREAM_MAIN" ]; then
+            echo "in_sync=true" >> "$GITHUB_OUTPUT"
+            echo "Vendored copy is up to date with upstream main."
+            exit 0
+          fi
+          # ahead_by = how many commits upstream main is ahead of our pinned commit.
+          AHEAD=$(gh api "repos/$UPSTREAM/compare/$PINNED...main" --jq .ahead_by)
+          echo "in_sync=false" >> "$GITHUB_OUTPUT"
+          echo "ahead=$AHEAD" >> "$GITHUB_OUTPUT"
+          echo "pinned=$PINNED" >> "$GITHUB_OUTPUT"
+          echo "upstream=$UPSTREAM_MAIN" >> "$GITHUB_OUTPUT"
+
+      - name: Open or update tracking issue
+        if: steps.compare.outputs.in_sync == 'false'
+        run: |
+          BODY=$(cat <<EOF
+          Upstream \`$UPSTREAM\` \`main\` is **${{ steps.compare.outputs.ahead }} commit(s) ahead** of the version DataPipe vendors.
+
+          - Pinned:   \`${{ steps.compare.outputs.pinned }}\`
+          - Upstream: \`${{ steps.compare.outputs.upstream }}\`
+          - Diff: https://github.com/$UPSTREAM/compare/${{ steps.compare.outputs.pinned }}...main
+
+          To re-sync (nothing is automatic):
+          \`\`\`
+          cd functions && npm run sync:metadata
+          \`\`\`
+          Then review the diff, run the tests, and open a PR. See \`functions/metadata/README.md\`.
+
+          _This issue is updated automatically by \`.github/workflows/metadata-drift-check.yml\`._
+          EOF
+          )
+          EXISTING=$(gh issue list --state open --search "$ISSUE_TITLE in:title" --json number --jq '.[0].number')
+          if [ -n "$EXISTING" ]; then
+            echo "Updating existing issue #$EXISTING"
+            gh issue edit "$EXISTING" --body "$BODY"
+          else
+            echo "Creating new tracking issue"
+            gh issue create --title "$ISSUE_TITLE" --body "$BODY"
+          fi

.github/workflows/node.js.yml

@@ -17,7 +17,6 @@ env:
   NEXT_PUBLIC_REDIRECT_URI: "https://datapipe-test.web.app/oauth2/callback"
   NEXT_PUBLIC_OAUTH_CALLBACK: "https://datapipe-test.web.app/api/oauth2callback"
   NEXT_PUBLIC_OAUTH_FINAL: "/admin"
-  NEXT_PUBLIC_OAUTH_REGENERATE: "https://datapipe-test.web.app/api/oauth2regenerate"
   NEXT_PUBLIC_OSF_ENV: ""
 
 jobs:
@@ -45,15 +44,14 @@ jobs:
         with:
           credentials_json: '${{ secrets.GOOGLE_TEST_CREDENTIALS }}' # Replace with the name of your GitHub Actions secret
       - name: Install firebase tools
-        run: npm install -g firebase-tools@15.8.0
+        run: npm install -g firebase-tools@15.28.1
       - name: Enable firebase webframeworks
         run: firebase experiments:enable webframeworks
       - run: npm ci
-      - name: Install dependencies and build metadata
-        working-directory: functions/metadata
-        run: |
-          npm ci
-          npm run build
+      - name: Lint
+        run: npm run lint
+      # functions/metadata is a pre-built vendored dist (see functions/metadata/README.md);
+      # it is installed as a file: dependency by the functions npm ci below.
       - name: Install dependencies and build functions
         working-directory: functions
         run: |
@@ -62,4 +60,7 @@ jobs:
       - name: Select project
         run: firebase use test
       - name: Launch firestore emulator and test
-        run: firebase emulators:exec 'npm run test-ci'
+        # maxWorkers=2: the emulator-backed suites contend under full
+        # parallel load (pre-existing data-emulator timing flake); capping
+        # workers reproduces consistently-green runs.
+        run: firebase emulators:exec 'npm run test-ci -- --maxWorkers=2'