Antalya 26.3 - Enhanced Flaky backport wf

.github/workflows/flaky_fix_sync.yml

@@ -4,15 +4,15 @@ on:
   workflow_dispatch:
     inputs:
       since_days:
-        description: 'How many days back to scan upstream master (default: 7)'
+        description: 'How many days back to scan upstream (default: 7)'
         required: false
         type: string
         default: '7'
       pattern:
         description: 'Commit subject pattern to match (case-insensitive)'
         required: false
         type: string
-        default: 'fix flaky test'
+        default: 'fix flaky'
       upstream_ref:
         description: 'Upstream branch/ref to scan'
         required: false
@@ -25,18 +25,39 @@ on:
         default: false
 
   schedule:
-    - cron: '0 12 * * 1'  # Every Monday at 08:00 EDT (12:00 UTC)
+    - cron: '0 10 * * 1'  # Every Monday at 06:00 EDT (10:00 UTC)
+
 
 jobs:
+  setup:
+    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
+    outputs:
+      branches: ${{ steps.set_branches.outputs.branches }}
+    steps:
+      - id: set_branches
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "branches=[\"${{ github.ref_name }}\"]" >> $GITHUB_OUTPUT
+          else
+            echo "branches=[\"antalya-26.3\", \"stable-26.3\", \"antalya-26.6\"]" >> $GITHUB_OUTPUT
+          fi
+
   check:
+    needs: setup
     runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
+    strategy:
+      matrix:
+        branch: ${{ fromJSON(needs.setup.outputs.branches) }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
           filter: blob:none
 
+      - name: Checkout target branch
+        run: git checkout ${{ matrix.branch }}
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -47,27 +68,30 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           python3 tests/ci/check_for_flaky_fixes.py \
-            --since-days "${{ inputs.since_days || '7' }}" \
-            --pattern "${{ inputs.pattern || 'fix flaky test' }}" \
-            --upstream-ref "${{ inputs.upstream_ref || 'master' }}" \
-            --md-output flaky_fix_report.md \
-            --json-output flaky_fix.json
+            --since-days "${{ github.event.inputs.since_days || '7' }}" \
+            --pattern "${{ github.event.inputs.pattern || 'fix flaky' }}" \
+            --upstream-ref "${{ github.event.inputs.upstream_ref || 'master' }}" \
+            --md-output flaky_fix_report_${{ matrix.branch }}.md \
+            --json-output flaky_fix_${{ matrix.branch }}.json
 
       - name: Write summary
         if: always()
-        run: cat flaky_fix_report.md >> "$GITHUB_STEP_SUMMARY"
+        run: cat flaky_fix_report_${{ matrix.branch }}.md >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload check results
         uses: actions/upload-artifact@v4
         with:
-          name: flaky-fix-check
+          name: flaky-fix-check-${{ matrix.branch }}
           path: |
-            flaky_fix_report.md
-            flaky_fix.json
+            flaky_fix_report_${{ matrix.branch }}.md
+            flaky_fix_${{ matrix.branch }}.json
 
   backport:
-    needs: check
-    if: ${{ github.event_name == 'schedule' || inputs.create_pr == true }}
+    needs: [check, setup]
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.create_pr == true }}
+    strategy:
+      matrix:
+        branch: ${{ fromJSON(needs.setup.outputs.branches) }}
     runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
     steps:
       - name: Check out repository
@@ -76,6 +100,9 @@ jobs:
           fetch-depth: 1
           filter: blob:none
 
+      - name: Checkout target branch
+        run: git checkout ${{ matrix.branch }}
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -84,7 +111,7 @@ jobs:
       - name: Download check results
         uses: actions/download-artifact@v4
         with:
-          name: flaky-fix-check
+          name: flaky-fix-check-${{ matrix.branch }}
 
       - name: Configure git identity
         run: |
@@ -102,5 +129,5 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           python3 tests/ci/backport_flaky_fixes.py \
-            --json-file flaky_fix.json \
+            --json-file flaky_fix_${{ matrix.branch }}.json \
             --repo "${{ github.repository }}"

tests/ci/backport_flaky_fixes.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python3
 """
-Read the JSON output from check_for_flaky_fix_backports.py, cherry-pick each
-missing commit onto a new branch, push it, and open a single PR.
+Read the JSON output from check_for_flaky_fixes.py and cherry-pick each missing
+commit onto a backport branch. Opens a new PR, or amends the open one from a
+previous run if it already exists for this base branch.
 """
 
 import argparse
@@ -86,6 +87,17 @@ def cherry_pick(sha: str) -> bool:
     return False
 
 
+def _commit_line(upstream_repo: str, sha, subject: str, date) -> str:
+    """Render one bullet. sha/date may be None for commits carried over from a
+    reused PR, where only the subject is known."""
+    link = ""
+    if sha:
+        url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha)
+        link = f"[`{sha[:12]}`]({url}) "
+    suffix = f"  _(committed {date})_" if date else ""
+    return f"- {link}{subject}{suffix}"
+
+
 def build_pr_body(
     upstream_repo: str,
     applied: list,
@@ -101,21 +113,89 @@ def build_pr_body(
         lines.append("### Applied")
         lines.append("")
         for sha, subject, date in applied:
-            url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha)
-            lines.append(f"- [`{sha[:12]}`]({url}) {subject}  _(committed {date})_")
+            lines.append(_commit_line(upstream_repo, sha, subject, date))
         lines.append("")
 
     if conflicted:
         lines.append("### Skipped (cherry-pick conflict — manual backport needed)")
         lines.append("")
         for sha, subject, date in conflicted:
-            url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha)
-            lines.append(f"- [`{sha[:12]}`]({url}) {subject}  _(committed {date})_")
+            lines.append(_commit_line(upstream_repo, sha, subject, date))
         lines.append("")
 
     return "\n".join(lines)
 
 
+def find_open_backport_pr(repo: str, base_branch: str):
+    """Return (number, head_branch) of an open flaky-fix backport PR for this base
+    branch, or (None, None). Lets a weekly run amend last week's PR instead of
+    opening a fresh one each time. If several match, reuses the newest."""
+    prefix = f"flaky-fix-backport/{base_branch}/"
+    result = subprocess.run(
+        ["gh", "pr", "list", "--repo", repo, "--state", "open", "--base", base_branch,
+         "--json", "number,headRefName,createdAt", "--limit", "100"],
+        text=True, capture_output=True,
+    )
+    if result.returncode != 0:
+        print(f"Warning: gh pr list failed, will open a new PR:\n{result.stderr}", file=sys.stderr)
+        return None, None
+    matches = [pr for pr in json.loads(result.stdout) if pr["headRefName"].startswith(prefix)]
+    if not matches:
+        return None, None
+    matches.sort(key=lambda pr: pr["createdAt"], reverse=True)
+    if len(matches) > 1:
+        others = ", ".join(f"#{pr['number']}" for pr in matches[1:])
+        print(
+            f"Warning: {len(matches)} open flaky-fix backport PRs for {base_branch}; "
+            f"reusing newest #{matches[0]['number']}, leaving {others} untouched.",
+            file=sys.stderr,
+        )
+    return matches[0]["number"], matches[0]["headRefName"]
+
+
+def pr_backport_commits(repo: str, number: int):
+    """Return [(upstream_sha_or_None, subject)] for commits already on the PR,
+    oldest-first, by reading the `(cherry picked from commit <sha>)` trailer that
+    `cherry-pick -x` records. Returns None if the PR commits could not be read."""
+    result = subprocess.run(
+        ["gh", "pr", "view", str(number), "--repo", repo, "--json", "commits"],
+        text=True, capture_output=True,
+    )
+    if result.returncode != 0:
+        print(f"Error: could not read commits of PR #{number}:\n{result.stderr}", file=sys.stderr)
+        return None
+    out = []
+    for c in json.loads(result.stdout).get("commits", []):
+        body = c.get("messageBody", "") or ""
+        m = re.search(r"cherry picked from commit ([0-9a-f]{7,40})", body)
+        out.append((m.group(1) if m else None, c.get("messageHeadline", "")))
+    return out
+
+
+def update_pr(repo: str, number: int, title: str, body: str, labels: list, dry_run: bool) -> None:
+    if dry_run:
+        print(f"DRY RUN: would update PR #{number} in {repo}:", file=sys.stderr)
+        print(f"  title:  {title}", file=sys.stderr)
+        print(f"  labels: {', '.join(labels)}", file=sys.stderr)
+        return
+
+    labels = existing_labels(repo, labels)
+
+    cmd = [
+        "gh", "pr", "edit", str(number),
+        "--repo", repo,
+        "--title", title,
+        "--body", body,
+    ]
+    for label in labels:
+        cmd += ["--add-label", label]
+
+    result = subprocess.run(cmd, text=True, capture_output=False)
+    if result.returncode != 0:
+        print("gh pr edit failed", file=sys.stderr)
+        sys.exit(1)
+
+
 def create_pr(repo: str, branch: str, base: str, title: str, body: str, labels: list, dry_run: bool) -> None:
     if dry_run:
         print(f"DRY RUN: would open PR in {repo}:", file=sys.stderr)
@@ -153,7 +233,7 @@ def parse_args() -> argparse.Namespace:
         "--json-file",
         required=True,
         metavar="PATH",
-        help="JSON output file produced by check_flaky_fix_backports.py",
+        help="JSON output file produced by check_for_flaky_fixes.py",
     )
     parser.add_argument(
         "--repo",
@@ -186,17 +266,43 @@ def main() -> None:
     # Sort oldest-first so cherry-picks apply in chronological order.
     missing.sort(key=lambda c: c["date"])
 
+    # Reuse an open PR from a previous run if one exists, so we amend a single
+    # rolling PR per base branch instead of opening a new one every week.
+    reuse_number, reuse_branch = find_open_backport_pr(args.repo, base_branch)
+
+    prior_applied = []
+    if reuse_branch:
+        print(f"Reusing open PR #{reuse_number} ({reuse_branch}).", file=sys.stderr)
+        prior_commits = pr_backport_commits(args.repo, reuse_number)
+        if prior_commits is None:
+            print(
+                f"Refusing to reuse PR #{reuse_number} without its commit list "
+                f"(would risk re-applying or rewriting the PR body).",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        prior_shas = {sha for sha, _ in prior_commits if sha}
+        prior_applied = [(sha, subject, None) for sha, subject in prior_commits]
+        missing = [c for c in missing if c["sha"] not in prior_shas]
+        if not missing:
+            print(f"PR #{reuse_number} already contains all missing commits. Nothing to do.", file=sys.stderr)
+            return
+
     prefetch_upstream_objects([c["sha"] for c in missing])
 
     date_tag = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-    backport_branch = f"flaky-fix-backport/{base_branch}/{date_tag}"
-
-    existing = run_git("branch", "--list", backport_branch).stdout.strip()
-    if existing:
-        print(f"Branch {backport_branch} already exists. Delete it first.", file=sys.stderr)
-        sys.exit(1)
 
-    run_git("checkout", "-b", backport_branch)
+    if reuse_branch:
+        backport_branch = reuse_branch
+        run_git("fetch", "--depth=1", "--no-tags", "origin", reuse_branch)
+        run_git("checkout", "-B", backport_branch, "FETCH_HEAD")
+    else:
+        backport_branch = f"flaky-fix-backport/{base_branch}/{date_tag}"
+        existing = run_git("branch", "--list", backport_branch).stdout.strip()
+        if existing:
+            print(f"Branch {backport_branch} already exists. Delete it first.", file=sys.stderr)
+            sys.exit(1)
+        run_git("checkout", "-b", backport_branch)
 
     applied = []
     conflicted = []
@@ -213,39 +319,49 @@ def main() -> None:
             conflicted.append((sha, subject, date))
             print(f"  Skipped {sha[:12]} (conflict)", file=sys.stderr)
 
-    if not applied:
+    if not reuse_branch and not applied:
         print("No commits could be applied (all conflicted). Cleaning up.", file=sys.stderr)
         run_git("checkout", base_branch)
         run_git("branch", "-D", backport_branch)
         sys.exit(0)
 
+    all_applied = prior_applied + applied
     pr_title = f"{base_branch.title()} - Backport flaky-fix commits from upstream ({date_tag})"
-    pr_body = build_pr_body(upstream_repo, applied, conflicted)
+    pr_body = build_pr_body(upstream_repo, all_applied, conflicted)
     pr_labels = labels_for_branch(base_branch)
 
     if args.dry_run:
+        action = f"amend PR #{reuse_number}" if reuse_branch else f"open PR against {base_branch}"
         print(
-            f"DRY RUN: would push {backport_branch} and open PR against {base_branch}.",
+            f"DRY RUN: would push {backport_branch} and {action}.",
             file=sys.stderr,
         )
-        print(f"  Applied: {len(applied)}  Conflicted/failed: {len(conflicted)}", file=sys.stderr)
+        print(f"  Newly applied: {len(applied)}  Carried over: {len(prior_applied)}  Conflicted: {len(conflicted)}", file=sys.stderr)
         print(f"\n--- PR title ---\n{pr_title}\n--- PR body ---\n{pr_body}---")
-        create_pr(repo=args.repo, branch=backport_branch, base=base_branch, title=pr_title, body=pr_body, labels=pr_labels, dry_run=True)
+        if reuse_branch:
+            update_pr(repo=args.repo, number=reuse_number, title=pr_title, body=pr_body, labels=pr_labels, dry_run=True)
+        else:
+            create_pr(repo=args.repo, branch=backport_branch, base=base_branch, title=pr_title, body=pr_body, labels=pr_labels, dry_run=True)
         run_git("checkout", base_branch)
         run_git("branch", "-D", backport_branch)
         return
 
-    run_git("push", "origin", backport_branch, capture=False)
-
-    create_pr(
-        repo=args.repo,
-        branch=backport_branch,
-        base=base_branch,
-        title=pr_title,
-        body=pr_body,
-        labels=pr_labels,
-        dry_run=False,
-    )
+    # Only push when we added commits; body-only refreshes still update the PR.
+    if applied:
+        run_git("push", "origin", backport_branch, capture=False)
+
+    if reuse_branch:
+        update_pr(repo=args.repo, number=reuse_number, title=pr_title, body=pr_body, labels=pr_labels, dry_run=False)
+    else:
+        create_pr(
+            repo=args.repo,
+            branch=backport_branch,
+            base=base_branch,
+            title=pr_title,
+            body=pr_body,
+            labels=pr_labels,
+            dry_run=False,
+        )
 
 
 if __name__ == "__main__":

tests/ci/check_for_flaky_fixes.py

@@ -144,7 +144,7 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--pattern",
-        default="fix flaky test",
+        default="fix flaky",
         help="case-insensitive substring to match in commit subjects",
     )
     parser.add_argument(