diff --git a/.github/workflows/flaky_fix_sync.yml b/.github/workflows/flaky_fix_sync.yml index f9bf8ef1cacf..569fe87eb7d0 100644 --- a/.github/workflows/flaky_fix_sync.yml +++ b/.github/workflows/flaky_fix_sync.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: since_days: - description: 'How many days back to scan upstream master (default: 7)' + description: 'How many days back to scan upstream (default: 7)' required: false type: string default: '7' @@ -12,7 +12,7 @@ on: description: 'Commit subject pattern to match (case-insensitive)' required: false type: string - default: 'fix flaky test' + default: 'fix flaky' upstream_ref: description: 'Upstream branch/ref to scan' required: false @@ -25,11 +25,29 @@ on: default: false schedule: - - cron: '0 12 * * 1' # Every Monday at 08:00 EDT (12:00 UTC) + - cron: '0 10 * * 1' # Every Monday at 06:00 EDT (10:00 UTC) + jobs: + setup: + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] + outputs: + branches: ${{ steps.set_branches.outputs.branches }} + steps: + - id: set_branches + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "branches=[\"${{ github.ref_name }}\"]" >> $GITHUB_OUTPUT + else + echo "branches=[\"antalya-26.3\", \"stable-26.3\", \"antalya-26.6\"]" >> $GITHUB_OUTPUT + fi + check: + needs: setup runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] + strategy: + matrix: + branch: ${{ fromJSON(needs.setup.outputs.branches) }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -37,6 +55,9 @@ jobs: fetch-depth: 0 filter: blob:none + - name: Checkout target branch + run: git checkout ${{ matrix.branch }} + - name: Set up Python uses: actions/setup-python@v4 with: @@ -47,27 +68,30 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | python3 tests/ci/check_for_flaky_fixes.py \ - --since-days "${{ inputs.since_days || '7' }}" \ - --pattern "${{ inputs.pattern || 'fix flaky test' }}" \ - --upstream-ref "${{ inputs.upstream_ref || 'master' }}" \ - --md-output flaky_fix_report.md \ - --json-output flaky_fix.json + --since-days "${{ github.event.inputs.since_days || '7' }}" \ + --pattern "${{ github.event.inputs.pattern || 'fix flaky' }}" \ + --upstream-ref "${{ github.event.inputs.upstream_ref || 'master' }}" \ + --md-output flaky_fix_report_${{ matrix.branch }}.md \ + --json-output flaky_fix_${{ matrix.branch }}.json - name: Write summary if: always() - run: cat flaky_fix_report.md >> "$GITHUB_STEP_SUMMARY" + run: cat flaky_fix_report_${{ matrix.branch }}.md >> "$GITHUB_STEP_SUMMARY" - name: Upload check results uses: actions/upload-artifact@v4 with: - name: flaky-fix-check + name: flaky-fix-check-${{ matrix.branch }} path: | - flaky_fix_report.md - flaky_fix.json + flaky_fix_report_${{ matrix.branch }}.md + flaky_fix_${{ matrix.branch }}.json backport: - needs: check - if: ${{ github.event_name == 'schedule' || inputs.create_pr == true }} + needs: [check, setup] + if: ${{ github.event_name == 'schedule' || github.event.inputs.create_pr == true }} + strategy: + matrix: + branch: ${{ fromJSON(needs.setup.outputs.branches) }} runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] steps: - name: Check out repository @@ -76,6 +100,9 @@ jobs: fetch-depth: 1 filter: blob:none + - name: Checkout target branch + run: git checkout ${{ matrix.branch }} + - name: Set up Python uses: actions/setup-python@v4 with: @@ -84,7 +111,7 @@ jobs: - name: Download check results uses: actions/download-artifact@v4 with: - name: flaky-fix-check + name: flaky-fix-check-${{ matrix.branch }} - name: Configure git identity run: | @@ -102,5 +129,5 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | python3 tests/ci/backport_flaky_fixes.py \ - --json-file flaky_fix.json \ + --json-file flaky_fix_${{ matrix.branch }}.json \ --repo "${{ github.repository }}" diff --git a/tests/ci/backport_flaky_fixes.py b/tests/ci/backport_flaky_fixes.py index 1724bedb1492..3b16bc3eb826 100755 --- a/tests/ci/backport_flaky_fixes.py +++ b/tests/ci/backport_flaky_fixes.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 """ -Read the JSON output from check_for_flaky_fix_backports.py, cherry-pick each -missing commit onto a new branch, push it, and open a single PR. +Read the JSON output from check_for_flaky_fixes.py and cherry-pick each missing +commit onto a backport branch. Opens a new PR, or amends the open one from a +previous run if it already exists for this base branch. """ import argparse @@ -86,6 +87,17 @@ def cherry_pick(sha: str) -> bool: return False +def _commit_line(upstream_repo: str, sha, subject: str, date) -> str: + """Render one bullet. sha/date may be None for commits carried over from a + reused PR, where only the subject is known.""" + link = "" + if sha: + url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha) + link = f"[`{sha[:12]}`]({url}) " + suffix = f" _(committed {date})_" if date else "" + return f"- {link}{subject}{suffix}" + + def build_pr_body( upstream_repo: str, applied: list, @@ -101,21 +113,89 @@ def build_pr_body( lines.append("### Applied") lines.append("") for sha, subject, date in applied: - url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha) - lines.append(f"- [`{sha[:12]}`]({url}) {subject} _(committed {date})_") + lines.append(_commit_line(upstream_repo, sha, subject, date)) lines.append("") if conflicted: lines.append("### Skipped (cherry-pick conflict — manual backport needed)") lines.append("") for sha, subject, date in conflicted: - url = UPSTREAM_COMMIT_URL.format(repo=upstream_repo, sha=sha) - lines.append(f"- [`{sha[:12]}`]({url}) {subject} _(committed {date})_") + lines.append(_commit_line(upstream_repo, sha, subject, date)) lines.append("") return "\n".join(lines) +def find_open_backport_pr(repo: str, base_branch: str): + """Return (number, head_branch) of an open flaky-fix backport PR for this base + branch, or (None, None). Lets a weekly run amend last week's PR instead of + opening a fresh one each time. If several match, reuses the newest.""" + prefix = f"flaky-fix-backport/{base_branch}/" + result = subprocess.run( + ["gh", "pr", "list", "--repo", repo, "--state", "open", "--base", base_branch, + "--json", "number,headRefName,createdAt", "--limit", "100"], + text=True, capture_output=True, + ) + if result.returncode != 0: + print(f"Warning: gh pr list failed, will open a new PR:\n{result.stderr}", file=sys.stderr) + return None, None + matches = [pr for pr in json.loads(result.stdout) if pr["headRefName"].startswith(prefix)] + if not matches: + return None, None + matches.sort(key=lambda pr: pr["createdAt"], reverse=True) + if len(matches) > 1: + others = ", ".join(f"#{pr['number']}" for pr in matches[1:]) + print( + f"Warning: {len(matches)} open flaky-fix backport PRs for {base_branch}; " + f"reusing newest #{matches[0]['number']}, leaving {others} untouched.", + file=sys.stderr, + ) + return matches[0]["number"], matches[0]["headRefName"] + + +def pr_backport_commits(repo: str, number: int): + """Return [(upstream_sha_or_None, subject)] for commits already on the PR, + oldest-first, by reading the `(cherry picked from commit )` trailer that + `cherry-pick -x` records. Returns None if the PR commits could not be read.""" + result = subprocess.run( + ["gh", "pr", "view", str(number), "--repo", repo, "--json", "commits"], + text=True, capture_output=True, + ) + if result.returncode != 0: + print(f"Error: could not read commits of PR #{number}:\n{result.stderr}", file=sys.stderr) + return None + out = [] + for c in json.loads(result.stdout).get("commits", []): + body = c.get("messageBody", "") or "" + m = re.search(r"cherry picked from commit ([0-9a-f]{7,40})", body) + out.append((m.group(1) if m else None, c.get("messageHeadline", ""))) + return out + + +def update_pr(repo: str, number: int, title: str, body: str, labels: list, dry_run: bool) -> None: + if dry_run: + print(f"DRY RUN: would update PR #{number} in {repo}:", file=sys.stderr) + print(f" title: {title}", file=sys.stderr) + print(f" labels: {', '.join(labels)}", file=sys.stderr) + return + + labels = existing_labels(repo, labels) + + cmd = [ + "gh", "pr", "edit", str(number), + "--repo", repo, + "--title", title, + "--body", body, + ] + for label in labels: + cmd += ["--add-label", label] + + result = subprocess.run(cmd, text=True, capture_output=False) + if result.returncode != 0: + print("gh pr edit failed", file=sys.stderr) + sys.exit(1) + + def create_pr(repo: str, branch: str, base: str, title: str, body: str, labels: list, dry_run: bool) -> None: if dry_run: print(f"DRY RUN: would open PR in {repo}:", file=sys.stderr) @@ -153,7 +233,7 @@ def parse_args() -> argparse.Namespace: "--json-file", required=True, metavar="PATH", - help="JSON output file produced by check_flaky_fix_backports.py", + help="JSON output file produced by check_for_flaky_fixes.py", ) parser.add_argument( "--repo", @@ -186,17 +266,43 @@ def main() -> None: # Sort oldest-first so cherry-picks apply in chronological order. missing.sort(key=lambda c: c["date"]) + # Reuse an open PR from a previous run if one exists, so we amend a single + # rolling PR per base branch instead of opening a new one every week. + reuse_number, reuse_branch = find_open_backport_pr(args.repo, base_branch) + + prior_applied = [] + if reuse_branch: + print(f"Reusing open PR #{reuse_number} ({reuse_branch}).", file=sys.stderr) + prior_commits = pr_backport_commits(args.repo, reuse_number) + if prior_commits is None: + print( + f"Refusing to reuse PR #{reuse_number} without its commit list " + f"(would risk re-applying or rewriting the PR body).", + file=sys.stderr, + ) + sys.exit(1) + prior_shas = {sha for sha, _ in prior_commits if sha} + prior_applied = [(sha, subject, None) for sha, subject in prior_commits] + missing = [c for c in missing if c["sha"] not in prior_shas] + if not missing: + print(f"PR #{reuse_number} already contains all missing commits. Nothing to do.", file=sys.stderr) + return + prefetch_upstream_objects([c["sha"] for c in missing]) date_tag = datetime.now(timezone.utc).strftime("%Y-%m-%d") - backport_branch = f"flaky-fix-backport/{base_branch}/{date_tag}" - - existing = run_git("branch", "--list", backport_branch).stdout.strip() - if existing: - print(f"Branch {backport_branch} already exists. Delete it first.", file=sys.stderr) - sys.exit(1) - run_git("checkout", "-b", backport_branch) + if reuse_branch: + backport_branch = reuse_branch + run_git("fetch", "--depth=1", "--no-tags", "origin", reuse_branch) + run_git("checkout", "-B", backport_branch, "FETCH_HEAD") + else: + backport_branch = f"flaky-fix-backport/{base_branch}/{date_tag}" + existing = run_git("branch", "--list", backport_branch).stdout.strip() + if existing: + print(f"Branch {backport_branch} already exists. Delete it first.", file=sys.stderr) + sys.exit(1) + run_git("checkout", "-b", backport_branch) applied = [] conflicted = [] @@ -213,39 +319,49 @@ def main() -> None: conflicted.append((sha, subject, date)) print(f" Skipped {sha[:12]} (conflict)", file=sys.stderr) - if not applied: + if not reuse_branch and not applied: print("No commits could be applied (all conflicted). Cleaning up.", file=sys.stderr) run_git("checkout", base_branch) run_git("branch", "-D", backport_branch) sys.exit(0) + all_applied = prior_applied + applied pr_title = f"{base_branch.title()} - Backport flaky-fix commits from upstream ({date_tag})" - pr_body = build_pr_body(upstream_repo, applied, conflicted) + pr_body = build_pr_body(upstream_repo, all_applied, conflicted) pr_labels = labels_for_branch(base_branch) if args.dry_run: + action = f"amend PR #{reuse_number}" if reuse_branch else f"open PR against {base_branch}" print( - f"DRY RUN: would push {backport_branch} and open PR against {base_branch}.", + f"DRY RUN: would push {backport_branch} and {action}.", file=sys.stderr, ) - print(f" Applied: {len(applied)} Conflicted/failed: {len(conflicted)}", file=sys.stderr) + print(f" Newly applied: {len(applied)} Carried over: {len(prior_applied)} Conflicted: {len(conflicted)}", file=sys.stderr) print(f"\n--- PR title ---\n{pr_title}\n--- PR body ---\n{pr_body}---") - create_pr(repo=args.repo, branch=backport_branch, base=base_branch, title=pr_title, body=pr_body, labels=pr_labels, dry_run=True) + if reuse_branch: + update_pr(repo=args.repo, number=reuse_number, title=pr_title, body=pr_body, labels=pr_labels, dry_run=True) + else: + create_pr(repo=args.repo, branch=backport_branch, base=base_branch, title=pr_title, body=pr_body, labels=pr_labels, dry_run=True) run_git("checkout", base_branch) run_git("branch", "-D", backport_branch) return - run_git("push", "origin", backport_branch, capture=False) - - create_pr( - repo=args.repo, - branch=backport_branch, - base=base_branch, - title=pr_title, - body=pr_body, - labels=pr_labels, - dry_run=False, - ) + # Only push when we added commits; body-only refreshes still update the PR. + if applied: + run_git("push", "origin", backport_branch, capture=False) + + if reuse_branch: + update_pr(repo=args.repo, number=reuse_number, title=pr_title, body=pr_body, labels=pr_labels, dry_run=False) + else: + create_pr( + repo=args.repo, + branch=backport_branch, + base=base_branch, + title=pr_title, + body=pr_body, + labels=pr_labels, + dry_run=False, + ) if __name__ == "__main__": diff --git a/tests/ci/check_for_flaky_fixes.py b/tests/ci/check_for_flaky_fixes.py index f3f9da129c19..b40b08c6eb54 100755 --- a/tests/ci/check_for_flaky_fixes.py +++ b/tests/ci/check_for_flaky_fixes.py @@ -144,7 +144,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--pattern", - default="fix flaky test", + default="fix flaky", help="case-insensitive substring to match in commit subjects", ) parser.add_argument(