Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,9 @@ add_test(NAME embed_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/embed_serve.sh $<TAR
add_test(NAME watchdog_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/watchdog_serve.sh $<TARGET_FILE:onebit>)
# serve's GET /metrics (docs/serve.md, "Metrics"): a fake backend, no GPU.
add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.sh $<TARGET_FILE:onebit>)
# engine#1945: scripts/bump-hrx-sync.sh tells AMD's own commit from a line AMD rebased apart from a commit that
# is not AMD's. Local git repositories only: no network, no GPU.
add_test(NAME bump_hrx_sync COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_sync_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-sync.sh)
add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $<TARGET_FILE:onebit>)
# serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU.
add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $<TARGET_FILE:onebit>)
Expand Down
8 changes: 7 additions & 1 deletion docs/hrx.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,13 @@ fails a pin that leaves a gate model's node off HRX.
`DRY_RUN=1` and pushes nothing):
- syncs the fork (`master` to ggml-org, `1bit/hrx-vulkan` to AMD's pin). When AMD
rewrites its history, the old `1bit/hrx-vulkan` tip is tagged `hrx-vulkan-<sha>` first,
and the run fails if that branch held a commit that is on no AMD branch;
and the run fails if that branch held a commit that is on no AMD branch. The failure
says which of two things that commit is, because they need opposite actions:
**rebased** (AMD's pin holds a commit with the same author, author date and subject:
AMD rebased its line and dropped the old one, so the commit is AMD's and the pin is merged
by hand) or **foreign** (no such commit: it is not AMD's and moves to
`1bit/hrx-vulkan-patched`). `scripts/bump-hrx-sync.sh off-branch <repo> <mirror> <pin>`
prints the classification alone, from a commits-only clone;
- merges AMD's pin into `1bit/hrx-vulkan-patched`, after tagging the previous tip
`patched-<sha>` so every commit the engine has pinned stays reachable. Conflicts under
`ggml/src/ggml-hrx` keep our side; a conflict anywhere else fails the run and names the files;
Expand Down
38 changes: 34 additions & 4 deletions scripts/bump-hrx-sync.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@
# an ancestor of the new pin. If every commit on the mirror is still on an AMD branch,
# the old tip is tagged and the mirror moves; the merge treats the squash commit as a
# child of its pre-squash equivalent, so only AMD's genuinely new changes are merged.
# A mirror commit that is on no AMD branch fails the run and is named.
# A mirror commit that is on no AMD branch fails the run and is named, as one of two
# things (off_branch_commits): AMD's own commit from a line AMD rebased and dropped, to be
# merged by hand, or a commit that is not AMD's, to be moved to our line.
# - hrx-system is held (config/hrx-system-hold.config). AMD's ggml-hrx is written against
# AMD's hrx-system (e.g. #139 moved the kernel corpus to the `multiple_of` predicate that
# our held hrx-system cannot parse), so only AMD's changes OUTSIDE the HRX backend paths
Expand Down Expand Up @@ -83,6 +85,27 @@ hold_sha() {
grep -v '^[[:space:]]*#' "$ENGINE_DIR/$HOLD_FILE" | grep -oE '^[0-9a-f]{40}' | head -n1 || true
}

# Why a mirror commit is on no AMD branch. Reads repo $1 (commits only are enough), whose refs/amd/*
# are AMD's branches; $2 is the mirror tip and $3 AMD's new pin. One line per such commit:
# rebased <old> <new> <subject> the new pin holds a commit with the same author, author date and
# subject. A rebase keeps exactly those and changes the hash, so this
# is AMD's commit from before AMD rebased its line and dropped the old one.
# foreign <old> <subject> no such commit: it is not AMD's, so it does not belong on the mirror.
# 2026-11-10 (engine#1945): AMD re-committed its HRX series on a base 1777 commits newer and left the
# old line on no branch. The run called the old tip, AMD's own #155, a commit of ours to move to
# 1bit/hrx-vulkan-patched, and a PR that did so was opened before anyone compared authors.
off_branch_commits() {
local repo=$1 mirror=$2 pin=$3 h ae at s
declare -A twin=()
while IFS=$'\t' read -r h ae at s; do
[ -n "${twin["$ae|$at|$s"]:-}" ] || twin["$ae|$at|$s"]=$h
done < <(git -C "$repo" log --format='%h%x09%ae%x09%at%x09%s' "$pin" --not "$mirror")
while IFS=$'\t' read -r h ae at s; do
if [ -n "${twin["$ae|$at|$s"]:-}" ]; then echo "rebased $h ${twin["$ae|$at|$s"]} $s"
else echo "foreign $h $s"; fi
done < <(git -C "$repo" log --format='%h%x09%ae%x09%at%x09%s' "$mirror" --not "$pin" --glob='refs/amd/*')
}

sync_llama() {
: "${AMD_LLAMA:?}" "${OURS_HRX:?}"
local hold refresh; hold=$(hold_sha)
Expand Down Expand Up @@ -114,10 +137,15 @@ sync_llama() {
git -C ../amdrefs fetch -q --filter=tree:0 https://github.com/AMD-Ecosystem/llama.cpp.git \
'+refs/heads/*:refs/amd/*' "$pin"
git -C ../amdrefs fetch -q --filter=tree:0 "$(remote_url 1bit-MONSTER/llama.cpp)" "+$mirror:refs/old-mirror"
local foreign
foreign=$(git -C ../amdrefs log --format='%h %s' "$mirror" --not "$pin" --glob='refs/amd/*')
local off foreign rebased
off=$(off_branch_commits ../amdrefs "$mirror" "$pin")
foreign=$(sed -n 's/^foreign //p' <<< "$off")
rebased=$(sed -n 's/^rebased //p' <<< "$off")
if [ -n "$foreign" ]; then
die "1bit/hrx-vulkan has commits that are on no branch of AMD-Ecosystem/llama.cpp; the mirror must hold AMD's commits only. Move these to 1bit/hrx-vulkan-patched by hand: $(printf '%s; ' "$foreign")"
die "1bit/hrx-vulkan has commits that are on no branch of AMD-Ecosystem/llama.cpp and have no same-author, same-date, same-subject commit in AMD's pin; the mirror must hold AMD's commits only. Move these to 1bit/hrx-vulkan-patched by hand: $(tr '\n' ';' <<< "$foreign")"
fi
if [ -n "$rebased" ]; then
die "AMD rebased its line and dropped the old one: $(wc -l <<< "$rebased" | tr -d ' ') commit(s) of 1bit/hrx-vulkan are on no AMD branch, and each has a commit in AMD's pin ${pin:0:12} with the same author, author date and subject (old new subject: $(tr '\n' ';' <<< "$rebased")). They are AMD's commits, NOT ours: do not move them to 1bit/hrx-vulkan-patched. Merge AMD's pin into 1bit/hrx-vulkan-patched by hand, then move the mirror to the pin."
fi
# AMD rewrote history. Find the newest commit of the new pin whose tree equals a commit that
# descends from the old mirror on some AMD branch (its pre-rewrite equivalent).
Expand Down Expand Up @@ -263,5 +291,7 @@ sync_hrx() {
case "$cmd" in
llama) sync_llama ;;
hrx) sync_hrx ;;
# For tests/bump_hrx_sync_test.sh: the classification alone, on a local repository, no network.
off-branch) off_branch_commits "${2:?repo}" "${3:?mirror}" "${4:?pin}" ;;
*) echo "usage: $0 llama|hrx" >&2; exit 2 ;;
esac
106 changes: 106 additions & 0 deletions tests/bump_hrx_sync_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# scripts/bump-hrx-sync.sh, the part that says WHY a commit of our AMD mirror is on no AMD branch
# (engine#1945). Two answers, and they need opposite actions:
# rebased AMD rebased its line and dropped the old one; the commit is AMD's and its twin is in the pin
# foreign the commit is not AMD's and has to leave the mirror
# Local repositories only: no network, no GPU, no token.
set -euo pipefail

script=${1:?usage: bump_hrx_sync_test.sh <scripts/bump-hrx-sync.sh>}
script=$(cd "$(dirname "$script")" && pwd)/$(basename "$script")
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
fails=0
ok() { echo "ok $*"; }
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
expect() { # expect <what> <actual> <wanted>
if [ "$2" = "$3" ]; then ok "$1"; else fail "$1"; printf ' got: %s\n wanted: %s\n' "$2" "$3"; fi
}

export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null
repo=$work/r
git init -q "$repo"
g() { git -C "$repo" "$@"; }
# commit <author name> <author email> <author date> <committer date> <subject>
commit() {
echo "$5 $RANDOM$RANDOM" >> "$repo/f"
g add f
GIT_AUTHOR_NAME=$1 GIT_AUTHOR_EMAIL=$2 GIT_AUTHOR_DATE=$3 \
GIT_COMMITTER_NAME=c GIT_COMMITTER_EMAIL=c@example.invalid GIT_COMMITTER_DATE=$4 \
git -C "$repo" commit -q -m "$5"
}
amd='amd dev'; amd_mail=dev@amd.example.invalid

# The old line: upstream base, then two AMD commits. The mirror is its tip.
commit up up@example.invalid 2026-01-01T00:00:00Z 2026-01-01T00:00:00Z "upstream base"
base=$(g rev-parse HEAD)
commit "$amd" "$amd_mail" 2026-08-17T03:02:45Z 2026-08-17T03:09:08Z "hrx: add the backend"
commit "$amd" "$amd_mail" 2026-10-28T03:17:36Z 2026-10-28T03:17:36Z "hrx: fix concat bounds (#155)"
mirror=$(g rev-parse HEAD)
old1=$(g rev-parse --short HEAD~1); old2=$(g rev-parse --short HEAD)

# The new line: a newer upstream base, the same two commits re-committed (same author, same author date,
# same subject, a later committer date -- what a rebase leaves), then one new commit. The pin is its tip.
g checkout -q -b newline "$base"
commit up up@example.invalid 2026-10-27T00:00:00Z 2026-10-27T00:00:00Z "upstream moved on"
commit "$amd" "$amd_mail" 2026-08-17T03:02:45Z 2026-10-28T09:00:00Z "hrx: add the backend"
new1=$(g rev-parse --short HEAD)
commit "$amd" "$amd_mail" 2026-10-28T03:17:36Z 2026-10-28T09:00:01Z "hrx: fix concat bounds (#155)"
new2=$(g rev-parse --short HEAD)
commit "$amd" "$amd_mail" 2026-11-09T15:29:43Z 2026-11-09T15:29:43Z "hrx: something new"
pin=$(g rev-parse HEAD)
g update-ref refs/amd/main "$pin" # AMD's only branch: the old line is on none

run() { bash "$script" off-branch "$repo" "$1" "$2"; }

got=$(run "$mirror" "$pin")
expect "a rebased line: both old commits are named with their twins in the pin, newest first" "$got" \
"rebased $old2 $new2 hrx: fix concat bounds (#155)
rebased $old1 $new1 hrx: add the backend"
if grep -q '^foreign ' <<< "$got"; then fail "a rebased AMD commit is not called foreign"; else ok "a rebased AMD commit is not called foreign"; fi

# While AMD still has the old line on a branch, nothing is off-branch at all.
g update-ref refs/amd/backup "$mirror"
expect "an old line AMD still keeps on a branch reports nothing" "$(run "$mirror" "$pin")" ""
g update-ref -d refs/amd/backup

# A commit of ours on top of the mirror: no twin in the pin, so it is foreign, and the AMD ones stay rebased.
g checkout -q --detach "$mirror"
commit "a lane" lane@example.invalid 2026-11-01T00:00:00Z 2026-11-01T00:00:00Z "hrx: a fix of ours"
ours=$(g rev-parse HEAD); ours_s=$(g rev-parse --short HEAD)
got=$(run "$ours" "$pin")
expect "a commit that is not AMD's is foreign, beside the rebased ones" "$got" \
"foreign $ours_s hrx: a fix of ours
rebased $old2 $new2 hrx: fix concat bounds (#155)
rebased $old1 $new1 hrx: add the backend"

# The subject alone is not identity: the same subject from another author, or at another author date, is foreign.
g checkout -q --detach "$mirror"
commit "a lane" lane@example.invalid 2026-11-09T15:29:43Z 2026-11-09T15:29:43Z "hrx: something new"
other=$(g rev-parse HEAD)
expect "the same subject from another author is foreign" "$(run "$other" "$pin" | head -n1)" \
"foreign $(g rev-parse --short "$other") hrx: something new"
g checkout -q --detach "$mirror"
commit "$amd" "$amd_mail" 2026-11-09T15:29:44Z 2026-11-09T15:29:44Z "hrx: something new"
other=$(g rev-parse HEAD)
expect "the same subject and author at another author date is foreign" "$(run "$other" "$pin" | head -n1)" \
"foreign $(g rev-parse --short "$other") hrx: something new"

if [ "$fails" -ne 0 ]; then echo "$fails check(s) failed"; exit 1; fi
echo "bump_hrx_sync: ok"
Loading