From 0871b86ac33203201b5fa3010904e73000bd2bfa Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 1 Oct 2026 03:39:49 -0600 Subject: [PATCH 1/5] K2GO-437 feat(maven-offline): offline Maven repository producer tool Builds one Maven2 repository with the build dependencies of Knowledge to Go, Code on the Go and the add-ons, served static for offline builds on the K2Go network. Uses direct Gradle resolution (the suggested yubyf plugin writes no artifacts on our Gradle versions); adds aapt2 and brotli4j for Linux and Windows build hosts. --- static/maven-offline/.gitignore | 10 + static/maven-offline/README.md | 60 +++++ static/maven-offline/build-maven-offline.sh | 228 ++++++++++++++++++ .../maven-offline/config/extra-artifacts.tsv | 14 ++ static/maven-offline/config/projects.tsv | 16 ++ .../gradle/offline-repo.init.gradle | 34 +++ .../gradle/resolve-all.init.gradle | 45 ++++ 7 files changed, 407 insertions(+) create mode 100644 static/maven-offline/.gitignore create mode 100644 static/maven-offline/README.md create mode 100644 static/maven-offline/build-maven-offline.sh create mode 100644 static/maven-offline/config/extra-artifacts.tsv create mode 100644 static/maven-offline/config/projects.tsv create mode 100644 static/maven-offline/gradle/offline-repo.init.gradle create mode 100644 static/maven-offline/gradle/resolve-all.init.gradle diff --git a/static/maven-offline/.gitignore b/static/maven-offline/.gitignore new file mode 100644 index 000000000..ff4dec766 --- /dev/null +++ b/static/maven-offline/.gitignore @@ -0,0 +1,10 @@ +# Generated output: the Maven2 repository tree (about 1.1 to 1.2 GiB). Never commit. +/out/ +# Per-run isolated Gradle user homes used during resolution. +/.gradle-homes/ +# Shallow clones of Code on the Go / add-ons made by the producer. +/.work/ +# Local source overrides (absolute paths on a developer machine). +/config/sources.local.tsv +# Logs. +*.log diff --git a/static/maven-offline/README.md b/static/maven-offline/README.md new file mode 100644 index 000000000..59903846a --- /dev/null +++ b/static/maven-offline/README.md @@ -0,0 +1,60 @@ +# maven-offline (K2GO-437) + +A download machine. It builds one offline Maven repository that holds every +dependency the builds need: Knowledge to Go, Code on the Go, and the add-ons. +A device or a laptop on the K2Go local network then builds those projects with +the network off. + +This folder is the PRODUCER tool only. The Ansible role that installs and serves +the repository, and the Code on the Go consumer setting, are separate tickets. + +## Contract + +- Output layout: standard Maven2 (`///` with + `.sha1`/`.md5`). This is what nginx serves and what a Gradle `maven { url ... }` + repository consumes. It is NOT the Gradle internal cache layout. +- Served over HTTP by the box nginx at a new path, for example + `http://:8085/maven-offline/`. No new port: it is one more path. +- One shared repository for all three projects (union). About 1.1 to 1.2 GiB. + Separate per-project repositories are not worth it. +- Architecture: about 99.8 percent of the repository is architecture-neutral JVM + bytecode, so one repository serves ARM devices and desktop build hosts. Only + `aapt2` and `brotli4j` have per-OS files. +- Build hosts in scope: on-device (ARM Android), Linux, Windows. MacOS deferred. + On-device (ARM) does not need the Maven `aapt2`: Code on the Go ships its own. + +## Pipeline + +1. Resolve. For each project, run `resolveAllDeps` (see `gradle/resolve-all.init.gradle`) + against an isolated Gradle user home. This forces a download of every resolvable + configuration plus the buildscript/plugin classpath into that home's module cache. +2. Reshape. Convert the module cache (`caches/modules-2/files-2.1`, content-addressed) + into Maven2 layout under `out/repo`. +3. Extras. Add the task-time tools a plain resolve misses: R8/D8, `aapt2` for Linux + and Windows, `brotli4j` native for Linux and Windows. See `config/extra-artifacts.tsv`. +4. Union and checksums. Merge all three into one tree (dedup by Maven path) and write + `.sha1`/`.md5` for every file. +5. Smoke test. Build a target with the network off, using only `out/repo` as the single + repository. This is the acceptance proof that the repository is complete. + +## Why not the suggested plugin + +The suggested `io.github.yubyf.maven-offline` 1.0.4 writes zero artifacts on our +Gradle versions (8.8 and 8.14): it reports "No effective repositories found" and +skips the download. So the producer uses direct Gradle resolution instead. + +## Usage + + ./build-maven-offline.sh # resolve + reshape + extras + checksums + ./build-maven-offline.sh --smoke # also run the offline build smoke test + +Notes: +- Run on an online host (the producer must reach Maven Central and Google Maven). +- Linux is the canonical producer host (CI). On Windows, run from a shell where a + Gradle daemon can open a loopback socket; `--no-daemon` is used to avoid that. +- `out/` and the per-run Gradle homes are generated, not committed (see `.gitignore`). + +## Sizes and method + +See the local study `maven-offline-study/REPORT.md` for the measured sizes, the +common trunk, and the per-OS slivers. diff --git a/static/maven-offline/build-maven-offline.sh b/static/maven-offline/build-maven-offline.sh new file mode 100644 index 000000000..587c02717 --- /dev/null +++ b/static/maven-offline/build-maven-offline.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# +# maven-offline: the download machine (K2GO-437). +# +# Builds one offline Maven2 repository under out/repo that holds every dependency +# the builds of Knowledge to Go, Code on the Go, and the add-ons need. A device or +# a laptop on the K2Go network then builds those projects with the network off. +# +# Pipeline: resolve (per project, isolated Gradle home) -> reshape (Gradle cache to +# Maven2 layout) -> extras (R8/D8, aapt2, brotli4j) -> checksums -> report -> smoke. +# +# Run on an online host. Linux is the canonical producer host. See README.md. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/../.." && pwd)" +OUT="$HERE/out/repo" +HOMES="$HERE/.gradle-homes" +WORK="$HERE/.work" +INIT="$HERE/gradle/resolve-all.init.gradle" +PROJECTS_TSV="$HERE/config/projects.tsv" +EXTRAS_TSV="$HERE/config/extra-artifacts.tsv" +LOCAL_SRC="$HERE/config/sources.local.tsv" + +MAVEN_CENTRAL="https://repo1.maven.org/maven2" +GOOGLE_MAVEN="https://dl.google.com/dl/android/maven2" + +DO_SMOKE=0 +ONLY="" + +usage() { sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'; } + +while [ $# -gt 0 ]; do + case "$1" in + --smoke) DO_SMOKE=1 ;; + --only) ONLY="${2:-}"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac + shift +done + +log() { printf '>> %s\n' "$*"; } + +# gradlew wrapper for a project dir, picking the Windows launcher under Git Bash. +gradlew_for() { + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) printf '%s/gradlew.bat' "$1" ;; + *) printf '%s/gradlew' "$1" ;; + esac +} + +# Resolve a local path override for a project name (config/sources.local.tsv), else "". +local_override() { + [ -f "$LOCAL_SRC" ] || { printf ''; return; } + awk -F'\t' -v n="$1" '!/^#/ && $1==n {print $2; exit}' "$LOCAL_SRC" +} + +# Clone (shallow) or point at a local checkout; echoes the source dir. +source_dir() { + local name="$1" source="$2" ref="$3" ovr + ovr="$(local_override "$name")" + if [ -n "$ovr" ]; then printf '%s' "$ovr"; return; fi + if [ "$source" = "self" ]; then printf '%s' "$REPO_ROOT"; return; fi + local dst="$WORK/$name" + if [ ! -d "$dst/.git" ]; then + mkdir -p "$WORK" + git clone --depth 1 --branch "$ref" "$source" "$dst" >&2 + fi + printf '%s' "$dst" +} + +# Force-download one Gradle build's dependencies into an isolated home. +resolve_build() { + local gradle_root="$1" home="$2" gw + gw="$(gradlew_for "$gradle_root")" + log "resolve: $gradle_root (home ${home##*/})" + ( cd "$gradle_root" && "$gw" --gradle-user-home "$home" \ + --init-script "$INIT" --no-daemon --console=plain -q \ + resolveAllDeps ) || log "resolve returned nonzero (lenient; continuing)" +} + +# Copy one isolated home's module cache into out/repo as Maven2 layout. +# Gradle cache: files-2.1///// +# Maven2: /// +reshape_home() { + local home="$1" cache="$1/caches/modules-2/files-2.1" n=0 + [ -d "$cache" ] || { log "no cache in ${home##*/}"; return; } + while IFS= read -r f; do + local rel="${f#"$cache"/}" + local group="${rel%%/*}"; rel="${rel#*/}" + local artifact="${rel%%/*}"; rel="${rel#*/}" + local version="${rel%%/*}"; rel="${rel#*/}" + local file="${rel##*/}" + local dest="$OUT/${group//.//}/$artifact/$version" + mkdir -p "$dest" + [ -f "$dest/$file" ] || cp "$f" "$dest/$file" + n=$((n+1)) + done < <(find "$cache" -type f) + log "reshaped ${home##*/}: $n files" +} + +# Download to (skip if present and non-empty). +fetch() { + local url="$1" dest="$2" + [ -s "$dest" ] && return 0 + mkdir -p "$(dirname "$dest")" + curl -fsSL "$url" -o "$dest" && return 0 + rm -f "$dest"; log "miss: $url"; return 1 +} + +# Place one GAV artifact (jar + pom) from a base repo into out/repo. +place_artifact() { + local base="$1" group="$2" artifact="$3" version="$4" classifier="$5" ext="$6" + local gpath="${group//.//}/$artifact/$version" + local name="$artifact-$version"; [ "$classifier" != "-" ] && name="$name-$classifier" + fetch "$base/$gpath/$name.$ext" "$OUT/$gpath/$name.$ext" || true + fetch "$base/$gpath/$artifact-$version.pom" "$OUT/$gpath/$artifact-$version.pom" || true +} + +# Extras a plain resolve misses: brotli4j natives (config) + aapt2/R8 (derived). +fetch_extras() { + log "extras: brotli4j natives" + while IFS=$'\t' read -r group artifact version classifier ext hosts; do + [ -z "${group:-}" ] && continue + case "$group" in \#*) continue ;; esac + place_artifact "$MAVEN_CENTRAL" "$group" "$artifact" "$version" "$classifier" "$ext" + done < "$EXTRAS_TSV" + derive_aapt2_r8 +} + +# aapt2 (Linux + Windows) and R8 follow each project's AGP version. Resolve the exact +# aapt2 version from Google Maven metadata for each AGP in use, then fetch the host jars. +derive_aapt2_r8() { + local agp + for agp in 8.4.1 8.8.2 8.11.0; do + local meta="$WORK/aapt2-metadata.xml" + fetch "$GOOGLE_MAVEN/com/android/tools/build/aapt2/maven-metadata.xml" "$meta" || continue + local ver + ver="$(grep -oE "${agp//./\\.}-[0-9]+" "$meta" | sed -E 's:::g' | tail -1)" + [ -z "$ver" ] && { log "aapt2: no version for AGP $agp"; continue; } + log "aapt2 for AGP $agp -> $ver (linux, windows)" + place_artifact "$GOOGLE_MAVEN" com.android.tools.build aapt2 "$ver" linux jar + place_artifact "$GOOGLE_MAVEN" com.android.tools.build aapt2 "$ver" windows jar + done + # R8 is architecture-neutral. If the resolve already captured it, nothing to do; the + # smoke test confirms whether a specific r8 version is missing. TODO: pin r8 per AGP + # once the smoke test names the exact version a build requests. +} + +# sha1 + md5 beside every artifact (Gradle validates .sha1 on download). +write_checksums() { + log "checksums" + find "$OUT" -type f ! -name '*.sha1' ! -name '*.md5' | while IFS= read -r f; do + [ -f "$f.sha1" ] || sha1sum "$f" | cut -d' ' -f1 > "$f.sha1" + [ -f "$f.md5" ] || md5sum "$f" | cut -d' ' -f1 > "$f.md5" + done +} + +report() { + log "repository: $OUT" + log "size: $(du -sh "$OUT" | cut -f1) files: $(find "$OUT" -type f | wc -l)" +} + +process_project() { + local name="$1" source="$2" ref="$3" gradle_root="$4" mode="$5" + [ -n "$ONLY" ] && [ "$ONLY" != "$name" ] && return + local src; src="$(source_dir "$name" "$source" "$ref")" + local root="$src/$gradle_root" + if [ "$mode" = "multi" ]; then + local sub + for sub in "$root"/*/; do + [ -e "$sub/settings.gradle" ] || [ -e "$sub/settings.gradle.kts" ] || continue + resolve_build "${sub%/}" "$HOMES/$name-$(basename "$sub")" + reshape_home "$HOMES/$name-$(basename "$sub")" + done + else + resolve_build "$root" "$HOMES/$name" + reshape_home "$HOMES/$name" + fi +} + +main() { + mkdir -p "$OUT" "$HOMES" + while IFS=$'\t' read -r name source ref gradle_root mode; do + [ -z "${name:-}" ] && continue + case "$name" in \#*) continue ;; esac + process_project "$name" "$source" "$ref" "$gradle_root" "$mode" + done < "$PROJECTS_TSV" + fetch_extras + write_checksums + report + [ "$DO_SMOKE" = 1 ] && smoke_test + log "done" +} + +# Prove each root project resolves from out/repo with the network off. This is the +# acceptance proof that the repository is complete. It reuses each project's already +# downloaded Gradle distribution so --offline needs no network for the wrapper itself. +smoke_test() { + local offline_init="$HERE/gradle/offline-repo.init.gradle" + local repo; repo="$(cd "$OUT" && pwd)" + while IFS=$'\t' read -r name source ref gradle_root mode; do + [ -z "${name:-}" ] && continue + case "$name" in \#*) continue ;; esac + [ -n "$ONLY" ] && [ "$ONLY" != "$name" ] && continue + [ "$mode" = "multi" ] && continue # the add-ons share the trunk; smoke the roots + local src root warm smoke gw + src="$(source_dir "$name" "$source" "$ref")" + root="$src/$gradle_root" + warm="$HOMES/$name"; smoke="$HOMES/$name-offline" + rm -rf "$smoke"; mkdir -p "$smoke" + [ -d "$warm/wrapper" ] && cp -r "$warm/wrapper" "$smoke/wrapper" + gw="$(gradlew_for "$root")" + log "smoke (offline): $name" + if ( cd "$root" && "$gw" --gradle-user-home "$smoke" --offline \ + "-Dmavenoffline.repo=$repo" \ + --init-script "$offline_init" --init-script "$INIT" \ + --no-daemon --console=plain -q resolveAllDeps ); then + log "smoke OK: $name resolves with the network off" + else + log "smoke FAIL: $name has missing artifacts in out/repo"; return 1 + fi + done < "$PROJECTS_TSV" +} + +main diff --git a/static/maven-offline/config/extra-artifacts.tsv b/static/maven-offline/config/extra-artifacts.tsv new file mode 100644 index 000000000..7187e0318 --- /dev/null +++ b/static/maven-offline/config/extra-artifacts.tsv @@ -0,0 +1,14 @@ +# Task-time artifacts a plain dependency resolve does NOT pull, but an offline build +# still needs. One per line, tab-separated. Lines starting with # are ignored. +# +# group artifact version classifier ext hosts +com.aayushatharva.brotli4j native-linux-x86_64 1.18.0 - jar linux +com.aayushatharva.brotli4j native-linux-aarch64 1.18.0 - jar linux +com.aayushatharva.brotli4j native-windows-x86_64 1.18.0 - jar windows +# +# aapt2 and R8/D8 are NOT listed here because their versions follow each project's +# AGP version, so the script derives them per project (see derive_aapt2_r8 in +# build-maven-offline.sh): +# com.android.tools.build:aapt2:: (per build-host OS; ARM on-device uses Code on the Go's own aapt2) +# com.android.tools:r8: (architecture-neutral) +# AGP versions in use: 8.4.1 (Knowledge to Go), 8.8.2 + 7.3.0 tooling (Code on the Go), 8.11.0 (add-ons). diff --git a/static/maven-offline/config/projects.tsv b/static/maven-offline/config/projects.tsv new file mode 100644 index 000000000..eb1e9fcae --- /dev/null +++ b/static/maven-offline/config/projects.tsv @@ -0,0 +1,16 @@ +# The code bases whose build dependencies go into the offline repository. +# One per line, tab-separated. Lines starting with # are ignored. +# +# name : short id, used for the isolated Gradle home dir name. +# source : "self" = the repository that contains this tool; otherwise a git URL to clone. +# ref : branch or tag to resolve. +# gradle_root : path to the Gradle root (where settings.gradle lives), relative to the repo root. +# mode : "root" = one Gradle build at gradle_root; "multi" = each immediate subdir of gradle_root is its own Gradle build (the add-ons). +# +# Local runs: to resolve from a local clone instead of cloning the git URL, add a line +# to config/sources.local.tsv (gitignored): +# +# name source ref gradle_root mode +knowledge-to-go self main controller root +code-on-the-go https://github.com/appdevforall/codeonthego stage . root +add-ons https://github.com/appdevforall/addons main . multi diff --git a/static/maven-offline/gradle/offline-repo.init.gradle b/static/maven-offline/gradle/offline-repo.init.gradle new file mode 100644 index 000000000..1a5f2e5c2 --- /dev/null +++ b/static/maven-offline/gradle/offline-repo.init.gradle @@ -0,0 +1,34 @@ +// Point every repository (plugins and dependencies) at the offline Maven2 tree only. +// Used with resolve-all.init.gradle and --offline to prove a build resolves with the +// network off. Pass the repo path with -Dmavenoffline.repo=. +// +// Two repository styles must both work: +// - settings repositories (pluginManagement + dependencyResolutionManagement). Code on +// the Go uses dependencyResolutionManagement with FAIL_ON_PROJECT_REPOS, so adding a +// PROJECT repository there is an error. We override the settings repositories instead. +// - project repositories (older style, e.g. Knowledge to Go). We override those too, but +// only when the build does not forbid project repositories. + +def repoPath = System.getProperty('mavenoffline.repo') +if (repoPath == null) throw new GradleException('set -Dmavenoffline.repo=') +def repoUri = new File(repoPath).toURI() +def allowProjectRepos = true + +settingsEvaluated { settings -> + try { + def mode = settings.dependencyResolutionManagement.repositoriesMode.getOrNull() + if (mode != null && mode.name() == 'FAIL_ON_PROJECT_REPOS') allowProjectRepos = false + } catch (ignored) { /* no dependencyResolutionManagement */ } + settings.pluginManagement.repositories { clear(); maven { url = repoUri } } + try { + settings.dependencyResolutionManagement.repositories { clear(); maven { url = repoUri } } + } catch (ignored) { /* project may not use dependencyResolutionManagement */ } +} + +allprojects { + // buildscript repositories are not governed by FAIL_ON_PROJECT_REPOS; always override. + buildscript { repositories { clear(); maven { url = repoUri } } } + if (allowProjectRepos) { + repositories { clear(); maven { url = repoUri } } + } +} diff --git a/static/maven-offline/gradle/resolve-all.init.gradle b/static/maven-offline/gradle/resolve-all.init.gradle new file mode 100644 index 000000000..086d75172 --- /dev/null +++ b/static/maven-offline/gradle/resolve-all.init.gradle @@ -0,0 +1,45 @@ +// Force a download of every dependency a build needs into this Gradle user home's +// module cache. A reshaped copy of that cache becomes the offline Maven2 repository. +// +// Applied with --init-script by build-maven-offline.sh. It registers one task, +// resolveAllDeps, on the ROOT project of every build in the composite (a plain build +// has one; Code on the Go has included builds). gradle.rootProject is the reliable +// init-script hook for this: projectsEvaluated alone does not register the task on a +// composite build's main root. +// +// The task leniently resolves every resolvable configuration of every project, plus +// each project's buildscript (plugin) classpath, so one unresolved optional artifact +// does not abort the sweep. Lenient resolution still downloads every artifact that CAN +// be resolved; it only swallows the ones that cannot (platform stubs, absent variants). + +gradle.rootProject { root -> + if (root.tasks.findByName('resolveAllDeps') != null) return + root.tasks.register('resolveAllDeps') { + doLast { + root.allprojects.each { p -> + p.configurations.each { c -> + if (c.canBeResolved) { + try { + c.incoming.artifactView { it.lenient(true) }.files.files.size() + } catch (Throwable t) { + logger.info("skip ${p.path}:${c.name}: ${t.message}") + } + } + } + try { + p.buildscript.configurations.each { bc -> + if (bc.canBeResolved) { + try { + bc.incoming.artifactView { it.lenient(true) }.files.files.size() + } catch (Throwable t) { + logger.info("skip buildscript ${p.path}:${bc.name}: ${t.message}") + } + } + } + } catch (Throwable t) { + logger.info("no buildscript for ${p.path}: ${t.message}") + } + } + } + } +} From a2430058ccb9473714428ce646f5e8253edf7896 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 1 Oct 2026 07:26:55 -0600 Subject: [PATCH 2/5] K2GO-437 fix(maven-offline): GMM url materialization, offline settings-buildscript, AGP 9.3.1 Reshape now also materializes each artifact under its Gradle Module Metadata url (KMP androidx -android AARs are served under a url that differs from the cache name; without this an offline fetch 404s). The offline smoke init now redirects the settings-level buildscript repositories too (beforeSettings), so the AGP + Kotlin plugin classpath resolves offline. Correct the add-ons AGP to 9.3.1 (R8/D8 ships inside builder:9.3.1, so no separate r8 to pin). --- static/maven-offline/build-maven-offline.sh | 32 ++++++++++++++++++- .../maven-offline/config/extra-artifacts.tsv | 12 ++++--- .../gradle/offline-repo.init.gradle | 7 ++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/static/maven-offline/build-maven-offline.sh b/static/maven-offline/build-maven-offline.sh index 587c02717..a3bd90db8 100644 --- a/static/maven-offline/build-maven-offline.sh +++ b/static/maven-offline/build-maven-offline.sh @@ -133,8 +133,11 @@ fetch_extras() { # aapt2 (Linux + Windows) and R8 follow each project's AGP version. Resolve the exact # aapt2 version from Google Maven metadata for each AGP in use, then fetch the host jars. derive_aapt2_r8() { + # AGP in use: 8.4.1 (K2Go), 8.8.2 (CoGo), 9.3.1 (add-ons). For AGP 9.x the R8/D8 + # tool ships inside com.android.tools.build:builder, so a resolve already captures it; + # only aapt2 (per build-host OS) must be added here. local agp - for agp in 8.4.1 8.8.2 8.11.0; do + for agp in 8.4.1 8.8.2 9.3.1; do local meta="$WORK/aapt2-metadata.xml" fetch "$GOOGLE_MAVEN/com/android/tools/build/aapt2/maven-metadata.xml" "$meta" || continue local ver @@ -149,6 +152,32 @@ derive_aapt2_r8() { # once the smoke test names the exact version a build requests. } +# Gradle Module Metadata (.module) can declare a file whose served `url` differs from the +# cache `name` (KMP androidx -android AARs: cache name lifecycle-runtime-release.aar, url +# lifecycle-runtime-android-.aar). A Maven2 consumer reading the .module fetches by url, +# so a reshape that keeps only the `name` 404s offline. Materialize a copy under each url. +materialize_module_urls() { + log "materialize GMM urls" + python3 - "$OUT" <<'PY' +import json, os, sys, shutil +root = sys.argv[1]; made = 0 +for dp, _, files in os.walk(root): + for fn in files: + if not fn.endswith('.module'): continue + try: + with open(os.path.join(dp, fn), encoding='utf-8') as f: mod = json.load(f) + except Exception: continue + for var in mod.get('variants', []): + for fe in var.get('files', []): + name, url = fe.get('name'), fe.get('url') + if not name or not url or name == url: continue + src, dst = os.path.join(dp, name), os.path.join(dp, url) + if os.path.exists(src) and not os.path.exists(dst): + shutil.copyfile(src, dst); made += 1 +print(f"materialized {made} url-named copies") +PY +} + # sha1 + md5 beside every artifact (Gradle validates .sha1 on download). write_checksums() { log "checksums" @@ -189,6 +218,7 @@ main() { process_project "$name" "$source" "$ref" "$gradle_root" "$mode" done < "$PROJECTS_TSV" fetch_extras + materialize_module_urls write_checksums report [ "$DO_SMOKE" = 1 ] && smoke_test diff --git a/static/maven-offline/config/extra-artifacts.tsv b/static/maven-offline/config/extra-artifacts.tsv index 7187e0318..556a5f57f 100644 --- a/static/maven-offline/config/extra-artifacts.tsv +++ b/static/maven-offline/config/extra-artifacts.tsv @@ -6,9 +6,11 @@ com.aayushatharva.brotli4j native-linux-x86_64 1.18.0 - com.aayushatharva.brotli4j native-linux-aarch64 1.18.0 - jar linux com.aayushatharva.brotli4j native-windows-x86_64 1.18.0 - jar windows # -# aapt2 and R8/D8 are NOT listed here because their versions follow each project's -# AGP version, so the script derives them per project (see derive_aapt2_r8 in -# build-maven-offline.sh): +# aapt2 version follows each project's AGP version, so the script derives it per project +# (see derive_aapt2_r8 in build-maven-offline.sh): # com.android.tools.build:aapt2:: (per build-host OS; ARM on-device uses Code on the Go's own aapt2) -# com.android.tools:r8: (architecture-neutral) -# AGP versions in use: 8.4.1 (Knowledge to Go), 8.8.2 + 7.3.0 tooling (Code on the Go), 8.11.0 (add-ons). +# R8/D8: no separate artifact for the add-ons' AGP 9.3.1 : it ships inside +# com.android.tools.build:builder:9.3.1, which a normal resolve captures. Older AGP +# (8.4 K2Go, 8.8 CoGo) may reference com.android.tools:r8 separately; pin it via an +# offline assemble oracle if/when the full offline BUILD of those ships (separate ticket). +# AGP versions in use: 8.4.1 (Knowledge to Go), 8.8.2 (Code on the Go), 9.3.1 (add-ons). diff --git a/static/maven-offline/gradle/offline-repo.init.gradle b/static/maven-offline/gradle/offline-repo.init.gradle index 1a5f2e5c2..8c3b3d018 100644 --- a/static/maven-offline/gradle/offline-repo.init.gradle +++ b/static/maven-offline/gradle/offline-repo.init.gradle @@ -14,6 +14,13 @@ if (repoPath == null) throw new GradleException('set -Dmavenoffline.repo= + settings.buildscript.repositories { clear(); maven { url = repoUri } } +} + settingsEvaluated { settings -> try { def mode = settings.dependencyResolutionManagement.repositoriesMode.getOrNull() From de2fe718fbb6c12d36d2a32672d418ab7b50fc04 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 1 Oct 2026 09:27:24 -0600 Subject: [PATCH 3/5] K2GO-437 refactor(maven-offline): derive aapt2 AGP versions from the repo, no hardcoded list The aapt2 set now follows the com.android.tools.build:gradle versions the resolve actually produced, so a project changing its AGP (e.g. K2Go 8.4 -> 8.8) needs no edit: re-running the tool adjusts the toolchains and the aapt2 set automatically. --- static/maven-offline/build-maven-offline.sh | 25 +++++++++++---------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/static/maven-offline/build-maven-offline.sh b/static/maven-offline/build-maven-offline.sh index a3bd90db8..4ddc312ac 100644 --- a/static/maven-offline/build-maven-offline.sh +++ b/static/maven-offline/build-maven-offline.sh @@ -133,23 +133,24 @@ fetch_extras() { # aapt2 (Linux + Windows) and R8 follow each project's AGP version. Resolve the exact # aapt2 version from Google Maven metadata for each AGP in use, then fetch the host jars. derive_aapt2_r8() { - # AGP in use: 8.4.1 (K2Go), 8.8.2 (CoGo), 9.3.1 (add-ons). For AGP 9.x the R8/D8 - # tool ships inside com.android.tools.build:builder, so a resolve already captures it; - # only aapt2 (per build-host OS) must be added here. - local agp - for agp in 8.4.1 8.8.2 9.3.1; do - local meta="$WORK/aapt2-metadata.xml" - fetch "$GOOGLE_MAVEN/com/android/tools/build/aapt2/maven-metadata.xml" "$meta" || continue - local ver + # Fully data-driven: the AGP versions are whatever the resolve actually pulled, i.e. the + # com.android.tools.build:gradle version dirs now in the repo. No hardcoded list, so a + # project moving (e.g. K2Go 8.4 -> 8.8) needs no edit here: the aapt2 set follows. + # For AGP 9.x the R8/D8 tool ships inside com.android.tools.build:builder (a resolve + # captures it); older AGP may reference com.android.tools:r8 separately, added via the + # offline assemble oracle when the full offline BUILD of those ships (separate ticket). + local gdir="$OUT/com/android/tools/build/gradle" + [ -d "$gdir" ] || { log "no AGP in repo yet; skipping aapt2"; return; } + local meta="$WORK/aapt2-metadata.xml" + fetch "$GOOGLE_MAVEN/com/android/tools/build/aapt2/maven-metadata.xml" "$meta" || return + local agp ver + for agp in $(ls "$gdir" 2>/dev/null); do ver="$(grep -oE "${agp//./\\.}-[0-9]+" "$meta" | sed -E 's:::g' | tail -1)" - [ -z "$ver" ] && { log "aapt2: no version for AGP $agp"; continue; } + [ -z "$ver" ] && { log "aapt2: no published version for AGP $agp"; continue; } log "aapt2 for AGP $agp -> $ver (linux, windows)" place_artifact "$GOOGLE_MAVEN" com.android.tools.build aapt2 "$ver" linux jar place_artifact "$GOOGLE_MAVEN" com.android.tools.build aapt2 "$ver" windows jar done - # R8 is architecture-neutral. If the resolve already captured it, nothing to do; the - # smoke test confirms whether a specific r8 version is missing. TODO: pin r8 per AGP - # once the smoke test names the exact version a build requests. } # Gradle Module Metadata (.module) can declare a file whose served `url` differs from the From 617f58afba10d7bc7b88e5514b6d92b120d16e91 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 1 Oct 2026 11:17:54 -0600 Subject: [PATCH 4/5] K2GO-437 fix(maven-offline): tab-separated configs, GMM relative urls, reuse-resolve mode The config TSVs were space-aligned but read with IFS tab, so projects.tsv and extra-artifacts.tsv parsed into one field (a full-tool run sent malformed extra URLs). Use real tabs. materialize_module_urls now normalizes a relative GMM url (a relocation like guava -android -> ../-jre) and creates its parent dir before copying. Add --no-resolve (rebuild the repo from existing Gradle homes) and --out, and point the add-ons gradle_root at plugins/ (the real addons repo layout). A full --no-resolve run reproduces the repo with zero artifacts missing. --- static/maven-offline/build-maven-offline.sh | 33 +++++++++++++++---- .../maven-offline/config/extra-artifacts.tsv | 26 +++++++-------- static/maven-offline/config/projects.tsv | 25 +++++++------- 3 files changed, 49 insertions(+), 35 deletions(-) diff --git a/static/maven-offline/build-maven-offline.sh b/static/maven-offline/build-maven-offline.sh index 4ddc312ac..71b38b48b 100644 --- a/static/maven-offline/build-maven-offline.sh +++ b/static/maven-offline/build-maven-offline.sh @@ -27,6 +27,7 @@ MAVEN_CENTRAL="https://repo1.maven.org/maven2" GOOGLE_MAVEN="https://dl.google.com/dl/android/maven2" DO_SMOKE=0 +NO_RESOLVE=0 ONLY="" usage() { sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'; } @@ -34,6 +35,8 @@ usage() { sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'; } while [ $# -gt 0 ]; do case "$1" in --smoke) DO_SMOKE=1 ;; + --no-resolve) NO_RESOLVE=1 ;; + --out) OUT="${2:-}"; shift ;; --only) ONLY="${2:-}"; shift ;; -h|--help) usage; exit 0 ;; *) echo "unknown arg: $1" >&2; exit 2 ;; @@ -172,8 +175,11 @@ for dp, _, files in os.walk(root): for fe in var.get('files', []): name, url = fe.get('name'), fe.get('url') if not name or not url or name == url: continue - src, dst = os.path.join(dp, name), os.path.join(dp, url) + # url may be relative with ../ (GMM relocations point to a sibling version dir) + src = os.path.join(dp, name) + dst = os.path.normpath(os.path.join(dp, url)) if os.path.exists(src) and not os.path.exists(dst): + os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.copyfile(src, dst); made += 1 print(f"materialized {made} url-named copies") PY @@ -211,13 +217,28 @@ process_project() { fi } +# Rebuild the repo from already-resolved homes, skipping Gradle. Lets you re-run the +# reshape / extras / checksums after a tool change without re-downloading, and reuse a +# resolve done elsewhere (drop its Gradle home under .gradle-homes/). +reshape_all_homes() { + local home + for home in "$HOMES"/*/; do + [ -d "$home/caches/modules-2/files-2.1" ] || continue + reshape_home "${home%/}" + done +} + main() { mkdir -p "$OUT" "$HOMES" - while IFS=$'\t' read -r name source ref gradle_root mode; do - [ -z "${name:-}" ] && continue - case "$name" in \#*) continue ;; esac - process_project "$name" "$source" "$ref" "$gradle_root" "$mode" - done < "$PROJECTS_TSV" + if [ "$NO_RESOLVE" = 1 ]; then + reshape_all_homes + else + while IFS=$'\t' read -r name source ref gradle_root mode; do + [ -z "${name:-}" ] && continue + case "$name" in \#*) continue ;; esac + process_project "$name" "$source" "$ref" "$gradle_root" "$mode" + done < "$PROJECTS_TSV" + fi fetch_extras materialize_module_urls write_checksums diff --git a/static/maven-offline/config/extra-artifacts.tsv b/static/maven-offline/config/extra-artifacts.tsv index 556a5f57f..3214785bd 100644 --- a/static/maven-offline/config/extra-artifacts.tsv +++ b/static/maven-offline/config/extra-artifacts.tsv @@ -1,16 +1,12 @@ # Task-time artifacts a plain dependency resolve does NOT pull, but an offline build -# still needs. One per line, tab-separated. Lines starting with # are ignored. -# -# group artifact version classifier ext hosts -com.aayushatharva.brotli4j native-linux-x86_64 1.18.0 - jar linux -com.aayushatharva.brotli4j native-linux-aarch64 1.18.0 - jar linux -com.aayushatharva.brotli4j native-windows-x86_64 1.18.0 - jar windows -# -# aapt2 version follows each project's AGP version, so the script derives it per project -# (see derive_aapt2_r8 in build-maven-offline.sh): -# com.android.tools.build:aapt2:: (per build-host OS; ARM on-device uses Code on the Go's own aapt2) -# R8/D8: no separate artifact for the add-ons' AGP 9.3.1 : it ships inside -# com.android.tools.build:builder:9.3.1, which a normal resolve captures. Older AGP -# (8.4 K2Go, 8.8 CoGo) may reference com.android.tools:r8 separately; pin it via an -# offline assemble oracle if/when the full offline BUILD of those ships (separate ticket). -# AGP versions in use: 8.4.1 (Knowledge to Go), 8.8.2 (Code on the Go), 9.3.1 (add-ons). +# still needs. Tab-separated (TSV). Lines starting with # are ignored. +# Columns: groupartifactversionclassifierexthosts (classifier "-" = none) +com.aayushatharva.brotli4j native-linux-x86_64 1.18.0 - jar linux +com.aayushatharva.brotli4j native-linux-aarch64 1.18.0 - jar linux +com.aayushatharva.brotli4j native-windows-x86_64 1.18.0 - jar windows +# aapt2 is NOT listed: its version follows each project's AGP version, so the script +# derives it from the com.android.tools.build:gradle versions actually in the repo (see +# derive_aapt2_r8). R8/D8 for AGP 9.x ships inside com.android.tools.build:builder, which a +# resolve captures; older AGP (8.4 K2Go, 8.8 CoGo) may reference com.android.tools:r8 +# separately, added via an offline assemble oracle when their full offline BUILD ships. +# AGP in use: 8.4.1 (Knowledge to Go), 8.8.2 (Code on the Go), 9.3.1 (add-ons). diff --git a/static/maven-offline/config/projects.tsv b/static/maven-offline/config/projects.tsv index eb1e9fcae..d2b753dbe 100644 --- a/static/maven-offline/config/projects.tsv +++ b/static/maven-offline/config/projects.tsv @@ -1,16 +1,13 @@ # The code bases whose build dependencies go into the offline repository. -# One per line, tab-separated. Lines starting with # are ignored. -# -# name : short id, used for the isolated Gradle home dir name. -# source : "self" = the repository that contains this tool; otherwise a git URL to clone. -# ref : branch or tag to resolve. -# gradle_root : path to the Gradle root (where settings.gradle lives), relative to the repo root. -# mode : "root" = one Gradle build at gradle_root; "multi" = each immediate subdir of gradle_root is its own Gradle build (the add-ons). -# -# Local runs: to resolve from a local clone instead of cloning the git URL, add a line +# Tab-separated (TSV). Lines starting with # are ignored. Columns: +# name short id, used for the isolated Gradle home dir name. +# source "self" = the repo that contains this tool; otherwise a git URL to clone. +# ref branch or tag to resolve. +# gradle_root path to the Gradle root (where settings.gradle lives), relative to the repo root. +# mode "root" = one Gradle build at gradle_root; "multi" = each immediate subdir +# of gradle_root is its own Gradle build (the add-ons under addons/plugins). +# Local runs: to resolve from a local clone instead of the git URL, add a tab-separated line # to config/sources.local.tsv (gitignored): -# -# name source ref gradle_root mode -knowledge-to-go self main controller root -code-on-the-go https://github.com/appdevforall/codeonthego stage . root -add-ons https://github.com/appdevforall/addons main . multi +knowledge-to-go self main controller root +code-on-the-go https://github.com/appdevforall/codeonthego stage . root +add-ons https://github.com/appdevforall/addons main plugins multi From 04adab5c64d270680f5cd68929a25421c4d89f47 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 1 Oct 2026 11:39:14 -0600 Subject: [PATCH 5/5] K2GO-437 fix(maven-offline): address code-review findings Degrade (not abort under set -e) when the aapt2 metadata fetch fails; make a missing per-build gradlew a loud WARN skip instead of a swallowed failure; guard --out/--only against a missing value; iterate AGP dirs with a glob instead of parsing ls; rename derive_aapt2_r8 to fetch_aapt2 (R8/D8 ships inside builder). --- static/maven-offline/build-maven-offline.sh | 38 ++++++++++++--------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/static/maven-offline/build-maven-offline.sh b/static/maven-offline/build-maven-offline.sh index 71b38b48b..c552b62ae 100644 --- a/static/maven-offline/build-maven-offline.sh +++ b/static/maven-offline/build-maven-offline.sh @@ -36,8 +36,8 @@ while [ $# -gt 0 ]; do case "$1" in --smoke) DO_SMOKE=1 ;; --no-resolve) NO_RESOLVE=1 ;; - --out) OUT="${2:-}"; shift ;; - --only) ONLY="${2:-}"; shift ;; + --out) OUT="${2:?--out requires a value}"; shift ;; + --only) ONLY="${2:?--only requires a value}"; shift ;; -h|--help) usage; exit 0 ;; *) echo "unknown arg: $1" >&2; exit 2 ;; esac @@ -78,6 +78,12 @@ source_dir() { resolve_build() { local gradle_root="$1" home="$2" gw gw="$(gradlew_for "$gradle_root")" + # Make a missing wrapper a loud skip, not a swallowed failure: some add-ons in the + # addons repo ship no per-build gradlew, which would otherwise drop their deps silently. + if [ ! -f "$gw" ]; then + log "WARN: no gradlew at $gradle_root; skipping (this build has no own wrapper)" + return 0 + fi log "resolve: $gradle_root (home ${home##*/})" ( cd "$gradle_root" && "$gw" --gradle-user-home "$home" \ --init-script "$INIT" --no-daemon --console=plain -q \ @@ -130,24 +136,24 @@ fetch_extras() { case "$group" in \#*) continue ;; esac place_artifact "$MAVEN_CENTRAL" "$group" "$artifact" "$version" "$classifier" "$ext" done < "$EXTRAS_TSV" - derive_aapt2_r8 + fetch_aapt2 } -# aapt2 (Linux + Windows) and R8 follow each project's AGP version. Resolve the exact -# aapt2 version from Google Maven metadata for each AGP in use, then fetch the host jars. -derive_aapt2_r8() { - # Fully data-driven: the AGP versions are whatever the resolve actually pulled, i.e. the - # com.android.tools.build:gradle version dirs now in the repo. No hardcoded list, so a - # project moving (e.g. K2Go 8.4 -> 8.8) needs no edit here: the aapt2 set follows. - # For AGP 9.x the R8/D8 tool ships inside com.android.tools.build:builder (a resolve - # captures it); older AGP may reference com.android.tools:r8 separately, added via the - # offline assemble oracle when the full offline BUILD of those ships (separate ticket). +# aapt2 (Linux + Windows) follows each project's AGP version. R8/D8 is NOT fetched here: +# for AGP 9.x it ships inside com.android.tools.build:builder, which a resolve captures. +fetch_aapt2() { + # Data-driven: the AGP versions are the com.android.tools.build:gradle dirs the resolve + # produced, so a project moving (e.g. K2Go 8.4 -> 8.8) needs no edit here. aapt2 is + # fetched for every AGP present (a few MB each): over-fetching is safe, under-fetching + # would break an offline build. local gdir="$OUT/com/android/tools/build/gradle" - [ -d "$gdir" ] || { log "no AGP in repo yet; skipping aapt2"; return; } + [ -d "$gdir" ] || { log "no AGP in repo yet; skipping aapt2"; return 0; } local meta="$WORK/aapt2-metadata.xml" - fetch "$GOOGLE_MAVEN/com/android/tools/build/aapt2/maven-metadata.xml" "$meta" || return - local agp ver - for agp in $(ls "$gdir" 2>/dev/null); do + fetch "$GOOGLE_MAVEN/com/android/tools/build/aapt2/maven-metadata.xml" "$meta" \ + || { log "aapt2: metadata unavailable; skipping aapt2"; return 0; } + local d agp ver + for d in "$gdir"/*/; do + agp="$(basename "$d")" ver="$(grep -oE "${agp//./\\.}-[0-9]+" "$meta" | sed -E 's:::g' | tail -1)" [ -z "$ver" ] && { log "aapt2: no published version for AGP $agp"; continue; } log "aapt2 for AGP $agp -> $ver (linux, windows)"