diff --git a/static/maven-offline/.gitignore b/static/maven-offline/.gitignore new file mode 100644 index 000000000..ff4dec766 --- /dev/null +++ b/static/maven-offline/.gitignore @@ -0,0 +1,10 @@ +# Generated output: the Maven2 repository tree (about 1.1 to 1.2 GiB). Never commit. +/out/ +# Per-run isolated Gradle user homes used during resolution. +/.gradle-homes/ +# Shallow clones of Code on the Go / add-ons made by the producer. +/.work/ +# Local source overrides (absolute paths on a developer machine). +/config/sources.local.tsv +# Logs. +*.log diff --git a/static/maven-offline/README.md b/static/maven-offline/README.md new file mode 100644 index 000000000..59903846a --- /dev/null +++ b/static/maven-offline/README.md @@ -0,0 +1,60 @@ +# maven-offline (K2GO-437) + +A download machine. It builds one offline Maven repository that holds every +dependency the builds need: Knowledge to Go, Code on the Go, and the add-ons. +A device or a laptop on the K2Go local network then builds those projects with +the network off. + +This folder is the PRODUCER tool only. The Ansible role that installs and serves +the repository, and the Code on the Go consumer setting, are separate tickets. + +## Contract + +- Output layout: standard Maven2 (`///` with + `.sha1`/`.md5`). This is what nginx serves and what a Gradle `maven { url ... }` + repository consumes. It is NOT the Gradle internal cache layout. +- Served over HTTP by the box nginx at a new path, for example + `http://:8085/maven-offline/`. No new port: it is one more path. +- One shared repository for all three projects (union). About 1.1 to 1.2 GiB. + Separate per-project repositories are not worth it. +- Architecture: about 99.8 percent of the repository is architecture-neutral JVM + bytecode, so one repository serves ARM devices and desktop build hosts. Only + `aapt2` and `brotli4j` have per-OS files. +- Build hosts in scope: on-device (ARM Android), Linux, Windows. MacOS deferred. + On-device (ARM) does not need the Maven `aapt2`: Code on the Go ships its own. + +## Pipeline + +1. Resolve. For each project, run `resolveAllDeps` (see `gradle/resolve-all.init.gradle`) + against an isolated Gradle user home. This forces a download of every resolvable + configuration plus the buildscript/plugin classpath into that home's module cache. +2. Reshape. Convert the module cache (`caches/modules-2/files-2.1`, content-addressed) + into Maven2 layout under `out/repo`. +3. Extras. Add the task-time tools a plain resolve misses: R8/D8, `aapt2` for Linux + and Windows, `brotli4j` native for Linux and Windows. See `config/extra-artifacts.tsv`. +4. Union and checksums. Merge all three into one tree (dedup by Maven path) and write + `.sha1`/`.md5` for every file. +5. Smoke test. Build a target with the network off, using only `out/repo` as the single + repository. This is the acceptance proof that the repository is complete. + +## Why not the suggested plugin + +The suggested `io.github.yubyf.maven-offline` 1.0.4 writes zero artifacts on our +Gradle versions (8.8 and 8.14): it reports "No effective repositories found" and +skips the download. So the producer uses direct Gradle resolution instead. + +## Usage + + ./build-maven-offline.sh # resolve + reshape + extras + checksums + ./build-maven-offline.sh --smoke # also run the offline build smoke test + +Notes: +- Run on an online host (the producer must reach Maven Central and Google Maven). +- Linux is the canonical producer host (CI). On Windows, run from a shell where a + Gradle daemon can open a loopback socket; `--no-daemon` is used to avoid that. +- `out/` and the per-run Gradle homes are generated, not committed (see `.gitignore`). + +## Sizes and method + +See the local study `maven-offline-study/REPORT.md` for the measured sizes, the +common trunk, and the per-OS slivers. diff --git a/static/maven-offline/build-maven-offline.sh b/static/maven-offline/build-maven-offline.sh new file mode 100644 index 000000000..c552b62ae --- /dev/null +++ b/static/maven-offline/build-maven-offline.sh @@ -0,0 +1,286 @@ +#!/usr/bin/env bash +# +# maven-offline: the download machine (K2GO-437). +# +# Builds one offline Maven2 repository under out/repo that holds every dependency +# the builds of Knowledge to Go, Code on the Go, and the add-ons need. A device or +# a laptop on the K2Go network then builds those projects with the network off. +# +# Pipeline: resolve (per project, isolated Gradle home) -> reshape (Gradle cache to +# Maven2 layout) -> extras (R8/D8, aapt2, brotli4j) -> checksums -> report -> smoke. +# +# Run on an online host. Linux is the canonical producer host. See README.md. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/../.." && pwd)" +OUT="$HERE/out/repo" +HOMES="$HERE/.gradle-homes" +WORK="$HERE/.work" +INIT="$HERE/gradle/resolve-all.init.gradle" +PROJECTS_TSV="$HERE/config/projects.tsv" +EXTRAS_TSV="$HERE/config/extra-artifacts.tsv" +LOCAL_SRC="$HERE/config/sources.local.tsv" + +MAVEN_CENTRAL="https://repo1.maven.org/maven2" +GOOGLE_MAVEN="https://dl.google.com/dl/android/maven2" + +DO_SMOKE=0 +NO_RESOLVE=0 +ONLY="" + +usage() { sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'; } + +while [ $# -gt 0 ]; do + case "$1" in + --smoke) DO_SMOKE=1 ;; + --no-resolve) NO_RESOLVE=1 ;; + --out) OUT="${2:?--out requires a value}"; shift ;; + --only) ONLY="${2:?--only requires a value}"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac + shift +done + +log() { printf '>> %s\n' "$*"; } + +# gradlew wrapper for a project dir, picking the Windows launcher under Git Bash. +gradlew_for() { + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) printf '%s/gradlew.bat' "$1" ;; + *) printf '%s/gradlew' "$1" ;; + esac +} + +# Resolve a local path override for a project name (config/sources.local.tsv), else "". +local_override() { + [ -f "$LOCAL_SRC" ] || { printf ''; return; } + awk -F'\t' -v n="$1" '!/^#/ && $1==n {print $2; exit}' "$LOCAL_SRC" +} + +# Clone (shallow) or point at a local checkout; echoes the source dir. +source_dir() { + local name="$1" source="$2" ref="$3" ovr + ovr="$(local_override "$name")" + if [ -n "$ovr" ]; then printf '%s' "$ovr"; return; fi + if [ "$source" = "self" ]; then printf '%s' "$REPO_ROOT"; return; fi + local dst="$WORK/$name" + if [ ! -d "$dst/.git" ]; then + mkdir -p "$WORK" + git clone --depth 1 --branch "$ref" "$source" "$dst" >&2 + fi + printf '%s' "$dst" +} + +# Force-download one Gradle build's dependencies into an isolated home. +resolve_build() { + local gradle_root="$1" home="$2" gw + gw="$(gradlew_for "$gradle_root")" + # Make a missing wrapper a loud skip, not a swallowed failure: some add-ons in the + # addons repo ship no per-build gradlew, which would otherwise drop their deps silently. + if [ ! -f "$gw" ]; then + log "WARN: no gradlew at $gradle_root; skipping (this build has no own wrapper)" + return 0 + fi + log "resolve: $gradle_root (home ${home##*/})" + ( cd "$gradle_root" && "$gw" --gradle-user-home "$home" \ + --init-script "$INIT" --no-daemon --console=plain -q \ + resolveAllDeps ) || log "resolve returned nonzero (lenient; continuing)" +} + +# Copy one isolated home's module cache into out/repo as Maven2 layout. +# Gradle cache: files-2.1///// +# Maven2: /// +reshape_home() { + local home="$1" cache="$1/caches/modules-2/files-2.1" n=0 + [ -d "$cache" ] || { log "no cache in ${home##*/}"; return; } + while IFS= read -r f; do + local rel="${f#"$cache"/}" + local group="${rel%%/*}"; rel="${rel#*/}" + local artifact="${rel%%/*}"; rel="${rel#*/}" + local version="${rel%%/*}"; rel="${rel#*/}" + local file="${rel##*/}" + local dest="$OUT/${group//.//}/$artifact/$version" + mkdir -p "$dest" + [ -f "$dest/$file" ] || cp "$f" "$dest/$file" + n=$((n+1)) + done < <(find "$cache" -type f) + log "reshaped ${home##*/}: $n files" +} + +# Download to (skip if present and non-empty). +fetch() { + local url="$1" dest="$2" + [ -s "$dest" ] && return 0 + mkdir -p "$(dirname "$dest")" + curl -fsSL "$url" -o "$dest" && return 0 + rm -f "$dest"; log "miss: $url"; return 1 +} + +# Place one GAV artifact (jar + pom) from a base repo into out/repo. +place_artifact() { + local base="$1" group="$2" artifact="$3" version="$4" classifier="$5" ext="$6" + local gpath="${group//.//}/$artifact/$version" + local name="$artifact-$version"; [ "$classifier" != "-" ] && name="$name-$classifier" + fetch "$base/$gpath/$name.$ext" "$OUT/$gpath/$name.$ext" || true + fetch "$base/$gpath/$artifact-$version.pom" "$OUT/$gpath/$artifact-$version.pom" || true +} + +# Extras a plain resolve misses: brotli4j natives (config) + aapt2/R8 (derived). +fetch_extras() { + log "extras: brotli4j natives" + while IFS=$'\t' read -r group artifact version classifier ext hosts; do + [ -z "${group:-}" ] && continue + case "$group" in \#*) continue ;; esac + place_artifact "$MAVEN_CENTRAL" "$group" "$artifact" "$version" "$classifier" "$ext" + done < "$EXTRAS_TSV" + fetch_aapt2 +} + +# aapt2 (Linux + Windows) follows each project's AGP version. R8/D8 is NOT fetched here: +# for AGP 9.x it ships inside com.android.tools.build:builder, which a resolve captures. +fetch_aapt2() { + # Data-driven: the AGP versions are the com.android.tools.build:gradle dirs the resolve + # produced, so a project moving (e.g. K2Go 8.4 -> 8.8) needs no edit here. aapt2 is + # fetched for every AGP present (a few MB each): over-fetching is safe, under-fetching + # would break an offline build. + local gdir="$OUT/com/android/tools/build/gradle" + [ -d "$gdir" ] || { log "no AGP in repo yet; skipping aapt2"; return 0; } + local meta="$WORK/aapt2-metadata.xml" + fetch "$GOOGLE_MAVEN/com/android/tools/build/aapt2/maven-metadata.xml" "$meta" \ + || { log "aapt2: metadata unavailable; skipping aapt2"; return 0; } + local d agp ver + for d in "$gdir"/*/; do + agp="$(basename "$d")" + ver="$(grep -oE "${agp//./\\.}-[0-9]+" "$meta" | sed -E 's:::g' | tail -1)" + [ -z "$ver" ] && { log "aapt2: no published version for AGP $agp"; continue; } + log "aapt2 for AGP $agp -> $ver (linux, windows)" + place_artifact "$GOOGLE_MAVEN" com.android.tools.build aapt2 "$ver" linux jar + place_artifact "$GOOGLE_MAVEN" com.android.tools.build aapt2 "$ver" windows jar + done +} + +# Gradle Module Metadata (.module) can declare a file whose served `url` differs from the +# cache `name` (KMP androidx -android AARs: cache name lifecycle-runtime-release.aar, url +# lifecycle-runtime-android-.aar). A Maven2 consumer reading the .module fetches by url, +# so a reshape that keeps only the `name` 404s offline. Materialize a copy under each url. +materialize_module_urls() { + log "materialize GMM urls" + python3 - "$OUT" <<'PY' +import json, os, sys, shutil +root = sys.argv[1]; made = 0 +for dp, _, files in os.walk(root): + for fn in files: + if not fn.endswith('.module'): continue + try: + with open(os.path.join(dp, fn), encoding='utf-8') as f: mod = json.load(f) + except Exception: continue + for var in mod.get('variants', []): + for fe in var.get('files', []): + name, url = fe.get('name'), fe.get('url') + if not name or not url or name == url: continue + # url may be relative with ../ (GMM relocations point to a sibling version dir) + src = os.path.join(dp, name) + dst = os.path.normpath(os.path.join(dp, url)) + if os.path.exists(src) and not os.path.exists(dst): + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copyfile(src, dst); made += 1 +print(f"materialized {made} url-named copies") +PY +} + +# sha1 + md5 beside every artifact (Gradle validates .sha1 on download). +write_checksums() { + log "checksums" + find "$OUT" -type f ! -name '*.sha1' ! -name '*.md5' | while IFS= read -r f; do + [ -f "$f.sha1" ] || sha1sum "$f" | cut -d' ' -f1 > "$f.sha1" + [ -f "$f.md5" ] || md5sum "$f" | cut -d' ' -f1 > "$f.md5" + done +} + +report() { + log "repository: $OUT" + log "size: $(du -sh "$OUT" | cut -f1) files: $(find "$OUT" -type f | wc -l)" +} + +process_project() { + local name="$1" source="$2" ref="$3" gradle_root="$4" mode="$5" + [ -n "$ONLY" ] && [ "$ONLY" != "$name" ] && return + local src; src="$(source_dir "$name" "$source" "$ref")" + local root="$src/$gradle_root" + if [ "$mode" = "multi" ]; then + local sub + for sub in "$root"/*/; do + [ -e "$sub/settings.gradle" ] || [ -e "$sub/settings.gradle.kts" ] || continue + resolve_build "${sub%/}" "$HOMES/$name-$(basename "$sub")" + reshape_home "$HOMES/$name-$(basename "$sub")" + done + else + resolve_build "$root" "$HOMES/$name" + reshape_home "$HOMES/$name" + fi +} + +# Rebuild the repo from already-resolved homes, skipping Gradle. Lets you re-run the +# reshape / extras / checksums after a tool change without re-downloading, and reuse a +# resolve done elsewhere (drop its Gradle home under .gradle-homes/). +reshape_all_homes() { + local home + for home in "$HOMES"/*/; do + [ -d "$home/caches/modules-2/files-2.1" ] || continue + reshape_home "${home%/}" + done +} + +main() { + mkdir -p "$OUT" "$HOMES" + if [ "$NO_RESOLVE" = 1 ]; then + reshape_all_homes + else + while IFS=$'\t' read -r name source ref gradle_root mode; do + [ -z "${name:-}" ] && continue + case "$name" in \#*) continue ;; esac + process_project "$name" "$source" "$ref" "$gradle_root" "$mode" + done < "$PROJECTS_TSV" + fi + fetch_extras + materialize_module_urls + write_checksums + report + [ "$DO_SMOKE" = 1 ] && smoke_test + log "done" +} + +# Prove each root project resolves from out/repo with the network off. This is the +# acceptance proof that the repository is complete. It reuses each project's already +# downloaded Gradle distribution so --offline needs no network for the wrapper itself. +smoke_test() { + local offline_init="$HERE/gradle/offline-repo.init.gradle" + local repo; repo="$(cd "$OUT" && pwd)" + while IFS=$'\t' read -r name source ref gradle_root mode; do + [ -z "${name:-}" ] && continue + case "$name" in \#*) continue ;; esac + [ -n "$ONLY" ] && [ "$ONLY" != "$name" ] && continue + [ "$mode" = "multi" ] && continue # the add-ons share the trunk; smoke the roots + local src root warm smoke gw + src="$(source_dir "$name" "$source" "$ref")" + root="$src/$gradle_root" + warm="$HOMES/$name"; smoke="$HOMES/$name-offline" + rm -rf "$smoke"; mkdir -p "$smoke" + [ -d "$warm/wrapper" ] && cp -r "$warm/wrapper" "$smoke/wrapper" + gw="$(gradlew_for "$root")" + log "smoke (offline): $name" + if ( cd "$root" && "$gw" --gradle-user-home "$smoke" --offline \ + "-Dmavenoffline.repo=$repo" \ + --init-script "$offline_init" --init-script "$INIT" \ + --no-daemon --console=plain -q resolveAllDeps ); then + log "smoke OK: $name resolves with the network off" + else + log "smoke FAIL: $name has missing artifacts in out/repo"; return 1 + fi + done < "$PROJECTS_TSV" +} + +main diff --git a/static/maven-offline/config/extra-artifacts.tsv b/static/maven-offline/config/extra-artifacts.tsv new file mode 100644 index 000000000..3214785bd --- /dev/null +++ b/static/maven-offline/config/extra-artifacts.tsv @@ -0,0 +1,12 @@ +# Task-time artifacts a plain dependency resolve does NOT pull, but an offline build +# still needs. Tab-separated (TSV). Lines starting with # are ignored. +# Columns: groupartifactversionclassifierexthosts (classifier "-" = none) +com.aayushatharva.brotli4j native-linux-x86_64 1.18.0 - jar linux +com.aayushatharva.brotli4j native-linux-aarch64 1.18.0 - jar linux +com.aayushatharva.brotli4j native-windows-x86_64 1.18.0 - jar windows +# aapt2 is NOT listed: its version follows each project's AGP version, so the script +# derives it from the com.android.tools.build:gradle versions actually in the repo (see +# derive_aapt2_r8). R8/D8 for AGP 9.x ships inside com.android.tools.build:builder, which a +# resolve captures; older AGP (8.4 K2Go, 8.8 CoGo) may reference com.android.tools:r8 +# separately, added via an offline assemble oracle when their full offline BUILD ships. +# AGP in use: 8.4.1 (Knowledge to Go), 8.8.2 (Code on the Go), 9.3.1 (add-ons). diff --git a/static/maven-offline/config/projects.tsv b/static/maven-offline/config/projects.tsv new file mode 100644 index 000000000..d2b753dbe --- /dev/null +++ b/static/maven-offline/config/projects.tsv @@ -0,0 +1,13 @@ +# The code bases whose build dependencies go into the offline repository. +# Tab-separated (TSV). Lines starting with # are ignored. Columns: +# name short id, used for the isolated Gradle home dir name. +# source "self" = the repo that contains this tool; otherwise a git URL to clone. +# ref branch or tag to resolve. +# gradle_root path to the Gradle root (where settings.gradle lives), relative to the repo root. +# mode "root" = one Gradle build at gradle_root; "multi" = each immediate subdir +# of gradle_root is its own Gradle build (the add-ons under addons/plugins). +# Local runs: to resolve from a local clone instead of the git URL, add a tab-separated line +# to config/sources.local.tsv (gitignored): +knowledge-to-go self main controller root +code-on-the-go https://github.com/appdevforall/codeonthego stage . root +add-ons https://github.com/appdevforall/addons main plugins multi diff --git a/static/maven-offline/gradle/offline-repo.init.gradle b/static/maven-offline/gradle/offline-repo.init.gradle new file mode 100644 index 000000000..8c3b3d018 --- /dev/null +++ b/static/maven-offline/gradle/offline-repo.init.gradle @@ -0,0 +1,41 @@ +// Point every repository (plugins and dependencies) at the offline Maven2 tree only. +// Used with resolve-all.init.gradle and --offline to prove a build resolves with the +// network off. Pass the repo path with -Dmavenoffline.repo=. +// +// Two repository styles must both work: +// - settings repositories (pluginManagement + dependencyResolutionManagement). Code on +// the Go uses dependencyResolutionManagement with FAIL_ON_PROJECT_REPOS, so adding a +// PROJECT repository there is an error. We override the settings repositories instead. +// - project repositories (older style, e.g. Knowledge to Go). We override those too, but +// only when the build does not forbid project repositories. + +def repoPath = System.getProperty('mavenoffline.repo') +if (repoPath == null) throw new GradleException('set -Dmavenoffline.repo=') +def repoUri = new File(repoPath).toURI() +def allowProjectRepos = true + +// The add-ons declare the AGP + Kotlin classpath in a settings-level buildscript{} block +// that hardcodes google()/mavenCentral(). Redirect it (before the settings script runs) +// so the plugin classpath resolves from the offline repo too. +gradle.beforeSettings { settings -> + settings.buildscript.repositories { clear(); maven { url = repoUri } } +} + +settingsEvaluated { settings -> + try { + def mode = settings.dependencyResolutionManagement.repositoriesMode.getOrNull() + if (mode != null && mode.name() == 'FAIL_ON_PROJECT_REPOS') allowProjectRepos = false + } catch (ignored) { /* no dependencyResolutionManagement */ } + settings.pluginManagement.repositories { clear(); maven { url = repoUri } } + try { + settings.dependencyResolutionManagement.repositories { clear(); maven { url = repoUri } } + } catch (ignored) { /* project may not use dependencyResolutionManagement */ } +} + +allprojects { + // buildscript repositories are not governed by FAIL_ON_PROJECT_REPOS; always override. + buildscript { repositories { clear(); maven { url = repoUri } } } + if (allowProjectRepos) { + repositories { clear(); maven { url = repoUri } } + } +} diff --git a/static/maven-offline/gradle/resolve-all.init.gradle b/static/maven-offline/gradle/resolve-all.init.gradle new file mode 100644 index 000000000..086d75172 --- /dev/null +++ b/static/maven-offline/gradle/resolve-all.init.gradle @@ -0,0 +1,45 @@ +// Force a download of every dependency a build needs into this Gradle user home's +// module cache. A reshaped copy of that cache becomes the offline Maven2 repository. +// +// Applied with --init-script by build-maven-offline.sh. It registers one task, +// resolveAllDeps, on the ROOT project of every build in the composite (a plain build +// has one; Code on the Go has included builds). gradle.rootProject is the reliable +// init-script hook for this: projectsEvaluated alone does not register the task on a +// composite build's main root. +// +// The task leniently resolves every resolvable configuration of every project, plus +// each project's buildscript (plugin) classpath, so one unresolved optional artifact +// does not abort the sweep. Lenient resolution still downloads every artifact that CAN +// be resolved; it only swallows the ones that cannot (platform stubs, absent variants). + +gradle.rootProject { root -> + if (root.tasks.findByName('resolveAllDeps') != null) return + root.tasks.register('resolveAllDeps') { + doLast { + root.allprojects.each { p -> + p.configurations.each { c -> + if (c.canBeResolved) { + try { + c.incoming.artifactView { it.lenient(true) }.files.files.size() + } catch (Throwable t) { + logger.info("skip ${p.path}:${c.name}: ${t.message}") + } + } + } + try { + p.buildscript.configurations.each { bc -> + if (bc.canBeResolved) { + try { + bc.incoming.artifactView { it.lenient(true) }.files.files.size() + } catch (Throwable t) { + logger.info("skip buildscript ${p.path}:${bc.name}: ${t.message}") + } + } + } + } catch (Throwable t) { + logger.info("no buildscript for ${p.path}: ${t.message}") + } + } + } + } +}