From 2cf2b6ef72740cd188fece6aa3577b5de5ec7fe0 Mon Sep 17 00:00:00 2001 From: "Luis Guzman (AppDevForAll)" Date: Sat, 3 Oct 2026 09:20:11 -0600 Subject: [PATCH 1/3] K2GO-443 feat(code-addons): add-ons gallery download on the aria2 job engine (dash-node 1.3.12) The gallery updater moves from the fire-and-forget wrapper to the durable job engine. Only the heavy add-on binaries (.cgp + source tarballs) go through aria2 (resume / pause / retry, survives a network change); the mirror stages the small files (shell, catalog, icons, pages) with its Cloudflare clean and catalog base rewrite. mirror_addons.py splits into --print-aria2-input (stage + emit the heavy input, with a #status marker so a small-file-only change still swaps) and --finalize-only (verify presence); one shared _stage core keeps the bake / legacy-wrapper path unchanged. --- .../k2go/addons/data/AddonsRefreshClient.java | 170 ------------------ static/dashboard/CHANGELOG.md | 1 + static/dashboard/package.json | 2 +- static/dashboard/routes.ts | 13 +- static/dashboard/server.ts | 1 + static/dashboard/sockets/code_addons.exec.ts | 144 +++++++++++++++ static/dashboard/sockets/jobs.ts | 2 +- .../roles/code_addons/files/mirror_addons.py | 159 +++++++++++++--- 8 files changed, 286 insertions(+), 206 deletions(-) delete mode 100644 controller/app/src/main/java/org/appdevforall/k2go/addons/data/AddonsRefreshClient.java create mode 100644 static/dashboard/sockets/code_addons.exec.ts diff --git a/controller/app/src/main/java/org/appdevforall/k2go/addons/data/AddonsRefreshClient.java b/controller/app/src/main/java/org/appdevforall/k2go/addons/data/AddonsRefreshClient.java deleted file mode 100644 index d39d1d0b4..000000000 --- a/controller/app/src/main/java/org/appdevforall/k2go/addons/data/AddonsRefreshClient.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * ============================================================================ - * Name : AddonsRefreshClient.java - * Author : AppDevForAll - * Copyright : Copyright (c) 2026 AppDevForAll - * Description : K2GO-99. App-side client of the dash-node add-ons refresh - * (static/dashboard routes.ts). The box re-mirrors the Code on the - * Go add-ons gallery while the server is UP: the device only POSTs - * to start and polls a coarse status, so a dropped app never stops - * the refresh (the box job is detached). Same POST-then-poll shape - * as ForgejoSeedClient.refresh(); the status carries downloaded and - * failed counts instead of per-repo outcomes. - * - * Contract: - * POST /k2go-api/addons/refresh -> 202 { state:"running" } - * GET /k2go-api/addons/refresh/status -> { state, lines, downloaded, reused, failed, upToDate } - * POST /k2go-api/addons/refresh/cancel -> { state:"cancelled" } - * ============================================================================ - */ -package org.appdevforall.k2go.addons.data; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import org.appdevforall.k2go.config.BoxEndpoints; -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; - -/** - * Drives the box add-ons refresh to a terminal state. {@link #refresh} BLOCKS (poll loop), so callers - * run it on an IO thread. The box job is detached, so a caller that dies mid-run does not stop it: a - * later {@link #refresh} re-attaches by reading the same status (a running refresh is left alone). - */ -public final class AddonsRefreshClient { - - /** Terminal verdict of a refresh. */ - public enum Result { DONE, ERROR, CANCELLED } - - /** Streamed status-tail lines, for a live one-line view. Optional (pass null to ignore). */ - public interface Listener { - void onLine(@NonNull String line); - } - - private static final String REFRESH_URL = BoxEndpoints.API + "/addons/refresh"; - private static final String REFRESH_STATUS_URL = BoxEndpoints.API + "/addons/refresh/status"; - private static final String REFRESH_CANCEL_URL = BoxEndpoints.API + "/addons/refresh/cancel"; - private static final long POLL_MS = 2000L; - private static final int MAX_POLL_ERRORS = 15; // ~30s of transient blips before giving up - // The mirror pulls the full set of .cgp files (hundreds of MB), so the run can take many minutes; - // cap the wait so a wedged box job cannot block forever (the box job keeps running detached). - private static final long MAX_WAIT_MS = 30 * 60 * 1000L; - - /** The last status-tail line handed to the listener, so a poll that did not advance stays quiet. */ - private String lastEmitted; - private int lastDownloaded = -1, lastReused = -1, lastFailed = -1; - private boolean lastUpToDate = false; - - /** Files downloaded in the last refresh; -1 if the box did not report it. */ - public int lastDownloaded() { return lastDownloaded; } - /** Files reused unchanged in the last refresh; -1 if the box did not report it (K2GO-441). */ - public int lastReused() { return lastReused; } - /** Files the last refresh could not fetch or verify; -1 if the box did not report it. */ - public int lastFailed() { return lastFailed; } - /** K2GO-441: true when the published set was unchanged, so the refresh downloaded nothing. */ - public boolean lastUpToDate() { return lastUpToDate; } - - /** - * Start the refresh if it is not already running, then poll to a terminal state. Each refresh is - * intentional, so there is no "done" short-circuit: unless one is already running (re-attach), it - * POSTs a fresh run. The box refresh is safe to re-run (it mirrors into a staging dir and only - * swaps the live gallery on success). Returns DONE when the box refresh finished, ERROR on an - * unreachable box or timeout, CANCELLED if the user stopped it. - */ - @NonNull - public Result refresh(@Nullable Listener l) { - String state = readState(l); - if (!"running".equals(state)) { - if (!post(REFRESH_URL)) return Result.ERROR; - } - return poll(l); - } - - /** Ask the box to stop a running refresh (best-effort; a poll then reads "cancelled"). */ - public void cancel() { - post(REFRESH_CANCEL_URL); - } - - @NonNull - private Result poll(@Nullable Listener l) { - final long deadline = System.currentTimeMillis() + MAX_WAIT_MS; - int pollErrors = 0; - while (System.currentTimeMillis() < deadline) { - try { Thread.sleep(POLL_MS); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); return Result.ERROR; } - String s = readState(l); - if (s == null) { if (++pollErrors > MAX_POLL_ERRORS) return Result.ERROR; continue; } - pollErrors = 0; - if ("done".equals(s)) return Result.DONE; - if ("error".equals(s)) return Result.ERROR; - if ("cancelled".equals(s)) return Result.CANCELLED; - // "running" (or an unknown transient) -> keep polling. - } - return Result.ERROR; // timed out; the box job may still finish, a later refresh re-checks - } - - /** POST a start/cancel endpoint; true if accepted (2xx) or already running (409). */ - private boolean post(@NonNull String url) { - try { - HttpURLConnection c = open("POST", url); - int code = c.getResponseCode(); - c.disconnect(); - return (code >= 200 && code < 300) || code == 409; - } catch (Exception e) { - return false; - } - } - - /** GET the status endpoint -> the state string, streaming any new tail line; null on a read error. */ - @Nullable - private String readState(@Nullable Listener l) { - try { - HttpURLConnection c = open("GET", REFRESH_STATUS_URL); - int code = c.getResponseCode(); - boolean ok = code >= 200 && code < 400; - String text = readAll(ok ? c.getInputStream() : c.getErrorStream()); - c.disconnect(); - if (!ok) return null; - JSONObject j = new JSONObject(text.isEmpty() ? "{}" : text); - if (j.has("downloaded")) lastDownloaded = j.optInt("downloaded", lastDownloaded); - if (j.has("reused")) lastReused = j.optInt("reused", lastReused); - if (j.has("failed")) lastFailed = j.optInt("failed", lastFailed); - if (j.has("upToDate")) lastUpToDate = j.optBoolean("upToDate", lastUpToDate); - if (l != null) { - JSONArray lines = j.optJSONArray("lines"); - if (lines != null && lines.length() > 0) { - String last = lines.optString(lines.length() - 1, ""); - if (!last.isEmpty() && !last.equals(lastEmitted)) { lastEmitted = last; l.onLine(last); } - } - } - return j.optString("state", ""); - } catch (Exception e) { - return null; - } - } - - private static HttpURLConnection open(String method, String urlStr) throws Exception { - HttpURLConnection c = (HttpURLConnection) new URL(urlStr).openConnection(); - c.setUseCaches(false); - c.setConnectTimeout(4000); - c.setReadTimeout(4000); - c.setRequestMethod(method); - c.setRequestProperty("Accept", "application/json"); - return c; - } - - private static String readAll(InputStream is) throws Exception { - if (is == null) return ""; - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - byte[] chunk = new byte[4096]; - int n; - while ((n = is.read(chunk)) != -1) buf.write(chunk, 0, n); - is.close(); - return buf.toString(StandardCharsets.UTF_8.name()); - } -} diff --git a/static/dashboard/CHANGELOG.md b/static/dashboard/CHANGELOG.md index ec9a929b3..c3655bf2b 100644 --- a/static/dashboard/CHANGELOG.md +++ b/static/dashboard/CHANGELOG.md @@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i (the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **1.3.12** - Add-ons gallery download on the durable job engine (K2GO-443). `code-addons` is now a job type: `POST /code-addons/download` plus `GET /code-addons/jobs/:id` (structured `{phase, percent, speed, detail}`) and pause/resume/retry/cancel over the generic `/:type/*` surface, like build-assets. The runner (`sockets/code_addons.exec.ts`) downloads only the heavy add-on binaries (.cgp + source tarballs) with aria2c (resilient: `--continue` resume, survives a network change) using the shared `downloadWithAria2` helper; the mirror stages the small files (shell, catalog, icons, pages) with its Cloudflare clean + catalog base rewrite and prints the aria2 input for the heavy ones (`mirror_addons.py --print-aria2-input`), then verifies them (`--finalize-only`), and the runner swaps the staged tree in atomically. The older `POST /addons/refresh` (wrapper) stays for now. Localhost-only. (K2GO-443) - **1.3.11** - Build-assets download on the durable job engine (K2GO-443). `code-assets` is now a job type: `POST /code-assets/download` plus `GET /code-assets/jobs/:id` (structured `{phase, percent, speed, detail}`) and pause/resume/retry/cancel over the generic `/:type/*` surface, like kiwix/maps. The runner (`sockets/code_assets.exec.ts`) downloads the build assets with aria2c (resilient: `--continue` resume, survives a full interface loss via the outer retry loop) using the shared `downloadWithAria2` helper, then the mirror verifies each file against its published `.md5` and writes the browse page (`mirror_code_assets.py --finalize-only`), and the runner swaps the staged tree in atomically. The older `POST /code-assets/refresh` (wrapper) stays for now. Localhost-only. (K2GO-443) - **1.3.10** - Code on the Go build-assets refresh (K2GO-437). New `POST /code-assets/refresh` re-mirrors the Code on the Go build assets into `/library/www/code-assets` LIVE (box up, no runrole) through a detached wrapper (`tools/code-assets-refresh.sh`) that mirrors into a staging dir and swaps it in only on success (a failed or cancelled refresh never serves a half-mirror), `GET /code-assets/refresh/status` returns `{state, lines, downloaded, reused, failed, upToDate}` to poll (counts parsed from the mirror's final log line; `upToDate` when nothing changed), and `POST /code-assets/refresh/cancel` stops a running refresh (SIGKILL to the wrapper process group; safe because the live tree is only replaced after a clean run). The wrapper reuses the role's `mirror_code_assets.py` (manifest-driven, per-file `.md5` verify and reuse, generates the browse page), so install (bake) and refresh (live) share one mechanism. Localhost-only. (K2GO-437) - **1.3.9** - Incremental add-ons refresh with an "up to date" report (K2GO-441). "Update add-ons" no longer re-downloads the whole gallery. `mirror_addons.py` takes `--reuse-from` the live tree and prints `result: up-to-date` when the published catalog is unchanged (nothing downloaded, no swap); it reads what changed straight from `catalog.json` (the single source of truth): it reuses unchanged `.cgp` and source tarballs by the catalog sha256, reuses each add-on's icon and page (both live inside the source tarball) when that sha is unchanged, and reuses shell assets by their content-hashed name, so only new or changed files download, with no local re-hash and no extra requests. `GET /addons/refresh/status` now also returns `{reused, upToDate}`. The app shows "already up to date" when nothing came down. (K2GO-441) diff --git a/static/dashboard/package.json b/static/dashboard/package.json index f325b99ab..48a5ce80e 100644 --- a/static/dashboard/package.json +++ b/static/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "dashboard-console", - "version": "1.3.11", + "version": "1.3.12", "description": "", "main": "index.js", "scripts": { diff --git a/static/dashboard/routes.ts b/static/dashboard/routes.ts index 354404e56..a39ff56c1 100644 --- a/static/dashboard/routes.ts +++ b/static/dashboard/routes.ts @@ -40,7 +40,7 @@ const ZIMS_DIR = '/library/zims/content/'; const KIWIX_INDEXER = '/usr/bin/iiab-make-kiwix-lib'; const ZIM_NAME_RE = /^[A-Za-z0-9._-]{1,150}\.zim$/; -const VALID_TYPES: JobType[] = ['kiwix', 'maps', 'books', 'kolibri', 'basemaps', 'code-assets']; +const VALID_TYPES: JobType[] = ['kiwix', 'maps', 'books', 'kolibri', 'basemaps', 'code-assets', 'code-addons']; function isType(t: string): t is JobType { return (VALID_TYPES as string[]).includes(t); } @@ -1117,13 +1117,14 @@ apiRouter.post('/:type/download', (req: Request, res: Response): void => { ? body.items : Array.isArray(body?.ids) ? body.ids : []; if (items.length === 0) { res.status(400).json({ error: 'items (or ids) required' }); return; } - // K2GO-443: code-assets stages into one shared tree (/library/www/code-assets.new), unlike kiwix's - // independent files, so only one build-assets job may run at a time. The app re-attaches via + // K2GO-443: these content types stage into ONE shared tree (/library/www/.new), unlike kiwix's + // independent files, so only one job per such type may run at a time. The app re-attaches via // start-or-attach; this is the hard guard behind it (two concurrent jobs would corrupt the staging). - if (type === 'code-assets' - && jobs.list('code-assets').some((j) => + const SINGLE_TREE_TYPES = ['code-assets', 'code-addons']; + if (SINGLE_TREE_TYPES.includes(type) + && jobs.list(type).some((j) => ['queued', 'downloading', 'indexing', 'processing', 'paused'].includes(j.phase))) { - res.status(409).json({ error: 'a build-assets job is already running' }); + res.status(409).json({ error: `a ${type} job is already running` }); return; } res.status(202).json(toApi(jobs.create(type, items))); diff --git a/static/dashboard/server.ts b/static/dashboard/server.ts index 0f9d3ee51..9ecf4b179 100644 --- a/static/dashboard/server.ts +++ b/static/dashboard/server.ts @@ -11,6 +11,7 @@ import './sockets/maps-base.exec'; import './sockets/books.exec'; import './sockets/kolibri.exec'; import './sockets/code_assets.exec'; // K2GO-443: build-assets runner (aria2 job engine) +import './sockets/code_addons.exec'; // K2GO-443: add-ons gallery runner (aria2 job engine) import { apiRouter } from './routes'; import { startServiceHeal } from './sockets/service-heal'; import { startLogRotation, stopLogRotation } from './sockets/log-rotate'; diff --git a/static/dashboard/sockets/code_addons.exec.ts b/static/dashboard/sockets/code_addons.exec.ts new file mode 100644 index 000000000..df9b83d6d --- /dev/null +++ b/static/dashboard/sockets/code_addons.exec.ts @@ -0,0 +1,144 @@ +// sockets/code_addons.exec.ts - K2GO-443 +// +// Code on the Go add-ons gallery runner for the durable job engine. Replaces the fire-and-forget +// wrapper (tools/code-addons-refresh.sh + POST /addons/refresh + status-file poll) with an aria2 +// download that gives percent / speed / pause / resume / retry and survives a network change, like +// build-assets / kiwix / maps. The gallery is a mix: many small files (shell, catalog, icons, pages) +// that need a transform (Cloudflare clean + catalog base rewrite), and the heavy add-on binaries +// (.cgp + source tarballs). Only the heavy binaries go through aria2; the mirror stages the rest. +// +// The mirror splits into a plan step and a verify step (mirror_addons.py): +// --print-aria2-input: stage the small files, print an aria2 input-file for the heavy binaries. Its +// first stdout line is a status marker. '#status=uptodate' means the same build is already served +// (keep the live tree, do not swap). '#status=stage' means STAGE is populated and the lines after +// are the aria2 input, which may be EMPTY when only small files changed (still finalize and swap). +// --finalize-only: verify the aria2-downloaded binaries are present in STAGE (aria2 already verified +// each by its sha-256). +// The runner then swaps the staged tree in atomically, restoring the previous tree if the swap fails. +import { jobs, RunnerContext, CanceledError, PausedError } from './jobs'; +import { downloadWithAria2, stopError } from './aria2-download'; +import { execFileSync } from 'child_process'; +import fs from 'fs'; + +const SERVE = '/library/www/code-addons'; +const STAGE = `${SERVE}.new`; +const OLD = `${SERVE}.old`; +const SERVE_BASE = '/code-addons'; // the catalog base the mirror rewrites to +// K2GO-440: prefer the mirror from the self-updating clone so a mirror fix ships via the dash-node +// rebuild with no rebake; fall back to the bake-time copy in the ansible roles dir. +const MIRROR_CLONE = '/opt/iiab-android/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py'; +const MIRROR_BAKED = '/opt/iiab/iiab/roles/code_addons/files/mirror_addons.py'; + +function mirrorPath(): string { + return fs.existsSync(MIRROR_CLONE) ? MIRROR_CLONE : MIRROR_BAKED; +} + +function rmrf(p: string): void { + try { fs.rmSync(p, { recursive: true, force: true }); } catch { /* best effort */ } +} + +/** Remove aria2 control/metadata so the served tree is only the gallery files. */ +function cleanAria2Files(dir: string): void { + try { execFileSync('find', [dir, '-name', '*.aria2', '-delete']); } catch { /* best effort */ } +} + +/** Run the mirror and resolve with its stdout; rejects (paused/canceled/error) like a download step. */ +function runMirror(ctx: RunnerContext, mirror: string, args: string[], capture: boolean): Promise { + return new Promise((resolve, reject) => { + let out = ''; + const p = ctx.spawn('python3', [mirror, ...args]); + p.stdout?.on('data', (d: Buffer) => { if (capture) out += d.toString(); else ctx.log(d.toString().trim()); }); + p.stderr?.on('data', (d: Buffer) => ctx.log(d.toString().trim())); + p.on('error', reject); + p.on('exit', (code, signal) => { + if (signal === 'SIGKILL' || ctx.isCanceled()) return reject(new CanceledError()); + if (code === 0) return resolve(out); + reject(new Error(`${args[0]} failed (exit ${code})`)); + }); + }); +} + +const codeAddonsRunner: (ctx: RunnerContext) => Promise = async (ctx) => { + const MIRROR = mirrorPath(); + if (!fs.existsSync(MIRROR)) throw new Error(`mirror script not found: ${MIRROR}`); + + // --- Plan (incremental) ------------------------------------------------- + // The mirror stages the small files into STAGE from the published site + the live tree and prints a + // status marker plus an aria2 input-file for the heavy binaries. STAGE is NOT pre-cleared, so a + // resume keeps its partials; the plan is deterministic and safe to re-run. + ctx.update({ phase: 'downloading', percent: -1, speed: 0, detail: 'add-ons' }); + let planned: string; + try { + planned = await runMirror(ctx, MIRROR, + ['--print-aria2-input', '--reuse-from', SERVE, '--out', STAGE, '--serve-base', SERVE_BASE], true); + } catch (e) { + const se = stopError(ctx); + if (se instanceof PausedError) throw se; + if (se instanceof CanceledError) { rmrf(STAGE); throw se; } + throw e; + } + ctx.throwIfCanceled(); + + const nl = planned.indexOf('\n'); + const marker = (nl === -1 ? planned : planned.slice(0, nl)).trim(); + const body = (nl === -1 ? '' : planned.slice(nl + 1)).trim(); + + if (marker === '#status=uptodate') { + // Same published build: keep the live tree, nothing to download or swap. + rmrf(STAGE); + ctx.update({ phase: 'done', percent: 100, speed: 0, detail: 'up to date' }); + return; + } + + // --- Download the changed heavy binaries (aria2, resilient) ------------- + // body may be empty when only small files changed: skip the download, but still finalize and swap + // so the staged small-file changes go live. + if (body !== '') { + const inputFile = `${STAGE}/.aria2-input`; + fs.mkdirSync(STAGE, { recursive: true }); + fs.writeFileSync(inputFile, body + '\n'); + try { + await downloadWithAria2(ctx, { destDir: STAGE, inputFile, phase: 'downloading' }); + } catch (e) { + const se = stopError(ctx); + if (se instanceof PausedError) throw se; // keep STAGE: resume continues + if (se instanceof CanceledError) { rmrf(STAGE); throw se; } // cancel discards the partial + throw e; // real error: keep STAGE for a retry/resume + } + try { fs.rmSync(inputFile, { force: true }); } catch { /* leave nothing non-served behind */ } + } + ctx.throwIfCanceled(); + + // --- Verify (aria2 checked each binary by sha-256; no re-hash) ---------- + ctx.update({ phase: 'processing', percent: -1, speed: 0, detail: 'finishing' }); + try { + await runMirror(ctx, MIRROR, ['--finalize-only', '--out', STAGE, '--serve-base', SERVE_BASE], false); + } catch (e) { + const se = stopError(ctx); + if (se) throw se; + throw e; + } + ctx.throwIfCanceled(); + + // --- Swap (atomic rename; restore the previous tree if the move fails) -- + ctx.update({ phase: 'processing', percent: 100, speed: 0, detail: 'installing' }); + cleanAria2Files(STAGE); + rmrf(OLD); + if (fs.existsSync(SERVE)) fs.renameSync(SERVE, OLD); + try { + fs.renameSync(STAGE, SERVE); + } catch (e) { + // A failed swap must never leave the box with no served gallery: put the previous tree back. + if (!fs.existsSync(SERVE) && fs.existsSync(OLD)) { + try { fs.renameSync(OLD, SERVE); } catch { /* best effort */ } + } + throw e; + } + rmrf(OLD); + + ctx.update({ phase: 'done', percent: 100, speed: 0 }); +}; + +jobs.registerRunner('code-addons', codeAddonsRunner); + +export { codeAddonsRunner }; diff --git a/static/dashboard/sockets/jobs.ts b/static/dashboard/sockets/jobs.ts index fad004693..b92b9805a 100644 --- a/static/dashboard/sockets/jobs.ts +++ b/static/dashboard/sockets/jobs.ts @@ -13,7 +13,7 @@ import fs from 'fs'; import path from 'path'; import { RollingLog, LogSlice } from './rolling-log'; -export type JobType = 'kiwix' | 'maps' | 'books' | 'kolibri' | 'basemaps' | 'code-assets'; +export type JobType = 'kiwix' | 'maps' | 'books' | 'kolibri' | 'basemaps' | 'code-assets' | 'code-addons'; export type JobPhase = | 'queued' | 'downloading' | 'indexing' | 'processing' // ADFA-4894 (control surface): 'paused' is a stopped-but-resumable state — like 'canceled' it diff --git a/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py b/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py index a992fa9a8..eb4e99828 100644 --- a/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py +++ b/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py @@ -285,8 +285,16 @@ def build_plan(source_base, catalog, served, index_unchanged, index_bytes): return paths, sha, size, reuse_ok -def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0, - verbose=True): +def _stage(source_base, serve_base, out, reuse_from, heavy_sink, limit_large, log): + """Stage the gallery into `out` and return a result dict (the single staging core). + + Fetches the small files (index, catalog, schema, icons, pages, shell), Cloudflare-cleans the + HTML, copies what the catalog proves is reusable, and rewrites the catalog base. When `heavy_sink` + is not None, a verified binary that would be downloaded (.cgp / source tarball) is NOT fetched: + its (url, rel, sha256, size) is appended to `heavy_sink` so a caller can hand it to aria2 (K2GO-443). + All human output goes through `log(msg, err=False)` so a caller can route it off stdout. Returns + {up_to_date, downloaded, reused, failed, headed}; on the same-build short-circuit only up_to_date + is meaningful.""" source_base = source_base.rstrip("/") serve_base = serve_base.rstrip("/") out = Path(out) @@ -295,16 +303,12 @@ def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0, catalog_bytes = fetch(f"{source_base}/{CATALOG}") rewritten_catalog = catalog_bytes.decode("utf-8").replace(source_base, serve_base) - # K2GO-441: same-build short-circuit. The catalog carries `generated` (a build - # timestamp), so a byte-identical catalog means the exact same published build: - # nothing changed, so there is nothing to download and nothing to swap. The - # wrapper reads "result: up-to-date" and keeps the live gallery. + # K2GO-441: same-build short-circuit. The catalog carries `generated` (a build timestamp), so a + # byte-identical catalog means the exact same published build: nothing to download, nothing to swap. if reuse_from is not None: try: if (reuse_from / CATALOG).read_text(encoding="utf-8") == rewritten_catalog: - print("result: up-to-date") - print("done: 0 downloaded, 0 reused, 0 failed") - return True + return {"up_to_date": True, "downloaded": 0, "reused": 0, "failed": 0, "headed": 0} except OSError: pass # no served catalog yet: fall through to a full mirror. @@ -325,9 +329,8 @@ def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0, paths, sha, size, reuse_ok = build_plan( source_base, catalog, served, index_unchanged, index_bytes) - if verbose: - print(f"plan: {len(paths)} files, {len(sha)} verified binaries, " - f"{len(catalog['addons'])} add-ons") + log(f"plan: {len(paths)} files, {len(sha)} verified binaries, " + f"{len(catalog['addons'])} add-ons") downloaded = reused = headed = failed = 0 @@ -356,14 +359,19 @@ def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0, reused += 1 continue + # K2GO-443: defer a verified binary (.cgp / tarball) to aria2 instead of fetching its body + # here, so the heavy download gets resume / pause / retry. aria2 verifies it by checksum=sha-256. + if heavy_sink is not None and rel in sha: + heavy_sink.append((url, rel, sha[rel], size.get(rel))) + continue + # Test only: HEAD a big binary instead of pulling its body. if limit_large and size.get(rel) and size[rel] > limit_large: st, length = head(url) ok = st == 200 and (length == size[rel] or length is None) headed += 1 - if verbose: - mark = "ok" if ok else f"MISMATCH(status={st},len={length})" - print(f" HEAD {rel} ({size[rel]} B): {mark}") + mark = "ok" if ok else f"MISMATCH(status={st},len={length})" + log(f" HEAD {rel} ({size[rel]} B): {mark}") if not ok: failed += 1 continue @@ -372,40 +380,120 @@ def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0, data = fetch(url) except RuntimeError as e: failed += 1 - print(f" FAIL {rel}: {e}", file=sys.stderr) + log(f" FAIL {rel}: {e}", err=True) continue if rel in sha: got = sha256_of(data) if got != sha[rel]: failed += 1 - print(f" SHA MISMATCH {rel}: got {got[:12]} " - f"want {sha[rel][:12]}", file=sys.stderr) + log(f" SHA MISMATCH {rel}: got {got[:12]} " + f"want {sha[rel][:12]}", err=True) continue if len(data) != size[rel]: failed += 1 - print(f" SIZE MISMATCH {rel}: got {len(data)} " - f"want {size[rel]}", file=sys.stderr) + log(f" SIZE MISMATCH {rel}: got {len(data)} " + f"want {size[rel]}", err=True) continue if rel.endswith(".html"): data = clean_html(data) dest.write_bytes(data) downloaded += 1 - if verbose and (downloaded % 10 == 0 or rel in sha): + if downloaded % 10 == 0 or rel in sha: tag = " (verified)" if rel in sha else "" - print(f" GET {rel} ({len(data)} B){tag}") + log(f" GET {rel} ({len(data)} B){tag}") # The one transform: point the catalog at the local serve base. catalog_file = out / CATALOG if catalog_file.exists(): n = catalog_file.read_text(encoding="utf-8").count(source_base) catalog_file.write_text(rewritten_catalog, encoding="utf-8") - if verbose: - print(f"rewrote catalog base: {source_base} -> {serve_base} " - f"({n} occurrences)") + log(f"rewrote catalog base: {source_base} -> {serve_base} " + f"({n} occurrences)") - tail = f", {headed} head-checked" if headed else "" - print(f"done: {downloaded} downloaded, {reused} reused{tail}, {failed} failed") - return failed == 0 + return {"up_to_date": False, "downloaded": downloaded, "reused": reused, + "failed": failed, "headed": headed} + + +def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0, + verbose=True): + """Full in-process mirror (bake and the legacy refresh wrapper). Prints the same stdout as before: + progress when verbose, then 'result: up-to-date' or the 'done: ...' summary the wrapper parses.""" + def log(msg, err=False): + if err: + print(msg, file=sys.stderr) + elif verbose: + print(msg) + + res = _stage(source_base, serve_base, out, reuse_from, None, limit_large, log) + if res["up_to_date"]: + print("result: up-to-date") + print("done: 0 downloaded, 0 reused, 0 failed") + return True + tail = f", {res['headed']} head-checked" if res["headed"] else "" + print(f"done: {res['downloaded']} downloaded, {res['reused']} reused{tail}, " + f"{res['failed']} failed") + return res["failed"] == 0 + + +def aria2_input(source_base, serve_base, out, reuse_from=None): + """K2GO-443: stage everything except the heavy verified binaries into `out`, and print an aria2 + input-file (the .cgp / tarball downloads) to STDOUT for the durable job runner. Line 1 is a status + marker: '#status=uptodate' (same build: keep the live tree, do not swap) or '#status=stage' (out is + staged; the lines after are the aria2 input). The input body may be EMPTY when only small files + changed: the runner must still finalize and swap. Heavy entries use an absolute per-entry dir= plus + a basename out= (aria2 drops a subdir in out=), and checksum=sha-256 so aria2 verifies each file. + All human logs go to stderr so stdout carries only the marker and the input.""" + def log(msg, err=False): + print(msg, file=sys.stderr) + + heavy = [] + res = _stage(source_base, serve_base, out, reuse_from, heavy, 0, log) + if res["up_to_date"]: + print("#status=uptodate") + return True + lines = ["#status=stage"] + out = Path(out) + for url, rel, want_sha, _want_size in heavy: + p = Path(rel) + lines.append(url) + lines.append(f" dir={out}/{p.parent}") + lines.append(f" out={p.name}") + lines.append(f" checksum=sha-256={want_sha}") + sys.stdout.write("\n".join(lines) + "\n") + return res["failed"] == 0 + + +def finalize(serve_base, out, verbose=True): + """K2GO-443: verify the heavy binaries aria2 downloaded are present in `out` with the catalog's + size. The small files and the catalog base-rewrite were already staged by --print-aria2-input, and + aria2 verified each heavy file by its sha-256, so this only confirms presence/size and reports. The + staged catalog is rewritten to serve_base, so its urls are serve-relative.""" + serve_base = serve_base.rstrip("/") + out = Path(out) + try: + catalog = json.loads((out / CATALOG).read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + print(f"finalize: cannot read staged catalog: {e}", file=sys.stderr) + return False + checked = missing = 0 + for addon in catalog.get("addons", []): + for key in ("download", "sourceTarball"): + entry = addon.get(key) or {} + rel = rel_to_base(entry.get("url"), serve_base) if entry.get("url") else None + if not rel: + continue + checked += 1 + f = out / rel + if not f.is_file(): + missing += 1 + print(f" MISSING {rel}", file=sys.stderr) + continue + want_size = entry.get("size") + if want_size is not None and f.stat().st_size != want_size: + missing += 1 + print(f" SIZE {rel}: {f.stat().st_size} != {want_size}", file=sys.stderr) + print(f"done: {checked - missing} present, {missing} missing") + return missing == 0 def main(argv=None): @@ -421,8 +509,23 @@ def main(argv=None): ap.add_argument("--limit-large", type=int, default=0, help="test only: HEAD files larger than this many bytes " "instead of downloading them (0 = download all)") + # K2GO-443: the durable job engine downloads the heavy binaries with aria2 (resume / pause / retry), + # so the mirror splits into a plan step and a verify step. --print-aria2-input stages the small files + # and prints the aria2 input for the .cgp / tarballs; --finalize-only verifies them after aria2 runs. + ap.add_argument("--print-aria2-input", action="store_true", + help="stage the small files and print an aria2 input-file for the heavy binaries " + "(for the job runner); stdout is a status marker plus the input") + ap.add_argument("--finalize-only", action="store_true", + help="verify the aria2-downloaded binaries are present in --out (used after the " + "job-engine download); no download") ap.add_argument("--quiet", action="store_true") args = ap.parse_args(argv) + + if args.print_aria2_input: + return 0 if aria2_input(args.source_base, args.serve_base, args.out, + reuse_from=args.reuse_from) else 1 + if args.finalize_only: + return 0 if finalize(args.serve_base, args.out, verbose=not args.quiet) else 1 ok = mirror(args.source_base, args.serve_base, args.out, reuse_from=args.reuse_from, limit_large=args.limit_large, verbose=not args.quiet) From 8f9ec44d2ac1b9d8c18920bdd24e7d745cbe1d75 Mon Sep 17 00:00:00 2001 From: "Luis Guzman (AppDevForAll)" Date: Sat, 3 Oct 2026 09:20:19 -0600 Subject: [PATCH 2/3] K2GO-443 feat(app): add-ons updater on the job engine; drop the refresh-wrapper client AddonsDownloadService extends ContentDownloadServiceBase (K2GO-449 shared base), and AddonsRefresh now drives it: a determinate bar with speed, pause/resume, cancel, injected after the trigger, re-attaching when the sheet reopens. Removes AddonsRefreshClient (the POST-then-poll wrapper client). ModuleActions already routes code_addons to AddonsRefresh.start. --- controller/app/src/main/AndroidManifest.xml | 9 + .../presentation/AddonsDownloadService.java | 98 ++++++++++ .../addons/presentation/AddonsRefresh.java | 177 +++++++++++------- 3 files changed, 218 insertions(+), 66 deletions(-) create mode 100644 controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsDownloadService.java diff --git a/controller/app/src/main/AndroidManifest.xml b/controller/app/src/main/AndroidManifest.xml index be8ed6b79..86d033ac9 100644 --- a/controller/app/src/main/AndroidManifest.xml +++ b/controller/app/src/main/AndroidManifest.xml @@ -203,6 +203,15 @@ android:value="Downloads the Code on the Go build assets through the co-located server REST engine." /> + + + + { - cancel.setEnabled(false); - label.setText(R.string.k2go_code_addons_update_cancelling); - AppExecutors.get().io().execute(() -> new AddonsRefreshClient().cancel()); + pauseBtn.setOnClickListener(v -> { + if (AddonsDownloadService.isPaused()) AddonsDownloadService.resume(ctx); + else AddonsDownloadService.pause(ctx); }); - - AppExecutors.get().io().execute(() -> { - final AddonsRefreshClient client = new AddonsRefreshClient(); - final AddonsRefreshClient.Result r = client.refresh(rawLine -> { - final String shown = rawLine.trim(); - main.post(() -> { if (liveLine.isAttachedToWindow()) liveLine.setText(shown); }); - }); - final int failed = client.lastFailed(); - final boolean upToDate = client.lastUpToDate(); - main.post(() -> { - if (!trigger.isAttachedToWindow()) return; - parent.removeView(progress); - trigger.setEnabled(true); - Snackbars.make(trigger, ctx.getString(messageFor(r, failed, upToDate))).show(); - }); + cancelBtn.setOnClickListener(v -> { + cancelBtn.setEnabled(false); + AddonsDownloadService.cancel(ctx); }); + + // Clear the session listener the moment this view leaves the window (sheet dismissed), so the + // static session never keeps a destroyed Activity alive while a download runs on. Deterministic: + // it does not wait for the next session event (a paused download emits none). + final View.OnAttachStateChangeListener detach = new View.OnAttachStateChangeListener() { + @Override public void onViewAttachedToWindow(@NonNull View v) {} + @Override public void onViewDetachedFromWindow(@NonNull View v) { + AddonsDownloadService.setListener(null); + } + }; + + final Runnable[] render = new Runnable[1]; + render[0] = () -> { + if (!trigger.isAttachedToWindow()) { AddonsDownloadService.setListener(null); return; } + final boolean running = AddonsDownloadService.isRunning(); + // hasFailed() BEFORE isComplete(): isComplete() is true for an all-FAILED session too + // (FAILED is "not in progress"), so a failed run must be caught first. + if (!running && AddonsDownloadService.hasFailed()) { + terminal(trigger, parent, progress, detach, R.string.k2go_code_addons_update_failed); + return; + } + if (AddonsDownloadService.isComplete()) { + terminal(trigger, parent, progress, detach, R.string.k2go_code_addons_update_done); + return; + } + if (!AddonsDownloadService.hasSession()) { // cancelled (purged) + terminal(trigger, parent, progress, detach, R.string.k2go_code_addons_update_cancelled); + return; + } + final boolean paused = AddonsDownloadService.isPaused(); + final int pct = AddonsDownloadService.percent(); + bar.setIndeterminate(pct < 0); + if (pct >= 0) bar.setProgressCompat(pct, true); + pauseBtn.setText(paused ? R.string.k2go_dl_resume : R.string.k2go_dl_pause); + if (AddonsDownloadService.reconnectAttempt() > 0) { + statusLine.setText(R.string.k2go_retrying); + } else if (paused) { + statusLine.setText(R.string.k2go_dl_paused); + } else { + final long spd = AddonsDownloadService.speed(); + statusLine.setText(spd > 0 + ? ByteFormatter.toHuman(spd) + "/s" + : trigger.getContext().getString(R.string.k2go_code_addons_updating)); + } + }; + + trigger.addOnAttachStateChangeListener(detach); + // publish() already posts to the main thread, so the listener runs on main: wire it directly. + AddonsDownloadService.setListener(render[0]::run); + AddonsDownloadService.start(ctx); + main.post(render[0]); // initial paint } - /** Map the refresh outcome to a user message covering every state. */ - private static int messageFor(AddonsRefreshClient.Result r, int failed, boolean upToDate) { - if (r == AddonsRefreshClient.Result.CANCELLED) return R.string.k2go_code_addons_update_cancelled; - if (r != AddonsRefreshClient.Result.DONE) return R.string.k2go_code_addons_update_failed; // box unreachable - if (failed > 0) return R.string.k2go_code_addons_update_some_failed; // some files could not be fetched - // K2GO-441: the box is the single source for "nothing changed" (result: up-to-date). A box that - // does not report it (pre-1.3.9) ran the old full mirror, so "updated" is the correct default. - if (upToDate) return R.string.k2go_code_addons_update_none; - return R.string.k2go_code_addons_update_done; + private static void terminal(@NonNull View trigger, @NonNull ViewGroup parent, @NonNull View progress, + @NonNull View.OnAttachStateChangeListener detach, int msgRes) { + AddonsDownloadService.setListener(null); + trigger.removeOnAttachStateChangeListener(detach); + if (progress.getParent() == parent) parent.removeView(progress); + trigger.setEnabled(true); + if (trigger.isAttachedToWindow()) { + Snackbars.make(trigger, trigger.getContext().getString(msgRes)).show(); + } + AddonsDownloadService.finishSession(); } } From a605667be117f6c4e461f264900229c6466c9192 Mon Sep 17 00:00:00 2001 From: "Luis Guzman (AppDevForAll)" Date: Sat, 3 Oct 2026 09:24:04 -0600 Subject: [PATCH 3/3] K2GO-443 fix(app): unique add-ons notification id (review) Notification id 8 is also ForgejoSeedService and CloneShareService; an add-ons download and a forgejo seed can be foreground at once, so give the add-ons service a free id (11). --- .../k2go/addons/presentation/AddonsDownloadService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsDownloadService.java b/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsDownloadService.java index c09e46742..7f7fd0116 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsDownloadService.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsDownloadService.java @@ -27,7 +27,7 @@ public final class AddonsDownloadService extends ContentDownloadServiceBase { private static final String CHANNEL_ID = "code_addons_download_channel"; - private static final int NOTIFICATION_ID = 8; + private static final int NOTIFICATION_ID = 11; // distinct: 8 is ForgejoSeedService / CloneShareService (may run concurrently) public static final String ACTION_START = "org.appdevforall.k2go.CODE_ADDONS_DOWNLOAD_START"; public static final String ACTION_PAUSE = "org.appdevforall.k2go.CODE_ADDONS_DOWNLOAD_PAUSE";