diff --git a/controller/app/src/main/AndroidManifest.xml b/controller/app/src/main/AndroidManifest.xml
index be8ed6b79..86d033ac9 100644
--- a/controller/app/src/main/AndroidManifest.xml
+++ b/controller/app/src/main/AndroidManifest.xml
@@ -203,6 +203,15 @@
android:value="Downloads the Code on the Go build assets through the co-located server REST engine." />
+
+
+
+
202 { state:"running" }
- * GET /k2go-api/addons/refresh/status -> { state, lines, downloaded, reused, failed, upToDate }
- * POST /k2go-api/addons/refresh/cancel -> { state:"cancelled" }
- * ============================================================================
- */
-package org.appdevforall.k2go.addons.data;
-
-import androidx.annotation.NonNull;
-import androidx.annotation.Nullable;
-
-import org.appdevforall.k2go.config.BoxEndpoints;
-import org.json.JSONArray;
-import org.json.JSONObject;
-
-import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-
-/**
- * Drives the box add-ons refresh to a terminal state. {@link #refresh} BLOCKS (poll loop), so callers
- * run it on an IO thread. The box job is detached, so a caller that dies mid-run does not stop it: a
- * later {@link #refresh} re-attaches by reading the same status (a running refresh is left alone).
- */
-public final class AddonsRefreshClient {
-
- /** Terminal verdict of a refresh. */
- public enum Result { DONE, ERROR, CANCELLED }
-
- /** Streamed status-tail lines, for a live one-line view. Optional (pass null to ignore). */
- public interface Listener {
- void onLine(@NonNull String line);
- }
-
- private static final String REFRESH_URL = BoxEndpoints.API + "/addons/refresh";
- private static final String REFRESH_STATUS_URL = BoxEndpoints.API + "/addons/refresh/status";
- private static final String REFRESH_CANCEL_URL = BoxEndpoints.API + "/addons/refresh/cancel";
- private static final long POLL_MS = 2000L;
- private static final int MAX_POLL_ERRORS = 15; // ~30s of transient blips before giving up
- // The mirror pulls the full set of .cgp files (hundreds of MB), so the run can take many minutes;
- // cap the wait so a wedged box job cannot block forever (the box job keeps running detached).
- private static final long MAX_WAIT_MS = 30 * 60 * 1000L;
-
- /** The last status-tail line handed to the listener, so a poll that did not advance stays quiet. */
- private String lastEmitted;
- private int lastDownloaded = -1, lastReused = -1, lastFailed = -1;
- private boolean lastUpToDate = false;
-
- /** Files downloaded in the last refresh; -1 if the box did not report it. */
- public int lastDownloaded() { return lastDownloaded; }
- /** Files reused unchanged in the last refresh; -1 if the box did not report it (K2GO-441). */
- public int lastReused() { return lastReused; }
- /** Files the last refresh could not fetch or verify; -1 if the box did not report it. */
- public int lastFailed() { return lastFailed; }
- /** K2GO-441: true when the published set was unchanged, so the refresh downloaded nothing. */
- public boolean lastUpToDate() { return lastUpToDate; }
-
- /**
- * Start the refresh if it is not already running, then poll to a terminal state. Each refresh is
- * intentional, so there is no "done" short-circuit: unless one is already running (re-attach), it
- * POSTs a fresh run. The box refresh is safe to re-run (it mirrors into a staging dir and only
- * swaps the live gallery on success). Returns DONE when the box refresh finished, ERROR on an
- * unreachable box or timeout, CANCELLED if the user stopped it.
- */
- @NonNull
- public Result refresh(@Nullable Listener l) {
- String state = readState(l);
- if (!"running".equals(state)) {
- if (!post(REFRESH_URL)) return Result.ERROR;
- }
- return poll(l);
- }
-
- /** Ask the box to stop a running refresh (best-effort; a poll then reads "cancelled"). */
- public void cancel() {
- post(REFRESH_CANCEL_URL);
- }
-
- @NonNull
- private Result poll(@Nullable Listener l) {
- final long deadline = System.currentTimeMillis() + MAX_WAIT_MS;
- int pollErrors = 0;
- while (System.currentTimeMillis() < deadline) {
- try { Thread.sleep(POLL_MS); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); return Result.ERROR; }
- String s = readState(l);
- if (s == null) { if (++pollErrors > MAX_POLL_ERRORS) return Result.ERROR; continue; }
- pollErrors = 0;
- if ("done".equals(s)) return Result.DONE;
- if ("error".equals(s)) return Result.ERROR;
- if ("cancelled".equals(s)) return Result.CANCELLED;
- // "running" (or an unknown transient) -> keep polling.
- }
- return Result.ERROR; // timed out; the box job may still finish, a later refresh re-checks
- }
-
- /** POST a start/cancel endpoint; true if accepted (2xx) or already running (409). */
- private boolean post(@NonNull String url) {
- try {
- HttpURLConnection c = open("POST", url);
- int code = c.getResponseCode();
- c.disconnect();
- return (code >= 200 && code < 300) || code == 409;
- } catch (Exception e) {
- return false;
- }
- }
-
- /** GET the status endpoint -> the state string, streaming any new tail line; null on a read error. */
- @Nullable
- private String readState(@Nullable Listener l) {
- try {
- HttpURLConnection c = open("GET", REFRESH_STATUS_URL);
- int code = c.getResponseCode();
- boolean ok = code >= 200 && code < 400;
- String text = readAll(ok ? c.getInputStream() : c.getErrorStream());
- c.disconnect();
- if (!ok) return null;
- JSONObject j = new JSONObject(text.isEmpty() ? "{}" : text);
- if (j.has("downloaded")) lastDownloaded = j.optInt("downloaded", lastDownloaded);
- if (j.has("reused")) lastReused = j.optInt("reused", lastReused);
- if (j.has("failed")) lastFailed = j.optInt("failed", lastFailed);
- if (j.has("upToDate")) lastUpToDate = j.optBoolean("upToDate", lastUpToDate);
- if (l != null) {
- JSONArray lines = j.optJSONArray("lines");
- if (lines != null && lines.length() > 0) {
- String last = lines.optString(lines.length() - 1, "");
- if (!last.isEmpty() && !last.equals(lastEmitted)) { lastEmitted = last; l.onLine(last); }
- }
- }
- return j.optString("state", "");
- } catch (Exception e) {
- return null;
- }
- }
-
- private static HttpURLConnection open(String method, String urlStr) throws Exception {
- HttpURLConnection c = (HttpURLConnection) new URL(urlStr).openConnection();
- c.setUseCaches(false);
- c.setConnectTimeout(4000);
- c.setReadTimeout(4000);
- c.setRequestMethod(method);
- c.setRequestProperty("Accept", "application/json");
- return c;
- }
-
- private static String readAll(InputStream is) throws Exception {
- if (is == null) return "";
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
- byte[] chunk = new byte[4096];
- int n;
- while ((n = is.read(chunk)) != -1) buf.write(chunk, 0, n);
- is.close();
- return buf.toString(StandardCharsets.UTF_8.name());
- }
-}
diff --git a/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsDownloadService.java b/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsDownloadService.java
new file mode 100644
index 000000000..7f7fd0116
--- /dev/null
+++ b/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsDownloadService.java
@@ -0,0 +1,98 @@
+/*
+ * ============================================================================
+ * Name : AddonsDownloadService.java
+ * Author : AppDevForAll
+ * Copyright : Copyright (c) 2026 AppDevForAll
+ * Description : K2GO-443 / K2GO-449. Foreground shell for the Code on the Go add-ons gallery download on
+ * the durable job engine (a single-item session, type "code-addons"). The generic service
+ * behavior (onStartCommand dispatch, notification, Host) lives in ContentDownloadServiceBase;
+ * this subclass only provides the module bits: the session singleton, the action strings,
+ * the notification text, and the START item (a sentinel, since the box runner reads the
+ * catalog itself). The heavy work (aria2, resume, verify, swap) runs on the box.
+ * ============================================================================
+ */
+package org.appdevforall.k2go.addons.presentation;
+
+import android.content.Context;
+import android.content.Intent;
+
+import androidx.core.content.ContextCompat;
+
+import org.appdevforall.k2go.R;
+import org.appdevforall.k2go.redesign.ContentDownloadServiceBase;
+import org.appdevforall.k2go.redesign.ContentDownloadSession;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+public final class AddonsDownloadService extends ContentDownloadServiceBase {
+
+ private static final String CHANNEL_ID = "code_addons_download_channel";
+ private static final int NOTIFICATION_ID = 11; // distinct: 8 is ForgejoSeedService / CloneShareService (may run concurrently)
+
+ public static final String ACTION_START = "org.appdevforall.k2go.CODE_ADDONS_DOWNLOAD_START";
+ public static final String ACTION_PAUSE = "org.appdevforall.k2go.CODE_ADDONS_DOWNLOAD_PAUSE";
+ public static final String ACTION_RESUME = "org.appdevforall.k2go.CODE_ADDONS_DOWNLOAD_RESUME";
+ public static final String ACTION_CANCEL = "org.appdevforall.k2go.CODE_ADDONS_DOWNLOAD_CANCEL";
+ public static final String ACTION_RETRY = "org.appdevforall.k2go.CODE_ADDONS_DOWNLOAD_RETRY";
+
+ // The box runner reads the catalog itself and ignores the job items, so a single sentinel
+ // satisfies POST /code-addons/download (which requires a non-empty items/ids).
+ private static final String SENTINEL = "add-ons";
+
+ private static final ContentDownloadSession SESSION = new ContentDownloadSession("code-addons");
+
+ /** Adapts to the session listener; the UI passes {@code this::render}. */
+ public interface Listener { void onUpdate(); }
+
+ // ---- static API the UI observes (delegates to the shared session) --------------------------
+ public static boolean isRunning() { return SESSION.isRunning(); }
+ public static boolean isPaused() { return SESSION.isPaused(); }
+ public static boolean hasSession() { return SESSION.hasSession(); }
+ public static boolean isComplete() { return SESSION.isComplete(); }
+ public static boolean hasFailed() { return SESSION.hasFailed(); }
+ public static int percent() { return SESSION.percent(); }
+ public static long speed() { return SESSION.speed(); }
+ public static int reconnectAttempt() { return SESSION.reconnectAttempt(); }
+ public static int reconnectTotal() { return SESSION.reconnectTotal(); }
+ public static void setListener(Listener l) { SESSION.setListener(l == null ? null : l::onUpdate); }
+
+ public static void start(Context ctx) { send(ctx, ACTION_START); }
+ public static void pause(Context ctx) { if (SESSION.isRunning()) send(ctx, ACTION_PAUSE); }
+ public static void resume(Context ctx) { send(ctx, ACTION_RESUME); }
+ public static void cancel(Context ctx) { send(ctx, ACTION_CANCEL); }
+
+ /** Re-queue the failed item and resume (the box resumes the partial via aria2 --continue). */
+ public static void retry(Context ctx) {
+ if (SESSION.requeueFailed() && !SESSION.isRunning()) send(ctx, ACTION_RETRY);
+ }
+
+ public static void finishSession() { SESSION.purge(); }
+
+ private static void send(Context ctx, String action) {
+ ContextCompat.startForegroundService(ctx,
+ new Intent(ctx, AddonsDownloadService.class).setAction(action));
+ }
+
+ // ---- per-module hooks for ContentDownloadServiceBase ---------------------------------------
+ @Override protected ContentDownloadSession session() { return SESSION; }
+ @Override protected int notificationId() { return NOTIFICATION_ID; }
+ @Override protected String channelId() { return CHANNEL_ID; }
+ @Override protected String channelName() { return getString(R.string.k2go_card_code_addons); }
+ @Override protected String notifTitle() { return getString(R.string.k2go_code_addons_updating); }
+ @Override protected String notifText() { return getString(R.string.k2go_card_code_addons); }
+ @Override protected String actionPause() { return ACTION_PAUSE; }
+ @Override protected String actionResume() { return ACTION_RESUME; }
+ @Override protected String actionCancel() { return ACTION_CANCEL; }
+ @Override protected String actionRetry() { return ACTION_RETRY; }
+
+ @Override
+ protected void beginFromIntent(Intent intent) {
+ String[] keys = { "code-addons" };
+ String[] labels = { getString(R.string.k2go_card_code_addons) };
+ long[] sizes = { 0L }; // count-based: the UI shows the item percent directly
+ JSONObject[] bodies = new JSONObject[1];
+ try { bodies[0] = new JSONObject().put("ids", new JSONArray().put(SENTINEL)); }
+ catch (Exception e) { bodies[0] = new JSONObject(); }
+ SESSION.begin(keys, labels, sizes, bodies);
+ }
+}
diff --git a/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsRefresh.java b/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsRefresh.java
index 73c9b2370..8ea212278 100644
--- a/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsRefresh.java
+++ b/controller/app/src/main/java/org/appdevforall/k2go/addons/presentation/AddonsRefresh.java
@@ -3,17 +3,16 @@
* Name : AddonsRefresh.java
* Author : AppDevForAll
* Copyright : Copyright (c) 2026 AppDevForAll
- * Description : K2GO-99. The single "Update add-ons" flow, shared by the module detail button and the
- * module action sheet row so neither duplicates it. Modeled on ForgejoRepoRefresh: it
- * gates like the dashboard update (needs internet, then metered consent), shows minimal
- * inline progress (a description, a live one-line output tail, an indeterminate bar and a
- * Cancel) injected right after the trigger view, runs the box refresh on an IO thread, and
- * reports the outcome in a snackbar.
+ * Description : K2GO-99 / K2GO-443. The single "Update add-ons" flow, shared by the module detail button
+ * and the module action sheet row. It gates like the dashboard update (needs internet, then
+ * metered consent), then drives the durable job engine through AddonsDownloadService: a
+ * determinate progress bar (percent + speed) with Pause / Resume and Cancel, injected right
+ * after the trigger view. The download runs in a foreground service and on the box, so it
+ * survives this view going away: re-opening the sheet re-attaches to the running session.
*
- * Lifecycle: there is NO persistent app-side state. The box refresh job is detached
- * (setsid), so a host that goes away mid-run just drops the UI updates (guarded by
- * View.isAttachedToWindow()); the box finishes on its own and the gallery is swapped in
- * only on success. The only state is the box's own status/pid files, which the box manages.
+ * Lifecycle: no persistent app-side state here. Pause/resume/cancel state lives in the
+ * session + the box job; this view only observes it and re-renders. A terminal state
+ * (done / failed / cancelled) removes the inline UI and reports it in a snackbar.
* ============================================================================
*/
package org.appdevforall.k2go.addons.presentation;
@@ -35,8 +34,7 @@
import com.google.android.material.progressindicator.LinearProgressIndicator;
import org.appdevforall.k2go.R;
-import org.appdevforall.k2go.addons.data.AddonsRefreshClient;
-import org.appdevforall.k2go.util.AppExecutors;
+import org.appdevforall.k2go.util.ByteFormatter;
import org.appdevforall.k2go.util.Snackbars;
public final class AddonsRefresh {
@@ -44,7 +42,7 @@ public final class AddonsRefresh {
private AddonsRefresh() {}
/**
- * Gate (internet, then metered consent) then run the refresh with progress injected right after
+ * Gate (internet, then metered consent) then run the update with progress injected right after
* {@code trigger}. The trigger stays in place (only disabled) as a visible anchor for the snackbar.
*/
public static void start(@NonNull Activity act, @NonNull View trigger) {
@@ -59,36 +57,36 @@ public static void start(@NonNull Activity act, @NonNull View trigger) {
private static void run(@NonNull View trigger) {
final ViewGroup parent = (ViewGroup) trigger.getParent();
if (parent == null || !trigger.isAttachedToWindow()) return; // host went away during the gate
- final Context ctx = trigger.getContext();
+ final Context ctx = trigger.getContext().getApplicationContext();
final Handler main = new Handler(Looper.getMainLooper());
- final float d = ctx.getResources().getDisplayMetrics().density;
+ final float d = trigger.getResources().getDisplayMetrics().density;
final int side = Math.round(20 * d);
- final LinearLayout progress = new LinearLayout(ctx);
+ final LinearLayout progress = new LinearLayout(trigger.getContext());
progress.setOrientation(LinearLayout.VERTICAL);
LinearLayout.LayoutParams plp = new LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT);
plp.leftMargin = side; plp.rightMargin = side; plp.topMargin = Math.round(8 * d);
progress.setLayoutParams(plp);
- final TextView label = new TextView(ctx);
+ final TextView label = new TextView(trigger.getContext());
label.setText(R.string.k2go_code_addons_updating);
label.setTextAppearance(com.google.android.material.R.style.TextAppearance_Material3_BodySmall);
- label.setTextColor(ContextCompat.getColor(ctx, R.color.k2go_muted));
+ label.setTextColor(ContextCompat.getColor(trigger.getContext(), R.color.k2go_muted));
progress.addView(label);
- final TextView liveLine = new TextView(ctx);
- liveLine.setTextAppearance(com.google.android.material.R.style.TextAppearance_Material3_BodySmall);
- liveLine.setTextColor(ContextCompat.getColor(ctx, R.color.k2go_muted));
- liveLine.setMaxLines(1);
- liveLine.setEllipsize(TextUtils.TruncateAt.END);
- LinearLayout.LayoutParams llp = new LinearLayout.LayoutParams(
+ final TextView statusLine = new TextView(trigger.getContext());
+ statusLine.setTextAppearance(com.google.android.material.R.style.TextAppearance_Material3_BodySmall);
+ statusLine.setTextColor(ContextCompat.getColor(trigger.getContext(), R.color.k2go_muted));
+ statusLine.setMaxLines(1);
+ statusLine.setEllipsize(TextUtils.TruncateAt.END);
+ LinearLayout.LayoutParams slp = new LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT);
- llp.topMargin = Math.round(2 * d);
- liveLine.setLayoutParams(llp);
- progress.addView(liveLine);
+ slp.topMargin = Math.round(2 * d);
+ statusLine.setLayoutParams(slp);
+ progress.addView(statusLine);
- final LinearLayout barLine = new LinearLayout(ctx);
+ final LinearLayout barLine = new LinearLayout(trigger.getContext());
barLine.setOrientation(LinearLayout.HORIZONTAL);
barLine.setGravity(Gravity.CENTER_VERTICAL);
LinearLayout.LayoutParams barLineLp = new LinearLayout.LayoutParams(
@@ -96,56 +94,103 @@ private static void run(@NonNull View trigger) {
barLineLp.topMargin = Math.round(4 * d);
barLine.setLayoutParams(barLineLp);
- final LinearProgressIndicator bar = new LinearProgressIndicator(ctx);
+ final LinearProgressIndicator bar = new LinearProgressIndicator(trigger.getContext());
bar.setIndeterminate(true);
LinearLayout.LayoutParams blp = new LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f);
bar.setLayoutParams(blp);
barLine.addView(bar);
- final TextView cancel = new TextView(ctx);
- cancel.setText(R.string.k2go_dash_cancel);
- cancel.setAllCaps(true);
- cancel.setTextAppearance(com.google.android.material.R.style.TextAppearance_Material3_LabelLarge);
- cancel.setTextColor(ContextCompat.getColor(ctx, R.color.k2go_teal));
- int hp = Math.round(12 * d), vp = Math.round(6 * d);
- cancel.setPadding(hp, vp, hp, vp);
- barLine.addView(cancel);
+ final int hp = Math.round(12 * d), vp = Math.round(6 * d);
+ final TextView pauseBtn = new TextView(trigger.getContext());
+ pauseBtn.setText(R.string.k2go_dl_pause);
+ pauseBtn.setAllCaps(true);
+ pauseBtn.setTextAppearance(com.google.android.material.R.style.TextAppearance_Material3_LabelLarge);
+ pauseBtn.setTextColor(ContextCompat.getColor(trigger.getContext(), R.color.k2go_teal));
+ pauseBtn.setPadding(hp, vp, hp, vp);
+ barLine.addView(pauseBtn);
+
+ final TextView cancelBtn = new TextView(trigger.getContext());
+ cancelBtn.setText(R.string.k2go_dash_cancel);
+ cancelBtn.setAllCaps(true);
+ cancelBtn.setTextAppearance(com.google.android.material.R.style.TextAppearance_Material3_LabelLarge);
+ cancelBtn.setTextColor(ContextCompat.getColor(trigger.getContext(), R.color.k2go_teal));
+ cancelBtn.setPadding(hp, vp, hp, vp);
+ barLine.addView(cancelBtn);
progress.addView(barLine);
parent.addView(progress, parent.indexOfChild(trigger) + 1);
- trigger.setEnabled(false); // stays in place as an anchor; re-enabled when the refresh settles
+ trigger.setEnabled(false); // stays in place as an anchor; re-enabled when the update settles
- cancel.setOnClickListener(cv -> {
- cancel.setEnabled(false);
- label.setText(R.string.k2go_code_addons_update_cancelling);
- AppExecutors.get().io().execute(() -> new AddonsRefreshClient().cancel());
+ pauseBtn.setOnClickListener(v -> {
+ if (AddonsDownloadService.isPaused()) AddonsDownloadService.resume(ctx);
+ else AddonsDownloadService.pause(ctx);
});
-
- AppExecutors.get().io().execute(() -> {
- final AddonsRefreshClient client = new AddonsRefreshClient();
- final AddonsRefreshClient.Result r = client.refresh(rawLine -> {
- final String shown = rawLine.trim();
- main.post(() -> { if (liveLine.isAttachedToWindow()) liveLine.setText(shown); });
- });
- final int failed = client.lastFailed();
- final boolean upToDate = client.lastUpToDate();
- main.post(() -> {
- if (!trigger.isAttachedToWindow()) return;
- parent.removeView(progress);
- trigger.setEnabled(true);
- Snackbars.make(trigger, ctx.getString(messageFor(r, failed, upToDate))).show();
- });
+ cancelBtn.setOnClickListener(v -> {
+ cancelBtn.setEnabled(false);
+ AddonsDownloadService.cancel(ctx);
});
+
+ // Clear the session listener the moment this view leaves the window (sheet dismissed), so the
+ // static session never keeps a destroyed Activity alive while a download runs on. Deterministic:
+ // it does not wait for the next session event (a paused download emits none).
+ final View.OnAttachStateChangeListener detach = new View.OnAttachStateChangeListener() {
+ @Override public void onViewAttachedToWindow(@NonNull View v) {}
+ @Override public void onViewDetachedFromWindow(@NonNull View v) {
+ AddonsDownloadService.setListener(null);
+ }
+ };
+
+ final Runnable[] render = new Runnable[1];
+ render[0] = () -> {
+ if (!trigger.isAttachedToWindow()) { AddonsDownloadService.setListener(null); return; }
+ final boolean running = AddonsDownloadService.isRunning();
+ // hasFailed() BEFORE isComplete(): isComplete() is true for an all-FAILED session too
+ // (FAILED is "not in progress"), so a failed run must be caught first.
+ if (!running && AddonsDownloadService.hasFailed()) {
+ terminal(trigger, parent, progress, detach, R.string.k2go_code_addons_update_failed);
+ return;
+ }
+ if (AddonsDownloadService.isComplete()) {
+ terminal(trigger, parent, progress, detach, R.string.k2go_code_addons_update_done);
+ return;
+ }
+ if (!AddonsDownloadService.hasSession()) { // cancelled (purged)
+ terminal(trigger, parent, progress, detach, R.string.k2go_code_addons_update_cancelled);
+ return;
+ }
+ final boolean paused = AddonsDownloadService.isPaused();
+ final int pct = AddonsDownloadService.percent();
+ bar.setIndeterminate(pct < 0);
+ if (pct >= 0) bar.setProgressCompat(pct, true);
+ pauseBtn.setText(paused ? R.string.k2go_dl_resume : R.string.k2go_dl_pause);
+ if (AddonsDownloadService.reconnectAttempt() > 0) {
+ statusLine.setText(R.string.k2go_retrying);
+ } else if (paused) {
+ statusLine.setText(R.string.k2go_dl_paused);
+ } else {
+ final long spd = AddonsDownloadService.speed();
+ statusLine.setText(spd > 0
+ ? ByteFormatter.toHuman(spd) + "/s"
+ : trigger.getContext().getString(R.string.k2go_code_addons_updating));
+ }
+ };
+
+ trigger.addOnAttachStateChangeListener(detach);
+ // publish() already posts to the main thread, so the listener runs on main: wire it directly.
+ AddonsDownloadService.setListener(render[0]::run);
+ AddonsDownloadService.start(ctx);
+ main.post(render[0]); // initial paint
}
- /** Map the refresh outcome to a user message covering every state. */
- private static int messageFor(AddonsRefreshClient.Result r, int failed, boolean upToDate) {
- if (r == AddonsRefreshClient.Result.CANCELLED) return R.string.k2go_code_addons_update_cancelled;
- if (r != AddonsRefreshClient.Result.DONE) return R.string.k2go_code_addons_update_failed; // box unreachable
- if (failed > 0) return R.string.k2go_code_addons_update_some_failed; // some files could not be fetched
- // K2GO-441: the box is the single source for "nothing changed" (result: up-to-date). A box that
- // does not report it (pre-1.3.9) ran the old full mirror, so "updated" is the correct default.
- if (upToDate) return R.string.k2go_code_addons_update_none;
- return R.string.k2go_code_addons_update_done;
+ private static void terminal(@NonNull View trigger, @NonNull ViewGroup parent, @NonNull View progress,
+ @NonNull View.OnAttachStateChangeListener detach, int msgRes) {
+ AddonsDownloadService.setListener(null);
+ trigger.removeOnAttachStateChangeListener(detach);
+ if (progress.getParent() == parent) parent.removeView(progress);
+ trigger.setEnabled(true);
+ if (trigger.isAttachedToWindow()) {
+ Snackbars.make(trigger, trigger.getContext().getString(msgRes)).show();
+ }
+ AddonsDownloadService.finishSession();
}
}
diff --git a/static/dashboard/CHANGELOG.md b/static/dashboard/CHANGELOG.md
index ec9a929b3..c3655bf2b 100644
--- a/static/dashboard/CHANGELOG.md
+++ b/static/dashboard/CHANGELOG.md
@@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i
(the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this
file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`.
+- **1.3.12** - Add-ons gallery download on the durable job engine (K2GO-443). `code-addons` is now a job type: `POST /code-addons/download` plus `GET /code-addons/jobs/:id` (structured `{phase, percent, speed, detail}`) and pause/resume/retry/cancel over the generic `/:type/*` surface, like build-assets. The runner (`sockets/code_addons.exec.ts`) downloads only the heavy add-on binaries (.cgp + source tarballs) with aria2c (resilient: `--continue` resume, survives a network change) using the shared `downloadWithAria2` helper; the mirror stages the small files (shell, catalog, icons, pages) with its Cloudflare clean + catalog base rewrite and prints the aria2 input for the heavy ones (`mirror_addons.py --print-aria2-input`), then verifies them (`--finalize-only`), and the runner swaps the staged tree in atomically. The older `POST /addons/refresh` (wrapper) stays for now. Localhost-only. (K2GO-443)
- **1.3.11** - Build-assets download on the durable job engine (K2GO-443). `code-assets` is now a job type: `POST /code-assets/download` plus `GET /code-assets/jobs/:id` (structured `{phase, percent, speed, detail}`) and pause/resume/retry/cancel over the generic `/:type/*` surface, like kiwix/maps. The runner (`sockets/code_assets.exec.ts`) downloads the build assets with aria2c (resilient: `--continue` resume, survives a full interface loss via the outer retry loop) using the shared `downloadWithAria2` helper, then the mirror verifies each file against its published `.md5` and writes the browse page (`mirror_code_assets.py --finalize-only`), and the runner swaps the staged tree in atomically. The older `POST /code-assets/refresh` (wrapper) stays for now. Localhost-only. (K2GO-443)
- **1.3.10** - Code on the Go build-assets refresh (K2GO-437). New `POST /code-assets/refresh` re-mirrors the Code on the Go build assets into `/library/www/code-assets` LIVE (box up, no runrole) through a detached wrapper (`tools/code-assets-refresh.sh`) that mirrors into a staging dir and swaps it in only on success (a failed or cancelled refresh never serves a half-mirror), `GET /code-assets/refresh/status` returns `{state, lines, downloaded, reused, failed, upToDate}` to poll (counts parsed from the mirror's final log line; `upToDate` when nothing changed), and `POST /code-assets/refresh/cancel` stops a running refresh (SIGKILL to the wrapper process group; safe because the live tree is only replaced after a clean run). The wrapper reuses the role's `mirror_code_assets.py` (manifest-driven, per-file `.md5` verify and reuse, generates the browse page), so install (bake) and refresh (live) share one mechanism. Localhost-only. (K2GO-437)
- **1.3.9** - Incremental add-ons refresh with an "up to date" report (K2GO-441). "Update add-ons" no longer re-downloads the whole gallery. `mirror_addons.py` takes `--reuse-from` the live tree and prints `result: up-to-date` when the published catalog is unchanged (nothing downloaded, no swap); it reads what changed straight from `catalog.json` (the single source of truth): it reuses unchanged `.cgp` and source tarballs by the catalog sha256, reuses each add-on's icon and page (both live inside the source tarball) when that sha is unchanged, and reuses shell assets by their content-hashed name, so only new or changed files download, with no local re-hash and no extra requests. `GET /addons/refresh/status` now also returns `{reused, upToDate}`. The app shows "already up to date" when nothing came down. (K2GO-441)
diff --git a/static/dashboard/package.json b/static/dashboard/package.json
index f325b99ab..48a5ce80e 100644
--- a/static/dashboard/package.json
+++ b/static/dashboard/package.json
@@ -1,6 +1,6 @@
{
"name": "dashboard-console",
- "version": "1.3.11",
+ "version": "1.3.12",
"description": "",
"main": "index.js",
"scripts": {
diff --git a/static/dashboard/routes.ts b/static/dashboard/routes.ts
index 354404e56..a39ff56c1 100644
--- a/static/dashboard/routes.ts
+++ b/static/dashboard/routes.ts
@@ -40,7 +40,7 @@ const ZIMS_DIR = '/library/zims/content/';
const KIWIX_INDEXER = '/usr/bin/iiab-make-kiwix-lib';
const ZIM_NAME_RE = /^[A-Za-z0-9._-]{1,150}\.zim$/;
-const VALID_TYPES: JobType[] = ['kiwix', 'maps', 'books', 'kolibri', 'basemaps', 'code-assets'];
+const VALID_TYPES: JobType[] = ['kiwix', 'maps', 'books', 'kolibri', 'basemaps', 'code-assets', 'code-addons'];
function isType(t: string): t is JobType {
return (VALID_TYPES as string[]).includes(t);
}
@@ -1117,13 +1117,14 @@ apiRouter.post('/:type/download', (req: Request, res: Response): void => {
? body.items
: Array.isArray(body?.ids) ? body.ids : [];
if (items.length === 0) { res.status(400).json({ error: 'items (or ids) required' }); return; }
- // K2GO-443: code-assets stages into one shared tree (/library/www/code-assets.new), unlike kiwix's
- // independent files, so only one build-assets job may run at a time. The app re-attaches via
+ // K2GO-443: these content types stage into ONE shared tree (/library/www/.new), unlike kiwix's
+ // independent files, so only one job per such type may run at a time. The app re-attaches via
// start-or-attach; this is the hard guard behind it (two concurrent jobs would corrupt the staging).
- if (type === 'code-assets'
- && jobs.list('code-assets').some((j) =>
+ const SINGLE_TREE_TYPES = ['code-assets', 'code-addons'];
+ if (SINGLE_TREE_TYPES.includes(type)
+ && jobs.list(type).some((j) =>
['queued', 'downloading', 'indexing', 'processing', 'paused'].includes(j.phase))) {
- res.status(409).json({ error: 'a build-assets job is already running' });
+ res.status(409).json({ error: `a ${type} job is already running` });
return;
}
res.status(202).json(toApi(jobs.create(type, items)));
diff --git a/static/dashboard/server.ts b/static/dashboard/server.ts
index 0f9d3ee51..9ecf4b179 100644
--- a/static/dashboard/server.ts
+++ b/static/dashboard/server.ts
@@ -11,6 +11,7 @@ import './sockets/maps-base.exec';
import './sockets/books.exec';
import './sockets/kolibri.exec';
import './sockets/code_assets.exec'; // K2GO-443: build-assets runner (aria2 job engine)
+import './sockets/code_addons.exec'; // K2GO-443: add-ons gallery runner (aria2 job engine)
import { apiRouter } from './routes';
import { startServiceHeal } from './sockets/service-heal';
import { startLogRotation, stopLogRotation } from './sockets/log-rotate';
diff --git a/static/dashboard/sockets/code_addons.exec.ts b/static/dashboard/sockets/code_addons.exec.ts
new file mode 100644
index 000000000..df9b83d6d
--- /dev/null
+++ b/static/dashboard/sockets/code_addons.exec.ts
@@ -0,0 +1,144 @@
+// sockets/code_addons.exec.ts - K2GO-443
+//
+// Code on the Go add-ons gallery runner for the durable job engine. Replaces the fire-and-forget
+// wrapper (tools/code-addons-refresh.sh + POST /addons/refresh + status-file poll) with an aria2
+// download that gives percent / speed / pause / resume / retry and survives a network change, like
+// build-assets / kiwix / maps. The gallery is a mix: many small files (shell, catalog, icons, pages)
+// that need a transform (Cloudflare clean + catalog base rewrite), and the heavy add-on binaries
+// (.cgp + source tarballs). Only the heavy binaries go through aria2; the mirror stages the rest.
+//
+// The mirror splits into a plan step and a verify step (mirror_addons.py):
+// --print-aria2-input: stage the small files, print an aria2 input-file for the heavy binaries. Its
+// first stdout line is a status marker. '#status=uptodate' means the same build is already served
+// (keep the live tree, do not swap). '#status=stage' means STAGE is populated and the lines after
+// are the aria2 input, which may be EMPTY when only small files changed (still finalize and swap).
+// --finalize-only: verify the aria2-downloaded binaries are present in STAGE (aria2 already verified
+// each by its sha-256).
+// The runner then swaps the staged tree in atomically, restoring the previous tree if the swap fails.
+import { jobs, RunnerContext, CanceledError, PausedError } from './jobs';
+import { downloadWithAria2, stopError } from './aria2-download';
+import { execFileSync } from 'child_process';
+import fs from 'fs';
+
+const SERVE = '/library/www/code-addons';
+const STAGE = `${SERVE}.new`;
+const OLD = `${SERVE}.old`;
+const SERVE_BASE = '/code-addons'; // the catalog base the mirror rewrites to
+// K2GO-440: prefer the mirror from the self-updating clone so a mirror fix ships via the dash-node
+// rebuild with no rebake; fall back to the bake-time copy in the ansible roles dir.
+const MIRROR_CLONE = '/opt/iiab-android/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py';
+const MIRROR_BAKED = '/opt/iiab/iiab/roles/code_addons/files/mirror_addons.py';
+
+function mirrorPath(): string {
+ return fs.existsSync(MIRROR_CLONE) ? MIRROR_CLONE : MIRROR_BAKED;
+}
+
+function rmrf(p: string): void {
+ try { fs.rmSync(p, { recursive: true, force: true }); } catch { /* best effort */ }
+}
+
+/** Remove aria2 control/metadata so the served tree is only the gallery files. */
+function cleanAria2Files(dir: string): void {
+ try { execFileSync('find', [dir, '-name', '*.aria2', '-delete']); } catch { /* best effort */ }
+}
+
+/** Run the mirror and resolve with its stdout; rejects (paused/canceled/error) like a download step. */
+function runMirror(ctx: RunnerContext, mirror: string, args: string[], capture: boolean): Promise {
+ return new Promise((resolve, reject) => {
+ let out = '';
+ const p = ctx.spawn('python3', [mirror, ...args]);
+ p.stdout?.on('data', (d: Buffer) => { if (capture) out += d.toString(); else ctx.log(d.toString().trim()); });
+ p.stderr?.on('data', (d: Buffer) => ctx.log(d.toString().trim()));
+ p.on('error', reject);
+ p.on('exit', (code, signal) => {
+ if (signal === 'SIGKILL' || ctx.isCanceled()) return reject(new CanceledError());
+ if (code === 0) return resolve(out);
+ reject(new Error(`${args[0]} failed (exit ${code})`));
+ });
+ });
+}
+
+const codeAddonsRunner: (ctx: RunnerContext) => Promise = async (ctx) => {
+ const MIRROR = mirrorPath();
+ if (!fs.existsSync(MIRROR)) throw new Error(`mirror script not found: ${MIRROR}`);
+
+ // --- Plan (incremental) -------------------------------------------------
+ // The mirror stages the small files into STAGE from the published site + the live tree and prints a
+ // status marker plus an aria2 input-file for the heavy binaries. STAGE is NOT pre-cleared, so a
+ // resume keeps its partials; the plan is deterministic and safe to re-run.
+ ctx.update({ phase: 'downloading', percent: -1, speed: 0, detail: 'add-ons' });
+ let planned: string;
+ try {
+ planned = await runMirror(ctx, MIRROR,
+ ['--print-aria2-input', '--reuse-from', SERVE, '--out', STAGE, '--serve-base', SERVE_BASE], true);
+ } catch (e) {
+ const se = stopError(ctx);
+ if (se instanceof PausedError) throw se;
+ if (se instanceof CanceledError) { rmrf(STAGE); throw se; }
+ throw e;
+ }
+ ctx.throwIfCanceled();
+
+ const nl = planned.indexOf('\n');
+ const marker = (nl === -1 ? planned : planned.slice(0, nl)).trim();
+ const body = (nl === -1 ? '' : planned.slice(nl + 1)).trim();
+
+ if (marker === '#status=uptodate') {
+ // Same published build: keep the live tree, nothing to download or swap.
+ rmrf(STAGE);
+ ctx.update({ phase: 'done', percent: 100, speed: 0, detail: 'up to date' });
+ return;
+ }
+
+ // --- Download the changed heavy binaries (aria2, resilient) -------------
+ // body may be empty when only small files changed: skip the download, but still finalize and swap
+ // so the staged small-file changes go live.
+ if (body !== '') {
+ const inputFile = `${STAGE}/.aria2-input`;
+ fs.mkdirSync(STAGE, { recursive: true });
+ fs.writeFileSync(inputFile, body + '\n');
+ try {
+ await downloadWithAria2(ctx, { destDir: STAGE, inputFile, phase: 'downloading' });
+ } catch (e) {
+ const se = stopError(ctx);
+ if (se instanceof PausedError) throw se; // keep STAGE: resume continues
+ if (se instanceof CanceledError) { rmrf(STAGE); throw se; } // cancel discards the partial
+ throw e; // real error: keep STAGE for a retry/resume
+ }
+ try { fs.rmSync(inputFile, { force: true }); } catch { /* leave nothing non-served behind */ }
+ }
+ ctx.throwIfCanceled();
+
+ // --- Verify (aria2 checked each binary by sha-256; no re-hash) ----------
+ ctx.update({ phase: 'processing', percent: -1, speed: 0, detail: 'finishing' });
+ try {
+ await runMirror(ctx, MIRROR, ['--finalize-only', '--out', STAGE, '--serve-base', SERVE_BASE], false);
+ } catch (e) {
+ const se = stopError(ctx);
+ if (se) throw se;
+ throw e;
+ }
+ ctx.throwIfCanceled();
+
+ // --- Swap (atomic rename; restore the previous tree if the move fails) --
+ ctx.update({ phase: 'processing', percent: 100, speed: 0, detail: 'installing' });
+ cleanAria2Files(STAGE);
+ rmrf(OLD);
+ if (fs.existsSync(SERVE)) fs.renameSync(SERVE, OLD);
+ try {
+ fs.renameSync(STAGE, SERVE);
+ } catch (e) {
+ // A failed swap must never leave the box with no served gallery: put the previous tree back.
+ if (!fs.existsSync(SERVE) && fs.existsSync(OLD)) {
+ try { fs.renameSync(OLD, SERVE); } catch { /* best effort */ }
+ }
+ throw e;
+ }
+ rmrf(OLD);
+
+ ctx.update({ phase: 'done', percent: 100, speed: 0 });
+};
+
+jobs.registerRunner('code-addons', codeAddonsRunner);
+
+export { codeAddonsRunner };
diff --git a/static/dashboard/sockets/jobs.ts b/static/dashboard/sockets/jobs.ts
index fad004693..b92b9805a 100644
--- a/static/dashboard/sockets/jobs.ts
+++ b/static/dashboard/sockets/jobs.ts
@@ -13,7 +13,7 @@ import fs from 'fs';
import path from 'path';
import { RollingLog, LogSlice } from './rolling-log';
-export type JobType = 'kiwix' | 'maps' | 'books' | 'kolibri' | 'basemaps' | 'code-assets';
+export type JobType = 'kiwix' | 'maps' | 'books' | 'kolibri' | 'basemaps' | 'code-assets' | 'code-addons';
export type JobPhase =
| 'queued' | 'downloading' | 'indexing' | 'processing'
// ADFA-4894 (control surface): 'paused' is a stopped-but-resumable state — like 'canceled' it
diff --git a/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py b/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py
index a992fa9a8..eb4e99828 100644
--- a/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py
+++ b/tools/upstream-patches/overlays/roles/code_addons/files/mirror_addons.py
@@ -285,8 +285,16 @@ def build_plan(source_base, catalog, served, index_unchanged, index_bytes):
return paths, sha, size, reuse_ok
-def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0,
- verbose=True):
+def _stage(source_base, serve_base, out, reuse_from, heavy_sink, limit_large, log):
+ """Stage the gallery into `out` and return a result dict (the single staging core).
+
+ Fetches the small files (index, catalog, schema, icons, pages, shell), Cloudflare-cleans the
+ HTML, copies what the catalog proves is reusable, and rewrites the catalog base. When `heavy_sink`
+ is not None, a verified binary that would be downloaded (.cgp / source tarball) is NOT fetched:
+ its (url, rel, sha256, size) is appended to `heavy_sink` so a caller can hand it to aria2 (K2GO-443).
+ All human output goes through `log(msg, err=False)` so a caller can route it off stdout. Returns
+ {up_to_date, downloaded, reused, failed, headed}; on the same-build short-circuit only up_to_date
+ is meaningful."""
source_base = source_base.rstrip("/")
serve_base = serve_base.rstrip("/")
out = Path(out)
@@ -295,16 +303,12 @@ def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0,
catalog_bytes = fetch(f"{source_base}/{CATALOG}")
rewritten_catalog = catalog_bytes.decode("utf-8").replace(source_base, serve_base)
- # K2GO-441: same-build short-circuit. The catalog carries `generated` (a build
- # timestamp), so a byte-identical catalog means the exact same published build:
- # nothing changed, so there is nothing to download and nothing to swap. The
- # wrapper reads "result: up-to-date" and keeps the live gallery.
+ # K2GO-441: same-build short-circuit. The catalog carries `generated` (a build timestamp), so a
+ # byte-identical catalog means the exact same published build: nothing to download, nothing to swap.
if reuse_from is not None:
try:
if (reuse_from / CATALOG).read_text(encoding="utf-8") == rewritten_catalog:
- print("result: up-to-date")
- print("done: 0 downloaded, 0 reused, 0 failed")
- return True
+ return {"up_to_date": True, "downloaded": 0, "reused": 0, "failed": 0, "headed": 0}
except OSError:
pass # no served catalog yet: fall through to a full mirror.
@@ -325,9 +329,8 @@ def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0,
paths, sha, size, reuse_ok = build_plan(
source_base, catalog, served, index_unchanged, index_bytes)
- if verbose:
- print(f"plan: {len(paths)} files, {len(sha)} verified binaries, "
- f"{len(catalog['addons'])} add-ons")
+ log(f"plan: {len(paths)} files, {len(sha)} verified binaries, "
+ f"{len(catalog['addons'])} add-ons")
downloaded = reused = headed = failed = 0
@@ -356,14 +359,19 @@ def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0,
reused += 1
continue
+ # K2GO-443: defer a verified binary (.cgp / tarball) to aria2 instead of fetching its body
+ # here, so the heavy download gets resume / pause / retry. aria2 verifies it by checksum=sha-256.
+ if heavy_sink is not None and rel in sha:
+ heavy_sink.append((url, rel, sha[rel], size.get(rel)))
+ continue
+
# Test only: HEAD a big binary instead of pulling its body.
if limit_large and size.get(rel) and size[rel] > limit_large:
st, length = head(url)
ok = st == 200 and (length == size[rel] or length is None)
headed += 1
- if verbose:
- mark = "ok" if ok else f"MISMATCH(status={st},len={length})"
- print(f" HEAD {rel} ({size[rel]} B): {mark}")
+ mark = "ok" if ok else f"MISMATCH(status={st},len={length})"
+ log(f" HEAD {rel} ({size[rel]} B): {mark}")
if not ok:
failed += 1
continue
@@ -372,40 +380,120 @@ def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0,
data = fetch(url)
except RuntimeError as e:
failed += 1
- print(f" FAIL {rel}: {e}", file=sys.stderr)
+ log(f" FAIL {rel}: {e}", err=True)
continue
if rel in sha:
got = sha256_of(data)
if got != sha[rel]:
failed += 1
- print(f" SHA MISMATCH {rel}: got {got[:12]} "
- f"want {sha[rel][:12]}", file=sys.stderr)
+ log(f" SHA MISMATCH {rel}: got {got[:12]} "
+ f"want {sha[rel][:12]}", err=True)
continue
if len(data) != size[rel]:
failed += 1
- print(f" SIZE MISMATCH {rel}: got {len(data)} "
- f"want {size[rel]}", file=sys.stderr)
+ log(f" SIZE MISMATCH {rel}: got {len(data)} "
+ f"want {size[rel]}", err=True)
continue
if rel.endswith(".html"):
data = clean_html(data)
dest.write_bytes(data)
downloaded += 1
- if verbose and (downloaded % 10 == 0 or rel in sha):
+ if downloaded % 10 == 0 or rel in sha:
tag = " (verified)" if rel in sha else ""
- print(f" GET {rel} ({len(data)} B){tag}")
+ log(f" GET {rel} ({len(data)} B){tag}")
# The one transform: point the catalog at the local serve base.
catalog_file = out / CATALOG
if catalog_file.exists():
n = catalog_file.read_text(encoding="utf-8").count(source_base)
catalog_file.write_text(rewritten_catalog, encoding="utf-8")
- if verbose:
- print(f"rewrote catalog base: {source_base} -> {serve_base} "
- f"({n} occurrences)")
+ log(f"rewrote catalog base: {source_base} -> {serve_base} "
+ f"({n} occurrences)")
- tail = f", {headed} head-checked" if headed else ""
- print(f"done: {downloaded} downloaded, {reused} reused{tail}, {failed} failed")
- return failed == 0
+ return {"up_to_date": False, "downloaded": downloaded, "reused": reused,
+ "failed": failed, "headed": headed}
+
+
+def mirror(source_base, serve_base, out, reuse_from=None, limit_large=0,
+ verbose=True):
+ """Full in-process mirror (bake and the legacy refresh wrapper). Prints the same stdout as before:
+ progress when verbose, then 'result: up-to-date' or the 'done: ...' summary the wrapper parses."""
+ def log(msg, err=False):
+ if err:
+ print(msg, file=sys.stderr)
+ elif verbose:
+ print(msg)
+
+ res = _stage(source_base, serve_base, out, reuse_from, None, limit_large, log)
+ if res["up_to_date"]:
+ print("result: up-to-date")
+ print("done: 0 downloaded, 0 reused, 0 failed")
+ return True
+ tail = f", {res['headed']} head-checked" if res["headed"] else ""
+ print(f"done: {res['downloaded']} downloaded, {res['reused']} reused{tail}, "
+ f"{res['failed']} failed")
+ return res["failed"] == 0
+
+
+def aria2_input(source_base, serve_base, out, reuse_from=None):
+ """K2GO-443: stage everything except the heavy verified binaries into `out`, and print an aria2
+ input-file (the .cgp / tarball downloads) to STDOUT for the durable job runner. Line 1 is a status
+ marker: '#status=uptodate' (same build: keep the live tree, do not swap) or '#status=stage' (out is
+ staged; the lines after are the aria2 input). The input body may be EMPTY when only small files
+ changed: the runner must still finalize and swap. Heavy entries use an absolute per-entry dir= plus
+ a basename out= (aria2 drops a subdir in out=), and checksum=sha-256 so aria2 verifies each file.
+ All human logs go to stderr so stdout carries only the marker and the input."""
+ def log(msg, err=False):
+ print(msg, file=sys.stderr)
+
+ heavy = []
+ res = _stage(source_base, serve_base, out, reuse_from, heavy, 0, log)
+ if res["up_to_date"]:
+ print("#status=uptodate")
+ return True
+ lines = ["#status=stage"]
+ out = Path(out)
+ for url, rel, want_sha, _want_size in heavy:
+ p = Path(rel)
+ lines.append(url)
+ lines.append(f" dir={out}/{p.parent}")
+ lines.append(f" out={p.name}")
+ lines.append(f" checksum=sha-256={want_sha}")
+ sys.stdout.write("\n".join(lines) + "\n")
+ return res["failed"] == 0
+
+
+def finalize(serve_base, out, verbose=True):
+ """K2GO-443: verify the heavy binaries aria2 downloaded are present in `out` with the catalog's
+ size. The small files and the catalog base-rewrite were already staged by --print-aria2-input, and
+ aria2 verified each heavy file by its sha-256, so this only confirms presence/size and reports. The
+ staged catalog is rewritten to serve_base, so its urls are serve-relative."""
+ serve_base = serve_base.rstrip("/")
+ out = Path(out)
+ try:
+ catalog = json.loads((out / CATALOG).read_text(encoding="utf-8"))
+ except (OSError, ValueError) as e:
+ print(f"finalize: cannot read staged catalog: {e}", file=sys.stderr)
+ return False
+ checked = missing = 0
+ for addon in catalog.get("addons", []):
+ for key in ("download", "sourceTarball"):
+ entry = addon.get(key) or {}
+ rel = rel_to_base(entry.get("url"), serve_base) if entry.get("url") else None
+ if not rel:
+ continue
+ checked += 1
+ f = out / rel
+ if not f.is_file():
+ missing += 1
+ print(f" MISSING {rel}", file=sys.stderr)
+ continue
+ want_size = entry.get("size")
+ if want_size is not None and f.stat().st_size != want_size:
+ missing += 1
+ print(f" SIZE {rel}: {f.stat().st_size} != {want_size}", file=sys.stderr)
+ print(f"done: {checked - missing} present, {missing} missing")
+ return missing == 0
def main(argv=None):
@@ -421,8 +509,23 @@ def main(argv=None):
ap.add_argument("--limit-large", type=int, default=0,
help="test only: HEAD files larger than this many bytes "
"instead of downloading them (0 = download all)")
+ # K2GO-443: the durable job engine downloads the heavy binaries with aria2 (resume / pause / retry),
+ # so the mirror splits into a plan step and a verify step. --print-aria2-input stages the small files
+ # and prints the aria2 input for the .cgp / tarballs; --finalize-only verifies them after aria2 runs.
+ ap.add_argument("--print-aria2-input", action="store_true",
+ help="stage the small files and print an aria2 input-file for the heavy binaries "
+ "(for the job runner); stdout is a status marker plus the input")
+ ap.add_argument("--finalize-only", action="store_true",
+ help="verify the aria2-downloaded binaries are present in --out (used after the "
+ "job-engine download); no download")
ap.add_argument("--quiet", action="store_true")
args = ap.parse_args(argv)
+
+ if args.print_aria2_input:
+ return 0 if aria2_input(args.source_base, args.serve_base, args.out,
+ reuse_from=args.reuse_from) else 1
+ if args.finalize_only:
+ return 0 if finalize(args.serve_base, args.out, verbose=not args.quiet) else 1
ok = mirror(args.source_base, args.serve_base, args.out,
reuse_from=args.reuse_from, limit_large=args.limit_large,
verbose=not args.quiet)