From f189a26f85e717cbfe3a2b2a6ddedc673b538ddd Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 7 Nov 2024 13:16:46 +0800 Subject: [PATCH 01/29] Document the job header and its contents --- dashboard/dashboard.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index a0304387..28187761 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -417,6 +417,9 @@

This page shows all of the crawls that ArchiveBot is currently running.

+

+ Each job has a header with the URL, note, request/queue stats, options and an identifier. +

To show or hide a job, click anywhere on its stats line. From 6c7dfe38a43b59c9cbd7bbcab42e041528acd80f Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:09:59 +0800 Subject: [PATCH 02/29] Refer to the job header rather than stats line --- dashboard/dashboard.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 28187761..077585bc 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -421,9 +421,9 @@ Each job has a header with the URL, note, request/queue stats, options and an identifier.

- To show or hide a job, click anywhere on its stats line. + To show or hide a job, click anywhere on its header. - The color coding for the job stats line is: + The color coding for the job header is: in progress, finished normally, finished with abort, From 49a8fc03a0c41a182dda8159a03af9c627507258 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:11:29 +0800 Subject: [PATCH 03/29] Clarify that the filter operates on job logs and is a regex --- dashboard/dashboard.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 077585bc..31bf1fbf 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -398,7 +398,7 @@ ArchiveBot tracking 0 crawls. See also pipeline or job reports. - Show: + Logs @@ -442,12 +442,12 @@ Keyboard shortcuts:

From 1ac40fb4fc52107caa59210fc16af337c3f8be1d Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:50:27 +0800 Subject: [PATCH 04/29] Document the title URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 31bf1fbf..83593486 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -467,6 +467,7 @@

  • To retain more lines in the log windows, add ?historyLines=1000 to the dashboard URL. The default is 500, or 250 on mobile.
  • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
  • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
    • +
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From 1fea779ba24a00b7a48eb2ec112affa4601fa0b3 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:50:52 +0800 Subject: [PATCH 05/29] Document the host URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 83593486..9516842c 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -468,6 +468,7 @@

  • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
  • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
    • +
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From fb017243e81cdb5b6671f753add0da0613c05d76 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:51:20 +0800 Subject: [PATCH 06/29] Document the dumpMax URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 9516842c..42585ff6 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -469,6 +469,7 @@

  • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
    • +
  • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From fe843b4c9679b63d87dadecd96b0b03faea190f3 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:51:34 +0800 Subject: [PATCH 07/29] Document the debug URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 42585ff6..c562c306 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -470,6 +470,7 @@

  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
  • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.
    • +
  • To enable debug messages in the browser console, add ?debug=1 to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From 04124216512866be6ac1e097a4907bb376f6b540 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:23:53 +0800 Subject: [PATCH 08/29] Add job notes to job URL title Makes it easier to access the notes when not in aligned mode. --- dashboard/assets/scripts/dashboard.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index f570f438..86d020df 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -225,10 +225,11 @@ class JobsTracker { } class JobRenderInfo { - constructor(logWindow, logSegment, statsElements, jobNote, lineCountWindow, lineCountSegments) { + constructor(logWindow, logSegment, statsElements, jobUrl, jobNote, lineCountWindow, lineCountSegments) { this.logWindow = logWindow; this.logSegment = logSegment; this.statsElements = statsElements; + this.jobUrl = jobUrl; this.jobNote = jobNote; this.lineCountWindow = lineCountWindow; this.lineCountSegments = lineCountSegments; @@ -411,13 +412,14 @@ class JobsRenderer { ], ), ]); + const jobUrl = statsElements.jobInfo.querySelector(".job-url"); const logWindow = h("div", logWindowAttrs, logSegment); const div = h("div", { className: "log-container", id: `log-container-${ident}` }, [ h("div", { className: "job-header" }, [statsElements.jobInfo, h("span", { className: "job-ident" }, ident)]), logWindow, ]); - this.renderInfo[ident] = new JobRenderInfo(logWindow, logSegment, statsElements, jobNote, 0, [0]); + this.renderInfo[ident] = new JobRenderInfo(logWindow, logSegment, statsElements, jobUrl, jobNote, 0, [0]); this.container.insertBefore(div, beforeElement); // Filter hasn't changed, but we might need to filter out the new job, or // add/remove log-window-expanded class @@ -580,6 +582,11 @@ class JobsRenderer { // Update note info.jobNote.textContent = isBlank(jobData.note) ? "" : ` (${jobData.note})`; + if (isBlank(jobData.note)) { + info.jobUrl.removeAttribute("title"); + } else { + info.jobUrl.title = jobData.note; + } info.lineCountWindow += linesRendered; info.lineCountSegments[info.lineCountSegments.length - 1] += linesRendered; From 8f048662ef008cd8b9bb9850744d488a43025a75 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:48:57 +0800 Subject: [PATCH 09/29] Document the mouseover info for the URL and queue count --- dashboard/dashboard.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index c562c306..55eb8389 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -430,7 +430,7 @@ finished with fatal exception.

    - Mouse over the job start date or the response count for additional information. + Mouse over the job URL, start date, response count or the queue count for additional information.

    To pause scrolling, move your mouse inside a log window. From 5713ae8487f5a8f860113dc13e84ae6ed30613c2 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Wed, 16 Oct 2024 13:02:38 +0800 Subject: [PATCH 10/29] Compile the filter regex only once when filtering jobs Previously it was compiled once for each job being filtered. Suggested-by: JustAnotherArchivist Fixes: commit 990b70d2d62b6528e1762b91e8baf6fbc0066807 --- dashboard/assets/scripts/dashboard.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 86d020df..0cdbe656 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -628,14 +628,14 @@ class JobsRenderer { } applyFilter() { - const query = this.filterBox.value; + const query = RegExp(this.filterBox.value); let matches = 0; const matchedWindows = []; const unmatchedWindows = []; this.firstFilterMatch = null; for (const job of this.jobs.sorted) { const w = this.renderInfo[job.ident].logWindow; - if (!RegExp(query).test(job.url)) { + if (!query.test(job.url)) { w.classList.add("log-window-hidden"); unmatchedWindows.push(w); From 40fd5cb7b2ee4e4adcf253a2f91208d07ba8acc7 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:13:28 +0800 Subject: [PATCH 11/29] Add identifiers for the filter modification buttons --- dashboard/dashboard.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 55eb8389..aafe75bc 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -399,8 +399,8 @@ tracking 0 crawls. See also pipeline or job reports. Logs - - + +

    😊
    From 31f65e5bd345bf21e256b05524492eb30c931c5d Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:21:58 +0800 Subject: [PATCH 12/29] Add a button and key to revert to the initial filter When the initial filter is your nick, this makes it easier to switch between monitoring only your personal jobs and monitoring all jobs or other jobs. --- dashboard/assets/scripts/dashboard.js | 18 ++++++++++++++++-- dashboard/dashboard.html | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 0cdbe656..f508402b 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1027,7 +1027,7 @@ class Dashboard { const batchMaxItems = args.batchMaxItems ? Number(args.batchMaxItems) : 250; const showNicks = args.showNicks ? Boolean(Number(args.showNicks)) : false; const contextMenu = args.contextMenu ? Boolean(Number(args.contextMenu)) : true; - const initialFilter = args.initialFilter ?? "^$"; + this.initialFilter = args.initialFilter ?? "^$"; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; this.debug = args.debug ? Boolean(Number(args.debug)) : false; @@ -1076,7 +1076,19 @@ class Dashboard { addPageStyles(".job-nick-aligned { width: 0; }"); } - this.setFilter(initialFilter); + if (args.initialFilter != null) { + byId("set-filter-none").after( + h("input", { + className: "button", + type: "button", + id: "set-filter-initial", + onclick: () => { ds.setFilter(ds.initialFilter) }, + value: "Initial", + }) + ); + byId("set-filter-none").after("\n"); + } + this.setFilter(this.initialFilter); const finishSetup = () => { byId("meta-info").innerHTML = ""; @@ -1198,6 +1210,8 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; ev.preventDefault(); byId("filter-box").focus(); byId("filter-box").select(); + } else if (ev.which === 105 /* i */) { + ds.setFilter(ds.initialFilter); } else if (ev.which === 118 /* v */) { window.open(this.jobsRenderer.firstFilterMatch.url); } diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index aafe75bc..a9592db0 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -447,6 +447,7 @@
  • a - show all job log
  • n - hide all job log
  • f - move focus to filter box +
  • i - use the initial job log filter
  • v - open the job URL of the first-shown job log
  • ? - show/hide help text From f2d89ad60abc3c3f79ef2d01a0cf0dce8373c49a Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:31:37 +0800 Subject: [PATCH 13/29] Allow hiding the job headers for hidden job logs This saves a lot of space when a lot of job logs are hidden. --- dashboard/assets/scripts/dashboard.js | 10 ++++++++++ dashboard/dashboard.html | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index f508402b..4d11f0bd 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1028,6 +1028,7 @@ class Dashboard { const showNicks = args.showNicks ? Boolean(Number(args.showNicks)) : false; const contextMenu = args.contextMenu ? Boolean(Number(args.contextMenu)) : true; this.initialFilter = args.initialFilter ?? "^$"; + const showAllHeaders = args.showAllHeaders ? Boolean(Number(args.showAllHeaders)) : true; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; this.debug = args.debug ? Boolean(Number(args.debug)) : false; @@ -1090,6 +1091,8 @@ class Dashboard { } this.setFilter(this.initialFilter); + this.showAllHeaders(showAllHeaders); + const finishSetup = () => { byId("meta-info").innerHTML = ""; @@ -1214,6 +1217,8 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; ds.setFilter(ds.initialFilter); } else if (ev.which === 118 /* v */) { window.open(this.jobsRenderer.firstFilterMatch.url); + } else if (ev.which === 104 /* h */) { + ds.showAllHeaders(!byId("show-all-headers").checked); } } @@ -1270,6 +1275,11 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; byId("filter-box").value = value; byId("filter-box").onchange(); } + + showAllHeaders(value) { + byId('show-all-headers').checked = value; + byId('hide-headers').sheet.disabled = value; + } } const ds = new Dashboard(); diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index a9592db0..5ff08c1b 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -391,6 +391,12 @@ top: 0; } +
    @@ -401,6 +407,8 @@ Logs + +
    😊
    @@ -449,6 +457,7 @@
  • f - move focus to filter box
  • i - use the initial job log filter
  • v - open the job URL of the first-shown job log +
  • h - show/hide headers for hidden job logs
  • ? - show/hide help text

    @@ -465,6 +474,7 @@

    • To specify an initial filter, add ?initialFilter=TEXT to the dashboard URL. The default is ^$.
      • +
    • To initially hide headers for hidden job logs, add ?showAllHeaders=0 to the dashboard URL. The default is to show them.
    • To retain more lines in the log windows, add ?historyLines=1000 to the dashboard URL. The default is 500, or 250 on mobile.
    • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
    • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
      • From 77af44aae14b4e5f82c2de036c0003d917a6bf50 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:33:01 +0800 Subject: [PATCH 14/29] Add an option to specify the port for the recent logs Allows using archivebot-dashboard-repeater on another port on localhost. See-also: https://github.com/iakat/archivebot-dashboard-repeater --- dashboard/assets/scripts/dashboard.js | 4 +++- dashboard/dashboard.html | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 4d11f0bd..037ed60e 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1038,6 +1038,8 @@ class Dashboard { } this.host = args.host ? args.host : location.hostname; + this.port = args.port ? `:${Number(args.port)}` : ''; + this.dumpTraffic = args.dumpMax && Number(args.dumpMax) > 0; if (this.dumpTraffic) { this.dumpMax = Number(args.dumpMax); @@ -1180,7 +1182,7 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; const size_mb = Math.round((100 * ev.total) / 1e6) / 100; byId("meta-info").textContent = `Recent data: ${percent}% (${size_mb}MB)`; }; - xhr.open("GET", `//${this.host}/logs/recent?cb=${Date.now()}${Math.random()}`); + xhr.open("GET", `//${this.host}${this.port}/logs/recent?cb=${Date.now()}${Math.random()}`); xhr.setRequestHeader("Accept", "application/json"); xhr.send(""); }); diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 5ff08c1b..58bcb2f4 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -480,6 +480,7 @@
    • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
    • To append some text to the page title, add ?title=TEXT to the dashboard URL.
    • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
      • +
    • To override the port of the recent logs server, add ?port=TEXT to the dashboard URL.
    • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.
    • To enable debug messages in the browser console, add ?debug=1 to the dashboard URL.
    From ed790267022423839908bdd6709c2db94e49c18f Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:54:36 +0800 Subject: [PATCH 15/29] Add an option to specify the URL for the logs stream Allows using archivebot-dashboard-repeater on localhost. See-also: https://github.com/iakat/archivebot-dashboard-repeater --- dashboard/assets/scripts/dashboard.js | 6 +++--- dashboard/dashboard.html | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 037ed60e..3f5fdee5 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1039,6 +1039,8 @@ class Dashboard { this.host = args.host ? args.host : location.hostname; this.port = args.port ? `:${Number(args.port)}` : ''; + const wsproto = window.location.protocol === "https:" ? "wss:" : "ws:"; + this.websocketUrl = args.websocketUrl ?? `${wsproto}//${this.host}:4568/stream`; this.dumpTraffic = args.dumpMax && Number(args.dumpMax) > 0; if (this.dumpTraffic) { @@ -1233,9 +1235,7 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; } connectWebSocket() { - const wsproto = window.location.protocol === "https:" ? "wss:" : "ws:"; - - this.ws = new WebSocket(`${wsproto}//${this.host}:4568/stream`); + this.ws = new WebSocket(this.websocketUrl); this.ws.onmessage = (ev) => { this.newItemsReceived += 1; diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 58bcb2f4..adfcca52 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -481,6 +481,7 @@
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
  • To override the port of the recent logs server, add ?port=TEXT to the dashboard URL.
    • +
  • To override the URL of the logs stream, add ?websocketUrl=ws://localhost:4568/stream to the dashboard URL.
  • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.
  • To enable debug messages in the browser console, add ?debug=1 to the dashboard URL.
    • From 21a2f93d910ec897d68d795294d0d833c610dc73 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:28:02 +0800 Subject: [PATCH 16/29] Add a slight delay when typing into the job log filter Avoids applying the filter multiple times while typing, since that can be too slow and block the UI sometimes. --- dashboard/assets/scripts/dashboard.js | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 3f5fdee5..18197f38 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -122,14 +122,6 @@ function regExpEscape(s) { return escaped; } -function addAnyChangeListener(elem, func) { - // DOM0 handler for convenient use by Clear button - elem.onchange = func; - elem.addEventListener("keydown", func, false); - elem.addEventListener("paste", func, false); - elem.addEventListener("input", func, false); -} - function scrollToBottom(elem) { // Scroll to the bottom. To avoid serious performance problems in Firefox, // use a big number instead of elem.scrollHeight. @@ -284,7 +276,25 @@ class JobsRenderer { constructor(container, filterBox, historyLines, showNicks, contextMenuRenderer) { this.container = container; this.filterBox = filterBox; - addAnyChangeListener(this.filterBox, () => this.applyFilter()); + this.filterTimeout = null; + this.filterBox.onchange = (e) => { + const repeats = [ + "insertText", + "deleteContent", + "deleteContentForward", + "deleteContentBackward", + ]; + let ms = e && e.inputType && repeats.includes(e.inputType) ? 100 : 0; + ms = !this.filterBox.value ? 0 : ms; + clearTimeout(this.filterTimeout); + this.filterTimeout = setTimeout(() => { + if (this.filterBox.value !== this.filterBox.old) { + this.applyFilter(); + this.filterBox.old = this.filterBox.value; + } + }, ms); + }; + this.filterBox.oninput = this.filterBox.onchange; this.filterBox.onkeypress = (ev) => { // Don't let `j` or `k` in the filter box cause the job window to switch ev.stopPropagation(); From 48c2ff5a9c9a40f2c2190a9a928d836f0aa41902 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:42:52 +0800 Subject: [PATCH 17/29] Allow hiding jobs based on their status Hiding finished jobs can save space when there are a lot of them. --- dashboard/assets/scripts/dashboard.js | 37 ++++++++++++++++++ dashboard/dashboard.html | 54 +++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 18197f38..ec45ff7e 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1039,6 +1039,10 @@ class Dashboard { const contextMenu = args.contextMenu ? Boolean(Number(args.contextMenu)) : true; this.initialFilter = args.initialFilter ?? "^$"; const showAllHeaders = args.showAllHeaders ? Boolean(Number(args.showAllHeaders)) : true; + const showRunningJobs = args.showRunningJobs ? Boolean(Number(args.showRunningJobs)) : true; + const showFinishedJobs = args.showFinishedJobs ? Boolean(Number(args.showFinishedJobs)) : true; + const showFatalJobs = args.showFatalJobs ? Boolean(Number(args.showFatalJobs)) : true; + const showAbortedJobs = args.showAbortedJobs ? Boolean(Number(args.showAbortedJobs)) : true; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; this.debug = args.debug ? Boolean(Number(args.debug)) : false; @@ -1107,6 +1111,11 @@ class Dashboard { this.showAllHeaders(showAllHeaders); + this.showRunningJobs(showRunningJobs); + this.showFinishedJobs(showFinishedJobs); + this.showFatalJobs(showFatalJobs); + this.showAbortedJobs(showAbortedJobs); + const finishSetup = () => { byId("meta-info").innerHTML = ""; @@ -1233,6 +1242,14 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; window.open(this.jobsRenderer.firstFilterMatch.url); } else if (ev.which === 104 /* h */) { ds.showAllHeaders(!byId("show-all-headers").checked); + } else if (ev.which === 114 /* r */) { + ds.showRunningJobs(!byId("show-running-jobs").checked); + } else if (ev.which === 100 /* d */) { + ds.showFinishedJobs(!byId("show-finished-jobs").checked); + } else if (ev.which === 99 /* c */) { + ds.showFatalJobs(!byId("show-fatal-jobs").checked); + } else if (ev.which === 115 /* s */) { + ds.showAbortedJobs(!byId("show-aborted-jobs").checked); } } @@ -1292,6 +1309,26 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; byId('show-all-headers').checked = value; byId('hide-headers').sheet.disabled = value; } + + showRunningJobs(value) { + byId('show-running-jobs').checked = value; + byId('hide-running').sheet.disabled = value; + } + + showFinishedJobs(value) { + byId('show-finished-jobs').checked = value; + byId('hide-done').sheet.disabled = value; + } + + showFatalJobs(value) { + byId('show-fatal-jobs').checked = value; + byId('hide-fatal').sheet.disabled = value; + } + + showAbortedJobs(value) { + byId('show-aborted-jobs').checked = value; + byId('hide-aborted').sheet.disabled = value; + } } const ds = new Dashboard(); diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index adfcca52..e6ec18a6 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -57,6 +57,11 @@ font-size: 18px; } +.drop-down { + display: inline flex; + flex-direction: column; +} + .padded-page { padding: 20px 27px 20px 27px; } @@ -397,6 +402,34 @@ content-visibility: hidden; } + + + +
    @@ -409,6 +442,17 @@ +
    + Jobs + +
    + +
    + +
    + +
    +
    😊
    @@ -458,6 +502,10 @@
  • i - use the initial job log filter
  • v - open the job URL of the first-shown job log
  • h - show/hide headers for hidden job logs +
  • r - show/hide header+log for running jobs +
  • d - show/hide header+log for finished jobs +
  • c - show/hide header+log for fatal jobs +
  • s - show/hide header+log for aborted jobs
  • ? - show/hide help text

    @@ -475,6 +523,12 @@

    • To specify an initial filter, add ?initialFilter=TEXT to the dashboard URL. The default is ^$.
    • To initially hide headers for hidden job logs, add ?showAllHeaders=0 to the dashboard URL. The default is to show them.
      • +
    • To initially hide different job types, add these to the dashboard URL. The default is to show them. + ?showRunningJobs=0 + ?showFinishedJobs=0 + ?showFatalJobs=0 + ?showAbortedJobs=0 +
    • To retain more lines in the log windows, add ?historyLines=1000 to the dashboard URL. The default is 500, or 250 on mobile.
    • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
    • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
      • From c5f0559964bbf578768e38c0cc704d259822e24e Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 16:00:37 +0800 Subject: [PATCH 18/29] Fix detection of finished jobs Fixes: commit e179725059e7752f6d97a5e72f4556256e78fa12 --- dashboard/assets/scripts/dashboard.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index ec45ff7e..45509c4c 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -492,10 +492,12 @@ class JobsRenderer { logSegment.appendChild(h("div", Reusable.obj_className_line_stdout, line)); renderedLines += 1; - // Check for 'Finished RsyncUpload for Item' - // instead of 'Starting MarkItemAsDone for Item' - // because the latter is often missing - if (/^Finished RsyncUpload for Item/.test(line)) { + // Check for several completion messages + // because some of them are often missing + // Ignore error jobs as they get done messages. + if (!info.statsElements.jobInfo.classList.contains("job-info-fatal") && + !info.statsElements.jobInfo.classList.contains("job-info-aborted") && + /^ *[1-9][0-9]* bytes\.$|^Starting (RelabelIfAborted|MarkItemAsDone) for Item$|^Finished (WgetDownload|MoveFiles|StopHeartbeat) for Item$/.test(line)) { info.statsElements.jobInfo.classList.add("job-info-done"); this.jobs.markFinished(ident); } else if ( From 62ca881b2a00bd96e9dc332bba39e2e24c2f84e7 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:47:53 +0800 Subject: [PATCH 19/29] Detect jobs that failed and allow filtering them out --- dashboard/assets/scripts/dashboard.js | 13 +++++++++++++ dashboard/dashboard.html | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 45509c4c..a9ceed08 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -497,9 +497,12 @@ class JobsRenderer { // Ignore error jobs as they get done messages. if (!info.statsElements.jobInfo.classList.contains("job-info-fatal") && !info.statsElements.jobInfo.classList.contains("job-info-aborted") && + !info.statsElements.jobInfo.classList.contains("job-info-failed") && /^ *[1-9][0-9]* bytes\.$|^Starting (RelabelIfAborted|MarkItemAsDone) for Item$|^Finished (WgetDownload|MoveFiles|StopHeartbeat) for Item$/.test(line)) { info.statsElements.jobInfo.classList.add("job-info-done"); this.jobs.markFinished(ident); + } else if (/^ *0 bytes\.$/.test(line)) { + info.statsElements.jobInfo.classList.add("job-info-failed"); } else if ( /^CRITICAL (Sorry|Please report)|^ERROR Fatal exception|No space left on device|^Fatal Python error:|^(Thread|Current thread) 0x/.test( line, @@ -520,6 +523,7 @@ class JobsRenderer { } else if (/^Received item /.test(line)) { // Clear other statuses if a job restarts with the same job ID info.statsElements.jobInfo.classList.remove("job-info-done"); + info.statsElements.jobInfo.classList.remove("job-info-failed"); info.statsElements.jobInfo.classList.remove("job-info-fatal"); info.statsElements.jobInfo.classList.remove("job-info-aborted"); this.jobs.markUnfinished(ident); @@ -1043,6 +1047,7 @@ class Dashboard { const showAllHeaders = args.showAllHeaders ? Boolean(Number(args.showAllHeaders)) : true; const showRunningJobs = args.showRunningJobs ? Boolean(Number(args.showRunningJobs)) : true; const showFinishedJobs = args.showFinishedJobs ? Boolean(Number(args.showFinishedJobs)) : true; + const showFailedJobs = args.showFailedJobs ? Boolean(Number(args.showFailedJobs)) : true; const showFatalJobs = args.showFatalJobs ? Boolean(Number(args.showFatalJobs)) : true; const showAbortedJobs = args.showAbortedJobs ? Boolean(Number(args.showAbortedJobs)) : true; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; @@ -1115,6 +1120,7 @@ class Dashboard { this.showRunningJobs(showRunningJobs); this.showFinishedJobs(showFinishedJobs); + this.showFailedJobs(showFailedJobs); this.showFatalJobs(showFatalJobs); this.showAbortedJobs(showAbortedJobs); @@ -1248,6 +1254,8 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; ds.showRunningJobs(!byId("show-running-jobs").checked); } else if (ev.which === 100 /* d */) { ds.showFinishedJobs(!byId("show-finished-jobs").checked); + } else if (ev.which === 98 /* b */) { + ds.showFailedJobs(!byId("show-failed-jobs").checked); } else if (ev.which === 99 /* c */) { ds.showFatalJobs(!byId("show-fatal-jobs").checked); } else if (ev.which === 115 /* s */) { @@ -1322,6 +1330,11 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; byId('hide-done').sheet.disabled = value; } + showFailedJobs(value) { + byId('show-failed-jobs').checked = value; + byId('hide-failed').sheet.disabled = value; + } + showFatalJobs(value) { byId('show-fatal-jobs').checked = value; byId('hide-fatal').sheet.disabled = value; diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index e6ec18a6..1d61d99b 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -133,6 +133,10 @@ color: #DD0000 !important; } +.job-info-failed { + color: #CC7676 !important; +} + .inline-stat { /* Needed for 'Align!' feature */ display: inline-block; @@ -405,6 +409,7 @@ +