diff --git a/.env.example b/.env.example
index 049c493..8ecf3ef 100644
--- a/.env.example
+++ b/.env.example
@@ -17,3 +17,21 @@ TG_CHAT_ID=123456789
# Optional: HTTP timeout in seconds for Telegram/Discord requests (must be positive, default 10)
# REQUEST_TIMEOUT=30
+
+# Optional: seconds between each speed test + device scan cycle (default 1800 = 30 min)
+# SLEEP_TIME=1800
+# Optional: how many cycles between detailed AI reports with graph (default 8, i.e. ~4 hours at the default SLEEP_TIME)
+# REPORT_CYCLE_COUNT=8
+# Optional: sets Ollama's num_ctx per-request via extra_body, to stop a local model's
+# default context window from silently truncating a long prompt + a day of history.
+# No effect on cloud OpenAI. Leave unset unless using a local/self-hosted AI backend.
+# AI_CONTEXT_SIZE=8192
+
+# Optional: instant outage/degradation alerting thresholds. A reading counts as
+# "bad" if it's below the download threshold OR above the ping threshold; a
+# speed test failing outright counts as bad too. An alert fires once per
+# episode (not every cycle) once N consecutive bad readings are seen, and
+# again once the connection recovers, with how long it lasted.
+# OUTAGE_DOWNLOAD_THRESHOLD_MBPS=20
+# OUTAGE_PING_THRESHOLD_MS=150
+# OUTAGE_CONSECUTIVE_READINGS=2
diff --git a/README.md b/README.md
index 861969a..cb39568 100644
--- a/README.md
+++ b/README.md
@@ -31,13 +31,14 @@ Every 4 hours, it delivers a **detailed report** complete with a 24-hour trend g
## Features & Workflow
-Every 30 minutes (`SLEEP_TIME` in `main.py`, default 1800 seconds):
+Every `SLEEP_TIME` seconds (default 1800 = 30 min, configurable):
1. **Speed Test:** Measures download/upload speeds, ping latency, ISP, and test server details using `speedtest-cli` (see [the note on measurement mode](#a-note-on-measurement-mode)).
-2. **LAN Scan:** Scans the local subnet using `nmap` ARP scan to count active connected devices.
-3. **Local Storage:** Saves metrics & device tallies directly to a local `metrics.sql` SQLite database.
+2. **LAN Scan:** Scans the local subnet using `nmap` ARP scan to identify active devices, including MAC address, vendor, and hostname where resolvable (see [Device Watch](#device-watch)).
+3. **Local Storage:** Saves metrics & device details directly to a local `metrics.sql` SQLite database.
4. **Status Alert:** Sends a concise status update to your chosen notifier (*"all good"* or *"line is dying"*).
-5. **24h AI Report:** Every 8th cycle (every 4h), generates a **24-hour trend graph** via `matplotlib` alongside a sarcastic LLM analysis of network load and speed fluctuations.
+5. **24h AI Report:** Every `REPORT_CYCLE_COUNT` cycles (default 8, i.e. ~4h), generates a **24-hour trend graph** via `matplotlib` alongside a sarcastic LLM analysis of network load, speed fluctuations, and any notable new devices on the network.
+6. **Instant Outage Alerting:** Watches every cycle for an outright failed speed test or a degraded reading, alerting immediately rather than waiting for the next scheduled report (see [Instant Outage & Degradation Alerting](#instant-outage--degradation-alerting)).
---
@@ -129,6 +130,9 @@ cp .env.example .env
| `DISCORD_WEBHOOK_URL` | Discord channel webhook URL — required if `NOTIFIER=discord` |
| `DB_PATH` | SQLite database file path (e.g. `metrics.sql`) |
| `REQUEST_TIMEOUT` | *Optional.* HTTP timeout in seconds for Telegram/Discord requests (positive integer, default `30`) |
+| `SLEEP_TIME` | *Optional.* Seconds between each speed test + device scan cycle (positive integer, default `1800`) |
+| `REPORT_CYCLE_COUNT` | *Optional.* How many cycles between detailed AI reports with graph (positive integer, default `8`) |
+| `AI_CONTEXT_SIZE` | *Optional.* Sets Ollama's `num_ctx` per-request, to stop a local model's default context window from silently truncating a long prompt + a day of history. No effect on cloud OpenAI — leave unset unless self-hosting the AI backend. |
> [!TIP]
> **You're not locked into OpenAI.** `ai.py` talks to any OpenAI-compatible endpoint, so a local inference server (e.g. [Ollama](https://ollama.com), LM Studio) works too — just point `AI_BASE_URL` at it. For report quality that holds up, use a model with **at least ~7B parameters**; a solid local pick is **Gemma 4 12B at 4-bit (QAT) quantization** (`gemma4:12b-it-qat` via Ollama), which fits comfortably on 16GB of RAM.
@@ -144,6 +148,9 @@ uv run main.py
> [!TIP]
> Run the bot inside `tmux`/`screen` or set it up as a system service (`systemd`/`launchd`) to keep it running 24/7 in the background.
+> [!TIP]
+> Pass `--test-ai` (`uv run main.py --test-ai`) to force the very first cycle to run the full detailed report (AI commentary + graph + notifier delivery) immediately, then resume the normal `REPORT_CYCLE_COUNT` schedule automatically — no config to remember to revert afterward. Useful for verifying your AI backend and notifier work without waiting for the regular cadence.
+
---
## Notifications: Telegram or Discord
@@ -192,6 +199,18 @@ Two consequences worth knowing:
Since netmon exists to track *trends*, consistency matters more than peak numbers: keep one measurement method for the lifetime of your database. Swapping the backend mid-history puts a step change in your 24-hour graph that the AI commentary will faithfully report as a real speed jump.
+### Jitter & Bufferbloat (Ookla backend only)
+
+Classic `speedtest-cli` has no jitter or packet-loss data. If you instead run netmon against an Ookla-compatible speed test backend that reports those fields, netmon picks them up automatically and surfaces them in both mini and detailed reports, alongside a note from the AI treating high jitter or nonzero packet loss as a sign of bufferbloat — a connection can have great raw Mbps numbers and still feel laggy under load if jitter is high. This is entirely additive: nothing changes in reports if your backend doesn't provide this data.
+
+---
+
+## Device Watch
+
+Every device scan records each device's MAC address, vendor (resolved from `nmap`'s built-in OUI database), and hostname where available. `nmap` can only resolve a MAC for hosts on the same local subnet it can ARP directly — off-subnet or otherwise hidden devices are still counted, just not identified.
+
+Using MAC address history, the detailed AI report includes a **Device Watch** section that flags any device whose MAC hasn't been seen on the network in the last 14 days, alongside a vendor-count breakdown of everything currently online. A device with no resolvable MAC is never flagged as new, since there's no reliable identity to compare against.
+
---
## Example Output
@@ -232,12 +251,16 @@ Server: New York
Download: 178.5 Mbps
Upload: 45.2 Mbps
Ping: 23.1 ms
+Jitter: 4.2 ms | Packet Loss: 0.0%
Devices Online: 9
24-Hour Dynamics Analysis
Over the last 24 hours, the download speed averaged 140 Mbps, but we saw a massive drop to 20 Mbps at 8:00 PM right as device count jumped from 4 to 11 devices. Clearly, someone's hogging the bandwidth or the ISP's mice were busy chewing on the fiber line again. Latency remained stable except for a brief spike during peak hours.
+Device Watch
+One new gadget joined the party today: a device with no vendor or hostname info at all — worth a second glance. Everything else is the same suspects as always.
+
Data Transfer (Latest Test)
Downloaded: 160.0 MB
@@ -248,6 +271,37 @@ Uploaded: 70.0 MB
Expect periodic speed drops whenever local freeloaders stream 4K movies or the ISP potato infrastructure struggles.
```
+> [!NOTE]
+> The AI is only ever asked for three short text fields (the dynamics analysis, the Device Watch line, and the conclusion) — the surrounding HTML structure above is assembled deterministically in code, not generated by the model. This keeps report formatting consistent regardless of which LLM is behind `AI_BASE_URL`, including smaller local models that would otherwise struggle to reproduce a long literal template reliably. The `Jitter` line only appears when your speed test backend reports it (see [Jitter & Bufferbloat](#jitter--bufferbloat-ookla-backend-only)) — it's silently omitted otherwise.
+
+---
+
+## Instant Outage & Degradation Alerting
+
+Waiting for the next scheduled detailed report to notice an outage could mean a multi-hour delay. netmon instead watches every cycle:
+
+* **Outage:** the speed test itself fails outright for `OUTAGE_CONSECUTIVE_READINGS` consecutive cycles.
+* **Degradation:** a successful reading falls below `OUTAGE_DOWNLOAD_THRESHOLD_MBPS` or above `OUTAGE_PING_THRESHOLD_MS` for the same number of consecutive cycles.
+
+Each fires an alert once per episode (not every cycle, to avoid spam), and again once the connection recovers, with how long the episode lasted.
+
+| Variable | Description |
+| :--- | :--- |
+| `OUTAGE_DOWNLOAD_THRESHOLD_MBPS` | *Optional.* Download speed below which a reading counts as degraded (positive number, default `20`) |
+| `OUTAGE_PING_THRESHOLD_MS` | *Optional.* Ping above which a reading counts as degraded (positive number, default `150`) |
+| `OUTAGE_CONSECUTIVE_READINGS` | *Optional.* Consecutive bad/failed readings before alerting (positive integer, default `2`) |
+
+> [!NOTE]
+> A failing or degraded speed test is treated as the exact condition this tool exists to detect, not a bug in netmon — it alerts and keeps retrying every cycle rather than crashing the process (see [Reliability](#reliability) below for the genuine-infra-failure case, which is handled differently on purpose).
+
+---
+
+## Reliability
+
+Speed test, device scan, database, or notifier-delivery failures are **never silently retried**. If one of these fails, netmon makes a best-effort attempt to post an alert to your configured notifier — so the failure is visible without checking server logs — then crashes rather than looping on a broken state. Check the service logs (`journalctl -u netmon` if running under `systemd`, or wherever your process manager sends output) for the full traceback, and your process manager's restart policy will bring it back up.
+
+This is deliberately different from how a *slow or unreachable AI backend* is handled: that degrades gracefully (the report still sends, just without AI commentary) rather than crashing, since a flaky LLM endpoint isn't the kind of infrastructure failure worth stopping the whole monitor over.
+
---
## Project Structure
diff --git a/ai.py b/ai.py
index 1a42980..b35493e 100644
--- a/ai.py
+++ b/ai.py
@@ -26,25 +26,43 @@ def init(cls, api_key: str, model: str, base_url: str) -> "Client":
return cls(OpenAI(api_key=api_key, base_url=base_url), model)
- def send_message(self, message:str, system_prompt: str) -> str:
+ def send_message(
+ self,
+ message: str,
+ system_prompt: str,
+ temperature: float = 0.9,
+ context_size: int | None = None,
+ ) -> str:
self._validate_str(message, "message")
+ # Ollama's default context window (often 2048-4096 tokens depending
+ # on the model) is easy to exceed once the system prompt plus a
+ # day's worth of historical readings are combined — and unlike a
+ # clear error, exceeding it just silently truncates the prompt
+ # (typically from the start), which can quietly drop persona/format
+ # instructions while leaving the raw data intact. Passing num_ctx
+ # via extra_body raises this per-request for Ollama specifically.
+ # This is a no-op / harmless on real OpenAI's API since it's only
+ # added when context_size is explicitly set (e.g. for a local
+ # Ollama backend), not unconditionally on every request.
+ extra_body = {}
+ if context_size is not None:
+ extra_body["options"] = {"num_ctx": context_size}
+
response = self.conn.chat.completions.create(
model=self.model,
+ temperature=temperature,
messages=[
{"role": "system", "content": system_prompt},
{"role": "user", "content": message}
- ])
-
+ ],
+ extra_body=extra_body or None,
+ )
+
if response.choices[0].message.content is not None:
return response.choices[0].message.content
-
+
raise RuntimeError("AI response is empty")
def close(self):
- self.conn.close()
-
-
-
-
-
\ No newline at end of file
+ self.conn.close()
\ No newline at end of file
diff --git a/config.py b/config.py
index d684d4c..0726470 100644
--- a/config.py
+++ b/config.py
@@ -6,6 +6,11 @@
logger = logging.getLogger(__name__)
DEFAULT_REQUEST_TIMEOUT = 30
+DEFAULT_SLEEP_TIME = 1800
+DEFAULT_REPORT_CYCLE_COUNT = 8
+DEFAULT_OUTAGE_DOWNLOAD_THRESHOLD_MBPS = 20.0
+DEFAULT_OUTAGE_PING_THRESHOLD_MS = 150.0
+DEFAULT_OUTAGE_CONSECUTIVE_READINGS = 2
class Config:
def __init__(
@@ -18,7 +23,14 @@ def __init__(
tg_bot_token: str = "",
tg_chat_id: str = "",
discord_webhook_url: str = "",
- request_timeout: int = DEFAULT_REQUEST_TIMEOUT
+ request_timeout: int = DEFAULT_REQUEST_TIMEOUT,
+ sleep_time: int = DEFAULT_SLEEP_TIME,
+ report_cycle_count: int = DEFAULT_REPORT_CYCLE_COUNT,
+ test_ai: bool = False,
+ ai_context_size: int | None = None,
+ outage_download_threshold_mbps: float = DEFAULT_OUTAGE_DOWNLOAD_THRESHOLD_MBPS,
+ outage_ping_threshold_ms: float = DEFAULT_OUTAGE_PING_THRESHOLD_MS,
+ outage_consecutive_readings: int = DEFAULT_OUTAGE_CONSECUTIVE_READINGS,
):
self.ai_api_key: str = ai_api_key
self.db_path: str = db_path
@@ -29,6 +41,13 @@ def __init__(
self.tg_chat_id: str = tg_chat_id
self.discord_webhook_url: str = discord_webhook_url
self.request_timeout: int = request_timeout
+ self.sleep_time: int = sleep_time
+ self.report_cycle_count: int = report_cycle_count
+ self.test_ai: bool = test_ai
+ self.ai_context_size: int | None = ai_context_size
+ self.outage_download_threshold_mbps: float = outage_download_threshold_mbps
+ self.outage_ping_threshold_ms: float = outage_ping_threshold_ms
+ self.outage_consecutive_readings: int = outage_consecutive_readings
@staticmethod
def _parse_args():
@@ -39,6 +58,15 @@ def _parse_args():
default=".env",
help="Path to the .env file (default: .env)"
)
+ parser.add_argument(
+ "--test-ai",
+ action="store_true",
+ help="Force the very first cycle to run the full detailed report "
+ "(AI commentary + graph + notifier delivery), then continue on "
+ "the normal REPORT_CYCLE_COUNT schedule for every cycle after. "
+ "Useful for verifying the AI backend and notifier work without "
+ "waiting for the regular cadence or permanently changing config."
+ )
return parser.parse_args()
@classmethod
@@ -72,6 +100,56 @@ def init(cls):
if request_timeout <= 0:
raise RuntimeError(f"REQUEST_TIMEOUT must be positive, got: {request_timeout}")
+ try:
+ sleep_time = int(os.getenv("SLEEP_TIME", DEFAULT_SLEEP_TIME))
+ except ValueError:
+ raise RuntimeError(f"SLEEP_TIME must be an integer number of seconds, got: {os.getenv('SLEEP_TIME')!r}")
+ if sleep_time <= 0:
+ raise RuntimeError(f"SLEEP_TIME must be positive, got: {sleep_time}")
+
+ try:
+ report_cycle_count = int(os.getenv("REPORT_CYCLE_COUNT", DEFAULT_REPORT_CYCLE_COUNT))
+ except ValueError:
+ raise RuntimeError(f"REPORT_CYCLE_COUNT must be an integer, got: {os.getenv('REPORT_CYCLE_COUNT')!r}")
+ if report_cycle_count <= 0:
+ raise RuntimeError(f"REPORT_CYCLE_COUNT must be positive, got: {report_cycle_count}")
+
+ # Optional and unset by default — only meaningful for local
+ # OpenAI-compatible servers like Ollama, whose default context
+ # window can silently truncate a long system prompt + a day's
+ # worth of history once combined. Left as None, nothing extra is
+ # sent, so cloud OpenAI usage is unaffected.
+ ai_context_size_raw = os.getenv("AI_CONTEXT_SIZE")
+ ai_context_size: int | None = None
+ if ai_context_size_raw is not None and ai_context_size_raw.strip() != "":
+ try:
+ ai_context_size = int(ai_context_size_raw)
+ except ValueError:
+ raise RuntimeError(f"AI_CONTEXT_SIZE must be an integer, got: {ai_context_size_raw!r}")
+ if ai_context_size <= 0:
+ raise RuntimeError(f"AI_CONTEXT_SIZE must be positive, got: {ai_context_size}")
+
+ try:
+ outage_download_threshold_mbps = float(os.getenv("OUTAGE_DOWNLOAD_THRESHOLD_MBPS", DEFAULT_OUTAGE_DOWNLOAD_THRESHOLD_MBPS))
+ except ValueError:
+ raise RuntimeError(f"OUTAGE_DOWNLOAD_THRESHOLD_MBPS must be a number, got: {os.getenv('OUTAGE_DOWNLOAD_THRESHOLD_MBPS')!r}")
+ if outage_download_threshold_mbps <= 0:
+ raise RuntimeError(f"OUTAGE_DOWNLOAD_THRESHOLD_MBPS must be positive, got: {outage_download_threshold_mbps}")
+
+ try:
+ outage_ping_threshold_ms = float(os.getenv("OUTAGE_PING_THRESHOLD_MS", DEFAULT_OUTAGE_PING_THRESHOLD_MS))
+ except ValueError:
+ raise RuntimeError(f"OUTAGE_PING_THRESHOLD_MS must be a number, got: {os.getenv('OUTAGE_PING_THRESHOLD_MS')!r}")
+ if outage_ping_threshold_ms <= 0:
+ raise RuntimeError(f"OUTAGE_PING_THRESHOLD_MS must be positive, got: {outage_ping_threshold_ms}")
+
+ try:
+ outage_consecutive_readings = int(os.getenv("OUTAGE_CONSECUTIVE_READINGS", DEFAULT_OUTAGE_CONSECUTIVE_READINGS))
+ except ValueError:
+ raise RuntimeError(f"OUTAGE_CONSECUTIVE_READINGS must be an integer, got: {os.getenv('OUTAGE_CONSECUTIVE_READINGS')!r}")
+ if outage_consecutive_readings <= 0:
+ raise RuntimeError(f"OUTAGE_CONSECUTIVE_READINGS must be positive, got: {outage_consecutive_readings}")
+
if notifier == "telegram":
if tg_bot_token.strip() == "":
raise RuntimeError("TG_BOT_TOKEN not found or empty in environment")
@@ -84,5 +162,7 @@ def init(cls):
return cls(
ai_key, db_path, model, base_url, notifier,
tg_bot_token, tg_chat_id, discord_webhook_url,
- request_timeout,
+ request_timeout, sleep_time, report_cycle_count,
+ args.test_ai, ai_context_size,
+ outage_download_threshold_mbps, outage_ping_threshold_ms, outage_consecutive_readings,
)
diff --git a/main.py b/main.py
index ddff21f..1bd2580 100644
--- a/main.py
+++ b/main.py
@@ -1,6 +1,9 @@
import signal
import sys
+import json
+import html
import logging
+from datetime import datetime, timezone
import graphs
import config as cfg
import sqlite
@@ -13,83 +16,107 @@
from notifier import ChatAction, Notifier
REPORT_SYSTEM_PROMPT = """
-You are a sarcastic, cynical network analyst bot. Your job is to output a short network speed test and 24-hour trend report in Telegram HTML format.
-You will receive a list of speed tests from the last 24 hours in chronological order (the last line is the latest test).
+You are a sarcastic, cynical network analyst bot.
-You must write the report in ENGLISH.
-You must follow the EXACT structure below. Do not deviate from this layout, header naming, or formatting.
+You will receive: (1) a chronological list of network speed test results from the last 24 hours, (2) a device vendor breakdown for the network (vendor name + how many devices of that vendor are currently online), and (3) a list of any devices that are NEW this cycle (their MAC address has not been seen on this network in the last 14 days), each with whatever vendor/hostname/IP is available.
-EXPECTED STRUCTURE:
-Network Speed Test Report (24h Analysis)
+Respond with ONLY a single raw JSON object -- no ```json code fences, no preamble, no explanation before or after it. Just the JSON object, starting with { and ending with }. It must have exactly these three string keys:
-Client: [Client ISP]
-Server: [Server Name]
+{
+ "dynamics_analysis": "2-3 short sentences analyzing speed/ping trends and drops over the given period. ONLY claim a link between device count and speed/latency swings if the numbers actually move together in the same window (e.g. speed visibly drops as device count rises). If device count swings while speed/ping stay flat, say plainly that device count does NOT explain it, and point at the ISP/line instead -- never invent a correlation the numbers don't support. If ping reads exactly 0.00 ms while download speed is very low, that means the real ping was too high to register and got floored to zero -- call it a red flag, not a strength, never a good sign. If jitter or packet loss data is present in the readings, treat high jitter (a few ms is normal; tens of ms is notable) or any nonzero packet loss as a sign of bufferbloat/an unstable line -- this matters even when the raw download/upload numbers look fine, since a connection can have great throughput but still feel laggy under load if jitter is high. If jitter/packet loss data is absent from the readings, don't mention it at all. Do NOT blame server changes for fluctuations -- assume the server choice is optimal. Wrap key numbers in ... tags, e.g. 148.31 Mbps.",
+ "device_watch": "1-2 short sentences. If the NEW devices list you were given is empty, say so plainly (e.g. 'No new devices -- same suspects as always.'). Otherwise mention only devices from that NEW list, identified by vendor/hostname if given, otherwise by IP. Call out anything more suspicious than the rest -- e.g. a NEW device with no vendor or hostname info at all is more worth a second glance than a NEW device from a recognizable vendor. Never invent a device, vendor, hostname, or IP not given to you, and do not restate the full vendor breakdown here -- that's background context only, not something to list out.",
+ "conclusion": "Exactly 1 short, witty, sarcastic sentence summarizing the network's overall quality/reliability over the period."
+}
+
+TONE (this matters more than anything else): sarcastic, informal, and funny throughout. Blame heavy users/leeches on the network or the ISP for problems -- e.g. "a bunch of idiots clogging the bandwidth", "the ISP dropping the ball", "mice chewing the optic fiber cables", "yet another gadget joining the freeloader party" -- but only when the data actually supports that story. A flat, neutral, corporate-analyst tone is a FAILED response even if the JSON is technically valid -- the personality is not optional decoration, it is the entire point of this bot. If in doubt, lean funnier and more informal, not safer and more clinical.
+
+LENGTH LIMITS (hard requirements): "dynamics_analysis" under 500 characters, "device_watch" under 250 characters, "conclusion" under 150 characters.
+
+Output ONLY the JSON object and nothing else -- no markdown formatting, no headers, no bullet points, no explanatory text, no restating of the raw data you were given.
+"""
+
+# The AI is only ever asked to produce the three free-text fields above --
+# never the surrounding HTML structure. This is deliberate: several
+# capable local models (tested: llama3.1:8b, qwen2.5:7b-instruct) reliably
+# abandon a long literal HTML template under a rich, multi-constraint
+# prompt and fall back to a generic "helpful assistant summarizing data"
+# response instead, even with a large context window. Handling the
+# skeleton in code guarantees correct, consistent formatting regardless of
+# which model is behind AI_BASE_URL, and only requires the model to
+# reliably produce three short strings in a JSON object -- a much easier
+# and more commonly well-supported task for small/local instruct models
+# than exact literal markup reproduction.
+REPORT_TEMPLATE_SHELL = """Network Speed Test Report (24h Analysis)
+
+Client: {client}
+Server: {server}
Latest Test Metrics
-Download: [Download Speed] Mbps
-Upload: [Upload Speed] Mbps
-Ping: [Ping Latency] ms
-Devices Online: [Device Count]
+Download: {download:.1f} Mbps
+Upload: {upload:.1f} Mbps
+Ping: {ping:.1f} ms
+{jitter_line}Devices Online: {device_count}
24-Hour Dynamics Analysis
-[Analyze the dynamics, drops, and load of the network over the last 24 hours. Note any major drops in download/upload speeds or ping spikes.
-Also look at how the device count changed over the same period. ONLY claim a link between device count and speed/latency swings if the numbers actually move together (e.g. speed visibly drops in the same window device count rises). If device count swings around while speed/ping stay flat, say plainly that device count does NOT explain it this period, and point at the ISP/line instead. Never invent a correlation that isn't supported by the numbers.
-If ping reads exactly 0.00 ms while download speed is very low (a few Mbps or less), do NOT describe that as a good/perfect ping. That reading means the real ping was too high to register and got floored to zero — call it a red flag, not a strength.
-Use a sarcastic, informal tone when describing speed drops, latency spikes, or a sudden herd of new devices, blaming heavy users/leeches on the network or the ISP (e.g. "a bunch of idiots clogging the bandwidth", "ISP dropping the ball", "mice chewing the optic fiber cables", or "yet another gadget joining the freeloader party") — but only when the data actually supports that story.
-CRITICAL: Do NOT blame server changes for fluctuations. Assume the server choice is optimal and fluctuations reflect real network load, device count, or ISP issues.
-Wrap key numbers in tags, e.g., 148.31 Mbps, 15.18 ms, or 7 devices.]
+{dynamics_analysis}
+
+Device Watch
+{device_watch}
Data Transfer (Latest Test)
-Downloaded: [Downloaded MB] MB
-Uploaded: [Uploaded MB] MB
+Downloaded: {download_mb:.1f} MB
+Uploaded: {upload_mb:.1f} MB
Conclusion
-[A sarcastic, witty 1 short sentence summary of the network's overall quality and reliability over the past day.]
+{conclusion}"""
+# Defensive per-field caps applied in code regardless of what the prompt
+# asked for -- a safety net, not the primary mechanism, since the prompt's
+# own instructed limits should normally keep fields well under these.
+_DYNAMICS_ANALYSIS_MAX_CHARS = 600
+_DEVICE_WATCH_MAX_CHARS = 320
+_CONCLUSION_MAX_CHARS = 200
-TEMPLATE EXAMPLE OF THE OUTPUT:
-Network Speed Test Report (24h Analysis)
-Client: nameserver
-Server: New York
+def _clip(text: str, max_chars: int) -> str:
+ text = (text or "").strip()
+ if len(text) <= max_chars:
+ return text
+ return text[: max_chars - 1].rstrip() + "…"
-Latest Test Metrics
-
-Download: 140.3 Mbps
-Upload: 62.8 Mbps
-Ping: 15.2 ms
-Devices Online: 7
-
-24-Hour Dynamics Analysis
-Over the last 24 hours, the download speed averaged 140 Mbps, but we saw a massive drop to 20 Mbps at 8:00 PM right as device count jumped from 4 to 11 devices. Clearly, a bunch of idiots decided to stream 4K movies all at once, or the ISP's mice were busy chewing on the fiber line again. Latency remained stable except for a brief spike to 95 ms during the speed dip.
+def _mini_report_jitter_line(metric: "models.NetworkMetric") -> str:
+ # Only the Ookla CLI backend provides jitter/packet loss; classic
+ # speedtest-cli users simply won't see this line at all.
+ if metric.jitter_ms is None:
+ return ""
+ parts = [f"Jitter: {metric.jitter_ms:.1f} ms"]
+ if metric.packet_loss_pct is not None:
+ parts.append(f"Packet loss: {metric.packet_loss_pct:.1f}%")
+ return " | ".join(parts) + "\n"
-Data Transfer (Latest Test)
-
-Downloaded: 160.0 MB
-Uploaded: 70.0 MB
-
-Conclusion
-Expect periodic speed deaths whenever the local leechers wake up or the ISP fails to maintain their potato infrastructure.
-
-
-CRITICAL RULES:
-1. Do NOT use
or
tags. For line breaks, use normal newlines.
-2. The entire report must be in English.
-3. Keep the "24-Hour Dynamics Analysis" to exactly 2-3 short sentences.
-4. Do NOT write any description text below the "Data Transfer (Latest Test)" pre-block.
-5. Keep the "Conclusion" to exactly 1 short sentence.
-6. Highlight all numeric metric values in the text using [Value].
-7. Do NOT output any markdown blocks like ```html. Output raw HTML tags directly.
-8. Make sure all HTML tags are closed correctly.
-9. Be sarcastic, informal, and funny when describing performance dips or network load.
-10. The entire output MUST be under 800 characters to ensure it easily fits within Telegram limits.
-"""
+def _shell_jitter_line(metric: "models.NetworkMetric") -> str:
+ if metric.jitter_ms is None:
+ return ""
+ line = f"Jitter: {metric.jitter_ms:.1f} ms"
+ if metric.packet_loss_pct is not None:
+ line += f" | Packet Loss: {metric.packet_loss_pct:.1f}%"
+ return line + "\n"
+
+
+def _ai_history_jitter_line(m: "models.NetworkMetric") -> str:
+ if m.jitter_ms is None:
+ return ""
+ line = f"- Jitter: {m.jitter_ms:.2f} ms\n"
+ if m.packet_loss_pct is not None:
+ line += f"- Packet Loss: {m.packet_loss_pct:.2f}%\n"
+ return line
+
REPORT_USER_TEMPLATE = """
Network speed test results:
@@ -97,7 +124,7 @@
- Download: {download:.2f} Mbps
- Upload: {upload:.2f} Mbps
- Ping: {ping:.2f} ms
-- Client: {client}
+{jitter_line}- Client: {client}
- Server: {server}
- Downloaded: {download_mb} MB
- Uploaded: {upload_mb} MB
@@ -105,6 +132,10 @@
- Devices online: {device_count}
"""
+VENDOR_COUNT_TEMPLATE = "- {vendor}: {count} device(s)"
+NEW_DEVICE_ENTRY_TEMPLATE = "- {ip}{vendor_part}{hostname_part}"
+
+
MINI_REPORT_TEMPLATE = """Network Status Update
Here is the latest snapshot of your internet speed:
@@ -116,12 +147,58 @@
Download: {download:.1f} Mbps
Upload: {upload:.1f} Mbps
Latency: {ping:.1f} ms
-
+{jitter_line}
Traffic used: {download_mb:.1f} MB down / {upload_mb:.1f} MB up
Current status: {status_text}"""
-SLEEP_TIME = 1800
+_ERROR_ALERT_MAX_EXC_CHARS = 500
+
+
+def _error_alert_message(exc: Exception) -> str:
+ # HTML-escaped so a stray < or & in an exception's own message text
+ # (e.g. a file path or repr containing special characters) can't break
+ # Telegram's HTML parse mode and cause the alert itself to fail to send.
+ exc_text = html.escape(_clip(f"{type(exc).__name__}: {exc}", _ERROR_ALERT_MAX_EXC_CHARS))
+ return (
+ "⚠️ netmon has hit an error and is stopping\n\n"
+ f"{exc_text}\n\n"
+ "This was not a transient AI hiccup — it's a real problem with the "
+ "speed test, device scan, database, or notifier delivery itself, so "
+ "the bot is not silently retrying. Check the service logs "
+ "(journalctl -u netmon) for full details."
+ )
+
+
+def _outage_down_alert_message(conf: "cfg.Config") -> str:
+ return (
+ "🔴 Outage detected\n\n"
+ f"The speed test itself has failed for {conf.outage_consecutive_readings} "
+ "consecutive cycles in a row. This looks like your actual internet "
+ "connection is down, not a netmon problem — monitoring will keep "
+ "trying every cycle and will post again once it's back."
+ )
+
+
+def _outage_degraded_alert_message(metric: "models.NetworkMetric", conf: "cfg.Config") -> str:
+ return (
+ "🟡 Degraded connection detected\n\n"
+ f"{metric.download / 10**6:.1f} Mbps down, {metric.ping:.1f} ms ping\n\n"
+ f"That's below the configured thresholds ({conf.outage_download_threshold_mbps:.0f} Mbps / "
+ f"{conf.outage_ping_threshold_ms:.0f} ms) for {conf.outage_consecutive_readings} readings in a row. "
+ "Could be your ISP, could be someone hogging the line — worth a look."
+ )
+
+
+def _outage_recovery_alert_message(previous_state: str, duration_seconds: float) -> str:
+ minutes = int(duration_seconds // 60)
+ if minutes < 60:
+ duration_str = f"{minutes} min" if minutes > 0 else "under a minute"
+ else:
+ duration_str = f"{minutes // 60}h {minutes % 60}m"
+ label = "Outage" if previous_state == "down" else "Degraded connection"
+ return f"✅ {label} resolved\n\nBack to normal after about {duration_str}."
+
log = logging.getLogger("netmon")
@@ -153,7 +230,27 @@ def main():
t = tg.Bot.init(conf.tg_bot_token, conf.tg_chat_id, conf.request_timeout)
r = runner.Runner()
- counter = 0
+ # Normally starts at 0 and climbs to conf.report_cycle_count before the
+ # first detailed report fires. --test-ai starts it already at threshold
+ # so the very first cycle exercises the AI + graph + notifier path; the
+ # detailed-report branch resets counter back to 0 on completion, so
+ # every cycle after that follows the normal schedule automatically —
+ # no config to remember to revert afterward.
+ counter = conf.report_cycle_count if conf.test_ai else 0
+ if conf.test_ai:
+ log.info("--test-ai passed: forcing a detailed AI report on the first cycle, then resuming normal schedule.")
+
+ # Instant outage/degradation alerting state. Tracked across loop
+ # iterations, separate from the crash-on-infra-failure handling below:
+ # the speed test genuinely failing, or reporting a genuinely bad
+ # connection, is the exact condition this tool exists to detect -- it
+ # is not a bug in netmon itself, so it does not crash the process the
+ # way an nmap/database/notifier failure does. Instead it alerts once
+ # per episode (not every cycle, to avoid spam) and again on recovery.
+ consecutive_bad_readings = 0
+ outage_alerted = False
+ outage_started_at: datetime | None = None
+ connection_state = "ok" # one of "ok", "degraded", "down"
with (
sqlite.DB.init(conf.db_path) as database,
@@ -161,88 +258,225 @@ def main():
):
log.info("The bot has been started.")
while True:
- t.send_chat_action(ChatAction.TYPING)
-
- metric = r.run_speedtest()
- all_devices = r.run_devices_scan()
-
- with database.transaction():
- database.add_metric(metric)
- device_scan_id = database.add_devices(all_devices)
- speedtest = models.SpeedTest.create(metric.id, device_scan_id)
- database.add_speedtest(speedtest)
- log.info(f"Speedtest has been added: {speedtest}")
-
- if counter >= 8: #send a detailed report with graph every 4 hours
- metrics, device_counts = database.get_metrics_with_device_counts()
-
- user_message = ""
- for m, device_count in zip(metrics, device_counts):
- user_message += REPORT_USER_TEMPLATE.format(
- timestamp=m.timestamp.astimezone().strftime("%Y-%m-%d %H:%M:%S"),
- download=round(m.download / 10**6, 1),
- upload=round(m.upload / 10**6, 1),
- ping=m.ping,
- client=m.client,
- server=m.server,
- download_mb=round(m.bytes_received / 10**6, 1),
- upload_mb=round(m.bytes_sent / 10**6, 1),
- share=m.share,
- device_count=device_count
- ) + "\n"
-
+ try:
t.send_chat_action(ChatAction.TYPING)
+
try:
- report = netmon_ai.send_message(user_message, REPORT_SYSTEM_PROMPT)
- report = report.replace("
", "\n").replace("
", "\n").replace("
", "\n")
- except Exception as e:
- # AI backend down/unreachable/misconfigured: don't lose the
- # whole report, just send the graph with a plain notice
- # instead of a sarcastic AI-written one.
- log.error(f"AI report generation failed, sending graph without commentary: {e}")
- report = (
- "Network Speed Test Report (24h Analysis)\n\n"
- "AI commentary unavailable this cycle — the AI backend "
- "could not be reached. Raw graph data is attached below."
- )
+ metric = r.run_speedtest()
+ except Exception as speedtest_exc:
+ log.error(f"Speed test failed: {speedtest_exc}")
+ consecutive_bad_readings += 1
+ if outage_started_at is None:
+ outage_started_at = datetime.now(timezone.utc)
+ if consecutive_bad_readings >= conf.outage_consecutive_readings and not outage_alerted:
+ try:
+ t.send_message(_outage_down_alert_message(conf))
+ except Exception as notify_err:
+ log.error(f"Failed to send outage alert: {notify_err}")
+ outage_alerted = True
+ connection_state = "down"
+ time.sleep(conf.sleep_time)
+ continue # no metric this cycle -- skip device scan/db/reports entirely
+
+ dl_speed_check = metric.download / 10**6
+ is_degraded = (
+ dl_speed_check < conf.outage_download_threshold_mbps
+ or metric.ping > conf.outage_ping_threshold_ms
+ )
- t.send_chat_action(ChatAction.UPLOAD_PHOTO)
- graph = graphs.NetmonGraph(metrics, device_counts)
- graph_name = graph.plot()
-
- with open(graph_name, "rb") as f:
- t.send_photo(f.read(), report)
-
- log.info("Detailed report has been sent.")
- counter = 0
- else:
- dl_speed = metric.download / 10**6
- ping = metric.ping
- if dl_speed >= 150 and ping <= 20:
- status_text = "Good speed and low latency"
- elif dl_speed < 60 or ping > 40:
- status_text = "A bunch of idiots decided to stream 4K movies all at once, or the ISP's mice were busy chewing on the fiber line again, whatever"
+ if is_degraded:
+ consecutive_bad_readings += 1
+ if outage_started_at is None:
+ outage_started_at = datetime.now(timezone.utc)
+ if consecutive_bad_readings >= conf.outage_consecutive_readings and not outage_alerted:
+ try:
+ t.send_message(_outage_degraded_alert_message(metric, conf))
+ except Exception as notify_err:
+ log.error(f"Failed to send degraded-connection alert: {notify_err}")
+ outage_alerted = True
+ connection_state = "degraded"
else:
- status_text = "At least it works, I guess"
-
- msg = MINI_REPORT_TEMPLATE.format(
- timestamp=metric.timestamp.astimezone().strftime("%Y-%m-%d %H:%M:%S"),
- download=dl_speed,
- upload=metric.upload / 10**6,
- ping=ping,
- device_count=len(all_devices),
- client=metric.client,
- server=metric.server,
- download_mb=metric.bytes_received / 10**6,
- upload_mb=metric.bytes_sent / 10**6,
- status_text=status_text,
- )
- t.send_message(msg)
- log.info("Mini report has been sent.")
+ if outage_alerted and outage_started_at is not None:
+ duration_seconds = (datetime.now(timezone.utc) - outage_started_at).total_seconds()
+ try:
+ t.send_message(_outage_recovery_alert_message(connection_state, duration_seconds))
+ except Exception as notify_err:
+ log.error(f"Failed to send recovery alert: {notify_err}")
+ consecutive_bad_readings = 0
+ outage_alerted = False
+ outage_started_at = None
+ connection_state = "ok"
+
+ all_devices = r.run_devices_scan()
+
+ with database.transaction():
+ database.add_metric(metric)
+ device_scan_id = database.add_devices(all_devices)
+ speedtest = models.SpeedTest.create(metric.id, device_scan_id)
+ database.add_speedtest(speedtest)
+ log.info(f"Speedtest has been added: {speedtest}")
+
+ if counter >= conf.report_cycle_count: #send a detailed report with graph every N cycles
+ metrics, device_counts = database.get_metrics_with_device_counts()
+
+ user_message = ""
+ for m, device_count in zip(metrics, device_counts):
+ user_message += REPORT_USER_TEMPLATE.format(
+ timestamp=m.timestamp.astimezone().strftime("%Y-%m-%d %H:%M:%S"),
+ download=round(m.download / 10**6, 1),
+ upload=round(m.upload / 10**6, 1),
+ ping=m.ping,
+ jitter_line=_ai_history_jitter_line(m),
+ client=m.client,
+ server=m.server,
+ download_mb=round(m.bytes_received / 10**6, 1),
+ upload_mb=round(m.bytes_sent / 10**6, 1),
+ share=m.share,
+ device_count=device_count
+ ) + "\n"
+
+ device_details = database.get_latest_devices_with_novelty()
+ if device_details:
+ vendor_counts: dict[str, int] = {}
+ for d in device_details:
+ if d["vendor"]:
+ key = d["vendor"]
+ elif d["mac"]:
+ key = "Unknown vendor"
+ else:
+ key = "Unidentified (off-subnet, no MAC resolved)"
+ vendor_counts[key] = vendor_counts.get(key, 0) + 1
+
+ vendor_lines = [
+ VENDOR_COUNT_TEMPLATE.format(vendor=vendor, count=count)
+ for vendor, count in sorted(vendor_counts.items(), key=lambda kv: -kv[1])
+ ]
+ user_message += "\nDevice vendor breakdown (currently online):\n" + "\n".join(vendor_lines) + "\n"
+
+ new_devices = [d for d in device_details if d["is_new"]]
+ if new_devices:
+ new_lines = []
+ for d in new_devices:
+ vendor_part = f" | Vendor: {d['vendor']}" if d["vendor"] else " | Vendor: unknown"
+ hostname_part = f" | Hostname: {d['hostname']}" if d["hostname"] else ""
+ new_lines.append(NEW_DEVICE_ENTRY_TEMPLATE.format(
+ ip=d["ip"],
+ vendor_part=vendor_part,
+ hostname_part=hostname_part,
+ ))
+ user_message += "\nNEW devices this cycle:\n" + "\n".join(new_lines) + "\n"
+ else:
+ user_message += "\nNEW devices this cycle: none.\n"
+ else:
+ user_message += "\nDevice vendor breakdown (currently online): none detected this cycle.\nNEW devices this cycle: none.\n"
+
+ t.send_chat_action(ChatAction.TYPING)
+ raw_response = None
+ try:
+ raw_response = netmon_ai.send_message(user_message, REPORT_SYSTEM_PROMPT, context_size=conf.ai_context_size)
+
+ # Defensive cleanup: some models wrap JSON in ```json
+ # fences despite being told not to -- strip those if present.
+ cleaned = raw_response.strip()
+ if cleaned.startswith("```"):
+ cleaned = cleaned.strip("`")
+ if cleaned.lower().startswith("json"):
+ cleaned = cleaned[4:]
+ cleaned = cleaned.strip()
+
+ parsed = json.loads(cleaned)
+
+ dynamics_analysis = _clip(str(parsed.get("dynamics_analysis", "")), _DYNAMICS_ANALYSIS_MAX_CHARS)
+ device_watch = _clip(str(parsed.get("device_watch", "")), _DEVICE_WATCH_MAX_CHARS)
+ conclusion = _clip(str(parsed.get("conclusion", "")), _CONCLUSION_MAX_CHARS)
+
+ if not dynamics_analysis or not device_watch or not conclusion:
+ raise ValueError(f"AI response missing one or more required fields: {parsed!r}")
+
+ except Exception as e:
+ # Covers both AI backend failures (unreachable/misconfigured)
+ # and the model returning malformed/incomplete JSON -- either
+ # way, don't lose the whole report, just fall back to plain
+ # non-AI text for the three commentary fields. The skeleton
+ # itself is unaffected either way since it's built in code.
+ log.error(
+ f"AI report generation failed or returned invalid data: {e}\n"
+ f"Raw AI response was: {raw_response!r}",
+ exc_info=True,
+ )
+ dynamics_analysis = "AI commentary unavailable this cycle — the AI backend could not be reached or returned an unexpected response."
+ device_watch = "AI commentary unavailable this cycle."
+ conclusion = "Raw graph data is attached below."
+
+ report = REPORT_TEMPLATE_SHELL.format(
+ client=metric.client,
+ server=metric.server,
+ download=metric.download / 10**6,
+ upload=metric.upload / 10**6,
+ ping=metric.ping,
+ jitter_line=_shell_jitter_line(metric),
+ device_count=len(all_devices),
+ dynamics_analysis=dynamics_analysis,
+ device_watch=device_watch,
+ download_mb=metric.bytes_received / 10**6,
+ upload_mb=metric.bytes_sent / 10**6,
+ conclusion=conclusion,
+ )
- counter += 1
+ t.send_chat_action(ChatAction.UPLOAD_PHOTO)
+ graph = graphs.NetmonGraph(metrics, device_counts)
+ graph_name = graph.plot()
+
+ with open(graph_name, "rb") as f:
+ t.send_photo(f.read(), report)
+
+ log.info("Detailed report has been sent.")
+ counter = 0
+ else:
+ dl_speed = metric.download / 10**6
+ ping = metric.ping
+ if dl_speed >= 150 and ping <= 20:
+ status_text = "Good speed and low latency"
+ elif dl_speed < 60 or ping > 40:
+ status_text = "A bunch of idiots decided to stream 4K movies all at once, or the ISP's mice were busy chewing on the fiber line again, whatever"
+ else:
+ status_text = "At least it works, I guess"
+
+ msg = MINI_REPORT_TEMPLATE.format(
+ timestamp=metric.timestamp.astimezone().strftime("%Y-%m-%d %H:%M:%S"),
+ download=dl_speed,
+ upload=metric.upload / 10**6,
+ ping=ping,
+ jitter_line=_mini_report_jitter_line(metric),
+ device_count=len(all_devices),
+ client=metric.client,
+ server=metric.server,
+ download_mb=metric.bytes_received / 10**6,
+ upload_mb=metric.bytes_sent / 10**6,
+ status_text=status_text,
+ )
+ t.send_message(msg)
+ log.info("Mini report has been sent.")
+
+ counter += 1
+
+ except Exception as e:
+ # Real infrastructure failures (speed test, device scan,
+ # database, or notifier delivery) are NOT retried silently --
+ # per the project's design, they should crash loudly so a
+ # persistent problem doesn't go unnoticed. Before crashing,
+ # make a best-effort attempt to post an alert to the
+ # configured notifier so the failure is visible outside of
+ # server logs, then re-raise so the process actually exits
+ # (systemd/your process manager handles the restart policy).
+ log.error(f"Unrecoverable error during monitoring cycle: {e}", exc_info=True)
+ try:
+ t.send_message(_error_alert_message(e))
+ except Exception as notify_err:
+ log.error(f"Additionally failed to notify about the error: {notify_err}")
+ raise
- time.sleep(SLEEP_TIME)
+ time.sleep(conf.sleep_time)
if __name__ == "__main__":
main()
\ No newline at end of file
diff --git a/models.py b/models.py
index f828c43..7ce4537 100644
--- a/models.py
+++ b/models.py
@@ -1,8 +1,9 @@
from dataclasses import dataclass
-import uuid
+import uuid
from uuid_extensions import uuid7str
from datetime import timezone
from datetime import datetime
+from typing import Optional
@dataclass(frozen=True, slots=True)
class NetworkMetric:
@@ -16,6 +17,8 @@ class NetworkMetric:
server: str
bytes_sent: int
bytes_received: int
+ jitter_ms: Optional[float] = None
+ packet_loss_pct: Optional[float] = None
@classmethod
def create(
@@ -27,7 +30,9 @@ def create(
client: str,
server: str,
bytes_sent: int,
- bytes_received: int
+ bytes_received: int,
+ jitter_ms: Optional[float] = None,
+ packet_loss_pct: Optional[float] = None,
) -> "NetworkMetric":
if download < 0:
raise ValueError("download must be non-negative")
@@ -43,18 +48,24 @@ def create(
raise ValueError("client cannot be empty")
if not server or not server.strip():
raise ValueError("server cannot be empty")
+ if jitter_ms is not None and jitter_ms < 0:
+ raise ValueError("jitter_ms must be non-negative")
+ if packet_loss_pct is not None and not (0 <= packet_loss_pct <= 100):
+ raise ValueError("packet_loss_pct must be between 0 and 100")
return cls(
- id=uuid.UUID(uuid7str()),
+ id=uuid.UUID(uuid7str()),
download=download,
upload=upload,
ping=ping,
- timestamp=datetime.now(timezone.utc),
+ timestamp=datetime.now(timezone.utc),
share=share,
client=client,
server=server,
bytes_sent=bytes_sent,
- bytes_received=bytes_received
+ bytes_received=bytes_received,
+ jitter_ms=jitter_ms,
+ packet_loss_pct=packet_loss_pct,
)
@@ -62,25 +73,34 @@ def create(
class NetworkDevice:
id: uuid.UUID
ip: str
- latency_ms: float
- timestamp: datetime
+ latency_ms: float
+ timestamp: datetime
+ mac: Optional[str] = None
+ vendor: Optional[str] = None
+ hostname: Optional[str] = None
@classmethod
def create(
cls,
ip: str,
latency_ms: float,
+ mac: Optional[str] = None,
+ vendor: Optional[str] = None,
+ hostname: Optional[str] = None,
) -> "NetworkDevice":
if not ip or not ip.strip():
raise ValueError("IP cannot be empty")
if latency_ms < 0:
raise ValueError("Latency must be non-negative")
-
+
return cls(
id=uuid.UUID(uuid7str()),
ip=ip,
latency_ms=latency_ms,
- timestamp=datetime.now(timezone.utc)
+ timestamp=datetime.now(timezone.utc),
+ mac=mac,
+ vendor=vendor,
+ hostname=hostname,
)
diff --git a/runner.py b/runner.py
index 3a5ab2a..e18a471 100644
--- a/runner.py
+++ b/runner.py
@@ -47,6 +47,12 @@ class _SpeedTestResponse(BaseModel):
bytes_received: int
share: Optional[str]
client: _ClientInfo
+ # Only present when using the Ookla CLI wrapper (install-ookla-speedtest.sh);
+ # the classic speedtest-cli JSON schema has no equivalent fields, so these
+ # stay None for that backend and any bufferbloat/jitter reporting is
+ # simply skipped rather than shown as a fake zero.
+ jitter: Optional[float] = None
+ packet_loss: Optional[float] = None
@field_validator('timestamp', mode='before')
@classmethod
@@ -80,23 +86,52 @@ def run_speedtest(self) -> models.NetworkMetric:
client=parsed_result.client.isp,
server=parsed_result.server.name,
bytes_sent=parsed_result.bytes_sent,
- bytes_received=parsed_result.bytes_received
+ bytes_received=parsed_result.bytes_received,
+ jitter_ms=parsed_result.jitter,
+ packet_loss_pct=parsed_result.packet_loss,
)
@staticmethod
def _parse_device(host: ET.Element) -> models.NetworkDevice | None:
- addr = host.find("address")
status = host.find("status")
- if addr is None or status is None or status.get("state") != "up":
+ if status is None or status.get("state") != "up":
return None
+ ip: str | None = None
+ mac: str | None = None
+ vendor: str | None = None
+ for addr in host.findall("address"):
+ addrtype = addr.get("addrtype")
+ if addrtype == "ipv4":
+ ip = addr.get("addr")
+ elif addrtype == "mac":
+ # nmap only reports a MAC (and resolves vendor from its
+ # built-in OUI database) for hosts on the same local subnet
+ # it can ARP directly — this needs no extra scan flags,
+ # since the existing sudo ARP scan already returns it.
+ mac = addr.get("addr")
+ vendor = addr.get("vendor") or None
+
+ if not ip:
+ return None
+
+ hostname: str | None = None
+ hostnames_el = host.find("hostnames")
+ if hostnames_el is not None:
+ hostname_el = hostnames_el.find("hostname")
+ if hostname_el is not None:
+ hostname = hostname_el.get("name") or None
+
times = host.find("times")
srtt = times.get("srtt") if times is not None else None
latency_ms = round(int(srtt) / 1000, 2) if srtt is not None else 0
return models.NetworkDevice.create(
- ip=addr.get("addr") or "",
- latency_ms=latency_ms
+ ip=ip,
+ latency_ms=latency_ms,
+ mac=mac,
+ vendor=vendor,
+ hostname=hostname,
)
def run_devices_scan(self) -> list[models.NetworkDevice]:
diff --git a/sqlite.py b/sqlite.py
index a43a214..8638718 100644
--- a/sqlite.py
+++ b/sqlite.py
@@ -48,17 +48,24 @@ def _create_schema(self):
server TEXT NOT NULL,
bytes_sent INTEGER NOT NULL,
bytes_received INTEGER NOT NULL,
+ jitter_ms REAL,
+ packet_loss_pct REAL,
timestamp DATETIME NOT NULL DEFAULT (datetime('now'))
);
""")
+ self._migrate_metrics_columns()
self.conn.execute("""
CREATE TABLE IF NOT EXISTS device_scans (
id TEXT PRIMARY KEY,
ips TEXT NOT NULL,
- latencies TEXT NOT NULL
+ latencies TEXT NOT NULL,
+ macs TEXT NOT NULL DEFAULT '[]',
+ vendors TEXT NOT NULL DEFAULT '[]',
+ hostnames TEXT NOT NULL DEFAULT '[]'
);
""")
+ self._migrate_device_scans_columns()
self.conn.execute("""
CREATE TABLE IF NOT EXISTS speedtest (
@@ -68,6 +75,30 @@ def _create_schema(self):
);
""")
+ def _migrate_metrics_columns(self):
+ # Databases created before jitter/packet-loss tracking was added
+ # won't have these columns yet; add them in place. Nullable since
+ # only the Ookla CLI backend provides this data -- classic
+ # speedtest-cli users will just have NULL here indefinitely, which
+ # is expected, not a migration failure.
+ cursor = self.conn.execute("PRAGMA table_info(metrics)")
+ existing_cols = {row[1] for row in cursor.fetchall()}
+ for col in ("jitter_ms", "packet_loss_pct"):
+ if col not in existing_cols:
+ self.conn.execute(f"ALTER TABLE metrics ADD COLUMN {col} REAL")
+
+ def _migrate_device_scans_columns(self):
+ # Databases created before mac/vendor/hostname tracking was added
+ # won't have these columns yet; add them in place so upgrading
+ # doesn't require recreating the database.
+ cursor = self.conn.execute("PRAGMA table_info(device_scans)")
+ existing_cols = {row[1] for row in cursor.fetchall()}
+ for col in ("macs", "vendors", "hostnames"):
+ if col not in existing_cols:
+ self.conn.execute(
+ f"ALTER TABLE device_scans ADD COLUMN {col} TEXT NOT NULL DEFAULT '[]'"
+ )
+
@contextmanager
def transaction(self):
depth = _tx_depth.get()
@@ -89,26 +120,29 @@ def add_metric(self, metric: models.NetworkMetric):
self.conn.execute("""
INSERT INTO metrics (
id, download, upload, ping, timestamp, share, client, server,
- bytes_sent, bytes_received
+ bytes_sent, bytes_received, jitter_ms, packet_loss_pct
)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
str(metric.id), metric.download, metric.upload,
metric.ping, metric.timestamp, metric.share,
metric.client, metric.server, metric.bytes_sent,
- metric.bytes_received
+ metric.bytes_received, metric.jitter_ms, metric.packet_loss_pct
))
def add_devices(self, devices: list[models.NetworkDevice]) -> uuid.UUID:
scan_id = uuid.UUID(uuid7str())
ips = json.dumps([d.ip for d in devices])
latencies = json.dumps([d.latency_ms for d in devices])
+ macs = json.dumps([d.mac for d in devices])
+ vendors = json.dumps([d.vendor for d in devices])
+ hostnames = json.dumps([d.hostname for d in devices])
with self.transaction():
self.conn.execute("""
- INSERT INTO device_scans (id, ips, latencies)
- VALUES (?, ?, ?)
- """, (str(scan_id), ips, latencies))
+ INSERT INTO device_scans (id, ips, latencies, macs, vendors, hostnames)
+ VALUES (?, ?, ?, ?, ?, ?)
+ """, (str(scan_id), ips, latencies, macs, vendors, hostnames))
return scan_id
def add_speedtest(self, speedtest: models.SpeedTest):
@@ -123,7 +157,8 @@ def get_metrics(self) -> list[models.NetworkMetric]:
with closing(self.conn.cursor()) as cursor:
cursor.execute("""
SELECT * FROM (
- SELECT id, download, upload, ping, timestamp, share, client, server, bytes_sent, bytes_received
+ SELECT id, download, upload, ping, timestamp, share, client, server, bytes_sent, bytes_received,
+ jitter_ms, packet_loss_pct
FROM metrics
WHERE timestamp > DATETIME('now', '-24 hours')
ORDER BY timestamp DESC
@@ -148,7 +183,9 @@ def get_metrics(self) -> list[models.NetworkMetric]:
client=row[6],
server=row[7],
bytes_sent=row[8],
- bytes_received=row[9]
+ bytes_received=row[9],
+ jitter_ms=row[10],
+ packet_loss_pct=row[11]
)
for row in rows
]
@@ -159,7 +196,7 @@ def get_metrics_with_device_counts(self) -> tuple[list[models.NetworkMetric], li
cursor.execute("""
SELECT * FROM (
SELECT m.id, m.download, m.upload, m.ping, m.timestamp, m.share, m.client, m.server,
- m.bytes_sent, m.bytes_received, ds.ips
+ m.bytes_sent, m.bytes_received, m.jitter_ms, m.packet_loss_pct, ds.ips
FROM metrics m
JOIN speedtest st ON st.metrics_id = m.id
JOIN device_scans ds ON ds.id = st.device_scans_id
@@ -186,11 +223,71 @@ def get_metrics_with_device_counts(self) -> tuple[list[models.NetworkMetric], li
client=row[6],
server=row[7],
bytes_sent=row[8],
- bytes_received=row[9]
+ bytes_received=row[9],
+ jitter_ms=row[10],
+ packet_loss_pct=row[11]
))
- device_counts.append(len(json.loads(row[10])))
+ device_counts.append(len(json.loads(row[12])))
return metrics, device_counts
+ def get_latest_devices_with_novelty(self, lookback_days: int = 14) -> list[dict]:
+ """
+ Returns the most recent device scan's devices, each annotated with
+ whether its MAC address has been seen in any earlier scan within
+ the lookback window — i.e. whether it's new to the network.
+ Devices without a resolvable MAC (nmap only resolves MAC for hosts
+ on the same local subnet) are never flagged as new, since there's
+ no reliable identity to compare against.
+ """
+ try:
+ with closing(self.conn.cursor()) as cursor:
+ cursor.execute("""
+ SELECT id, ips, latencies, macs, vendors, hostnames
+ FROM device_scans
+ ORDER BY rowid DESC
+ LIMIT 1
+ """)
+ latest = cursor.fetchone()
+ if latest is None:
+ return []
+
+ latest_id = latest[0]
+ ips = json.loads(latest[1])
+ latencies = json.loads(latest[2])
+ macs = json.loads(latest[3])
+ vendors = json.loads(latest[4])
+ hostnames = json.loads(latest[5])
+
+ cursor.execute("""
+ SELECT ds.macs
+ FROM device_scans ds
+ JOIN speedtest st ON st.device_scans_id = ds.id
+ JOIN metrics m ON m.id = st.metrics_id
+ WHERE ds.id != ?
+ AND m.timestamp > DATETIME('now', ?)
+ """, (latest_id, f'-{lookback_days} days'))
+
+ known_macs: set[str] = set()
+ for (macs_json,) in cursor.fetchall():
+ for m in json.loads(macs_json):
+ if m:
+ known_macs.add(m)
+ except sqlite3.Error as e:
+ raise RuntimeError(f"Failed to get latest devices with novelty: {e}")
+
+ devices = []
+ for i, ip in enumerate(ips):
+ mac = macs[i] if i < len(macs) else None
+ devices.append({
+ "ip": ip,
+ "mac": mac,
+ "vendor": vendors[i] if i < len(vendors) else None,
+ "hostname": hostnames[i] if i < len(hostnames) else None,
+ "latency_ms": latencies[i] if i < len(latencies) else None,
+ "is_new": bool(mac) and mac not in known_macs,
+ })
+ return devices
+
def close(self):
self.conn.close()
\ No newline at end of file