diff --git a/.github/prerelease.json b/.github/prerelease.json new file mode 100644 index 0000000..a45a224 --- /dev/null +++ b/.github/prerelease.json @@ -0,0 +1,3 @@ +{ + "tag": "v1.2.0" +} diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml new file mode 100644 index 0000000..2eb68fb --- /dev/null +++ b/.github/workflows/prerelease.yml @@ -0,0 +1,107 @@ +name: Prerelease + +on: + push: + branches: [main] + paths: ['.github/prerelease.json'] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: publish-prerelease + cancel-in-progress: false + +jobs: + prepare: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.version.outputs.tag }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - id: version + name: Validate prerelease request + run: | + python - <<'PY' + import json, os, re + from pathlib import Path + tag = json.loads(Path('.github/prerelease.json').read_text())['tag'] + manifest = json.loads(Path('custom_components/entity_replacer/manifest.json').read_text()) + assert re.fullmatch(r'v\d+\.\d+\.\d+', tag), 'Invalid version tag' + assert tag == 'v' + manifest['version'], 'Tag and manifest version differ' + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'tag={tag}\n') + PY + + tests: + needs: prepare + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - uses: actions/setup-node@v4 + with: + node-version: '22' + - run: python -m pip install -r requirements-dev.txt + - run: python -m unittest discover -s tests -v + - run: node tests/frontend_unit.cjs + - run: node --check custom_components/entity_replacer/frontend/panel.js + - run: python scripts/build_release.py --check-only --tag '${{ needs.prepare.outputs.tag }}' + + recorder-tests: + needs: prepare + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.14' + - run: python -m pip install homeassistant==2026.3.0 + - run: python -m unittest discover -s ha_tests -v + + hassfest: + needs: prepare + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: home-assistant/actions/hassfest@master + + hacs: + needs: prepare + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: hacs/action@main + with: + category: integration + + publish: + needs: [prepare, tests, recorder-tests, hassfest, hacs] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Build installation ZIP + env: + RELEASE_TAG: ${{ needs.prepare.outputs.tag }} + run: python scripts/build_release.py --tag "$RELEASE_TAG" + - name: Publish GitHub prerelease + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.prepare.outputs.tag }} + run: | + gh release create "$RELEASE_TAG" dist/*.zip dist/SHA256SUMS.txt \ + --repo "$GITHUB_REPOSITORY" --target "$GITHUB_SHA" \ + --prerelease --latest=false --title "Device Replacer $RELEASE_TAG — prerelease" \ + --notes-file RELEASE_NOTES.md diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 3c71d28..ea6affe 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -26,6 +26,16 @@ jobs: - run: node --check custom_components/entity_replacer/frontend/panel.js - run: python scripts/build_release.py --check-only + recorder-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.14' + - run: python -m pip install homeassistant==2026.3.0 + - run: python -m unittest discover -s ha_tests -v + hassfest: runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1511d7c..5865d2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 1.2.0 — prerelease + +- Add a separate reviewed statistics transfer for compatible disabled measurement sensors with an unused destination statistics ID. +- Use recorder metadata APIs, preserve both statistics tables, verify completion, and leave raw history/configuration unchanged. +- Block existing destination statistics, cumulative counters, incompatible units/classes, and unverifiable sources. +- Persist statistics audit records, prevent blind retries after uncertain outcomes, and exclude them from configuration Restore. +- Add standalone and real Home Assistant 2026.3 SQLite recorder tests. + ## 1.1.3 — 2026-10-10 - Hide the panel header sidebar toggle in desktop/wide mode. diff --git a/README.md b/README.md index 3fe3aa7..20a7920 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The integration updates literal references in YAML and saved UI dashboards. Comp It adds a **Device Replacer** page to the administrator sidebar. There is no dashboard card, frontend resource entry, or `configuration.yaml` setup. The internal domain, installation directory and panel URL remain `entity_replacer` and `/entity-replacer` so existing Entity Replacer installs upgrade in place. -Maintained by [FortranFour](https://github.com/FortranFour). Current release: **1.1.3**. Requires **Home Assistant 2026.3.0 or newer**, including the bundled local brand images. It uses Home Assistant's existing Python dependencies and has no frontend CDN or additional pip requirements. +Maintained by [FortranFour](https://github.com/FortranFour). Prerelease version: **1.2.0** (for testing; latest stable remains **1.1.3**). Requires **Home Assistant 2026.3.0 or newer**, including the bundled local brand images. It uses Home Assistant's existing Python dependencies and has no frontend CDN or additional pip requirements. Screenshot 2026-10-05 104154 @@ -137,11 +137,11 @@ The limits are 8 MiB per file or dashboard, 64 MiB of filesystem content, 10,000 **UI helpers and integration settings:** Open **Settings → Devices & services**, select the named integration or helper, and edit its referenced entity through the owning UI. Some integrations require a helper to be recreated. The report shows the physical storage path and location for identification, but the integration never rewrites those internal stores directly. -**Energy dashboard:** Enable **Additional scan options → Report references in other internal storage (manual review)** to report literal matching references in Energy settings. Change the configured sensor under **Settings → Dashboards → Energy**. Energy configuration is report-only in this release. Replacing an entity reference does not migrate or merge the old sensor's historical energy statistics. +**Energy dashboard:** Enable **Additional scan options → Report references in other internal storage (manual review)** to report literal matching references in Energy settings. Change the configured sensor under **Settings → Dashboards → Energy**. Energy configuration is report-only in this release. Replacing an entity reference does not migrate or merge the old sensor's historical energy statistics. The separate statistics transfer below intentionally excludes cumulative energy counters. **Cross-domain changes:** Replacing a `switch` with a `light`, or a `sensor` with a `binary_sensor`, can require different actions, values, conditions, and card types. This release restricts automatic replacement to the same domain. Even within a domain, inspect device capabilities, service data, thresholds, and units in the preview. -**Computed references and external systems:** IDs assembled from pieces, wildcard/regex selections, aliases selected through areas or labels, former device/entity IDs you have not supplied after registry removal, and add-on-owned configurations such as Node-RED are not fully discoverable by a literal scan. Review those separately. Replacing references does not rename or delete the source entity, migrate recorder history or statistics, pair hardware, or transfer entity/device registry identity. +**Computed references and external systems:** IDs assembled from pieces, wildcard/regex selections, aliases selected through areas or labels, former device/entity IDs you have not supplied after registry removal, and add-on-owned configurations such as Node-RED are not fully discoverable by a literal scan. Review those separately. Replacing references does not rename or delete the source entity, migrate recorder history or statistics, pair hardware, or transfer entity/device registry identity. Statistics transfer is a separate, explicitly reviewed operation with the limits below. ## Report-only use @@ -230,3 +230,44 @@ Report issues at [FortranFour/device-replacer/issues](https://github.com/Fortran - [Native storage persistence source](https://github.com/home-assistant/core/blob/dev/homeassistant/helpers/storage.py) The integration reads version-dependent Lovelace objects to enumerate dashboards and identify their storage paths. If those interfaces are unavailable, unsupported dashboard references remain manual-review items rather than being written by an unsafe fallback. + +## Historical statistics transfer (limited first version) + +This operation addresses [issue #1](https://github.com/FortranFour/device-replacer/issues/1) for compatible **measurement** sensors. It renames the existing recorder statistics series to the replacement entity ID through Home Assistant's Python recorder API. Hourly and retained five-minute statistics remain attached to the same numeric metadata ID. It does not copy measurements, merge two series, change raw state history, rewrite configuration, or change Energy Dashboard settings. + +Requirements are deliberately strict: + +- A running recorder and the supported metadata API (Home Assistant 2026.3+). +- Both entities remain registered, are disabled, and have no loaded state. Simply being unavailable is insufficient. Removed sources cannot be verified in this version. +- Both registry entries declare `state_class: measurement`, the same known device class, and the same nonempty unit. The source's stored unit must match exactly. No unit conversion is attempted. +- The source contains recorder-owned arithmetic-mean statistics and at least one retained hourly or five-minute record. +- The replacement has **no statistics metadata**, including an empty metadata record. Existing replacement statistics are never deleted to make room. +- `total`, `total_increasing`, cumulative energy counters, external statistics and circular means are unsupported. + +### Transfer a fresh replacement + +1. Create and verify a current backup that includes your recorder database. Device Replacer's configuration backups do **not** contain that database. For an external database, use its appropriate backup process. +2. Disable the old sensor and keep it disabled. Pair the replacement, verify its role, then disable its sensor before it accumulates statistics. Wait for both sensors to unload; reload their integration or restart Home Assistant if necessary. Do not delete either entity from the registry. +3. In Device Replacer, choose the pair in **Single entity**, or load and confirm whole-device entity mappings. Open **Transfer historical statistics** and click **Check statistics** for one pair. This read-only check works even if there are no configuration references to replace. +4. Review the pair, class, units and latest retained periods. Confirm the measurement role and database backup, then select **Transfer reviewed statistics**. The preview is session-bound, expires after 15 minutes, and is checked again at execution. +5. Wait for the verified result, then enable **only the replacement sensor**. Review its statistics in Developer tools. Replace configuration references separately; if the replacement is disabled during that scan, enable **Allow unavailable or disabled replacements** for reference edits only. + +Home Assistant can refuse a rename without raising an exception. Device Replacer therefore waits for recorder completion and verifies the destination metadata and numeric ID rather than trusting the callback. No database schema migration or direct SQL is used. + +### Audit and recovery + +Each attempt writes a durable audit manifest under `.entity_replacer_backups//manifest.json` before queuing recorder work. Statistics audit entries appear separately in the panel and cannot be selected for configuration Restore; the backend also rejects that operation. There is no automatic database undo after recording resumes. + +A timeout, cancellation, failed verification, or interrupted transfer has an **unverified outcome**: recorder work may still finish. The reviewed token is consumed, and unresolved audit entries block further transfers involving either ID even after restart. Do not retry blindly. Inspect the database, Home Assistant log and audit manifest; recover using your database-inclusive backup if needed. Use **Check outcome** in the statistics audit to verify a late completion without submitting another recorder job. It clears the unresolved audit only when the original numeric metadata ID is provably at the destination and the source is absent. Otherwise it stays blocked: inspect/recover the database and audit record. This version does not offer a force/retry button. Do not change these sensors' registry settings from another session while a transfer is running. + +### Development checks + +The ordinary unit suite uses HA stubs. Run real recorder tests separately so those stubs cannot mask API compatibility: + +```bash +uv venv --python 3.14 /tmp/device-replacer-ha +uv pip install --python /tmp/device-replacer-ha/bin/python homeassistant==2026.3.0 +/tmp/device-replacer-ha/bin/python -m unittest discover -s ha_tests -v +``` + +These tests use a temporary SQLite database and real Home Assistant recorder APIs. They check hourly/five-minute row preservation, metadata ID continuity, unchanged raw history, and collision refusal. No production Home Assistant connection is used. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f82dc10..83ddd04 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,9 @@ -# Device Replacer 1.1.3 +# Device Replacer 1.2.0 — prerelease -Hide the duplicate hamburger menu in desktop/wide view. Retain it in Home Assistant’s mobile/narrow view so navigation remains accessible. Visibility follows the narrow layout property supplied by Home Assistant and updates when the layout changes. +Add a separate statistics transfer workflow for compatible arithmetic measurement sensors. The source and replacement must both be disabled and unloaded, retain verifiable registry metadata, and use matching units/classes. The replacement must have no stored statistics metadata. -The cyan icon, device replacement, single-entity replacement, backups and restore remain available. Update through HACS, restart Home Assistant, and refresh the browser/app to load the new frontend. No new integration entry is needed. +Transfer uses recorder's metadata API and verifies the result. Hourly and retained five-minute measurements keep their existing metadata ID; raw state history and configuration are unchanged. Cumulative counters (including Energy sensors), merges, unit conversion and removed/unverifiable sources are excluded in this first version. + +A database-inclusive backup is required for recovery. Statistics operations have separate audit records and are never undone by configuration Restore. Uncertain recorder outcomes consume the preview and block further transfers involving the pair until reconciled. + +This is a prerelease for testing before a stable release. Real recorder tests target Home Assistant 2026.3.0 with SQLite; other database engines and production installations still need validation. diff --git a/custom_components/entity_replacer/const.py b/custom_components/entity_replacer/const.py index 03a97d2..5008330 100644 --- a/custom_components/entity_replacer/const.py +++ b/custom_components/entity_replacer/const.py @@ -1,7 +1,7 @@ """Constants for Device Replacer.""" DOMAIN = "entity_replacer" -VERSION = "1.1.3" +VERSION = "1.2.0" PANEL_PATH = "entity-replacer" PANEL_NAME = "entity-replacer-panel" STATIC_PATH = "/entity_replacer_static" diff --git a/custom_components/entity_replacer/device_manager.py b/custom_components/entity_replacer/device_manager.py index efafea6..e62a758 100644 --- a/custom_components/entity_replacer/device_manager.py +++ b/custom_components/entity_replacer/device_manager.py @@ -311,7 +311,7 @@ async def scan_device(self, owner, source, target, mappings, confirmed_mappings= mode="device", mappings=rows, source_device=source, target_device=target, registry_snapshot=fingerprint, device_blocks=blocks) if old is None: warnings.append("The old device is removed. Only its entered device ID and explicitly supplied entity/registry IDs can be found; omitted former entities cannot be discovered.") - warnings.append("Review unassigned entities and manual device blocks. Pairing, identities, areas/labels, and historical statistics are not transferred.") + warnings.append("Review unassigned entities and manual device blocks. Pairing, identities, areas/labels, and historical statistics are not transferred by reference replacement; eligible measurement statistics have a separate transfer action.") return await self._public_preview(preview_id, preview, warnings + notes, file_count, dashboard_count) async def _check_device_preview(self, preview, selected): diff --git a/custom_components/entity_replacer/engine.py b/custom_components/entity_replacer/engine.py index 55475a1..3882ad3 100644 --- a/custom_components/entity_replacer/engine.py +++ b/custom_components/entity_replacer/engine.py @@ -709,13 +709,36 @@ def history(self) -> list[dict]: continue try: manifest = self.load(directory.name) - result.append({key: manifest[key] for key in ("job_id", "created", "source", "target", "status")}) + result.append({**{key: manifest[key] for key in ("job_id", "created", "source", "target", "status")}, "mode": manifest.get("mode", "entity")}) except (OSError, ValueError, KeyError, ReplacementError): continue if len(result) >= 30: break return result + def statistics_conflicts(self, source: str, target: str) -> list[str]: + """Find unresolved recorder operations, including across HA restarts. + + Unlike the UI history this scan is not capped at 30 records. Refuse a + damaged manifest rather than overlooking an unknown pending operation. + """ + base = safe_path(self.root, BACKUP_DIR) + if not base.exists(): + return [] + conflicts = [] + for directory in base.iterdir(): + if not JOB_ID.fullmatch(directory.name): + continue + try: + manifest = self.load(directory.name) + except (OSError, ValueError, KeyError, ReplacementError) as err: + raise ReplacementError(f"Cannot verify audit record {directory.name}; inspect it before transferring statistics.") from err + if (manifest.get("mode") == "statistics" + and manifest.get("status") in {"statistics_applying", "statistics_outcome_unknown"} + and {source, target} & {manifest["source"], manifest["target"]}): + conflicts.append(manifest["job_id"]) + return conflicts + def render_report(preview: dict) -> str: lines = [ @@ -728,7 +751,7 @@ def render_report(preview: dict) -> str: lines += ["## Confirmed entity mappings", "", "| Source entity | Replacement entity | Old registry ID | New registry ID |", "| --- | --- | --- | --- |"] for row in preview.get("mappings", []): lines.append(f"| `{row['source']}` | {('`' + row['target'] + '`') if row['target'] else 'Unmapped; unchanged'} | {row.get('source_registry_id') or 'Unknown'} | {row.get('target_registry_id') or 'None'} |") - lines += ["", "Device automation selector groups must be selected together. Unsupported blocks must be rebuilt in their owning editor.", "Pairing, unique IDs, areas/labels and historical statistics are not transferred.", ""] + lines += ["", "Device automation selector groups must be selected together. Unsupported blocks must be rebuilt in their owning editor.", "Pairing, unique IDs, areas/labels and historical statistics are not transferred by reference replacement; eligible measurement statistics have a separate transfer action.", ""] for warning in preview["warnings"]: lines += ["- " + warning] for document in preview["documents"]: diff --git a/custom_components/entity_replacer/frontend/panel.js b/custom_components/entity_replacer/frontend/panel.js index 2a09747..a19cdaf 100644 --- a/custom_components/entity_replacer/frontend/panel.js +++ b/custom_components/entity_replacer/frontend/panel.js @@ -1,4 +1,4 @@ -/* Device Replacer 1.1.3. No external frontend libraries or build step. */ +/* Device Replacer 1.2.0. No external frontend libraries or build step. */ const escapeHtml = (value) => String(value ?? "").replace(/[&<>"']/g, (character) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'", }[character])); @@ -30,6 +30,7 @@ class EntityReplacerPanel extends HTMLElement { this._notice = ""; this._preview = null; this._review = null; + this._statisticsPreview = null; this._started = false; this._render(); } @@ -96,6 +97,9 @@ class EntityReplacerPanel extends HTMLElement { } _invalidate() { + this._statisticsPreview = null; + const statistics = this._query("#statistics-section"); + if (statistics) { statistics.innerHTML = this._renderStatistics(); this._bindStatistics(); } this._preview = null; this._review = null; this._selected.clear(); @@ -242,6 +246,59 @@ class EntityReplacerPanel extends HTMLElement { }); } + _statisticsPairs() { + return this._mode === "device" + ? (this._mappingConfirmed ? this._mappings.filter((row) => row.source.startsWith("sensor.") && row.target) : []) + : (this._source.trim() && this._target.trim() ? [{ source: this._source.trim(), target: this._target.trim() }] : []); + } + + async _previewStatistics(index) { + const pair = this._statisticsPairs()[index]; + if (!pair) return; + this._statisticsPreview = null; + await this._run("Checking stored statistics and disabled sensors…", async () => { + this._statisticsPreview = await this._request("statistics_preview", { source: pair.source.trim(), target: pair.target.trim() }); + }); + } + + async _applyStatistics() { + if (!this._statisticsPreview?.eligible || !this._query("#statistics-confirm")?.checked) return; + const previewId = this._statisticsPreview.preview_id; + await this._run("Transferring statistics through recorder and verifying the result…", async () => { + let result; + try { + result = await this._request("statistics_apply", { preview_id: previewId, confirmed: true }); + } finally { + // A timeout can leave recorder work queued. Never keep a retry button. + this._statisticsPreview = null; + } + this._notice = `${result.message} Audit: ${result.job_id}`; + await this._refreshHistory(); + }); + } + + _statisticsTime(timestamp) { + return timestamp == null ? "none" : new Date(timestamp * 1000).toLocaleString(); + } + + async _verifyStatistics(jobId) { + await this._run("Checking recorder metadata without retrying the transfer…", async () => { + const result = await this._request("statistics_verify", { job_id: jobId }); + this._notice = result.message; + await this._refreshHistory(); + }); + } + + _renderStatistics() { + const pairs = this._statisticsPairs(); + const preview = this._statisticsPreview; + const audit = (this._history || []).filter((item) => item.mode === "statistics"); + return `

Transfer historical statistics (limited preview)

Separate from reference replacement. Supports arithmetic measurement sensors with identical units; existing replacement statistics and cumulative counters are blocked.

Keep both sensors registered, disable them in Settings → Devices & services → Entities, and wait until unloaded. The old sensor must remain disabled afterward. No measurements or raw state history are copied.

+ ${pairs.length ? pairs.map((pair, index) => `
${escapeHtml(pair.source)} → ${escapeHtml(pair.target)}
`).join("") : '

Choose a sensor pair above, or confirm the device entity mappings.

'} + ${preview ? `
${escapeHtml(preview.source)} → ${escapeHtml(preview.target)}${preview.eligible ? `

${escapeHtml(preview.device_class)} · ${escapeHtml(preview.unit)}. The destination has no stored statistics metadata.

Latest retained periods: hourly ${escapeHtml(this._statisticsTime(preview.latest.hourly[0]?.start))}; five-minute ${escapeHtml(this._statisticsTime(preview.latest.five_minute[0]?.start))}.

The existing series will be renamed. Configuration and Energy settings stay unchanged. Configuration Restore cannot undo this operation.

` : `

${escapeHtml(preview.reason)}

`}
` : ""} + ${audit.length ? `
Statistics transfer audit${audit.map((item) => `

${escapeHtml(item.job_id)} · ${escapeHtml(item.source)} → ${escapeHtml(item.target)} · ${escapeHtml(item.status)} ${["statistics_applying", "statistics_outcome_unknown"].includes(item.status) ? `` : ""}

`).join("")}

Audit records contain no database backup. An unverified outcome blocks further transfers involving those sensors until the audit/database is reconciled.

` : ""}
`; + } + async _restore() { const jobId = this._query("#backup").value; if (!jobId) return; @@ -347,14 +404,14 @@ class EntityReplacerPanel extends HTMLElement { .reference { display:flex; gap:12px; padding:12px 0; border-top:1px solid var(--divider-color,#e4e8ec); } .reference-body { min-width:0; flex:1; } .manual { display:block; padding:2px 6px; border-radius:5px; font-size:12px; background:rgba(255,170,0,.18); } pre { margin:6px 0; white-space:pre-wrap; overflow-wrap:anywhere; font:13px/1.5 monospace; background:var(--secondary-background-color,#f4f6f8); border-radius:7px; padding:10px; } .diff { border-top:1px solid var(--divider-color,#dce1e7); } .diff pre { max-height:520px; overflow:auto; white-space:pre; } .coverage { margin:20px 0; } li { margin:6px 0; } .empty { padding:20px; } .progress { color:var(--primary-color,#0288d1); margin:12px 0; } .backup-row { display:flex; flex-wrap:wrap; align-items:center; gap:12px; } .backup-row select { flex:1; min-width:230px; } .restore-check { margin-top:16px; } @media(max-width:640px) { main { padding:12px; } .fields,.mapping-row { grid-template-columns:1fr; gap:14px; } .card { padding:15px; } header { padding:12px; } .reference { gap:8px; } button { min-height:44px; } .backup-row select { min-width:0; flex-basis:100%; } } -
Device Replacer${escapeHtml(this._version || "1.1.3")}
+
Device Replacer${escapeHtml(this._version || "1.2.0")}
Replace a device and its entity references

Choose the old and new devices, confirm their entity roles, then review and apply selected changes with backups. Single-entity replacement is also available.

${this._error ? `` : ""}${this._notice ? `
${escapeHtml(this._notice)}
` : ""}
${this._renderChooser()}
Additional scan options

YAML formatting and comments are preserved. UI dashboards are saved through Home Assistant. Device blocks are selectable only when their selectors and replacement capabilities can be validated. Unsupported blocks and other storage get manual instructions.

- ${this._busy ? `

${escapeHtml(this._busyMessage)}

` : ""}
${this._renderResults()}
-

Restore a replacement

Restore checks every file/dashboard first and refuses to overwrite newer edits. Backups survive Home Assistant restarts.

${this._history.length ? `
` : '

No replacements have been applied yet.

'}
+ ${this._busy ? `

${escapeHtml(this._busyMessage)}

` : ""}
${this._renderResults()}
${this._renderStatistics()}
+

Restore a replacement

Restore checks every file/dashboard first and refuses to overwrite newer edits. Backups survive Home Assistant restarts.

${this._history.some((item) => item.mode !== "statistics") ? `
` : '

No replacements have been applied yet.

'}
`; this._bind(); this._updateControls(); @@ -456,6 +513,21 @@ class EntityReplacerPanel extends HTMLElement { restore.disabled = true; }); on("#restore", "click", () => this._restore()); + this._bindStatistics(); + } + + _bindStatistics() { + const on = (selector, event, handler) => this._query(selector)?.addEventListener(event, handler); + for (const button of this.shadowRoot.querySelectorAll("[data-statistics-preview]")) { + button.addEventListener("click", () => this._previewStatistics(Number(button.dataset.statisticsPreview))); + } + on("#statistics-confirm", "change", (event) => { + this._query("#statistics-apply").disabled = this._busy || !event.target.checked; + }); + on("#statistics-apply", "click", () => this._applyStatistics()); + for (const button of this.shadowRoot.querySelectorAll("[data-statistics-verify]")) { + button.addEventListener("click", () => this._verifyStatistics(button.dataset.statisticsVerify)); + } } _updateFileChecks() { diff --git a/custom_components/entity_replacer/manager.py b/custom_components/entity_replacer/manager.py index 17c5a9f..7c99084 100644 --- a/custom_components/entity_replacer/manager.py +++ b/custom_components/entity_replacer/manager.py @@ -22,6 +22,7 @@ from .const import PREVIEW_TTL, VERSION from .device import DeviceBlock from .device_manager import DeviceWorkflow +from .statistics import StatisticsWorkflow from .engine import ( BackupStore, Document, MAX_FILE_BYTES, ReplacementError, build_candidate, digest, json_bytes, read_file, render_report, replace_file, scan_dashboard, @@ -49,7 +50,7 @@ class Preview: device_blocks: list[DeviceBlock] = field(default_factory=list) -class EntityReplacer(DeviceWorkflow): +class EntityReplacer(DeviceWorkflow, StatisticsWorkflow): """All mutations are serialized; external editors are guarded by hashes.""" def __init__(self, hass: HomeAssistant) -> None: @@ -57,6 +58,7 @@ def __init__(self, hass: HomeAssistant) -> None: self.root = Path(hass.config.config_dir) self.backups = BackupStore(self.root) self.previews: dict[str, Preview] = {} + self.statistics_previews: dict[str, dict] = {} self.lock = asyncio.Lock() async def _io(self, function, *args): @@ -212,7 +214,7 @@ async def _public_preview(self, preview_id, preview, warnings, file_count, dashb "Other active storage and additional text/code files are manual-review only when their scan options are enabled. Known binary storage formats are skipped quietly.", "Secrets, entity/device identity registries, history caches, vendor code, backups, symlinks, external add-ons, and files outside the configuration directory are excluded.", "Computed/concatenated IDs, area/label selections, blueprint device inputs, and unsupported device blocks need manual follow-up.", - "Device pairing, unique IDs, entity/device registries, areas/labels, and historical statistics are not transferred.", + "Device pairing, unique IDs, entity/device registries, areas/labels, and historical statistics are not transferred by reference replacement; eligible measurement statistics have a separate transfer action.", ], } result["report"] = await self._io(render_report, result) @@ -386,6 +388,8 @@ async def _rollback(self, manifest: dict, records: list[dict]) -> list[str]: async def restore(self, job_id: str) -> dict: async with self.lock: manifest = await self._io(self.backups.load, job_id) + if manifest.get("mode") == "statistics": + raise ReplacementError("Statistics audit records are not configuration backups. Database recovery requires your database-inclusive backup; this Restore action cannot undo statistics transfers.") candidates = [] for record in manifest["items"]: before = await self._io(self.backups.payload, manifest, record, "before") diff --git a/custom_components/entity_replacer/manifest.json b/custom_components/entity_replacer/manifest.json index 4571518..70556a5 100644 --- a/custom_components/entity_replacer/manifest.json +++ b/custom_components/entity_replacer/manifest.json @@ -1,6 +1,9 @@ { "domain": "entity_replacer", "name": "Device Replacer", + "after_dependencies": [ + "recorder" + ], "codeowners": [ "@FortranFour" ], @@ -19,5 +22,5 @@ "issue_tracker": "https://github.com/FortranFour/device-replacer/issues", "requirements": [], "single_config_entry": true, - "version": "1.1.3" + "version": "1.2.0" } diff --git a/custom_components/entity_replacer/statistics.py b/custom_components/entity_replacer/statistics.py new file mode 100644 index 0000000..d557603 --- /dev/null +++ b/custom_components/entity_replacer/statistics.py @@ -0,0 +1,206 @@ +"""Conservative recorder statistics transfer, separate from reference replacement. + +Recorder imports are lazy: ordinary reference replacement works without recorder. +Only its metadata API writes to the database; no SQL or measurement rows are edited. +""" + +from __future__ import annotations + +import asyncio +from functools import partial +import inspect +import time +import uuid + +from homeassistant.helpers import entity_registry as er + +from .const import PREVIEW_TTL +from .engine import ReplacementError, digest, json_bytes, validate_pair + +TRANSFER_TIMEOUT = 30 + + +class RecorderStatistics: + """Small adapter for HA's recorder APIs (2026.3+), never a raw DB writer.""" + + def __init__(self, hass): + try: + from homeassistant.components.recorder import get_instance + from homeassistant.components.recorder import statistics + + self.instance = get_instance(hass) + self.api = statistics + required = inspect.signature(statistics.async_update_statistics_metadata).parameters + if not {"new_statistic_id", "on_done"} <= required.keys(): + raise ValueError("Unsupported recorder metadata API") + if not self.instance.is_alive(): + raise ValueError("Recorder is not running") + except (ImportError, KeyError, ValueError, AttributeError) as err: + raise ReplacementError("Statistics transfer needs a running, compatible recorder (Home Assistant 2026.3+).") from err + self.hass = hass + + async def metadata(self, source, target): + return await self.instance.async_add_executor_job( + partial(self.api.get_metadata, self.hass, statistic_ids={source, target}) + ) + + async def latest(self, source): + async def read(function): + rows = await self.instance.async_add_executor_job( + function, self.hass, 1, source, False, {"mean", "min", "max"} + ) + return rows.get(source, []) + + hourly, short_term = await asyncio.gather( + read(self.api.get_last_statistics), read(self.api.get_last_short_term_statistics) + ) + return {"hourly": hourly, "five_minute": short_term} + + async def rename(self, source, target): + done = asyncio.Event() + loop = asyncio.get_running_loop() + self.api.async_update_statistics_metadata( + self.hass, source, new_statistic_id=target, + on_done=lambda: loop.call_soon_threadsafe(done.set), + ) + # Timing out does not cancel the queued recorder job. + await asyncio.wait_for(done.wait(), TRANSFER_TIMEOUT) + + +def _identity(hass, entity_id): + entry = er.async_get(hass).async_get(entity_id) + if entry is None: + raise ReplacementError(f"Keep {entity_id} registered and disabled so its measurement role can be verified.") + if entry.disabled_by is None or hass.states.get(entity_id) is not None: + raise ReplacementError(f"Disable {entity_id} in Settings → Devices & services → Entities and wait until it is unloaded. Unavailable alone is insufficient.") + capabilities = getattr(entry, "capabilities", None) or {} + device_class = getattr(entry, "device_class", None) or getattr(entry, "original_device_class", None) + if capabilities.get("state_class") != "measurement" or not device_class: + raise ReplacementError(f"{entity_id} needs a known device class and state_class measurement. Cumulative counters and unknown roles are not supported.") + return { + "registry_id": entry.id, "unique_id": entry.unique_id, + "platform": entry.platform, "device_id": entry.device_id, + "device_class": device_class, "state_class": capabilities["state_class"], + "unit": getattr(entry, "unit_of_measurement", None), + "disabled_by": str(entry.disabled_by), + } + + +class StatisticsWorkflow: + """Session-bound review and single-pair execution under the manager's lock.""" + + def _statistics_adapter(self): + return RecorderStatistics(self.hass) + + async def statistics_verify(self, job_id): + """Reconcile only a provably completed rename; never retry recorder work.""" + async with self.lock: + manifest = await self._io(self.backups.load, job_id) + if manifest.get("mode") != "statistics": + raise ReplacementError("This audit record is not a statistics transfer.") + snapshot = manifest["statistics"] + source, target = manifest["source"], manifest["target"] + after = await self._statistics_adapter().metadata(source, target) + expected = dict(snapshot["metadata"], statistic_id=target) + verified = source not in after and after.get(target) == (snapshot["metadata_id"], expected) + if verified: + manifest["status"] = "statistics_transferred" + await self._io(self.backups.save, manifest) + return {"job_id": job_id, "verified": verified, + "message": "Statistics rename verified; the audit is reconciled. Keep the old sensor disabled." if verified else + "Rename cannot be verified. No database changes or retries were made; the audit remains unresolved. Inspect the database and recorder log before recovery."} + + async def _statistics_check(self, source, target): + validate_pair(source, target) + if not source.startswith("sensor."): + raise ReplacementError("Only sensor statistics can be transferred.") + conflicts = await self._io(self.backups.statistics_conflicts, source, target) + if conflicts: + raise ReplacementError("An earlier statistics transfer has an unresolved outcome. Inspect the recorder/database and audit record before further transfers: " + ", ".join(conflicts)) + old, new = _identity(self.hass, source), _identity(self.hass, target) + if old["device_class"] != new["device_class"] or old["unit"] != new["unit"] or not old["unit"]: + raise ReplacementError("Sensors must have the same known device class and unit; unit conversion is not supported.") + adapter = self._statistics_adapter() + metadata = await adapter.metadata(source, target) + if target in metadata: + raise ReplacementError("The replacement already has statistics metadata. Merging or deleting its history is not supported.") + if source not in metadata: + raise ReplacementError("No stored statistics metadata exists for the old sensor.") + metadata_id, meta = metadata[source] + # ARITHMETIC is 1 in the supported HA releases. Reject circular means. + if (meta.get("source") != "recorder" or meta.get("has_sum") + or meta.get("mean_type") != 1 or meta.get("unit_of_measurement") != old["unit"]): + raise ReplacementError("Only recorder-owned arithmetic measurement statistics with identical stored units are supported.") + snapshot = {"source_identity": old, "target_identity": new, + "metadata_id": metadata_id, "metadata": dict(meta)} + return adapter, snapshot + + async def statistics_preview(self, owner, source, target): + async with self.lock: + try: + adapter, snapshot = await self._statistics_check(source, target) + except ReplacementError as err: + return {"source": source, "target": target, "eligible": False, "reason": str(err)} + latest = await adapter.latest(source) + if not any(latest.values()): + return {"source": source, "target": target, "eligible": False, "reason": "The source has metadata but no retained hourly or five-minute statistics."} + now = time.monotonic() + self.statistics_previews = {key: item for key, item in self.statistics_previews.items() + if now - item["created"] < PREVIEW_TTL} + while len(self.statistics_previews) >= 3: + self.statistics_previews.pop(next(iter(self.statistics_previews))) + token = uuid.uuid4().hex + self.statistics_previews[token] = { + "owner": owner, "source": source, "target": target, "created": now, + "snapshot": snapshot, "fingerprint": digest(json_bytes(snapshot)), + } + return {"preview_id": token, "source": source, "target": target, + "eligible": True, "unit": snapshot["metadata"]["unit_of_measurement"], + "device_class": snapshot["source_identity"]["device_class"], + "latest": latest, "expires_in_seconds": PREVIEW_TTL} + + async def statistics_apply(self, owner, preview_id, confirmed=False): + async with self.lock: + preview = self.statistics_previews.get(preview_id) + if (not preview or preview["owner"] != owner + or time.monotonic() - preview["created"] >= PREVIEW_TTL): + raise ReplacementError("Statistics preview expired or belongs to another session. Preview again.") + if confirmed is not True: + raise ReplacementError("Confirm the reviewed pair, measurement role and database-inclusive backup before transfer.") + source, target = preview["source"], preview["target"] + adapter, snapshot = await self._statistics_check(source, target) + if digest(json_bytes(snapshot)) != preview["fingerprint"]: + raise ReplacementError("Sensor identity or statistics metadata changed. Preview again.") + # Persist intent before queuing recorder work. Never mix this audit + # record into the file restore transaction or promise a DB rollback. + manifest = await self._io(self.backups.create, source, target, []) + manifest.update(mode="statistics", status="statistics_applying", statistics=snapshot) + await self._io(self.backups.save, manifest) + self.statistics_previews.pop(preview_id, None) + try: + # No await between this final runtime check and entering rename. + if (_identity(self.hass, source) != snapshot["source_identity"] + or _identity(self.hass, target) != snapshot["target_identity"]): + raise ReplacementError("Sensor identities changed before queuing the transfer.") + except Exception: + manifest["status"] = "statistics_not_queued" + await self._io(self.backups.save, manifest) + raise + try: + await adapter.rename(source, target) + after = await adapter.metadata(source, target) + expected = dict(snapshot["metadata"], statistic_id=target) + if source in after or after.get(target) != (snapshot["metadata_id"], expected): + raise ReplacementError("Recorder did not verify the requested rename; no success is assumed.") + manifest["status"] = "statistics_transferred" + await self._io(self.backups.save, manifest) + except (Exception, asyncio.CancelledError) as err: + manifest["status"] = "statistics_outcome_unknown" + manifest["errors"] = [str(err) or type(err).__name__] + await self._io(self.backups.save, manifest) + if isinstance(err, asyncio.CancelledError): + raise + raise ReplacementError(f"Statistics outcome is unverified. Recorder work may still complete; do not retry blindly. Audit {manifest['job_id']}. Configuration was not changed.") from err + return {"job_id": manifest["job_id"], "source": source, "target": target, + "status": manifest["status"], + "message": "Statistics metadata rename verified. Enable only the replacement sensor. Raw state history and configuration were unchanged. Configuration restore cannot undo this transfer."} diff --git a/custom_components/entity_replacer/websocket.py b/custom_components/entity_replacer/websocket.py index 32b0119..a40fcdb 100644 --- a/custom_components/entity_replacer/websocket.py +++ b/custom_components/entity_replacer/websocket.py @@ -39,6 +39,12 @@ async def _respond(hass, connection, msg, action: str) -> None: result = await manager.apply(connection.user.id, msg["preview_id"], msg["selected_ids"]) elif action == "review": result = await manager.review(connection.user.id, msg["preview_id"], msg["selected_ids"]) + elif action == "statistics_preview": + result = await manager.statistics_preview(connection.user.id, msg["source"].strip(), msg["target"].strip()) + elif action == "statistics_apply": + result = await manager.statistics_apply(connection.user.id, msg["preview_id"], msg["confirmed"]) + elif action == "statistics_verify": + result = await manager.statistics_verify(msg["job_id"]) else: result = await manager.restore(msg["job_id"]) connection.send_result(msg["id"], result) @@ -129,6 +135,39 @@ async def ws_restore(hass, connection, msg): await _respond(hass, connection, msg, "restore") +@websocket_api.websocket_command({ + vol.Required("type"): DOMAIN + "/statistics_preview", + vol.Required("source"): str, + vol.Required("target"): str, +}) +@websocket_api.require_admin +@websocket_api.async_response +async def ws_statistics_preview(hass, connection, msg): + await _respond(hass, connection, msg, "statistics_preview") + + +@websocket_api.websocket_command({ + vol.Required("type"): DOMAIN + "/statistics_apply", + vol.Required("preview_id"): str, + vol.Required("confirmed"): bool, +}) +@websocket_api.require_admin +@websocket_api.async_response +async def ws_statistics_apply(hass, connection, msg): + await _respond(hass, connection, msg, "statistics_apply") + + +@websocket_api.websocket_command({ + vol.Required("type"): DOMAIN + "/statistics_verify", + vol.Required("job_id"): str, +}) +@websocket_api.require_admin +@websocket_api.async_response +async def ws_statistics_verify(hass, connection, msg): + await _respond(hass, connection, msg, "statistics_verify") + + def register_commands(hass) -> None: - for handler in (ws_status, ws_scan, ws_device_plan, ws_scan_device, ws_review, ws_apply, ws_restore): + for handler in (ws_status, ws_scan, ws_device_plan, ws_scan_device, ws_review, ws_apply, ws_restore, + ws_statistics_preview, ws_statistics_apply, ws_statistics_verify): websocket_api.async_register_command(hass, handler) diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index ad9135e..5e6f444 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -188,3 +188,10 @@ Concept, requirements, design decisions, field testing and maintenance: FortranF - [HACS integration requirements](https://www.hacs.dev/docs/publish/integration/) - [HACS custom repositories](https://www.hacs.dev/docs/faq/custom_repositories/) - [Home Assistant local brand images](https://developers.home-assistant.io/docs/core/integration/brand_images/) + + +## Publish a prerelease + +The separate `Prerelease` workflow publishes with `--prerelease --latest=false` and includes the installation ZIP and checksums. It runs only on `main` when `.github/prerelease.json` changes, or when manually dispatched on `main`. Set its `tag` to match the integration manifest version, and merge the reviewed change. The workflow reruns unit/frontend tests, the pinned real recorder tests, hassfest and HACS checks before publishing. It creates the tag at the checked merge commit; the token-created tag does not recursively trigger the stable tag workflow. + +Do not push that tag separately through the existing stable `Release` workflow. A prerelease does not replace the latest stable release. Promoting it to stable is a separate deliberate action after installation testing. diff --git a/ha_tests/test_statistics_recorder.py b/ha_tests/test_statistics_recorder.py new file mode 100644 index 0000000..f5f5dbc --- /dev/null +++ b/ha_tests/test_statistics_recorder.py @@ -0,0 +1,174 @@ +"""Real HA 2026.3 recorder tests; run separately from stub-based unit tests. + + /path/to/ha-venv/bin/python -m unittest discover -s ha_tests -v +""" + +import asyncio +from datetime import datetime, timedelta, timezone +import importlib +from pathlib import Path +import sys +import tempfile +import types +import unittest +from unittest.mock import patch + +from homeassistant.core import HomeAssistant +from homeassistant import loader, config_entries +from homeassistant.setup import async_setup_component +from homeassistant.helpers import entity_registry as er +from homeassistant.helpers.recorder import async_initialize_recorder +from homeassistant.components.recorder import get_instance +from homeassistant.components.recorder.models import StatisticMeanType +from homeassistant.components.recorder.statistics import ( + async_import_statistics, get_last_statistics, get_last_short_term_statistics, +) +from homeassistant.components.recorder.db_schema import Statistics, StatisticsShortTerm, States +from homeassistant.components.recorder.util import session_scope +from homeassistant.components.recorder.tasks import AddRecorderPlatformTask, StatisticsTask, SynchronizeTask +from homeassistant.components.sensor import recorder as sensor_recorder +from sqlalchemy import select + +ROOT = Path(__file__).resolve().parents[1] +package = types.ModuleType("er_live") +package.__path__ = [str(ROOT / "custom_components/entity_replacer")] +sys.modules["er_live"] = package +manager_module = importlib.import_module("er_live.manager") +statistics_module = importlib.import_module("er_live.statistics") + +OLD, NEW = "sensor.old", "sensor.new" + + +class RealRecorderTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.hass = HomeAssistant(self.tmp.name) + self.addAsyncCleanup(self.hass.async_stop) + self.hass.config.time_zone = "UTC" + loader.async_setup(self.hass) + self.hass.config_entries = config_entries.ConfigEntries(self.hass, {}) + await self.hass.config_entries.async_initialize() + async_initialize_recorder(self.hass) + self.assertTrue(await async_setup_component(self.hass, "recorder", { + "recorder": {"db_url": "sqlite:///" + self.tmp.name + "/recorder.db", "purge_keep_days": 1000, "auto_purge": False}, + })) + await self.hass.async_start() + self.recorder = get_instance(self.hass) + await self.recording_done() + registry = er.async_get(self.hass) + await registry.async_load(load_empty=True) + for name in ("old", "new"): + registry.async_get_or_create( + "sensor", "test", name, suggested_object_id=name, + disabled_by=er.RegistryEntryDisabler.USER, + original_device_class="temperature", unit_of_measurement="°C", + capabilities={"state_class": "measurement"}, + ) + self.manager = manager_module.EntityReplacer(self.hass) + self.start = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) - timedelta(hours=2) + self.meta = {"statistic_id": OLD, "source": "recorder", "name": None, + "mean_type": StatisticMeanType.ARITHMETIC, "has_sum": False, + "unit_class": "temperature", "unit_of_measurement": "°C"} + for table in (Statistics, StatisticsShortTerm): + self.recorder.async_import_statistics(self.meta, [ + {"start": self.start, "mean": 20.5, "min": 19.0, "max": 22.0}, + ], table) + # A raw state for a different historical entity must remain untouched. + self.hass.states.async_set("sensor.raw_history", "123") + await self.hass.async_block_till_done() + await self.recording_done() + + async def asyncTearDown(self): + await self.hass.async_stop() + self.tmp.cleanup() + + async def recording_done(self): + # HA's async_block_till_done can return while a task has already been + # taken off its queue but is still committing a separate DB session. + # Always enqueue a barrier so snapshots cannot race an import/compile. + await self.hass.async_block_till_done() + done = asyncio.get_running_loop().create_future() + self.recorder.queue_task(SynchronizeTask(done)) + await asyncio.wait_for(done, 10) + + async def rows(self): + def read(): + with session_scope(hass=self.hass, read_only=True) as session: + return { + "hourly": list(session.execute(select(Statistics.id, Statistics.metadata_id, Statistics.start_ts, Statistics.mean, Statistics.min, Statistics.max)).tuples()), + "five_minute": list(session.execute(select(StatisticsShortTerm.id, StatisticsShortTerm.metadata_id, StatisticsShortTerm.start_ts, StatisticsShortTerm.mean, StatisticsShortTerm.min, StatisticsShortTerm.max)).tuples()), + "states": list(session.execute(select(States.state_id, States.metadata_id, States.state)).tuples()), + } + return await self.recorder.async_add_executor_job(read) + + async def test_transfer_keeps_both_statistics_tables_and_raw_history(self): + before = await self.rows() + self.assertEqual(len(before["hourly"]), 1) + self.assertEqual(len(before["five_minute"]), 1) + preview = await self.manager.statistics_preview("owner", OLD, NEW) + self.assertTrue(preview["eligible"], preview) + result = await self.manager.statistics_apply("owner", preview["preview_id"], True) + self.assertEqual(result["status"], "statistics_transferred") + self.assertEqual(await self.rows(), before) + adapter = self.manager._statistics_adapter() + self.assertNotIn(OLD, await adapter.metadata(OLD, NEW)) + for function in (get_last_statistics, get_last_short_term_statistics): + old = await self.recorder.async_add_executor_job(function, self.hass, 1, OLD, False, {"mean"}) + new = await self.recorder.async_add_executor_job(function, self.hass, 1, NEW, False, {"mean"}) + self.assertFalse(old.get(OLD)) + self.assertEqual(new[NEW][0]["mean"], 20.5) + # Ensure queued statistics work after the rename does not resurrect old data. + await self.recording_done() + self.assertEqual(await self.rows(), before) + + async def test_destination_collision_after_preview_does_not_change_any_rows(self): + preview = await self.manager.statistics_preview("owner", OLD, NEW) + self.assertTrue(preview["eligible"], preview) + async_import_statistics(self.hass, dict(self.meta, statistic_id=NEW), [ + {"start": self.start, "mean": 99.0, "min": 99.0, "max": 99.0}, + ]) + await self.hass.async_block_till_done() + await self.recording_done() + before = await self.rows() + with self.assertRaisesRegex(manager_module.ReplacementError, "already has"): + await self.manager.statistics_apply("owner", preview["preview_id"], True) + self.assertEqual(await self.rows(), before) + + async def test_replacement_collects_new_statistics_under_the_same_metadata_id(self): + preview = await self.manager.statistics_preview("owner", OLD, NEW) + await self.manager.statistics_apply("owner", preview["preview_id"], True) + old_rows = await self.rows() + metadata_id = old_rows["five_minute"][0][1] + er.async_get(self.hass).async_update_entity(NEW, disabled_by=None) + self.hass.states.async_set(NEW, "25", { + "device_class": "temperature", "state_class": "measurement", "unit_of_measurement": "°C", + }) + await self.hass.async_block_till_done() + await self.recording_done() + now = datetime.now(timezone.utc) + start = now.replace(minute=now.minute - now.minute % 5, second=0, microsecond=0) + self.recorder.queue_task(AddRecorderPlatformTask("sensor", sensor_recorder)) + self.recorder.queue_task(StatisticsTask(start, False)) + await self.recording_done() + after = await self.rows() + self.assertEqual(len(after["five_minute"]), 2) + self.assertTrue(all(row[1] == metadata_id for row in after["five_minute"])) + self.assertAlmostEqual(after["five_minute"][-1][3], 25) + self.assertNotIn(OLD, await self.manager._statistics_adapter().metadata(OLD, NEW)) + + async def test_timeout_then_late_completion_is_verified_without_retry(self): + preview = await self.manager.statistics_preview("owner", OLD, NEW) + before = await self.rows() + with patch.object(statistics_module, "TRANSFER_TIMEOUT", 0): + with self.assertRaisesRegex(manager_module.ReplacementError, "unverified"): + await self.manager.statistics_apply("owner", preview["preview_id"], True) + await self.recording_done() + job_id = self.manager.backups.statistics_conflicts(OLD, NEW)[0] + result = await self.manager.statistics_verify(job_id) + self.assertTrue(result["verified"]) + self.assertEqual(self.manager.backups.statistics_conflicts(OLD, NEW), []) + self.assertEqual(await self.rows(), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/build_release.py b/scripts/build_release.py index 87d9bf2..25cbbc4 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -56,7 +56,7 @@ def check_repository(tag: str | None) -> tuple[str, list[Path]]: if hacs.get("homeassistant") != "2026.3.0" or hacs.get("content_in_root", False) or hacs.get("zip_release", False): raise ValueError("This package uses the standard repository layout and local brands from HA 2026.3.") for name in ("__init__.py", "config_flow.py", "compat.py", "const.py", "engine.py", "manager.py", - "device.py", "device_manager.py", "websocket.py", "strings.json", "translations/en.json", "frontend/panel.js", "frontend/icon.svg"): + "device.py", "device_manager.py", "statistics.py", "websocket.py", "strings.json", "translations/en.json", "frontend/panel.js", "frontend/icon.svg"): if not (COMPONENT / name).is_file(): raise ValueError(f"Missing runtime file: {name}") for path in COMPONENT.rglob("*.json"): diff --git a/tests/frontend_smoke.cjs b/tests/frontend_smoke.cjs index 5399ac4..88face2 100644 --- a/tests/frontend_smoke.cjs +++ b/tests/frontend_smoke.cjs @@ -5,7 +5,7 @@ const assert = require("node:assert/strict"); const { chromium } = require(process.env.PLAYWRIGHT_MODULE || "playwright"); (async () => { - const browser = await chromium.launch({ headless: true, args: ["--no-sandbox"] }); + const browser = await chromium.launch({ headless: true, executablePath: process.env.CHROMIUM_EXECUTABLE, args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"] }); try { for (const viewport of [{ width: 1360, height: 1000 }, { width: 390, height: 844 }]) { for (const mode of ["device", "entity"]) { @@ -73,6 +73,14 @@ const { chromium } = require(process.env.PLAYWRIGHT_MODULE || "playwright"); case "entity_replacer/apply": status.history = [{ job_id: "20261002T170000Z-abcdef123456", created: "2026-10-02T17:00:00+00:00", source: "sensor.old", target: "sensor.new", status: "applied" }]; return { references_changed: message.selected_ids.length, documents_changed: message.selected_ids.length, message: "Changes saved. Check configuration, then restart Home Assistant.", backup_path: "/config/.entity_replacer_backups/demo" }; + case "entity_replacer/statistics_preview": return { + eligible: true, preview_id: "statistics-demo", source: message.source, target: message.target, + device_class: "temperature", unit: "°C", latest: { hourly: [{ start: 1700000000 }], five_minute: [] }, + }; + case "entity_replacer/statistics_apply": + if (!message.confirmed || message.preview_id !== "statistics-demo") throw new Error("Unreviewed statistics write"); + status.history.push({ mode: "statistics", job_id: "20261024T170000Z-abcdef123456", source: "sensor.old", target: "sensor.new", status: "statistics_transferred" }); + return { message: "Statistics rename verified; configuration restore cannot undo it.", job_id: "statistics-demo" }; case "entity_replacer/restore": window.restored = true; return { documents_restored: 2, message: "Backup restored." }; default: throw new Error("Unexpected request"); } @@ -109,6 +117,7 @@ const { chromium } = require(process.env.PLAYWRIGHT_MODULE || "playwright"); await panel.locator("#review").click(); await panel.locator("#apply").waitFor(); assert.equal(await panel.locator("#apply").isEnabled(), true); + await panel.locator(".file summary").first().click(); await panel.locator("[data-ref]").first().check(); assert.equal(await panel.locator("#apply").count(), 0); await panel.locator("#review").click(); @@ -133,6 +142,15 @@ const { chromium } = require(process.env.PLAYWRIGHT_MODULE || "playwright"); await panel.locator("#restore-confirm").check(); await panel.locator("#restore").click(); await page.waitForFunction(() => window.restored); + await panel.locator("[data-statistics-preview]").click(); + await panel.locator("#statistics-confirm").waitFor(); + assert.equal(await panel.locator("#statistics-apply").isDisabled(), true); + await panel.locator("#statistics-confirm").check(); + assert.equal(await panel.locator("#statistics-apply").isDisabled(), false); + await panel.locator("#statistics-apply").click(); + await page.waitForFunction(() => window.panel.shadowRoot.textContent.includes("Audit: statistics-demo")); + assert.equal(await panel.locator("#statistics-apply").count(), 0); + assert.equal(await panel.locator('#backup option[value="20261024T170000Z-abcdef123456"]').count(), 0); assert.deepEqual(errors, []); await page.close(); console.log(`Frontend ${mode} workflow and overflow checks passed at ${viewport.width}px.`); diff --git a/tests/frontend_unit.cjs b/tests/frontend_unit.cjs index b11a02a..a3612e3 100644 --- a/tests/frontend_unit.cjs +++ b/tests/frontend_unit.cjs @@ -46,9 +46,10 @@ function panel() { }; value._review = null; value._render = () => {}; + value.shadowRoot = { querySelectorAll: () => [] }; const nodes = new Map(); value._query = (selector) => { - if (!nodes.has(selector)) nodes.set(selector, { disabled: false, textContent: "", innerHTML: "", scrollIntoView() {} }); + if (!nodes.has(selector)) nodes.set(selector, { disabled: false, textContent: "", innerHTML: "", scrollIntoView() {}, addEventListener() {} }); return nodes.get(selector); }; return value; @@ -177,4 +178,34 @@ function panel() { value._updateControls(); assert.equal(value._query("#load-mappings").disabled, true); console.log("A removed selection becomes explicit, and a missing replacement cannot load mappings."); + value = panel(); + assert.equal(value._statisticsPairs().length, 1); + value._mode = "device"; + value._mappings = [{ source: "sensor.old", target: "sensor.new" }]; + assert.equal(value._statisticsPairs().length, 0); + value._mappingConfirmed = true; + assert.equal(value._statisticsPairs().length, 1); + value._statisticsPreview = { source: "", target: "sensor.new", eligible: false, reason: "" }; + html = value._renderStatistics(); + assert.ok(html.includes("<blocked>")); + assert.ok(!html.includes('id="statistics-apply"')); + value._statisticsPreview = { source: "sensor.old", target: "sensor.new", eligible: true, preview_id: "stats", + unit: "°C", device_class: "temperature", latest: { hourly: [], five_minute: [] } }; + html = value._renderStatistics(); + assert.ok(html.includes('id="statistics-apply" class="primary" disabled')); + assert.ok(html.includes("database-inclusive backup")); + let statsCalls = 0; + value._request = async () => { statsCalls++; throw new Error("Recorder timed out"); }; + value._refreshHistory = async () => {}; + await value._applyStatistics(); + assert.equal(statsCalls, 0, "A statistics write needs explicit confirmation"); + value._query("#statistics-confirm").checked = true; + await value._applyStatistics(); + assert.equal(statsCalls, 1); + assert.equal(value._statisticsPreview, null, "An uncertain statistics write cannot be retried with its old token"); + assert.ok(value._error.includes("timed out")); + value._statisticsPreview = { eligible: true }; + value._invalidate(); + assert.equal(value._statisticsPreview, null, "Changing the pair invalidates statistics review"); + console.log("Statistics role selection, escaping, consent and uncertain-write gating passed."); })().catch((error) => { console.error(error); process.exitCode = 1; }); diff --git a/tests/test_statistics.py b/tests/test_statistics.py new file mode 100644 index 0000000..93daba5 --- /dev/null +++ b/tests/test_statistics.py @@ -0,0 +1,197 @@ +"""Statistics control-flow tests. Real recorder coverage lives in ha_tests/.""" + +import asyncio +import copy +import time +import types +import unittest +from unittest.mock import patch + +import test_manager as fixtures +from test_manager import OLD, NEW, manager_module, engine + + +class MemoryRecorder: + def __init__(self): + self.meta = {OLD: (7, {"statistic_id": OLD, "source": "recorder", "has_sum": False, + "mean_type": 1, "unit_of_measurement": "°C", "unit_class": "temperature", "name": None})} + self.rows = {"hourly": [{"start": 1000.0, "mean": 20.0}], "five_minute": []} + self.calls = 0 + self.outcome = "success" + + async def metadata(self, source, target): + return copy.deepcopy(self.meta) + + async def latest(self, source): + return copy.deepcopy(self.rows) + + async def rename(self, source, target): + self.calls += 1 + if self.outcome == "timeout": + raise TimeoutError() + if self.outcome == "cancel": + raise asyncio.CancelledError() + if self.outcome == "silent_refusal": + return + id_, meta = self.meta.pop(source) + meta["statistic_id"] = target + self.meta[target] = (id_, meta) + + +# Reuse fixtures, not the unrelated inherited test methods. +class StatisticsTests(unittest.IsolatedAsyncioTestCase): + asyncTearDown = fixtures.ManagerTests.asyncTearDown + + async def asyncSetUp(self): + await fixtures.ManagerTests.asyncSetUp(self) + for index, name in enumerate((OLD, NEW)): + self.hass.states.values.pop(name) + entry = self.hass.registry.entities[name] + entry.id = str(index) * 32 + entry.unique_id = f"unique-{index}" + entry.platform = "test" + entry.disabled_by = "user" + entry.original_device_class = "temperature" + entry.capabilities = {"state_class": "measurement"} + entry.unit_of_measurement = "°C" + self.recorder = MemoryRecorder() + self.manager._statistics_adapter = lambda: self.recorder + + async def preview(self): + return await self.manager.statistics_preview("owner", OLD, NEW) + + async def transfer(self, preview=None): + preview = preview or await self.preview() + return await self.manager.statistics_apply("owner", preview["preview_id"], True) + + async def test_verified_transfer_keeps_metadata_id_and_configuration_unchanged(self): + before = self.yaml.read_bytes() + preview = await self.preview() + self.assertTrue(preview["eligible"]) + self.assertEqual(self.recorder.calls, 0) + result = await self.transfer(preview) + self.assertEqual(result["status"], "statistics_transferred") + self.assertEqual(self.recorder.meta[NEW][0], 7) + self.assertEqual(self.yaml.read_bytes(), before) + self.assertEqual(self.hass.dashboard.calls, []) + with self.assertRaisesRegex(engine.ReplacementError, "not configuration backups"): + await self.manager.restore(result["job_id"]) + with self.assertRaises(engine.ReplacementError): + await self.transfer(preview) + + async def test_existing_destination_including_empty_metadata_is_blocked(self): + self.recorder.meta[NEW] = (8, dict(self.recorder.meta[OLD][1], statistic_id=NEW)) + self.assertIn("already has", (await self.preview())["reason"]) + self.assertEqual(self.recorder.calls, 0) + + async def test_source_missing_external_sum_circular_units_and_no_rows_are_blocked(self): + original = copy.deepcopy(self.recorder.meta) + for changes in ({"source": "external"}, {"has_sum": True}, {"mean_type": 2}, {"unit_of_measurement": "°F"}): + self.recorder.meta = copy.deepcopy(original) + self.recorder.meta[OLD][1].update(changes) + self.assertFalse((await self.preview())["eligible"]) + self.recorder.meta = {} + self.assertIn("No stored", (await self.preview())["reason"]) + self.recorder.meta = original + self.recorder.rows = {"hourly": [], "five_minute": []} + self.assertFalse((await self.preview())["eligible"]) + + async def test_disabled_unloaded_known_compatible_measurements_required(self): + for name in (OLD, NEW): + entry = self.hass.registry.entities[name] + entry.disabled_by = None + self.assertIn("Disable", (await self.preview())["reason"]) + entry.disabled_by = "user" + self.hass.states.values[name] = types.SimpleNamespace(state="unavailable") + self.assertFalse((await self.preview())["eligible"]) + self.hass.states.values.pop(name) + for attribute, bad in (("unit_of_measurement", "°F"), ("original_device_class", "humidity"), ("capabilities", {"state_class": "total_increasing"})): + entry = self.hass.registry.entities[NEW] + original = getattr(entry, attribute) + setattr(entry, attribute, bad) + self.assertFalse((await self.preview())["eligible"]) + setattr(entry, attribute, original) + self.hass.registry.entities.pop(OLD) + self.assertFalse((await self.preview())["eligible"]) + + async def test_owner_expiry_and_explicit_confirmation(self): + preview = await self.preview() + token = preview["preview_id"] + with self.assertRaises(engine.ReplacementError): + await self.manager.statistics_apply("other", token, True) + with self.assertRaises(engine.ReplacementError): + await self.manager.statistics_apply("owner", token, False) + self.manager.statistics_previews[token]["created"] = time.monotonic() - 10000 + with self.assertRaises(engine.ReplacementError): + await self.transfer(preview) + self.assertEqual(self.recorder.calls, 0) + + async def test_changed_registry_or_metadata_and_destination_race_refuse_before_write(self): + preview = await self.preview() + self.hass.registry.entities[NEW].unique_id = "changed" + with self.assertRaisesRegex(engine.ReplacementError, "changed"): + await self.transfer(preview) + preview = await self.preview() + self.recorder.meta[OLD][1]["name"] = "changed" + with self.assertRaisesRegex(engine.ReplacementError, "changed"): + await self.transfer(preview) + preview = await self.preview() + self.recorder.meta[NEW] = (8, {}) + with self.assertRaisesRegex(engine.ReplacementError, "already has"): + await self.transfer(preview) + self.assertEqual(self.recorder.calls, 0) + + async def test_silent_refusal_timeout_and_cancel_never_report_success_or_allow_retry(self): + for outcome in ("silent_refusal", "timeout", "cancel"): + with self.subTest(outcome=outcome): + self.recorder.outcome = outcome + preview = await self.preview() + expected = asyncio.CancelledError if outcome == "cancel" else engine.ReplacementError + with self.assertRaises(expected): + await self.transfer(preview) + manifest = self.manager.backups.load(self.manager.backups.statistics_conflicts(OLD, NEW)[0]) + self.assertEqual(manifest["status"], "statistics_outcome_unknown") + restarted = manager_module.EntityReplacer(self.hass) + restarted._statistics_adapter = lambda: self.recorder + blocked = await restarted.statistics_preview("owner", OLD, NEW) + self.assertIn("unresolved", blocked["reason"]) + # Only isolate the next subtest; production never clears this automatically. + manifest["status"] = "statistics_not_queued" + self.manager.backups.save(manifest) + + async def test_pending_audit_and_corrupt_audit_fail_closed(self): + manifest = self.manager.backups.create(OLD, NEW, []) + manifest.update(mode="statistics", status="statistics_applying") + self.manager.backups.save(manifest) + self.assertIn("unresolved", (await self.preview())["reason"]) + path = self.manager.backups._job(manifest["job_id"]) / "manifest.json" + path.write_text("broken") + self.assertIn("Cannot verify audit", (await self.preview())["reason"]) + + async def test_no_manifest_write_means_no_recorder_write(self): + preview = await self.preview() + with patch.object(self.manager.backups, "save", side_effect=OSError("disk full")): + with self.assertRaises(OSError): + await self.transfer(preview) + self.assertEqual(self.recorder.calls, 0) + + async def test_only_sensors_and_distinct_pairs(self): + for old, new in ((OLD, OLD), ("switch.old", "switch.new"), (OLD, "switch.new")): + self.assertFalse((await self.manager.statistics_preview("owner", old, new))["eligible"]) + + async def test_late_completion_can_be_verified_without_retrying(self): + self.recorder.outcome = "timeout" + with self.assertRaises(engine.ReplacementError): + await self.transfer() + job_id = self.manager.backups.statistics_conflicts(OLD, NEW)[0] + unresolved = await self.manager.statistics_verify(job_id) + self.assertFalse(unresolved["verified"]) + self.assertEqual(self.recorder.calls, 1) + # Simulate the already queued recorder job completing after timeout. + self.recorder.outcome = "success" + await self.recorder.rename(OLD, NEW) + calls = self.recorder.calls + result = await self.manager.statistics_verify(job_id) + self.assertTrue(result["verified"]) + self.assertEqual(self.recorder.calls, calls) + self.assertEqual(self.manager.backups.statistics_conflicts(OLD, NEW), [])