From 36da2fa5155f1079c5744dd048d868a0143bafd3 Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Wed, 18 Nov 2026 20:01:46 -0600 Subject: [PATCH] fix(parse): facts under undeclared contexts are counted, and kept when the filing states the period A fact whose contextRef names an id the instance never declares was skipped with no warning and no count. Some Indian exchange results filings put nearly every fact under two such ids, so the model held 4 facts of 99. Every fact the walk leaves out is now counted on model.parse_report by reason and context id, logged, and shown in the load receipt and describe_filing. An undeclared context is rebuilt when the filing reports exactly one start and one end date under that id; its facts carry provenance kind "recovered_context", which the holon and TAVI projections already keep. Refs #111 --- tests/test_parse_contexts.py | 303 +++++++++++++++++++++++++++++++++++ xbrlkit/model.py | 61 +++++++ xbrlkit/parse/README.md | 32 ++++ xbrlkit/parse/to_model.py | 236 ++++++++++++++++++++++++++- xbrlkit/serve/README.md | 5 + xbrlkit/serve/tools.py | 59 ++++++- 6 files changed, 688 insertions(+), 8 deletions(-) create mode 100644 tests/test_parse_contexts.py diff --git a/tests/test_parse_contexts.py b/tests/test_parse_contexts.py new file mode 100644 index 0000000..41ae0fd --- /dev/null +++ b/tests/test_parse_contexts.py @@ -0,0 +1,303 @@ +"""Facts the parse cannot place: counted, never dropped silently, and kept when +the filing states the period of an undeclared context itself. + +The instances are synthetic and load offline: a three-concept schema and an +instance whose facts name context ids it never declares, the pattern some +Indian exchange results filings follow (``OneD`` for the quarter, ``FourD`` for +the year to date). +""" + +from __future__ import annotations + +import logging +from datetime import date +from pathlib import Path + +import pytest + +from xbrlkit.model import RECOVERED_CONTEXT, FilingMeta, XbrlModel +from xbrlkit.parse import close, load_model, to_xbrl_model + +SCHEMA = """ + + + + + + + +""" + +HEAD = """ + + +iso4217:INR +""" + + +def _context(context_id: str, entity: str, start: str, end: str) -> str: + return ( + f'' + f'{entity}' + f"{start}" + f"{end}\n" + ) + + +def _dates(context_id: str, start: str, end: str) -> str: + return ( + f'{start}' + "\n" + f'{end}' + "\n" + ) + + +def _fact(name: str, context_id: str, value: int) -> str: + return ( + f'{value}' + f"\n" + ) + + +# The quarter and the year to date, neither declared; one declared context +# that dates a year-to-date breakdown as the quarter, as the real filings do. +RESULTS = ( + _context("FourDetailD", "ACME", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-10-01", "2022-12-31") + + _dates("FourD", "2022-04-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Revenue", "FourD", 300) + + _fact("Revenue", "FourDetailD", 300) +) + + +def _parse(tmp_path: Path, body: str) -> XbrlModel: + (tmp_path / "t.xsd").write_text(SCHEMA) + instance = tmp_path / "i.xml" + instance.write_text(HEAD + body + "\n") + mx = load_model(str(instance), cache_dir=tmp_path / "cache", offline=True) + try: + return to_xbrl_model(mx, FilingMeta(accession="acc", cik="0000000042")) + finally: + close(mx.modelManager.cntlr) + + +def _by_context_value(model: XbrlModel) -> dict[str, tuple[date | None, date | None]]: + periods = {p.id: p for p in model.periods} + return { + f.value_str or "": (periods[f.period_id].start, periods[f.period_id].end) + for f in model.facts + if f.concept_qname == "t:Revenue" and f.provenance is not None + } + + +def test_undeclared_contexts_are_rebuilt_from_the_filings_own_dates(tmp_path: Path): + model = _parse(tmp_path, RESULTS) + + assert len(model.facts) == 7 + assert _by_context_value(model) == { + "100": (date(2022, 10, 1), date(2022, 12, 31)), + "300": (date(2022, 4, 1), date(2022, 12, 31)), + } + report = model.parse_report + assert report is not None and report.dropped == [] + assert [ + (r.context_id, r.start, r.end, r.facts) for r in report.recovered_contexts + ] == [ + ("OneD", date(2022, 10, 1), date(2022, 12, 31), 3), + ("FourD", date(2022, 4, 1), date(2022, 12, 31), 3), + ] + assert report.recovered_contexts[0].entity_from == "declared_contexts" + assert report.recovered_contexts[0].period_from == [ + "t:DateOfStartOfReportingPeriod", + "t:DateOfEndOfReportingPeriod", + ] + + +def test_recovered_facts_are_marked_and_declared_ones_are_not(tmp_path: Path): + model = _parse(tmp_path, RESULTS) + + marked = [f for f in model.facts if f.provenance is not None] + declared = [f for f in model.facts if f.provenance is None] + assert len(marked) == 6 and len(declared) == 1 + assert {f.provenance.kind for f in marked if f.provenance} == {RECOVERED_CONTEXT} + assert all(f.entity_identifier == "ACME" and f.dims == [] for f in marked) + # The declared context keeps the dates the filing gave it, right or wrong. + period = next(p for p in model.periods if p.id == declared[0].period_id) + assert (period.start, period.end) == (date(2022, 10, 1), date(2022, 12, 31)) + + +def test_recovered_facts_of_one_concept_get_distinct_ids(tmp_path: Path): + # Arelle hashes a fact with no context by its qname alone, so every orphan + # of one concept shares a hash — across contexts and within one. + model = _parse(tmp_path, RESULTS + _fact("Revenue", "OneD", 101)) + + ids = [f.id for f in model.facts] + assert len(set(ids)) == len(ids) + hashes = [f.source_hash for f in model.facts if f.provenance is not None] + assert len(set(hashes)) == len(hashes) + + +def test_a_context_with_no_stated_period_is_dropped_and_counted( + tmp_path: Path, caplog: pytest.LogCaptureFixture +): + body = ( + _context("Declared", "ACME", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-10-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Revenue", "FourD", 300) + + _fact("Revenue", "FourD", 301) + + _fact("Revenue", "Declared", 5) + ) + with caplog.at_level(logging.WARNING, logger="xbrlkit.parse.to_model"): + model = _parse(tmp_path, body) + + report = model.parse_report + assert report is not None + assert [(d.reason, d.context_id, d.facts, d.concepts) for d in report.dropped] == [ + ("undeclared_context", "FourD", 2, ["t:Revenue"]) + ] + assert report.dropped_facts == 2 and report.recovered_facts == 3 + assert len(model.facts) == 4 + assert "2 fact(s) dropped (undeclared_context: 2)" in caplog.text + assert "rebuilt context(s) OneD" in caplog.text + + +def test_half_a_stated_period_is_not_enough(tmp_path: Path): + body = ( + '2022-12-31' + "\n" + _fact("Revenue", "OneD", 100) + ) + model = _parse(tmp_path, body) + + assert model.facts == [] + assert model.parse_report is not None + assert model.parse_report.recovered_contexts == [] + assert model.parse_report.dropped_facts == 2 + + +def test_conflicting_stated_dates_are_not_enough(tmp_path: Path): + body = ( + _dates("OneD", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-07-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + ) + model = _parse(tmp_path, body) + + assert model.facts == [] + assert model.parse_report is not None and model.parse_report.dropped_facts == 5 + + +def test_an_instant_concept_is_not_placed_in_a_rebuilt_duration(tmp_path: Path): + body = ( + _dates("OneD", "2022-10-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Assets", "OneD", 900) + ) + model = _parse(tmp_path, body) + + assert "t:Assets" not in {f.concept_qname for f in model.facts} + report = model.parse_report + assert report is not None + assert [(d.context_id, d.facts, d.concepts) for d in report.dropped] == [ + ("OneD", 1, ["t:Assets"]) + ] + + +def test_declared_contexts_that_disagree_on_the_entity_rebuild_nothing(tmp_path: Path): + body = ( + _context("A", "ACME", "2022-10-01", "2022-12-31") + + _context("B", "OTHER", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-10-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Revenue", "A", 1) + + _fact("Revenue", "B", 2) + ) + model = _parse(tmp_path, body) + + assert len(model.facts) == 2 + report = model.parse_report + assert report is not None and report.recovered_contexts == [] + assert report.dropped_facts == 3 + + +def test_with_no_declared_context_the_filings_identifier_stands_in(tmp_path: Path): + model = _parse( + tmp_path, _dates("OneD", "2022-10-01", "2022-12-31") + _fact("Revenue", "OneD", 100) + ) + + assert len(model.facts) == 3 + assert {f.entity_cik for f in model.facts} == {"0000000042"} + assert {f.entity_scheme for f in model.facts} == {None} + assert model.parse_report is not None + assert model.parse_report.recovered_contexts[0].entity_from == "filing" + + +def test_a_filing_that_declares_its_contexts_has_no_parse_report( + tmp_path: Path, caplog: pytest.LogCaptureFixture +): + body = _context("OneD", "ACME", "2022-10-01", "2022-12-31") + _fact( + "Revenue", "OneD", 100 + ) + with caplog.at_level(logging.WARNING, logger="xbrlkit.parse.to_model"): + model = _parse(tmp_path, body) + + assert len(model.facts) == 1 and model.facts[0].provenance is None + assert model.parse_report is None + assert caplog.text == "" + + +def test_the_recovered_mark_travels_through_holon_and_tavi(tmp_path: Path): + from xbrlkit.deserialize.holon import from_holon_json + from xbrlkit.deserialize.tavi import from_tavi_json + from xbrlkit.serialize import to_holon, to_tavi + + model = _parse(tmp_path, RESULTS) + + for back in (from_holon_json(to_holon(model)), from_tavi_json(to_tavi(model))): + kinds = [f.provenance.kind if f.provenance else None for f in back.facts] + assert kinds.count(RECOVERED_CONTEXT) == 6 and kinds.count(None) == 1 + + +def test_the_load_receipt_and_describe_filing_say_what_the_parse_did(tmp_path: Path): + from xbrlkit.serve import tools + from xbrlkit.serve.session import FilingSession + + (tmp_path / "t.xsd").write_text(SCHEMA) + instance = tmp_path / "i.xml" + instance.write_text( + HEAD + + _context("Declared", "ACME", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-10-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Revenue", "FourD", 300) + + _fact("Revenue", "Declared", 5) + + "\n" + ) + loaded = FilingSession().load(str(instance)) + + for payload in (tools.load_receipt(loaded), tools.describe_filing(loaded)): + report = payload["parse_report"] + assert report["facts_dropped"] == 1 + assert report["dropped"] == [ + { + "reason": "undeclared_context", + "context": "FourD", + "facts": 1, + "concepts": ["t:Revenue"], + } + ] + assert report["facts_in_recovered_contexts"] == 3 + assert report["recovered_contexts"][0]["context"] == "OneD" + assert report["recovered_contexts"][0]["end"] == "2022-12-31" diff --git a/xbrlkit/model.py b/xbrlkit/model.py index 533f936..c88de1c 100644 --- a/xbrlkit/model.py +++ b/xbrlkit/model.py @@ -324,6 +324,8 @@ class XbrlFact(BaseModel): section_role: str | None = None # Where the fact came from, when the producer said. A filing leaves it # ``None``; an authored report may carry it, and both projections keep it. + # The parse sets it in one case: a fact whose context the instance never + # declared and the parse rebuilt carries ``kind="recovered_context"``. provenance: FactProvenance | None = None @@ -387,6 +389,63 @@ class Network(BaseModel): fact_set_id: str | None = None +# ``FactProvenance.kind`` of a fact whose context the parse rebuilt. +RECOVERED_CONTEXT = "recovered_context" + +DropReason = Literal["undeclared_context", "undefined_concept", "invalid_period"] + + +class DroppedFacts(BaseModel): + """Facts in the instance that are not in the model, for one reason and context.""" + + reason: DropReason + # The ``contextRef`` the facts name; for ``undeclared_context`` an id the + # instance never declares. + context_id: str | None = None + facts: int = 0 + # A sample of the concepts lost, in instance order. + concepts: list[str] = Field(default_factory=list) + + +class RecoveredContext(BaseModel): + """A context the instance references but never declares, rebuilt by the parse. + + The period is read from the filing's own reporting-period date facts under + that context id, never borrowed from another context. The facts placed in + it carry ``provenance.kind == "recovered_context"``. + """ + + context_id: str + start: date + end: date + period_id: str + facts: int = 0 + # The two facts the period was read from. + period_from: list[str] = Field(default_factory=list) + # ``declared_contexts`` when every declared context names one entity; + # ``filing`` when the instance declares none and the caller's id stands in. + entity_from: Literal["declared_contexts", "filing"] + + +class ParseReport(BaseModel): + """What the parse could not place as the instance wrote it. + + ``None`` on a model means the parse had nothing to report, or the model was + not parsed from XBRL at all. + """ + + dropped: list[DroppedFacts] = Field(default_factory=list) + recovered_contexts: list[RecoveredContext] = Field(default_factory=list) + + @property + def dropped_facts(self) -> int: + return sum(d.facts for d in self.dropped) + + @property + def recovered_facts(self) -> int: + return sum(r.facts for r in self.recovered_contexts) + + class XbrlModel(BaseModel): """The whole filing as neutral objects — the contract between parse and serialize.""" @@ -397,3 +456,5 @@ class XbrlModel(BaseModel): units: list[Unit] = Field(default_factory=list) facts: list[XbrlFact] = Field(default_factory=list) networks: list[Network] = Field(default_factory=list) + # Facts the parse dropped or placed in a rebuilt context; ``None`` when none. + parse_report: ParseReport | None = None diff --git a/xbrlkit/parse/README.md b/xbrlkit/parse/README.md index 52f42e3..0654917 100644 --- a/xbrlkit/parse/README.md +++ b/xbrlkit/parse/README.md @@ -36,6 +36,38 @@ so all three have to agree exactly. every qname the slice touches — abstract headers, subtotals, axes, domains, members — so labels and structural flags are available for all of them. +## Facts the walk cannot place + +A fact is left out of the model for three reasons, and each is counted on +`model.parse_report` (by reason and context id, with a sample of the concepts) +and logged as a warning, never dropped silently: + +| Reason | What happened | +| --- | --- | +| `undeclared_context` | The fact's `contextRef` names an id the instance never declares. Arelle loads the fact with no context | +| `undefined_concept` | The fact has no concept behind it | +| `invalid_period` | The context's dates could not be read | + +`parse_report` is `None` when there was nothing to report. + +**An undeclared context is rebuilt when the filing states its period itself.** +Some Indian exchange results filings put most of their facts under `OneD` (the +quarter) and `FourD` (the year to date) and declare neither, but report +`DateOfStartOfReportingPeriod` and `DateOfEndOfReportingPeriod` under each. When +exactly one start and one end date sit under an undeclared id, the parse builds +a duration from them and keeps the facts. The rules are narrow on purpose: + +- The period comes only from those two facts, never from a declared sibling + context: the same filings date some of those wrongly. +- The entity is the one every declared context names. If they disagree, nothing + is rebuilt; if the instance declares none, the caller's filer id stands in. +- Only duration concepts are placed, and a rebuilt context has no dimensions. + +A fact kept this way carries `provenance.kind == "recovered_context"`, which +the holon and TAVI projections write and their readers restore, and the +context is listed in `parse_report.recovered_contexts`. Arelle hashes a fact +with its context, so these facts' ids fold in the context id. + ## The Arelle cache Arelle resolves a filing's DTS by fetching every schema and linkbase it diff --git a/xbrlkit/parse/to_model.py b/xbrlkit/parse/to_model.py index e2ad756..7f86e66 100644 --- a/xbrlkit/parse/to_model.py +++ b/xbrlkit/parse/to_model.py @@ -12,25 +12,39 @@ - Arelle stores instant/end dates as the *exclusive* next midnight, so every instant/end date is rolled back one day (``- timedelta(1)``). - Measures resolve to prefixed tokens (``iso4217:USD``) from the QName. + +A fact the walk cannot place is never dropped silently: it is counted on the +model's ``parse_report`` by reason, and a fact under a context the instance +references but never declares is kept when the filing states that context's +period itself (see :func:`_recoverable_contexts`). """ from __future__ import annotations +import hashlib +import logging +from dataclasses import dataclass, field from datetime import date, datetime, timedelta from typing import TYPE_CHECKING, Any from arelle import XbrlConst from xbrlkit.model import ( + RECOVERED_CONTEXT, Arc, Concept, DimQualifier, + DroppedFacts, + DropReason, EntityIdentity, + FactProvenance, FilingMeta, Label, Network, NetworkKind, + ParseReport, Period, + RecoveredContext, Reference, Unit, XbrlFact, @@ -42,6 +56,8 @@ if TYPE_CHECKING: from arelle.ModelXbrl import ModelXbrl +logger = logging.getLogger(__name__) + # Extended-link roles that carry no statement/disclosure network — the label # and reference linkbases (standard link role) plus the enumeration-list roles. ROLES_FILTERED = { @@ -54,6 +70,17 @@ _DEI_FISCAL_PERIOD = "dei:DocumentFiscalPeriodFocus" _DEI_FISCAL_YEAR_END = "dei:CurrentFiscalYearEndDate" +# Concept local names (start, end) under which a filing states the period of +# the context they sit in. The Indian exchanges' results taxonomies report +# these under each column's context, including the ones some filings never +# declare. +_REPORTING_PERIOD_FACTS = ( + ("DateOfStartOfReportingPeriod", "DateOfEndOfReportingPeriod"), +) + +# Concepts sampled per dropped group on the parse report. +_DROPPED_CONCEPT_SAMPLE = 5 + def to_xbrl_model( mx: ModelXbrl, @@ -86,11 +113,23 @@ def to_xbrl_model( fiscal_period_focus: str | None = None fiscal_year_end_month: str | None = None + tally = _ParseTally() + rebuilt_contexts = _recoverable_contexts(mx, main_cik) + for f in mx.facts: - if f.context is None: - continue + context = f.context + context_id = getattr(f, "contextID", None) concept = f.concept + rebuilt: _RebuiltContext | None = None + if context is None: + # A rebuilt context is a duration; an instant concept cannot sit in it. + if getattr(concept, "periodType", None) == "duration": + rebuilt = rebuilt_contexts.get(context_id or "") + if rebuilt is None: + tally.drop("undeclared_context", context_id, f) + continue if concept is None or concept.qname is None: + tally.drop("undefined_concept", context_id, f) continue qname_str = str(concept.qname) @@ -112,8 +151,9 @@ def to_xbrl_model( _ensure_concept(mx, concepts, namespaces, concept) # Period (deduped by content-derived id). Skip facts with invalid dates. - period = _make_period(f.context) + period = rebuilt.period if rebuilt else _make_period(context) if period is None: + tally.drop("invalid_period", context_id, f) continue if period.id not in periods: periods[period.id] = period @@ -128,7 +168,11 @@ def to_xbrl_model( unit_ref = unit.id # Entity identity (prefer the context whose CIK is the filer's). - scheme, raw_cik = f.context.entityIdentifier + if rebuilt: + scheme, raw_cik = rebuilt.scheme, rebuilt.identifier + rebuilt.facts += 1 + else: + scheme, raw_cik = context.entityIdentifier norm_cik = _normalize_cik(raw_cik) if entity_scheme is None or norm_cik == main_cik: entity_scheme = scheme @@ -142,27 +186,39 @@ def to_xbrl_model( numeric_value = None raw_value = getattr(f, "value", None) + source_hash = _md5(f) + if rebuilt: + source_hash = _orphan_hash(f, source_hash, context_id, unit_ref) facts.append( XbrlFact( - id=fact_id(report_uri, f.md5sum.value), + id=fact_id(report_uri, source_hash if rebuilt else f.md5sum.value), concept_qname=qname_str, period_id=period.id, unit_id=unit_ref, entity_cik=norm_cik, entity_scheme=scheme, entity_identifier=str(raw_cik), - dims=_make_dims(f.context, mx, concepts, namespaces), + dims=[] if rebuilt else _make_dims(context, mx, concepts, namespaces), value_str=_value_str(f), raw_value=str(raw_value) if raw_value is not None else None, - source_hash=_md5(f), + source_hash=source_hash, numeric_value=numeric_value, decimals=(str(f.decimals) if (is_numeric and f.decimals is not None) else None), value_kind="numeric" if is_numeric else "text", language=getattr(f, "xmlLang", None), is_nil=bool(getattr(f, "isNil", False)), + provenance=( + FactProvenance(kind=RECOVERED_CONTEXT, attributed_to="xbrlkit") + if rebuilt + else None + ), ) ) + parse_report = tally.report(rebuilt_contexts) + if parse_report is not None: + logger.warning("%s: %s", report_uri, _describe_report(parse_report)) + # Networks last: their arc endpoints (abstract headers, subtotals, hypercube # wiring) complete the concept coverage and add their namespaces. networks = _make_networks(mx, concepts, namespaces) @@ -204,9 +260,175 @@ def to_xbrl_model( units=list(units.values()), facts=facts, networks=networks, + parse_report=parse_report, ) +@dataclass +class _RebuiltContext: + """An undeclared context's period and entity, as the parse rebuilt them.""" + + period: Period + start: date + end: date + scheme: str | None + identifier: str + period_from: list[str] + entity_from: str + facts: int = 0 + + +@dataclass +class _ParseTally: + """Counts the facts the walk leaves out, by reason and context id.""" + + groups: dict[tuple[DropReason, str | None], DroppedFacts] = field( + default_factory=dict + ) + + def drop(self, reason: DropReason, context_id: str | None, fact: Any) -> None: + group = self.groups.setdefault( + (reason, context_id), DroppedFacts(reason=reason, context_id=context_id) + ) + group.facts += 1 + qname = str(getattr(fact, "qname", None) or "") + if ( + qname + and qname not in group.concepts + and len(group.concepts) < _DROPPED_CONCEPT_SAMPLE + ): + group.concepts.append(qname) + + def report(self, rebuilt: dict[str, _RebuiltContext]) -> ParseReport | None: + recovered = [ + RecoveredContext( + context_id=context_id, + start=ctx.start, + end=ctx.end, + period_id=ctx.period.id, + facts=ctx.facts, + period_from=ctx.period_from, + entity_from=ctx.entity_from, # pyright: ignore[reportArgumentType] + ) + for context_id, ctx in rebuilt.items() + if ctx.facts + ] + if not self.groups and not recovered: + return None + return ParseReport(dropped=list(self.groups.values()), recovered_contexts=recovered) + + +def _describe_report(report: ParseReport) -> str: + """One line for the log: what was dropped and what was rebuilt.""" + parts: list[str] = [] + if report.dropped: + by_reason: dict[str, int] = {} + for group in report.dropped: + by_reason[group.reason] = by_reason.get(group.reason, 0) + group.facts + parts.append( + f"{report.dropped_facts} fact(s) dropped (" + + ", ".join(f"{reason}: {count}" for reason, count in by_reason.items()) + + ")" + ) + if report.recovered_contexts: + parts.append( + f"{report.recovered_facts} fact(s) placed in rebuilt context(s) " + + ", ".join(r.context_id for r in report.recovered_contexts) + ) + return "; ".join(parts) + + +def _recoverable_contexts(mx: ModelXbrl, filing_id: str) -> dict[str, _RebuiltContext]: + """Rebuild the contexts the instance references but never declares. + + Some filings name a context id on most of their facts and declare no + ```` for it; Arelle loads those facts with no context. A + context is rebuilt only when the filing states its period itself: exactly + one start date and one end date reported under that same id (see + ``_REPORTING_PERIOD_FACTS``). The period is never taken from a declared + sibling context — the same filings date some of those wrongly. + + The entity is the one every declared context names. When the declared + contexts disagree nothing is rebuilt; when the instance declares none, + ``filing_id`` (the caller's identifier for the filer) stands in, with no + scheme. A rebuilt context has no dimensions. + """ + orphans = [f for f in mx.facts if f.context is None and getattr(f, "contextID", None)] + if not orphans: + return {} + + entities = {c.entityIdentifier for c in (mx.contexts or {}).values()} + if len(entities) > 1: + return {} + if entities: + scheme, identifier = next(iter(entities)) + entity_from = "declared_contexts" + else: + scheme, identifier, entity_from = None, filing_id, "filing" + + stated: dict[str, dict[str, dict[date, str]]] = {} + for f in orphans: + qname = getattr(f, "qname", None) + local = getattr(qname, "localName", None) + if not any(local in pair for pair in _REPORTING_PERIOD_FACTS): + continue + value = _iso_date(getattr(f, "value", None)) + if value is not None: + stated.setdefault(f.contextID, {}).setdefault(local, {})[value] = str(qname) + + rebuilt: dict[str, _RebuiltContext] = {} + for context_id, by_name in stated.items(): + for start_name, end_name in _REPORTING_PERIOD_FACTS: + starts, ends = by_name.get(start_name, {}), by_name.get(end_name, {}) + if len(starts) != 1 or len(ends) != 1: + continue + (start, start_qname), (end, end_qname) = ( + next(iter(starts.items())), + next(iter(ends.items())), + ) + if start > end: + continue + rebuilt[context_id] = _RebuiltContext( + period=duration_period(start, end), + start=start, + end=end, + scheme=scheme, + identifier=str(identifier), + period_from=[start_qname, end_qname], + entity_from=entity_from, + ) + break + return rebuilt + + +def _orphan_hash( + fact: Any, arelle_hash: str | None, context_id: str | None, unit_ref: str | None +) -> str: + """A hash for a fact Arelle loaded with no context. + + Arelle hashes such a fact by its qname alone — the value, language, nil and + unit only enter beside a context — so every orphan of one concept shares a + hash. This folds in what Arelle would have, with the context id standing + where the context's own hash goes. + """ + parts = [ + arelle_hash or "", + context_id or "", + getattr(fact, "xmlLang", None) or "", + "nil" if getattr(fact, "isNil", False) else str(getattr(fact, "value", "") or ""), + unit_ref or "", + ] + return hashlib.md5("|".join(parts).encode(), usedforsecurity=False).hexdigest() + + +def _iso_date(value: Any) -> date | None: + """An ISO date from a fact's lexical value, or ``None``.""" + try: + return _to_date(date.fromisoformat(str(value).strip()[:10])) + except ValueError: + return None + + def _ensure_concept( mx: ModelXbrl, concepts: dict[str, Concept], diff --git a/xbrlkit/serve/README.md b/xbrlkit/serve/README.md index a6b931c..d407a77 100644 --- a/xbrlkit/serve/README.md +++ b/xbrlkit/serve/README.md @@ -56,6 +56,11 @@ A filing loaded without a taxonomy whose host has gone — every Danish ESEF report cites one — says that too: `missing_taxonomy` names the documents and counts the facts that are in the report and not in the model. +So does a filing whose facts the parse could not place as written: +`parse_report` counts the facts dropped, by reason and context id, and names +any context the instance never declared that was +[rebuilt from the filing's own dates](../parse/README.md#facts-the-walk-cannot-place). + To read the filing itself, parse it from EDGAR: - `--pure` does so by default (see below). diff --git a/xbrlkit/serve/tools.py b/xbrlkit/serve/tools.py index 55fae69..3b0631a 100644 --- a/xbrlkit/serve/tools.py +++ b/xbrlkit/serve/tools.py @@ -35,7 +35,16 @@ from xbrlkit.check import precision as _precision from xbrlkit.check import tolerance as _tolerance from xbrlkit.config import CONFIG, Config -from xbrlkit.model import Arc, Concept, Network, Period, Unit, XbrlFact, XbrlModel +from xbrlkit.model import ( + Arc, + Concept, + Network, + ParseReport, + Period, + Unit, + XbrlFact, + XbrlModel, +) from xbrlkit.serialize import classify_network, root_qname from xbrlkit.serve.session import ( FilingSession, @@ -485,6 +494,7 @@ def list_filings(session: FilingSession) -> dict[str, Any]: "read_from", "taxonomy", "missing_taxonomy", + "parse_report", "filing", "entity", "counts", @@ -653,6 +663,11 @@ def describe_filing( if lf.missing_taxonomy else {} ), + **( + {"parse_report": _describe_parse_report(model.parse_report)} + if model.parse_report + else {} + ), "filing": { "id": lf.id, "source": lf.source, @@ -747,6 +762,48 @@ def _describe_missing(missing: MissingTaxonomy) -> dict[str, Any]: } +def _describe_parse_report(report: ParseReport) -> dict[str, Any]: + """What the parse left out or rebuilt, so no answer is read as the whole.""" + out: dict[str, Any] = {} + notes: list[str] = [] + if report.dropped: + out["facts_dropped"] = report.dropped_facts + out["dropped"] = [ + { + "reason": d.reason, + "context": d.context_id, + "facts": d.facts, + "concepts": d.concepts, + } + for d in report.dropped + ] + notes.append( + "the dropped facts are in the report and not in this model: " + "`undeclared_context` names a context id the instance references and " + "never declares, `invalid_period` a context whose dates could not be read" + ) + if report.recovered_contexts: + out["facts_in_recovered_contexts"] = report.recovered_facts + out["recovered_contexts"] = [ + { + "context": r.context_id, + "start": r.start.isoformat(), + "end": r.end.isoformat(), + "facts": r.facts, + "period_from": r.period_from, + "entity_from": r.entity_from, + } + for r in report.recovered_contexts + ] + notes.append( + "a recovered context is referenced by the instance and never declared; " + "its period was read from the filing's own reporting-period facts, and " + "it carries no dimensions" + ) + out["note"] = "; ".join(notes) + "." + return out + + def _describe_taxonomy(taxonomy: TaxonomyEntry) -> dict[str, Any]: """Which entry point a taxonomy package was loaded from, and the others it offers — the choice stated, never silent."""