diff --git a/tests/test_parse_contexts.py b/tests/test_parse_contexts.py new file mode 100644 index 0000000..41ae0fd --- /dev/null +++ b/tests/test_parse_contexts.py @@ -0,0 +1,303 @@ +"""Facts the parse cannot place: counted, never dropped silently, and kept when +the filing states the period of an undeclared context itself. + +The instances are synthetic and load offline: a three-concept schema and an +instance whose facts name context ids it never declares, the pattern some +Indian exchange results filings follow (``OneD`` for the quarter, ``FourD`` for +the year to date). +""" + +from __future__ import annotations + +import logging +from datetime import date +from pathlib import Path + +import pytest + +from xbrlkit.model import RECOVERED_CONTEXT, FilingMeta, XbrlModel +from xbrlkit.parse import close, load_model, to_xbrl_model + +SCHEMA = """ + + + + + + + +""" + +HEAD = """ + + +iso4217:INR +""" + + +def _context(context_id: str, entity: str, start: str, end: str) -> str: + return ( + f'' + f'{entity}' + f"{start}" + f"{end}\n" + ) + + +def _dates(context_id: str, start: str, end: str) -> str: + return ( + f'{start}' + "\n" + f'{end}' + "\n" + ) + + +def _fact(name: str, context_id: str, value: int) -> str: + return ( + f'{value}' + f"\n" + ) + + +# The quarter and the year to date, neither declared; one declared context +# that dates a year-to-date breakdown as the quarter, as the real filings do. +RESULTS = ( + _context("FourDetailD", "ACME", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-10-01", "2022-12-31") + + _dates("FourD", "2022-04-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Revenue", "FourD", 300) + + _fact("Revenue", "FourDetailD", 300) +) + + +def _parse(tmp_path: Path, body: str) -> XbrlModel: + (tmp_path / "t.xsd").write_text(SCHEMA) + instance = tmp_path / "i.xml" + instance.write_text(HEAD + body + "\n") + mx = load_model(str(instance), cache_dir=tmp_path / "cache", offline=True) + try: + return to_xbrl_model(mx, FilingMeta(accession="acc", cik="0000000042")) + finally: + close(mx.modelManager.cntlr) + + +def _by_context_value(model: XbrlModel) -> dict[str, tuple[date | None, date | None]]: + periods = {p.id: p for p in model.periods} + return { + f.value_str or "": (periods[f.period_id].start, periods[f.period_id].end) + for f in model.facts + if f.concept_qname == "t:Revenue" and f.provenance is not None + } + + +def test_undeclared_contexts_are_rebuilt_from_the_filings_own_dates(tmp_path: Path): + model = _parse(tmp_path, RESULTS) + + assert len(model.facts) == 7 + assert _by_context_value(model) == { + "100": (date(2022, 10, 1), date(2022, 12, 31)), + "300": (date(2022, 4, 1), date(2022, 12, 31)), + } + report = model.parse_report + assert report is not None and report.dropped == [] + assert [ + (r.context_id, r.start, r.end, r.facts) for r in report.recovered_contexts + ] == [ + ("OneD", date(2022, 10, 1), date(2022, 12, 31), 3), + ("FourD", date(2022, 4, 1), date(2022, 12, 31), 3), + ] + assert report.recovered_contexts[0].entity_from == "declared_contexts" + assert report.recovered_contexts[0].period_from == [ + "t:DateOfStartOfReportingPeriod", + "t:DateOfEndOfReportingPeriod", + ] + + +def test_recovered_facts_are_marked_and_declared_ones_are_not(tmp_path: Path): + model = _parse(tmp_path, RESULTS) + + marked = [f for f in model.facts if f.provenance is not None] + declared = [f for f in model.facts if f.provenance is None] + assert len(marked) == 6 and len(declared) == 1 + assert {f.provenance.kind for f in marked if f.provenance} == {RECOVERED_CONTEXT} + assert all(f.entity_identifier == "ACME" and f.dims == [] for f in marked) + # The declared context keeps the dates the filing gave it, right or wrong. + period = next(p for p in model.periods if p.id == declared[0].period_id) + assert (period.start, period.end) == (date(2022, 10, 1), date(2022, 12, 31)) + + +def test_recovered_facts_of_one_concept_get_distinct_ids(tmp_path: Path): + # Arelle hashes a fact with no context by its qname alone, so every orphan + # of one concept shares a hash — across contexts and within one. + model = _parse(tmp_path, RESULTS + _fact("Revenue", "OneD", 101)) + + ids = [f.id for f in model.facts] + assert len(set(ids)) == len(ids) + hashes = [f.source_hash for f in model.facts if f.provenance is not None] + assert len(set(hashes)) == len(hashes) + + +def test_a_context_with_no_stated_period_is_dropped_and_counted( + tmp_path: Path, caplog: pytest.LogCaptureFixture +): + body = ( + _context("Declared", "ACME", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-10-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Revenue", "FourD", 300) + + _fact("Revenue", "FourD", 301) + + _fact("Revenue", "Declared", 5) + ) + with caplog.at_level(logging.WARNING, logger="xbrlkit.parse.to_model"): + model = _parse(tmp_path, body) + + report = model.parse_report + assert report is not None + assert [(d.reason, d.context_id, d.facts, d.concepts) for d in report.dropped] == [ + ("undeclared_context", "FourD", 2, ["t:Revenue"]) + ] + assert report.dropped_facts == 2 and report.recovered_facts == 3 + assert len(model.facts) == 4 + assert "2 fact(s) dropped (undeclared_context: 2)" in caplog.text + assert "rebuilt context(s) OneD" in caplog.text + + +def test_half_a_stated_period_is_not_enough(tmp_path: Path): + body = ( + '2022-12-31' + "\n" + _fact("Revenue", "OneD", 100) + ) + model = _parse(tmp_path, body) + + assert model.facts == [] + assert model.parse_report is not None + assert model.parse_report.recovered_contexts == [] + assert model.parse_report.dropped_facts == 2 + + +def test_conflicting_stated_dates_are_not_enough(tmp_path: Path): + body = ( + _dates("OneD", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-07-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + ) + model = _parse(tmp_path, body) + + assert model.facts == [] + assert model.parse_report is not None and model.parse_report.dropped_facts == 5 + + +def test_an_instant_concept_is_not_placed_in_a_rebuilt_duration(tmp_path: Path): + body = ( + _dates("OneD", "2022-10-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Assets", "OneD", 900) + ) + model = _parse(tmp_path, body) + + assert "t:Assets" not in {f.concept_qname for f in model.facts} + report = model.parse_report + assert report is not None + assert [(d.context_id, d.facts, d.concepts) for d in report.dropped] == [ + ("OneD", 1, ["t:Assets"]) + ] + + +def test_declared_contexts_that_disagree_on_the_entity_rebuild_nothing(tmp_path: Path): + body = ( + _context("A", "ACME", "2022-10-01", "2022-12-31") + + _context("B", "OTHER", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-10-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Revenue", "A", 1) + + _fact("Revenue", "B", 2) + ) + model = _parse(tmp_path, body) + + assert len(model.facts) == 2 + report = model.parse_report + assert report is not None and report.recovered_contexts == [] + assert report.dropped_facts == 3 + + +def test_with_no_declared_context_the_filings_identifier_stands_in(tmp_path: Path): + model = _parse( + tmp_path, _dates("OneD", "2022-10-01", "2022-12-31") + _fact("Revenue", "OneD", 100) + ) + + assert len(model.facts) == 3 + assert {f.entity_cik for f in model.facts} == {"0000000042"} + assert {f.entity_scheme for f in model.facts} == {None} + assert model.parse_report is not None + assert model.parse_report.recovered_contexts[0].entity_from == "filing" + + +def test_a_filing_that_declares_its_contexts_has_no_parse_report( + tmp_path: Path, caplog: pytest.LogCaptureFixture +): + body = _context("OneD", "ACME", "2022-10-01", "2022-12-31") + _fact( + "Revenue", "OneD", 100 + ) + with caplog.at_level(logging.WARNING, logger="xbrlkit.parse.to_model"): + model = _parse(tmp_path, body) + + assert len(model.facts) == 1 and model.facts[0].provenance is None + assert model.parse_report is None + assert caplog.text == "" + + +def test_the_recovered_mark_travels_through_holon_and_tavi(tmp_path: Path): + from xbrlkit.deserialize.holon import from_holon_json + from xbrlkit.deserialize.tavi import from_tavi_json + from xbrlkit.serialize import to_holon, to_tavi + + model = _parse(tmp_path, RESULTS) + + for back in (from_holon_json(to_holon(model)), from_tavi_json(to_tavi(model))): + kinds = [f.provenance.kind if f.provenance else None for f in back.facts] + assert kinds.count(RECOVERED_CONTEXT) == 6 and kinds.count(None) == 1 + + +def test_the_load_receipt_and_describe_filing_say_what_the_parse_did(tmp_path: Path): + from xbrlkit.serve import tools + from xbrlkit.serve.session import FilingSession + + (tmp_path / "t.xsd").write_text(SCHEMA) + instance = tmp_path / "i.xml" + instance.write_text( + HEAD + + _context("Declared", "ACME", "2022-10-01", "2022-12-31") + + _dates("OneD", "2022-10-01", "2022-12-31") + + _fact("Revenue", "OneD", 100) + + _fact("Revenue", "FourD", 300) + + _fact("Revenue", "Declared", 5) + + "\n" + ) + loaded = FilingSession().load(str(instance)) + + for payload in (tools.load_receipt(loaded), tools.describe_filing(loaded)): + report = payload["parse_report"] + assert report["facts_dropped"] == 1 + assert report["dropped"] == [ + { + "reason": "undeclared_context", + "context": "FourD", + "facts": 1, + "concepts": ["t:Revenue"], + } + ] + assert report["facts_in_recovered_contexts"] == 3 + assert report["recovered_contexts"][0]["context"] == "OneD" + assert report["recovered_contexts"][0]["end"] == "2022-12-31" diff --git a/xbrlkit/model.py b/xbrlkit/model.py index 533f936..c88de1c 100644 --- a/xbrlkit/model.py +++ b/xbrlkit/model.py @@ -324,6 +324,8 @@ class XbrlFact(BaseModel): section_role: str | None = None # Where the fact came from, when the producer said. A filing leaves it # ``None``; an authored report may carry it, and both projections keep it. + # The parse sets it in one case: a fact whose context the instance never + # declared and the parse rebuilt carries ``kind="recovered_context"``. provenance: FactProvenance | None = None @@ -387,6 +389,63 @@ class Network(BaseModel): fact_set_id: str | None = None +# ``FactProvenance.kind`` of a fact whose context the parse rebuilt. +RECOVERED_CONTEXT = "recovered_context" + +DropReason = Literal["undeclared_context", "undefined_concept", "invalid_period"] + + +class DroppedFacts(BaseModel): + """Facts in the instance that are not in the model, for one reason and context.""" + + reason: DropReason + # The ``contextRef`` the facts name; for ``undeclared_context`` an id the + # instance never declares. + context_id: str | None = None + facts: int = 0 + # A sample of the concepts lost, in instance order. + concepts: list[str] = Field(default_factory=list) + + +class RecoveredContext(BaseModel): + """A context the instance references but never declares, rebuilt by the parse. + + The period is read from the filing's own reporting-period date facts under + that context id, never borrowed from another context. The facts placed in + it carry ``provenance.kind == "recovered_context"``. + """ + + context_id: str + start: date + end: date + period_id: str + facts: int = 0 + # The two facts the period was read from. + period_from: list[str] = Field(default_factory=list) + # ``declared_contexts`` when every declared context names one entity; + # ``filing`` when the instance declares none and the caller's id stands in. + entity_from: Literal["declared_contexts", "filing"] + + +class ParseReport(BaseModel): + """What the parse could not place as the instance wrote it. + + ``None`` on a model means the parse had nothing to report, or the model was + not parsed from XBRL at all. + """ + + dropped: list[DroppedFacts] = Field(default_factory=list) + recovered_contexts: list[RecoveredContext] = Field(default_factory=list) + + @property + def dropped_facts(self) -> int: + return sum(d.facts for d in self.dropped) + + @property + def recovered_facts(self) -> int: + return sum(r.facts for r in self.recovered_contexts) + + class XbrlModel(BaseModel): """The whole filing as neutral objects — the contract between parse and serialize.""" @@ -397,3 +456,5 @@ class XbrlModel(BaseModel): units: list[Unit] = Field(default_factory=list) facts: list[XbrlFact] = Field(default_factory=list) networks: list[Network] = Field(default_factory=list) + # Facts the parse dropped or placed in a rebuilt context; ``None`` when none. + parse_report: ParseReport | None = None diff --git a/xbrlkit/parse/README.md b/xbrlkit/parse/README.md index 52f42e3..0654917 100644 --- a/xbrlkit/parse/README.md +++ b/xbrlkit/parse/README.md @@ -36,6 +36,38 @@ so all three have to agree exactly. every qname the slice touches — abstract headers, subtotals, axes, domains, members — so labels and structural flags are available for all of them. +## Facts the walk cannot place + +A fact is left out of the model for three reasons, and each is counted on +`model.parse_report` (by reason and context id, with a sample of the concepts) +and logged as a warning, never dropped silently: + +| Reason | What happened | +| --- | --- | +| `undeclared_context` | The fact's `contextRef` names an id the instance never declares. Arelle loads the fact with no context | +| `undefined_concept` | The fact has no concept behind it | +| `invalid_period` | The context's dates could not be read | + +`parse_report` is `None` when there was nothing to report. + +**An undeclared context is rebuilt when the filing states its period itself.** +Some Indian exchange results filings put most of their facts under `OneD` (the +quarter) and `FourD` (the year to date) and declare neither, but report +`DateOfStartOfReportingPeriod` and `DateOfEndOfReportingPeriod` under each. When +exactly one start and one end date sit under an undeclared id, the parse builds +a duration from them and keeps the facts. The rules are narrow on purpose: + +- The period comes only from those two facts, never from a declared sibling + context: the same filings date some of those wrongly. +- The entity is the one every declared context names. If they disagree, nothing + is rebuilt; if the instance declares none, the caller's filer id stands in. +- Only duration concepts are placed, and a rebuilt context has no dimensions. + +A fact kept this way carries `provenance.kind == "recovered_context"`, which +the holon and TAVI projections write and their readers restore, and the +context is listed in `parse_report.recovered_contexts`. Arelle hashes a fact +with its context, so these facts' ids fold in the context id. + ## The Arelle cache Arelle resolves a filing's DTS by fetching every schema and linkbase it diff --git a/xbrlkit/parse/to_model.py b/xbrlkit/parse/to_model.py index e2ad756..7f86e66 100644 --- a/xbrlkit/parse/to_model.py +++ b/xbrlkit/parse/to_model.py @@ -12,25 +12,39 @@ - Arelle stores instant/end dates as the *exclusive* next midnight, so every instant/end date is rolled back one day (``- timedelta(1)``). - Measures resolve to prefixed tokens (``iso4217:USD``) from the QName. + +A fact the walk cannot place is never dropped silently: it is counted on the +model's ``parse_report`` by reason, and a fact under a context the instance +references but never declares is kept when the filing states that context's +period itself (see :func:`_recoverable_contexts`). """ from __future__ import annotations +import hashlib +import logging +from dataclasses import dataclass, field from datetime import date, datetime, timedelta from typing import TYPE_CHECKING, Any from arelle import XbrlConst from xbrlkit.model import ( + RECOVERED_CONTEXT, Arc, Concept, DimQualifier, + DroppedFacts, + DropReason, EntityIdentity, + FactProvenance, FilingMeta, Label, Network, NetworkKind, + ParseReport, Period, + RecoveredContext, Reference, Unit, XbrlFact, @@ -42,6 +56,8 @@ if TYPE_CHECKING: from arelle.ModelXbrl import ModelXbrl +logger = logging.getLogger(__name__) + # Extended-link roles that carry no statement/disclosure network — the label # and reference linkbases (standard link role) plus the enumeration-list roles. ROLES_FILTERED = { @@ -54,6 +70,17 @@ _DEI_FISCAL_PERIOD = "dei:DocumentFiscalPeriodFocus" _DEI_FISCAL_YEAR_END = "dei:CurrentFiscalYearEndDate" +# Concept local names (start, end) under which a filing states the period of +# the context they sit in. The Indian exchanges' results taxonomies report +# these under each column's context, including the ones some filings never +# declare. +_REPORTING_PERIOD_FACTS = ( + ("DateOfStartOfReportingPeriod", "DateOfEndOfReportingPeriod"), +) + +# Concepts sampled per dropped group on the parse report. +_DROPPED_CONCEPT_SAMPLE = 5 + def to_xbrl_model( mx: ModelXbrl, @@ -86,11 +113,23 @@ def to_xbrl_model( fiscal_period_focus: str | None = None fiscal_year_end_month: str | None = None + tally = _ParseTally() + rebuilt_contexts = _recoverable_contexts(mx, main_cik) + for f in mx.facts: - if f.context is None: - continue + context = f.context + context_id = getattr(f, "contextID", None) concept = f.concept + rebuilt: _RebuiltContext | None = None + if context is None: + # A rebuilt context is a duration; an instant concept cannot sit in it. + if getattr(concept, "periodType", None) == "duration": + rebuilt = rebuilt_contexts.get(context_id or "") + if rebuilt is None: + tally.drop("undeclared_context", context_id, f) + continue if concept is None or concept.qname is None: + tally.drop("undefined_concept", context_id, f) continue qname_str = str(concept.qname) @@ -112,8 +151,9 @@ def to_xbrl_model( _ensure_concept(mx, concepts, namespaces, concept) # Period (deduped by content-derived id). Skip facts with invalid dates. - period = _make_period(f.context) + period = rebuilt.period if rebuilt else _make_period(context) if period is None: + tally.drop("invalid_period", context_id, f) continue if period.id not in periods: periods[period.id] = period @@ -128,7 +168,11 @@ def to_xbrl_model( unit_ref = unit.id # Entity identity (prefer the context whose CIK is the filer's). - scheme, raw_cik = f.context.entityIdentifier + if rebuilt: + scheme, raw_cik = rebuilt.scheme, rebuilt.identifier + rebuilt.facts += 1 + else: + scheme, raw_cik = context.entityIdentifier norm_cik = _normalize_cik(raw_cik) if entity_scheme is None or norm_cik == main_cik: entity_scheme = scheme @@ -142,27 +186,39 @@ def to_xbrl_model( numeric_value = None raw_value = getattr(f, "value", None) + source_hash = _md5(f) + if rebuilt: + source_hash = _orphan_hash(f, source_hash, context_id, unit_ref) facts.append( XbrlFact( - id=fact_id(report_uri, f.md5sum.value), + id=fact_id(report_uri, source_hash if rebuilt else f.md5sum.value), concept_qname=qname_str, period_id=period.id, unit_id=unit_ref, entity_cik=norm_cik, entity_scheme=scheme, entity_identifier=str(raw_cik), - dims=_make_dims(f.context, mx, concepts, namespaces), + dims=[] if rebuilt else _make_dims(context, mx, concepts, namespaces), value_str=_value_str(f), raw_value=str(raw_value) if raw_value is not None else None, - source_hash=_md5(f), + source_hash=source_hash, numeric_value=numeric_value, decimals=(str(f.decimals) if (is_numeric and f.decimals is not None) else None), value_kind="numeric" if is_numeric else "text", language=getattr(f, "xmlLang", None), is_nil=bool(getattr(f, "isNil", False)), + provenance=( + FactProvenance(kind=RECOVERED_CONTEXT, attributed_to="xbrlkit") + if rebuilt + else None + ), ) ) + parse_report = tally.report(rebuilt_contexts) + if parse_report is not None: + logger.warning("%s: %s", report_uri, _describe_report(parse_report)) + # Networks last: their arc endpoints (abstract headers, subtotals, hypercube # wiring) complete the concept coverage and add their namespaces. networks = _make_networks(mx, concepts, namespaces) @@ -204,9 +260,175 @@ def to_xbrl_model( units=list(units.values()), facts=facts, networks=networks, + parse_report=parse_report, ) +@dataclass +class _RebuiltContext: + """An undeclared context's period and entity, as the parse rebuilt them.""" + + period: Period + start: date + end: date + scheme: str | None + identifier: str + period_from: list[str] + entity_from: str + facts: int = 0 + + +@dataclass +class _ParseTally: + """Counts the facts the walk leaves out, by reason and context id.""" + + groups: dict[tuple[DropReason, str | None], DroppedFacts] = field( + default_factory=dict + ) + + def drop(self, reason: DropReason, context_id: str | None, fact: Any) -> None: + group = self.groups.setdefault( + (reason, context_id), DroppedFacts(reason=reason, context_id=context_id) + ) + group.facts += 1 + qname = str(getattr(fact, "qname", None) or "") + if ( + qname + and qname not in group.concepts + and len(group.concepts) < _DROPPED_CONCEPT_SAMPLE + ): + group.concepts.append(qname) + + def report(self, rebuilt: dict[str, _RebuiltContext]) -> ParseReport | None: + recovered = [ + RecoveredContext( + context_id=context_id, + start=ctx.start, + end=ctx.end, + period_id=ctx.period.id, + facts=ctx.facts, + period_from=ctx.period_from, + entity_from=ctx.entity_from, # pyright: ignore[reportArgumentType] + ) + for context_id, ctx in rebuilt.items() + if ctx.facts + ] + if not self.groups and not recovered: + return None + return ParseReport(dropped=list(self.groups.values()), recovered_contexts=recovered) + + +def _describe_report(report: ParseReport) -> str: + """One line for the log: what was dropped and what was rebuilt.""" + parts: list[str] = [] + if report.dropped: + by_reason: dict[str, int] = {} + for group in report.dropped: + by_reason[group.reason] = by_reason.get(group.reason, 0) + group.facts + parts.append( + f"{report.dropped_facts} fact(s) dropped (" + + ", ".join(f"{reason}: {count}" for reason, count in by_reason.items()) + + ")" + ) + if report.recovered_contexts: + parts.append( + f"{report.recovered_facts} fact(s) placed in rebuilt context(s) " + + ", ".join(r.context_id for r in report.recovered_contexts) + ) + return "; ".join(parts) + + +def _recoverable_contexts(mx: ModelXbrl, filing_id: str) -> dict[str, _RebuiltContext]: + """Rebuild the contexts the instance references but never declares. + + Some filings name a context id on most of their facts and declare no + ```` for it; Arelle loads those facts with no context. A + context is rebuilt only when the filing states its period itself: exactly + one start date and one end date reported under that same id (see + ``_REPORTING_PERIOD_FACTS``). The period is never taken from a declared + sibling context — the same filings date some of those wrongly. + + The entity is the one every declared context names. When the declared + contexts disagree nothing is rebuilt; when the instance declares none, + ``filing_id`` (the caller's identifier for the filer) stands in, with no + scheme. A rebuilt context has no dimensions. + """ + orphans = [f for f in mx.facts if f.context is None and getattr(f, "contextID", None)] + if not orphans: + return {} + + entities = {c.entityIdentifier for c in (mx.contexts or {}).values()} + if len(entities) > 1: + return {} + if entities: + scheme, identifier = next(iter(entities)) + entity_from = "declared_contexts" + else: + scheme, identifier, entity_from = None, filing_id, "filing" + + stated: dict[str, dict[str, dict[date, str]]] = {} + for f in orphans: + qname = getattr(f, "qname", None) + local = getattr(qname, "localName", None) + if not any(local in pair for pair in _REPORTING_PERIOD_FACTS): + continue + value = _iso_date(getattr(f, "value", None)) + if value is not None: + stated.setdefault(f.contextID, {}).setdefault(local, {})[value] = str(qname) + + rebuilt: dict[str, _RebuiltContext] = {} + for context_id, by_name in stated.items(): + for start_name, end_name in _REPORTING_PERIOD_FACTS: + starts, ends = by_name.get(start_name, {}), by_name.get(end_name, {}) + if len(starts) != 1 or len(ends) != 1: + continue + (start, start_qname), (end, end_qname) = ( + next(iter(starts.items())), + next(iter(ends.items())), + ) + if start > end: + continue + rebuilt[context_id] = _RebuiltContext( + period=duration_period(start, end), + start=start, + end=end, + scheme=scheme, + identifier=str(identifier), + period_from=[start_qname, end_qname], + entity_from=entity_from, + ) + break + return rebuilt + + +def _orphan_hash( + fact: Any, arelle_hash: str | None, context_id: str | None, unit_ref: str | None +) -> str: + """A hash for a fact Arelle loaded with no context. + + Arelle hashes such a fact by its qname alone — the value, language, nil and + unit only enter beside a context — so every orphan of one concept shares a + hash. This folds in what Arelle would have, with the context id standing + where the context's own hash goes. + """ + parts = [ + arelle_hash or "", + context_id or "", + getattr(fact, "xmlLang", None) or "", + "nil" if getattr(fact, "isNil", False) else str(getattr(fact, "value", "") or ""), + unit_ref or "", + ] + return hashlib.md5("|".join(parts).encode(), usedforsecurity=False).hexdigest() + + +def _iso_date(value: Any) -> date | None: + """An ISO date from a fact's lexical value, or ``None``.""" + try: + return _to_date(date.fromisoformat(str(value).strip()[:10])) + except ValueError: + return None + + def _ensure_concept( mx: ModelXbrl, concepts: dict[str, Concept], diff --git a/xbrlkit/serve/README.md b/xbrlkit/serve/README.md index a6b931c..d407a77 100644 --- a/xbrlkit/serve/README.md +++ b/xbrlkit/serve/README.md @@ -56,6 +56,11 @@ A filing loaded without a taxonomy whose host has gone — every Danish ESEF report cites one — says that too: `missing_taxonomy` names the documents and counts the facts that are in the report and not in the model. +So does a filing whose facts the parse could not place as written: +`parse_report` counts the facts dropped, by reason and context id, and names +any context the instance never declared that was +[rebuilt from the filing's own dates](../parse/README.md#facts-the-walk-cannot-place). + To read the filing itself, parse it from EDGAR: - `--pure` does so by default (see below). diff --git a/xbrlkit/serve/tools.py b/xbrlkit/serve/tools.py index 55fae69..3b0631a 100644 --- a/xbrlkit/serve/tools.py +++ b/xbrlkit/serve/tools.py @@ -35,7 +35,16 @@ from xbrlkit.check import precision as _precision from xbrlkit.check import tolerance as _tolerance from xbrlkit.config import CONFIG, Config -from xbrlkit.model import Arc, Concept, Network, Period, Unit, XbrlFact, XbrlModel +from xbrlkit.model import ( + Arc, + Concept, + Network, + ParseReport, + Period, + Unit, + XbrlFact, + XbrlModel, +) from xbrlkit.serialize import classify_network, root_qname from xbrlkit.serve.session import ( FilingSession, @@ -485,6 +494,7 @@ def list_filings(session: FilingSession) -> dict[str, Any]: "read_from", "taxonomy", "missing_taxonomy", + "parse_report", "filing", "entity", "counts", @@ -653,6 +663,11 @@ def describe_filing( if lf.missing_taxonomy else {} ), + **( + {"parse_report": _describe_parse_report(model.parse_report)} + if model.parse_report + else {} + ), "filing": { "id": lf.id, "source": lf.source, @@ -747,6 +762,48 @@ def _describe_missing(missing: MissingTaxonomy) -> dict[str, Any]: } +def _describe_parse_report(report: ParseReport) -> dict[str, Any]: + """What the parse left out or rebuilt, so no answer is read as the whole.""" + out: dict[str, Any] = {} + notes: list[str] = [] + if report.dropped: + out["facts_dropped"] = report.dropped_facts + out["dropped"] = [ + { + "reason": d.reason, + "context": d.context_id, + "facts": d.facts, + "concepts": d.concepts, + } + for d in report.dropped + ] + notes.append( + "the dropped facts are in the report and not in this model: " + "`undeclared_context` names a context id the instance references and " + "never declares, `invalid_period` a context whose dates could not be read" + ) + if report.recovered_contexts: + out["facts_in_recovered_contexts"] = report.recovered_facts + out["recovered_contexts"] = [ + { + "context": r.context_id, + "start": r.start.isoformat(), + "end": r.end.isoformat(), + "facts": r.facts, + "period_from": r.period_from, + "entity_from": r.entity_from, + } + for r in report.recovered_contexts + ] + notes.append( + "a recovered context is referenced by the instance and never declared; " + "its period was read from the filing's own reporting-period facts, and " + "it carries no dimensions" + ) + out["note"] = "; ".join(notes) + "." + return out + + def _describe_taxonomy(taxonomy: TaxonomyEntry) -> dict[str, Any]: """Which entry point a taxonomy package was loaded from, and the others it offers — the choice stated, never silent."""