diff --git a/tests/test_parse_contexts.py b/tests/test_parse_contexts.py
new file mode 100644
index 0000000..41ae0fd
--- /dev/null
+++ b/tests/test_parse_contexts.py
@@ -0,0 +1,303 @@
+"""Facts the parse cannot place: counted, never dropped silently, and kept when
+the filing states the period of an undeclared context itself.
+
+The instances are synthetic and load offline: a three-concept schema and an
+instance whose facts name context ids it never declares, the pattern some
+Indian exchange results filings follow (``OneD`` for the quarter, ``FourD`` for
+the year to date).
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import date
+from pathlib import Path
+
+import pytest
+
+from xbrlkit.model import RECOVERED_CONTEXT, FilingMeta, XbrlModel
+from xbrlkit.parse import close, load_model, to_xbrl_model
+
+SCHEMA = """
+
+
+
+
+
+
+
+"""
+
+HEAD = """
+
+
+iso4217:INR
+"""
+
+
+def _context(context_id: str, entity: str, start: str, end: str) -> str:
+ return (
+ f''
+ f'{entity}'
+ f"{start}"
+ f"{end}\n"
+ )
+
+
+def _dates(context_id: str, start: str, end: str) -> str:
+ return (
+ f'{start}'
+ "\n"
+ f'{end}'
+ "\n"
+ )
+
+
+def _fact(name: str, context_id: str, value: int) -> str:
+ return (
+ f'{value}'
+ f"\n"
+ )
+
+
+# The quarter and the year to date, neither declared; one declared context
+# that dates a year-to-date breakdown as the quarter, as the real filings do.
+RESULTS = (
+ _context("FourDetailD", "ACME", "2022-10-01", "2022-12-31")
+ + _dates("OneD", "2022-10-01", "2022-12-31")
+ + _dates("FourD", "2022-04-01", "2022-12-31")
+ + _fact("Revenue", "OneD", 100)
+ + _fact("Revenue", "FourD", 300)
+ + _fact("Revenue", "FourDetailD", 300)
+)
+
+
+def _parse(tmp_path: Path, body: str) -> XbrlModel:
+ (tmp_path / "t.xsd").write_text(SCHEMA)
+ instance = tmp_path / "i.xml"
+ instance.write_text(HEAD + body + "\n")
+ mx = load_model(str(instance), cache_dir=tmp_path / "cache", offline=True)
+ try:
+ return to_xbrl_model(mx, FilingMeta(accession="acc", cik="0000000042"))
+ finally:
+ close(mx.modelManager.cntlr)
+
+
+def _by_context_value(model: XbrlModel) -> dict[str, tuple[date | None, date | None]]:
+ periods = {p.id: p for p in model.periods}
+ return {
+ f.value_str or "": (periods[f.period_id].start, periods[f.period_id].end)
+ for f in model.facts
+ if f.concept_qname == "t:Revenue" and f.provenance is not None
+ }
+
+
+def test_undeclared_contexts_are_rebuilt_from_the_filings_own_dates(tmp_path: Path):
+ model = _parse(tmp_path, RESULTS)
+
+ assert len(model.facts) == 7
+ assert _by_context_value(model) == {
+ "100": (date(2022, 10, 1), date(2022, 12, 31)),
+ "300": (date(2022, 4, 1), date(2022, 12, 31)),
+ }
+ report = model.parse_report
+ assert report is not None and report.dropped == []
+ assert [
+ (r.context_id, r.start, r.end, r.facts) for r in report.recovered_contexts
+ ] == [
+ ("OneD", date(2022, 10, 1), date(2022, 12, 31), 3),
+ ("FourD", date(2022, 4, 1), date(2022, 12, 31), 3),
+ ]
+ assert report.recovered_contexts[0].entity_from == "declared_contexts"
+ assert report.recovered_contexts[0].period_from == [
+ "t:DateOfStartOfReportingPeriod",
+ "t:DateOfEndOfReportingPeriod",
+ ]
+
+
+def test_recovered_facts_are_marked_and_declared_ones_are_not(tmp_path: Path):
+ model = _parse(tmp_path, RESULTS)
+
+ marked = [f for f in model.facts if f.provenance is not None]
+ declared = [f for f in model.facts if f.provenance is None]
+ assert len(marked) == 6 and len(declared) == 1
+ assert {f.provenance.kind for f in marked if f.provenance} == {RECOVERED_CONTEXT}
+ assert all(f.entity_identifier == "ACME" and f.dims == [] for f in marked)
+ # The declared context keeps the dates the filing gave it, right or wrong.
+ period = next(p for p in model.periods if p.id == declared[0].period_id)
+ assert (period.start, period.end) == (date(2022, 10, 1), date(2022, 12, 31))
+
+
+def test_recovered_facts_of_one_concept_get_distinct_ids(tmp_path: Path):
+ # Arelle hashes a fact with no context by its qname alone, so every orphan
+ # of one concept shares a hash — across contexts and within one.
+ model = _parse(tmp_path, RESULTS + _fact("Revenue", "OneD", 101))
+
+ ids = [f.id for f in model.facts]
+ assert len(set(ids)) == len(ids)
+ hashes = [f.source_hash for f in model.facts if f.provenance is not None]
+ assert len(set(hashes)) == len(hashes)
+
+
+def test_a_context_with_no_stated_period_is_dropped_and_counted(
+ tmp_path: Path, caplog: pytest.LogCaptureFixture
+):
+ body = (
+ _context("Declared", "ACME", "2022-10-01", "2022-12-31")
+ + _dates("OneD", "2022-10-01", "2022-12-31")
+ + _fact("Revenue", "OneD", 100)
+ + _fact("Revenue", "FourD", 300)
+ + _fact("Revenue", "FourD", 301)
+ + _fact("Revenue", "Declared", 5)
+ )
+ with caplog.at_level(logging.WARNING, logger="xbrlkit.parse.to_model"):
+ model = _parse(tmp_path, body)
+
+ report = model.parse_report
+ assert report is not None
+ assert [(d.reason, d.context_id, d.facts, d.concepts) for d in report.dropped] == [
+ ("undeclared_context", "FourD", 2, ["t:Revenue"])
+ ]
+ assert report.dropped_facts == 2 and report.recovered_facts == 3
+ assert len(model.facts) == 4
+ assert "2 fact(s) dropped (undeclared_context: 2)" in caplog.text
+ assert "rebuilt context(s) OneD" in caplog.text
+
+
+def test_half_a_stated_period_is_not_enough(tmp_path: Path):
+ body = (
+ '2022-12-31'
+ "\n" + _fact("Revenue", "OneD", 100)
+ )
+ model = _parse(tmp_path, body)
+
+ assert model.facts == []
+ assert model.parse_report is not None
+ assert model.parse_report.recovered_contexts == []
+ assert model.parse_report.dropped_facts == 2
+
+
+def test_conflicting_stated_dates_are_not_enough(tmp_path: Path):
+ body = (
+ _dates("OneD", "2022-10-01", "2022-12-31")
+ + _dates("OneD", "2022-07-01", "2022-12-31")
+ + _fact("Revenue", "OneD", 100)
+ )
+ model = _parse(tmp_path, body)
+
+ assert model.facts == []
+ assert model.parse_report is not None and model.parse_report.dropped_facts == 5
+
+
+def test_an_instant_concept_is_not_placed_in_a_rebuilt_duration(tmp_path: Path):
+ body = (
+ _dates("OneD", "2022-10-01", "2022-12-31")
+ + _fact("Revenue", "OneD", 100)
+ + _fact("Assets", "OneD", 900)
+ )
+ model = _parse(tmp_path, body)
+
+ assert "t:Assets" not in {f.concept_qname for f in model.facts}
+ report = model.parse_report
+ assert report is not None
+ assert [(d.context_id, d.facts, d.concepts) for d in report.dropped] == [
+ ("OneD", 1, ["t:Assets"])
+ ]
+
+
+def test_declared_contexts_that_disagree_on_the_entity_rebuild_nothing(tmp_path: Path):
+ body = (
+ _context("A", "ACME", "2022-10-01", "2022-12-31")
+ + _context("B", "OTHER", "2022-10-01", "2022-12-31")
+ + _dates("OneD", "2022-10-01", "2022-12-31")
+ + _fact("Revenue", "OneD", 100)
+ + _fact("Revenue", "A", 1)
+ + _fact("Revenue", "B", 2)
+ )
+ model = _parse(tmp_path, body)
+
+ assert len(model.facts) == 2
+ report = model.parse_report
+ assert report is not None and report.recovered_contexts == []
+ assert report.dropped_facts == 3
+
+
+def test_with_no_declared_context_the_filings_identifier_stands_in(tmp_path: Path):
+ model = _parse(
+ tmp_path, _dates("OneD", "2022-10-01", "2022-12-31") + _fact("Revenue", "OneD", 100)
+ )
+
+ assert len(model.facts) == 3
+ assert {f.entity_cik for f in model.facts} == {"0000000042"}
+ assert {f.entity_scheme for f in model.facts} == {None}
+ assert model.parse_report is not None
+ assert model.parse_report.recovered_contexts[0].entity_from == "filing"
+
+
+def test_a_filing_that_declares_its_contexts_has_no_parse_report(
+ tmp_path: Path, caplog: pytest.LogCaptureFixture
+):
+ body = _context("OneD", "ACME", "2022-10-01", "2022-12-31") + _fact(
+ "Revenue", "OneD", 100
+ )
+ with caplog.at_level(logging.WARNING, logger="xbrlkit.parse.to_model"):
+ model = _parse(tmp_path, body)
+
+ assert len(model.facts) == 1 and model.facts[0].provenance is None
+ assert model.parse_report is None
+ assert caplog.text == ""
+
+
+def test_the_recovered_mark_travels_through_holon_and_tavi(tmp_path: Path):
+ from xbrlkit.deserialize.holon import from_holon_json
+ from xbrlkit.deserialize.tavi import from_tavi_json
+ from xbrlkit.serialize import to_holon, to_tavi
+
+ model = _parse(tmp_path, RESULTS)
+
+ for back in (from_holon_json(to_holon(model)), from_tavi_json(to_tavi(model))):
+ kinds = [f.provenance.kind if f.provenance else None for f in back.facts]
+ assert kinds.count(RECOVERED_CONTEXT) == 6 and kinds.count(None) == 1
+
+
+def test_the_load_receipt_and_describe_filing_say_what_the_parse_did(tmp_path: Path):
+ from xbrlkit.serve import tools
+ from xbrlkit.serve.session import FilingSession
+
+ (tmp_path / "t.xsd").write_text(SCHEMA)
+ instance = tmp_path / "i.xml"
+ instance.write_text(
+ HEAD
+ + _context("Declared", "ACME", "2022-10-01", "2022-12-31")
+ + _dates("OneD", "2022-10-01", "2022-12-31")
+ + _fact("Revenue", "OneD", 100)
+ + _fact("Revenue", "FourD", 300)
+ + _fact("Revenue", "Declared", 5)
+ + "\n"
+ )
+ loaded = FilingSession().load(str(instance))
+
+ for payload in (tools.load_receipt(loaded), tools.describe_filing(loaded)):
+ report = payload["parse_report"]
+ assert report["facts_dropped"] == 1
+ assert report["dropped"] == [
+ {
+ "reason": "undeclared_context",
+ "context": "FourD",
+ "facts": 1,
+ "concepts": ["t:Revenue"],
+ }
+ ]
+ assert report["facts_in_recovered_contexts"] == 3
+ assert report["recovered_contexts"][0]["context"] == "OneD"
+ assert report["recovered_contexts"][0]["end"] == "2022-12-31"
diff --git a/xbrlkit/model.py b/xbrlkit/model.py
index 533f936..c88de1c 100644
--- a/xbrlkit/model.py
+++ b/xbrlkit/model.py
@@ -324,6 +324,8 @@ class XbrlFact(BaseModel):
section_role: str | None = None
# Where the fact came from, when the producer said. A filing leaves it
# ``None``; an authored report may carry it, and both projections keep it.
+ # The parse sets it in one case: a fact whose context the instance never
+ # declared and the parse rebuilt carries ``kind="recovered_context"``.
provenance: FactProvenance | None = None
@@ -387,6 +389,63 @@ class Network(BaseModel):
fact_set_id: str | None = None
+# ``FactProvenance.kind`` of a fact whose context the parse rebuilt.
+RECOVERED_CONTEXT = "recovered_context"
+
+DropReason = Literal["undeclared_context", "undefined_concept", "invalid_period"]
+
+
+class DroppedFacts(BaseModel):
+ """Facts in the instance that are not in the model, for one reason and context."""
+
+ reason: DropReason
+ # The ``contextRef`` the facts name; for ``undeclared_context`` an id the
+ # instance never declares.
+ context_id: str | None = None
+ facts: int = 0
+ # A sample of the concepts lost, in instance order.
+ concepts: list[str] = Field(default_factory=list)
+
+
+class RecoveredContext(BaseModel):
+ """A context the instance references but never declares, rebuilt by the parse.
+
+ The period is read from the filing's own reporting-period date facts under
+ that context id, never borrowed from another context. The facts placed in
+ it carry ``provenance.kind == "recovered_context"``.
+ """
+
+ context_id: str
+ start: date
+ end: date
+ period_id: str
+ facts: int = 0
+ # The two facts the period was read from.
+ period_from: list[str] = Field(default_factory=list)
+ # ``declared_contexts`` when every declared context names one entity;
+ # ``filing`` when the instance declares none and the caller's id stands in.
+ entity_from: Literal["declared_contexts", "filing"]
+
+
+class ParseReport(BaseModel):
+ """What the parse could not place as the instance wrote it.
+
+ ``None`` on a model means the parse had nothing to report, or the model was
+ not parsed from XBRL at all.
+ """
+
+ dropped: list[DroppedFacts] = Field(default_factory=list)
+ recovered_contexts: list[RecoveredContext] = Field(default_factory=list)
+
+ @property
+ def dropped_facts(self) -> int:
+ return sum(d.facts for d in self.dropped)
+
+ @property
+ def recovered_facts(self) -> int:
+ return sum(r.facts for r in self.recovered_contexts)
+
+
class XbrlModel(BaseModel):
"""The whole filing as neutral objects — the contract between parse and serialize."""
@@ -397,3 +456,5 @@ class XbrlModel(BaseModel):
units: list[Unit] = Field(default_factory=list)
facts: list[XbrlFact] = Field(default_factory=list)
networks: list[Network] = Field(default_factory=list)
+ # Facts the parse dropped or placed in a rebuilt context; ``None`` when none.
+ parse_report: ParseReport | None = None
diff --git a/xbrlkit/parse/README.md b/xbrlkit/parse/README.md
index 52f42e3..0654917 100644
--- a/xbrlkit/parse/README.md
+++ b/xbrlkit/parse/README.md
@@ -36,6 +36,38 @@ so all three have to agree exactly.
every qname the slice touches — abstract headers, subtotals, axes, domains,
members — so labels and structural flags are available for all of them.
+## Facts the walk cannot place
+
+A fact is left out of the model for three reasons, and each is counted on
+`model.parse_report` (by reason and context id, with a sample of the concepts)
+and logged as a warning, never dropped silently:
+
+| Reason | What happened |
+| --- | --- |
+| `undeclared_context` | The fact's `contextRef` names an id the instance never declares. Arelle loads the fact with no context |
+| `undefined_concept` | The fact has no concept behind it |
+| `invalid_period` | The context's dates could not be read |
+
+`parse_report` is `None` when there was nothing to report.
+
+**An undeclared context is rebuilt when the filing states its period itself.**
+Some Indian exchange results filings put most of their facts under `OneD` (the
+quarter) and `FourD` (the year to date) and declare neither, but report
+`DateOfStartOfReportingPeriod` and `DateOfEndOfReportingPeriod` under each. When
+exactly one start and one end date sit under an undeclared id, the parse builds
+a duration from them and keeps the facts. The rules are narrow on purpose:
+
+- The period comes only from those two facts, never from a declared sibling
+ context: the same filings date some of those wrongly.
+- The entity is the one every declared context names. If they disagree, nothing
+ is rebuilt; if the instance declares none, the caller's filer id stands in.
+- Only duration concepts are placed, and a rebuilt context has no dimensions.
+
+A fact kept this way carries `provenance.kind == "recovered_context"`, which
+the holon and TAVI projections write and their readers restore, and the
+context is listed in `parse_report.recovered_contexts`. Arelle hashes a fact
+with its context, so these facts' ids fold in the context id.
+
## The Arelle cache
Arelle resolves a filing's DTS by fetching every schema and linkbase it
diff --git a/xbrlkit/parse/to_model.py b/xbrlkit/parse/to_model.py
index e2ad756..7f86e66 100644
--- a/xbrlkit/parse/to_model.py
+++ b/xbrlkit/parse/to_model.py
@@ -12,25 +12,39 @@
- Arelle stores instant/end dates as the *exclusive* next midnight, so every
instant/end date is rolled back one day (``- timedelta(1)``).
- Measures resolve to prefixed tokens (``iso4217:USD``) from the QName.
+
+A fact the walk cannot place is never dropped silently: it is counted on the
+model's ``parse_report`` by reason, and a fact under a context the instance
+references but never declares is kept when the filing states that context's
+period itself (see :func:`_recoverable_contexts`).
"""
from __future__ import annotations
+import hashlib
+import logging
+from dataclasses import dataclass, field
from datetime import date, datetime, timedelta
from typing import TYPE_CHECKING, Any
from arelle import XbrlConst
from xbrlkit.model import (
+ RECOVERED_CONTEXT,
Arc,
Concept,
DimQualifier,
+ DroppedFacts,
+ DropReason,
EntityIdentity,
+ FactProvenance,
FilingMeta,
Label,
Network,
NetworkKind,
+ ParseReport,
Period,
+ RecoveredContext,
Reference,
Unit,
XbrlFact,
@@ -42,6 +56,8 @@
if TYPE_CHECKING:
from arelle.ModelXbrl import ModelXbrl
+logger = logging.getLogger(__name__)
+
# Extended-link roles that carry no statement/disclosure network — the label
# and reference linkbases (standard link role) plus the enumeration-list roles.
ROLES_FILTERED = {
@@ -54,6 +70,17 @@
_DEI_FISCAL_PERIOD = "dei:DocumentFiscalPeriodFocus"
_DEI_FISCAL_YEAR_END = "dei:CurrentFiscalYearEndDate"
+# Concept local names (start, end) under which a filing states the period of
+# the context they sit in. The Indian exchanges' results taxonomies report
+# these under each column's context, including the ones some filings never
+# declare.
+_REPORTING_PERIOD_FACTS = (
+ ("DateOfStartOfReportingPeriod", "DateOfEndOfReportingPeriod"),
+)
+
+# Concepts sampled per dropped group on the parse report.
+_DROPPED_CONCEPT_SAMPLE = 5
+
def to_xbrl_model(
mx: ModelXbrl,
@@ -86,11 +113,23 @@ def to_xbrl_model(
fiscal_period_focus: str | None = None
fiscal_year_end_month: str | None = None
+ tally = _ParseTally()
+ rebuilt_contexts = _recoverable_contexts(mx, main_cik)
+
for f in mx.facts:
- if f.context is None:
- continue
+ context = f.context
+ context_id = getattr(f, "contextID", None)
concept = f.concept
+ rebuilt: _RebuiltContext | None = None
+ if context is None:
+ # A rebuilt context is a duration; an instant concept cannot sit in it.
+ if getattr(concept, "periodType", None) == "duration":
+ rebuilt = rebuilt_contexts.get(context_id or "")
+ if rebuilt is None:
+ tally.drop("undeclared_context", context_id, f)
+ continue
if concept is None or concept.qname is None:
+ tally.drop("undefined_concept", context_id, f)
continue
qname_str = str(concept.qname)
@@ -112,8 +151,9 @@ def to_xbrl_model(
_ensure_concept(mx, concepts, namespaces, concept)
# Period (deduped by content-derived id). Skip facts with invalid dates.
- period = _make_period(f.context)
+ period = rebuilt.period if rebuilt else _make_period(context)
if period is None:
+ tally.drop("invalid_period", context_id, f)
continue
if period.id not in periods:
periods[period.id] = period
@@ -128,7 +168,11 @@ def to_xbrl_model(
unit_ref = unit.id
# Entity identity (prefer the context whose CIK is the filer's).
- scheme, raw_cik = f.context.entityIdentifier
+ if rebuilt:
+ scheme, raw_cik = rebuilt.scheme, rebuilt.identifier
+ rebuilt.facts += 1
+ else:
+ scheme, raw_cik = context.entityIdentifier
norm_cik = _normalize_cik(raw_cik)
if entity_scheme is None or norm_cik == main_cik:
entity_scheme = scheme
@@ -142,27 +186,39 @@ def to_xbrl_model(
numeric_value = None
raw_value = getattr(f, "value", None)
+ source_hash = _md5(f)
+ if rebuilt:
+ source_hash = _orphan_hash(f, source_hash, context_id, unit_ref)
facts.append(
XbrlFact(
- id=fact_id(report_uri, f.md5sum.value),
+ id=fact_id(report_uri, source_hash if rebuilt else f.md5sum.value),
concept_qname=qname_str,
period_id=period.id,
unit_id=unit_ref,
entity_cik=norm_cik,
entity_scheme=scheme,
entity_identifier=str(raw_cik),
- dims=_make_dims(f.context, mx, concepts, namespaces),
+ dims=[] if rebuilt else _make_dims(context, mx, concepts, namespaces),
value_str=_value_str(f),
raw_value=str(raw_value) if raw_value is not None else None,
- source_hash=_md5(f),
+ source_hash=source_hash,
numeric_value=numeric_value,
decimals=(str(f.decimals) if (is_numeric and f.decimals is not None) else None),
value_kind="numeric" if is_numeric else "text",
language=getattr(f, "xmlLang", None),
is_nil=bool(getattr(f, "isNil", False)),
+ provenance=(
+ FactProvenance(kind=RECOVERED_CONTEXT, attributed_to="xbrlkit")
+ if rebuilt
+ else None
+ ),
)
)
+ parse_report = tally.report(rebuilt_contexts)
+ if parse_report is not None:
+ logger.warning("%s: %s", report_uri, _describe_report(parse_report))
+
# Networks last: their arc endpoints (abstract headers, subtotals, hypercube
# wiring) complete the concept coverage and add their namespaces.
networks = _make_networks(mx, concepts, namespaces)
@@ -204,9 +260,175 @@ def to_xbrl_model(
units=list(units.values()),
facts=facts,
networks=networks,
+ parse_report=parse_report,
)
+@dataclass
+class _RebuiltContext:
+ """An undeclared context's period and entity, as the parse rebuilt them."""
+
+ period: Period
+ start: date
+ end: date
+ scheme: str | None
+ identifier: str
+ period_from: list[str]
+ entity_from: str
+ facts: int = 0
+
+
+@dataclass
+class _ParseTally:
+ """Counts the facts the walk leaves out, by reason and context id."""
+
+ groups: dict[tuple[DropReason, str | None], DroppedFacts] = field(
+ default_factory=dict
+ )
+
+ def drop(self, reason: DropReason, context_id: str | None, fact: Any) -> None:
+ group = self.groups.setdefault(
+ (reason, context_id), DroppedFacts(reason=reason, context_id=context_id)
+ )
+ group.facts += 1
+ qname = str(getattr(fact, "qname", None) or "")
+ if (
+ qname
+ and qname not in group.concepts
+ and len(group.concepts) < _DROPPED_CONCEPT_SAMPLE
+ ):
+ group.concepts.append(qname)
+
+ def report(self, rebuilt: dict[str, _RebuiltContext]) -> ParseReport | None:
+ recovered = [
+ RecoveredContext(
+ context_id=context_id,
+ start=ctx.start,
+ end=ctx.end,
+ period_id=ctx.period.id,
+ facts=ctx.facts,
+ period_from=ctx.period_from,
+ entity_from=ctx.entity_from, # pyright: ignore[reportArgumentType]
+ )
+ for context_id, ctx in rebuilt.items()
+ if ctx.facts
+ ]
+ if not self.groups and not recovered:
+ return None
+ return ParseReport(dropped=list(self.groups.values()), recovered_contexts=recovered)
+
+
+def _describe_report(report: ParseReport) -> str:
+ """One line for the log: what was dropped and what was rebuilt."""
+ parts: list[str] = []
+ if report.dropped:
+ by_reason: dict[str, int] = {}
+ for group in report.dropped:
+ by_reason[group.reason] = by_reason.get(group.reason, 0) + group.facts
+ parts.append(
+ f"{report.dropped_facts} fact(s) dropped ("
+ + ", ".join(f"{reason}: {count}" for reason, count in by_reason.items())
+ + ")"
+ )
+ if report.recovered_contexts:
+ parts.append(
+ f"{report.recovered_facts} fact(s) placed in rebuilt context(s) "
+ + ", ".join(r.context_id for r in report.recovered_contexts)
+ )
+ return "; ".join(parts)
+
+
+def _recoverable_contexts(mx: ModelXbrl, filing_id: str) -> dict[str, _RebuiltContext]:
+ """Rebuild the contexts the instance references but never declares.
+
+ Some filings name a context id on most of their facts and declare no
+ ```` for it; Arelle loads those facts with no context. A
+ context is rebuilt only when the filing states its period itself: exactly
+ one start date and one end date reported under that same id (see
+ ``_REPORTING_PERIOD_FACTS``). The period is never taken from a declared
+ sibling context — the same filings date some of those wrongly.
+
+ The entity is the one every declared context names. When the declared
+ contexts disagree nothing is rebuilt; when the instance declares none,
+ ``filing_id`` (the caller's identifier for the filer) stands in, with no
+ scheme. A rebuilt context has no dimensions.
+ """
+ orphans = [f for f in mx.facts if f.context is None and getattr(f, "contextID", None)]
+ if not orphans:
+ return {}
+
+ entities = {c.entityIdentifier for c in (mx.contexts or {}).values()}
+ if len(entities) > 1:
+ return {}
+ if entities:
+ scheme, identifier = next(iter(entities))
+ entity_from = "declared_contexts"
+ else:
+ scheme, identifier, entity_from = None, filing_id, "filing"
+
+ stated: dict[str, dict[str, dict[date, str]]] = {}
+ for f in orphans:
+ qname = getattr(f, "qname", None)
+ local = getattr(qname, "localName", None)
+ if not any(local in pair for pair in _REPORTING_PERIOD_FACTS):
+ continue
+ value = _iso_date(getattr(f, "value", None))
+ if value is not None:
+ stated.setdefault(f.contextID, {}).setdefault(local, {})[value] = str(qname)
+
+ rebuilt: dict[str, _RebuiltContext] = {}
+ for context_id, by_name in stated.items():
+ for start_name, end_name in _REPORTING_PERIOD_FACTS:
+ starts, ends = by_name.get(start_name, {}), by_name.get(end_name, {})
+ if len(starts) != 1 or len(ends) != 1:
+ continue
+ (start, start_qname), (end, end_qname) = (
+ next(iter(starts.items())),
+ next(iter(ends.items())),
+ )
+ if start > end:
+ continue
+ rebuilt[context_id] = _RebuiltContext(
+ period=duration_period(start, end),
+ start=start,
+ end=end,
+ scheme=scheme,
+ identifier=str(identifier),
+ period_from=[start_qname, end_qname],
+ entity_from=entity_from,
+ )
+ break
+ return rebuilt
+
+
+def _orphan_hash(
+ fact: Any, arelle_hash: str | None, context_id: str | None, unit_ref: str | None
+) -> str:
+ """A hash for a fact Arelle loaded with no context.
+
+ Arelle hashes such a fact by its qname alone — the value, language, nil and
+ unit only enter beside a context — so every orphan of one concept shares a
+ hash. This folds in what Arelle would have, with the context id standing
+ where the context's own hash goes.
+ """
+ parts = [
+ arelle_hash or "",
+ context_id or "",
+ getattr(fact, "xmlLang", None) or "",
+ "nil" if getattr(fact, "isNil", False) else str(getattr(fact, "value", "") or ""),
+ unit_ref or "",
+ ]
+ return hashlib.md5("|".join(parts).encode(), usedforsecurity=False).hexdigest()
+
+
+def _iso_date(value: Any) -> date | None:
+ """An ISO date from a fact's lexical value, or ``None``."""
+ try:
+ return _to_date(date.fromisoformat(str(value).strip()[:10]))
+ except ValueError:
+ return None
+
+
def _ensure_concept(
mx: ModelXbrl,
concepts: dict[str, Concept],
diff --git a/xbrlkit/serve/README.md b/xbrlkit/serve/README.md
index a6b931c..d407a77 100644
--- a/xbrlkit/serve/README.md
+++ b/xbrlkit/serve/README.md
@@ -56,6 +56,11 @@ A filing loaded without a taxonomy whose host has gone — every Danish ESEF
report cites one — says that too: `missing_taxonomy` names the documents and
counts the facts that are in the report and not in the model.
+So does a filing whose facts the parse could not place as written:
+`parse_report` counts the facts dropped, by reason and context id, and names
+any context the instance never declared that was
+[rebuilt from the filing's own dates](../parse/README.md#facts-the-walk-cannot-place).
+
To read the filing itself, parse it from EDGAR:
- `--pure` does so by default (see below).
diff --git a/xbrlkit/serve/tools.py b/xbrlkit/serve/tools.py
index 55fae69..3b0631a 100644
--- a/xbrlkit/serve/tools.py
+++ b/xbrlkit/serve/tools.py
@@ -35,7 +35,16 @@
from xbrlkit.check import precision as _precision
from xbrlkit.check import tolerance as _tolerance
from xbrlkit.config import CONFIG, Config
-from xbrlkit.model import Arc, Concept, Network, Period, Unit, XbrlFact, XbrlModel
+from xbrlkit.model import (
+ Arc,
+ Concept,
+ Network,
+ ParseReport,
+ Period,
+ Unit,
+ XbrlFact,
+ XbrlModel,
+)
from xbrlkit.serialize import classify_network, root_qname
from xbrlkit.serve.session import (
FilingSession,
@@ -485,6 +494,7 @@ def list_filings(session: FilingSession) -> dict[str, Any]:
"read_from",
"taxonomy",
"missing_taxonomy",
+ "parse_report",
"filing",
"entity",
"counts",
@@ -653,6 +663,11 @@ def describe_filing(
if lf.missing_taxonomy
else {}
),
+ **(
+ {"parse_report": _describe_parse_report(model.parse_report)}
+ if model.parse_report
+ else {}
+ ),
"filing": {
"id": lf.id,
"source": lf.source,
@@ -747,6 +762,48 @@ def _describe_missing(missing: MissingTaxonomy) -> dict[str, Any]:
}
+def _describe_parse_report(report: ParseReport) -> dict[str, Any]:
+ """What the parse left out or rebuilt, so no answer is read as the whole."""
+ out: dict[str, Any] = {}
+ notes: list[str] = []
+ if report.dropped:
+ out["facts_dropped"] = report.dropped_facts
+ out["dropped"] = [
+ {
+ "reason": d.reason,
+ "context": d.context_id,
+ "facts": d.facts,
+ "concepts": d.concepts,
+ }
+ for d in report.dropped
+ ]
+ notes.append(
+ "the dropped facts are in the report and not in this model: "
+ "`undeclared_context` names a context id the instance references and "
+ "never declares, `invalid_period` a context whose dates could not be read"
+ )
+ if report.recovered_contexts:
+ out["facts_in_recovered_contexts"] = report.recovered_facts
+ out["recovered_contexts"] = [
+ {
+ "context": r.context_id,
+ "start": r.start.isoformat(),
+ "end": r.end.isoformat(),
+ "facts": r.facts,
+ "period_from": r.period_from,
+ "entity_from": r.entity_from,
+ }
+ for r in report.recovered_contexts
+ ]
+ notes.append(
+ "a recovered context is referenced by the instance and never declared; "
+ "its period was read from the filing's own reporting-period facts, and "
+ "it carries no dimensions"
+ )
+ out["note"] = "; ".join(notes) + "."
+ return out
+
+
def _describe_taxonomy(taxonomy: TaxonomyEntry) -> dict[str, Any]:
"""Which entry point a taxonomy package was loaded from, and the others it
offers — the choice stated, never silent."""