From f3397106d3645069605a44a4bbe9df2c59b4b202 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Tue, 21 Jul 2026 13:37:41 -0500 Subject: [PATCH 1/4] updated to ignore the bank and only pick up the actual matched line --- .../plugins/inband/dmesg/dmesg_analyzer.py | 12 +- nodescraper/plugins/inband/dmesg/mce_utils.py | 138 ++++++------------ test/unit/plugin/test_dmesg_analyzer.py | 87 +++++++++-- test/unit/plugin/test_mce_utils.py | 61 +++++++- 4 files changed, 185 insertions(+), 113 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 04c8737b..5ae53f77 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -38,8 +38,8 @@ from .mce_utils import ( compile_mce_ce_status_regex, compile_mce_uc_status_regex, - ignored_mce_block_line_indices, - mce_block_all_line_indices, + mce_known_regex_skip_line_indices, + mce_unknown_suppress_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, trim_mce_status_match_content, @@ -718,8 +718,8 @@ def analyze_data( dmesg_content = data.dmesg_content ignore_match_rules, ignore_mce_banks = parse_ignore_match_rules(args.ignore_match_rules) - ignored_mce_block_lines = ignored_mce_block_line_indices(dmesg_content, ignore_mce_banks) - mce_block_lines = mce_block_all_line_indices(dmesg_content) + known_skip_lines = mce_known_regex_skip_line_indices(dmesg_content, ignore_mce_banks) + unknown_skip_lines = mce_unknown_suppress_line_indices(dmesg_content) known_err_events = self.check_all_regexes( content=dmesg_content, @@ -728,7 +728,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, - skip_line_indices=ignored_mce_block_lines, + skip_line_indices=known_skip_lines, ) for event in known_err_events: if event.description in ("MCE Corrected Error", "MCE Uncorrected Error"): @@ -764,7 +764,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, - skip_line_indices=mce_block_lines, + skip_line_indices=unknown_skip_lines, ) for err_event in err_events: diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index a8eeccf3..55222aaa 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -35,29 +35,6 @@ _MCE_STATUS_START_RE = re.compile(r"\bMC\d+_STATUS\[", re.IGNORECASE) _MCE_DETAIL_LINE_RE = re.compile(r"\[Hardware Error\]:") -_CORRECTABLE_SUMMARY_RE = re.compile( - r"(?P\d+)\s+correctable hardware errors detected in total in (?P\w+) block" - r"(?:\s+on\s+(?PCPU:?\d+))?", - re.IGNORECASE, -) - -_UNCORRECTABLE_SUMMARY_RE = re.compile( - r"(?P\d+)\s+uncorrectable hardware errors detected in (?P\w+) block", - re.IGNORECASE, -) - -_GPU_CORRECTABLE_RE = re.compile( - r"amdgpu\s+(?P[\w:.]+):.*?(?P\d+)\s+correctable hardware errors detected in total in " - r"(?P\w+) block", - re.IGNORECASE, -) - -_GPU_UNCORRECTABLE_RE = re.compile( - r"amdgpu\s+(?P[\w:.]+):.*?(?P\d+)\s+uncorrectable hardware errors detected in " - r"(?P\w+) block", - re.IGNORECASE, -) - _MCE_CE_STATUS_RE = re.compile( r"\[Hardware Error\]:.*?(?PCPU:?\d+).*?MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\]", re.IGNORECASE, @@ -116,33 +93,6 @@ def _add_count(counts: dict[str, int], part: str, amount: int) -> None: counts[part] = counts.get(part, 0) + amount -def _part_label( - *, - cpu: Optional[str] = None, - block: Optional[str] = None, - bdf: Optional[str] = None, - gpu_index: Optional[int] = None, -) -> str: - if bdf is not None: - block_suffix = f"/{block}" if block else "" - if gpu_index is not None: - return f"GPU{gpu_index}{block_suffix}" - return f"GPU {bdf}{block_suffix}" - if cpu and block: - return f"{cpu}/{block}" - if cpu: - return cpu - if block: - return block - return "unknown" - - -def _gpu_index_for_bdf(bdf: str, bdf_order: list[str]) -> int: - if bdf not in bdf_order: - bdf_order.append(bdf) - return bdf_order.index(bdf) - - def _is_mce_primary_starter(line: str) -> bool: return _MCE_PRIMARY_START_RE.search(line) is not None @@ -192,6 +142,48 @@ def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: return False +def mce_defining_status_line_indices(content: str) -> frozenset[int]: + """Return line indices for MCn_STATUS rows that define a corrected or uncorrected MCE.""" + lines = content.splitlines() + indices: set[int] = set() + for index, line in enumerate(lines): + if _MCE_CE_STATUS_LINE_RE.search(line) or _MCE_UC_STATUS_LINE_RE.search(line): + indices.add(index) + return frozenset(indices) + + +def mce_hardware_error_line_indices(content: str) -> frozenset[int]: + """Return every line index containing [Hardware Error]:.""" + lines = content.splitlines() + return frozenset(index for index, line in enumerate(lines) if _is_mce_detail_line(line)) + + +def mce_non_status_hardware_error_line_indices(content: str) -> frozenset[int]: + """Return [Hardware Error]: detail lines that are not defining MCn_STATUS CE/UC rows.""" + defining = mce_defining_status_line_indices(content) + return frozenset( + index for index in mce_hardware_error_line_indices(content) if index not in defining + ) + + +def mce_known_regex_skip_line_indices( + content: str, + ignore_banks: Optional[FrozenSet[int]] = None, +) -> frozenset[int]: + """Skip non-defining MCE detail lines and ignored-bank incidents during known regex scan.""" + ignored = ignore_banks or frozenset() + skipped = set(mce_non_status_hardware_error_line_indices(content)) + skipped.update(ignored_mce_block_line_indices(content, ignored)) + return frozenset(skipped) + + +def mce_unknown_suppress_line_indices(content: str) -> frozenset[int]: + """Suppress MCE block context and every [Hardware Error]: line from unknown scan.""" + suppressed = set(mce_block_all_line_indices(content)) + suppressed.update(mce_hardware_error_line_indices(content)) + return frozenset(suppressed) + + def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, int]]: """Return (start, end) line index ranges for MCE incident blocks. @@ -264,39 +256,14 @@ def parse_correctable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, ) -> dict[str, int]: - """Count correctable MCE / RAS hardware errors per component from dmesg text. - - Handles summary lines (for example ``mce: 3 correctable ... on CPU1``), - amdgpu block summaries, and per-event ``MCn_STATUS[|CE|]`` hardware error lines. - """ + """Count correctable MCE hardware errors per CPU from MCn_STATUS[|CE|] rows.""" counts: dict[str, int] = {} - gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() ignored_block_lines = ignored_mce_block_line_indices(content, ignored) for line_no, line in enumerate(content.splitlines()): if line_no in ignored_block_lines: continue - gpu_match = _GPU_CORRECTABLE_RE.search(line) - if gpu_match: - bdf = gpu_match.group("bdf") - part = _part_label( - bdf=bdf, - block=gpu_match.group("block"), - gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order), - ) - _add_count(counts, part, int(gpu_match.group("count"))) - continue - - summary_match = _CORRECTABLE_SUMMARY_RE.search(line) - if summary_match: - cpu = summary_match.group("cpu") - part = _part_label( - cpu=_normalize_cpu_label(cpu) if cpu else None, - block=summary_match.group("block"), - ) - _add_count(counts, part, int(summary_match.group("count"))) - continue status_match = _MCE_CE_STATUS_RE.search(line) if status_match: @@ -314,31 +281,14 @@ def parse_uncorrectable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, ) -> dict[str, int]: - """Count uncorrectable MCE / RAS hardware errors per component from dmesg text.""" + """Count uncorrectable MCE hardware errors per CPU from MCn_STATUS[|UC|] rows.""" counts: dict[str, int] = {} - gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() ignored_block_lines = ignored_mce_block_line_indices(content, ignored) for line_no, line in enumerate(content.splitlines()): if line_no in ignored_block_lines: continue - gpu_match = _GPU_UNCORRECTABLE_RE.search(line) - if gpu_match: - bdf = gpu_match.group("bdf") - part = _part_label( - bdf=bdf, - block=gpu_match.group("block"), - gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order), - ) - _add_count(counts, part, int(gpu_match.group("count"))) - continue - - summary_match = _UNCORRECTABLE_SUMMARY_RE.search(line) - if summary_match: - part = _part_label(block=summary_match.group("block")) - _add_count(counts, part, int(summary_match.group("count"))) - continue status_match = _MCE_UC_STATUS_RE.search(line) if status_match: diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index 87477acf..9b009785 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -1022,11 +1022,11 @@ def test_priority_override_updates_unkown_dmesg_error(system_info): assert res.events[0].priority == EventPriority.ERROR -def test_mce_threshold_raises_error_for_gpu(system_info): +def test_mce_threshold_raises_error_for_status_lines(system_info): dmesg_content = ( - "kern :err : 2024-10-07T10:17:15,145363-04:00 " - "amdgpu 0000:c1:00.0: amdgpu: socket: 4, die: 0 " - "3 correctable hardware errors detected in total in gfx block\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x2\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x3\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1038,7 +1038,7 @@ def test_mce_threshold_raises_error_for_gpu(system_info): threshold_events = [e for e in res.events if e.data.get("mce_threshold") == 3] assert len(threshold_events) == 1 assert threshold_events[0].priority == EventPriority.ERROR - assert threshold_events[0].data["part"] == "GPU0/gfx" + assert threshold_events[0].data["part"] == "CPU0" assert threshold_events[0].data["correctable_mce_count"] == 3 assert res.status == ExecutionStatus.ERROR @@ -1065,8 +1065,8 @@ def test_mce_threshold_raises_error_for_cpu_colon_status(system_info): def test_mce_threshold_not_triggered_below_limit(system_info): dmesg_content = ( - "kern :warn : 2024-06-11T14:30:00,123456+00:00 " - "mce: 2 correctable hardware errors detected in total in mc0 block on CPU1\n" + "[Hardware Error]: CPU1 MC0_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU1 MC0_STATUS[0x0|CE|]: 0x2\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1260,7 +1260,11 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" - "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1270,8 +1274,14 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): ) descriptions = {event.description for event in res.events} + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + mce_events = [event for event in res.events if event.description == "MCE Corrected Error"] + assert unknown_events == [] + assert len(mce_events) == 1 + assert "MC60_STATUS" in str(mce_events[0].data["match_content"]) assert "Unknown dmesg error" not in descriptions - assert "MCE Corrected Error" in descriptions or "RAS Corrected Error" in descriptions + assert "MCE Corrected Error" in descriptions + assert "RAS Corrected Error" not in descriptions def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info): @@ -1328,6 +1338,65 @@ def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info) assert unknown_events[0].data["match_content"] == "dummy harness fault outside mce blocks" +def test_orphan_mce_detail_lines_not_reported_as_unknown(system_info): + """When analysis window omits MCn_STATUS, trailing detail lines must not become unknowns.""" + dmesg_content = ( + "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " + "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" + "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs(check_unknown_dmesg_errors=True), + ) + + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + assert len(unknown_events) == 1 + assert unknown_events[0].data["match_content"] == "unrelated plugin failure" + + +def test_filtered_window_orphan_mce_tail_not_unknown(system_info): + """i198-maxcorestim: analysis window can start after MCn_STATUS but before IPID/cache tail.""" + dmesg_content = ( + "kern :info : 2026-07-14T22:21:10,254124-07:00 mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2026-07-14T22:21:10,266984-07:00 [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2026-07-14T22:21:10,280615-07:00 " + "[Hardware Error]: CPU:56 (1a:51:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xd8202000000c080b\n" + "kern :emerg : 2026-07-14T22:21:10,303837-07:00 [Hardware Error]: PPIN: 0x00831e03ffcb8015\n" + "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " + "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" + "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + ) + analysis_range_start = datetime.datetime.fromisoformat( + "2026-07-14 22:21:10.314-07:00" + ).astimezone(datetime.timezone.utc) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=True, + analysis_range_start=analysis_range_start, + ), + ) + + filtered_artifact = next( + artifact for artifact in res.artifacts if artifact.filename == "filtered_dmesg.log" + ) + assert "MC60_STATUS" not in filtered_artifact.contents + assert "IPID:" in filtered_artifact.contents + + descriptions = {event.description for event in res.events} + assert "Unknown dmesg error" not in descriptions + assert "MCE Corrected Error" not in descriptions + assert "RAS Corrected Error" not in descriptions + + def test_mce_match_content_is_single_status_line(system_info): """Adjacent incidents: events.json match_content must be one MCn_STATUS row.""" dmesg_content = ( diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index 2e60c73a..e62d6115 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -27,6 +27,10 @@ hardware_error_block_line_indices, ignored_mce_block_line_indices, iter_hardware_error_block_ranges, + mce_defining_status_line_indices, + mce_hardware_error_line_indices, + mce_non_status_hardware_error_line_indices, + mce_unknown_suppress_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, trim_mce_status_match_content, @@ -43,7 +47,7 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): counts = parse_correctable_mce_counts(content) - assert counts == {"CPU1/mc0": 3, "CPU0": 1} + assert counts == {"CPU0": 1} def test_parse_correctable_mce_counts_gpu_summary(): @@ -55,7 +59,7 @@ def test_parse_correctable_mce_counts_gpu_summary(): counts = parse_correctable_mce_counts(content) - assert counts == {"GPU0/gfx": 3} + assert counts == {} def test_parse_uncorrectable_mce_counts(): @@ -67,7 +71,7 @@ def test_parse_uncorrectable_mce_counts(): counts = parse_uncorrectable_mce_counts(content) - assert counts == {"CPU1": 1, "GPU0/gfx": 2} + assert counts == {"CPU1": 1} def test_parse_correctable_mce_counts_skips_ignored_banks(): @@ -192,6 +196,55 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): assert 6 not in ignored_mce_block_line_indices(content, frozenset({60})) +def test_mce_non_status_hardware_error_line_indices(): + content = ( + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" + "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + ) + + assert mce_defining_status_line_indices(content) == frozenset({1}) + assert mce_non_status_hardware_error_line_indices(content) == frozenset({0, 2}) + + +def test_mce_unknown_suppress_orphan_detail_lines(): + """Tail of an MCE block without the defining MCn_STATUS row still suppresses unknowns.""" + content = ( + "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " + "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" + "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + ) + + suppressed = mce_unknown_suppress_line_indices(content) + + assert 0 in suppressed + assert 1 in suppressed + assert 2 not in suppressed + + +def test_mce_unknown_suppress_status_and_tail_lines(): + """Status row plus PPIN/IPID/cache tail must never reach unknown dmesg matching.""" + content = ( + "kern :emerg : 2026-07-14T22:21:10,266984-07:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2026-07-14T22:21:10,280615-07:00 " + "[Hardware Error]: CPU:56 (1a:51:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xabc\n" + "kern :emerg : 2026-07-14T22:21:10,303837-07:00 [Hardware Error]: PPIN: 0x00831e03ffcb8015\n" + "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " + "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" + "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + ) + + suppressed = mce_unknown_suppress_line_indices(content) + + assert suppressed == frozenset({0, 1, 2, 3, 4}) + assert mce_hardware_error_line_indices(content) == frozenset({0, 1, 2, 3, 4}) + + def test_trim_mce_status_match_content_keeps_status_row_only(): multiline = ( "[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" @@ -228,7 +281,7 @@ def test_parse_correctable_mce_counts_both_cpu_formats(): counts = parse_correctable_mce_counts(content) - assert counts == {"CPU0": 1, "CPU72": 1, "CPU1/mc0": 2} + assert counts == {"CPU0": 1, "CPU72": 1} def test_parse_uncorrectable_mce_counts_cpu_colon_status(): From b718ea95dca6151f85bdb71302e6114fbc8268db Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Tue, 21 Jul 2026 15:13:03 -0500 Subject: [PATCH 2/4] utest dummies --- test/unit/plugin/test_dmesg_analyzer.py | 32 ++++---- test/unit/plugin/test_mce_utils.py | 102 +++++++++++++----------- 2 files changed, 74 insertions(+), 60 deletions(-) diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index 9b009785..784b5453 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -1285,7 +1285,7 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info): - """Dummy excerpt modeled on dmesg_mce_interleave.log: warn/blank lines inside MCE blocks.""" + """Dummy excerpt: warn/blank lines interleaved inside MCE hardware-error blocks.""" dmesg_content = ( "kern :info : 2038-01-19T00:00:00,000000+00:00 " "mce: [Hardware Error]: Machine check events logged\n" @@ -1341,11 +1341,11 @@ def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info) def test_orphan_mce_detail_lines_not_reported_as_unknown(system_info): """When analysis window omits MCn_STATUS, trailing detail lines must not become unknowns.""" dmesg_content = ( - "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " - "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" - "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" - "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 unrelated plugin failure\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1360,20 +1360,22 @@ def test_orphan_mce_detail_lines_not_reported_as_unknown(system_info): def test_filtered_window_orphan_mce_tail_not_unknown(system_info): - """i198-maxcorestim: analysis window can start after MCn_STATUS but before IPID/cache tail.""" + """Analysis window can start after MCn_STATUS but before IPID/cache tail lines.""" dmesg_content = ( - "kern :info : 2026-07-14T22:21:10,254124-07:00 mce: [Hardware Error]: Machine check events logged\n" - "kern :emerg : 2026-07-14T22:21:10,266984-07:00 [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : 2026-07-14T22:21:10,280615-07:00 " - "[Hardware Error]: CPU:56 (1a:51:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xd8202000000c080b\n" - "kern :emerg : 2026-07-14T22:21:10,303837-07:00 [Hardware Error]: PPIN: 0x00831e03ffcb8015\n" - "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " - "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" - "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "kern :info : 2038-01-19T00:00:10,254124+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:10,266984+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:10,280615+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:10,303837+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:10,315164+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:10,335516+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) analysis_range_start = datetime.datetime.fromisoformat( - "2026-07-14 22:21:10.314-07:00" + "2038-01-19T00:00:10.314000+00:00" ).astimezone(datetime.timezone.utc) analyzer = DmesgAnalyzer(system_info=system_info) diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index e62d6115..4a3034e6 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -42,7 +42,7 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): "kern :warn : 2038-01-19T00:00:00,000000+00:00 " "mce: 3 correctable hardware errors detected in total in mc0 block on CPU1\n" "kern :warn : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xabc\n" + "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xaaa\n" ) counts = parse_correctable_mce_counts(content) @@ -65,7 +65,7 @@ def test_parse_correctable_mce_counts_gpu_summary(): def test_parse_uncorrectable_mce_counts(): content = ( "kern :err : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: Machine Check: CPU1 MC1_STATUS[0xfeed|UC|AddrV]: 0x0\n" + "[Hardware Error]: Machine Check: CPU1 MC1_STATUS[0xbbb|UC|AddrV]: 0x0\n" "amdgpu 0000:de:ad.0: amdgpu: socket: 0 2 uncorrectable hardware errors detected in gfx block\n" ) @@ -101,10 +101,12 @@ def test_parse_correctable_mce_counts_skips_ignored_banks_in_separate_block(): def test_hardware_error_block_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" - "kern :info : ts unrelated line\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :info : 2038-01-19T00:00:03,000000+00:00 dummy: unrelated line\n" ) assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2}) @@ -118,10 +120,10 @@ def test_hardware_error_block_splits_info_preamble_from_emerg_details(): "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" - "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " - "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" - "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) lines = content.splitlines() @@ -133,11 +135,15 @@ def test_hardware_error_block_splits_info_preamble_from_emerg_details(): def test_hardware_error_block_includes_interleaved_non_mce_lines(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts amdgpu 0000:de:ad.0: amdgpu: unrelated reset notice\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" - "kern :emerg : ts [Hardware Error]: cache level: L3/GEN\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "amdgpu 0000:de:ad.0: amdgpu: dummy unrelated reset notice\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN\n" ) assert hardware_error_block_line_indices(content) == frozenset({0, 1, 3, 4}) @@ -146,12 +152,16 @@ def test_hardware_error_block_includes_interleaved_non_mce_lines(): def test_ignored_mce_block_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" "\n" - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :err : ts [Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0x3\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :err : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0xccc\n" ) ignored = ignored_mce_block_line_indices(content, frozenset({60})) @@ -173,7 +183,7 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): "workqueue: dummy_mce_worker hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND\n" "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " - "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" "\n" "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" @@ -182,7 +192,7 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): "kern :emerg : 2038-01-19T00:00:08,000000+00:00 " "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:09,000000+00:00 " - "[Hardware Error]: CPU:24 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "[Hardware Error]: CPU:99 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" "kern :emerg : 2038-01-19T00:00:10,000000+00:00 [Hardware Error]: PPIN: 0xcccccccccccccccc\n" ) lines = content.splitlines() @@ -198,9 +208,11 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): def test_mce_non_status_hardware_error_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" ) assert mce_defining_status_line_indices(content) == frozenset({1}) @@ -210,11 +222,11 @@ def test_mce_non_status_hardware_error_line_indices(): def test_mce_unknown_suppress_orphan_detail_lines(): """Tail of an MCE block without the defining MCn_STATUS row still suppresses unknowns.""" content = ( - "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " - "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" - "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" - "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 dummy: unrelated plugin failure\n" ) suppressed = mce_unknown_suppress_line_indices(content) @@ -227,16 +239,16 @@ def test_mce_unknown_suppress_orphan_detail_lines(): def test_mce_unknown_suppress_status_and_tail_lines(): """Status row plus PPIN/IPID/cache tail must never reach unknown dmesg matching.""" content = ( - "kern :emerg : 2026-07-14T22:21:10,266984-07:00 " + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " "[Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : 2026-07-14T22:21:10,280615-07:00 " - "[Hardware Error]: CPU:56 (1a:51:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xabc\n" - "kern :emerg : 2026-07-14T22:21:10,303837-07:00 [Hardware Error]: PPIN: 0x00831e03ffcb8015\n" - "kern :emerg : 2026-07-14T22:21:10,315164-07:00 " - "[Hardware Error]: IPID: 0x000001e11ccc0005, Syndrome: 0x000000005a800001\n" - "kern :emerg : 2026-07-14T22:21:10,335516-07:00 " + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" - "kern :err : 2026-07-14T22:21:10,400000-07:00 unrelated plugin failure\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 dummy: unrelated plugin failure\n" ) suppressed = mce_unknown_suppress_line_indices(content) @@ -247,14 +259,14 @@ def test_mce_unknown_suppress_status_and_tail_lines(): def test_trim_mce_status_match_content_keeps_status_row_only(): multiline = ( - "[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" + "[Hardware Error]: CPU:42 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" "[Hardware Error]: Corrected error, no action required.\n" - "[Hardware Error]: CPU:8 (00:00:0) MC60_STATUS[Over|CE|MiscV]: 0xccc\n" + "[Hardware Error]: CPU:99 (00:00:0) MC60_STATUS[Over|CE|MiscV]: 0xccc\n" ) trimmed = trim_mce_status_match_content(multiline) - assert trimmed == ("[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb") + assert trimmed == ("[Hardware Error]: CPU:42 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb") assert "MC60_STATUS" not in trimmed assert "\n" not in trimmed @@ -262,34 +274,34 @@ def test_trim_mce_status_match_content_keeps_status_row_only(): def test_parse_correctable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:00,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xaaa\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU72": 1} + assert counts == {"CPU12": 1} def test_parse_correctable_mce_counts_both_cpu_formats(): content = ( "[Hardware Error]: CPU0 MC1_STATUS[0x0|CE|]: 0x1\n" "kern :err : 2038-01-19T00:00:00,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xaaa\n" "kern :warn : 2038-01-19T00:00:02,000000+00:00 " "mce: 2 correctable hardware errors detected in total in mc0 block on CPU:1\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU0": 1, "CPU72": 1} + assert counts == {"CPU0": 1, "CPU12": 1} def test_parse_uncorrectable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|UC|Misc]: 0xdef\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|UC|Misc]: 0xbbb\n" ) counts = parse_uncorrectable_mce_counts(content) - assert counts == {"CPU72": 1} + assert counts == {"CPU12": 1} From 9f08eb6d46b9948145605747b2deaf3d87c15378 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Wed, 22 Jul 2026 14:25:33 -0500 Subject: [PATCH 3/4] utest fix --- nodescraper/plugins/inband/dmesg/mce_utils.py | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index 55222aaa..2b492bcf 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -55,6 +55,21 @@ re.IGNORECASE, ) +# MCE incident rows only +_MCE_INCIDENT_LINE_RE = re.compile( + r"\[Hardware Error\]:\s*(?:" + r"(?:Corrected error|Uncorrected error|Machine check events logged)|" + r"Machine Check:|" + r"CPU:?\d|" + r"MC\d+_STATUS|" + r"PPIN:|" + r"IPID:|" + r"Syndrome:|" + r"cache level:" + r")", + re.IGNORECASE, +) + def compile_mce_ce_status_regex() -> re.Pattern[str]: """Return a single-line regex for corrected MCn_STATUS hardware error rows.""" @@ -110,6 +125,10 @@ def _is_mce_detail_line(line: str) -> bool: return _MCE_DETAIL_LINE_RE.search(line) is not None +def _is_mce_incident_line(line: str) -> bool: + return _MCE_INCIDENT_LINE_RE.search(line) is not None + + def _mce_detail_line_indices_in_range(lines: Sequence[str], start: int, end: int) -> set[int]: return {index for index in range(start, end) if _is_mce_detail_line(lines[index])} @@ -127,8 +146,8 @@ def _is_mce_status_only_starter(line: str) -> bool: return not _is_mce_primary_starter(line) and _MCE_STATUS_START_RE.search(line) is not None -def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: - """Return True when another [Hardware Error]: line appears before the next incident.""" +def _has_mce_incident_line_ahead(lines: Sequence[str], start_index: int) -> bool: + """Return True when another MCE incident [Hardware Error]: row appears before the next block.""" for idx in range(start_index + 1, len(lines)): line = lines[idx] if not line.strip(): @@ -137,8 +156,10 @@ def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: return False if _is_mce_status_only_starter(line): return False - if _is_mce_detail_line(line): + if _is_mce_incident_line(line): return True + if _is_mce_detail_line(line): + return False return False @@ -172,7 +193,12 @@ def mce_known_regex_skip_line_indices( ) -> frozenset[int]: """Skip non-defining MCE detail lines and ignored-bank incidents during known regex scan.""" ignored = ignore_banks or frozenset() - skipped = set(mce_non_status_hardware_error_line_indices(content)) + lines = content.splitlines() + defining = mce_defining_status_line_indices(content) + primary_starters = frozenset( + index for index, line in enumerate(lines) if _is_mce_primary_starter(line) + ) + skipped = set(hardware_error_block_line_indices(content)) - defining - primary_starters skipped.update(ignored_mce_block_line_indices(content, ignored)) return frozenset(skipped) @@ -190,9 +216,9 @@ def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, in A block begins at a primary MCE header (Corrected/Uncorrected/Machine check logged) or at the first MCn_STATUS line when not already inside a block. The block then includes subsequent lines until the next primary header, a blank line before a bare - MCn_STATUS starter, trailing non-MCE lines with no further [Hardware Error]: detail, - or EOF. Blank lines, warn/err noise, and other non-MCE dmesg lines between detail - entries belong to the same block. + MCn_STATUS starter, a non-MCE [Hardware Error]: row, trailing lines with no further + MCE incident detail, or EOF. Blank lines, warn/err noise, and other non-MCE dmesg + lines between MCE incident entries belong to the same block. """ blocks: list[tuple[int, int]] = [] index = 0 @@ -208,9 +234,11 @@ def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, in next_line = _next_non_blank_line_index(lines, index) if next_line is not None and _is_mce_status_only_starter(lines[next_line]): break + elif _is_mce_detail_line(lines[index]) and not _is_mce_incident_line(lines[index]): + break elif _is_mce_block_starter(lines[index], in_block=True): break - elif not _is_mce_detail_line(lines[index]) and not _has_mce_detail_line_ahead( + elif not _is_mce_incident_line(lines[index]) and not _has_mce_incident_line_ahead( lines, index ): break From ea69717338c024e2ea1e1854329e314a69682498 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Wed, 22 Jul 2026 14:49:38 -0500 Subject: [PATCH 4/4] fix for doubling down on same err --- nodescraper/plugins/inband/dmesg/mce_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index 2b492bcf..9ec57246 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -187,6 +187,18 @@ def mce_non_status_hardware_error_line_indices(content: str) -> frozenset[int]: ) +def _primary_starters_in_mce_status_blocks( + lines: Sequence[str], defining: frozenset[int] +) -> frozenset[int]: + """Return primary MCE header lines that belong to blocks with MCn_STATUS CE/UC rows.""" + primary_starters = {index for index, line in enumerate(lines) if _is_mce_primary_starter(line)} + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + if any(index in defining for index in range(start, end)): + suppressed.update(index for index in range(start, end) if index in primary_starters) + return frozenset(suppressed) + + def mce_known_regex_skip_line_indices( content: str, ignore_banks: Optional[FrozenSet[int]] = None, @@ -198,7 +210,9 @@ def mce_known_regex_skip_line_indices( primary_starters = frozenset( index for index, line in enumerate(lines) if _is_mce_primary_starter(line) ) + primary_in_status_blocks = _primary_starters_in_mce_status_blocks(lines, defining) skipped = set(hardware_error_block_line_indices(content)) - defining - primary_starters + skipped.update(primary_in_status_blocks) skipped.update(ignored_mce_block_line_indices(content, ignored)) return frozenset(skipped)