diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 04c8737b..5ae53f77 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -38,8 +38,8 @@ from .mce_utils import ( compile_mce_ce_status_regex, compile_mce_uc_status_regex, - ignored_mce_block_line_indices, - mce_block_all_line_indices, + mce_known_regex_skip_line_indices, + mce_unknown_suppress_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, trim_mce_status_match_content, @@ -718,8 +718,8 @@ def analyze_data( dmesg_content = data.dmesg_content ignore_match_rules, ignore_mce_banks = parse_ignore_match_rules(args.ignore_match_rules) - ignored_mce_block_lines = ignored_mce_block_line_indices(dmesg_content, ignore_mce_banks) - mce_block_lines = mce_block_all_line_indices(dmesg_content) + known_skip_lines = mce_known_regex_skip_line_indices(dmesg_content, ignore_mce_banks) + unknown_skip_lines = mce_unknown_suppress_line_indices(dmesg_content) known_err_events = self.check_all_regexes( content=dmesg_content, @@ -728,7 +728,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, - skip_line_indices=ignored_mce_block_lines, + skip_line_indices=known_skip_lines, ) for event in known_err_events: if event.description in ("MCE Corrected Error", "MCE Uncorrected Error"): @@ -764,7 +764,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, - skip_line_indices=mce_block_lines, + skip_line_indices=unknown_skip_lines, ) for err_event in err_events: diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index a8eeccf3..9ec57246 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -35,29 +35,6 @@ _MCE_STATUS_START_RE = re.compile(r"\bMC\d+_STATUS\[", re.IGNORECASE) _MCE_DETAIL_LINE_RE = re.compile(r"\[Hardware Error\]:") -_CORRECTABLE_SUMMARY_RE = re.compile( - r"(?P\d+)\s+correctable hardware errors detected in total in (?P\w+) block" - r"(?:\s+on\s+(?PCPU:?\d+))?", - re.IGNORECASE, -) - -_UNCORRECTABLE_SUMMARY_RE = re.compile( - r"(?P\d+)\s+uncorrectable hardware errors detected in (?P\w+) block", - re.IGNORECASE, -) - -_GPU_CORRECTABLE_RE = re.compile( - r"amdgpu\s+(?P[\w:.]+):.*?(?P\d+)\s+correctable hardware errors detected in total in " - r"(?P\w+) block", - re.IGNORECASE, -) - -_GPU_UNCORRECTABLE_RE = re.compile( - r"amdgpu\s+(?P[\w:.]+):.*?(?P\d+)\s+uncorrectable hardware errors detected in " - r"(?P\w+) block", - re.IGNORECASE, -) - _MCE_CE_STATUS_RE = re.compile( r"\[Hardware Error\]:.*?(?PCPU:?\d+).*?MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\]", re.IGNORECASE, @@ -78,6 +55,21 @@ re.IGNORECASE, ) +# MCE incident rows only +_MCE_INCIDENT_LINE_RE = re.compile( + r"\[Hardware Error\]:\s*(?:" + r"(?:Corrected error|Uncorrected error|Machine check events logged)|" + r"Machine Check:|" + r"CPU:?\d|" + r"MC\d+_STATUS|" + r"PPIN:|" + r"IPID:|" + r"Syndrome:|" + r"cache level:" + r")", + re.IGNORECASE, +) + def compile_mce_ce_status_regex() -> re.Pattern[str]: """Return a single-line regex for corrected MCn_STATUS hardware error rows.""" @@ -116,33 +108,6 @@ def _add_count(counts: dict[str, int], part: str, amount: int) -> None: counts[part] = counts.get(part, 0) + amount -def _part_label( - *, - cpu: Optional[str] = None, - block: Optional[str] = None, - bdf: Optional[str] = None, - gpu_index: Optional[int] = None, -) -> str: - if bdf is not None: - block_suffix = f"/{block}" if block else "" - if gpu_index is not None: - return f"GPU{gpu_index}{block_suffix}" - return f"GPU {bdf}{block_suffix}" - if cpu and block: - return f"{cpu}/{block}" - if cpu: - return cpu - if block: - return block - return "unknown" - - -def _gpu_index_for_bdf(bdf: str, bdf_order: list[str]) -> int: - if bdf not in bdf_order: - bdf_order.append(bdf) - return bdf_order.index(bdf) - - def _is_mce_primary_starter(line: str) -> bool: return _MCE_PRIMARY_START_RE.search(line) is not None @@ -160,6 +125,10 @@ def _is_mce_detail_line(line: str) -> bool: return _MCE_DETAIL_LINE_RE.search(line) is not None +def _is_mce_incident_line(line: str) -> bool: + return _MCE_INCIDENT_LINE_RE.search(line) is not None + + def _mce_detail_line_indices_in_range(lines: Sequence[str], start: int, end: int) -> set[int]: return {index for index in range(start, end) if _is_mce_detail_line(lines[index])} @@ -177,8 +146,8 @@ def _is_mce_status_only_starter(line: str) -> bool: return not _is_mce_primary_starter(line) and _MCE_STATUS_START_RE.search(line) is not None -def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: - """Return True when another [Hardware Error]: line appears before the next incident.""" +def _has_mce_incident_line_ahead(lines: Sequence[str], start_index: int) -> bool: + """Return True when another MCE incident [Hardware Error]: row appears before the next block.""" for idx in range(start_index + 1, len(lines)): line = lines[idx] if not line.strip(): @@ -187,20 +156,83 @@ def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: return False if _is_mce_status_only_starter(line): return False - if _is_mce_detail_line(line): + if _is_mce_incident_line(line): return True + if _is_mce_detail_line(line): + return False return False +def mce_defining_status_line_indices(content: str) -> frozenset[int]: + """Return line indices for MCn_STATUS rows that define a corrected or uncorrected MCE.""" + lines = content.splitlines() + indices: set[int] = set() + for index, line in enumerate(lines): + if _MCE_CE_STATUS_LINE_RE.search(line) or _MCE_UC_STATUS_LINE_RE.search(line): + indices.add(index) + return frozenset(indices) + + +def mce_hardware_error_line_indices(content: str) -> frozenset[int]: + """Return every line index containing [Hardware Error]:.""" + lines = content.splitlines() + return frozenset(index for index, line in enumerate(lines) if _is_mce_detail_line(line)) + + +def mce_non_status_hardware_error_line_indices(content: str) -> frozenset[int]: + """Return [Hardware Error]: detail lines that are not defining MCn_STATUS CE/UC rows.""" + defining = mce_defining_status_line_indices(content) + return frozenset( + index for index in mce_hardware_error_line_indices(content) if index not in defining + ) + + +def _primary_starters_in_mce_status_blocks( + lines: Sequence[str], defining: frozenset[int] +) -> frozenset[int]: + """Return primary MCE header lines that belong to blocks with MCn_STATUS CE/UC rows.""" + primary_starters = {index for index, line in enumerate(lines) if _is_mce_primary_starter(line)} + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + if any(index in defining for index in range(start, end)): + suppressed.update(index for index in range(start, end) if index in primary_starters) + return frozenset(suppressed) + + +def mce_known_regex_skip_line_indices( + content: str, + ignore_banks: Optional[FrozenSet[int]] = None, +) -> frozenset[int]: + """Skip non-defining MCE detail lines and ignored-bank incidents during known regex scan.""" + ignored = ignore_banks or frozenset() + lines = content.splitlines() + defining = mce_defining_status_line_indices(content) + primary_starters = frozenset( + index for index, line in enumerate(lines) if _is_mce_primary_starter(line) + ) + primary_in_status_blocks = _primary_starters_in_mce_status_blocks(lines, defining) + skipped = set(hardware_error_block_line_indices(content)) - defining - primary_starters + skipped.update(primary_in_status_blocks) + skipped.update(ignored_mce_block_line_indices(content, ignored)) + return frozenset(skipped) + + +def mce_unknown_suppress_line_indices(content: str) -> frozenset[int]: + """Suppress MCE block context and every [Hardware Error]: line from unknown scan.""" + suppressed = set(mce_block_all_line_indices(content)) + suppressed.update(mce_hardware_error_line_indices(content)) + return frozenset(suppressed) + + def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, int]]: """Return (start, end) line index ranges for MCE incident blocks. A block begins at a primary MCE header (Corrected/Uncorrected/Machine check logged) or at the first MCn_STATUS line when not already inside a block. The block then includes subsequent lines until the next primary header, a blank line before a bare - MCn_STATUS starter, trailing non-MCE lines with no further [Hardware Error]: detail, - or EOF. Blank lines, warn/err noise, and other non-MCE dmesg lines between detail - entries belong to the same block. + MCn_STATUS starter, a non-MCE [Hardware Error]: row, trailing lines with no further + MCE incident detail, or EOF. Blank lines, warn/err noise, and other non-MCE dmesg + lines between MCE incident entries belong to the same block. """ blocks: list[tuple[int, int]] = [] index = 0 @@ -216,9 +248,11 @@ def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, in next_line = _next_non_blank_line_index(lines, index) if next_line is not None and _is_mce_status_only_starter(lines[next_line]): break + elif _is_mce_detail_line(lines[index]) and not _is_mce_incident_line(lines[index]): + break elif _is_mce_block_starter(lines[index], in_block=True): break - elif not _is_mce_detail_line(lines[index]) and not _has_mce_detail_line_ahead( + elif not _is_mce_incident_line(lines[index]) and not _has_mce_incident_line_ahead( lines, index ): break @@ -264,39 +298,14 @@ def parse_correctable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, ) -> dict[str, int]: - """Count correctable MCE / RAS hardware errors per component from dmesg text. - - Handles summary lines (for example ``mce: 3 correctable ... on CPU1``), - amdgpu block summaries, and per-event ``MCn_STATUS[|CE|]`` hardware error lines. - """ + """Count correctable MCE hardware errors per CPU from MCn_STATUS[|CE|] rows.""" counts: dict[str, int] = {} - gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() ignored_block_lines = ignored_mce_block_line_indices(content, ignored) for line_no, line in enumerate(content.splitlines()): if line_no in ignored_block_lines: continue - gpu_match = _GPU_CORRECTABLE_RE.search(line) - if gpu_match: - bdf = gpu_match.group("bdf") - part = _part_label( - bdf=bdf, - block=gpu_match.group("block"), - gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order), - ) - _add_count(counts, part, int(gpu_match.group("count"))) - continue - - summary_match = _CORRECTABLE_SUMMARY_RE.search(line) - if summary_match: - cpu = summary_match.group("cpu") - part = _part_label( - cpu=_normalize_cpu_label(cpu) if cpu else None, - block=summary_match.group("block"), - ) - _add_count(counts, part, int(summary_match.group("count"))) - continue status_match = _MCE_CE_STATUS_RE.search(line) if status_match: @@ -314,31 +323,14 @@ def parse_uncorrectable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, ) -> dict[str, int]: - """Count uncorrectable MCE / RAS hardware errors per component from dmesg text.""" + """Count uncorrectable MCE hardware errors per CPU from MCn_STATUS[|UC|] rows.""" counts: dict[str, int] = {} - gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() ignored_block_lines = ignored_mce_block_line_indices(content, ignored) for line_no, line in enumerate(content.splitlines()): if line_no in ignored_block_lines: continue - gpu_match = _GPU_UNCORRECTABLE_RE.search(line) - if gpu_match: - bdf = gpu_match.group("bdf") - part = _part_label( - bdf=bdf, - block=gpu_match.group("block"), - gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order), - ) - _add_count(counts, part, int(gpu_match.group("count"))) - continue - - summary_match = _UNCORRECTABLE_SUMMARY_RE.search(line) - if summary_match: - part = _part_label(block=summary_match.group("block")) - _add_count(counts, part, int(summary_match.group("count"))) - continue status_match = _MCE_UC_STATUS_RE.search(line) if status_match: diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index 87477acf..784b5453 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -1022,11 +1022,11 @@ def test_priority_override_updates_unkown_dmesg_error(system_info): assert res.events[0].priority == EventPriority.ERROR -def test_mce_threshold_raises_error_for_gpu(system_info): +def test_mce_threshold_raises_error_for_status_lines(system_info): dmesg_content = ( - "kern :err : 2024-10-07T10:17:15,145363-04:00 " - "amdgpu 0000:c1:00.0: amdgpu: socket: 4, die: 0 " - "3 correctable hardware errors detected in total in gfx block\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x2\n" + "[Hardware Error]: CPU0 MC0_STATUS[0x0|CE|]: 0x3\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1038,7 +1038,7 @@ def test_mce_threshold_raises_error_for_gpu(system_info): threshold_events = [e for e in res.events if e.data.get("mce_threshold") == 3] assert len(threshold_events) == 1 assert threshold_events[0].priority == EventPriority.ERROR - assert threshold_events[0].data["part"] == "GPU0/gfx" + assert threshold_events[0].data["part"] == "CPU0" assert threshold_events[0].data["correctable_mce_count"] == 3 assert res.status == ExecutionStatus.ERROR @@ -1065,8 +1065,8 @@ def test_mce_threshold_raises_error_for_cpu_colon_status(system_info): def test_mce_threshold_not_triggered_below_limit(system_info): dmesg_content = ( - "kern :warn : 2024-06-11T14:30:00,123456+00:00 " - "mce: 2 correctable hardware errors detected in total in mc0 block on CPU1\n" + "[Hardware Error]: CPU1 MC0_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU1 MC0_STATUS[0x0|CE|]: 0x2\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1260,7 +1260,11 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" - "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) analyzer = DmesgAnalyzer(system_info=system_info) @@ -1270,12 +1274,18 @@ def test_hardware_error_block_reports_mce_without_ignore(system_info): ) descriptions = {event.description for event in res.events} + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + mce_events = [event for event in res.events if event.description == "MCE Corrected Error"] + assert unknown_events == [] + assert len(mce_events) == 1 + assert "MC60_STATUS" in str(mce_events[0].data["match_content"]) assert "Unknown dmesg error" not in descriptions - assert "MCE Corrected Error" in descriptions or "RAS Corrected Error" in descriptions + assert "MCE Corrected Error" in descriptions + assert "RAS Corrected Error" not in descriptions def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info): - """Dummy excerpt modeled on dmesg_mce_interleave.log: warn/blank lines inside MCE blocks.""" + """Dummy excerpt: warn/blank lines interleaved inside MCE hardware-error blocks.""" dmesg_content = ( "kern :info : 2038-01-19T00:00:00,000000+00:00 " "mce: [Hardware Error]: Machine check events logged\n" @@ -1328,6 +1338,67 @@ def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info) assert unknown_events[0].data["match_content"] == "dummy harness fault outside mce blocks" +def test_orphan_mce_detail_lines_not_reported_as_unknown(system_info): + """When analysis window omits MCn_STATUS, trailing detail lines must not become unknowns.""" + dmesg_content = ( + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 unrelated plugin failure\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs(check_unknown_dmesg_errors=True), + ) + + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + assert len(unknown_events) == 1 + assert unknown_events[0].data["match_content"] == "unrelated plugin failure" + + +def test_filtered_window_orphan_mce_tail_not_unknown(system_info): + """Analysis window can start after MCn_STATUS but before IPID/cache tail lines.""" + dmesg_content = ( + "kern :info : 2038-01-19T00:00:10,254124+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:10,266984+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:10,280615+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:10,303837+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:10,315164+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:10,335516+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + ) + analysis_range_start = datetime.datetime.fromisoformat( + "2038-01-19T00:00:10.314000+00:00" + ).astimezone(datetime.timezone.utc) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=True, + analysis_range_start=analysis_range_start, + ), + ) + + filtered_artifact = next( + artifact for artifact in res.artifacts if artifact.filename == "filtered_dmesg.log" + ) + assert "MC60_STATUS" not in filtered_artifact.contents + assert "IPID:" in filtered_artifact.contents + + descriptions = {event.description for event in res.events} + assert "Unknown dmesg error" not in descriptions + assert "MCE Corrected Error" not in descriptions + assert "RAS Corrected Error" not in descriptions + + def test_mce_match_content_is_single_status_line(system_info): """Adjacent incidents: events.json match_content must be one MCn_STATUS row.""" dmesg_content = ( diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index 2e60c73a..4a3034e6 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -27,6 +27,10 @@ hardware_error_block_line_indices, ignored_mce_block_line_indices, iter_hardware_error_block_ranges, + mce_defining_status_line_indices, + mce_hardware_error_line_indices, + mce_non_status_hardware_error_line_indices, + mce_unknown_suppress_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, trim_mce_status_match_content, @@ -38,12 +42,12 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): "kern :warn : 2038-01-19T00:00:00,000000+00:00 " "mce: 3 correctable hardware errors detected in total in mc0 block on CPU1\n" "kern :warn : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xabc\n" + "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xaaa\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU1/mc0": 3, "CPU0": 1} + assert counts == {"CPU0": 1} def test_parse_correctable_mce_counts_gpu_summary(): @@ -55,19 +59,19 @@ def test_parse_correctable_mce_counts_gpu_summary(): counts = parse_correctable_mce_counts(content) - assert counts == {"GPU0/gfx": 3} + assert counts == {} def test_parse_uncorrectable_mce_counts(): content = ( "kern :err : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: Machine Check: CPU1 MC1_STATUS[0xfeed|UC|AddrV]: 0x0\n" + "[Hardware Error]: Machine Check: CPU1 MC1_STATUS[0xbbb|UC|AddrV]: 0x0\n" "amdgpu 0000:de:ad.0: amdgpu: socket: 0 2 uncorrectable hardware errors detected in gfx block\n" ) counts = parse_uncorrectable_mce_counts(content) - assert counts == {"CPU1": 1, "GPU0/gfx": 2} + assert counts == {"CPU1": 1} def test_parse_correctable_mce_counts_skips_ignored_banks(): @@ -97,10 +101,12 @@ def test_parse_correctable_mce_counts_skips_ignored_banks_in_separate_block(): def test_hardware_error_block_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" - "kern :info : ts unrelated line\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :info : 2038-01-19T00:00:03,000000+00:00 dummy: unrelated line\n" ) assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2}) @@ -114,10 +120,10 @@ def test_hardware_error_block_splits_info_preamble_from_emerg_details(): "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" - "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " - "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" - "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" ) lines = content.splitlines() @@ -129,11 +135,15 @@ def test_hardware_error_block_splits_info_preamble_from_emerg_details(): def test_hardware_error_block_includes_interleaved_non_mce_lines(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts amdgpu 0000:de:ad.0: amdgpu: unrelated reset notice\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" - "kern :emerg : ts [Hardware Error]: cache level: L3/GEN\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "amdgpu 0000:de:ad.0: amdgpu: dummy unrelated reset notice\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN\n" ) assert hardware_error_block_line_indices(content) == frozenset({0, 1, 3, 4}) @@ -142,12 +152,16 @@ def test_hardware_error_block_includes_interleaved_non_mce_lines(): def test_ignored_mce_block_line_indices(): content = ( - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" - "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" "\n" - "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" - "kern :err : ts [Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0x3\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :err : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0xccc\n" ) ignored = ignored_mce_block_line_indices(content, frozenset({60})) @@ -169,7 +183,7 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): "workqueue: dummy_mce_worker hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND\n" "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " - "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" "\n" "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" @@ -178,7 +192,7 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): "kern :emerg : 2038-01-19T00:00:08,000000+00:00 " "[Hardware Error]: Corrected error, no action required.\n" "kern :emerg : 2038-01-19T00:00:09,000000+00:00 " - "[Hardware Error]: CPU:24 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "[Hardware Error]: CPU:99 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" "kern :emerg : 2038-01-19T00:00:10,000000+00:00 [Hardware Error]: PPIN: 0xcccccccccccccccc\n" ) lines = content.splitlines() @@ -192,16 +206,67 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): assert 6 not in ignored_mce_block_line_indices(content, frozenset({60})) +def test_mce_non_status_hardware_error_line_indices(): + content = ( + "kern :emerg : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + ) + + assert mce_defining_status_line_indices(content) == frozenset({1}) + assert mce_non_status_hardware_error_line_indices(content) == frozenset({0, 2}) + + +def test_mce_unknown_suppress_orphan_detail_lines(): + """Tail of an MCE block without the defining MCn_STATUS row still suppresses unknowns.""" + content = ( + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 dummy: unrelated plugin failure\n" + ) + + suppressed = mce_unknown_suppress_line_indices(content) + + assert 0 in suppressed + assert 1 in suppressed + assert 2 not in suppressed + + +def test_mce_unknown_suppress_status_and_tail_lines(): + """Status row plus PPIN/IPID/cache tail must never reach unknown dmesg matching.""" + content = ( + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000002\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2038-01-19T00:00:06,000000+00:00 dummy: unrelated plugin failure\n" + ) + + suppressed = mce_unknown_suppress_line_indices(content) + + assert suppressed == frozenset({0, 1, 2, 3, 4}) + assert mce_hardware_error_line_indices(content) == frozenset({0, 1, 2, 3, 4}) + + def test_trim_mce_status_match_content_keeps_status_row_only(): multiline = ( - "[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" + "[Hardware Error]: CPU:42 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n" "[Hardware Error]: Corrected error, no action required.\n" - "[Hardware Error]: CPU:8 (00:00:0) MC60_STATUS[Over|CE|MiscV]: 0xccc\n" + "[Hardware Error]: CPU:99 (00:00:0) MC60_STATUS[Over|CE|MiscV]: 0xccc\n" ) trimmed = trim_mce_status_match_content(multiline) - assert trimmed == ("[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb") + assert trimmed == ("[Hardware Error]: CPU:42 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb") assert "MC60_STATUS" not in trimmed assert "\n" not in trimmed @@ -209,34 +274,34 @@ def test_trim_mce_status_match_content_keeps_status_row_only(): def test_parse_correctable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:00,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xaaa\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU72": 1} + assert counts == {"CPU12": 1} def test_parse_correctable_mce_counts_both_cpu_formats(): content = ( "[Hardware Error]: CPU0 MC1_STATUS[0x0|CE|]: 0x1\n" "kern :err : 2038-01-19T00:00:00,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xaaa\n" "kern :warn : 2038-01-19T00:00:02,000000+00:00 " "mce: 2 correctable hardware errors detected in total in mc0 block on CPU:1\n" ) counts = parse_correctable_mce_counts(content) - assert counts == {"CPU0": 1, "CPU72": 1, "CPU1/mc0": 2} + assert counts == {"CPU0": 1, "CPU12": 1} def test_parse_uncorrectable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:01,000000+00:00 " - "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|UC|Misc]: 0xdef\n" + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[-|UC|Misc]: 0xbbb\n" ) counts = parse_uncorrectable_mce_counts(content) - assert counts == {"CPU72": 1} + assert counts == {"CPU12": 1}