diff --git a/lars/preprocessing/__init__.py b/lars/preprocessing/__init__.py index f5a0d98..3720720 100644 --- a/lars/preprocessing/__init__.py +++ b/lars/preprocessing/__init__.py @@ -1,2 +1,2 @@ from .radar_preprocessing import preprocess_radar_data # noqa: F401 -from .labels import load_labels, save_labels, change_file_path, copy_labels, apply_criteria_to_labels, combine_labels # noqa: F401 \ No newline at end of file +from .labels import load_labels, save_labels, change_file_path, copy_labels, apply_criteria_to_labels, combine_labels, standardize_labels # noqa: F401 \ No newline at end of file diff --git a/lars/preprocessing/labels.py b/lars/preprocessing/labels.py index 6955944..1b62f64 100644 --- a/lars/preprocessing/labels.py +++ b/lars/preprocessing/labels.py @@ -1,6 +1,44 @@ import pandas as pd import os +# Maps non-canonical label spellings (matched case- and whitespace- +# insensitively) to their canonical codebook form. +STANDARD_LABEL_MAP = { + "ambiguous": "Ambiguous / Uncertain", + "unknown": "Ambiguous / Uncertain", + "stratiform": "Stratiform Precipitation", + "ambiguous / uncertain": "Ambiguous / Uncertain", +} + + +def standardize_labels(df, label_column='label'): + """ + Standardize inconsistent label spellings to their canonical codebook form. + + Maps the ambiguous/unknown labels ('Ambiguous', 'UNKNOWN', and any casing + thereof) to the single canonical value 'Ambiguous / Uncertain', and maps + the bare 'Stratiform' label to the full codebook name 'Stratiform + Precipitation'. Matching is case- and whitespace-insensitive. Values that + don't match one of these variants (including labels already in their + canonical form, e.g. 'Stratiform Precipitation') are left unchanged. + + Parameters + ---------- + df (pd.DataFrame): DataFrame containing a label column to standardize. + label_column (str): Name of the column containing labels. Default 'label'. + + Returns + ------- + pd.DataFrame + A copy of ``df`` with standardized values in ``label_column``. + """ + df = df.copy() + keys = df[label_column].astype(str).str.strip().str.lower() + mapped = keys.map(STANDARD_LABEL_MAP) + df[label_column] = mapped.where(mapped.notna(), df[label_column]) + return df + + def change_file_path(radar_df, new_path): """ Change the file paths in the radar DataFrame to a new path. diff --git a/lars/util/__init__.py b/lars/util/__init__.py index d7a1c04..70c5cda 100644 --- a/lars/util/__init__.py +++ b/lars/util/__init__.py @@ -1,3 +1,7 @@ from .confusion_matrix import plot_confusion_matrix, calculate_cohen_kappa # noqa: F401 from .image_grid import plot_label_images # noqa: F401 from .label_timeseries import plot_label_timeseries # noqa: F401 +from .kappa_matrix import plot_kappa_matrix, calculate_kappa_matrix # noqa: F401 +from .label_rate_matrix import plot_label_rate_diff_matrix, calculate_label_rate_diff_matrix # noqa: F401 +from .label_disagreement_matrix import plot_label_disagreement_matrix, calculate_label_disagreement_matrix # noqa: F401 +from .dawid_skene import fit_dawid_skene, score_against_consensus, plot_dawid_skene_confusion # noqa: F401 diff --git a/lars/util/dawid_skene.py b/lars/util/dawid_skene.py new file mode 100644 index 0000000..5f10398 --- /dev/null +++ b/lars/util/dawid_skene.py @@ -0,0 +1,257 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.metrics import accuracy_score, f1_score, cohen_kappa_score + + +def _log_sum_exp(log_values, axis): + max_val = np.max(log_values, axis=axis, keepdims=True) + max_val = np.where(np.isfinite(max_val), max_val, 0.0) + summed = np.sum(np.exp(log_values - max_val), axis=axis, keepdims=True) + return (max_val + np.log(summed)).squeeze(axis) + + +def fit_dawid_skene(df, columns=None, max_iter=100, tol=1e-4, smoothing=0.1): + """ + Fit a Dawid-Skene model to estimate a consensus label per item and a + confusion matrix per rater, without requiring any rater's labels to be + treated as ground truth. + + Each rater's label is modeled as a noisy observation of an unknown true + class. The EM algorithm jointly estimates (a) each rater's confusion + matrix ``P(observed | true)`` and (b) a posterior distribution over the + true class for every item. The entropy of that posterior is a per-item + uncertainty estimate: near zero when raters agree, higher when they + don't. + + Parameters + ---------- + df (pd.DataFrame): DataFrame containing one column per rater/source. + Missing values (NaN) are treated as "this rater did not label this + item". + columns (list of str or None): Columns (raters) to fit on. Defaults to + all columns in ``df``. + max_iter (int): Maximum number of EM iterations. + tol (float): Convergence tolerance on the maximum absolute change in the + item posterior between iterations. + smoothing (float): Additive (Laplace) smoothing applied to each rater's + confusion matrix during the M-step, to avoid zero probabilities. + + Returns + ------- + dict with keys: + consensus (pd.Series): MAP consensus label per item, indexed like + ``df``. NaN for items with no non-missing rater. + consensus_proba (pd.DataFrame): Item x class posterior probabilities. + item_entropy (pd.Series): Shannon entropy (bits) of each item's + posterior -- the per-item uncertainty estimate. + confusion_matrices (dict[str, pd.DataFrame]): Per-rater confusion + matrix, indexed by true class and labeled by observed class. + Each row sums to 1. + class_prior (pd.Series): Estimated prevalence of each true class. + columns (list of str): Columns used to fit the model. + n_iter (int): Number of EM iterations actually run. + log_likelihood (list of float): Observed-data log-likelihood at each + iteration (should increase monotonically). + """ + if columns is None: + columns = list(df.columns) + + labels = df[columns].apply(lambda s: s.astype(str).str.lower()) + labels = labels.where(df[columns].notna()) + + classes = sorted(set(labels.values.flatten()) - {None, np.nan}) + classes = [c for c in classes if isinstance(c, str)] + n_classes = len(classes) + class_index = {c: k for k, c in enumerate(classes)} + + n_items = len(df) + n_raters = len(columns) + + observed = np.full((n_items, n_raters), -1, dtype=int) + for j, col in enumerate(columns): + for i, value in enumerate(labels[col].values): + if isinstance(value, str): + observed[i, j] = class_index[value] + + has_any_label = (observed >= 0).any(axis=1) + + q = np.full((n_items, n_classes), 1.0 / n_classes) + for i in range(n_items): + if not has_any_label[i]: + continue + votes = np.zeros(n_classes) + for j in range(n_raters): + if observed[i, j] >= 0: + votes[observed[i, j]] += 1 + q[i] = votes / votes.sum() + + log_likelihood_history = [] + n_iter = 0 + + for n_iter in range(1, max_iter + 1): + # M-step + class_prior = q[has_any_label].mean(axis=0) + confusion = np.zeros((n_raters, n_classes, n_classes)) + for j in range(n_raters): + mask = observed[:, j] >= 0 + numer = np.full((n_classes, n_classes), smoothing) + denom = np.full(n_classes, smoothing * n_classes) + for k in range(n_classes): + weight = q[mask, k] + denom[k] += weight.sum() + for l in range(n_classes): + numer[k, l] += weight[observed[mask, j] == l].sum() + confusion[j] = numer / denom[:, None] + + # E-step + log_confusion = np.log(confusion) + log_prior = np.log(class_prior) + log_q_unnorm = np.tile(log_prior, (n_items, 1)) + for j in range(n_raters): + mask = observed[:, j] >= 0 + log_q_unnorm[mask] += log_confusion[j, :, observed[mask, j]] + + log_norm = _log_sum_exp(log_q_unnorm, axis=1) + log_likelihood_history.append(log_norm[has_any_label].sum()) + + new_q = np.exp(log_q_unnorm - log_norm[:, None]) + new_q[~has_any_label] = 1.0 / n_classes + + delta = np.max(np.abs(new_q - q)) + q = new_q + if delta < tol: + break + + index = df.index + consensus_proba = pd.DataFrame(q, index=index, columns=classes) + consensus_proba.loc[~has_any_label, :] = np.nan + + consensus = pd.Series( + [classes[k] for k in np.argmax(q, axis=1)], index=index + ) + consensus[~has_any_label] = np.nan + + with np.errstate(divide="ignore", invalid="ignore"): + entropy_terms = np.where(q > 0, q * np.log2(q), 0.0) + item_entropy = pd.Series(-entropy_terms.sum(axis=1), index=index) + item_entropy[~has_any_label] = np.nan + + confusion_matrices = { + col: pd.DataFrame(confusion[j], index=classes, columns=classes) + for j, col in enumerate(columns) + } + + return { + "consensus": consensus, + "consensus_proba": consensus_proba, + "item_entropy": item_entropy, + "confusion_matrices": confusion_matrices, + "class_prior": pd.Series(class_prior, index=classes), + "columns": columns, + "n_iter": n_iter, + "log_likelihood": log_likelihood_history, + } + + +def score_against_consensus(df, result, columns=None): + """ + Score one or more raters/experiments against a Dawid-Skene consensus + label, e.g. to rank several LLM labelling experiments without treating + any single human rater as ground truth. + + Parameters + ---------- + df (pd.DataFrame): DataFrame containing the columns to score. Must share + an index with the DataFrame ``result`` was fit on. + result (dict): Return value of ``fit_dawid_skene``. + columns (list of str or None): Columns to score. Defaults to the columns + the model was fit on (``result['columns']``). + + Returns + ------- + pd.DataFrame + Indexed by column name, with columns ``accuracy``, ``macro_f1``, and + ``kappa`` computed against the consensus label (rows missing either + the consensus or the rater's label are excluded per column). Sorted + by ``macro_f1`` descending, so the top row is the best-agreeing + experiment. + """ + if columns is None: + columns = result["columns"] + + consensus = result["consensus"] + rows = [] + for col in columns: + pair = pd.DataFrame({"consensus": consensus, "rater": df[col]}).dropna() + true_values = pair["consensus"] + pred_values = pair["rater"].astype(str).str.lower() + rows.append( + { + "column": col, + "accuracy": accuracy_score(true_values, pred_values), + "macro_f1": f1_score( + true_values, pred_values, average="macro", zero_division=0 + ), + "kappa": cohen_kappa_score(true_values, pred_values), + } + ) + + return ( + pd.DataFrame(rows) + .set_index("column") + .sort_values("macro_f1", ascending=False) + ) + + +def plot_dawid_skene_confusion(result, column, ax=None, cmap=None, annot=True): + """ + Plot one rater's estimated Dawid-Skene confusion matrix as a heatmap. + + Parameters + ---------- + result (dict): Return value of ``fit_dawid_skene``. + column (str): Which rater's confusion matrix to plot (a key of + ``result['confusion_matrices']``). + ax (matplotlib axis handle): The axis handle to plot on. Set to None to + use the current axis. + cmap (matplotlib colormap or None): Colormap for the heatmap. Defaults + to ``plt.cm.Blues``. + annot (bool): Whether to annotate each cell with its value. + + Returns + ------- + matplotlib.axes.Axes + The axis the confusion matrix was drawn on. + """ + matrix = result["confusion_matrices"][column] + classes = list(matrix.columns) + + if ax is None: + ax = plt.gca() + if cmap is None: + cmap = plt.cm.Blues + + im = ax.imshow(matrix.values, cmap=cmap, vmin=0, vmax=1) + + n = len(classes) + ax.set_xticks(range(n)) + ax.set_yticks(range(n)) + ax.set_xticklabels(classes, rotation=45, ha="right") + ax.set_yticklabels(classes) + + if annot: + for i in range(n): + for j in range(n): + value = matrix.values[i, j] + ax.text( + j, i, f"{value:.2f}", ha="center", va="center", + color="white" if value > 0.5 else "black", + ) + + ax.figure.colorbar(im, ax=ax, label="P(observed | true)") + ax.set_xlabel("Observed label") + ax.set_ylabel("True label (estimated)") + ax.set_title(f"Dawid-Skene Confusion Matrix: {column}") + + return ax diff --git a/lars/util/kappa_matrix.py b/lars/util/kappa_matrix.py new file mode 100644 index 0000000..ac5b45d --- /dev/null +++ b/lars/util/kappa_matrix.py @@ -0,0 +1,96 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.metrics import cohen_kappa_score + + +def calculate_kappa_matrix(df, columns=None): + """ + Compute pairwise Cohen's kappa scores between multiple label columns. + + Parameters + ---------- + df (pd.DataFrame): DataFrame containing one column per labeler/source. + columns (list of str or None): Columns to compare. Defaults to all + columns in ``df``. + + Returns + ------- + pd.DataFrame + Square DataFrame indexed and labeled by ``columns``, where entry + (i, j) is the Cohen's kappa score between columns i and j (1.0 on + the diagonal). Rows with a missing value in either of the two + compared columns are excluded from that pair's calculation. + """ + if columns is None: + columns = list(df.columns) + + matrix = pd.DataFrame(np.eye(len(columns)), index=columns, columns=columns) + + for i, col_i in enumerate(columns): + for col_j in columns[i + 1:]: + pair = df[[col_i, col_j]].dropna() + a = pair[col_i].astype(str).str.lower() + b = pair[col_j].astype(str).str.lower() + kappa = cohen_kappa_score(a, b) if len(pair) > 0 else np.nan + matrix.loc[col_i, col_j] = kappa + matrix.loc[col_j, col_i] = kappa + + return matrix + + +def plot_kappa_matrix(df, columns=None, labels=None, ax=None, cmap=None, annot=True, vmin=-1, vmax=1, + matrix=None): + """ + Plot a matrix of pairwise Cohen's kappa scores between multiple label + columns as a heatmap. + + Parameters + ---------- + df (pd.DataFrame): DataFrame containing one column per labeler/source. + columns (list of str or None): Columns to compare. Defaults to all + columns in ``df``. + labels (list of str or None): Display names for the tick labels, in the + same order as ``columns``. Defaults to the column names. + ax (matplotlib axis handle): The axis handle to plot on. Set to None to + use the current axis. + cmap (matplotlib colormap or None): Colormap for the heatmap. Defaults + to ``plt.cm.RdYlGn``. + annot (bool): Whether to annotate each cell with its kappa value. + vmin (float): Minimum value for the colormap. Defaults to -1. + vmax (float): Maximum value for the colormap. Defaults to 1. + + Returns + ------- + matplotlib.axes.Axes + The axis the kappa matrix was drawn on. + """ + if matrix is None: + matrix = calculate_kappa_matrix(df, columns=columns) + display_labels = labels if labels is not None else list(matrix.columns) + + if ax is None: + ax = plt.gca() + if cmap is None: + cmap = plt.cm.RdYlGn + + im = ax.imshow(matrix.values, cmap=cmap, vmin=vmin, vmax=vmax) + + n = len(display_labels) + ax.set_xticks(range(n)) + ax.set_yticks(range(n)) + ax.set_xticklabels(display_labels, rotation=45, ha="right") + ax.set_yticklabels(display_labels) + + if annot: + for i in range(n): + for j in range(n): + value = matrix.values[i, j] + text = "" if np.isnan(value) else f"{value:.2f}" + color = "white" if (not np.isnan(value)) and value < 0.5 else "black" + ax.text(j, i, text, ha="center", va="center", color=color) + + ax.figure.colorbar(im, ax=ax) + ax.set_title("Cohen's Kappa Matrix") + + return ax diff --git a/lars/util/label_disagreement_matrix.py b/lars/util/label_disagreement_matrix.py new file mode 100644 index 0000000..e6bb0d7 --- /dev/null +++ b/lars/util/label_disagreement_matrix.py @@ -0,0 +1,115 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + + +def calculate_label_disagreement_matrix(df, label, columns=None): + """ + Compute a matrix of pairwise item-level disagreement rates for a single + label, treated as a one-vs-rest binary classification. + + For each pair of labellers, only rows where both columns have a + non-missing value are considered. Each value is binarized to whether it + equals ``label`` (case-insensitively), and entry (i, j) of the returned + matrix is the percentage of those rows where the two labellers' + binarized values differ. + + Unlike ``calculate_label_rate_diff_matrix`` (see ``label_rate_matrix.py``), + which only compares how often each labeller uses ``label`` overall, this + compares labellers item-by-item, so it can be high even when two + labellers use ``label`` at the same overall rate but disagree on which + items it applies to. + + Parameters + ---------- + df (pd.DataFrame): DataFrame containing one column per labeller/source. + label (str): The label value to treat as the positive class. Compared + case-insensitively. + columns (list of str or None): Columns (labellers) to compare. Defaults + to all columns in ``df``. + + Returns + ------- + pd.DataFrame + Square DataFrame indexed and labeled by ``columns``, where entry + (i, j) is the item-level disagreement rate (0-100) between columns + i and j (0.0 on the diagonal). NaN if a pair shares no non-missing + rows. + """ + if columns is None: + columns = list(df.columns) + + label_lower = str(label).lower() + matrix = pd.DataFrame(0.0, index=columns, columns=columns) + + for i, col_i in enumerate(columns): + for col_j in columns[i + 1:]: + pair = df[[col_i, col_j]].dropna() + if len(pair) == 0: + diff = np.nan + else: + a = pair[col_i].astype(str).str.lower() == label_lower + b = pair[col_j].astype(str).str.lower() == label_lower + diff = 100.0 * (a != b).mean() + matrix.loc[col_i, col_j] = diff + matrix.loc[col_j, col_i] = diff + + return matrix + + +def plot_label_disagreement_matrix(df, label, columns=None, labels=None, ax=None, + cmap=None, annot=True): + """ + Plot a matrix of pairwise item-level disagreement rates for a single + label as a heatmap. + + Parameters + ---------- + df (pd.DataFrame): DataFrame containing one column per labeller/source. + label (str): The label value to treat as the positive class. + columns (list of str or None): Columns to compare. Defaults to all + columns in ``df``. + labels (list of str or None): Display names for the tick labels, in the + same order as ``columns``. Defaults to the column names. + ax (matplotlib axis handle): The axis handle to plot on. Set to None to + use the current axis. + cmap (matplotlib colormap or None): Colormap for the heatmap. Defaults + to ``plt.cm.Reds``. + annot (bool): Whether to annotate each cell with its value. + + Returns + ------- + matplotlib.axes.Axes + The axis the matrix was drawn on. + """ + matrix = calculate_label_disagreement_matrix(df, label, columns=columns) + display_labels = labels if labels is not None else list(matrix.columns) + + if ax is None: + ax = plt.gca() + if cmap is None: + cmap = plt.cm.Reds + + finite_values = matrix.values[~np.isnan(matrix.values)] + vmax = finite_values.max() if finite_values.size > 0 and finite_values.max() > 0 else 100.0 + + im = ax.imshow(matrix.values, cmap=cmap, vmin=0, vmax=vmax) + + n = len(display_labels) + ax.set_xticks(range(n)) + ax.set_yticks(range(n)) + ax.set_xticklabels(display_labels, rotation=45, ha="right") + ax.set_yticklabels(display_labels) + + if annot: + for i in range(n): + for j in range(n): + value = matrix.values[i, j] + text = "" if np.isnan(value) else f"{value:.1f}" + color = "white" if (not np.isnan(value)) and value > vmax / 2 else "black" + ax.text(j, i, text, ha="center", va="center", color=color) + + ax.figure.colorbar(im, ax=ax, label="Disagreement rate (%)") + ax.set_title(f"Item-Level Disagreement: '{label}'") + + return ax diff --git a/lars/util/label_rate_matrix.py b/lars/util/label_rate_matrix.py new file mode 100644 index 0000000..828b013 --- /dev/null +++ b/lars/util/label_rate_matrix.py @@ -0,0 +1,111 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + + +def calculate_label_rate_diff_matrix(df, label, columns=None): + """ + Compute a matrix of pairwise usage-rate differences for a single label + across multiple labellers. + + For each column (labeller), the usage rate is the percentage of its + non-missing values that equal ``label``. Entry (i, j) of the returned + matrix is the absolute difference, in percentage points, between the + usage rates of labellers i and j. + + Unlike Cohen's kappa (see ``kappa_matrix.py``), this does not require + labellers to agree item-by-item -- it only compares how often each + labeller applies ``label`` overall, so it can flag labellers that are + systematically biased toward or away from a class even when they are + scored on different items. + + Parameters + ---------- + df (pd.DataFrame): DataFrame containing one column per labeller/source. + label (str): The label value to compute usage rates for. Compared + case-insensitively. + columns (list of str or None): Columns (labellers) to compare. Defaults + to all columns in ``df``. + + Returns + ------- + pd.DataFrame + Square DataFrame indexed and labeled by ``columns``, where entry + (i, j) is the absolute usage-rate difference (0-100) between + columns i and j (0.0 on the diagonal). + """ + if columns is None: + columns = list(df.columns) + + label_lower = str(label).lower() + rates = {} + for col in columns: + values = df[col].dropna().astype(str).str.lower() + rates[col] = 100.0 * (values == label_lower).mean() if len(values) > 0 else np.nan + + matrix = pd.DataFrame(0.0, index=columns, columns=columns) + for i, col_i in enumerate(columns): + for col_j in columns[i + 1:]: + diff = abs(rates[col_i] - rates[col_j]) + matrix.loc[col_i, col_j] = diff + matrix.loc[col_j, col_i] = diff + + return matrix + + +def plot_label_rate_diff_matrix(df, label, columns=None, labels=None, ax=None, + cmap=None, annot=True): + """ + Plot a matrix of pairwise usage-rate differences for a single label + across multiple labellers as a heatmap. + + Parameters + ---------- + df (pd.DataFrame): DataFrame containing one column per labeller/source. + label (str): The label value to compute usage rates for. + columns (list of str or None): Columns to compare. Defaults to all + columns in ``df``. + labels (list of str or None): Display names for the tick labels, in the + same order as ``columns``. Defaults to the column names. + ax (matplotlib axis handle): The axis handle to plot on. Set to None to + use the current axis. + cmap (matplotlib colormap or None): Colormap for the heatmap. Defaults + to ``plt.cm.Reds``. + annot (bool): Whether to annotate each cell with its value. + + Returns + ------- + matplotlib.axes.Axes + The axis the matrix was drawn on. + """ + matrix = calculate_label_rate_diff_matrix(df, label, columns=columns) + display_labels = labels if labels is not None else list(matrix.columns) + + if ax is None: + ax = plt.gca() + if cmap is None: + cmap = plt.cm.Reds + + finite_values = matrix.values[~np.isnan(matrix.values)] + vmax = finite_values.max() if finite_values.size > 0 and finite_values.max() > 0 else 100.0 + + im = ax.imshow(matrix.values, cmap=cmap, vmin=0, vmax=vmax) + + n = len(display_labels) + ax.set_xticks(range(n)) + ax.set_yticks(range(n)) + ax.set_xticklabels(display_labels, rotation=45, ha="right") + ax.set_yticklabels(display_labels) + + if annot: + for i in range(n): + for j in range(n): + value = matrix.values[i, j] + text = "" if np.isnan(value) else f"{value:.1f}" + color = "white" if (not np.isnan(value)) and value > vmax / 2 else "black" + ax.text(j, i, text, ha="center", va="center", color=color) + + ax.figure.colorbar(im, ax=ax, label="Usage rate difference (pp)") + ax.set_title(f"Usage Rate Difference: '{label}'") + + return ax diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.000231.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.000231.png new file mode 100644 index 0000000..d7a2c5d Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.000231.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.001232.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.001232.png new file mode 100644 index 0000000..bc93a09 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.001232.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.002234.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.002234.png new file mode 100644 index 0000000..df586d0 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.002234.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.003237.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.003237.png new file mode 100644 index 0000000..1362111 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.003237.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.004239.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.004239.png new file mode 100644 index 0000000..6cb3246 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.004239.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.005240.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.005240.png new file mode 100644 index 0000000..18e671f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.005240.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.010243.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.010243.png new file mode 100644 index 0000000..cddb026 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.010243.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.011001.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.011001.png new file mode 100644 index 0000000..d9a0784 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.011001.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.012002.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.012002.png new file mode 100644 index 0000000..6588580 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.012002.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.013005.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.013005.png new file mode 100644 index 0000000..623a3d2 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.013005.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.014022.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.014022.png new file mode 100644 index 0000000..313c223 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.014022.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.015024.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.015024.png new file mode 100644 index 0000000..bf2371f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.015024.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.020026.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.020026.png new file mode 100644 index 0000000..cb374cb Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.020026.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.022000.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.022000.png new file mode 100644 index 0000000..7cacd30 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.022000.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.023004.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.023004.png new file mode 100644 index 0000000..26aaae7 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.023004.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.024005.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.024005.png new file mode 100644 index 0000000..49fa451 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.024005.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.025006.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.025006.png new file mode 100644 index 0000000..14da98f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.025006.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.030007.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.030007.png new file mode 100644 index 0000000..288f927 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.030007.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.031008.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.031008.png new file mode 100644 index 0000000..8070932 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.031008.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.032009.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.032009.png new file mode 100644 index 0000000..c6df1f6 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.032009.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.033010.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.033010.png new file mode 100644 index 0000000..f9ecc3a Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.033010.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.034011.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.034011.png new file mode 100644 index 0000000..1ba551e Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.034011.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.035012.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.035012.png new file mode 100644 index 0000000..15245b5 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.035012.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.040012.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.040012.png new file mode 100644 index 0000000..f9906df Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.040012.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.041013.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.041013.png new file mode 100644 index 0000000..05c8355 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.041013.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.042014.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.042014.png new file mode 100644 index 0000000..63d2ebb Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.042014.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.043004.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.043004.png new file mode 100644 index 0000000..a3a0f01 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.043004.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.044016.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.044016.png new file mode 100644 index 0000000..4bd84b2 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.044016.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.045052.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.045052.png new file mode 100644 index 0000000..f409f68 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.045052.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.050053.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.050053.png new file mode 100644 index 0000000..e811151 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.050053.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.051054.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.051054.png new file mode 100644 index 0000000..384223d Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.051054.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.052055.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.052055.png new file mode 100644 index 0000000..656ace1 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.052055.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.053002.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.053002.png new file mode 100644 index 0000000..72010b1 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.053002.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.054001.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.054001.png new file mode 100644 index 0000000..eee4be4 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.054001.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.055004.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.055004.png new file mode 100644 index 0000000..4632a12 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.055004.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.060005.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.060005.png new file mode 100644 index 0000000..6442f19 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.060005.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.061007.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.061007.png new file mode 100644 index 0000000..dfeb81b Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.061007.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.062009.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.062009.png new file mode 100644 index 0000000..2ff14a3 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.062009.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.063013.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.063013.png new file mode 100644 index 0000000..4562f6e Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.063013.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.064016.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.064016.png new file mode 100644 index 0000000..e3774da Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.064016.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.065019.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.065019.png new file mode 100644 index 0000000..96eaa40 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.065019.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.070021.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.070021.png new file mode 100644 index 0000000..11888da Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.070021.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.071025.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.071025.png new file mode 100644 index 0000000..5af8362 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.071025.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.072027.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.072027.png new file mode 100644 index 0000000..9d0b404 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.072027.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.073030.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.073030.png new file mode 100644 index 0000000..b7270ab Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.073030.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.074018.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.074018.png new file mode 100644 index 0000000..250d6ac Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.074018.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.075002.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.075002.png new file mode 100644 index 0000000..1275951 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.075002.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.080020.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.080020.png new file mode 100644 index 0000000..ccfe116 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.080020.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.081025.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.081025.png new file mode 100644 index 0000000..f2018fa Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.081025.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.082027.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.082027.png new file mode 100644 index 0000000..bb8a499 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.082027.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.083029.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.083029.png new file mode 100644 index 0000000..b477843 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.083029.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.084033.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.084033.png new file mode 100644 index 0000000..c803508 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.084033.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.085035.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.085035.png new file mode 100644 index 0000000..b700a4c Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.085035.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.090105.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.090105.png new file mode 100644 index 0000000..2083e18 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.090105.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.091107.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.091107.png new file mode 100644 index 0000000..4172737 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.091107.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.092110.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.092110.png new file mode 100644 index 0000000..fe382a8 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.092110.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.093112.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.093112.png new file mode 100644 index 0000000..1dc33a9 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.093112.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.094114.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.094114.png new file mode 100644 index 0000000..406c258 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.094114.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.095117.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.095117.png new file mode 100644 index 0000000..079a15f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.095117.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.100119.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.100119.png new file mode 100644 index 0000000..0651a5f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.100119.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.101141.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.101141.png new file mode 100644 index 0000000..214ebe6 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.101141.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.102143.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.102143.png new file mode 100644 index 0000000..e308b03 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.102143.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.103147.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.103147.png new file mode 100644 index 0000000..17f5293 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.103147.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.104149.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.104149.png new file mode 100644 index 0000000..aaf518f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.104149.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.105153.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.105153.png new file mode 100644 index 0000000..e74f167 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.105153.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.110155.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.110155.png new file mode 100644 index 0000000..2d2e42e Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.110155.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.111158.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.111158.png new file mode 100644 index 0000000..e08528b Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.111158.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.112201.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.112201.png new file mode 100644 index 0000000..982a292 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.112201.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.113205.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.113205.png new file mode 100644 index 0000000..2587fab Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.113205.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.114207.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.114207.png new file mode 100644 index 0000000..ccb373b Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.114207.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.115209.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.115209.png new file mode 100644 index 0000000..170fb1b Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.115209.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.120212.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.120212.png new file mode 100644 index 0000000..2e217c0 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.120212.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.121214.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.121214.png new file mode 100644 index 0000000..f41c25f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.121214.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.122216.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.122216.png new file mode 100644 index 0000000..bf67fb4 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.122216.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.123002.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.123002.png new file mode 100644 index 0000000..de64029 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.123002.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.124006.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.124006.png new file mode 100644 index 0000000..68e2105 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.124006.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.125010.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.125010.png new file mode 100644 index 0000000..6544090 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.125010.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.130013.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.130013.png new file mode 100644 index 0000000..b1192e4 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.130013.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.131015.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.131015.png new file mode 100644 index 0000000..875cd3d Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.131015.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.132017.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.132017.png new file mode 100644 index 0000000..09fd9ec Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.132017.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.133021.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.133021.png new file mode 100644 index 0000000..922b53b Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.133021.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.134023.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.134023.png new file mode 100644 index 0000000..4b348dc Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.134023.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.135027.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.135027.png new file mode 100644 index 0000000..09db53f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.135027.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.140030.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.140030.png new file mode 100644 index 0000000..b4d17f0 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.140030.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.141032.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.141032.png new file mode 100644 index 0000000..55a7315 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.141032.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.142036.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.142036.png new file mode 100644 index 0000000..f9ab597 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.142036.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.143002.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.143002.png new file mode 100644 index 0000000..12bf3fa Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.143002.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.144002.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.144002.png new file mode 100644 index 0000000..664d9b1 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.144002.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.145003.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.145003.png new file mode 100644 index 0000000..06f72a1 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.145003.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.150003.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.150003.png new file mode 100644 index 0000000..cccb49b Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.150003.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.151003.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.151003.png new file mode 100644 index 0000000..d5873c5 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.151003.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.152015.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.152015.png new file mode 100644 index 0000000..854db41 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.152015.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.153016.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.153016.png new file mode 100644 index 0000000..981d853 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.153016.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.154017.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.154017.png new file mode 100644 index 0000000..df2eaea Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.154017.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.155024.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.155024.png new file mode 100644 index 0000000..dba4892 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.155024.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.160025.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.160025.png new file mode 100644 index 0000000..bb0caf1 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.160025.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.161028.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.161028.png new file mode 100644 index 0000000..9a81b19 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.161028.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.162028.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.162028.png new file mode 100644 index 0000000..0924211 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.162028.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.163030.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.163030.png new file mode 100644 index 0000000..37b5fc2 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.163030.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.164032.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.164032.png new file mode 100644 index 0000000..994f572 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.164032.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.165034.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.165034.png new file mode 100644 index 0000000..b645098 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.165034.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.170035.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.170035.png new file mode 100644 index 0000000..0b79008 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.170035.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.171036.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.171036.png new file mode 100644 index 0000000..52e223e Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.171036.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.172038.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.172038.png new file mode 100644 index 0000000..d2fd062 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.172038.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.173040.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.173040.png new file mode 100644 index 0000000..238b0bb Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.173040.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.174042.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.174042.png new file mode 100644 index 0000000..bb59727 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.174042.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.175043.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.175043.png new file mode 100644 index 0000000..a806ed3 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.175043.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.180045.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.180045.png new file mode 100644 index 0000000..c1d6d37 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.180045.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.181047.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.181047.png new file mode 100644 index 0000000..1b21b43 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.181047.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.182048.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.182048.png new file mode 100644 index 0000000..a9f9fbc Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.182048.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.183049.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.183049.png new file mode 100644 index 0000000..cecdd66 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.183049.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.184050.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.184050.png new file mode 100644 index 0000000..2a5288f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.184050.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.185050.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.185050.png new file mode 100644 index 0000000..8f9a0fe Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.185050.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.190050.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.190050.png new file mode 100644 index 0000000..43e2362 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.190050.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.191051.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.191051.png new file mode 100644 index 0000000..bc1c1e2 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.191051.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.192115.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.192115.png new file mode 100644 index 0000000..6c50714 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.192115.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.193116.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.193116.png new file mode 100644 index 0000000..2d5fbe0 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.193116.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.194116.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.194116.png new file mode 100644 index 0000000..b8c1aa9 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.194116.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.200000.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.200000.png new file mode 100644 index 0000000..932a205 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.200000.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.201002.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.201002.png new file mode 100644 index 0000000..f3a25a8 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.201002.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.202004.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.202004.png new file mode 100644 index 0000000..0a698be Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.202004.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.203006.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.203006.png new file mode 100644 index 0000000..3cbdf0c Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.203006.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.204009.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.204009.png new file mode 100644 index 0000000..82a638f Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.204009.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.205011.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.205011.png new file mode 100644 index 0000000..3e3b0ca Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.205011.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.210012.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.210012.png new file mode 100644 index 0000000..858fb91 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.210012.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.211014.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.211014.png new file mode 100644 index 0000000..1968358 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.211014.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.212016.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.212016.png new file mode 100644 index 0000000..95117e9 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.212016.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.213018.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.213018.png new file mode 100644 index 0000000..4e23571 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.213018.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.214020.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.214020.png new file mode 100644 index 0000000..1d093be Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.214020.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.215048.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.215048.png new file mode 100644 index 0000000..3c5500c Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.215048.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.220050.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.220050.png new file mode 100644 index 0000000..c252638 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.220050.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.221053.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.221053.png new file mode 100644 index 0000000..bb26409 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.221053.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.222055.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.222055.png new file mode 100644 index 0000000..f7c5f60 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.222055.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.223057.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.223057.png new file mode 100644 index 0000000..9e857d7 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.223057.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.224030.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.224030.png new file mode 100644 index 0000000..9891e22 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.224030.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.225032.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.225032.png new file mode 100644 index 0000000..462b7c7 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.225032.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.230034.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.230034.png new file mode 100644 index 0000000..9336fcc Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.230034.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.231036.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.231036.png new file mode 100644 index 0000000..2f5d7af Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.231036.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.232038.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.232038.png new file mode 100644 index 0000000..54fd341 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.232038.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.233040.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.233040.png new file mode 100644 index 0000000..605c325 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.233040.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.234042.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.234042.png new file mode 100644 index 0000000..dca0386 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.234042.png differ diff --git a/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.235045.png b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.235045.png new file mode 100644 index 0000000..65ae550 Binary files /dev/null and b/tests/data/test_images/bnfcsapr2cmacS3.c1.20250527.235045.png differ diff --git a/tests/test_dawid_skene.py b/tests/test_dawid_skene.py new file mode 100644 index 0000000..561e903 --- /dev/null +++ b/tests/test_dawid_skene.py @@ -0,0 +1,123 @@ +import pytest +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + + +@pytest.fixture +def sample_df(): + n_cycles = 10 + true_pattern = ["convective", "stratiform", "anvil"] * n_cycles + + rater_a = list(true_pattern) + rater_b = list(true_pattern) + rater_b[0], rater_b[1] = "stratiform", "convective" + rater_c = ["convective" if label == "anvil" else label for label in true_pattern] + + # An extra row where all three raters pick a different class: genuinely + # ambiguous, unlike the systematic rater_c bias above (which the model + # learns to discount and remains confident about). + rater_a.append("convective") + rater_b.append("stratiform") + rater_c.append("anvil") + exp_good = true_pattern + ["convective"] + exp_bad = ["anvil"] * len(true_pattern) + ["anvil"] + + return pd.DataFrame({ + "rater_a": rater_a, + "rater_b": rater_b, + "rater_c": rater_c, + "exp_good": exp_good, + "exp_bad": exp_bad, + }) + + +@pytest.fixture(autouse=True) +def close_figures(): + yield + plt.close("all") + + +@pytest.fixture +def fitted(sample_df): + from lars.util.dawid_skene import fit_dawid_skene + + return fit_dawid_skene(sample_df, columns=["rater_a", "rater_b", "rater_c"]) + + +def test_confusion_matrices_are_row_stochastic(fitted): + for matrix in fitted["confusion_matrices"].values(): + np.testing.assert_array_almost_equal(matrix.sum(axis=1).values, np.ones(len(matrix))) + + +def test_confusion_matrices_indexed_by_columns(fitted, sample_df): + assert set(fitted["confusion_matrices"].keys()) == {"rater_a", "rater_b", "rater_c"} + for matrix in fitted["confusion_matrices"].values(): + assert set(matrix.index) == {"convective", "stratiform", "anvil"} + assert set(matrix.columns) == {"convective", "stratiform", "anvil"} + + +def test_noisy_rater_has_lower_diagonal_mass_for_biased_class(fitted): + clean = fitted["confusion_matrices"]["rater_a"].loc["anvil", "anvil"] + noisy = fitted["confusion_matrices"]["rater_c"].loc["anvil", "anvil"] + assert clean > noisy + + +def test_consensus_recovers_true_pattern(fitted, sample_df): + cyclic_rows = slice(0, -1) + assert (fitted["consensus"].values[cyclic_rows] == sample_df["exp_good"].values[cyclic_rows]).all() + + +def test_unanimous_row_has_near_zero_entropy(fitted): + assert fitted["item_entropy"].iloc[3] < 0.01 + + +def test_three_way_disagreement_has_higher_entropy_than_unanimous(fitted): + assert fitted["item_entropy"].iloc[-1] > fitted["item_entropy"].iloc[3] + + +def test_consensus_proba_rows_sum_to_one(fitted): + np.testing.assert_array_almost_equal( + fitted["consensus_proba"].sum(axis=1).values, np.ones(len(fitted["consensus_proba"])) + ) + + +def test_subset_of_columns(sample_df): + from lars.util.dawid_skene import fit_dawid_skene + + result = fit_dawid_skene(sample_df, columns=["rater_a", "rater_c"]) + assert result["columns"] == ["rater_a", "rater_c"] + assert set(result["confusion_matrices"].keys()) == {"rater_a", "rater_c"} + + +def test_score_against_consensus_ranks_good_above_bad(fitted, sample_df): + from lars.util.dawid_skene import score_against_consensus + + scores = score_against_consensus(sample_df, fitted, columns=["exp_good", "exp_bad"]) + assert list(scores.index)[0] == "exp_good" + assert scores.loc["exp_good", "accuracy"] == pytest.approx(1.0) + assert scores.loc["exp_good", "macro_f1"] > scores.loc["exp_bad", "macro_f1"] + + +def test_score_against_consensus_sorted_by_macro_f1(fitted, sample_df): + from lars.util.dawid_skene import score_against_consensus + + scores = score_against_consensus(sample_df, fitted, columns=["exp_bad", "exp_good"]) + assert scores["macro_f1"].is_monotonic_decreasing + + +def test_plot_returns_axes(fitted): + from lars.util.dawid_skene import plot_dawid_skene_confusion + + _, ax = plt.subplots() + result = plot_dawid_skene_confusion(fitted, "rater_a", ax=ax) + assert result is ax + + +def test_plot_tick_labels_match_classes(fitted): + from lars.util.dawid_skene import plot_dawid_skene_confusion + + _, ax = plt.subplots() + plot_dawid_skene_confusion(fitted, "rater_a", ax=ax) + labels = set(t.get_text() for t in ax.get_xticklabels()) + assert labels == {"convective", "stratiform", "anvil"} diff --git a/tests/test_kappa_matrix.py b/tests/test_kappa_matrix.py new file mode 100644 index 0000000..91eb533 --- /dev/null +++ b/tests/test_kappa_matrix.py @@ -0,0 +1,114 @@ +import io +import pytest +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +CSV_DATA = """\ +rater_a,rater_b,rater_c +stratiform,stratiform,stratiform +convective,convective,stratiform +stratiform,convective,convective +convective,stratiform,convective +anvil,anvil,anvil +stratiform,stratiform,convective +""" + + +@pytest.fixture +def sample_df(): + return pd.read_csv(io.StringIO(CSV_DATA)) + + +@pytest.fixture(autouse=True) +def close_figures(): + yield + plt.close("all") + + +def test_matrix_is_square_and_symmetric(sample_df): + from lars.util.kappa_matrix import calculate_kappa_matrix + + matrix = calculate_kappa_matrix(sample_df) + assert list(matrix.columns) == ["rater_a", "rater_b", "rater_c"] + assert list(matrix.index) == ["rater_a", "rater_b", "rater_c"] + np.testing.assert_array_almost_equal(matrix.values, matrix.values.T) + + +def test_diagonal_is_one(sample_df): + from lars.util.kappa_matrix import calculate_kappa_matrix + + matrix = calculate_kappa_matrix(sample_df) + np.testing.assert_array_almost_equal(np.diag(matrix.values), np.ones(3)) + + +def test_matrix_matches_pairwise_kappa(sample_df): + from lars.util.kappa_matrix import calculate_kappa_matrix + from lars.util.confusion_matrix import calculate_cohen_kappa + + matrix = calculate_kappa_matrix(sample_df) + expected = calculate_cohen_kappa( + sample_df.rename(columns={"rater_a": "label", "rater_b": "llm_label"}) + ) + assert matrix.loc["rater_a", "rater_b"] == pytest.approx(expected) + + +def test_subset_of_columns(sample_df): + from lars.util.kappa_matrix import calculate_kappa_matrix + + matrix = calculate_kappa_matrix(sample_df, columns=["rater_a", "rater_c"]) + assert list(matrix.columns) == ["rater_a", "rater_c"] + + +def test_excludes_rows_with_missing_values(): + from lars.util.kappa_matrix import calculate_kappa_matrix + + df = pd.DataFrame({ + "a": ["x", "y", "x", None], + "b": ["x", "y", "y", "x"], + }) + matrix = calculate_kappa_matrix(df) + assert not np.isnan(matrix.loc["a", "b"]) + + +def test_plot_returns_axes(sample_df): + from lars.util.kappa_matrix import plot_kappa_matrix + + _, ax = plt.subplots() + result = plot_kappa_matrix(sample_df, ax=ax) + assert result is ax + + +def test_plot_title_is_set(sample_df): + from lars.util.kappa_matrix import plot_kappa_matrix + + _, ax = plt.subplots() + plot_kappa_matrix(sample_df, ax=ax) + assert ax.get_title() == "Cohen's Kappa Matrix" + + +def test_plot_tick_labels_match_columns(sample_df): + from lars.util.kappa_matrix import plot_kappa_matrix + + _, ax = plt.subplots() + plot_kappa_matrix(sample_df, ax=ax) + labels = [t.get_text() for t in ax.get_xticklabels()] + assert labels == ["rater_a", "rater_b", "rater_c"] + + +def test_plot_custom_labels(sample_df): + from lars.util.kappa_matrix import plot_kappa_matrix + + _, ax = plt.subplots() + plot_kappa_matrix(sample_df, ax=ax, labels=["Alice", "Bob", "LLM"]) + labels = [t.get_text() for t in ax.get_xticklabels()] + assert labels == ["Alice", "Bob", "LLM"] + + +def test_plot_uses_gca_when_no_ax(sample_df): + from lars.util.kappa_matrix import plot_kappa_matrix + + fig, ax = plt.subplots() + plt.sca(ax) + plot_kappa_matrix(sample_df) + assert ax.get_title() == "Cohen's Kappa Matrix" diff --git a/tests/test_label_disagreement_matrix.py b/tests/test_label_disagreement_matrix.py new file mode 100644 index 0000000..c6f3f7b --- /dev/null +++ b/tests/test_label_disagreement_matrix.py @@ -0,0 +1,141 @@ +import io +import pytest +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +CSV_DATA = """\ +rater_a,rater_b,rater_c +stratiform,stratiform,stratiform +convective,convective,stratiform +stratiform,convective,convective +convective,stratiform,convective +anvil,anvil,anvil +stratiform,stratiform,convective +""" + + +@pytest.fixture +def sample_df(): + return pd.read_csv(io.StringIO(CSV_DATA)) + + +@pytest.fixture(autouse=True) +def close_figures(): + yield + plt.close("all") + + +def test_matrix_is_square_and_symmetric(sample_df): + from lars.util.label_disagreement_matrix import calculate_label_disagreement_matrix + + matrix = calculate_label_disagreement_matrix(sample_df, "stratiform") + assert list(matrix.columns) == ["rater_a", "rater_b", "rater_c"] + assert list(matrix.index) == ["rater_a", "rater_b", "rater_c"] + np.testing.assert_array_almost_equal(matrix.values, matrix.values.T) + + +def test_diagonal_is_zero(sample_df): + from lars.util.label_disagreement_matrix import calculate_label_disagreement_matrix + + matrix = calculate_label_disagreement_matrix(sample_df, "stratiform") + np.testing.assert_array_almost_equal(np.diag(matrix.values), np.zeros(3)) + + +def test_disagreement_values(sample_df): + """Binarized 'stratiform' (T/F) per row: + a: T F T F F T + b: T F F T F T + c: T T F F F F + a vs b disagree at rows 2,3 -> 2/6 + a vs c disagree at rows 1,2,5 -> 3/6 + b vs c disagree at rows 1,3,5 -> 3/6 + """ + from lars.util.label_disagreement_matrix import calculate_label_disagreement_matrix + + matrix = calculate_label_disagreement_matrix(sample_df, "stratiform") + assert matrix.loc["rater_a", "rater_b"] == pytest.approx(100 / 3) + assert matrix.loc["rater_a", "rater_c"] == pytest.approx(50.0) + assert matrix.loc["rater_b", "rater_c"] == pytest.approx(50.0) + + +def test_label_is_case_insensitive(sample_df): + from lars.util.label_disagreement_matrix import calculate_label_disagreement_matrix + + lower = calculate_label_disagreement_matrix(sample_df, "stratiform") + upper = calculate_label_disagreement_matrix(sample_df, "STRATIFORM") + np.testing.assert_array_almost_equal(lower.values, upper.values) + + +def test_subset_of_columns(sample_df): + from lars.util.label_disagreement_matrix import calculate_label_disagreement_matrix + + matrix = calculate_label_disagreement_matrix(sample_df, "stratiform", columns=["rater_a", "rater_c"]) + assert list(matrix.columns) == ["rater_a", "rater_c"] + + +def test_excludes_rows_with_missing_values(): + from lars.util.label_disagreement_matrix import calculate_label_disagreement_matrix + + df = pd.DataFrame({ + "a": ["x", "y", "x", None], + "b": ["x", "y", "y", "x"], + }) + matrix = calculate_label_disagreement_matrix(df, "x") + # Row 4 dropped (a missing). Remaining rows: (x,x) agree, (y,y) agree + # (both non-x -> agree), (x,y) disagree -> 1/3 disagreement. + assert matrix.loc["a", "b"] == pytest.approx(100 / 3) + + +def test_nan_when_no_shared_rows(): + from lars.util.label_disagreement_matrix import calculate_label_disagreement_matrix + + df = pd.DataFrame({ + "a": ["x", None], + "b": [None, "y"], + }) + matrix = calculate_label_disagreement_matrix(df, "x") + assert np.isnan(matrix.loc["a", "b"]) + + +def test_plot_returns_axes(sample_df): + from lars.util.label_disagreement_matrix import plot_label_disagreement_matrix + + _, ax = plt.subplots() + result = plot_label_disagreement_matrix(sample_df, "stratiform", ax=ax) + assert result is ax + + +def test_plot_title_includes_label(sample_df): + from lars.util.label_disagreement_matrix import plot_label_disagreement_matrix + + _, ax = plt.subplots() + plot_label_disagreement_matrix(sample_df, "stratiform", ax=ax) + assert ax.get_title() == "Item-Level Disagreement: 'stratiform'" + + +def test_plot_tick_labels_match_columns(sample_df): + from lars.util.label_disagreement_matrix import plot_label_disagreement_matrix + + _, ax = plt.subplots() + plot_label_disagreement_matrix(sample_df, "stratiform", ax=ax) + labels = [t.get_text() for t in ax.get_xticklabels()] + assert labels == ["rater_a", "rater_b", "rater_c"] + + +def test_plot_custom_labels(sample_df): + from lars.util.label_disagreement_matrix import plot_label_disagreement_matrix + + _, ax = plt.subplots() + plot_label_disagreement_matrix(sample_df, "stratiform", ax=ax, labels=["Alice", "Bob", "LLM"]) + labels = [t.get_text() for t in ax.get_xticklabels()] + assert labels == ["Alice", "Bob", "LLM"] + + +def test_plot_uses_gca_when_no_ax(sample_df): + from lars.util.label_disagreement_matrix import plot_label_disagreement_matrix + + fig, ax = plt.subplots() + plt.sca(ax) + plot_label_disagreement_matrix(sample_df, "stratiform") + assert ax.get_title() == "Item-Level Disagreement: 'stratiform'" diff --git a/tests/test_label_rate_matrix.py b/tests/test_label_rate_matrix.py new file mode 100644 index 0000000..f825319 --- /dev/null +++ b/tests/test_label_rate_matrix.py @@ -0,0 +1,122 @@ +import io +import pytest +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +CSV_DATA = """\ +rater_a,rater_b,rater_c +stratiform,stratiform,stratiform +convective,convective,stratiform +stratiform,convective,convective +convective,stratiform,convective +anvil,anvil,anvil +stratiform,stratiform,convective +""" + + +@pytest.fixture +def sample_df(): + return pd.read_csv(io.StringIO(CSV_DATA)) + + +@pytest.fixture(autouse=True) +def close_figures(): + yield + plt.close("all") + + +def test_matrix_is_square_and_symmetric(sample_df): + from lars.util.label_rate_matrix import calculate_label_rate_diff_matrix + + matrix = calculate_label_rate_diff_matrix(sample_df, "stratiform") + assert list(matrix.columns) == ["rater_a", "rater_b", "rater_c"] + assert list(matrix.index) == ["rater_a", "rater_b", "rater_c"] + np.testing.assert_array_almost_equal(matrix.values, matrix.values.T) + + +def test_diagonal_is_zero(sample_df): + from lars.util.label_rate_matrix import calculate_label_rate_diff_matrix + + matrix = calculate_label_rate_diff_matrix(sample_df, "stratiform") + np.testing.assert_array_almost_equal(np.diag(matrix.values), np.zeros(3)) + + +def test_usage_rate_diff_values(sample_df): + """rater_a: 3/6=50% stratiform, rater_b: 3/6=50%, rater_c: 2/6=33.33%.""" + from lars.util.label_rate_matrix import calculate_label_rate_diff_matrix + + matrix = calculate_label_rate_diff_matrix(sample_df, "stratiform") + assert matrix.loc["rater_a", "rater_b"] == pytest.approx(0.0) + assert matrix.loc["rater_a", "rater_c"] == pytest.approx(100 / 6) + assert matrix.loc["rater_b", "rater_c"] == pytest.approx(100 / 6) + + +def test_label_is_case_insensitive(sample_df): + from lars.util.label_rate_matrix import calculate_label_rate_diff_matrix + + lower = calculate_label_rate_diff_matrix(sample_df, "stratiform") + upper = calculate_label_rate_diff_matrix(sample_df, "STRATIFORM") + np.testing.assert_array_almost_equal(lower.values, upper.values) + + +def test_subset_of_columns(sample_df): + from lars.util.label_rate_matrix import calculate_label_rate_diff_matrix + + matrix = calculate_label_rate_diff_matrix(sample_df, "stratiform", columns=["rater_a", "rater_c"]) + assert list(matrix.columns) == ["rater_a", "rater_c"] + + +def test_ignores_missing_values(): + from lars.util.label_rate_matrix import calculate_label_rate_diff_matrix + + df = pd.DataFrame({ + "a": ["x", "x", None], + "b": ["x", "y", "y"], + }) + matrix = calculate_label_rate_diff_matrix(df, "x") + # a: 2/2 non-missing are 'x' -> 100%; b: 1/3 are 'x' -> 33.33% + assert matrix.loc["a", "b"] == pytest.approx(100.0 - 100.0 / 3) + + +def test_plot_returns_axes(sample_df): + from lars.util.label_rate_matrix import plot_label_rate_diff_matrix + + _, ax = plt.subplots() + result = plot_label_rate_diff_matrix(sample_df, "stratiform", ax=ax) + assert result is ax + + +def test_plot_title_includes_label(sample_df): + from lars.util.label_rate_matrix import plot_label_rate_diff_matrix + + _, ax = plt.subplots() + plot_label_rate_diff_matrix(sample_df, "stratiform", ax=ax) + assert ax.get_title() == "Usage Rate Difference: 'stratiform'" + + +def test_plot_tick_labels_match_columns(sample_df): + from lars.util.label_rate_matrix import plot_label_rate_diff_matrix + + _, ax = plt.subplots() + plot_label_rate_diff_matrix(sample_df, "stratiform", ax=ax) + labels = [t.get_text() for t in ax.get_xticklabels()] + assert labels == ["rater_a", "rater_b", "rater_c"] + + +def test_plot_custom_labels(sample_df): + from lars.util.label_rate_matrix import plot_label_rate_diff_matrix + + _, ax = plt.subplots() + plot_label_rate_diff_matrix(sample_df, "stratiform", ax=ax, labels=["Alice", "Bob", "LLM"]) + labels = [t.get_text() for t in ax.get_xticklabels()] + assert labels == ["Alice", "Bob", "LLM"] + + +def test_plot_uses_gca_when_no_ax(sample_df): + from lars.util.label_rate_matrix import plot_label_rate_diff_matrix + + fig, ax = plt.subplots() + plt.sca(ax) + plot_label_rate_diff_matrix(sample_df, "stratiform") + assert ax.get_title() == "Usage Rate Difference: 'stratiform'" diff --git a/tests/test_labels.py b/tests/test_labels.py new file mode 100644 index 0000000..888edb7 --- /dev/null +++ b/tests/test_labels.py @@ -0,0 +1,168 @@ +import os +import pytest +import pandas as pd + + +def _write_csv(path, contents): + with open(path, "w") as f: + f.write(contents) + return str(path) + + +def test_combine_labels_matches_on_file_path(tmp_path): + from lars.preprocessing.labels import combine_labels + + human_csv = _write_csv( + tmp_path / "human.csv", + "file_path,label\n" + "/a/img1.png,No Precipitation\n" + "/a/img2.png,Stratiform Precipitation\n", + ) + llm_csv = _write_csv( + tmp_path / "llm.csv", + "file_path,label\n" + "/b/img1.png,No Precipitation\n" + "/b/img2.png,Isolated Convection\n", + ) + + combined = combine_labels( + [human_csv, llm_csv], ["human_alice", "llm_l4scout"] + ) + + assert list(combined.columns) == ["file_path", "label", "source"] + assert len(combined) == 4 + assert set(combined["source"]) == {"human_alice", "llm_l4scout"} + assert set(combined["file_path"]) == {"img1.png", "img2.png"} + + +def test_combine_labels_matches_on_time(tmp_path): + from lars.preprocessing.labels import combine_labels + + human_csv = _write_csv( + tmp_path / "human.csv", + "time,label\n3/4/25 0:00,No Precipitation\n3/4/25 0:12,Isolated Convection\n", + ) + llm_csv = _write_csv( + tmp_path / "llm.csv", + "time,label\n3/4/25 0:00,No Precipitation\n3/4/25 0:12,Stratiform Precipitation\n", + ) + + combined = combine_labels( + [human_csv, llm_csv], ["human_bob", "llm_l4scout"], match_on="time" + ) + + assert list(combined.columns) == ["time", "label", "source"] + assert len(combined) == 4 + assert set(combined["time"]) == {"3/4/25 0:00", "3/4/25 0:12"} + + +def test_combine_labels_custom_label_column(tmp_path): + from lars.preprocessing.labels import combine_labels + + csv1 = _write_csv( + tmp_path / "a.csv", + "file_path,llm_label\n/a/img1.png,No Precipitation\n", + ) + csv2 = _write_csv( + tmp_path / "b.csv", + "file_path,llm_label\n/b/img1.png,Isolated Convection\n", + ) + + combined = combine_labels( + [csv1, csv2], ["run1", "run2"], label_column="llm_label" + ) + + assert list(combined.columns) == ["file_path", "llm_label", "source"] + assert combined["llm_label"].tolist() == [ + "No Precipitation", "Isolated Convection" + ] + + +def test_combine_labels_mismatched_lengths_raises(tmp_path): + from lars.preprocessing.labels import combine_labels + + csv1 = _write_csv(tmp_path / "a.csv", "file_path,label\n/a/img1.png,x\n") + + with pytest.raises(ValueError): + combine_labels([csv1], ["only_one", "too_many"]) + + +def test_combine_labels_invalid_match_on_raises(tmp_path): + from lars.preprocessing.labels import combine_labels + + csv1 = _write_csv(tmp_path / "a.csv", "file_path,label\n/a/img1.png,x\n") + + with pytest.raises(ValueError): + combine_labels([csv1], ["source1"], match_on="bogus") + + +def test_standardize_labels_maps_ambiguous_and_unknown_variants(): + from lars.preprocessing.labels import standardize_labels + + df = pd.DataFrame({ + "label": ["Ambiguous", "UNKNOWN", "unknown", " Unknown ", "ambiguous"], + }) + + result = standardize_labels(df) + + assert result["label"].tolist() == ["Ambiguous / Uncertain"] * 5 + + +def test_standardize_labels_maps_bare_stratiform(): + from lars.preprocessing.labels import standardize_labels + + df = pd.DataFrame({"label": ["Stratiform", "stratiform", "STRATIFORM"]}) + + result = standardize_labels(df) + + assert result["label"].tolist() == ["Stratiform Precipitation"] * 3 + + +def test_standardize_labels_leaves_canonical_and_other_labels_unchanged(): + from lars.preprocessing.labels import standardize_labels + + df = pd.DataFrame({ + "label": [ + "No Precipitation", + "Stratiform Precipitation", + "Isolated Convection", + "Mesoscale Convective System", + ], + }) + + result = standardize_labels(df) + + assert result["label"].tolist() == df["label"].tolist() + + +def test_standardize_labels_preserves_missing_values(): + from lars.preprocessing.labels import standardize_labels + + df = pd.DataFrame({"label": ["Ambiguous", None]}) + + result = standardize_labels(df) + + assert result["label"].iloc[0] == "Ambiguous / Uncertain" + assert pd.isna(result["label"].iloc[1]) + + +def test_standardize_labels_custom_label_column(): + from lars.preprocessing.labels import standardize_labels + + df = pd.DataFrame({"llm_label": ["UNKNOWN", "Stratiform"]}) + + result = standardize_labels(df, label_column="llm_label") + + assert result["llm_label"].tolist() == [ + "Ambiguous / Uncertain", "Stratiform Precipitation" + ] + + +def test_standardize_labels_does_not_mutate_input(): + from lars.preprocessing.labels import standardize_labels + + df = pd.DataFrame({"label": ["UNKNOWN"]}) + + standardize_labels(df) + + assert df["label"].tolist() == ["UNKNOWN"]