From a2dd71f5ef141d8da04e85e925ede49f1da5ed82 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Mon, 10 Aug 2026 16:28:19 -0500 Subject: [PATCH 1/3] Support flag_values/flag_meanings for gate_id categories in quicklooks The PPI and RHI quicklook hydrometeor ID plots only understood the colon-separated notes attribute for gate_id categories. Add a shared gate_id helper that also recognizes the CF-style flag_values/flag_meanings attributes (space-separated meanings paired with integer codes). --- cmac/cmac_ppi_quicklooks.py | 13 ++++----- cmac/cmac_rhi_quicklooks.py | 13 ++++----- cmac/gate_id.py | 53 +++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 cmac/gate_id.py diff --git a/cmac/cmac_ppi_quicklooks.py b/cmac/cmac_ppi_quicklooks.py index 6855b05..30f99d3 100644 --- a/cmac/cmac_ppi_quicklooks.py +++ b/cmac/cmac_ppi_quicklooks.py @@ -16,6 +16,7 @@ generate_radar_name, generate_radar_time_begin) from .config import get_plot_values, get_field_names +from .gate_id import get_gate_id_categories, gate_id_has_category plt.switch_backend('agg') @@ -190,13 +191,12 @@ def _range(key, default): # Four panel plot of gate_id, velocity_texture, reflectivity, and # cross_correlation_ratio. - cat_dict = {} + gate_id_field = radar.fields['gate_id'] + cat_dict = get_gate_id_categories(gate_id_field) print('##') print('## Keys for each gate id are as follows:') - for i, pair_str in enumerate(radar.fields['gate_id']['notes'].split(',')): - pair_str = pair_str.split(':')[1].strip() - print('## ', str(pair_str)) - cat_dict.update({pair_str: i}) + for label in sorted(cat_dict, key=cat_dict.get): + print('## ', str(label)) sorted_cats = sorted(cat_dict.items(), key=operator.itemgetter(1)) cat_colors = dict(cat_colors_cfg) lab_colors = [cat_colors[kitty[0]] for kitty in sorted_cats] @@ -218,7 +218,8 @@ def _range(key, default): colors='k') cbax = ax[0, 0] - if 'ground_clutter' in radar.fields.keys() or 'terrain_blockage' in radar.fields['gate_id']['notes']: + if ('ground_clutter' in radar.fields.keys() + or gate_id_has_category(gate_id_field, 'terrain_blockage')): tick_locs = np.linspace( 0, len(sorted_cats) - 1, len(sorted_cats)) + 0.5 else: diff --git a/cmac/cmac_rhi_quicklooks.py b/cmac/cmac_rhi_quicklooks.py index 73051e7..fcbb5f7 100644 --- a/cmac/cmac_rhi_quicklooks.py +++ b/cmac/cmac_rhi_quicklooks.py @@ -15,6 +15,7 @@ generate_radar_name, generate_radar_time_begin) from .config import get_plot_values, get_field_names +from .gate_id import get_gate_id_categories, gate_id_has_category plt.switch_backend('agg') @@ -106,13 +107,12 @@ def _range(key, default): # Four panel plot of gate_id, velocity_texture, reflectivity, and # cross_correlation_ratio. - cat_dict = {} + gate_id_field = radar.fields['gate_id'] + cat_dict = get_gate_id_categories(gate_id_field) print('##') print('## Keys for each gate id are as follows:') - for i, pair_str in enumerate(radar.fields['gate_id']['notes'].split(',')): - pair_str = pair_str.split(':')[1].strip() - print('## ', str(pair_str)) - cat_dict.update({pair_str: i}) + for label in sorted(cat_dict, key=cat_dict.get): + print('## ', str(label)) sorted_cats = sorted(cat_dict.items(), key=operator.itemgetter(1)) cat_colors = dict(cat_colors_cfg) @@ -126,7 +126,8 @@ def _range(key, default): cmap=cmap, vmin=0, vmax=6) cbax = ax[0, 0] - if 'ground_clutter' in radar.fields.keys() or 'terrain_blockage' in radar.fields['gate_id']['notes']: + if ('ground_clutter' in radar.fields.keys() + or gate_id_has_category(gate_id_field, 'terrain_blockage')): tick_locs = np.linspace( 0, len(sorted_cats) - 1, len(sorted_cats)) + 0.5 else: diff --git a/cmac/gate_id.py b/cmac/gate_id.py new file mode 100644 index 0000000..91cf65e --- /dev/null +++ b/cmac/gate_id.py @@ -0,0 +1,53 @@ +""" +Helpers for interpreting the category metadata attached to a ``gate_id`` +(hydrometeor ID) radar field. + +CMAC's own gate id fields document their categories with a ``notes`` +attribute: a comma separated list of ``"index: label"`` pairs, e.g. +``"0: multi_trip, 1: rain, 2: snow"``. Fields that follow the CF +conventions instead (or radar objects re-read from a file that converted +``notes`` on save) may document the same information with a +``flag_meanings`` attribute (a space separated list of labels) and a +parallel ``flag_values`` attribute (the matching integer codes). +""" + + +def get_gate_id_categories(gate_id_field): + """ + Return a dict mapping each gate id category label to its integer code. + + Parameters + ---------- + gate_id_field : dict + A Py-ART field dictionary, e.g. ``radar.fields['gate_id']``. + + """ + if 'notes' in gate_id_field: + cat_dict = {} + for i, pair_str in enumerate(gate_id_field['notes'].split(',')): + label = pair_str.split(':')[1].strip() + cat_dict[label] = i + return cat_dict + + if 'flag_meanings' in gate_id_field and 'flag_values' in gate_id_field: + labels = gate_id_field['flag_meanings'].split() + values = gate_id_field['flag_values'] + return {label: int(value) for label, value in zip(labels, values)} + + raise KeyError( + "The 'gate_id' field must have either a 'notes' attribute or " + "'flag_values'/'flag_meanings' attributes describing its " + "categories.") + + +def gate_id_has_category(gate_id_field, category): + """ + Return True if ``category`` is one of the documented categories of a + ``gate_id`` field, whether documented via ``notes`` or via + ``flag_meanings``. + """ + if 'notes' in gate_id_field: + return category in gate_id_field['notes'] + if 'flag_meanings' in gate_id_field: + return category in gate_id_field['flag_meanings'].split() + return False From 6a709c31019009b031315bbae5c50ae08e0c9c60 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Mon, 10 Aug 2026 16:47:42 -0500 Subject: [PATCH 2/3] Accept space-separated notes and comma-separated flag_meanings notes pairs may separate the index from the label with whitespace instead of a colon, and flag_meanings is generally comma separated rather than the CF-standard whitespace separation. Handle both forms for each attribute. --- cmac/gate_id.py | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/cmac/gate_id.py b/cmac/gate_id.py index 91cf65e..4008ce3 100644 --- a/cmac/gate_id.py +++ b/cmac/gate_id.py @@ -4,13 +4,37 @@ CMAC's own gate id fields document their categories with a ``notes`` attribute: a comma separated list of ``"index: label"`` pairs, e.g. -``"0: multi_trip, 1: rain, 2: snow"``. Fields that follow the CF -conventions instead (or radar objects re-read from a file that converted -``notes`` on save) may document the same information with a -``flag_meanings`` attribute (a space separated list of labels) and a -parallel ``flag_values`` attribute (the matching integer codes). +``"0: multi_trip, 1: rain, 2: snow"``. The index/label separator within +each pair may be either a colon or plain whitespace (``"0 multi_trip"``). +Fields that follow the CF conventions instead (or radar objects re-read +from a file that converted ``notes`` on save) may document the same +information with a ``flag_meanings`` attribute and a parallel +``flag_values`` attribute (the matching integer codes). ``flag_meanings`` +is generally comma separated, though a plain whitespace separated string +is also accepted. """ +import re + +_PAIR_SEP_RE = re.compile(r'[:\s]+') + + +def _label_from_pair(pair_str): + """Extract the category label from a single ``"index: label"`` pair, + where the index/label separator is a colon, whitespace, or both.""" + parts = _PAIR_SEP_RE.split(pair_str.strip(), maxsplit=1) + return parts[-1].strip() + + +def _split_flag_meanings(flag_meanings): + """Split a ``flag_meanings`` attribute into its individual labels, + whether it is comma separated or plain whitespace separated.""" + if ',' in flag_meanings: + parts = flag_meanings.split(',') + else: + parts = flag_meanings.split() + return [part.strip() for part in parts if part.strip()] + def get_gate_id_categories(gate_id_field): """ @@ -25,12 +49,11 @@ def get_gate_id_categories(gate_id_field): if 'notes' in gate_id_field: cat_dict = {} for i, pair_str in enumerate(gate_id_field['notes'].split(',')): - label = pair_str.split(':')[1].strip() - cat_dict[label] = i + cat_dict[_label_from_pair(pair_str)] = i return cat_dict if 'flag_meanings' in gate_id_field and 'flag_values' in gate_id_field: - labels = gate_id_field['flag_meanings'].split() + labels = _split_flag_meanings(gate_id_field['flag_meanings']) values = gate_id_field['flag_values'] return {label: int(value) for label, value in zip(labels, values)} @@ -49,5 +72,5 @@ def gate_id_has_category(gate_id_field, category): if 'notes' in gate_id_field: return category in gate_id_field['notes'] if 'flag_meanings' in gate_id_field: - return category in gate_id_field['flag_meanings'].split() + return category in _split_flag_meanings(gate_id_field['flag_meanings']) return False From f11845542874f390690733e22eab8e2ba417939e Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Mon, 10 Aug 2026 16:50:41 -0500 Subject: [PATCH 3/3] Handle plain, unindexed notes lists Fix a KeyError seen in production: a gate_id field whose notes attribute is just a whitespace-separated list of labels with no indices or commas (e.g. "rain snow no_scatter melting clutter terrain_blockage") was being misparsed as an "index label" pair, collapsing all but the first label into a single bogus category key. Detect indexed vs. plain notes by whether the comma split yields more than one piece or the first piece contains a colon. --- cmac/gate_id.py | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/cmac/gate_id.py b/cmac/gate_id.py index 4008ce3..fe11f4d 100644 --- a/cmac/gate_id.py +++ b/cmac/gate_id.py @@ -3,9 +3,16 @@ (hydrometeor ID) radar field. CMAC's own gate id fields document their categories with a ``notes`` -attribute: a comma separated list of ``"index: label"`` pairs, e.g. -``"0: multi_trip, 1: rain, 2: snow"``. The index/label separator within -each pair may be either a colon or plain whitespace (``"0 multi_trip"``). +attribute, which in practice shows up in a few different shapes: + +- ``"0: multi_trip, 1: rain, 2: snow"`` -- comma separated ``"index: label"`` + pairs, colon separated. +- ``"0 multi_trip, 1 rain, 2 snow"`` -- comma separated ``"index label"`` + pairs, whitespace separated. +- ``"multi_trip rain snow melting no_scatter clutter terrain_blockage"`` -- + a plain, unindexed list of labels in order, with no indices or commas at + all. + Fields that follow the CF conventions instead (or radar objects re-read from a file that converted ``notes`` on save) may document the same information with a ``flag_meanings`` attribute and a parallel @@ -26,16 +33,26 @@ def _label_from_pair(pair_str): return parts[-1].strip() -def _split_flag_meanings(flag_meanings): - """Split a ``flag_meanings`` attribute into its individual labels, - whether it is comma separated or plain whitespace separated.""" - if ',' in flag_meanings: - parts = flag_meanings.split(',') +def _split_list(text): + """Split a comma or whitespace separated list of labels into its + individual, stripped entries.""" + if ',' in text: + parts = text.split(',') else: - parts = flag_meanings.split() + parts = text.split() return [part.strip() for part in parts if part.strip()] +def _labels_from_notes(notes): + """Return the ordered list of category labels encoded in a ``notes`` + attribute, handling both indexed ``"index: label"``/``"index label"`` + pairs and a plain, unindexed list of labels.""" + pieces = [p.strip() for p in notes.split(',') if p.strip()] + if len(pieces) > 1 or (pieces and ':' in pieces[0]): + return [_label_from_pair(piece) for piece in pieces] + return _split_list(notes) + + def get_gate_id_categories(gate_id_field): """ Return a dict mapping each gate id category label to its integer code. @@ -47,13 +64,11 @@ def get_gate_id_categories(gate_id_field): """ if 'notes' in gate_id_field: - cat_dict = {} - for i, pair_str in enumerate(gate_id_field['notes'].split(',')): - cat_dict[_label_from_pair(pair_str)] = i - return cat_dict + labels = _labels_from_notes(gate_id_field['notes']) + return {label: i for i, label in enumerate(labels)} if 'flag_meanings' in gate_id_field and 'flag_values' in gate_id_field: - labels = _split_flag_meanings(gate_id_field['flag_meanings']) + labels = _split_list(gate_id_field['flag_meanings']) values = gate_id_field['flag_values'] return {label: int(value) for label, value in zip(labels, values)} @@ -70,7 +85,7 @@ def gate_id_has_category(gate_id_field, category): ``flag_meanings``. """ if 'notes' in gate_id_field: - return category in gate_id_field['notes'] + return category in _labels_from_notes(gate_id_field['notes']) if 'flag_meanings' in gate_id_field: - return category in _split_flag_meanings(gate_id_field['flag_meanings']) + return category in _split_list(gate_id_field['flag_meanings']) return False