fix(ci): green builds — real bugs, not guards

.github/workflows/python-arabic.yml

@@ -8,7 +8,6 @@ on:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    continue-on-error: true  # 36 pre-existing violations; see TODO.complete/08-ruff-rababa-python.md
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-python@v7
@@ -51,6 +50,10 @@ jobs:
 
   train:
     runs-on: ubuntu-latest
+    env:
+      # tensorboard's generated descriptors predate protobuf 4; the pure-python
+      # implementation tolerates them without pinning protobuf
+      PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
     strategy:
       fail-fast: false
       matrix:

lib/rababa/arabic/cleaner.rb

@@ -6,7 +6,7 @@ class Cleaner < Rababa::Cleaner
       # filter arabic only + basic cleaner
       def clean(text)
         text = text.chars.select { |c| VALID_ARABIC.include? c }.join
-        text = super(text)
+        text = super
         text.strip
       end
     end

lib/rababa/arabic/encoders.rb

@@ -16,7 +16,6 @@ class TextEncoder
         def initialize(input_chars, target_chars,
           cleaner_type,
           reverse_input)
-
           # cleaner fcts
           @cleaner = get_text_cleaner(cleaner_type)
 
@@ -69,7 +68,6 @@ class BasicArabicEncoder < TextEncoder
         def initialize(cleaner_type = "basic_cleaners",
           reverse_input: false,
           reverse_target: false)
-
           input_chars = "بض.غىهظخة؟:طس،؛فندؤلوئآك-يذاصشحزءمأجإ ترقعث".chars
           target_chars = ALL_POSSIBLE_HARAQAT.keys
 
@@ -86,7 +84,6 @@ class ArabicEncoderWithStartSymbol < BasicArabicEncoder
         def initialize(cleaner_type = "basic_cleaners",
           reverse_input: false,
           reverse_target: false)
-
           super
           @start_symbol_id = @target_symbol_to_id["s"]
         end

lib/rababa/arabic/harakats.rb

@@ -24,7 +24,7 @@ def extract_stack(stack, correct_reversed)
         elsif ALL_POSSIBLE_HARAQAT.include?(reversed_full_haraqah) && correct_reversed
           out = reversed_full_haraqah
         else
-          val = full_haraqah.map { |diac| \
+          val = full_haraqah.map { |diac|
             ALL_POSSIBLE_HARAQAT[diac]
           }.join("|")
 

python/arabic/config_manager.py

@@ -240,7 +240,7 @@ def get_text_encoder(self):
     def get_loss_type(self):
         try:
             loss_type = LossType[self.config["loss_type"]]
-        except:
+        except KeyError:
             raise Exception(f"The loss type is not correct {self.config['loss_type']}")
         return loss_type
 

python/arabic/setup.py

@@ -21,7 +21,7 @@
     author_email="open.source@ribose.com",
     license="MIT",
     description="Rababa for Arabic diacriticization",
-    # packages=['rababa'],
+    packages=setuptools.find_namespace_packages(include=["util*", "modules*"]),
     url="https://www.interscript.org",
     python_requires=">=3.6, <4",
     project_urls={

python/arabic/util/text_encoders.py

@@ -1,8 +1,7 @@
 from typing import Optional
 
-from util.constants import ALL_POSSIBLE_HARAQAT
-
 from util import text_cleaners
+from util.constants import ALL_POSSIBLE_HARAQAT
 
 
 class TextEncoder:

python/arabic/util/utils.py

@@ -6,6 +6,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 import torch
+from diacritization_evaluation import der, wer
 from torch import nn
 from util.decorators import ignore_exception
 
@@ -115,17 +116,6 @@ def plot_multi_head(model, path, global_step):
         display_attention(encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}")
 
 
-def make_src_mask(src, pad_idx=0):
-
-    # src = [batch size, src len]
-
-    src_mask = (src != pad_idx).unsqueeze(1).unsqueeze(2)
-
-    # src_mask = [batch size, 1, 1, src len]
-
-    return src_mask
-
-
 def get_angles(pos, i, model_dim):
     angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(model_dim))
     return pos * angle_rates

python/hebrew/config_manager.py

@@ -180,7 +180,7 @@ def load_model(self, model_path: str = None, load_optimizer: bool = False):
             optimizer_stat_dict = saved_model["optimizer_state_dict"] if load_optimizer else None
             global_step = saved_model["global_step"] + 1
 
-        except:
+        except Exception:
             print("model_path:: ", model_path)
             print("WARNING:: Model not found under model_state_dict,")
             print("starting with a fresh model.")
@@ -244,6 +244,6 @@ def get_text_encoder(self):
     def get_loss_type(self):
         try:
             loss_type = LossType[self.config["loss_type"]]
-        except:
+        except KeyError:
             raise Exception(f"The loss type is not correct {self.config['loss_type']}")
         return loss_type

python/hebrew/convert_torch_model_to_onnx.py

@@ -4,6 +4,7 @@
 import onnx
 import onnxruntime
 import torch
+import yaml
 from diacritizer import Diacritizer
 
 """

python/hebrew/dataset.py

@@ -6,7 +6,6 @@
 
 from config_manager import ConfigManager
 from torch.utils.data import DataLoader, Dataset
-
 from util import nakdimon_dataset
 from util import nakdimon_hebrew_model as hebrew
 from util import nakdimon_utils as utils

python/hebrew/diacritizer.py

@@ -3,7 +3,6 @@
 from config_manager import ConfigManager
 from dataset import DiacritizationDataset, collate_fn
 from torch.utils.data import DataLoader
-
 from util import nakdimon_dataset  # as dataset
 from util import nakdimon_hebrew_model as hebrew
 from util import nakdimon_utils as utils

python/hebrew/setup.py

@@ -21,7 +21,7 @@
     author_email="open.source@ribose.com",
     license="MIT",
     description="Rababa for Arabic diacriticization",
-    # packages=['rababa'],
+    packages=setuptools.find_namespace_packages(include=["util*", "modules*"]),
     url="https://www.interscript.org",
     python_requires=">=3.6, <4",
     project_urls={

python/hebrew/trainer.py

@@ -10,6 +10,7 @@
 from torch.cuda.amp import autocast
 from torch.utils.tensorboard.writer import SummaryWriter
 from tqdm import trange
+from util import nakdimon_dataset, nakdimon_metrics
 from util.learning_rates import LearningRateDecay
 from util.utils import (
     count_parameters,
@@ -18,8 +19,6 @@
     repeater,
 )
 
-from util import nakdimon_dataset, nakdimon_metrics
-
 
 class Trainer:
     def run(self):
@@ -82,7 +81,7 @@ def print_losses(self, step_results, tqdm):
 
         for pos, n_steps in enumerate(self.config["n_steps_avg_losses"]):
             if len(self.losses) > n_steps:
-                d_losses = process_losses(step_results[-n_steps:])
+                d_losses = self.process_losses(step_results[-n_steps:])
                 for k in d_losses.keys():
                     for i, k in enumerate(d_losses.keys()):
                         tqdm.display(

python/hebrew/util/nakdimon_dataset.py

@@ -2,7 +2,6 @@
 
 import numpy as np
 import torch
-
 from util import nakdimon_hebrew_model as hebrew
 from util import nakdimon_utils as utils
 
@@ -128,7 +127,7 @@ def to_device(self, device):
 
     def get_idces(self, idces):
 
-        if type(idces) == int:
+        if isinstance(idces, int):
             idces = [idces]
 
         return Data(

python/hebrew/util/nakdimon_hebrew_model.py

@@ -1,7 +1,13 @@
+import unicodedata
 from collections.abc import Iterable, Iterator
 from functools import lru_cache
 from typing import NamedTuple
 
+
+def name_of(c: str) -> str:
+    return unicodedata.name(c, f"U+{ord(c):04X}")
+
+
 # "rafe" denotes a letter to which it would have been valid to add a diacritic of some category
 # but instead it is decided not to. This makes the metrics less biased.
 RAFE = "\u05bf"

python/hebrew/util/utils.py

@@ -6,6 +6,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 import torch
+from diacritization_evaluation import der, wer
 from torch import nn
 from util.decorators import ignore_exception
 
@@ -115,17 +116,6 @@ def plot_multi_head(model, path, global_step):
         display_attention(encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}")
 
 
-def make_src_mask(src, pad_idx=0):
-
-    # src = [batch size, src len]
-
-    src_mask = (src != pad_idx).unsqueeze(1).unsqueeze(2)
-
-    # src_mask = [batch size, 1, 1, src len]
-
-    return src_mask
-
-
 def get_angles(pos, i, model_dim):
     angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(model_dim))
     return pos * angle_rates

rababa.gemspec

@@ -13,10 +13,10 @@ Gem::Specification.new do |spec|
   spec.homepage = "https://www.interscript.org"
   spec.required_ruby_version = ">= 3.3.0"
 
-  spec.metadata["homepage_uri"]         = spec.homepage
-  spec.metadata["source_code_uri"]      = "https://github.com/interscript/rababa"
-  spec.metadata["changelog_uri"]        = "https://github.com/interscript/rababa/releases"
-  spec.metadata["bug_tracker_uri"]      = "https://github.com/interscript/rababa/issues"
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/interscript/rababa"
+  spec.metadata["changelog_uri"] = "https://github.com/interscript/rababa/releases"
+  spec.metadata["bug_tracker_uri"] = "https://github.com/interscript/rababa/issues"
   spec.metadata["rubygems_mfa_required"] = "true"
 
   spec.files = Dir.chdir(__dir__) do