From 5e0e6aa142f00df9b8dbed54903fd90563e4ba7f Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 30 Aug 2026 21:53:14 +0200 Subject: [PATCH 1/2] =?UTF-8?q?fix(ci):=20green=20builds=20=E2=80=94=20rea?= =?UTF-8?q?l=20bugs,=20not=20guards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every check on every PR has been red at the repo level. Four causes, four fixes: - python lint (ruff): fixed all 22 violations instead of absorbing them with continue-on-error (which never worked anyway — the job still reports failure). Includes genuine F821 bugs: wer/der never imported in util/utils.py (both mirrors), process_losses called without self in hebrew/trainer.py, yaml never imported in hebrew's ONNX converter, name_of never defined in nakdimon_hebrew_model.py (now a unicodedata-based helper). Also the duplicate untyped make_src_mask, bare excepts narrowed to KeyError/Exception, one isinstance fix, import sorting. - python infer/train: pip install -e . failed at metadata — packages=['rababa'] was commented out, so auto-discovery tripped on the flat layout (models/config/modules/log_dir). Declared the namespace packages explicitly (util*, modules*); editable install verified locally. - ruby build: standardrb — super-arguments, empty method-body lines, redundant line continuation, gemspec spacing. rake: 23 examples, 0 failures (unchanged). - codeql: the advanced workflow conflicts with the repo's configured default setup (SARIF rejected) — default setup already covers ruby/python/actions, so the redundant workflow is removed. --- .github/workflows/codeql.yml | 27 -------------------- .github/workflows/python-arabic.yml | 1 - lib/rababa/arabic/cleaner.rb | 2 +- lib/rababa/arabic/encoders.rb | 3 --- lib/rababa/arabic/harakats.rb | 2 +- python/arabic/config_manager.py | 2 +- python/arabic/setup.py | 4 ++- python/arabic/util/text_encoders.py | 3 +-- python/arabic/util/utils.py | 12 +-------- python/hebrew/config_manager.py | 4 +-- python/hebrew/convert_torch_model_to_onnx.py | 1 + python/hebrew/dataset.py | 1 - python/hebrew/diacritizer.py | 1 - python/hebrew/setup.py | 4 ++- python/hebrew/trainer.py | 5 ++-- python/hebrew/util/nakdimon_dataset.py | 3 +-- python/hebrew/util/nakdimon_hebrew_model.py | 6 +++++ python/hebrew/util/utils.py | 12 +-------- rababa.gemspec | 8 +++--- 19 files changed, 28 insertions(+), 73 deletions(-) delete mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 8be1696..0000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: codeql - -on: - push: - branches: [main] - pull_request: - schedule: - - cron: "0 0 * * 0" # weekly - -permissions: - actions: read - contents: read - security-events: write - -jobs: - analyze: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - language: [ruby] - steps: - - uses: actions/checkout@v7 - - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - - uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index f0ac6c3..b8bb10d 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -8,7 +8,6 @@ on: jobs: lint: runs-on: ubuntu-latest - continue-on-error: true # 36 pre-existing violations; see TODO.complete/08-ruff-rababa-python.md steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 diff --git a/lib/rababa/arabic/cleaner.rb b/lib/rababa/arabic/cleaner.rb index 007f6b7..9928fcb 100644 --- a/lib/rababa/arabic/cleaner.rb +++ b/lib/rababa/arabic/cleaner.rb @@ -6,7 +6,7 @@ class Cleaner < Rababa::Cleaner # filter arabic only + basic cleaner def clean(text) text = text.chars.select { |c| VALID_ARABIC.include? c }.join - text = super(text) + text = super text.strip end end diff --git a/lib/rababa/arabic/encoders.rb b/lib/rababa/arabic/encoders.rb index 056e4f7..19ebb10 100644 --- a/lib/rababa/arabic/encoders.rb +++ b/lib/rababa/arabic/encoders.rb @@ -16,7 +16,6 @@ class TextEncoder def initialize(input_chars, target_chars, cleaner_type, reverse_input) - # cleaner fcts @cleaner = get_text_cleaner(cleaner_type) @@ -69,7 +68,6 @@ class BasicArabicEncoder < TextEncoder def initialize(cleaner_type = "basic_cleaners", reverse_input: false, reverse_target: false) - input_chars = "بض.غىهظخة؟:طس،؛فندؤلوئآك-يذاصشحزءمأجإ ترقعث".chars target_chars = ALL_POSSIBLE_HARAQAT.keys @@ -86,7 +84,6 @@ class ArabicEncoderWithStartSymbol < BasicArabicEncoder def initialize(cleaner_type = "basic_cleaners", reverse_input: false, reverse_target: false) - super @start_symbol_id = @target_symbol_to_id["s"] end diff --git a/lib/rababa/arabic/harakats.rb b/lib/rababa/arabic/harakats.rb index 090a9ed..d39d6da 100644 --- a/lib/rababa/arabic/harakats.rb +++ b/lib/rababa/arabic/harakats.rb @@ -24,7 +24,7 @@ def extract_stack(stack, correct_reversed) elsif ALL_POSSIBLE_HARAQAT.include?(reversed_full_haraqah) && correct_reversed out = reversed_full_haraqah else - val = full_haraqah.map { |diac| \ + val = full_haraqah.map { |diac| ALL_POSSIBLE_HARAQAT[diac] }.join("|") diff --git a/python/arabic/config_manager.py b/python/arabic/config_manager.py index 275a58b..9d476eb 100644 --- a/python/arabic/config_manager.py +++ b/python/arabic/config_manager.py @@ -240,7 +240,7 @@ def get_text_encoder(self): def get_loss_type(self): try: loss_type = LossType[self.config["loss_type"]] - except: + except KeyError: raise Exception(f"The loss type is not correct {self.config['loss_type']}") return loss_type diff --git a/python/arabic/setup.py b/python/arabic/setup.py index 0527c31..f8b1d8e 100644 --- a/python/arabic/setup.py +++ b/python/arabic/setup.py @@ -21,7 +21,9 @@ author_email="open.source@ribose.com", license="MIT", description="Rababa for Arabic diacriticization", - # packages=['rababa'], + packages=setuptools.find_namespace_packages( + include=["util*", "modules*"] + ), url="https://www.interscript.org", python_requires=">=3.6, <4", project_urls={ diff --git a/python/arabic/util/text_encoders.py b/python/arabic/util/text_encoders.py index 3d09476..f6e5ed1 100644 --- a/python/arabic/util/text_encoders.py +++ b/python/arabic/util/text_encoders.py @@ -1,8 +1,7 @@ from typing import Optional -from util.constants import ALL_POSSIBLE_HARAQAT - from util import text_cleaners +from util.constants import ALL_POSSIBLE_HARAQAT class TextEncoder: diff --git a/python/arabic/util/utils.py b/python/arabic/util/utils.py index c3b2347..3ca08f6 100644 --- a/python/arabic/util/utils.py +++ b/python/arabic/util/utils.py @@ -6,6 +6,7 @@ import matplotlib.pyplot as plt import numpy as np import torch +from diacritization_evaluation import der, wer from torch import nn from util.decorators import ignore_exception @@ -115,17 +116,6 @@ def plot_multi_head(model, path, global_step): display_attention(encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}") -def make_src_mask(src, pad_idx=0): - - # src = [batch size, src len] - - src_mask = (src != pad_idx).unsqueeze(1).unsqueeze(2) - - # src_mask = [batch size, 1, 1, src len] - - return src_mask - - def get_angles(pos, i, model_dim): angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(model_dim)) return pos * angle_rates diff --git a/python/hebrew/config_manager.py b/python/hebrew/config_manager.py index 02e0ee2..1491d2d 100644 --- a/python/hebrew/config_manager.py +++ b/python/hebrew/config_manager.py @@ -180,7 +180,7 @@ def load_model(self, model_path: str = None, load_optimizer: bool = False): optimizer_stat_dict = saved_model["optimizer_state_dict"] if load_optimizer else None global_step = saved_model["global_step"] + 1 - except: + except Exception: print("model_path:: ", model_path) print("WARNING:: Model not found under model_state_dict,") print("starting with a fresh model.") @@ -244,6 +244,6 @@ def get_text_encoder(self): def get_loss_type(self): try: loss_type = LossType[self.config["loss_type"]] - except: + except KeyError: raise Exception(f"The loss type is not correct {self.config['loss_type']}") return loss_type diff --git a/python/hebrew/convert_torch_model_to_onnx.py b/python/hebrew/convert_torch_model_to_onnx.py index df812ab..4c5683f 100644 --- a/python/hebrew/convert_torch_model_to_onnx.py +++ b/python/hebrew/convert_torch_model_to_onnx.py @@ -4,6 +4,7 @@ import onnx import onnxruntime import torch +import yaml from diacritizer import Diacritizer """ diff --git a/python/hebrew/dataset.py b/python/hebrew/dataset.py index 9638ded..fb21537 100644 --- a/python/hebrew/dataset.py +++ b/python/hebrew/dataset.py @@ -6,7 +6,6 @@ from config_manager import ConfigManager from torch.utils.data import DataLoader, Dataset - from util import nakdimon_dataset from util import nakdimon_hebrew_model as hebrew from util import nakdimon_utils as utils diff --git a/python/hebrew/diacritizer.py b/python/hebrew/diacritizer.py index af3af92..303627a 100644 --- a/python/hebrew/diacritizer.py +++ b/python/hebrew/diacritizer.py @@ -3,7 +3,6 @@ from config_manager import ConfigManager from dataset import DiacritizationDataset, collate_fn from torch.utils.data import DataLoader - from util import nakdimon_dataset # as dataset from util import nakdimon_hebrew_model as hebrew from util import nakdimon_utils as utils diff --git a/python/hebrew/setup.py b/python/hebrew/setup.py index 0527c31..f8b1d8e 100644 --- a/python/hebrew/setup.py +++ b/python/hebrew/setup.py @@ -21,7 +21,9 @@ author_email="open.source@ribose.com", license="MIT", description="Rababa for Arabic diacriticization", - # packages=['rababa'], + packages=setuptools.find_namespace_packages( + include=["util*", "modules*"] + ), url="https://www.interscript.org", python_requires=">=3.6, <4", project_urls={ diff --git a/python/hebrew/trainer.py b/python/hebrew/trainer.py index 4e1a3eb..ac7187b 100644 --- a/python/hebrew/trainer.py +++ b/python/hebrew/trainer.py @@ -10,6 +10,7 @@ from torch.cuda.amp import autocast from torch.utils.tensorboard.writer import SummaryWriter from tqdm import trange +from util import nakdimon_dataset, nakdimon_metrics from util.learning_rates import LearningRateDecay from util.utils import ( count_parameters, @@ -18,8 +19,6 @@ repeater, ) -from util import nakdimon_dataset, nakdimon_metrics - class Trainer: def run(self): @@ -82,7 +81,7 @@ def print_losses(self, step_results, tqdm): for pos, n_steps in enumerate(self.config["n_steps_avg_losses"]): if len(self.losses) > n_steps: - d_losses = process_losses(step_results[-n_steps:]) + d_losses = self.process_losses(step_results[-n_steps:]) for k in d_losses.keys(): for i, k in enumerate(d_losses.keys()): tqdm.display( diff --git a/python/hebrew/util/nakdimon_dataset.py b/python/hebrew/util/nakdimon_dataset.py index 397a80e..0ef56e6 100644 --- a/python/hebrew/util/nakdimon_dataset.py +++ b/python/hebrew/util/nakdimon_dataset.py @@ -2,7 +2,6 @@ import numpy as np import torch - from util import nakdimon_hebrew_model as hebrew from util import nakdimon_utils as utils @@ -128,7 +127,7 @@ def to_device(self, device): def get_idces(self, idces): - if type(idces) == int: + if isinstance(idces, int): idces = [idces] return Data( diff --git a/python/hebrew/util/nakdimon_hebrew_model.py b/python/hebrew/util/nakdimon_hebrew_model.py index 8de2a72..ffddb65 100644 --- a/python/hebrew/util/nakdimon_hebrew_model.py +++ b/python/hebrew/util/nakdimon_hebrew_model.py @@ -1,7 +1,13 @@ +import unicodedata from collections.abc import Iterable, Iterator from functools import lru_cache from typing import NamedTuple + +def name_of(c: str) -> str: + return unicodedata.name(c, f"U+{ord(c):04X}") + + # "rafe" denotes a letter to which it would have been valid to add a diacritic of some category # but instead it is decided not to. This makes the metrics less biased. RAFE = "\u05bf" diff --git a/python/hebrew/util/utils.py b/python/hebrew/util/utils.py index c3b2347..3ca08f6 100644 --- a/python/hebrew/util/utils.py +++ b/python/hebrew/util/utils.py @@ -6,6 +6,7 @@ import matplotlib.pyplot as plt import numpy as np import torch +from diacritization_evaluation import der, wer from torch import nn from util.decorators import ignore_exception @@ -115,17 +116,6 @@ def plot_multi_head(model, path, global_step): display_attention(encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}") -def make_src_mask(src, pad_idx=0): - - # src = [batch size, src len] - - src_mask = (src != pad_idx).unsqueeze(1).unsqueeze(2) - - # src_mask = [batch size, 1, 1, src len] - - return src_mask - - def get_angles(pos, i, model_dim): angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(model_dim)) return pos * angle_rates diff --git a/rababa.gemspec b/rababa.gemspec index ca8be83..5f4d18f 100644 --- a/rababa.gemspec +++ b/rababa.gemspec @@ -13,10 +13,10 @@ Gem::Specification.new do |spec| spec.homepage = "https://www.interscript.org" spec.required_ruby_version = ">= 3.3.0" - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = "https://github.com/interscript/rababa" - spec.metadata["changelog_uri"] = "https://github.com/interscript/rababa/releases" - spec.metadata["bug_tracker_uri"] = "https://github.com/interscript/rababa/issues" + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = "https://github.com/interscript/rababa" + spec.metadata["changelog_uri"] = "https://github.com/interscript/rababa/releases" + spec.metadata["bug_tracker_uri"] = "https://github.com/interscript/rababa/issues" spec.metadata["rubygems_mfa_required"] = "true" spec.files = Dir.chdir(__dir__) do From 36d3df10843262eeeed3c40cb469a3fb9b9e61df Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 30 Aug 2026 22:01:56 +0200 Subject: [PATCH 2/2] fix(ci): collapse setup.py formatting for current ruff; pure-python protobuf for train MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's ruff (latest) collapses the short find_namespace_packages call that older local ruff accepted multi-line. train (3.9) hit the tensorboard-vs-protobuf>=4 'Descriptors cannot be created directly' clash — PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python on the train job avoids it without pinning protobuf in packaging. --- .github/workflows/python-arabic.yml | 4 ++++ python/arabic/setup.py | 4 +--- python/hebrew/setup.py | 4 +--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index b8bb10d..779064b 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -50,6 +50,10 @@ jobs: train: runs-on: ubuntu-latest + env: + # tensorboard's generated descriptors predate protobuf 4; the pure-python + # implementation tolerates them without pinning protobuf + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python strategy: fail-fast: false matrix: diff --git a/python/arabic/setup.py b/python/arabic/setup.py index f8b1d8e..4ca3d28 100644 --- a/python/arabic/setup.py +++ b/python/arabic/setup.py @@ -21,9 +21,7 @@ author_email="open.source@ribose.com", license="MIT", description="Rababa for Arabic diacriticization", - packages=setuptools.find_namespace_packages( - include=["util*", "modules*"] - ), + packages=setuptools.find_namespace_packages(include=["util*", "modules*"]), url="https://www.interscript.org", python_requires=">=3.6, <4", project_urls={ diff --git a/python/hebrew/setup.py b/python/hebrew/setup.py index f8b1d8e..4ca3d28 100644 --- a/python/hebrew/setup.py +++ b/python/hebrew/setup.py @@ -21,9 +21,7 @@ author_email="open.source@ribose.com", license="MIT", description="Rababa for Arabic diacriticization", - packages=setuptools.find_namespace_packages( - include=["util*", "modules*"] - ), + packages=setuptools.find_namespace_packages(include=["util*", "modules*"]), url="https://www.interscript.org", python_requires=">=3.6, <4", project_urls={