diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 8be1696..0000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: codeql - -on: - push: - branches: [main] - pull_request: - schedule: - - cron: "0 0 * * 0" # weekly - -permissions: - actions: read - contents: read - security-events: write - -jobs: - analyze: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - language: [ruby] - steps: - - uses: actions/checkout@v7 - - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - - uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index f0ac6c3..779064b 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -8,7 +8,6 @@ on: jobs: lint: runs-on: ubuntu-latest - continue-on-error: true # 36 pre-existing violations; see TODO.complete/08-ruff-rababa-python.md steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 @@ -51,6 +50,10 @@ jobs: train: runs-on: ubuntu-latest + env: + # tensorboard's generated descriptors predate protobuf 4; the pure-python + # implementation tolerates them without pinning protobuf + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python strategy: fail-fast: false matrix: diff --git a/lib/rababa/arabic/cleaner.rb b/lib/rababa/arabic/cleaner.rb index 007f6b7..9928fcb 100644 --- a/lib/rababa/arabic/cleaner.rb +++ b/lib/rababa/arabic/cleaner.rb @@ -6,7 +6,7 @@ class Cleaner < Rababa::Cleaner # filter arabic only + basic cleaner def clean(text) text = text.chars.select { |c| VALID_ARABIC.include? c }.join - text = super(text) + text = super text.strip end end diff --git a/lib/rababa/arabic/encoders.rb b/lib/rababa/arabic/encoders.rb index 056e4f7..19ebb10 100644 --- a/lib/rababa/arabic/encoders.rb +++ b/lib/rababa/arabic/encoders.rb @@ -16,7 +16,6 @@ class TextEncoder def initialize(input_chars, target_chars, cleaner_type, reverse_input) - # cleaner fcts @cleaner = get_text_cleaner(cleaner_type) @@ -69,7 +68,6 @@ class BasicArabicEncoder < TextEncoder def initialize(cleaner_type = "basic_cleaners", reverse_input: false, reverse_target: false) - input_chars = "بض.غىهظخة؟:طس،؛فندؤلوئآك-يذاصشحزءمأجإ ترقعث".chars target_chars = ALL_POSSIBLE_HARAQAT.keys @@ -86,7 +84,6 @@ class ArabicEncoderWithStartSymbol < BasicArabicEncoder def initialize(cleaner_type = "basic_cleaners", reverse_input: false, reverse_target: false) - super @start_symbol_id = @target_symbol_to_id["s"] end diff --git a/lib/rababa/arabic/harakats.rb b/lib/rababa/arabic/harakats.rb index 090a9ed..d39d6da 100644 --- a/lib/rababa/arabic/harakats.rb +++ b/lib/rababa/arabic/harakats.rb @@ -24,7 +24,7 @@ def extract_stack(stack, correct_reversed) elsif ALL_POSSIBLE_HARAQAT.include?(reversed_full_haraqah) && correct_reversed out = reversed_full_haraqah else - val = full_haraqah.map { |diac| \ + val = full_haraqah.map { |diac| ALL_POSSIBLE_HARAQAT[diac] }.join("|") diff --git a/python/arabic/config_manager.py b/python/arabic/config_manager.py index 275a58b..9d476eb 100644 --- a/python/arabic/config_manager.py +++ b/python/arabic/config_manager.py @@ -240,7 +240,7 @@ def get_text_encoder(self): def get_loss_type(self): try: loss_type = LossType[self.config["loss_type"]] - except: + except KeyError: raise Exception(f"The loss type is not correct {self.config['loss_type']}") return loss_type diff --git a/python/arabic/setup.py b/python/arabic/setup.py index 0527c31..4ca3d28 100644 --- a/python/arabic/setup.py +++ b/python/arabic/setup.py @@ -21,7 +21,7 @@ author_email="open.source@ribose.com", license="MIT", description="Rababa for Arabic diacriticization", - # packages=['rababa'], + packages=setuptools.find_namespace_packages(include=["util*", "modules*"]), url="https://www.interscript.org", python_requires=">=3.6, <4", project_urls={ diff --git a/python/arabic/util/text_encoders.py b/python/arabic/util/text_encoders.py index 3d09476..f6e5ed1 100644 --- a/python/arabic/util/text_encoders.py +++ b/python/arabic/util/text_encoders.py @@ -1,8 +1,7 @@ from typing import Optional -from util.constants import ALL_POSSIBLE_HARAQAT - from util import text_cleaners +from util.constants import ALL_POSSIBLE_HARAQAT class TextEncoder: diff --git a/python/arabic/util/utils.py b/python/arabic/util/utils.py index c3b2347..3ca08f6 100644 --- a/python/arabic/util/utils.py +++ b/python/arabic/util/utils.py @@ -6,6 +6,7 @@ import matplotlib.pyplot as plt import numpy as np import torch +from diacritization_evaluation import der, wer from torch import nn from util.decorators import ignore_exception @@ -115,17 +116,6 @@ def plot_multi_head(model, path, global_step): display_attention(encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}") -def make_src_mask(src, pad_idx=0): - - # src = [batch size, src len] - - src_mask = (src != pad_idx).unsqueeze(1).unsqueeze(2) - - # src_mask = [batch size, 1, 1, src len] - - return src_mask - - def get_angles(pos, i, model_dim): angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(model_dim)) return pos * angle_rates diff --git a/python/hebrew/config_manager.py b/python/hebrew/config_manager.py index 02e0ee2..1491d2d 100644 --- a/python/hebrew/config_manager.py +++ b/python/hebrew/config_manager.py @@ -180,7 +180,7 @@ def load_model(self, model_path: str = None, load_optimizer: bool = False): optimizer_stat_dict = saved_model["optimizer_state_dict"] if load_optimizer else None global_step = saved_model["global_step"] + 1 - except: + except Exception: print("model_path:: ", model_path) print("WARNING:: Model not found under model_state_dict,") print("starting with a fresh model.") @@ -244,6 +244,6 @@ def get_text_encoder(self): def get_loss_type(self): try: loss_type = LossType[self.config["loss_type"]] - except: + except KeyError: raise Exception(f"The loss type is not correct {self.config['loss_type']}") return loss_type diff --git a/python/hebrew/convert_torch_model_to_onnx.py b/python/hebrew/convert_torch_model_to_onnx.py index df812ab..4c5683f 100644 --- a/python/hebrew/convert_torch_model_to_onnx.py +++ b/python/hebrew/convert_torch_model_to_onnx.py @@ -4,6 +4,7 @@ import onnx import onnxruntime import torch +import yaml from diacritizer import Diacritizer """ diff --git a/python/hebrew/dataset.py b/python/hebrew/dataset.py index 9638ded..fb21537 100644 --- a/python/hebrew/dataset.py +++ b/python/hebrew/dataset.py @@ -6,7 +6,6 @@ from config_manager import ConfigManager from torch.utils.data import DataLoader, Dataset - from util import nakdimon_dataset from util import nakdimon_hebrew_model as hebrew from util import nakdimon_utils as utils diff --git a/python/hebrew/diacritizer.py b/python/hebrew/diacritizer.py index af3af92..303627a 100644 --- a/python/hebrew/diacritizer.py +++ b/python/hebrew/diacritizer.py @@ -3,7 +3,6 @@ from config_manager import ConfigManager from dataset import DiacritizationDataset, collate_fn from torch.utils.data import DataLoader - from util import nakdimon_dataset # as dataset from util import nakdimon_hebrew_model as hebrew from util import nakdimon_utils as utils diff --git a/python/hebrew/setup.py b/python/hebrew/setup.py index 0527c31..4ca3d28 100644 --- a/python/hebrew/setup.py +++ b/python/hebrew/setup.py @@ -21,7 +21,7 @@ author_email="open.source@ribose.com", license="MIT", description="Rababa for Arabic diacriticization", - # packages=['rababa'], + packages=setuptools.find_namespace_packages(include=["util*", "modules*"]), url="https://www.interscript.org", python_requires=">=3.6, <4", project_urls={ diff --git a/python/hebrew/trainer.py b/python/hebrew/trainer.py index 4e1a3eb..ac7187b 100644 --- a/python/hebrew/trainer.py +++ b/python/hebrew/trainer.py @@ -10,6 +10,7 @@ from torch.cuda.amp import autocast from torch.utils.tensorboard.writer import SummaryWriter from tqdm import trange +from util import nakdimon_dataset, nakdimon_metrics from util.learning_rates import LearningRateDecay from util.utils import ( count_parameters, @@ -18,8 +19,6 @@ repeater, ) -from util import nakdimon_dataset, nakdimon_metrics - class Trainer: def run(self): @@ -82,7 +81,7 @@ def print_losses(self, step_results, tqdm): for pos, n_steps in enumerate(self.config["n_steps_avg_losses"]): if len(self.losses) > n_steps: - d_losses = process_losses(step_results[-n_steps:]) + d_losses = self.process_losses(step_results[-n_steps:]) for k in d_losses.keys(): for i, k in enumerate(d_losses.keys()): tqdm.display( diff --git a/python/hebrew/util/nakdimon_dataset.py b/python/hebrew/util/nakdimon_dataset.py index 397a80e..0ef56e6 100644 --- a/python/hebrew/util/nakdimon_dataset.py +++ b/python/hebrew/util/nakdimon_dataset.py @@ -2,7 +2,6 @@ import numpy as np import torch - from util import nakdimon_hebrew_model as hebrew from util import nakdimon_utils as utils @@ -128,7 +127,7 @@ def to_device(self, device): def get_idces(self, idces): - if type(idces) == int: + if isinstance(idces, int): idces = [idces] return Data( diff --git a/python/hebrew/util/nakdimon_hebrew_model.py b/python/hebrew/util/nakdimon_hebrew_model.py index 8de2a72..ffddb65 100644 --- a/python/hebrew/util/nakdimon_hebrew_model.py +++ b/python/hebrew/util/nakdimon_hebrew_model.py @@ -1,7 +1,13 @@ +import unicodedata from collections.abc import Iterable, Iterator from functools import lru_cache from typing import NamedTuple + +def name_of(c: str) -> str: + return unicodedata.name(c, f"U+{ord(c):04X}") + + # "rafe" denotes a letter to which it would have been valid to add a diacritic of some category # but instead it is decided not to. This makes the metrics less biased. RAFE = "\u05bf" diff --git a/python/hebrew/util/utils.py b/python/hebrew/util/utils.py index c3b2347..3ca08f6 100644 --- a/python/hebrew/util/utils.py +++ b/python/hebrew/util/utils.py @@ -6,6 +6,7 @@ import matplotlib.pyplot as plt import numpy as np import torch +from diacritization_evaluation import der, wer from torch import nn from util.decorators import ignore_exception @@ -115,17 +116,6 @@ def plot_multi_head(model, path, global_step): display_attention(encoder_attentions[0][0], path, global_step, f"encoder-layer {i + 1}") -def make_src_mask(src, pad_idx=0): - - # src = [batch size, src len] - - src_mask = (src != pad_idx).unsqueeze(1).unsqueeze(2) - - # src_mask = [batch size, 1, 1, src len] - - return src_mask - - def get_angles(pos, i, model_dim): angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(model_dim)) return pos * angle_rates diff --git a/rababa.gemspec b/rababa.gemspec index ca8be83..5f4d18f 100644 --- a/rababa.gemspec +++ b/rababa.gemspec @@ -13,10 +13,10 @@ Gem::Specification.new do |spec| spec.homepage = "https://www.interscript.org" spec.required_ruby_version = ">= 3.3.0" - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = "https://github.com/interscript/rababa" - spec.metadata["changelog_uri"] = "https://github.com/interscript/rababa/releases" - spec.metadata["bug_tracker_uri"] = "https://github.com/interscript/rababa/issues" + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = "https://github.com/interscript/rababa" + spec.metadata["changelog_uri"] = "https://github.com/interscript/rababa/releases" + spec.metadata["bug_tracker_uri"] = "https://github.com/interscript/rababa/issues" spec.metadata["rubygems_mfa_required"] = "true" spec.files = Dir.chdir(__dir__) do