From 9378e6f89dfdb4c223743a4e3217db864a3aa055 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 29 Apr 2025 21:35:22 +0800 Subject: [PATCH 1/8] feat: modernize and run with python --- .github/workflows/python-arabic.yml | 118 +++++- README.adoc | 238 +++++++++++- python/arabic/config_manager.py | 6 +- python/arabic/convert_torch_model_to_onnx.py | 384 ++++++++++--------- python/arabic/diacritize.py | 38 +- python/arabic/test.py | 16 +- python/arabic/train.py | 21 +- python/hebrew/config/cbhg.yml | 4 + python/hebrew/convert_torch_model_to_onnx.py | 349 +++++++++-------- python/hebrew/diacritize.py | 39 +- python/hebrew/run_experiments_wandb.py | 50 +-- python/hebrew/test.py | 16 +- python/hebrew/train.py | 21 +- 13 files changed, 832 insertions(+), 468 deletions(-) diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index 68488c9..983e8f3 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -1,34 +1,83 @@ -name: python +name: Python CI on: push: branches: [ main ] + paths: + - 'python/**' + - '.github/workflows/python-arabic.yml' pull_request: + paths: + - 'python/**' + - '.github/workflows/python-arabic.yml' + +# Cancel in-progress runs for the same workflow and branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + security-events: write jobs: + dependency-review: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Dependency Review + uses: actions/dependency-review-action@v3 + + codeql: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: python + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + infer: runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - name: Checkout code + uses: actions/checkout@v4 - - uses: actions/cache@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('python/arabic/setup.py') }}-${{ hashFiles('python/arabic/requirements.txt') }} + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + python/arabic/requirements.txt + python/arabic/setup.py - name: Install requirements working-directory: ./python/arabic run: | + python -m pip install --upgrade pip pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . + - name: Create model directory + working-directory: ./python/arabic + run: | + mkdir -p log_dir/CA_MSA.base.cbhg/models/ + - name: Download PyTorch model working-directory: ./python/arabic run: | @@ -42,25 +91,29 @@ jobs: train: runs-on: ubuntu-latest + timeout-minutes: 20 strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - name: Checkout code + uses: actions/checkout@v4 - - uses: actions/cache@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('python/setup.py') }}-${{ hashFiles('python/requirements.txt') }} + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + python/arabic/requirements.txt + python/arabic/setup.py - name: Install requirements working-directory: ./python/arabic run: | + python -m pip install --upgrade pip pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . - name: Prepare dataset @@ -78,4 +131,35 @@ jobs: - name: Try training (WIP) working-directory: ./python/arabic run: | + # Fix for Python 3.9+ multiprocessing issue + if [[ "${{ matrix.python-version }}" == "3.9" || "${{ matrix.python-version }}" > "3.9" ]]; then + sed -i '1s/^/import multiprocessing\nif __name__ == "__main__":\n multiprocessing.freeze_support()\n/' train.py + fi python train.py --model "cbhg" --config config/test_cbhg.yml + + security-scan: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install safety + run: | + python -m pip install --upgrade pip + pip install safety + + - name: Check for vulnerabilities in Arabic requirements + working-directory: ./python/arabic + run: | + safety check -r requirements.txt --full-report || true + + - name: Check for vulnerabilities in Hebrew requirements + working-directory: ./python/hebrew + run: | + safety check -r requirements.txt --full-report || true diff --git a/README.adoc b/README.adoc index ea9e65f..611fc80 100644 --- a/README.adoc +++ b/README.adoc @@ -3,22 +3,15 @@ Middle-Eastern Language diacritization is useful for several practical business cases like text to speech or Romanization of texts or scripts. -As of now, this library supports Hebrew and Arabic. - == Purpose This repository contains everything to train a diacritization model in Python and run it in Python and Ruby. -== Try out Rababa - -Rababa can be run both in Python and Ruby. Go the directory corresponding to the -language you prefer to use. - -Please see the following README's, under the "`Try out Rababa`" section: +It supports the following languages: -* https://github.com/interscript/rababa/tree/main/python[Python] -* https://github.com/interscript/rababa/tree/main/lib[Ruby] +* Arabic +* Hebrew == Library @@ -74,12 +67,185 @@ In a similar fashion that a Rababa produces melody from a simple strings and pieces of wood, our library and diacritization gives a whole palette of colour and meanings to arabic scripts. + +== Try out Rababa + +Rababa can be run both in Python and Ruby. Go to the directory corresponding to the +language you prefer to use. + +Training of Rababa models are only available in Python. + + +=== Python (3.8+) + +==== Prerequisites + +The Python version of Rababa supports Python 3.8 and later. We recommend using a +version manager like https://asdf-vm.com[asdf] or +https://github.com/pyenv/pyenv[pyenv]. + +[IMPORTANT] +==== +*PyTorch Compatibility:* Rababa works with PyTorch 1.7.0 and later. Different +versions of PyTorch may handle model loading differently, especially with the +`weights_only` parameter introduced in PyTorch 2.6+. + +If you encounter model loading errors mentioning `weights_only`, you might be +using a newer PyTorch version that is incompatible with the saved models. The +setup steps below have been tested and should work across PyTorch versions. +==== + +With asdf: + +[source,bash] +---- +# Install asdf (if not already installed) +# See: https://asdf-vm.com/guide/getting-started.html + +# Install Python plugin +asdf plugin add python + +# Install desired Python version +asdf install python 3.9.13 + +# Set Python version for this project +asdf local python 3.9.13 + +# Verify your Python version +python --version +---- + +With pyenv: + +[source,bash] +---- +# Install pyenv (if not already installed) +brew install pyenv + +# Install desired Python version +pyenv install 3.9.13 + +# Set Python version for this project +pyenv local 3.9.13 + +# Verify your Python version +python --version +---- + +==== Arabic Diacritization + +. Install the required dependencies: ++ +[source,bash] +---- +cd python/arabic +pip install --upgrade pip +pip install torch numpy matplotlib pandas ruamel.yaml tensorboard diacritization-evaluation tqdm onnx onnxruntime pyyaml +---- + +. Download the pre-trained model: ++ +[source,bash] +---- +mkdir -p log_dir/CA_MSA.base.cbhg/models +curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt +---- + +. Run diacritization: ++ +[source,bash] +---- +# Diacritize a single text +python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر' + +# Diacritize a text file +python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text_file path_to_input_file --diacritized_text_file path_to_output_file +---- + +.Example Output +[source] +---- +# Input: قطر +# Output: قَطُرُ + +# Input: مرحبا كيف حالك انا بخير +# Output: مَرْحَبًا كَيْفَ حَالُك انَّا بِخَيْرٍ +---- + +[NOTE] +==== +If you encounter model loading errors with newer versions of PyTorch (especially +2.6+), you may see messages related to the `weights_only` parameter. The code +has been updated to handle this compatibility issue, but if you still experience +problems, please refer to the Troubleshooting section at the end of this +document. +==== + +==== Hebrew Diacritization + +. Install the required dependencies: ++ +[source,bash] +---- +cd python/hebrew +pip install --upgrade pip +pip install torch numpy matplotlib pandas ruamel.yaml tensorboard diacritization-evaluation tqdm onnx onnxruntime pyyaml +---- + +. Download the pre-trained model and configuration: ++ +[source,bash] +---- +mkdir -p log_dir/base.cbhg +curl -sSL https://github.com/secryst/rababa-models/releases/download/hebrew.0.1/bchg_len90_dec0.9766_wor_0.8877.yml -o config/model_hebrew.yml +curl -sSL https://github.com/secryst/rababa-models/releases/download/hebrew.0.1/model_len90_dec0.9766_wor_0.8877.pt -o log_dir/base.cbhg/model.pt +---- + +. Run diacritization: ++ +[source,bash] +---- +# Diacritize a single text +python diacritize.py --model_kind "cbhg" --config config/model_hebrew.yml --text 'מה שלומך' + +# Diacritize a text file +python diacritize.py --model_kind "cbhg" --config config/model_hebrew.yml --text_file path_to_input_file --diacritized_text_file path_to_output_file +---- + +.Example Output +[source] +---- +# Input: מה שלומך +# Output: מָה שֶׁלוֹמְךַ + +# Input: אני אוהב לקרוא ספרים +# Output: אֲנִי אוֹהֵב לִקְרוֹא סְפָרִים +---- + +[NOTE] +==== +The Hebrew model has similar PyTorch compatibility considerations as the Arabic +model. The fixes applied ensure smooth operation across different PyTorch +versions. +==== + +For more detailed instructions, please see the language-specific READMEs: + +* https://github.com/interscript/rababa/tree/main/python/arabic[Arabic] +* https://github.com/interscript/rababa/tree/main/python/hebrew[Hebrew] +* https://github.com/interscript/rababa/tree/main/lib[Ruby] + + +// ==== Training + +// === Ruby + == Under development We are working on the following improvements: * Enhancing architecture and encoding -* Enhance datasets to improve models +* Enhancing datasets to improve models == License and copyright @@ -132,3 +298,53 @@ The Tashkeela dataset used for training is provided under GPL v2 license: * Processed dataset by Hamza Abbad: https://sourceforge.net/projects/tashkeela-processed/ * License: GPL v2 + +== Troubleshooting + +=== PyTorch Compatibility Issues + +If you encounter model loading errors related to the `weights_only` parameter in PyTorch, try one of the following solutions: + +1. *For Hebrew models:* ++ +If you see errors when loading the Hebrew model, verify that the fix for the `weights_only` parameter is applied in the Hebrew config_manager.py file. The load_model function should *not* include the `weights_only` parameter: ++ +[source,python] +---- +saved_model = ( + torch.load(model_path) + if torch.cuda.is_available() + else torch.load(model_path, map_location=torch.device("cpu")) +) +---- + +2. *For Arabic models:* ++ +If you see errors when loading the Arabic model, the `weights_only` parameter should be explicitly set to `False` in the config_manager.py file: ++ +[source,python] +---- +saved_model = torch.load(last_model_path, weights_only=False) if torch.cuda.is_available() else torch.load(last_model_path, map_location=torch.device('cpu'), weights_only=False) +---- + +3. *Using a specific PyTorch version:* ++ +If problems persist, installing a specific PyTorch version compatible with the saved model formats may help: ++ +[source,bash] +---- +pip install torch==1.13.1 +---- + +4. *Model conversion:* ++ +For persistent issues, you may need to load and re-save the models with your current PyTorch version: ++ +[source,python] +---- +# Load model with explicit weights_only=False setting +old_model = torch.load("model.pt", weights_only=False) + +# Save model in a compatible format +torch.save(old_model, "new_model.pt") +---- diff --git a/python/arabic/config_manager.py b/python/arabic/config_manager.py index 2a486c7..1df6d02 100644 --- a/python/arabic/config_manager.py +++ b/python/arabic/config_manager.py @@ -188,9 +188,9 @@ def load_model(self, model_path: str = None): return model, 1 else: last_model_path = model_path - - saved_model = torch.load(last_model_path) if torch.cuda.is_available() else torch.load(last_model_path, map_location=torch.device('cpu')) - + + saved_model = torch.load(last_model_path, weights_only=False) if torch.cuda.is_available() else torch.load(last_model_path, map_location=torch.device('cpu'), weights_only=False) + out = model.load_state_dict(saved_model["model_state_dict"]) # print(out) check... global_step = saved_model["global_step"] + 1 diff --git a/python/arabic/convert_torch_model_to_onnx.py b/python/arabic/convert_torch_model_to_onnx.py index 48f7296..def7f55 100644 --- a/python/arabic/convert_torch_model_to_onnx.py +++ b/python/arabic/convert_torch_model_to_onnx.py @@ -1,175 +1,139 @@ import torch import pickle +import multiprocessing +import random import numpy as np import yaml from diacritizer import Diacritizer +import onnx +import onnxruntime -""" - Key Params: - max_len: - is the max length for the arabic strings to be diacritized - batch size: - has to do with the model training and usage -""" - -d_params = yaml.load(open("config/convert_torch_onnx.yml")) - -max_len = d_params["max_len"] # 600 for the original length -batch_size = d_params["batch_size"] -config_str = d_params["config_str"] -model_kind_str = d_params["model_kind_str"] -onnx_model_filename = d_params["onnx_model_filename"] - +def main(): + """ + Key Params: + max_len: + is the max length for the arabic strings to be diacritized + batch size: + has to do with the model training and usage + """ -""" - example and mock data: - we found that populating all the data, removing the zeros gives better results. -""" + d_params = yaml.load(open("config/convert_torch_onnx.yml")) -src = torch.Tensor([[1 for i in range(max_len)] for i in range(batch_size)]).long() -lengths = torch.Tensor([max_len for i in range(batch_size)]).long() + max_len = d_params["max_len"] # 600 for the original length + batch_size = d_params["batch_size"] + config_str = d_params["config_str"] + model_kind_str = d_params["model_kind_str"] + onnx_model_filename = d_params["onnx_model_filename"] -""" - Instantiate Diacritization model -""" + """ + example and mock data: + we found that populating all the data, removing the zeros gives better results. + """ -dia = Diacritizer(config_str, model_kind_str, True) + src = torch.Tensor([[1 for i in range(max_len)] for i in range(batch_size)]).long() + lengths = torch.Tensor([max_len for i in range(batch_size)]).long() -# set model to inference mode -dia.model.to(dia.device) -dia.model.eval() -# run model -torch_out = dia.model(src, lengths) + """ + Instantiate Diacritization model + """ -""" - Load ONNX libs and export models into onnx -""" + dia = Diacritizer(config_str, model_kind_str, True) -import torch -import onnx -import onnxruntime + # set model to inference mode + dia.model.to(dia.device) + dia.model.eval() + # run model + torch_out = dia.model(src, lengths) -# export model -torch.onnx.export( - dia.model, - (src, lengths), - onnx_model_filename, - verbose=False, - opset_version=11, - input_names=["src", "lengths"], - output_names=["output"], - dynamic_axes={ - "src": [1], # [0,1,2], #[0,1,2], - #'input_2':{0:'batch'}, - "output": [1], - }, -) - -print("Model printed in rel. path:", onnx_model_filename) - - -""" - Load ONNX versions of model -""" - -# load model -onnx_model = onnx.load(onnx_model_filename) -# check model -onnx.checker.check_model(onnx_model) -# inference session -ort_session = onnxruntime.InferenceSession(onnx_model_filename) - -# onnx inputs and outputs names -# ort_session.get_inputs(), ort_session.get_outputs() - - -""" - Run ONNX model on sample data -""" - -# prepare onnx input -ort_inputs = { - ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), - ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), -} - -# run onnx model -ort_outs = ort_session.run(None, ort_inputs) - -print("outs:: ", ort_outs) -print("src:: ", src.detach().numpy().astype(np.int64)) -print("lengths: ", lengths.detach().numpy().astype(np.int64)) - - -for i in range(batch_size): - np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), - ort_outs[0][i], - rtol=1e-03, - atol=1e-03, + """ + Load ONNX libs and export models into onnx + """ + + # export model + torch.onnx.export( + dia.model, + (src, lengths), + onnx_model_filename, + verbose=False, + opset_version=11, + input_names=["src", "lengths"], + output_names=["output"], + dynamic_axes={ + "src": [1], # [0,1,2], #[0,1,2], + #'input_2':{0:'batch'}, + "output": [1], + }, ) -print( - "\n!!!Exported model has been tested with ONNXRuntime, result looks good within given tolerance!!!" -) + print("Model printed in rel. path:", onnx_model_filename) -vec = [[41, 12, 40] for i in range(batch_size)] -src = torch.Tensor(vec).long() + """ + Load ONNX versions of model + """ -lengths = torch.Tensor([3 for i in range(batch_size)]).long() + # load model + onnx_model = onnx.load(onnx_model_filename) + # check model + onnx.checker.check_model(onnx_model) + # inference session + ort_session = onnxruntime.InferenceSession(onnx_model_filename) -ort_inputs = { - ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), - ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), -} + # onnx inputs and outputs names + # ort_session.get_inputs(), ort_session.get_outputs() -ort_outs = ort_session.run(None, ort_inputs) + """ + Run ONNX model on sample data + """ -torch_out = dia.model(src, lengths) - -for i in range(batch_size): - np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), - ort_outs[0][i], - rtol=1e-03, - atol=1e-03, - ) + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), + ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), + } + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) -""" - Test ONNX model on randomized data -""" + print("outs:: ", ort_outs) + print("src:: ", src.detach().numpy().astype(np.int64)) + print("lengths: ", lengths.detach().numpy().astype(np.int64)) -import random -test_id = 0 + for i in range(batch_size): + np.testing.assert_allclose( + torch_out["diacritics"][i].detach().numpy(), + ort_outs[0][i], + rtol=1e-03, + atol=1e-03, + ) -print("***** Test MAX size :: Random Boolean vectors: *****") + print( + "\n!!!Exported model has been tested with ONNXRuntime, result looks good within given tolerance!!!" + ) -for test_run in range(3): - vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] + vec = [[41, 12, 40] for i in range(batch_size)] src = torch.Tensor(vec).long() - lengths = torch.Tensor([max_len for i in range(batch_size)]).long() - torch_out = dia.model(src, lengths) + lengths = torch.Tensor([3 for i in range(batch_size)]).long() - # prepare onnx input ort_inputs = { ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), } - # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + torch_out = dia.model(src, lengths) + for i in range(batch_size): np.testing.assert_allclose( torch_out["diacritics"][i].detach().numpy(), @@ -178,93 +142,133 @@ atol=1e-03, ) - print("test :: ", test_run) - print("Result looks good within given tolerance!!!") + """ + Test ONNX model on randomized data + """ -print("***** Test MAX size :: Random float, vectors within 0:16 *****") + test_id = 0 -for test_run in range(3): + print("***** Test MAX size :: Random Boolean vectors: *****") - vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] - src = torch.Tensor(vec).long() - torch_out = dia.model(src, lengths) + for test_run in range(3): - # my_list = torch_out['diacritics'].detach().numpy().tolist() - # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), - ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), - } + vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] + src = torch.Tensor(vec).long() + lengths = torch.Tensor([max_len for i in range(batch_size)]).long() - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + torch_out = dia.model(src, lengths) - for i in range(batch_size): - np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), ort_outs[0][i], rtol=1, atol=1 - ) + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), + ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), + } - print("test :: ", test_run) - print("Result looks good within given tolerance!!!") + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + for i in range(batch_size): + np.testing.assert_allclose( + torch_out["diacritics"][i].detach().numpy(), + ort_outs[0][i], + rtol=1e-03, + atol=1e-03, + ) -print("***** Test Dynamical sizes :: Random Boolean vectors: *****") + print("test :: ", test_run) + print("Result looks good within given tolerance!!!") -for l in [2, 10, 40, 100, 150]: - print("length:: ", l) + print("***** Test MAX size :: Random float, vectors within 0:16 *****") - vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) - src = torch.Tensor(vec).long() - lengths = torch.Tensor([l for i in range(batch_size)]).long() + for test_run in range(3): - torch_out = dia.model(src, lengths) + vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] + src = torch.Tensor(vec).long() + torch_out = dia.model(src, lengths) - # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), - ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), - } + # my_list = torch_out['diacritics'].detach().numpy().tolist() + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), + ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), + } - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) - for i in range(batch_size): - np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), - ort_outs[0][i], - rtol=1e-03, - atol=1e-03, - ) + for i in range(batch_size): + np.testing.assert_allclose( + torch_out["diacritics"][i].detach().numpy(), ort_outs[0][i], rtol=1, atol=1 + ) - print("test :: ", l) - print("Result looks good within given tolerance!!!") + print("test :: ", test_run) + print("Result looks good within given tolerance!!!") -print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") + print("***** Test Dynamical sizes :: Random Boolean vectors: *****") -for l in [2, 10, 40, 100, 150]: + for l in [2, 10, 40, 100, 150]: - vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] - src = torch.Tensor(vec).long() - lengths = torch.Tensor([l for i in range(batch_size)]).long() + print("length:: ", l) - torch_out = dia.model(src, lengths) + vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) + src = torch.Tensor(vec).long() + lengths = torch.Tensor([l for i in range(batch_size)]).long() - # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), - ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), - } + torch_out = dia.model(src, lengths) - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), + ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), + } + + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + + for i in range(batch_size): + np.testing.assert_allclose( + torch_out["diacritics"][i].detach().numpy(), + ort_outs[0][i], + rtol=1e-03, + atol=1e-03, + ) + + print("test :: ", l) + print("Result looks good within given tolerance!!!") + + + print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") + + for l in [2, 10, 40, 100, 150]: + + vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] + src = torch.Tensor(vec).long() + lengths = torch.Tensor([l for i in range(batch_size)]).long() + + torch_out = dia.model(src, lengths) + + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), + ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), + } + + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + + for i in range(batch_size): + np.testing.assert_allclose( + torch_out["diacritics"][i].detach().numpy(), ort_outs[0][i], rtol=1, atol=1 + ) + + print("test :: ", l) + print("Result looks good within given tolerance!!!") - for i in range(batch_size): - np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), ort_outs[0][i], rtol=1, atol=1 - ) - print("test :: ", l) - print("Result looks good within given tolerance!!!") +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/arabic/diacritize.py b/python/arabic/diacritize.py index 30f1222..a0c3448 100644 --- a/python/arabic/diacritize.py +++ b/python/arabic/diacritize.py @@ -2,6 +2,7 @@ from diacritizer import Diacritizer from itertools import repeat import random +import multiprocessing import numpy as np import torch @@ -25,22 +26,29 @@ def diacritization_parser(): return parser -parser = diacritization_parser() -args = parser.parse_args() +def main(): + parser = diacritization_parser() + args = parser.parse_args() -if args.text is None and args.text_file is None: - raise ValueError("text or text_file params required!") + if args.text is None and args.text_file is None: + raise ValueError("text or text_file params required!") -if args.model_kind == "cbhg": - diacritizer = Diacritizer(args.config, args.model_kind, 'log_dir') -elif args.model_kind == "baseline": - diacritizer = Diacritizer(args.config, args.model_kind, 'log_dir') -else: - raise ValueError("The model kind is not supported") + if args.model_kind == "cbhg": + diacritizer = Diacritizer(args.config, args.model_kind, 'log_dir') + elif args.model_kind == "baseline": + diacritizer = Diacritizer(args.config, args.model_kind, 'log_dir') + else: + raise ValueError("The model kind is not supported") -if args.text_file is None: - txt = diacritizer.diacritize_text(args.text) - print(txt) -else: - for txt in diacritizer.diacritize_file(args.text_file): + if args.text_file is None: + txt = diacritizer.diacritize_text(args.text) print(txt) + else: + for txt in diacritizer.diacritize_file(args.text_file): + print(txt) + + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/arabic/test.py b/python/arabic/test.py index d98834b..ef1a343 100644 --- a/python/arabic/test.py +++ b/python/arabic/test.py @@ -1,5 +1,6 @@ import argparse import random +import multiprocessing from tester import DiacritizationTester import numpy as np @@ -23,8 +24,15 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() +def main(): + parser = train_parser() + args = parser.parse_args() -tester = DiacritizationTester(args.config, args.model_kind) -tester.run() + tester = DiacritizationTester(args.config, args.model_kind) + tester.run() + + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/arabic/train.py b/python/arabic/train.py index 5dba6d8..e47d976 100644 --- a/python/arabic/train.py +++ b/python/arabic/train.py @@ -1,6 +1,7 @@ import argparse import random +import multiprocessing import numpy as np import torch @@ -31,13 +32,19 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() +def main(): + parser = train_parser() + args = parser.parse_args() + if args.model_kind in ['baseline',"cbhg"]: + trainer = CBHGTrainer(args.config, args.model_kind) + else: + raise ValueError("The model kind is not supported") -if args.model_kind in ['baseline',"cbhg"]: - trainer = CBHGTrainer(args.config, args.model_kind) -else: - raise ValueError("The model kind is not supported") + trainer.run() -trainer.run() + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/config/cbhg.yml b/python/hebrew/config/cbhg.yml index 454c955..033aa99 100644 --- a/python/hebrew/config/cbhg.yml +++ b/python/hebrew/config/cbhg.yml @@ -55,4 +55,8 @@ n_steps_avg_losses: [100, 500, 1_000, 5_000] # command line display of average l error_rates_n_batches: 10000 # if calculating error rate is slow, then you can specify the number of batches to be calculated test_model_path: null # load the last saved model + +# Path to model weights file (pytorch model) +# This line is added to support testing without an existing model +model_path: null train_resume_model_path: null # load last saved model diff --git a/python/hebrew/convert_torch_model_to_onnx.py b/python/hebrew/convert_torch_model_to_onnx.py index 1ece73f..153bc72 100644 --- a/python/hebrew/convert_torch_model_to_onnx.py +++ b/python/hebrew/convert_torch_model_to_onnx.py @@ -1,6 +1,8 @@ import torch import pickle import random +import multiprocessing +import yaml import torch import onnx @@ -11,238 +13,245 @@ from diacritizer import Diacritizer -""" - Key Params: - max_len: - is the max length for the arabic strings to be diacritized - batch size: - has to do with the model training and usage -""" +def main(): + """ + Key Params: + max_len: + is the max length for the arabic strings to be diacritized + batch size: + has to do with the model training and usage + """ -d_params = yaml.load(open("config/convert_torch_onnx.yml")) -max_len = d_params["max_len"] # 600 for the original length -batch_size = d_params["batch_size"] -config_str = d_params["config_str"] -model_kind_str = d_params["model_kind_str"] -onnx_model_filename = d_params["onnx_model_filename"] -device = d_params["device"] + d_params = yaml.load(open("config/convert_torch_onnx.yml")) + max_len = d_params["max_len"] # 600 for the original length + batch_size = d_params["batch_size"] + config_str = d_params["config_str"] + model_kind_str = d_params["model_kind_str"] + onnx_model_filename = d_params["onnx_model_filename"] + device = d_params["device"] -""" - example and mock data: - we found that populating all the data, removing the zeros gives better results. -""" + """ + example and mock data: + we found that populating all the data, removing the zeros gives better results. + """ -normalized = torch.Tensor( - [[1 for i in range(max_len)] for i in range(batch_size)] -).long() + normalized = torch.Tensor( + [[1 for i in range(max_len)] for i in range(batch_size)] + ).long() -""" - Instantiate Diacritization model -""" + """ + Instantiate Diacritization model + """ -dia = Diacritizer(config_str, model_kind_str, True) + dia = Diacritizer(config_str, model_kind_str, True) -# set model to inference mode -dia.model.to(device) -dia.model.eval() -normalized = normalized.to(device) + # set model to inference mode + dia.model.to(device) + dia.model.eval() + normalized = normalized.to(device) -# run model -niqqud, dagesh, sin = dia.model(normalized) -torch_outs = dia.model(normalized) # niqqud, dagesh, sin + # run model + niqqud, dagesh, sin = dia.model(normalized) + torch_outs = dia.model(normalized) # niqqud, dagesh, sin -""" - Load ONNX libs and export models into onnx -""" + """ + Load ONNX libs and export models into onnx + """ -onnx_model_filename = "../models-data/diacritization_model.onnx" + onnx_model_filename = "../models-data/diacritization_model.onnx" -# export model -torch.onnx.export( - dia.model, - normalized, - onnx_model_filename, - verbose=False, - opset_version=11, - input_names=["normalized"], - output_names=["niqqud", "dagesh", "sin"], - dynamic_axes={"normalized": [1], "output": [1], "dagesh": [1], "sin": [1]}, -) + # export model + torch.onnx.export( + dia.model, + normalized, + onnx_model_filename, + verbose=False, + opset_version=11, + input_names=["normalized"], + output_names=["niqqud", "dagesh", "sin"], + dynamic_axes={"normalized": [1], "output": [1], "dagesh": [1], "sin": [1]}, + ) -print("Model printed in rel. path:", onnx_model_filename) + print("Model printed in rel. path:", onnx_model_filename) -""" - Load ONNX versions of model -""" + """ + Load ONNX versions of model + """ -# load model -onnx_model = onnx.load(onnx_model_filename) -# check model -onnx.checker.check_model(onnx_model) -# inference session -ort_session = onnxruntime.InferenceSession(onnx_model_filename) + # load model + onnx_model = onnx.load(onnx_model_filename) + # check model + onnx.checker.check_model(onnx_model) + # inference session + ort_session = onnxruntime.InferenceSession(onnx_model_filename) -# get onnx inputs and outputs names -# ort_session.get_inputs(), ort_session.get_outputs() + # get onnx inputs and outputs names + # ort_session.get_inputs(), ort_session.get_outputs() -""" - Run ONNX model on sample data -""" + """ + Run ONNX model on sample data + """ -# prepare onnx input -ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) -} + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } -# run onnx model -ort_outs = ort_session.run(None, ort_inputs) + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) -for i in range(batch_size): - for dim in range(3): # niqqud, dagesh, sin - np.testing.assert_allclose( - torch_outs[dim][i].detach().numpy(), - ort_outs[dim][i], - rtol=1e-02, - atol=1e-02, - ) + for i in range(batch_size): + for dim in range(3): # niqqud, dagesh, sin + np.testing.assert_allclose( + torch_outs[dim][i].detach().numpy(), + ort_outs[dim][i], + rtol=1e-02, + atol=1e-02, + ) -print( - "\n!!!Exported model has been tested with ONNXRuntime, \ - result looks good within given tolerance!!!" -) + print( + "\n!!!Exported model has been tested with ONNXRuntime, \ + result looks good within given tolerance!!!" + ) -vec = [[41, 12, 40] for i in range(batch_size)] -normalized = torch.Tensor(vec).long() + vec = [[41, 12, 40] for i in range(batch_size)] + normalized = torch.Tensor(vec).long() -ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) -} + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } -""" - Test ONNX model on randomized data -""" + """ + Test ONNX model on randomized data + """ -test_id = 0 + test_id = 0 -print("***** Test MAX size :: Random Boolean vectors: *****") -print(max_len) + print("***** Test MAX size :: Random Boolean vectors: *****") + print(max_len) -for test_run in range(3): + for test_run in range(3): - vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] - normalized = torch.Tensor(vec).long() + vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] + normalized = torch.Tensor(vec).long() - torch_outs = dia.model(normalized) - # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) - } + torch_outs = dia.model(normalized) + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) - for i in range(batch_size): - for dim in range(3): - np.testing.assert_allclose( - torch_outs[dim][i].detach().numpy(), - ort_outs[dim][i], - rtol=1e-01, - atol=1e-01, - ) + for i in range(batch_size): + for dim in range(3): + np.testing.assert_allclose( + torch_outs[dim][i].detach().numpy(), + ort_outs[dim][i], + rtol=1e-01, + atol=1e-01, + ) - print("test :: ", test_run) - print("Result looks good within given tolerance!!!") + print("test :: ", test_run) + print("Result looks good within given tolerance!!!") -print("***** Test MAX size :: Random float, vectors within 0:16 *****") -print(max_len) + print("***** Test MAX size :: Random float, vectors within 0:16 *****") + print(max_len) -for test_run in range(3): + for test_run in range(3): - vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] - normalized = torch.Tensor(vec).long() - torch_out = dia.model(normalized) + vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] + normalized = torch.Tensor(vec).long() + torch_out = dia.model(normalized) - # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) - } + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) - for i in range(batch_size): - for dim in range(3): - np.testing.assert_allclose( - torch_out[dim][i].detach().numpy(), ort_outs[dim][i], rtol=1, atol=1 - ) + for i in range(batch_size): + for dim in range(3): + np.testing.assert_allclose( + torch_out[dim][i].detach().numpy(), ort_outs[dim][i], rtol=1, atol=1 + ) - print("test :: ", test_run) - print("Result looks good within given tolerance!!!") + print("test :: ", test_run) + print("Result looks good within given tolerance!!!") -print("***** Test Dynamical sizes :: Random Boolean vectors: *****") + print("***** Test Dynamical sizes :: Random Boolean vectors: *****") -for l in [2, 10, 40, 100, 150]: + for l in [2, 10, 40, 100, 150]: - print("length:: ", l) + print("length:: ", l) - vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) - normalized = torch.Tensor(vec).long() + vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) + normalized = torch.Tensor(vec).long() - torch_out = dia.model(normalized) + torch_out = dia.model(normalized) - # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) - } + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) - for i in range(batch_size): - for dim in range(3): - np.testing.assert_allclose( - torch_out[dim][i].detach().numpy(), - ort_outs[dim][i], - rtol=1e-02, - atol=1e-02, - ) + for i in range(batch_size): + for dim in range(3): + np.testing.assert_allclose( + torch_out[dim][i].detach().numpy(), + ort_outs[dim][i], + rtol=1e-02, + atol=1e-02, + ) - print("test :: ", l) - print("Result looks good within given tolerance!!!") + print("test :: ", l) + print("Result looks good within given tolerance!!!") -print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") + print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") -for l in [2, 10, 40, 100, 150]: + for l in [2, 10, 40, 100, 150]: - vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] - normalized = torch.Tensor(vec).long() + vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] + normalized = torch.Tensor(vec).long() - torch_out = dia.model(normalized) + torch_out = dia.model(normalized) - # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) - } + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + + for i in range(batch_size): + for dim in range(3): + np.testing.assert_allclose( + torch_out[dim][i].detach().numpy(), ort_outs[dim][i], rtol=1, atol=1 + ) + + print("test :: ", l) + print("Result looks good within given tolerance!!!") - for i in range(batch_size): - for dim in range(3): - np.testing.assert_allclose( - torch_out[dim][i].detach().numpy(), ort_outs[dim][i], rtol=1, atol=1 - ) - print("test :: ", l) - print("Result looks good within given tolerance!!!") +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/diacritize.py b/python/hebrew/diacritize.py index fcda882..00e499b 100644 --- a/python/hebrew/diacritize.py +++ b/python/hebrew/diacritize.py @@ -2,6 +2,7 @@ from diacritizer import Diacritizer from itertools import repeat import random +import multiprocessing import numpy as np import torch @@ -31,23 +32,29 @@ def diacritization_parser(): return parser -parser = diacritization_parser() -args = parser.parse_args() +def main(): + parser = diacritization_parser() + args = parser.parse_args() + if args.text is None and args.text_file is None: + raise ValueError("text or text_file/diacritized_text_file params required!") -if args.text is None and args.text_file is None: - raise ValueError("text or text_file/diacritized_text_file params required!") + if args.model_kind == "cbhg": + diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") + elif args.model_kind == "baseline": + diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") + else: + raise ValueError("The model kind is not supported") -if args.model_kind == "cbhg": - diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") -elif args.model_kind == "baseline": - diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") -else: - raise ValueError("The model kind is not supported") + if args.text_file is None: + txt = diacritizer.diacritize_text(args.text) + print(txt) + else: + diacritizer.diacritize_file(args.text_file, args.diacritized_text_file) + print("done!!! written in: ", args.diacritized_text_file) -if args.text_file is None: - txt = diacritizer.diacritize_text(args.text) - print(txt) -else: - diacritizer.diacritize_file(args.text_file, args.diacritized_text_file) - print("done!!! written in: ", args.diacritized_text_file) + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/run_experiments_wandb.py b/python/hebrew/run_experiments_wandb.py index 2ed0379..9099fdf 100644 --- a/python/hebrew/run_experiments_wandb.py +++ b/python/hebrew/run_experiments_wandb.py @@ -1,6 +1,7 @@ import argparse import random +import multiprocessing import numpy as np import torch @@ -36,9 +37,6 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() - # Define Experiments using Wandb sweep_config = { # search method @@ -46,7 +44,7 @@ def train_parser(): # metric and objective 'metric': { 'name': 'dec', - 'goal': 'maximize' #'minimize' + 'goal': 'maximize' #'minimize' }, # define search parameters 'parameters': { @@ -69,7 +67,7 @@ def train_parser(): 'post_cbhg_layers_units': { 'values': [[256, 256]] }, - + 'optimizer': { 'values': ['Adam', 'SGD'] }, @@ -78,26 +76,28 @@ def train_parser(): }, 'prenet_sizes': { 'values': [[512, 256]] - } + } } } -# train code, with the search preprocessing logic +# train code, with the search preprocessing logic def train(): + parser = train_parser() + args = parser.parse_args() with open('config/train.yml', "rb") as model_yaml: config = yaml.load(model_yaml) - + # load default config - config_defaults = config + config_defaults = config wandb.init(config=config_defaults) # , magic=True) config_wandb = wandb.config - + # overwrite initial config - config = { **config, + config = { **config, **config_wandb } - + tmp_config_path = 'config/sweep_tmp.yml' with open(tmp_config_path, 'w') as yaml_file: yaml.dump(config, yaml_file, default_flow_style=False) @@ -108,19 +108,21 @@ def train(): raise ValueError("The model kind is not supported") trainer.run(config_wandb) - - -################################## -# MAIN # -################################## -# Run name -run_name = "hyperparams search" +def main(): + # Run name + run_name = "hyperparams search" + + # Init wandb and search + wandb.login() + sweep_id = wandb.sweep(sweep_config, project=run_name) + + # Run search + wandb.agent(sweep_id, train) -# Init wandb and search -wandb.login() -sweep_id = wandb.sweep(sweep_config, project=run_name) -# Run search -wandb.agent(sweep_id, train) +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/test.py b/python/hebrew/test.py index d98834b..ef1a343 100644 --- a/python/hebrew/test.py +++ b/python/hebrew/test.py @@ -1,5 +1,6 @@ import argparse import random +import multiprocessing from tester import DiacritizationTester import numpy as np @@ -23,8 +24,15 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() +def main(): + parser = train_parser() + args = parser.parse_args() -tester = DiacritizationTester(args.config, args.model_kind) -tester.run() + tester = DiacritizationTester(args.config, args.model_kind) + tester.run() + + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/train.py b/python/hebrew/train.py index 3e9ae93..2ec489b 100644 --- a/python/hebrew/train.py +++ b/python/hebrew/train.py @@ -1,6 +1,7 @@ import argparse import random +import multiprocessing import numpy as np import torch @@ -32,13 +33,19 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() +def main(): + parser = train_parser() + args = parser.parse_args() + if args.model_kind in ['baseline',"cbhg"]: + trainer = CBHGTrainer(args.config, args.model_kind) + else: + raise ValueError("The model kind is not supported") -if args.model_kind in ['baseline',"cbhg"]: - trainer = CBHGTrainer(args.config, args.model_kind) -else: - raise ValueError("The model kind is not supported") + trainer.run() -trainer.run() + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() From 87465dc8beedb69cf75cec0d0ad1a9153e57a82a Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 30 Apr 2025 00:40:51 +0800 Subject: [PATCH 2/8] fix: gha --- .github/workflows/python-arabic.yml | 4 ---- python/arabic/convert_torch_model_to_onnx.py | 2 +- python/arabic/requirements.txt | 20 +++++++++--------- python/arabic/setup.py | 2 +- python/arabic/train.py | 1 - python/hebrew/convert_torch_model_to_onnx.py | 2 +- python/hebrew/requirements.txt | 22 ++++++++++---------- python/hebrew/setup.py | 2 +- 8 files changed, 25 insertions(+), 30 deletions(-) diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index 983e8f3..ac7f45d 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -131,10 +131,6 @@ jobs: - name: Try training (WIP) working-directory: ./python/arabic run: | - # Fix for Python 3.9+ multiprocessing issue - if [[ "${{ matrix.python-version }}" == "3.9" || "${{ matrix.python-version }}" > "3.9" ]]; then - sed -i '1s/^/import multiprocessing\nif __name__ == "__main__":\n multiprocessing.freeze_support()\n/' train.py - fi python train.py --model "cbhg" --config config/test_cbhg.yml security-scan: diff --git a/python/arabic/convert_torch_model_to_onnx.py b/python/arabic/convert_torch_model_to_onnx.py index def7f55..d88e28f 100644 --- a/python/arabic/convert_torch_model_to_onnx.py +++ b/python/arabic/convert_torch_model_to_onnx.py @@ -19,7 +19,7 @@ def main(): has to do with the model training and usage """ - d_params = yaml.load(open("config/convert_torch_onnx.yml")) + d_params = yaml.safe_load(open("config/convert_torch_onnx.yml")) max_len = d_params["max_len"] # 600 for the original length batch_size = d_params["batch_size"] diff --git a/python/arabic/requirements.txt b/python/arabic/requirements.txt index c7f0f70..e711c23 100644 --- a/python/arabic/requirements.txt +++ b/python/arabic/requirements.txt @@ -1,11 +1,11 @@ -torch==1.9.0 -numpy==1.19.5 -matplotlib==3.3.3 -pandas==1.1.5 -ruamel.yaml==0.16.12 -tensorboard==2.4.0 +torch>=1.9.0,<2.0.0 +numpy>=1.20.0,<2.0.0 +matplotlib>=3.3.3 +pandas>=1.3.0 +ruamel.yaml>=0.16.12 +tensorboard>=2.4.0 diacritization-evaluation==0.5 -tqdm==4.56.0 -onnx==1.9.0 -onnxruntime==1.8.1 -pyyaml==5.4.1 +tqdm>=4.56.0 +onnx>=1.9.0 +onnxruntime>=1.8.1 +pyyaml>=5.4.1 diff --git a/python/arabic/setup.py b/python/arabic/setup.py index 88bea0a..fd8ada6 100644 --- a/python/arabic/setup.py +++ b/python/arabic/setup.py @@ -23,7 +23,7 @@ description='Rababa for Arabic diacriticization', # packages=['rababa'], url='https://www.interscript.org', - python_requires='>=3.6, <4', + python_requires='>=3.8, <4', project_urls={ 'Documentation': 'https://github.com/interscript/rababa', 'Source': 'https://github.com/interscript/rababa', diff --git a/python/arabic/train.py b/python/arabic/train.py index e47d976..676cf67 100644 --- a/python/arabic/train.py +++ b/python/arabic/train.py @@ -1,4 +1,3 @@ - import argparse import random import multiprocessing diff --git a/python/hebrew/convert_torch_model_to_onnx.py b/python/hebrew/convert_torch_model_to_onnx.py index 153bc72..02e8301 100644 --- a/python/hebrew/convert_torch_model_to_onnx.py +++ b/python/hebrew/convert_torch_model_to_onnx.py @@ -22,7 +22,7 @@ def main(): has to do with the model training and usage """ - d_params = yaml.load(open("config/convert_torch_onnx.yml")) + d_params = yaml.safe_load(open("config/convert_torch_onnx.yml")) max_len = d_params["max_len"] # 600 for the original length batch_size = d_params["batch_size"] config_str = d_params["config_str"] diff --git a/python/hebrew/requirements.txt b/python/hebrew/requirements.txt index 4b932e4..eb9ffe7 100644 --- a/python/hebrew/requirements.txt +++ b/python/hebrew/requirements.txt @@ -1,12 +1,12 @@ -torch==1.9.0 -numpy==1.19.5 -matplotlib==3.3.3 -pandas==1.1.5 -ruamel.yaml==0.16.12 -tensorboard==2.4.0 +torch>=1.9.0,<2.0.0 +numpy>=1.20.0,<2.0.0 +matplotlib>=3.3.3 +pandas>=1.3.0 +ruamel.yaml>=0.16.12 +tensorboard>=2.4.0 diacritization-evaluation==0.5 -tqdm==4.56.0 -onnx==1.9.0 -onnxruntime==1.8.1 -pyyaml==5.4.1 -wandb==0.12.4 \ No newline at end of file +tqdm>=4.56.0 +onnx>=1.9.0 +onnxruntime>=1.8.1 +pyyaml>=5.4.1 +wandb>=0.12.4 diff --git a/python/hebrew/setup.py b/python/hebrew/setup.py index 88bea0a..fd8ada6 100644 --- a/python/hebrew/setup.py +++ b/python/hebrew/setup.py @@ -23,7 +23,7 @@ description='Rababa for Arabic diacriticization', # packages=['rababa'], url='https://www.interscript.org', - python_requires='>=3.6, <4', + python_requires='>=3.8, <4', project_urls={ 'Documentation': 'https://github.com/interscript/rababa', 'Source': 'https://github.com/interscript/rababa', From f8ee7ecdd4d3f432555751c2327e1fb0297873d3 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 1 May 2025 10:23:50 +0800 Subject: [PATCH 3/8] Fix GitHub Actions workflows and code compatibility issues - Added new Python Hebrew workflow file - Updated Ruby workflow to use newer Ruby versions (2.7-3.3) and latest actions - Fixed Python package setup.py files for both Arabic and Hebrew modules - Made wandb optional in Python code to avoid breaking builds - Modified test configuration files with correct parameters - Updated torch dependency to support newer versions (<3.0.0 instead of <2.0.0) - Added test data files for Hebrew module testing --- .github/workflows/python-arabic.yml | 2 +- .github/workflows/python-hebrew.yml | 122 ++++++++++++++++++++++++++++ .github/workflows/ruby.yml | 25 +++++- python/arabic/requirements.txt | 4 +- python/arabic/setup.py | 38 ++++----- python/hebrew/config/test_cbhg.yml | 38 +++++---- python/hebrew/data/eval/test.txt | 6 ++ python/hebrew/requirements.txt | 4 +- python/hebrew/setup.py | 41 +++++----- python/hebrew/train.py | 9 +- python/hebrew/trainer.py | 11 ++- 11 files changed, 233 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/python-hebrew.yml create mode 100644 python/hebrew/data/eval/test.txt diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index ac7f45d..178297d 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -1,4 +1,4 @@ -name: Python CI +name: Python Arabic CI on: push: diff --git a/.github/workflows/python-hebrew.yml b/.github/workflows/python-hebrew.yml new file mode 100644 index 0000000..e822517 --- /dev/null +++ b/.github/workflows/python-hebrew.yml @@ -0,0 +1,122 @@ +name: Python Hebrew CI + +on: + push: + branches: [ main ] + paths: + - 'python/hebrew/**' + - '.github/workflows/python-hebrew.yml' + pull_request: + paths: + - 'python/hebrew/**' + - '.github/workflows/python-hebrew.yml' + +# Cancel in-progress runs for the same workflow and branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + security-events: write + +jobs: + dependency-review: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Dependency Review + uses: actions/dependency-review-action@v3 + + codeql: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: python + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + + infer: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + python/hebrew/requirements.txt + python/hebrew/setup.py + + - name: Install requirements + working-directory: ./python/hebrew + run: | + python -m pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . + + - name: Create model directory + working-directory: ./python/hebrew + run: | + mkdir -p log_dir/base.cbhg/ + + - name: Run diacriticization + working-directory: ./python/hebrew + run: | + python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'שלום' + + train: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + python/hebrew/requirements.txt + python/hebrew/setup.py + + - name: Install requirements + working-directory: ./python/hebrew + run: | + python -m pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . + + - name: Prepare test data + working-directory: ./python/hebrew + run: | + mkdir -p data/test + echo "שלום עולם" > data/test/test.txt + + - name: Try training (WIP) + working-directory: ./python/hebrew + run: | + python train.py --model "cbhg" --config config/test_cbhg.yml diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index f8ebdbf..00bcb9d 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -1,9 +1,26 @@ -name: ruby +name: Ruby CI on: push: branches: [ main ] + paths: + - 'lib/**' + - 'spec/**' + - 'Gemfile' + - 'rababa.gemspec' + - '.github/workflows/ruby.yml' pull_request: + paths: + - 'lib/**' + - 'spec/**' + - 'Gemfile' + - 'rababa.gemspec' + - '.github/workflows/ruby.yml' + +# Cancel in-progress runs for the same workflow and branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: build: @@ -11,13 +28,13 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ['2.6', '2.7', '3.0', '3.1', '3.2'] + ruby-version: ['2.7', '3.0', '3.1', '3.2', '3.3'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Ruby - uses: ruby/setup-ruby@v1 + uses: ruby/setup-ruby@v1.171.0 with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true diff --git a/python/arabic/requirements.txt b/python/arabic/requirements.txt index e711c23..468be4a 100644 --- a/python/arabic/requirements.txt +++ b/python/arabic/requirements.txt @@ -1,10 +1,10 @@ -torch>=1.9.0,<2.0.0 +torch>=1.9.0,<3.0.0 numpy>=1.20.0,<2.0.0 matplotlib>=3.3.3 pandas>=1.3.0 ruamel.yaml>=0.16.12 tensorboard>=2.4.0 -diacritization-evaluation==0.5 +diacritization-evaluation>=0.5 tqdm>=4.56.0 onnx>=1.9.0 onnxruntime>=1.8.1 diff --git a/python/arabic/setup.py b/python/arabic/setup.py index fd8ada6..89092be 100644 --- a/python/arabic/setup.py +++ b/python/arabic/setup.py @@ -2,6 +2,7 @@ from os import environ import setuptools +from setuptools import find_packages with open("README.adoc", "r", encoding="utf-8") as fh: LONG_DESCRIPTION = fh.read() @@ -15,13 +16,18 @@ PKG_VERSION = TAG_VERSION.group(1) setuptools.setup( - name='rababa', + name='rababa-arabic', version=PKG_VERSION, author="Ribose", author_email="open.source@ribose.com", license='MIT', description='Rababa for Arabic diacriticization', - # packages=['rababa'], + packages=find_packages(include=[ + "*", + "models.*", + "modules.*", + "util.*", + ]), url='https://www.interscript.org', python_requires='>=3.8, <4', project_urls={ @@ -30,23 +36,17 @@ 'Tracker': 'https://github.com/interscript/rababa/issues', }, install_requires=[ - 'torch>=1.9.0', - 'numpy', - 'matplotlib', - 'pandas', - 'ruamel.yaml', - 'tensorboard', - 'diacritization-evaluation', - 'tqdm', - 'onnx', - 'onnxruntime', - 'pyyaml', + 'torch>=1.9.0,<3.0.0', + 'numpy>=1.20.0,<2.0.0', + 'matplotlib>=3.3.3', + 'pandas>=1.3.0', + 'ruamel.yaml>=0.16.12', + 'tensorboard>=2.4.0', + 'diacritization-evaluation>=0.5', + 'tqdm>=4.56.0', + 'onnx>=1.9.0', + 'onnxruntime>=1.8.1', + 'pyyaml>=5.4.1', ], - # extras_require={'plotting': ['matplotlib>=2.2.0', 'jupyter']}, setup_requires=['pytest-runner'], - tests_require=['pytest'], - # entry_points={ - # 'console_scripts': ['my-command=exampleproject.example:main'] - # }, - # package_data={'exampleproject': ['data/schema.json']} ) diff --git a/python/hebrew/config/test_cbhg.yml b/python/hebrew/config/test_cbhg.yml index cfdaa42..f3e7769 100644 --- a/python/hebrew/config/test_cbhg.yml +++ b/python/hebrew/config/test_cbhg.yml @@ -1,20 +1,21 @@ session_name: base data_directory: "data" -data_type: "CA_MSA" +data_type: "test" log_directory: "log_dir" load_training_data: true load_test_data: false load_validation_data: true -n_training_examples: null # null load all training examples, good for fast loading +n_training_examples: 5 # Using a small number for testing n_test_examples: null # null load all test examples n_validation_examples: null # null load all validation examples -test_file_name: "test.csv" -is_data_preprocessed: false # The data file is organized as (original text | text | diacritics) -data_separator: '|' # Required if the data already processed -diacritics_separator: '*' # Required if the data already processed -text_encoder: ArabicEncoderWithStartSymbol -text_cleaner: valid_arabic_cleaners # a white list that uses only Arabic letters, punctuations, and a space +train_file_name: "test.txt" +test_file_name: "test.txt" +is_data_preprocessed: false +data_separator: '|' +diacritics_separator: '*' +text_encoder: HebrewEncoder # Use Hebrew encoder +text_cleaner: basic_cleaners # Adjusted for Hebrew max_len: 600 # sentences larger than this size will not be used reconcile: true @@ -36,16 +37,23 @@ post_cbhg_use_batch_norm: true use_mixed_precision: false optimizer_type: Adam -device: cuda +device: cpu # Using CPU for testing + +# GEOMETRY +len_input_symbols: 90 +len_niqqud_symbols: 16 +len_dagesh_symbols: 3 +len_sin_symbols: 4 # LOGGING -evaluate_frequency: 5000 -evaluate_with_error_rates_frequency: 5000 -n_predicted_text_tensorboard: 10 # To be written to the tensorboard -model_save_frequency: 5000 +evaluate_frequency: 10 +evaluate_with_error_rates_frequency: 10 +n_predicted_text_tensorboard: 5 # To be written to the tensorboard +model_save_frequency: 10 train_plotting_frequency: 50000000 # No plotting for this model -n_steps_avg_losses: [100, 500, 1_000, 5_000] # command line display of average loss values for the last n steps -error_rates_n_batches: 10000 # if calculating error rate is slow, then you can specify the number of batches to be calculated +n_steps_avg_losses: [10, 20, 30, 40] # Reduced for testing +error_rates_n_batches: 5 # Reduced for testing test_model_path: null # load the last saved model train_resume_model_path: null # load last saved model +model_path: null diff --git a/python/hebrew/data/eval/test.txt b/python/hebrew/data/eval/test.txt new file mode 100644 index 0000000..63861de --- /dev/null +++ b/python/hebrew/data/eval/test.txt @@ -0,0 +1,6 @@ +שלום עולם +זה מבחן +בדיקה ניקוד +מערכת ניקוד עברית +ירושלים +תל אביב diff --git a/python/hebrew/requirements.txt b/python/hebrew/requirements.txt index eb9ffe7..fa9a890 100644 --- a/python/hebrew/requirements.txt +++ b/python/hebrew/requirements.txt @@ -1,10 +1,10 @@ -torch>=1.9.0,<2.0.0 +torch>=1.9.0,<3.0.0 numpy>=1.20.0,<2.0.0 matplotlib>=3.3.3 pandas>=1.3.0 ruamel.yaml>=0.16.12 tensorboard>=2.4.0 -diacritization-evaluation==0.5 +diacritization-evaluation>=0.5 tqdm>=4.56.0 onnx>=1.9.0 onnxruntime>=1.8.1 diff --git a/python/hebrew/setup.py b/python/hebrew/setup.py index fd8ada6..5cb37f4 100644 --- a/python/hebrew/setup.py +++ b/python/hebrew/setup.py @@ -2,6 +2,7 @@ from os import environ import setuptools +from setuptools import find_packages with open("README.adoc", "r", encoding="utf-8") as fh: LONG_DESCRIPTION = fh.read() @@ -15,13 +16,18 @@ PKG_VERSION = TAG_VERSION.group(1) setuptools.setup( - name='rababa', + name='rababa-hebrew', version=PKG_VERSION, author="Ribose", author_email="open.source@ribose.com", license='MIT', - description='Rababa for Arabic diacriticization', - # packages=['rababa'], + description='Rababa for Hebrew diacriticization', + packages=find_packages(include=[ + "*", + "models.*", + "modules.*", + "util.*", + ]), url='https://www.interscript.org', python_requires='>=3.8, <4', project_urls={ @@ -30,23 +36,18 @@ 'Tracker': 'https://github.com/interscript/rababa/issues', }, install_requires=[ - 'torch>=1.9.0', - 'numpy', - 'matplotlib', - 'pandas', - 'ruamel.yaml', - 'tensorboard', - 'diacritization-evaluation', - 'tqdm', - 'onnx', - 'onnxruntime', - 'pyyaml', + 'torch>=1.9.0,<3.0.0', + 'numpy>=1.20.0,<2.0.0', + 'matplotlib>=3.3.3', + 'pandas>=1.3.0', + 'ruamel.yaml>=0.16.12', + 'tensorboard>=2.4.0', + 'diacritization-evaluation>=0.5', + 'tqdm>=4.56.0', + 'onnx>=1.9.0', + 'onnxruntime>=1.8.1', + 'pyyaml>=5.4.1', + 'wandb>=0.12.4', ], - # extras_require={'plotting': ['matplotlib>=2.2.0', 'jupyter']}, setup_requires=['pytest-runner'], - tests_require=['pytest'], - # entry_points={ - # 'console_scripts': ['my-command=exampleproject.example:main'] - # }, - # package_data={'exampleproject': ['data/schema.json']} ) diff --git a/python/hebrew/train.py b/python/hebrew/train.py index 2ec489b..d6c389e 100644 --- a/python/hebrew/train.py +++ b/python/hebrew/train.py @@ -5,7 +5,14 @@ import numpy as np import torch -import wandb + +# Make wandb optional +try: + import wandb + WANDB_AVAILABLE = True +except ImportError: + WANDB_AVAILABLE = False + print("Warning: wandb not available, training will proceed without logging to wandb") from trainer import ( CBHGTrainer diff --git a/python/hebrew/trainer.py b/python/hebrew/trainer.py index 16e4feb..cab811f 100644 --- a/python/hebrew/trainer.py +++ b/python/hebrew/trainer.py @@ -29,7 +29,13 @@ from util import nakdimon_hebrew_model as hebrew from util import nakdimon_metrics -import wandb +# Make wandb optional +try: + import wandb + WANDB_AVAILABLE = True +except ImportError: + WANDB_AVAILABLE = False + print("Warning: wandb not available in trainer.py, training will proceed without wandb logging") class Trainer: @@ -249,8 +255,7 @@ def run(self, config_wandb=None): validation_iterator, tqdm_error_rates ) - if not config_wandb is None: - + if not config_wandb is None and WANDB_AVAILABLE: wandb.log({**d_scores, **scores}) print("scores:: ", scores) From 40ec3872410f10052fa99049dd3e950bc4f1f44f Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 3 Sep 2026 00:48:29 +0200 Subject: [PATCH 4/8] fix(ci): green the modernized python workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - drop EOL python 3.8 from matrixes (list[int] annotations need 3.9+) - hebrew train job: create data/train + data/eval test files the test_cbhg config actually reads (was data/test) - drop the hebrew workflow codeql job (duplicates the repo configured default CodeQL setup — same rationale as #61) - bump actions to checkout@v7 / setup-python@v7 - ruff: fix 5 I001 import sorts + format (verified in a clean checkout — local untracked dirs had masked isort classification) --- .github/workflows/python-arabic.yml | 4 +-- .github/workflows/python-hebrew.yml | 34 ++++++++------------------ python/arabic/util/text_encoders.py | 3 +-- python/hebrew/dataset.py | 1 - python/hebrew/diacritizer.py | 1 - python/hebrew/trainer.py | 3 +-- python/hebrew/util/nakdimon_dataset.py | 1 - 7 files changed, 14 insertions(+), 33 deletions(-) diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index e16d588..fd230a2 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -48,7 +48,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout code @@ -95,7 +95,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout code diff --git a/.github/workflows/python-hebrew.yml b/.github/workflows/python-hebrew.yml index e822517..a837e3c 100644 --- a/.github/workflows/python-hebrew.yml +++ b/.github/workflows/python-hebrew.yml @@ -26,40 +26,25 @@ jobs: if: github.event_name == 'pull_request' steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Dependency Review uses: actions/dependency-review-action@v3 - codeql: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: python - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - infer: runs-on: ubuntu-latest timeout-minutes: 10 strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -89,14 +74,14 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -113,8 +98,9 @@ jobs: - name: Prepare test data working-directory: ./python/hebrew run: | - mkdir -p data/test - echo "שלום עולם" > data/test/test.txt + mkdir -p data/train data/eval + printf "שלום עולם\nשלום עולם\nשלום עולם\nשלום עולם\nשלום עולם\n" > data/train/test.txt + printf "שלום עולם\nשלום עולם\n" > data/eval/test.txt - name: Try training (WIP) working-directory: ./python/hebrew diff --git a/python/arabic/util/text_encoders.py b/python/arabic/util/text_encoders.py index 3d09476..f6e5ed1 100644 --- a/python/arabic/util/text_encoders.py +++ b/python/arabic/util/text_encoders.py @@ -1,8 +1,7 @@ from typing import Optional -from util.constants import ALL_POSSIBLE_HARAQAT - from util import text_cleaners +from util.constants import ALL_POSSIBLE_HARAQAT class TextEncoder: diff --git a/python/hebrew/dataset.py b/python/hebrew/dataset.py index 9638ded..fb21537 100644 --- a/python/hebrew/dataset.py +++ b/python/hebrew/dataset.py @@ -6,7 +6,6 @@ from config_manager import ConfigManager from torch.utils.data import DataLoader, Dataset - from util import nakdimon_dataset from util import nakdimon_hebrew_model as hebrew from util import nakdimon_utils as utils diff --git a/python/hebrew/diacritizer.py b/python/hebrew/diacritizer.py index af3af92..303627a 100644 --- a/python/hebrew/diacritizer.py +++ b/python/hebrew/diacritizer.py @@ -3,7 +3,6 @@ from config_manager import ConfigManager from dataset import DiacritizationDataset, collate_fn from torch.utils.data import DataLoader - from util import nakdimon_dataset # as dataset from util import nakdimon_hebrew_model as hebrew from util import nakdimon_utils as utils diff --git a/python/hebrew/trainer.py b/python/hebrew/trainer.py index 8d19941..dbb4a3c 100644 --- a/python/hebrew/trainer.py +++ b/python/hebrew/trainer.py @@ -9,6 +9,7 @@ from torch.cuda.amp import autocast from torch.utils.tensorboard.writer import SummaryWriter from tqdm import trange +from util import nakdimon_dataset, nakdimon_metrics from util.learning_rates import LearningRateDecay from util.utils import ( count_parameters, @@ -17,8 +18,6 @@ repeater, ) -from util import nakdimon_dataset, nakdimon_metrics - # Make wandb optional try: import wandb diff --git a/python/hebrew/util/nakdimon_dataset.py b/python/hebrew/util/nakdimon_dataset.py index cdcba4a..0ef56e6 100644 --- a/python/hebrew/util/nakdimon_dataset.py +++ b/python/hebrew/util/nakdimon_dataset.py @@ -2,7 +2,6 @@ import numpy as np import torch - from util import nakdimon_hebrew_model as hebrew from util import nakdimon_utils as utils From a43b1ac1ad2b3fdd97e7d70e1a2dca3b1f7dfeaf Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 3 Sep 2026 00:52:39 +0200 Subject: [PATCH 5/8] fix(hebrew): test config needs eval_file_name for the modernized trainer The dataset loader reads config["eval_file_name"] for the validation set (train.yml already uses it); test_cbhg.yml predates the rename, so the CI train job died with KeyError after the data-prep fix. --- python/hebrew/config/test_cbhg.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/hebrew/config/test_cbhg.yml b/python/hebrew/config/test_cbhg.yml index f3e7769..7d1b0d4 100644 --- a/python/hebrew/config/test_cbhg.yml +++ b/python/hebrew/config/test_cbhg.yml @@ -10,6 +10,7 @@ n_training_examples: 5 # Using a small number for testing n_test_examples: null # null load all test examples n_validation_examples: null # null load all validation examples train_file_name: "test.txt" +eval_file_name: "test.txt" test_file_name: "test.txt" is_data_preprocessed: false data_separator: '|' From 8353ae82295d86fc48a19aeac99475ffee74d67d Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 3 Sep 2026 01:15:27 +0200 Subject: [PATCH 6/8] =?UTF-8?q?fix(ci):=20arabic=20train=20job=20needs=206?= =?UTF-8?q?0min=20=E2=80=94=2020min=20timeout=20killed=20step=2026/51?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/python-arabic.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index fd230a2..88e5a4b 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -87,7 +87,8 @@ jobs: train: runs-on: ubuntu-latest - timeout-minutes: 20 + # the WIP 51-step CPU train needs ~35 min after dataset download + timeout-minutes: 60 env: # tensorboard's generated descriptors predate protobuf 4; the pure-python # implementation tolerates them without pinning protobuf From 22c6d4516f0426fb3a6e0b830ecdeaeea658bbe7 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 3 Sep 2026 01:28:14 +0200 Subject: [PATCH 7/8] =?UTF-8?q?fix(ci):=20infer=20job=2020min=20=E2=80=94?= =?UTF-8?q?=20cold-cache=20torch=20download=20exceeds=2010?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/python-arabic.yml | 3 ++- .github/workflows/python-hebrew.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index 88e5a4b..daa3ace 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -44,7 +44,8 @@ jobs: infer: runs-on: ubuntu-latest - timeout-minutes: 10 + # cold pip cache: the eager torch wheel is ~5GB + timeout-minutes: 20 strategy: fail-fast: false matrix: diff --git a/.github/workflows/python-hebrew.yml b/.github/workflows/python-hebrew.yml index a837e3c..735410f 100644 --- a/.github/workflows/python-hebrew.yml +++ b/.github/workflows/python-hebrew.yml @@ -33,7 +33,8 @@ jobs: infer: runs-on: ubuntu-latest - timeout-minutes: 10 + # cold pip cache: the eager torch wheel is ~5GB + timeout-minutes: 20 strategy: fail-fast: false matrix: From 00dae934094392990b5201f702dfc83d58f1d556 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 3 Sep 2026 01:53:59 +0200 Subject: [PATCH 8/8] =?UTF-8?q?fix(ci):=20hebrew=20train=20job=2060min=20t?= =?UTF-8?q?oo=20=E2=80=94=20the=20WIP=20train=20is=2051=20steps=20x=2042s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/python-hebrew.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-hebrew.yml b/.github/workflows/python-hebrew.yml index 735410f..1f52273 100644 --- a/.github/workflows/python-hebrew.yml +++ b/.github/workflows/python-hebrew.yml @@ -71,7 +71,8 @@ jobs: train: runs-on: ubuntu-latest - timeout-minutes: 20 + # the WIP 51-step CPU train needs ~36 min after install + timeout-minutes: 60 strategy: fail-fast: false matrix: