diff --git a/.github/workflows/python-arabic.yml b/.github/workflows/python-arabic.yml index 779064b..daa3ace 100644 --- a/.github/workflows/python-arabic.yml +++ b/.github/workflows/python-arabic.yml @@ -1,9 +1,24 @@ -name: python +name: Python Arabic CI on: push: branches: [ main ] + paths: + - 'python/**' + - '.github/workflows/python-arabic.yml' pull_request: + paths: + - 'python/**' + - '.github/workflows/python-arabic.yml' + +# Cancel in-progress runs for the same workflow and branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + security-events: write jobs: lint: @@ -17,39 +32,64 @@ jobs: - run: ruff check python/ - run: ruff format --check python/ + dependency-review: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Dependency Review + uses: actions/dependency-review-action@v3 + infer: runs-on: ubuntu-latest + # cold pip cache: the eager torch wheel is ~5GB + timeout-minutes: 20 strategy: fail-fast: false matrix: - python-version: ["3.9"] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: python/arabic/requirements.txt - - - name: Install requirements - working-directory: ./python/arabic - run: | - pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . - - - name: Download PyTorch model - working-directory: ./python/arabic - run: | - curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt \ - -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt - - - name: Run diacriticization - working-directory: ./python/arabic - run: | - python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر' + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + python/arabic/requirements.txt + python/arabic/setup.py + + - name: Install requirements + working-directory: ./python/arabic + run: | + python -m pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . + + - name: Create model directory + working-directory: ./python/arabic + run: | + mkdir -p log_dir/CA_MSA.base.cbhg/models/ + + - name: Download PyTorch model + working-directory: ./python/arabic + run: | + curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt \ + -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt + + - name: Run diacriticization + working-directory: ./python/arabic + run: | + python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر' train: runs-on: ubuntu-latest + # the WIP 51-step CPU train needs ~35 min after dataset download + timeout-minutes: 60 env: # tensorboard's generated descriptors predate protobuf 4; the pure-python # implementation tolerates them without pinning protobuf @@ -57,34 +97,67 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9"] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: python/arabic/requirements.txt - - - name: Install requirements - working-directory: ./python/arabic - run: | - pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . - - - name: Prepare dataset - working-directory: ./python/arabic - run: | - mkdir -p data/CA_MSA - touch data/CA_MSA/{eval,train,test}.csv - cd data - curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip - unzip tashkeela.zip - for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done - for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done - for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done - - - name: Try training (WIP) - working-directory: ./python/arabic - run: | - python train.py --model "cbhg" --config config/test_cbhg.yml + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + python/arabic/requirements.txt + python/arabic/setup.py + + - name: Install requirements + working-directory: ./python/arabic + run: | + python -m pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . + + - name: Prepare dataset + working-directory: ./python/arabic + run: | + mkdir -p data/CA_MSA + touch data/CA_MSA/{eval,train,test}.csv + cd data + curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip + unzip tashkeela.zip + for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done + for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done + for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done + + - name: Try training (WIP) + working-directory: ./python/arabic + run: | + python train.py --model "cbhg" --config config/test_cbhg.yml + + security-scan: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.11' + + - name: Install safety + run: | + python -m pip install --upgrade pip + pip install safety + + - name: Check for vulnerabilities in Arabic requirements + working-directory: ./python/arabic + run: | + safety check -r requirements.txt --full-report || true + + - name: Check for vulnerabilities in Hebrew requirements + working-directory: ./python/hebrew + run: | + safety check -r requirements.txt --full-report || true diff --git a/.github/workflows/python-hebrew.yml b/.github/workflows/python-hebrew.yml new file mode 100644 index 0000000..1f52273 --- /dev/null +++ b/.github/workflows/python-hebrew.yml @@ -0,0 +1,110 @@ +name: Python Hebrew CI + +on: + push: + branches: [ main ] + paths: + - 'python/hebrew/**' + - '.github/workflows/python-hebrew.yml' + pull_request: + paths: + - 'python/hebrew/**' + - '.github/workflows/python-hebrew.yml' + +# Cancel in-progress runs for the same workflow and branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + security-events: write + +jobs: + dependency-review: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Dependency Review + uses: actions/dependency-review-action@v3 + + infer: + runs-on: ubuntu-latest + # cold pip cache: the eager torch wheel is ~5GB + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + python/hebrew/requirements.txt + python/hebrew/setup.py + + - name: Install requirements + working-directory: ./python/hebrew + run: | + python -m pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . + + - name: Create model directory + working-directory: ./python/hebrew + run: | + mkdir -p log_dir/base.cbhg/ + + - name: Run diacriticization + working-directory: ./python/hebrew + run: | + python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'שלום' + + train: + runs-on: ubuntu-latest + # the WIP 51-step CPU train needs ~36 min after install + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + python/hebrew/requirements.txt + python/hebrew/setup.py + + - name: Install requirements + working-directory: ./python/hebrew + run: | + python -m pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r requirements.txt -e . + + - name: Prepare test data + working-directory: ./python/hebrew + run: | + mkdir -p data/train data/eval + printf "שלום עולם\nשלום עולם\nשלום עולם\nשלום עולם\nשלום עולם\n" > data/train/test.txt + printf "שלום עולם\nשלום עולם\n" > data/eval/test.txt + + - name: Try training (WIP) + working-directory: ./python/hebrew + run: | + python train.py --model "cbhg" --config config/test_cbhg.yml diff --git a/README.adoc b/README.adoc index ca9da92..d80cf2d 100644 --- a/README.adoc +++ b/README.adoc @@ -4,22 +4,15 @@ image:https://github.com/interscript/rababa/actions/workflows/ruby.yml/badge.svg Middle-Eastern Language diacritization is useful for several practical business cases like text to speech or Romanization of texts or scripts. -As of now, this library supports Hebrew and Arabic. - == Purpose This repository contains everything to train a diacritization model in Python and run it in Python and Ruby. -== Try out Rababa - -Rababa can be run both in Python and Ruby. Go the directory corresponding to the -language you prefer to use. - -Please see the following README's, under the "`Try out Rababa`" section: +It supports the following languages: -* https://github.com/interscript/rababa/tree/main/python[Python] -* https://github.com/interscript/rababa/tree/main/lib[Ruby] +* Arabic +* Hebrew == Library @@ -75,12 +68,185 @@ In a similar fashion that a Rababa produces melody from a simple strings and pieces of wood, our library and diacritization gives a whole palette of colour and meanings to arabic scripts. + +== Try out Rababa + +Rababa can be run both in Python and Ruby. Go to the directory corresponding to the +language you prefer to use. + +Training of Rababa models are only available in Python. + + +=== Python (3.8+) + +==== Prerequisites + +The Python version of Rababa supports Python 3.8 and later. We recommend using a +version manager like https://asdf-vm.com[asdf] or +https://github.com/pyenv/pyenv[pyenv]. + +[IMPORTANT] +==== +*PyTorch Compatibility:* Rababa works with PyTorch 1.7.0 and later. Different +versions of PyTorch may handle model loading differently, especially with the +`weights_only` parameter introduced in PyTorch 2.6+. + +If you encounter model loading errors mentioning `weights_only`, you might be +using a newer PyTorch version that is incompatible with the saved models. The +setup steps below have been tested and should work across PyTorch versions. +==== + +With asdf: + +[source,bash] +---- +# Install asdf (if not already installed) +# See: https://asdf-vm.com/guide/getting-started.html + +# Install Python plugin +asdf plugin add python + +# Install desired Python version +asdf install python 3.9.13 + +# Set Python version for this project +asdf local python 3.9.13 + +# Verify your Python version +python --version +---- + +With pyenv: + +[source,bash] +---- +# Install pyenv (if not already installed) +brew install pyenv + +# Install desired Python version +pyenv install 3.9.13 + +# Set Python version for this project +pyenv local 3.9.13 + +# Verify your Python version +python --version +---- + +==== Arabic Diacritization + +. Install the required dependencies: ++ +[source,bash] +---- +cd python/arabic +pip install --upgrade pip +pip install torch numpy matplotlib pandas ruamel.yaml tensorboard diacritization-evaluation tqdm onnx onnxruntime pyyaml +---- + +. Download the pre-trained model: ++ +[source,bash] +---- +mkdir -p log_dir/CA_MSA.base.cbhg/models +curl -sSL https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt -o log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt +---- + +. Run diacritization: ++ +[source,bash] +---- +# Diacritize a single text +python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر' + +# Diacritize a text file +python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text_file path_to_input_file --diacritized_text_file path_to_output_file +---- + +.Example Output +[source] +---- +# Input: قطر +# Output: قَطُرُ + +# Input: مرحبا كيف حالك انا بخير +# Output: مَرْحَبًا كَيْفَ حَالُك انَّا بِخَيْرٍ +---- + +[NOTE] +==== +If you encounter model loading errors with newer versions of PyTorch (especially +2.6+), you may see messages related to the `weights_only` parameter. The code +has been updated to handle this compatibility issue, but if you still experience +problems, please refer to the Troubleshooting section at the end of this +document. +==== + +==== Hebrew Diacritization + +. Install the required dependencies: ++ +[source,bash] +---- +cd python/hebrew +pip install --upgrade pip +pip install torch numpy matplotlib pandas ruamel.yaml tensorboard diacritization-evaluation tqdm onnx onnxruntime pyyaml +---- + +. Download the pre-trained model and configuration: ++ +[source,bash] +---- +mkdir -p log_dir/base.cbhg +curl -sSL https://github.com/secryst/rababa-models/releases/download/hebrew.0.1/bchg_len90_dec0.9766_wor_0.8877.yml -o config/model_hebrew.yml +curl -sSL https://github.com/secryst/rababa-models/releases/download/hebrew.0.1/model_len90_dec0.9766_wor_0.8877.pt -o log_dir/base.cbhg/model.pt +---- + +. Run diacritization: ++ +[source,bash] +---- +# Diacritize a single text +python diacritize.py --model_kind "cbhg" --config config/model_hebrew.yml --text 'מה שלומך' + +# Diacritize a text file +python diacritize.py --model_kind "cbhg" --config config/model_hebrew.yml --text_file path_to_input_file --diacritized_text_file path_to_output_file +---- + +.Example Output +[source] +---- +# Input: מה שלומך +# Output: מָה שֶׁלוֹמְךַ + +# Input: אני אוהב לקרוא ספרים +# Output: אֲנִי אוֹהֵב לִקְרוֹא סְפָרִים +---- + +[NOTE] +==== +The Hebrew model has similar PyTorch compatibility considerations as the Arabic +model. The fixes applied ensure smooth operation across different PyTorch +versions. +==== + +For more detailed instructions, please see the language-specific READMEs: + +* https://github.com/interscript/rababa/tree/main/python/arabic[Arabic] +* https://github.com/interscript/rababa/tree/main/python/hebrew[Hebrew] +* https://github.com/interscript/rababa/tree/main/lib[Ruby] + + +// ==== Training + +// === Ruby + == Under development We are working on the following improvements: * Enhancing architecture and encoding -* Enhance datasets to improve models +* Enhancing datasets to improve models == License and copyright @@ -133,3 +299,53 @@ The Tashkeela dataset used for training is provided under GPL v2 license: * Processed dataset by Hamza Abbad: https://sourceforge.net/projects/tashkeela-processed/ * License: GPL v2 + +== Troubleshooting + +=== PyTorch Compatibility Issues + +If you encounter model loading errors related to the `weights_only` parameter in PyTorch, try one of the following solutions: + +1. *For Hebrew models:* ++ +If you see errors when loading the Hebrew model, verify that the fix for the `weights_only` parameter is applied in the Hebrew config_manager.py file. The load_model function should *not* include the `weights_only` parameter: ++ +[source,python] +---- +saved_model = ( + torch.load(model_path) + if torch.cuda.is_available() + else torch.load(model_path, map_location=torch.device("cpu")) +) +---- + +2. *For Arabic models:* ++ +If you see errors when loading the Arabic model, the `weights_only` parameter should be explicitly set to `False` in the config_manager.py file: ++ +[source,python] +---- +saved_model = torch.load(last_model_path, weights_only=False) if torch.cuda.is_available() else torch.load(last_model_path, map_location=torch.device('cpu'), weights_only=False) +---- + +3. *Using a specific PyTorch version:* ++ +If problems persist, installing a specific PyTorch version compatible with the saved model formats may help: ++ +[source,bash] +---- +pip install torch==1.13.1 +---- + +4. *Model conversion:* ++ +For persistent issues, you may need to load and re-save the models with your current PyTorch version: ++ +[source,python] +---- +# Load model with explicit weights_only=False setting +old_model = torch.load("model.pt", weights_only=False) + +# Save model in a compatible format +torch.save(old_model, "new_model.pt") +---- diff --git a/python/arabic/config_manager.py b/python/arabic/config_manager.py index 9d476eb..3c1eb62 100644 --- a/python/arabic/config_manager.py +++ b/python/arabic/config_manager.py @@ -173,9 +173,9 @@ def load_model(self, model_path: str = None): last_model_path = model_path saved_model = ( - torch.load(last_model_path) + torch.load(last_model_path, weights_only=False) if torch.cuda.is_available() - else torch.load(last_model_path, map_location=torch.device("cpu")) + else torch.load(last_model_path, map_location=torch.device("cpu"), weights_only=False) ) model.load_state_dict(saved_model["model_state_dict"]) @@ -240,7 +240,7 @@ def get_text_encoder(self): def get_loss_type(self): try: loss_type = LossType[self.config["loss_type"]] - except KeyError: + except Exception: raise Exception(f"The loss type is not correct {self.config['loss_type']}") return loss_type diff --git a/python/arabic/convert_torch_model_to_onnx.py b/python/arabic/convert_torch_model_to_onnx.py index 52f112a..9732728 100644 --- a/python/arabic/convert_torch_model_to_onnx.py +++ b/python/arabic/convert_torch_model_to_onnx.py @@ -1,160 +1,90 @@ +import multiprocessing +import random + import numpy as np +import onnx +import onnxruntime import torch import yaml from diacritizer import Diacritizer -""" + +def main(): + """ Key Params: max_len: is the max length for the arabic strings to be diacritized batch size: has to do with the model training and usage -""" - -d_params = yaml.load(open("config/convert_torch_onnx.yml")) - -max_len = d_params["max_len"] # 600 for the original length -batch_size = d_params["batch_size"] -config_str = d_params["config_str"] -model_kind_str = d_params["model_kind_str"] -onnx_model_filename = d_params["onnx_model_filename"] - - -""" - example and mock data: - we found that populating all the data, removing the zeros gives better results. -""" - -src = torch.Tensor([[1 for i in range(max_len)] for i in range(batch_size)]).long() -lengths = torch.Tensor([max_len for i in range(batch_size)]).long() - - -""" - Instantiate Diacritization model -""" - -dia = Diacritizer(config_str, model_kind_str, True) - -# set model to inference mode -dia.model.to(dia.device) -dia.model.eval() -# run model -torch_out = dia.model(src, lengths) - - -""" - Load ONNX libs and export models into onnx -""" - -import onnx -import onnxruntime -import torch - -# export model -torch.onnx.export( - dia.model, - (src, lengths), - onnx_model_filename, - verbose=False, - opset_version=11, - input_names=["src", "lengths"], - output_names=["output"], - dynamic_axes={ - "src": [1], # [0,1,2], #[0,1,2], - #'input_2':{0:'batch'}, - "output": [1], - }, -) - -print("Model printed in rel. path:", onnx_model_filename) - - -""" - Load ONNX versions of model -""" - -# load model -onnx_model = onnx.load(onnx_model_filename) -# check model -onnx.checker.check_model(onnx_model) -# inference session -ort_session = onnxruntime.InferenceSession(onnx_model_filename) - -# onnx inputs and outputs names -# ort_session.get_inputs(), ort_session.get_outputs() - - -""" - Run ONNX model on sample data -""" - -# prepare onnx input -ort_inputs = { - ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), - ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), -} - -# run onnx model -ort_outs = ort_session.run(None, ort_inputs) - -print("outs:: ", ort_outs) -print("src:: ", src.detach().numpy().astype(np.int64)) -print("lengths: ", lengths.detach().numpy().astype(np.int64)) - - -for i in range(batch_size): - np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), - ort_outs[0][i], - rtol=1e-03, - atol=1e-03, - ) + """ -print( - "\n!!!Exported model has been tested with ONNXRuntime, result looks good within given tolerance!!!" -) + d_params = yaml.safe_load(open("config/convert_torch_onnx.yml")) + max_len = d_params["max_len"] # 600 for the original length + batch_size = d_params["batch_size"] + config_str = d_params["config_str"] + model_kind_str = d_params["model_kind_str"] + onnx_model_filename = d_params["onnx_model_filename"] -vec = [[41, 12, 40] for i in range(batch_size)] -src = torch.Tensor(vec).long() + """ + example and mock data: + we found that populating all the data, removing the zeros gives better results. + """ -lengths = torch.Tensor([3 for i in range(batch_size)]).long() - -ort_inputs = { - ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), - ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), -} + src = torch.Tensor([[1 for i in range(max_len)] for i in range(batch_size)]).long() + lengths = torch.Tensor([max_len for i in range(batch_size)]).long() + """ + Instantiate Diacritization model + """ -ort_outs = ort_session.run(None, ort_inputs) + dia = Diacritizer(config_str, model_kind_str, True) -torch_out = dia.model(src, lengths) + # set model to inference mode + dia.model.to(dia.device) + dia.model.eval() + # run model + torch_out = dia.model(src, lengths) -for i in range(batch_size): - np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), - ort_outs[0][i], - rtol=1e-03, - atol=1e-03, + """ + Load ONNX libs and export models into onnx + """ + + # export model + torch.onnx.export( + dia.model, + (src, lengths), + onnx_model_filename, + verbose=False, + opset_version=11, + input_names=["src", "lengths"], + output_names=["output"], + dynamic_axes={ + "src": [1], # [0,1,2], #[0,1,2], + #'input_2':{0:'batch'}, + "output": [1], + }, ) + print("Model printed in rel. path:", onnx_model_filename) -""" - Test ONNX model on randomized data -""" + """ + Load ONNX versions of model + """ -import random + # load model + onnx_model = onnx.load(onnx_model_filename) + # check model + onnx.checker.check_model(onnx_model) + # inference session + ort_session = onnxruntime.InferenceSession(onnx_model_filename) -test_id = 0 + # onnx inputs and outputs names + # ort_session.get_inputs(), ort_session.get_outputs() -print("***** Test MAX size :: Random Boolean vectors: *****") - -for test_run in range(3): - vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] - src = torch.Tensor(vec).long() - lengths = torch.Tensor([max_len for i in range(batch_size)]).long() - - torch_out = dia.model(src, lengths) + """ + Run ONNX model on sample data + """ # prepare onnx input ort_inputs = { @@ -165,6 +95,10 @@ # run onnx model ort_outs = ort_session.run(None, ort_inputs) + print("outs:: ", ort_outs) + print("src:: ", src.detach().numpy().astype(np.int64)) + print("lengths: ", lengths.detach().numpy().astype(np.int64)) + for i in range(batch_size): np.testing.assert_allclose( torch_out["diacritics"][i].detach().numpy(), @@ -173,90 +107,151 @@ atol=1e-03, ) - print("test :: ", test_run) - print("Result looks good within given tolerance!!!") - - -print("***** Test MAX size :: Random float, vectors within 0:16 *****") + print( + "\n!!!Exported model has been tested with ONNXRuntime, result looks good within given tolerance!!!" + ) -for test_run in range(3): - vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] + vec = [[41, 12, 40] for i in range(batch_size)] src = torch.Tensor(vec).long() - torch_out = dia.model(src, lengths) - # my_list = torch_out['diacritics'].detach().numpy().tolist() - # prepare onnx input + lengths = torch.Tensor([3 for i in range(batch_size)]).long() + ort_inputs = { ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), } - # run onnx model ort_outs = ort_session.run(None, ort_inputs) + torch_out = dia.model(src, lengths) + for i in range(batch_size): np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), ort_outs[0][i], rtol=1, atol=1 + torch_out["diacritics"][i].detach().numpy(), + ort_outs[0][i], + rtol=1e-03, + atol=1e-03, ) - print("test :: ", test_run) - print("Result looks good within given tolerance!!!") + """ + Test ONNX model on randomized data + """ + # test_id = 0 -print("***** Test Dynamical sizes :: Random Boolean vectors: *****") + print("***** Test MAX size :: Random Boolean vectors: *****") -for l in [2, 10, 40, 100, 150]: - print("length:: ", l) + for test_run in range(3): + vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] + src = torch.Tensor(vec).long() + lengths = torch.Tensor([max_len for i in range(batch_size)]).long() - vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) - src = torch.Tensor(vec).long() - lengths = torch.Tensor([l for i in range(batch_size)]).long() + torch_out = dia.model(src, lengths) - torch_out = dia.model(src, lengths) + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), + ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), + } - # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), - ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), - } + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + for i in range(batch_size): + np.testing.assert_allclose( + torch_out["diacritics"][i].detach().numpy(), + ort_outs[0][i], + rtol=1e-03, + atol=1e-03, + ) - for i in range(batch_size): - np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), - ort_outs[0][i], - rtol=1e-03, - atol=1e-03, - ) + print("test :: ", test_run) + print("Result looks good within given tolerance!!!") - print("test :: ", l) - print("Result looks good within given tolerance!!!") + print("***** Test MAX size :: Random float, vectors within 0:16 *****") + for test_run in range(3): + vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] + src = torch.Tensor(vec).long() + torch_out = dia.model(src, lengths) -print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") + # my_list = torch_out['diacritics'].detach().numpy().tolist() + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), + ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), + } -for l in [2, 10, 40, 100, 150]: - vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] - src = torch.Tensor(vec).long() - lengths = torch.Tensor([l for i in range(batch_size)]).long() + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) - torch_out = dia.model(src, lengths) + for i in range(batch_size): + np.testing.assert_allclose( + torch_out["diacritics"][i].detach().numpy(), ort_outs[0][i], rtol=1, atol=1 + ) - # prepare onnx input - ort_inputs = { - ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), - ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), - } + print("test :: ", test_run) + print("Result looks good within given tolerance!!!") - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + print("***** Test Dynamical sizes :: Random Boolean vectors: *****") + + for l in [2, 10, 40, 100, 150]: + print("length:: ", l) + + vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) + src = torch.Tensor(vec).long() + lengths = torch.Tensor([l for i in range(batch_size)]).long() + + torch_out = dia.model(src, lengths) + + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), + ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), + } + + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + + for i in range(batch_size): + np.testing.assert_allclose( + torch_out["diacritics"][i].detach().numpy(), + ort_outs[0][i], + rtol=1e-03, + atol=1e-03, + ) + + print("test :: ", l) + print("Result looks good within given tolerance!!!") + + print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") + + for l in [2, 10, 40, 100, 150]: + vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] + src = torch.Tensor(vec).long() + lengths = torch.Tensor([l for i in range(batch_size)]).long() + + torch_out = dia.model(src, lengths) + + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64), + ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64), + } + + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + + for i in range(batch_size): + np.testing.assert_allclose( + torch_out["diacritics"][i].detach().numpy(), ort_outs[0][i], rtol=1, atol=1 + ) + + print("test :: ", l) + print("Result looks good within given tolerance!!!") - for i in range(batch_size): - np.testing.assert_allclose( - torch_out["diacritics"][i].detach().numpy(), ort_outs[0][i], rtol=1, atol=1 - ) - print("test :: ", l) - print("Result looks good within given tolerance!!!") +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/arabic/diacritize.py b/python/arabic/diacritize.py index fb518c4..c5a6172 100644 --- a/python/arabic/diacritize.py +++ b/python/arabic/diacritize.py @@ -1,4 +1,5 @@ import argparse +import multiprocessing import random import numpy as np @@ -23,22 +24,29 @@ def diacritization_parser(): return parser -parser = diacritization_parser() -args = parser.parse_args() +def main(): + parser = diacritization_parser() + args = parser.parse_args() -if args.text is None and args.text_file is None: - raise ValueError("text or text_file params required!") + if args.text is None and args.text_file is None: + raise ValueError("text or text_file params required!") -if args.model_kind == "cbhg": - diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") -elif args.model_kind == "baseline": - diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") -else: - raise ValueError("The model kind is not supported") + if args.model_kind == "cbhg": + diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") + elif args.model_kind == "baseline": + diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") + else: + raise ValueError("The model kind is not supported") -if args.text_file is None: - txt = diacritizer.diacritize_text(args.text) - print(txt) -else: - for txt in diacritizer.diacritize_file(args.text_file): + if args.text_file is None: + txt = diacritizer.diacritize_text(args.text) print(txt) + else: + for txt in diacritizer.diacritize_file(args.text_file): + print(txt) + + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/arabic/requirements.txt b/python/arabic/requirements.txt index c7f0f70..468be4a 100644 --- a/python/arabic/requirements.txt +++ b/python/arabic/requirements.txt @@ -1,11 +1,11 @@ -torch==1.9.0 -numpy==1.19.5 -matplotlib==3.3.3 -pandas==1.1.5 -ruamel.yaml==0.16.12 -tensorboard==2.4.0 -diacritization-evaluation==0.5 -tqdm==4.56.0 -onnx==1.9.0 -onnxruntime==1.8.1 -pyyaml==5.4.1 +torch>=1.9.0,<3.0.0 +numpy>=1.20.0,<2.0.0 +matplotlib>=3.3.3 +pandas>=1.3.0 +ruamel.yaml>=0.16.12 +tensorboard>=2.4.0 +diacritization-evaluation>=0.5 +tqdm>=4.56.0 +onnx>=1.9.0 +onnxruntime>=1.8.1 +pyyaml>=5.4.1 diff --git a/python/arabic/setup.py b/python/arabic/setup.py index 4ca3d28..54eb6f0 100644 --- a/python/arabic/setup.py +++ b/python/arabic/setup.py @@ -2,6 +2,7 @@ from os import environ import setuptools +from setuptools import find_packages with open("README.adoc", encoding="utf-8") as fh: LONG_DESCRIPTION = fh.read() @@ -15,38 +16,39 @@ PKG_VERSION = TAG_VERSION.group(1) setuptools.setup( - name="rababa", + name="rababa-arabic", version=PKG_VERSION, author="Ribose", author_email="open.source@ribose.com", license="MIT", description="Rababa for Arabic diacriticization", - packages=setuptools.find_namespace_packages(include=["util*", "modules*"]), + packages=find_packages( + include=[ + "*", + "models.*", + "modules.*", + "util.*", + ] + ), url="https://www.interscript.org", - python_requires=">=3.6, <4", + python_requires=">=3.8, <4", project_urls={ "Documentation": "https://github.com/interscript/rababa", "Source": "https://github.com/interscript/rababa", "Tracker": "https://github.com/interscript/rababa/issues", }, install_requires=[ - "torch>=1.9.0", - "numpy", - "matplotlib", - "pandas", - "ruamel.yaml", - "tensorboard", - "diacritization-evaluation", - "tqdm", - "onnx", - "onnxruntime", - "pyyaml", + "torch>=1.9.0,<3.0.0", + "numpy>=1.20.0,<2.0.0", + "matplotlib>=3.3.3", + "pandas>=1.3.0", + "ruamel.yaml>=0.16.12", + "tensorboard>=2.4.0", + "diacritization-evaluation>=0.5", + "tqdm>=4.56.0", + "onnx>=1.9.0", + "onnxruntime>=1.8.1", + "pyyaml>=5.4.1", ], - # extras_require={'plotting': ['matplotlib>=2.2.0', 'jupyter']}, setup_requires=["pytest-runner"], - tests_require=["pytest"], - # entry_points={ - # 'console_scripts': ['my-command=exampleproject.example:main'] - # }, - # package_data={'exampleproject': ['data/schema.json']} ) diff --git a/python/arabic/test.py b/python/arabic/test.py index c5d4bde..d2412df 100644 --- a/python/arabic/test.py +++ b/python/arabic/test.py @@ -1,4 +1,5 @@ import argparse +import multiprocessing import random import numpy as np @@ -22,8 +23,15 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() +def main(): + parser = train_parser() + args = parser.parse_args() -tester = DiacritizationTester(args.config, args.model_kind) -tester.run() + tester = DiacritizationTester(args.config, args.model_kind) + tester.run() + + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/arabic/train.py b/python/arabic/train.py index dba06c6..134bf5a 100644 --- a/python/arabic/train.py +++ b/python/arabic/train.py @@ -1,4 +1,5 @@ import argparse +import multiprocessing import random import numpy as np @@ -27,13 +28,19 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() +def main(): + parser = train_parser() + args = parser.parse_args() + if args.model_kind in ["baseline", "cbhg"]: + trainer = CBHGTrainer(args.config, args.model_kind) + else: + raise ValueError("The model kind is not supported") -if args.model_kind in ["baseline", "cbhg"]: - trainer = CBHGTrainer(args.config, args.model_kind) -else: - raise ValueError("The model kind is not supported") + trainer.run() -trainer.run() + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/config/cbhg.yml b/python/hebrew/config/cbhg.yml index 454c955..033aa99 100644 --- a/python/hebrew/config/cbhg.yml +++ b/python/hebrew/config/cbhg.yml @@ -55,4 +55,8 @@ n_steps_avg_losses: [100, 500, 1_000, 5_000] # command line display of average l error_rates_n_batches: 10000 # if calculating error rate is slow, then you can specify the number of batches to be calculated test_model_path: null # load the last saved model + +# Path to model weights file (pytorch model) +# This line is added to support testing without an existing model +model_path: null train_resume_model_path: null # load last saved model diff --git a/python/hebrew/config/test_cbhg.yml b/python/hebrew/config/test_cbhg.yml index cfdaa42..7d1b0d4 100644 --- a/python/hebrew/config/test_cbhg.yml +++ b/python/hebrew/config/test_cbhg.yml @@ -1,20 +1,22 @@ session_name: base data_directory: "data" -data_type: "CA_MSA" +data_type: "test" log_directory: "log_dir" load_training_data: true load_test_data: false load_validation_data: true -n_training_examples: null # null load all training examples, good for fast loading +n_training_examples: 5 # Using a small number for testing n_test_examples: null # null load all test examples n_validation_examples: null # null load all validation examples -test_file_name: "test.csv" -is_data_preprocessed: false # The data file is organized as (original text | text | diacritics) -data_separator: '|' # Required if the data already processed -diacritics_separator: '*' # Required if the data already processed -text_encoder: ArabicEncoderWithStartSymbol -text_cleaner: valid_arabic_cleaners # a white list that uses only Arabic letters, punctuations, and a space +train_file_name: "test.txt" +eval_file_name: "test.txt" +test_file_name: "test.txt" +is_data_preprocessed: false +data_separator: '|' +diacritics_separator: '*' +text_encoder: HebrewEncoder # Use Hebrew encoder +text_cleaner: basic_cleaners # Adjusted for Hebrew max_len: 600 # sentences larger than this size will not be used reconcile: true @@ -36,16 +38,23 @@ post_cbhg_use_batch_norm: true use_mixed_precision: false optimizer_type: Adam -device: cuda +device: cpu # Using CPU for testing + +# GEOMETRY +len_input_symbols: 90 +len_niqqud_symbols: 16 +len_dagesh_symbols: 3 +len_sin_symbols: 4 # LOGGING -evaluate_frequency: 5000 -evaluate_with_error_rates_frequency: 5000 -n_predicted_text_tensorboard: 10 # To be written to the tensorboard -model_save_frequency: 5000 +evaluate_frequency: 10 +evaluate_with_error_rates_frequency: 10 +n_predicted_text_tensorboard: 5 # To be written to the tensorboard +model_save_frequency: 10 train_plotting_frequency: 50000000 # No plotting for this model -n_steps_avg_losses: [100, 500, 1_000, 5_000] # command line display of average loss values for the last n steps -error_rates_n_batches: 10000 # if calculating error rate is slow, then you can specify the number of batches to be calculated +n_steps_avg_losses: [10, 20, 30, 40] # Reduced for testing +error_rates_n_batches: 5 # Reduced for testing test_model_path: null # load the last saved model train_resume_model_path: null # load last saved model +model_path: null diff --git a/python/hebrew/convert_torch_model_to_onnx.py b/python/hebrew/convert_torch_model_to_onnx.py index 4c5683f..bf52410 100644 --- a/python/hebrew/convert_torch_model_to_onnx.py +++ b/python/hebrew/convert_torch_model_to_onnx.py @@ -1,3 +1,4 @@ +import multiprocessing import random import numpy as np @@ -7,129 +8,84 @@ import yaml from diacritizer import Diacritizer -""" + +def main(): + """ Key Params: max_len: is the max length for the arabic strings to be diacritized batch size: has to do with the model training and usage -""" - -d_params = yaml.load(open("config/convert_torch_onnx.yml")) -max_len = d_params["max_len"] # 600 for the original length -batch_size = d_params["batch_size"] -config_str = d_params["config_str"] -model_kind_str = d_params["model_kind_str"] -onnx_model_filename = d_params["onnx_model_filename"] -device = d_params["device"] - - -""" - example and mock data: - we found that populating all the data, removing the zeros gives better results. -""" - -normalized = torch.Tensor([[1 for i in range(max_len)] for i in range(batch_size)]).long() - - -""" - Instantiate Diacritization model -""" - -dia = Diacritizer(config_str, model_kind_str, True) - -# set model to inference mode -dia.model.to(device) -dia.model.eval() -normalized = normalized.to(device) - -# run model -niqqud, dagesh, sin = dia.model(normalized) -torch_outs = dia.model(normalized) # niqqud, dagesh, sin - - -""" - Load ONNX libs and export models into onnx -""" - -onnx_model_filename = "../models-data/diacritization_model.onnx" - -# export model -torch.onnx.export( - dia.model, - normalized, - onnx_model_filename, - verbose=False, - opset_version=11, - input_names=["normalized"], - output_names=["niqqud", "dagesh", "sin"], - dynamic_axes={"normalized": [1], "output": [1], "dagesh": [1], "sin": [1]}, -) - -print("Model printed in rel. path:", onnx_model_filename) - - -""" - Load ONNX versions of model -""" - -# load model -onnx_model = onnx.load(onnx_model_filename) -# check model -onnx.checker.check_model(onnx_model) -# inference session -ort_session = onnxruntime.InferenceSession(onnx_model_filename) - -# get onnx inputs and outputs names -# ort_session.get_inputs(), ort_session.get_outputs() + """ + + d_params = yaml.safe_load(open("config/convert_torch_onnx.yml")) + max_len = d_params["max_len"] # 600 for the original length + batch_size = d_params["batch_size"] + config_str = d_params["config_str"] + model_kind_str = d_params["model_kind_str"] + onnx_model_filename = d_params["onnx_model_filename"] + device = d_params["device"] + + """ + example and mock data: + we found that populating all the data, removing the zeros gives better results. + """ + + normalized = torch.Tensor([[1 for i in range(max_len)] for i in range(batch_size)]).long() + + """ + Instantiate Diacritization model + """ + + dia = Diacritizer(config_str, model_kind_str, True) + + # set model to inference mode + dia.model.to(device) + dia.model.eval() + normalized = normalized.to(device) + + # run model + niqqud, dagesh, sin = dia.model(normalized) + torch_outs = dia.model(normalized) # niqqud, dagesh, sin + + """ + Load ONNX libs and export models into onnx + """ + + onnx_model_filename = "../models-data/diacritization_model.onnx" + + # export model + torch.onnx.export( + dia.model, + normalized, + onnx_model_filename, + verbose=False, + opset_version=11, + input_names=["normalized"], + output_names=["niqqud", "dagesh", "sin"], + dynamic_axes={"normalized": [1], "output": [1], "dagesh": [1], "sin": [1]}, + ) + + print("Model printed in rel. path:", onnx_model_filename) + + """ + Load ONNX versions of model + """ + + # load model + onnx_model = onnx.load(onnx_model_filename) + # check model + onnx.checker.check_model(onnx_model) + # inference session + ort_session = onnxruntime.InferenceSession(onnx_model_filename) + + # get onnx inputs and outputs names + # ort_session.get_inputs(), ort_session.get_outputs() + + """ + Run ONNX model on sample data + """ - -""" - Run ONNX model on sample data -""" - -# prepare onnx input -ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} - -# run onnx model -ort_outs = ort_session.run(None, ort_inputs) - - -for i in range(batch_size): - for dim in range(3): # niqqud, dagesh, sin - np.testing.assert_allclose( - torch_outs[dim][i].detach().numpy(), - ort_outs[dim][i], - rtol=1e-02, - atol=1e-02, - ) - -print( - "\n!!!Exported model has been tested with ONNXRuntime, \ - result looks good within given tolerance!!!" -) - - -vec = [[41, 12, 40] for i in range(batch_size)] -normalized = torch.Tensor(vec).long() - -ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} - - -""" - Test ONNX model on randomized data -""" - -test_id = 0 - -print("***** Test MAX size :: Random Boolean vectors: *****") -print(max_len) - -for test_run in range(3): - vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] - normalized = torch.Tensor(vec).long() - - torch_outs = dia.model(normalized) # prepare onnx input ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} @@ -137,90 +93,140 @@ ort_outs = ort_session.run(None, ort_inputs) for i in range(batch_size): - for dim in range(3): + for dim in range(3): # niqqud, dagesh, sin np.testing.assert_allclose( torch_outs[dim][i].detach().numpy(), ort_outs[dim][i], - rtol=1e-01, - atol=1e-01, + rtol=1e-02, + atol=1e-02, ) - print("test :: ", test_run) - print("Result looks good within given tolerance!!!") - + print( + "\n!!!Exported model has been tested with ONNXRuntime, \ + result looks good within given tolerance!!!" + ) -print("***** Test MAX size :: Random float, vectors within 0:16 *****") -print(max_len) - -for test_run in range(3): - vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] + vec = [[41, 12, 40] for i in range(batch_size)] normalized = torch.Tensor(vec).long() - torch_out = dia.model(normalized) - # prepare onnx input ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + """ + Test ONNX model on randomized data + """ - for i in range(batch_size): - for dim in range(3): - np.testing.assert_allclose( - torch_out[dim][i].detach().numpy(), ort_outs[dim][i], rtol=1, atol=1 - ) + # test_id = 0 - print("test :: ", test_run) - print("Result looks good within given tolerance!!!") + print("***** Test MAX size :: Random Boolean vectors: *****") + print(max_len) + for test_run in range(3): + vec = [[random.randint(0, 1) for i in range(max_len)] for i in range(batch_size)] + normalized = torch.Tensor(vec).long() -print("***** Test Dynamical sizes :: Random Boolean vectors: *****") + torch_outs = dia.model(normalized) + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } -for l in [2, 10, 40, 100, 150]: - print("length:: ", l) + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) - vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) - normalized = torch.Tensor(vec).long() + for i in range(batch_size): + for dim in range(3): + np.testing.assert_allclose( + torch_outs[dim][i].detach().numpy(), + ort_outs[dim][i], + rtol=1e-01, + atol=1e-01, + ) - torch_out = dia.model(normalized) + print("test :: ", test_run) + print("Result looks good within given tolerance!!!") - # prepare onnx input - ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} + print("***** Test MAX size :: Random float, vectors within 0:16 *****") + print(max_len) - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + for test_run in range(3): + vec = [[random.randint(0, 17) for i in range(max_len)] for i in range(batch_size)] + normalized = torch.Tensor(vec).long() + torch_out = dia.model(normalized) - for i in range(batch_size): - for dim in range(3): - np.testing.assert_allclose( - torch_out[dim][i].detach().numpy(), - ort_outs[dim][i], - rtol=1e-02, - atol=1e-02, - ) + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } - print("test :: ", l) - print("Result looks good within given tolerance!!!") + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + for i in range(batch_size): + for dim in range(3): + np.testing.assert_allclose( + torch_out[dim][i].detach().numpy(), ort_outs[dim][i], rtol=1, atol=1 + ) -print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") + print("test :: ", test_run) + print("Result looks good within given tolerance!!!") -for l in [2, 10, 40, 100, 150]: - vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] - normalized = torch.Tensor(vec).long() + print("***** Test Dynamical sizes :: Random Boolean vectors: *****") - torch_out = dia.model(normalized) + for l in [2, 10, 40, 100, 150]: + print("length:: ", l) - # prepare onnx input - ort_inputs = {ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64)} + vec = [[1 for i in range(l)] for i in range(batch_size)] # random.randint(0,1) + normalized = torch.Tensor(vec).long() - # run onnx model - ort_outs = ort_session.run(None, ort_inputs) + torch_out = dia.model(normalized) + + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } + + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + + for i in range(batch_size): + for dim in range(3): + np.testing.assert_allclose( + torch_out[dim][i].detach().numpy(), + ort_outs[dim][i], + rtol=1e-02, + atol=1e-02, + ) + + print("test :: ", l) + print("Result looks good within given tolerance!!!") + + print("***** Test Dynamical sizes :: Random float, vectors within 0:16 *****") + + for l in [2, 10, 40, 100, 150]: + vec = [[random.randint(0, 17) for i in range(l)] for i in range(batch_size)] + normalized = torch.Tensor(vec).long() + + torch_out = dia.model(normalized) + + # prepare onnx input + ort_inputs = { + ort_session.get_inputs()[0].name: normalized.detach().numpy().astype(np.int64) + } + + # run onnx model + ort_outs = ort_session.run(None, ort_inputs) + + for i in range(batch_size): + for dim in range(3): + np.testing.assert_allclose( + torch_out[dim][i].detach().numpy(), ort_outs[dim][i], rtol=1, atol=1 + ) + + print("test :: ", l) + print("Result looks good within given tolerance!!!") - for i in range(batch_size): - for dim in range(3): - np.testing.assert_allclose( - torch_out[dim][i].detach().numpy(), ort_outs[dim][i], rtol=1, atol=1 - ) - print("test :: ", l) - print("Result looks good within given tolerance!!!") +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/data/eval/test.txt b/python/hebrew/data/eval/test.txt new file mode 100644 index 0000000..63861de --- /dev/null +++ b/python/hebrew/data/eval/test.txt @@ -0,0 +1,6 @@ +שלום עולם +זה מבחן +בדיקה ניקוד +מערכת ניקוד עברית +ירושלים +תל אביב diff --git a/python/hebrew/diacritize.py b/python/hebrew/diacritize.py index c628850..5dafc31 100644 --- a/python/hebrew/diacritize.py +++ b/python/hebrew/diacritize.py @@ -1,4 +1,5 @@ import argparse +import multiprocessing import random import numpy as np @@ -29,23 +30,29 @@ def diacritization_parser(): return parser -parser = diacritization_parser() -args = parser.parse_args() +def main(): + parser = diacritization_parser() + args = parser.parse_args() + if args.text is None and args.text_file is None: + raise ValueError("text or text_file/diacritized_text_file params required!") -if args.text is None and args.text_file is None: - raise ValueError("text or text_file/diacritized_text_file params required!") + if args.model_kind == "cbhg": + diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") + elif args.model_kind == "baseline": + diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") + else: + raise ValueError("The model kind is not supported") -if args.model_kind == "cbhg": - diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") -elif args.model_kind == "baseline": - diacritizer = Diacritizer(args.config, args.model_kind, "log_dir") -else: - raise ValueError("The model kind is not supported") + if args.text_file is None: + txt = diacritizer.diacritize_text(args.text) + print(txt) + else: + diacritizer.diacritize_file(args.text_file, args.diacritized_text_file) + print("done!!! written in: ", args.diacritized_text_file) -if args.text_file is None: - txt = diacritizer.diacritize_text(args.text) - print(txt) -else: - diacritizer.diacritize_file(args.text_file, args.diacritized_text_file) - print("done!!! written in: ", args.diacritized_text_file) + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/requirements.txt b/python/hebrew/requirements.txt index 4b932e4..fa9a890 100644 --- a/python/hebrew/requirements.txt +++ b/python/hebrew/requirements.txt @@ -1,12 +1,12 @@ -torch==1.9.0 -numpy==1.19.5 -matplotlib==3.3.3 -pandas==1.1.5 -ruamel.yaml==0.16.12 -tensorboard==2.4.0 -diacritization-evaluation==0.5 -tqdm==4.56.0 -onnx==1.9.0 -onnxruntime==1.8.1 -pyyaml==5.4.1 -wandb==0.12.4 \ No newline at end of file +torch>=1.9.0,<3.0.0 +numpy>=1.20.0,<2.0.0 +matplotlib>=3.3.3 +pandas>=1.3.0 +ruamel.yaml>=0.16.12 +tensorboard>=2.4.0 +diacritization-evaluation>=0.5 +tqdm>=4.56.0 +onnx>=1.9.0 +onnxruntime>=1.8.1 +pyyaml>=5.4.1 +wandb>=0.12.4 diff --git a/python/hebrew/run_experiments_wandb.py b/python/hebrew/run_experiments_wandb.py index fd52824..7371c86 100644 --- a/python/hebrew/run_experiments_wandb.py +++ b/python/hebrew/run_experiments_wandb.py @@ -1,4 +1,5 @@ import argparse +import multiprocessing import random import numpy as np @@ -31,9 +32,6 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() - # Define Experiments using Wandb sweep_config = { # search method @@ -64,6 +62,8 @@ def train_parser(): # train code, with the search preprocessing logic def train(): + parser = train_parser() + args = parser.parse_args() with open("config/train.yml", "rb") as model_yaml: config = yaml.load(model_yaml) @@ -88,16 +88,19 @@ def train(): trainer.run(config_wandb) -################################## -# MAIN # -################################## +def main(): + # Run name + run_name = "hyperparams search" + + # Init wandb and search + wandb.login() + sweep_id = wandb.sweep(sweep_config, project=run_name) -# Run name -run_name = "hyperparams search" + # Run search + wandb.agent(sweep_id, train) -# Init wandb and search -wandb.login() -sweep_id = wandb.sweep(sweep_config, project=run_name) -# Run search -wandb.agent(sweep_id, train) +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/setup.py b/python/hebrew/setup.py index 4ca3d28..cfa7cae 100644 --- a/python/hebrew/setup.py +++ b/python/hebrew/setup.py @@ -2,6 +2,7 @@ from os import environ import setuptools +from setuptools import find_packages with open("README.adoc", encoding="utf-8") as fh: LONG_DESCRIPTION = fh.read() @@ -15,38 +16,40 @@ PKG_VERSION = TAG_VERSION.group(1) setuptools.setup( - name="rababa", + name="rababa-hebrew", version=PKG_VERSION, author="Ribose", author_email="open.source@ribose.com", license="MIT", - description="Rababa for Arabic diacriticization", - packages=setuptools.find_namespace_packages(include=["util*", "modules*"]), + description="Rababa for Hebrew diacriticization", + packages=find_packages( + include=[ + "*", + "models.*", + "modules.*", + "util.*", + ] + ), url="https://www.interscript.org", - python_requires=">=3.6, <4", + python_requires=">=3.8, <4", project_urls={ "Documentation": "https://github.com/interscript/rababa", "Source": "https://github.com/interscript/rababa", "Tracker": "https://github.com/interscript/rababa/issues", }, install_requires=[ - "torch>=1.9.0", - "numpy", - "matplotlib", - "pandas", - "ruamel.yaml", - "tensorboard", - "diacritization-evaluation", - "tqdm", - "onnx", - "onnxruntime", - "pyyaml", + "torch>=1.9.0,<3.0.0", + "numpy>=1.20.0,<2.0.0", + "matplotlib>=3.3.3", + "pandas>=1.3.0", + "ruamel.yaml>=0.16.12", + "tensorboard>=2.4.0", + "diacritization-evaluation>=0.5", + "tqdm>=4.56.0", + "onnx>=1.9.0", + "onnxruntime>=1.8.1", + "pyyaml>=5.4.1", + "wandb>=0.12.4", ], - # extras_require={'plotting': ['matplotlib>=2.2.0', 'jupyter']}, setup_requires=["pytest-runner"], - tests_require=["pytest"], - # entry_points={ - # 'console_scripts': ['my-command=exampleproject.example:main'] - # }, - # package_data={'exampleproject': ['data/schema.json']} ) diff --git a/python/hebrew/test.py b/python/hebrew/test.py index c5d4bde..d2412df 100644 --- a/python/hebrew/test.py +++ b/python/hebrew/test.py @@ -1,4 +1,5 @@ import argparse +import multiprocessing import random import numpy as np @@ -22,8 +23,15 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() +def main(): + parser = train_parser() + args = parser.parse_args() -tester = DiacritizationTester(args.config, args.model_kind) -tester.run() + tester = DiacritizationTester(args.config, args.model_kind) + tester.run() + + +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/train.py b/python/hebrew/train.py index dba06c6..bbef02d 100644 --- a/python/hebrew/train.py +++ b/python/hebrew/train.py @@ -1,8 +1,19 @@ import argparse +import multiprocessing import random import numpy as np import torch + +# Make wandb optional +try: + import wandb # noqa: F401 — availability probe + + WANDB_AVAILABLE = True +except ImportError: + WANDB_AVAILABLE = False + print("Warning: wandb not available, training will proceed without logging to wandb") + from trainer import CBHGTrainer SEED = 1234 @@ -27,13 +38,19 @@ def train_parser(): return parser -parser = train_parser() -args = parser.parse_args() +def main(): + parser = train_parser() + args = parser.parse_args() + + if args.model_kind in ["baseline", "cbhg"]: + trainer = CBHGTrainer(args.config, args.model_kind) + else: + raise ValueError("The model kind is not supported") + trainer.run() -if args.model_kind in ["baseline", "cbhg"]: - trainer = CBHGTrainer(args.config, args.model_kind) -else: - raise ValueError("The model kind is not supported") -trainer.run() +if __name__ == "__main__": + # Fix for Python 3.9+ multiprocessing issues + multiprocessing.freeze_support() + main() diff --git a/python/hebrew/trainer.py b/python/hebrew/trainer.py index ac7187b..dbb4a3c 100644 --- a/python/hebrew/trainer.py +++ b/python/hebrew/trainer.py @@ -1,7 +1,6 @@ import os import torch -import wandb from config_manager import ConfigManager from dataset import load_iterators from diacritizer import Diacritizer @@ -19,6 +18,15 @@ repeater, ) +# Make wandb optional +try: + import wandb + + WANDB_AVAILABLE = True +except ImportError: + WANDB_AVAILABLE = False + print("Warning: wandb not available in trainer.py, training will proceed without wandb logging") + class Trainer: def run(self): @@ -72,7 +80,7 @@ def print_config(self): def load_diacritizer(self): if self.model_kind in ["cbhg", "baseline"]: - self.diacritizer = Diacritizer(self.config_path, self.model_kind) # , load_model) + self.diacritizer = Diacritizer(self.config_path, self.model_kind) else: print("model not found") exit() @@ -191,12 +199,6 @@ def run(self, config_wandb=None): torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.config["CLIP"]) self.optimizer.step() - { - "N": float(step_results["N"]), - "S": float(step_results["S"]), - "D": float(step_results["D"]), - } - self.print_losses(step_results, tqdm) if self.global_step % self.config["model_save_frequency"] == 0: @@ -218,7 +220,7 @@ def run(self, config_wandb=None): scores, _ = self.evaluate_with_error_rates(validation_iterator, tqdm_error_rates) - if config_wandb is not None: + if config_wandb is not None and WANDB_AVAILABLE: wandb.log({**d_scores, **scores}) print("scores:: ", scores)