From 7616272a1b0b580fb43e3a06701fd11db9a2706a Mon Sep 17 00:00:00 2001 From: SBrandeis <33657802+SBrandeis@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:07:09 +0200 Subject: [PATCH] release: v1.0.0-rc.2 --- bindings/node/Cargo.lock | 10 +++++----- bindings/node/Cargo.toml | 2 +- bindings/node/package.json | 2 +- bindings/python/Cargo.lock | 10 +++++----- bindings/python/Cargo.toml | 2 +- tokenizers/Cargo.lock | 10 +++++----- tokenizers/Cargo.toml | 8 ++++---- tokenizers/bitcannon/Cargo.toml | 2 +- tokenizers/tk-convert/Cargo.toml | 4 ++-- tokenizers/tk-encode/Cargo.toml | 4 ++-- tokenizers/tk-serialize/Cargo.toml | 6 +++--- tokenizers/tk-train/Cargo.toml | 6 +++--- 12 files changed, 33 insertions(+), 33 deletions(-) diff --git a/bindings/node/Cargo.lock b/bindings/node/Cargo.lock index 671e28af67..01fe182859 100644 --- a/bindings/node/Cargo.lock +++ b/bindings/node/Cargo.lock @@ -68,7 +68,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitcannon" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "ahash", ] @@ -702,7 +702,7 @@ dependencies = [ [[package]] name = "node" -version = "1.0.0-dev.0" +version = "1.0.0-rc.2" dependencies = [ "napi", "napi-build", @@ -1190,7 +1190,7 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tk-convert" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "bitcannon", "serde_json", @@ -1199,7 +1199,7 @@ dependencies = [ [[package]] name = "tk-encode" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "ahash", "bitcannon", @@ -1225,7 +1225,7 @@ dependencies = [ [[package]] name = "tk-serialize" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "base64 0.22.1", "bitcannon", diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml index 2bc6f39a93..68da968acc 100644 --- a/bindings/node/Cargo.toml +++ b/bindings/node/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Nicolas Patry "] edition = "2024" name = "node" -version = "1.0.0-dev.0" +version = "1.0.0-rc.2" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/bindings/node/package.json b/bindings/node/package.json index 83902e7da6..1877478c05 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "tokenizers", - "version": "1.0.0-dev.0", + "version": "1.0.0-rc.2", "repository": { "type": "git", "url": "git+https://github.com/huggingface/tokenizers.git" diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index 72db3746c1..e1736e363f 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -68,7 +68,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitcannon" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "ahash", ] @@ -1179,7 +1179,7 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tk-convert" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "bitcannon", "serde_json", @@ -1188,7 +1188,7 @@ dependencies = [ [[package]] name = "tk-encode" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "ahash", "bitcannon", @@ -1214,7 +1214,7 @@ dependencies = [ [[package]] name = "tk-serialize" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "base64 0.22.1", "bitcannon", @@ -1224,7 +1224,7 @@ dependencies = [ [[package]] name = "tokenizers-python" -version = "1.0.0-dev.0" +version = "1.0.0-rc.2" dependencies = [ "numpy", "pyo3", diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index e264ffc896..b648a5fdfd 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tokenizers-python" -version = "1.0.0-dev.0" +version = "1.0.0-rc.2" edition = "2024" publish = false authors = [ diff --git a/tokenizers/Cargo.lock b/tokenizers/Cargo.lock index e3a4ba0373..ad53724dfb 100644 --- a/tokenizers/Cargo.lock +++ b/tokenizers/Cargo.lock @@ -189,7 +189,7 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitcannon" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "ahash", "onig", @@ -3145,7 +3145,7 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tk-convert" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "bitcannon", "criterion 0.8.2", @@ -3159,7 +3159,7 @@ dependencies = [ [[package]] name = "tk-encode" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "ahash", "assert_approx_eq", @@ -3195,7 +3195,7 @@ dependencies = [ [[package]] name = "tk-serialize" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" dependencies = [ "base64 0.22.1", "bitcannon", @@ -3242,7 +3242,7 @@ dependencies = [ [[package]] name = "tokenizers" -version = "1.0.0-dev.0" +version = "1.0.0-rc.2" dependencies = [ "ahash", "assert_approx_eq", diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml index 04c2639ade..e25d406728 100644 --- a/tokenizers/Cargo.toml +++ b/tokenizers/Cargo.toml @@ -13,7 +13,7 @@ authors = [ ] edition = "2024" name = "tokenizers" -version = "1.0.0-dev.0" +version = "1.0.0-rc.2" homepage = "https://github.com/huggingface/tokenizers" repository = "https://github.com/huggingface/tokenizers" documentation = "https://docs.rs/tokenizers/" @@ -41,7 +41,7 @@ path = "src/lib.rs" bench = false [dependencies] -tk-encode = { path = "tk-encode", version = "0.1.0-dev.0", default-features = false, features = [ +tk-encode = { path = "tk-encode", version = "0.1.0-rc.2", default-features = false, features = [ "parallelism", "normalizers", "bpe", @@ -50,10 +50,10 @@ tk-encode = { path = "tk-encode", version = "0.1.0-dev.0", default-features = fa "wordlevel", "unicode-scripts", ] } -tk-convert = { path = "tk-convert", version = "0.1.0-dev.0", default-features = false } +tk-convert = { path = "tk-convert", version = "0.1.0-rc.2", default-features = false } # The reader and the writer. Same component features as `tk-encode` above: the reader has one match # arm per component, so a component compiled out of the runtime cannot be read into it either. -tk-serialize = { path = "tk-serialize", version = "0.1.0-dev.0", default-features = false, features = [ +tk-serialize = { path = "tk-serialize", version = "0.1.0-rc.2", default-features = false, features = [ "deserialize", "serialize", "parallelism", diff --git a/tokenizers/bitcannon/Cargo.toml b/tokenizers/bitcannon/Cargo.toml index 8f5e69c3a2..49916855a1 100644 --- a/tokenizers/bitcannon/Cargo.toml +++ b/tokenizers/bitcannon/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bitcannon" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" edition = "2024" rust-version = "1.89" # AVX-512 VBMI intrinsics (simd_avx_classify) are stable only since 1.89 authors = [ diff --git a/tokenizers/tk-convert/Cargo.toml b/tokenizers/tk-convert/Cargo.toml index 4f5730b3dc..8fe16f4ea1 100644 --- a/tokenizers/tk-convert/Cargo.toml +++ b/tokenizers/tk-convert/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tk-convert" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" edition = "2024" authors = [ "Arthur Zucker ", @@ -31,7 +31,7 @@ thiserror = "2" # The GPT-2 pattern a `ByteLevel` lowers to. `bitcannon::regexes` is the single source of truth for # it -- tk-encode's runtime recognizer reads the same const -- so a copy here could drift and cost # the FSM fast path without changing any test. -bitcannon = { path = "../bitcannon", version = "0.1.0-dev.0" } +bitcannon = { path = "../bitcannon", version = "0.1.0-rc.2" } # Latest released tokenizers, the independent id/decode oracle `tests/oracle.rs` compares against. tokenizers-release = { package = "tokenizers", version = "=0.23.2", optional = true } diff --git a/tokenizers/tk-encode/Cargo.toml b/tokenizers/tk-encode/Cargo.toml index dd36507e99..17246be89b 100644 --- a/tokenizers/tk-encode/Cargo.toml +++ b/tokenizers/tk-encode/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tk-encode" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" edition = "2024" authors = [ "Anthony MOI ", @@ -29,7 +29,7 @@ path = "src/lib.rs" [dependencies] ############ Required dependencies ############################### -bitcannon = { path = "../bitcannon", version = "0.1.0-dev.0" } +bitcannon = { path = "../bitcannon", version = "0.1.0-rc.2" } rand = "0.10" ptr_hash = { version = "2.0.2", default-features = false } unicode_categories = "0.1" diff --git a/tokenizers/tk-serialize/Cargo.toml b/tokenizers/tk-serialize/Cargo.toml index f6bf317d0e..0f94562771 100644 --- a/tokenizers/tk-serialize/Cargo.toml +++ b/tokenizers/tk-serialize/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tk-serialize" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" edition = "2024" authors = [ "Arthur Zucker ", @@ -38,10 +38,10 @@ hifijson = "0.5.0" base64 = "0.22" # The runtime this crate builds. No `config`: the whole point is that reading a canonical # `tokenizer.json` needs no serde layer at all. -tk-encode = { path = "../tk-encode", version = "0.1.0-dev.0", default-features = false } +tk-encode = { path = "../tk-encode", version = "0.1.0-rc.2", default-features = false } # The GPT-2 pre-tokenizer pattern is a constant in `bitcannon`, and a `ByteLevel` pre-tokenizer is # spelled as a `Split` on it. Nothing else here reaches the FSM crate directly. -bitcannon = { path = "../bitcannon", version = "0.1.0-dev.0" } +bitcannon = { path = "../bitcannon", version = "0.1.0-rc.2" } [features] # Reading a canonical `tokenizer.json` into a `PipelineTokenizer`. On by default -- it is what the diff --git a/tokenizers/tk-train/Cargo.toml b/tokenizers/tk-train/Cargo.toml index 00480b8823..6df23e8b02 100644 --- a/tokenizers/tk-train/Cargo.toml +++ b/tokenizers/tk-train/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tk-train" -version = "0.1.0-dev.0" +version = "0.1.0-rc.2" edition = "2024" authors = [ "Arthur Zucker