release: v1.0.0-rc.2

bindings/node/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Nicolas Patry <nicolas@huggingface.co>"]
 edition = "2024"
 name    = "node"
-version = "1.0.0-dev.0"
+version = "1.0.0-rc.2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

bindings/node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tokenizers",
-  "version": "1.0.0-dev.0",
+  "version": "1.0.0-rc.2",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/huggingface/tokenizers.git"

bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers-python"
-version = "1.0.0-dev.0"
+version = "1.0.0-rc.2"
 edition = "2024"
 publish = false
 authors = [

tokenizers/Cargo.toml

@@ -13,7 +13,7 @@ authors = [
 ]
 edition = "2024"
 name = "tokenizers"
-version = "1.0.0-dev.0"
+version = "1.0.0-rc.2"
 homepage = "https://github.com/huggingface/tokenizers"
 repository = "https://github.com/huggingface/tokenizers"
 documentation = "https://docs.rs/tokenizers/"
@@ -41,7 +41,7 @@ path = "src/lib.rs"
 bench = false
 
 [dependencies]
-tk-encode = { path = "tk-encode", version = "0.1.0-dev.0", default-features = false, features = [
+tk-encode = { path = "tk-encode", version = "0.1.0-rc.2", default-features = false, features = [
   "parallelism",
   "normalizers",
   "bpe",
@@ -50,10 +50,10 @@ tk-encode = { path = "tk-encode", version = "0.1.0-dev.0", default-features = fa
   "wordlevel",
   "unicode-scripts",
 ] }
-tk-convert = { path = "tk-convert", version = "0.1.0-dev.0", default-features = false }
+tk-convert = { path = "tk-convert", version = "0.1.0-rc.2", default-features = false }
 # The reader and the writer. Same component features as `tk-encode` above: the reader has one match
 # arm per component, so a component compiled out of the runtime cannot be read into it either.
-tk-serialize = { path = "tk-serialize", version = "0.1.0-dev.0", default-features = false, features = [
+tk-serialize = { path = "tk-serialize", version = "0.1.0-rc.2", default-features = false, features = [
   "deserialize",
   "serialize",
   "parallelism",

tokenizers/bitcannon/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bitcannon"
-version = "0.1.0-dev.0"
+version = "0.1.0-rc.2"
 edition = "2024"
 rust-version = "1.89" # AVX-512 VBMI intrinsics (simd_avx_classify) are stable only since 1.89
 authors = [

tokenizers/tk-convert/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tk-convert"
-version = "0.1.0-dev.0"
+version = "0.1.0-rc.2"
 edition = "2024"
 authors = [
   "Arthur Zucker <arthur@huggingface.co>",
@@ -31,7 +31,7 @@ thiserror = "2"
 # The GPT-2 pattern a `ByteLevel` lowers to. `bitcannon::regexes` is the single source of truth for
 # it -- tk-encode's runtime recognizer reads the same const -- so a copy here could drift and cost
 # the FSM fast path without changing any test.
-bitcannon = { path = "../bitcannon", version = "0.1.0-dev.0" }
+bitcannon = { path = "../bitcannon", version = "0.1.0-rc.2" }
 
 # Latest released tokenizers, the independent id/decode oracle `tests/oracle.rs` compares against.
 tokenizers-release = { package = "tokenizers", version = "=0.23.2", optional = true }

tokenizers/tk-encode/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tk-encode"
-version = "0.1.0-dev.0"
+version = "0.1.0-rc.2"
 edition = "2024"
 authors = [
   "Anthony MOI <m.anthony.moi@gmail.com>",
@@ -29,7 +29,7 @@ path = "src/lib.rs"
 
 [dependencies]
 ############ Required dependencies ###############################
-bitcannon = { path = "../bitcannon", version = "0.1.0-dev.0" }
+bitcannon = { path = "../bitcannon", version = "0.1.0-rc.2" }
 rand = "0.10"
 ptr_hash = { version = "2.0.2", default-features = false }
 unicode_categories = "0.1"

tokenizers/tk-serialize/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tk-serialize"
-version = "0.1.0-dev.0"
+version = "0.1.0-rc.2"
 edition = "2024"
 authors = [
   "Arthur Zucker <arthur@huggingface.co>",
@@ -38,10 +38,10 @@ hifijson = "0.5.0"
 base64 = "0.22"
 # The runtime this crate builds. No `config`: the whole point is that reading a canonical
 # `tokenizer.json` needs no serde layer at all.
-tk-encode = { path = "../tk-encode", version = "0.1.0-dev.0", default-features = false }
+tk-encode = { path = "../tk-encode", version = "0.1.0-rc.2", default-features = false }
 # The GPT-2 pre-tokenizer pattern is a constant in `bitcannon`, and a `ByteLevel` pre-tokenizer is
 # spelled as a `Split` on it. Nothing else here reaches the FSM crate directly.
-bitcannon = { path = "../bitcannon", version = "0.1.0-dev.0" }
+bitcannon = { path = "../bitcannon", version = "0.1.0-rc.2" }
 
 [features]
 # Reading a canonical `tokenizer.json` into a `PipelineTokenizer`. On by default -- it is what the

tokenizers/tk-train/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tk-train"
-version = "0.1.0-dev.0"
+version = "0.1.0-rc.2"
 edition = "2024"
 authors = [
   "Arthur Zucker <arthur@huggingface.co",
@@ -25,7 +25,7 @@ path = "src/lib.rs"
 # Every trainer needs its model, and the training extension goes through `Tokenizer`, so the
 # training half wants the full inference surface -- `tk-encode`'s slim defaults are for consumers
 # that only encode.
-tk-encode = { path = "../tk-encode", version = "0.1.0-dev.0", default-features = false, features = [
+tk-encode = { path = "../tk-encode", version = "0.1.0-rc.2", default-features = false, features = [
   "parallelism",
   "normalizers",
   "bpe",
@@ -34,7 +34,7 @@ tk-encode = { path = "../tk-encode", version = "0.1.0-dev.0", default-features =
   "wordlevel",
 ] }
 # `ModelWrapper`, `TokenizerImpl` and `AddedToken` live in the config layer.
-tk-convert = { path = "../tk-convert", version = "0.1.0-dev.0", default-features = false }
+tk-convert = { path = "../tk-convert", version = "0.1.0-rc.2", default-features = false }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 ahash = { version = "0.8.11", features = ["serde"] }