diff --git a/Cargo.lock b/Cargo.lock index 143fcef..c24da48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3593,9 +3593,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.19.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd" +checksum = "67b67c92f6d705e2a1d106fb0b28c696f9074901a9c656ee5d9f5de204c39bf7" dependencies = [ "aho-corasick", "derive_builder", diff --git a/Cargo.toml b/Cargo.toml index 1d73070..e4462e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ serde_json = "1.0.127" toml = "0.8.15" clap = { version = "4.5.9", features = ["derive", "color", "env"] } clap-serde-derive = "0.2.1" -tokenizers = "0.19.1" +tokenizers = "0.20.3" hf-hub = "0.3.2" candle-transformers = "0.6.0" candle-core = "0.6.0"