diff --git a/Cargo.lock b/Cargo.lock index 143fcef..423f3fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3593,9 +3593,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.19.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd" +checksum = "9ecededfed68a69bc657e486510089e255e53c3d38cc7d4d59c8742668ca2cae" dependencies = [ "aho-corasick", "derive_builder", diff --git a/Cargo.toml b/Cargo.toml index 1d73070..c64d2cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ serde_json = "1.0.127" toml = "0.8.15" clap = { version = "4.5.9", features = ["derive", "color", "env"] } clap-serde-derive = "0.2.1" -tokenizers = "0.19.1" +tokenizers = "0.21.0" hf-hub = "0.3.2" candle-transformers = "0.6.0" candle-core = "0.6.0"