From 6cecd9378f43f180f11945e6322b86e138cfbb15 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 27 Nov 2024 13:59:20 +0000
Subject: [PATCH] Bump tokenizers from 0.19.1 to 0.21.0

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.19.1 to 0.21.0.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.19.1...v0.21.0)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 143fcef..423f3fc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3593,9 +3593,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokenizers"
-version = "0.19.1"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd"
+checksum = "9ecededfed68a69bc657e486510089e255e53c3d38cc7d4d59c8742668ca2cae"
 dependencies = [
  "aho-corasick",
  "derive_builder",
diff --git a/Cargo.toml b/Cargo.toml
index 1d73070..c64d2cf 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -47,7 +47,7 @@ serde_json = "1.0.127"
 toml = "0.8.15"
 clap = { version = "4.5.9", features = ["derive", "color", "env"] }
 clap-serde-derive = "0.2.1"
-tokenizers = "0.19.1"
+tokenizers = "0.21.0"
 hf-hub = "0.3.2"
 candle-transformers = "0.6.0"
 candle-core = "0.6.0"