From ce014ad281d69e92886d8f61d45a06366a75e9cd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Sep 2024 13:47:18 +0000
Subject: [PATCH] Bump tokenizers from 0.19.1 to 0.20.0

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.19.1 to 0.20.0.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.19.1...v0.20.0)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 143fcef..cf7404d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3593,9 +3593,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokenizers"
-version = "0.19.1"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd"
+checksum = "c8a24d7f7d6be5b9d1377418b893ab1808af0074f5d1bb2c64784452ddd2aa70"
 dependencies = [
  "aho-corasick",
  "derive_builder",
diff --git a/Cargo.toml b/Cargo.toml
index 1d73070..c258c18 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -47,7 +47,7 @@ serde_json = "1.0.127"
 toml = "0.8.15"
 clap = { version = "4.5.9", features = ["derive", "color", "env"] }
 clap-serde-derive = "0.2.1"
-tokenizers = "0.19.1"
+tokenizers = "0.20.0"
 hf-hub = "0.3.2"
 candle-transformers = "0.6.0"
 candle-core = "0.6.0"