From 9747c5e8a72c20dfc322911b90674bb6f42406a8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 6 Nov 2024 13:37:02 +0000
Subject: [PATCH] Bump tokenizers from 0.19.1 to 0.20.3

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.19.1 to 0.20.3.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.19.1...v0.20.3)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 143fcef..c24da48 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3593,9 +3593,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokenizers"
-version = "0.19.1"
+version = "0.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd"
+checksum = "67b67c92f6d705e2a1d106fb0b28c696f9074901a9c656ee5d9f5de204c39bf7"
 dependencies = [
  "aho-corasick",
  "derive_builder",
diff --git a/Cargo.toml b/Cargo.toml
index 1d73070..e4462e4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -47,7 +47,7 @@ serde_json = "1.0.127"
 toml = "0.8.15"
 clap = { version = "4.5.9", features = ["derive", "color", "env"] }
 clap-serde-derive = "0.2.1"
-tokenizers = "0.19.1"
+tokenizers = "0.20.3"
 hf-hub = "0.3.2"
 candle-transformers = "0.6.0"
 candle-core = "0.6.0"