From 57cbd464c0a5a8dba2276e48e16a2787e74b1e32 Mon Sep 17 00:00:00 2001
From: Xin Yang <xyang19@gmail.com>
Date: Mon, 16 Sep 2024 14:28:01 -0700
Subject: [PATCH] [tokenizers] Use tokenizers from rust.io

---
 extensions/tokenizers/build.sh        | 3 ---
 extensions/tokenizers/rust/Cargo.toml | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/extensions/tokenizers/build.sh b/extensions/tokenizers/build.sh
index ca7f3957c0b..d23594de6d6 100755
--- a/extensions/tokenizers/build.sh
+++ b/extensions/tokenizers/build.sh
@@ -9,9 +9,6 @@ ARCH=$2
 FLAVOR=$3
 
 pushd "$WORK_DIR"
-if [ ! -d "tokenizers" ]; then
-  git clone https://github.com/huggingface/tokenizers -b "$VERSION"
-fi
 
 if [ ! -d "build" ]; then
   mkdir build
diff --git a/extensions/tokenizers/rust/Cargo.toml b/extensions/tokenizers/rust/Cargo.toml
index c177a6681e0..fe9346fad9d 100644
--- a/extensions/tokenizers/rust/Cargo.toml
+++ b/extensions/tokenizers/rust/Cargo.toml
@@ -13,7 +13,7 @@ candle-flash-attn = { version = "*", optional = true }
 candle-cublaslt = { git = "https://github.com/huggingface/candle-cublaslt", rev = "cf789b7dd6d4abb19b03b9556442f94f0588b4a0", optional = true }
 candle-layer-norm = { git = "https://github.com/xyang16/candle-layer-norm", rev = "e574de6a7f88bafbede8edf9ee43170c6a8ce51a", optional = true }
 candle-rotary = { git = "https://github.com/huggingface/candle-rotary", rev = "0a718a0856569a92f3112e64f10d07e4447822e8", optional = true }
-tokenizers = { path = "../tokenizers/tokenizers", version = "*", features = ["http"] }
+tokenizers = { version = "0.20.0", features = ["http"] }
 half = "2.4.0"
 tracing = "0.1.40"
 safetensors = "0.4.3"