[tokenizer] add optional tokenizerPath Prior to modelPath (#3120)

* [tokenizer] add optional tokenizerPath Prior to modelPath --------- Co-authored-by: Frank Liu <[email protected]>
deepjavalibrary · Apr 25, 2024 · 6efe660 · 6efe660
1 parent c03669f
commit 6efe660
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
@@ -30,6 +30,7 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
@@ -686,7 +687,6 @@ static PaddingStrategy fromValue(String value) {
  /** The builder for creating huggingface tokenizer. */
  public static final class Builder {
 
- private Path tokenizerPath;
  private NDManager manager;
  private Map<String, String> options;
 
@@ -724,7 +724,7 @@ public Builder optTokenizerName(String tokenizerName) {
  * @return this builder
  */
  public Builder optTokenizerPath(Path tokenizerPath) {
- this.tokenizerPath = tokenizerPath;
+ options.putIfAbsent("tokenizerPath", tokenizerPath.toString());
  return this;
  }
 
@@ -894,9 +894,11 @@ public HuggingFaceTokenizer build() throws IOException {
  if (tokenizerName != null) {
  return managed(HuggingFaceTokenizer.newInstance(tokenizerName, options));
  }
- if (tokenizerPath == null) {
+ String path = options.get("tokenizerPath");
+ if (path == null) {
  throw new IllegalArgumentException("Missing tokenizer path.");
  }
+ Path tokenizerPath = Paths.get(path);
  if (Files.isDirectory(tokenizerPath)) {
  Path tokenizerFile = tokenizerPath.resolve("tokenizer.json");
  if (Files.exists(tokenizerFile)) {

diff --git a/...ns/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/CrossEncoderTranslatorTest.java b/...ns/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/CrossEncoderTranslatorTest.java
@@ -64,6 +64,7 @@ public void testCrossEncoderTranslator()
  .optBlock(block)
  .optEngine("PyTorch")
  .optArgument("tokenizer", "bert-base-cased")
+ .optArgument("tokenizerPath", modelDir)
  .optOption("hasParameter", "false")
  .optTranslatorFactory(new CrossEncoderTranslatorFactory())
  .build();