add sentence piece IS loading option

extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java

@@ -13,9 +13,11 @@
 package ai.djl.sentencepiece;
 
 import ai.djl.modality.nlp.preprocess.Tokenizer;
+import ai.djl.util.Utils;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
@@ -61,6 +63,17 @@ public SpTokenizer(byte[] serializedModel) {
  processor.loadModelFromBytes(serializedModel);
  }
 
+ /**
+ * Creates a SentencePiece Tokenizer from inputStream.
+ *
+ * @param is {@link InputStream} of the serialized model
+ * @throws IOException when IO operation fails in loading a resource
+ */
+ public SpTokenizer(InputStream is) throws IOException {
+ this.processor = SpProcessor.newInstance();
+ processor.loadModelFromBytes(Utils.toByteArray(is));
+ }
+
  /** {@inheritDoc} */
  @Override
  public List<String> tokenize(String sentence) {

extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java

@@ -21,6 +21,7 @@
 import org.testng.annotations.Test;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -55,6 +56,21 @@ public void testLoadFromBytes() throws IOException {
  }
  }
 
+ @Test
+ public void testLoadFromInputStream() throws IOException {
+ Path modelPath = Paths.get("build/test/sp_model/sp_model.model");
+ try (InputStream is = Files.newInputStream(modelPath)) {
+ try (SpTokenizer tokenizer = new SpTokenizer(is)) {
+ String original = "Hello World";
+ List<String> tokens = tokenizer.tokenize(original);
+ List<String> expected = Arrays.asList("▁He", "ll", "o", "▁", "W", "or", "l", "d");
+ Assert.assertEquals(tokens, expected);
+ String recovered = tokenizer.buildSentence(tokens);
+ Assert.assertEquals(original, recovered);
+ }
+ }
+ }
+
  @Test
  public void testTokenize() throws IOException {
  TestRequirements.notWindows();