Raise an error when piece encoders are called before initialization

spacy_curated_transformers/tests/tokenization/test_bbpe_encoder.py

@@ -100,3 +100,9 @@ def _check_roberta_base_encoder(encoding):
  ops.xp.testing.assert_array_equal(
  encoding[1].dataXd, [0, 5625, 52, 40, 3529, 181, 48344, 5749, 4, 2]
  )
+
+
+def test_uninitialized_bbpe_encoder(sample_docs):
+ encoder = build_byte_bpe_encoder_v1()
+ with pytest.raises(ValueError, match="not initialized"):
+ encoder.predict(sample_docs)

spacy_curated_transformers/tests/tokenization/test_char_encoder.py

@@ -111,3 +111,9 @@ def test_loader_rejects_incorrect_encoder(test_dir):
  encoder.init = build_char_encoder_loader_v1(path=test_dir / "toy-chars.txt")
  with pytest.raises(ValueError, match="incompatible model"):
  encoder.initialize()
+
+
+def test_uninitialized_char_encoder(sample_docs):
+ encoder = build_char_encoder_v1()
+ with pytest.raises(ValueError, match="not initialized"):
+ encoder.predict(sample_docs)

spacy_curated_transformers/tests/tokenization/test_sentencepiece_encoder.py

@@ -80,3 +80,9 @@ def _check_toy_encoder(encoding):
  encoding[1].dataXd,
  [1, 483, 546, 112, 171, 567, 62, 20, 45, 0, 84, 115, 27, 7, 4, 2],
  )
+
+
+def test_uninitialized_sentencepiece_encoder(sample_docs):
+ encoder = build_sentencepiece_encoder_v1()
+ with pytest.raises(ValueError, match="not initialized"):
+ encoder.predict(sample_docs)

spacy_curated_transformers/tests/tokenization/test_wordpiece_encoder.py

@@ -224,3 +224,13 @@ def _check_toy_encoder(encoding):
  encoding[1].dataXd,
  [2, 824, 98, 189, 311, 417, 65, 155, 503, 99, 1, 416, 117, 88, 17, 3],
  )
+
+
+def test_uninitialized_wordpiece_encoder(sample_docs):
+ encoder = build_wordpiece_encoder_v1()
+ with pytest.raises(ValueError, match="not initialized"):
+ encoder.predict(sample_docs)
+
+ encoder = build_bert_wordpiece_encoder_v1()
+ with pytest.raises(ValueError, match="not initialized"):
+ encoder.predict(sample_docs)

spacy_curated_transformers/tests/tokenization/test_xlmr_adapter.py

@@ -32,6 +32,7 @@ def toy_model(sentencepiece_toy_model_path):
 def toy_encoder(toy_model):
  encoder = build_xlmr_sentencepiece_encoder_v1()
  encoder.get_ref("encoder").attrs["sentencepiece_processor"] = toy_model
+ encoder.get_ref("encoder").attrs["initialized"] = True
  return encoder
 
 

spacy_curated_transformers/tokenization/bbpe_encoder.py

@@ -38,6 +38,7 @@ def build_byte_bpe_encoder_v1() -> Tok2PiecesModelT:
  "unk_piece": "<unk>",
  "bos_piece": "<s>",
  "eos_piece": "</s>",
+ "initialized": False,
  },
  )
 
@@ -49,6 +50,12 @@ def byte_bpe_encoder_forward(
  bos_piece: str = model.attrs["bos_piece"]
  eos_piece: str = model.attrs["eos_piece"]
  unk_piece: str = model.attrs["unk_piece"]
+ initialized: bool = model.attrs["initialized"]
+ if not initialized:
+ raise ValueError(
+ "Byte-BPE piece encoder was not initialized with an appropriate loader"
+ )
+
  bos_id = bbp.piece_id(bos_piece)
  if bos_id is None:
  raise ValueError(
@@ -116,6 +123,7 @@ def load(model, X=None, Y=None):
  model.attrs["byte_bpe_processor"] = ByteBPEProcessor.load_from_files(
  vocab=vocab_path, merges=merges_path
  )
+ model.attrs["initialized"] = True
  return model
 
  return load

spacy_curated_transformers/tokenization/char_encoder.py

@@ -28,6 +28,7 @@ def build_char_encoder_v1() -> Tok2PiecesModelT:
  "unk_piece": "[UNK]",
  "normalize": "NFKC",
  "vocab": None,
+ "initialized": False,
  },
  )
 
@@ -42,6 +43,11 @@ def char_encoder_forward(
  This model must be separately initialized using an appropriate
  loader.
  """
+ initialized: bool = model.attrs["initialized"]
+ if not initialized:
+ raise ValueError(
+ "Character piece encoder was not initialized with an appropriate loader"
+ )
  vocab: Optional[Dict[str, int]] = model.attrs["vocab"]
  if vocab is None:
  raise ValueError(
@@ -123,6 +129,7 @@ def load(model, X=None, Y=None):
  char = unicodedata.normalize(normalize, char)
  vocab[char] = len(vocab)
  model.attrs["vocab"] = vocab
+ model.attrs["initialized"] = True
  return model
 
  return load

spacy_curated_transformers/tokenization/hf_loader.py

@@ -82,6 +82,7 @@ def _convert_byte_bpe_encoder(
  model.attrs["bos_piece"] = tokenizer.bos_token # type: ignore
  model.attrs["eos_piece"] = tokenizer.eos_token # type: ignore
  model.attrs["unk_piece"] = tokenizer.unk_token # type: ignore
+ model.attrs["initialized"] = True
 
  return model
 
@@ -95,6 +96,8 @@ def _convert_sentencepiece_encoder(
  ] = SentencePieceProcessor.from_file(
  tokenizer.vocab_file # type: ignore
  )
+ model.get_ref("encoder").attrs["initialized"] = True
+
  return model
 
 
@@ -120,6 +123,7 @@ def _convert_wordpiece_encoder(
  model.attrs["strip_accents"] = strip_accents or (
  strip_accents is not False and lowercase
  )
+ model.attrs["initialized"] = True
 
  return model
 
@@ -148,5 +152,6 @@ def _convert_bert_japanese_encoder(
  "NFKC" if tokenizer.subword_tokenizer.normalize_text else None
  )
  model.attrs["vocab"] = tokenizer.vocab.copy()
+ model.attrs["initialized"] = True
 
  return model

spacy_curated_transformers/tokenization/sentencepiece_encoder.py

@@ -47,7 +47,10 @@ def build_sentencepiece_encoder_v1() -> Tok2PiecesModelT:
  model: Tok2PiecesModelT = Model(
  "sentencepiece_encoder",
  forward=sentencepiece_encoder_forward,
- attrs={"sentencepiece_processor": SentencePieceProcessor()},
+ attrs={
+ "sentencepiece_processor": SentencePieceProcessor(),
+ "initialized": False,
+ },
  )
  model.set_ref("encoder", model)
  return model
@@ -70,6 +73,11 @@ def build_xlmr_sentencepiece_encoder_v1() -> Tok2PiecesModelT:
 def sentencepiece_encoder_forward(
  model: Model, X: Tok2PiecesInT, is_train: bool
 ) -> Tuple[Tok2PiecesOutT, Tok2PiecesBackpropT]:
+ initialized: bool = model.attrs["initialized"]
+ if not initialized:
+ raise ValueError(
+ "SentencePiece piece encoder was not initialized with an appropriate loader"
+ )
  spp: SentencePieceProcessor = model.attrs["sentencepiece_processor"]
 
  pieces = []
@@ -114,6 +122,7 @@ def load(model, X=None, Y=None):
  model.attrs["sentencepiece_processor"] = SentencePieceProcessor.from_file(
  str(path)
  )
+ model.attrs["initialized"] = True
  return model
 
  return load

spacy_curated_transformers/tokenization/wordpiece_encoder.py

@@ -42,6 +42,7 @@ def build_bert_wordpiece_encoder_v1() -> Tok2PiecesModelT:
  "lowercase": False,
  "preprocess": _bert_preprocess,
  "strip_accents": False,
+ "initialized": False,
  },
  )
 
@@ -65,6 +66,7 @@ def build_wordpiece_encoder_v1() -> Tok2PiecesModelT:
  "lowercase": False,
  "preprocess": lambda t: [t],
  "strip_accents": False,
+ "initialized": False,
  },
  )
 
@@ -79,6 +81,13 @@ def wordpiece_encoder_forward(
  lowercase: bool = model.attrs["lowercase"]
  preprocess: Callable[[str], str] = model.attrs["preprocess"]
  strip_accents: bool = model.attrs["strip_accents"]
+
+ initialized: bool = model.attrs["initialized"]
+ if not initialized:
+ raise ValueError(
+ "WordPiece piece encoder was not initialized with an appropriate loader"
+ )
+
  bos_id = wpp.get_initial(bos_piece)
  eos_id = wpp.get_initial(eos_piece)
  unk_id = wpp.get_initial(unk_piece)
@@ -143,6 +152,7 @@ def load(model, X=None, Y=None):
  model.attrs["unk_piece"] = unk_piece
  model.attrs["lowercase"] = lowercase
  model.attrs["strip_accents"] = strip_accents
+ model.attrs["initialized"] = True
  return model
 
  return load