Skip RoFormer ONNX test if rjieba not installed (huggingface#16981)

* Skip RoFormer ONNX test if rjieba not installed * Update deps table * Skip RoFormer serialization test * Fix RoFormer vocab * Add rjieba to CircleCI
elusenji · Jun 12, 2022 · 9b49ff0 · 9b49ff0
1 parent 998eff5
commit 9b49ff0
Show file tree

Hide file tree

Showing 6 changed files with 13 additions and 5 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -549,7 +549,7 @@ jobs:
  - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
  - v0.4-{{ checksum "setup.py" }}
  - run: pip install --upgrade pip
- - run: pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy]
+ - run: pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]
  - run: python -m unidic download
  - save_cache:
  key: v0.4-custom_tokenizers-{{ checksum "setup.py" }}
@@ -785,7 +785,7 @@ jobs:
  - v0.4-torch-{{ checksum "setup.py" }}
  - v0.4-{{ checksum "setup.py" }}
  - run: pip install --upgrade pip
- - run: pip install .[torch,testing,sentencepiece,onnxruntime,vision]
+ - run: pip install .[torch,testing,sentencepiece,onnxruntime,vision,rjieba]
  - save_cache:
  key: v0.4-onnx-{{ checksum "setup.py" }}
  paths:

diff --git a/setup.py b/setup.py
@@ -140,6 +140,7 @@
  "ray[tune]",
  "regex!=2019.12.17",
  "requests",
+ "rjieba",
  "rouge-score",
  "sacrebleu>=1.4.12,<2.0.0",
  "sacremoses",
@@ -288,7 +289,8 @@ def run(self):
  "nltk",
  "GitPython",
  "hf-doc-builder",
- 'sacremoses'
+ "sacremoses",
+ "rjieba"
  )
  + extras["retrieval"]
  + extras["modelcreation"]

diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
@@ -47,6 +47,7 @@
  "ray[tune]": "ray[tune]",
  "regex": "regex!=2019.12.17",
  "requests": "requests",
+ "rjieba": "rjieba",
  "rouge-score": "rouge-score",
  "sacrebleu": "sacrebleu>=1.4.12,<2.0.0",
  "sacremoses": "sacremoses",

diff --git a/src/transformers/models/roformer/tokenization_roformer_fast.py b/src/transformers/models/roformer/tokenization_roformer_fast.py
@@ -27,7 +27,7 @@
 
 logger = logging.get_logger(__name__)
 
-VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
 PRETRAINED_VOCAB_FILES_MAP = {
  "vocab_file": {

diff --git a/tests/models/roformer/test_tokenization_roformer.py b/tests/models/roformer/test_tokenization_roformer.py
@@ -71,3 +71,7 @@ def test_training_new_tokenizer(self):
  # can't train new_tokenizer via Tokenizers lib
  def test_training_new_tokenizer_with_special_tokens_change(self):
  pass
+
+ # can't serialise custom PreTokenizer
+ def test_save_slow_from_fast_and_reload_fast(self):
+ pass
diff --git a/tests/onnx/test_onnx_v2.py b/tests/onnx/test_onnx_v2.py
@@ -16,7 +16,7 @@
  validate_model_outputs,
 )
 from transformers.onnx.utils import compute_effective_axis_dimension, compute_serialized_parameters_size
-from transformers.testing_utils import require_onnx, require_tf, require_torch, require_vision, slow
+from transformers.testing_utils import require_onnx, require_rjieba, require_tf, require_torch, require_vision, slow
 
 
 if is_torch_available() or is_tf_available():
@@ -287,6 +287,7 @@ def _onnx_export(self, test_name, name, model_name, feature, onnx_config_class_c
  @slow
  @require_torch
  @require_vision
+ @require_rjieba
  def test_pytorch_export(self, test_name, name, model_name, feature, onnx_config_class_constructor):
  self._onnx_export(test_name, name, model_name, feature, onnx_config_class_constructor)