Bug fixes and options for TextCategorizer

examples/training/train_textcat.py

@@ -43,7 +43,11 @@ def main(model=None, output_dir=None, n_iter=20, n_texts=2000, init_tok2vec=None
  # nlp.create_pipe works for built-ins that are registered with spaCy
  if "textcat" not in nlp.pipe_names:
  textcat = nlp.create_pipe(
- "textcat", config={"architecture": "simple_cnn", "exclusive_classes": True}
+ "textcat",
+ config={
+ "exclusive_classes": True,
+ "architecture": "simple_cnn",
+ }
  )
  nlp.add_pipe(textcat, last=True)
  # otherwise, get it, so we can add labels to it
@@ -56,7 +60,9 @@ def main(model=None, output_dir=None, n_iter=20, n_texts=2000, init_tok2vec=None
 
  # load the IMDB dataset
  print("Loading IMDB data...")
- (train_texts, train_cats), (dev_texts, dev_cats) = load_data(limit=n_texts)
+ (train_texts, train_cats), (dev_texts, dev_cats) = load_data()
+ train_texts = train_texts[:n_texts]
+ train_cats = train_cats[:n_texts]
  print(
  "Using {} examples ({} training, {} evaluation)".format(
  n_texts, len(train_texts), len(dev_texts)

spacy/_ml.py

@@ -81,18 +81,6 @@ def _zero_init_impl(self, *args, **kwargs):
  return model
 
 
-@layerize
-def _preprocess_doc(docs, drop=0.0):
- keys = [doc.to_array(LOWER) for doc in docs]
- # The dtype here matches what thinc is expecting -- which differs per
- # platform (by int definition). This should be fixed once the problem
- # is fixed on Thinc's side.
- lengths = numpy.array([arr.shape[0] for arr in keys], dtype=numpy.int_)
- keys = numpy.concatenate(keys)
- vals = numpy.zeros(keys.shape, dtype='f')
- return (keys, vals, lengths), None
-
-
 def with_cpu(ops, model):
  """Wrap a model that should run on CPU, transferring inputs and outputs
  as necessary."""
@@ -133,20 +121,31 @@ def _to_device(ops, X):
  return ops.asarray(X)
 
 
-@layerize
-def _preprocess_doc_bigrams(docs, drop=0.0):
- unigrams = [doc.to_array(LOWER) for doc in docs]
- ops = Model.ops
- bigrams = [ops.ngrams(2, doc_unis) for doc_unis in unigrams]
- keys = [ops.xp.concatenate(feats) for feats in zip(unigrams, bigrams)]
- keys, vals = zip(*[ops.xp.unique(k, return_counts=True) for k in keys])
- # The dtype here matches what thinc is expecting -- which differs per
- # platform (by int definition). This should be fixed once the problem
- # is fixed on Thinc's side.
- lengths = ops.asarray([arr.shape[0] for arr in keys], dtype=numpy.int_)
- keys = ops.xp.concatenate(keys)
- vals = ops.asarray(ops.xp.concatenate(vals), dtype="f")
- return (keys, vals, lengths), None
+class extract_ngrams(Model):
+ def __init__(self, ngram_size, attr=LOWER):
+ Model.__init__(self)
+ self.ngram_size = ngram_size
+ self.attr = attr
+
+ def begin_update(self, docs, drop=0.0):
+ batch_keys = []
+ batch_vals = []
+ for doc in docs:
+ unigrams = doc.to_array([self.attr])
+ ngrams = [unigrams]
+ for n in range(2, self.ngram_size + 1):
+ ngrams.append(self.ops.ngrams(n, unigrams))
+ keys = self.ops.xp.concatenate(ngrams)
+ keys, vals = self.ops.xp.unique(keys, return_counts=True)
+ batch_keys.append(keys)
+ batch_vals.append(vals)
+ # The dtype here matches what thinc is expecting -- which differs per
+ # platform (by int definition). This should be fixed once the problem
+ # is fixed on Thinc's side.
+ lengths = self.ops.asarray([arr.shape[0] for arr in batch_keys], dtype=numpy.int_)
+ batch_keys = self.ops.xp.concatenate(batch_keys)
+ batch_vals = self.ops.asarray(self.ops.xp.concatenate(batch_vals), dtype="f")
+ return (batch_keys, batch_vals, lengths), None
 
 
 @describe.on_data(
@@ -486,16 +485,6 @@ def _zero_init_impl(self, X, y):
  return model
 
 
-@layerize
-def preprocess_doc(docs, drop=0.0):
- keys = [doc.to_array([LOWER]) for doc in docs]
- ops = Model.ops
- lengths = ops.asarray([arr.shape[0] for arr in keys])
- keys = ops.xp.concatenate(keys)
- vals = ops.allocate(keys.shape[0]) + 1
- return (keys, vals, lengths), None
-
-
 def getitem(i):
  def getitem_fwd(X, drop=0.0):
  return X[i], None
@@ -602,10 +591,8 @@ def build_text_classifier(nr_class, width=64, **cfg):
  >> zero_init(Affine(nr_class, width, drop_factor=0.0))
  )
 
- linear_model = (
- _preprocess_doc
- >> with_cpu(Model.ops, LinearModel(nr_class))
- )
+ linear_model = build_bow_text_classifier(
+ nr_class, ngram_size=cfg.get("ngram_size", 1), no_output_layer=True)
  if cfg.get('exclusive_classes'):
  output_layer = Softmax(nr_class, nr_class * 2)
  else:
@@ -623,6 +610,33 @@ def build_text_classifier(nr_class, width=64, **cfg):
  return model
 
 
+def build_bow_text_classifier(nr_class, ngram_size=1, exclusive_classes=False,
+ no_output_layer=False, **cfg):
+ with Model.define_operators({">>": chain}):
+ model = (
+ extract_ngrams(ngram_size, attr=ORTH) 
+ >> with_cpu(Model.ops,
+ LinearModel(nr_class)
+ )
+ )
+ if not no_output_layer:
+ model = model >> (cpu_softmax if exclusive_classes else logistic)
+ model.nO = nr_class
+ return model
+
+
+@layerize
+def cpu_softmax(X, drop=0.):
+ ops = NumpyOps()
+
+ Y = ops.softmax(X)
+
+ def cpu_softmax_backward(dY, sgd=None):
+ return dY
+
+ return ops.softmax(X), cpu_softmax_backward
+
+
 def build_simple_cnn_text_classifier(tok2vec, nr_class, exclusive_classes=False, **cfg):
  """
  Build a simple CNN text classifier, given a token-to-vector model as inputs.

spacy/pipeline/pipes.pyx

@@ -25,6 +25,7 @@ from ..attrs import POS, ID
 from ..parts_of_speech import X
 from .._ml import Tok2Vec, build_tagger_model
 from .._ml import build_text_classifier, build_simple_cnn_text_classifier
+from .._ml import build_bow_text_classifier
 from .._ml import link_vectors_to_models, zero_init, flatten
 from .._ml import masked_language_model, create_default_optimizer
 from ..errors import Errors, TempErrors
@@ -876,6 +877,8 @@ class TextCategorizer(Pipe):
  if cfg.get("architecture") == "simple_cnn":
  tok2vec = Tok2Vec(token_vector_width, embed_size, **cfg)
  return build_simple_cnn_text_classifier(tok2vec, nr_class, **cfg)
+ elif cfg.get("architecture") == "bow":
+ return build_bow_text_classifier(nr_class, **cfg)
  else:
  return build_text_classifier(nr_class, **cfg)
 

website/docs/api/textcategorizer.md

@@ -58,8 +58,9 @@ argument.
 
 | Name | Description |
 | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `"ensemble"` | **Default:** Stacked ensemble of a unigram bag-of-words model and a neural network model. The neural network uses a CNN with mean pooling and attention. |
-| `"simple_cnn"` | A neural network model where token vectors are calculated using a CNN. The vectors are mean pooled and used as features in a feed-forward network. |
+| `"ensemble"` | **Default:** Stacked ensemble of a bag-of-words model and a neural network model. The neural network uses a CNN with mean pooling and attention. The "ngram_size" and "attr" arguments can be used to configure the feature extraction for the bag-of-words model.
+| `"simple_cnn"` | A neural network model where token vectors are calculated using a CNN. The vectors are mean pooled and used as features in a feed-forward network. This architecture is usually less accurate than the ensemble, but runs faster. |
+| `"bow"` | An ngram "bag-of-words" model. This architecture should run much faster than the others, but may not be as accurate, especially if texts are short. The features extracted can be controlled using the keyword arguments ngram_size and attr. For instance, `ngram_size=3` and `attr="lower"` would give lower-cased unigram, trigram and bigram features. 2, 3 or 4 are usually good choices of ngram size. |
 
 ## TextCategorizer.\_\_call\_\_ {#call tag="method"}