fixes for data loading updates in core (#192)

* fixes for data loading updates in core

* fix

allennlp_models/generation/dataset_readers/cnn_dm.py

@@ -59,7 +59,7 @@ def __init__(
  **kwargs,
  ) -> None:
  super().__init__(
- manual_distributed_sharding=True, manual_multi_process_sharding=True, **kwargs
+ manual_distributed_sharding=True, manual_multiprocess_sharding=True, **kwargs
  )
  self._source_tokenizer = source_tokenizer or SpacyTokenizer()
  self._target_tokenizer = target_tokenizer or self._source_tokenizer

allennlp_models/generation/dataset_readers/copynet_seq2seq.py

@@ -96,7 +96,7 @@ def __init__(
  **kwargs,
  ) -> None:
  super().__init__(
- manual_distributed_sharding=True, manual_multi_process_sharding=True, **kwargs
+ manual_distributed_sharding=True, manual_multiprocess_sharding=True, **kwargs
  )
  self._target_namespace = target_namespace
  self._source_tokenizer = source_tokenizer or SpacyTokenizer()

allennlp_models/generation/dataset_readers/seq2seq.py

@@ -84,7 +84,7 @@ def __init__(
  **kwargs,
  ) -> None:
  super().__init__(
- manual_distributed_sharding=True, manual_multi_process_sharding=True, **kwargs
+ manual_distributed_sharding=True, manual_multiprocess_sharding=True, **kwargs
  )
  self._source_tokenizer = source_tokenizer or SpacyTokenizer()
  self._target_tokenizer = target_tokenizer or self._source_tokenizer

allennlp_models/pair_classification/dataset_readers/snli.py

@@ -46,7 +46,7 @@ def __init__(
  **kwargs,
  ) -> None:
  super().__init__(
- manual_distributed_sharding=True, manual_multi_process_sharding=True, **kwargs
+ manual_distributed_sharding=True, manual_multiprocess_sharding=True, **kwargs
  )
  self._tokenizer = tokenizer or SpacyTokenizer()
  if isinstance(self._tokenizer, PretrainedTransformerTokenizer):

allennlp_models/rc/dataset_readers/squad.py

@@ -91,7 +91,7 @@ def __init__(
  skip_impossible_questions = kwargs.pop("skip_invalid_examples")
 
  super().__init__(
- manual_distributed_sharding=True, manual_multi_process_sharding=True, **kwargs
+ manual_distributed_sharding=True, manual_multiprocess_sharding=True, **kwargs
  )
  self._tokenizer = tokenizer or SpacyTokenizer()
  self._token_indexers = token_indexers or {"tokens": SingleIdTokenIndexer()}

allennlp_models/rc/dataset_readers/transformer_squad.py

@@ -103,7 +103,7 @@ def __init__(
  skip_impossible_questions = kwargs.pop("skip_invalid_examples")
 
  super().__init__(
- manual_distributed_sharding=True, manual_multi_process_sharding=True, **kwargs
+ manual_distributed_sharding=True, manual_multiprocess_sharding=True, **kwargs
  )
  self._tokenizer = PretrainedTransformerTokenizer(
  transformer_model_name,

allennlp_models/structured_prediction/tools/write_srl_predictions_to_conll_format.py

@@ -7,7 +7,6 @@
 from typing import List
 
 import torch
-from allennlp.data.samplers import PyTorchSequentialSampler
 
 from allennlp_models.structured_prediction.models.srl import write_to_conll_eval_file
 
@@ -16,7 +15,8 @@
 from allennlp.common.tqdm import Tqdm
 from allennlp.common import Params
 from allennlp.models.archival import load_archive
-from allennlp.data import DatasetReader, DataLoader
+from allennlp.data import DatasetReader
+from allennlp.data.data_loaders import SimpleDataLoader
 from allennlp.nn.util import move_to_device
 
 
@@ -63,10 +63,10 @@ def main(serialization_directory: str, device: int, data: str, prefix: str, doma
 
  # Load the evaluation data and index it.
  print("reading evaluation data from {}".format(evaluation_data_path))
- dataset = dataset_reader.read(evaluation_data_path)
+ dataset = list(dataset_reader.read(evaluation_data_path))
 
  with torch.autograd.no_grad():
- loader = DataLoader(dataset, sampler=PyTorchSequentialSampler(dataset), batch_size=32)
+ loader = SimpleDataLoader(dataset, 32)
  model_predictions: List[List[str]] = []
  for batch in Tqdm.tqdm(loader):
  batch = move_to_device(batch, device)