[Maps][File upload] Parse geojson files in chunks to avoid thread blocking

x-pack/legacy/plugins/file_upload/public/components/json_index_file_picker.js

@@ -10,6 +10,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import { i18n } from '@kbn/i18n';
 import { parseFile } from '../util/file_parser';
 import { MAX_FILE_SIZE } from '../../common/constants/file_import';
+import _ from 'lodash';
 
 const ACCEPTABLE_FILETYPES = [
  'json',
@@ -20,7 +21,10 @@ export class JsonIndexFilePicker extends Component {
 
  state = {
  fileUploadError: '',
- fileParsingProgress: '',
+ percentageProcessed: 0,
+ featuresProcessed: 0,
+ fileParseActive: false,
+ currentFileTracker: null,
  };
 
  async componentDidMount() {
@@ -31,16 +35,28 @@ export class JsonIndexFilePicker extends Component {
  this._isMounted = false;
  }
 
+ getFileParseActive = () => this._isMounted && this.state.fileParseActive;
+
  _fileHandler = fileList => {
  const fileArr = Array.from(fileList);
  this.props.resetFileAndIndexSettings();
- this.setState({ fileUploadError: '' });
+ this.setState({
+ fileUploadError: '',
+ percentageProcessed: 0,
+ featuresProcessed: 0,
+ });
  if (fileArr.length === 0) { // Remove
+ this.setState({
+ fileParseActive: false
+ });
  return;
  }
  const file = fileArr[0];
 
- this._parseFile(file);
+ this.setState({
+ fileParseActive: true,
+ currentFileTracker: Symbol()
+ }, () => this._parseFile(file));
  };
 
  _checkFileSize = ({ size }) => {
@@ -106,7 +122,17 @@ export class JsonIndexFilePicker extends Component {
  return fileNameOnly.toLowerCase();
  }
 
+ // It's necessary to throttle progress. Updates that are too frequent cause
+ // issues (update failure) in the nested progress component
+ setFileProgress = _.debounce(({ featuresProcessed, bytesProcessed, totalBytes }) => {
+ const percentageProcessed = parseInt((100 * bytesProcessed) / totalBytes);
+ if (this.getFileParseActive()) {
+ this.setState({ featuresProcessed, percentageProcessed });
+ }
+ }, 150);
+
  async _parseFile(file) {
+ const { currentFileTracker } = this.state;
  const {
  setFileRef, setParsedFile, resetFileAndIndexSettings, onFileUpload,
  transformDetails, setIndexName
@@ -119,15 +145,19 @@ export class JsonIndexFilePicker extends Component {
  return;
  }
  // Parse file
- this.setState({ fileParsingProgress: i18n.translate(
- 'xpack.fileUpload.jsonIndexFilePicker.parsingFile',
- { defaultMessage: 'Parsing file...' })
- });
- const parsedFileResult = await parseFile(
- file, transformDetails, onFileUpload
- ).catch(err => {
+
+ const fileResult = await parseFile({
+ file,
+ transformDetails,
+ onFileUpload,
+ setFileProgress: this.setFileProgress,
+ getFileParseActive: this.getFileParseActive
+ }).catch(err => {
  if (this._isMounted) {
  this.setState({
+ fileParseActive: false,
+ percentageProcessed: 0,
+ featuresProcessed: 0,
  fileUploadError: (
  <FormattedMessage
  id="xpack.fileUpload.jsonIndexFilePicker.unableParseFile"
@@ -143,25 +173,59 @@ export class JsonIndexFilePicker extends Component {
  if (!this._isMounted) {
  return;
  }
- this.setState({ fileParsingProgress: '' });
- if (!parsedFileResult) {
+
+ // If another file is replacing this one, leave file parse active
+ this.setState({
+ percentageProcessed: 0,
+ featuresProcessed: 0,
+ fileParseActive: currentFileTracker !== this.state.currentFileTracker
+ });
+ if (!fileResult) {
  resetFileAndIndexSettings();
  return;
  }
+ const { errors, parsedGeojson } = fileResult;
 
+ if (errors.length) {
+ // Set only the first error for now (since there's only one).
+ // TODO: Add handling in case of further errors
+ const error = errors[0];
+ this.setState({
+ fileUploadError: (
+ <FormattedMessage
+ id="xpack.fileUpload.jsonIndexFilePicker.fileParseError"
+ defaultMessage="File parse error(s) detected: {error}"
+ values={{ error }}
+ />
+ )
+ });
+ }
  setIndexName(defaultIndexName);
  setFileRef(file);
- setParsedFile(parsedFileResult);
+ setParsedFile(parsedGeojson);
  }
 
 
 
  render() {
- const { fileParsingProgress, fileUploadError } = this.state;
+ const {
+ fileUploadError,
+ percentageProcessed,
+ featuresProcessed,
+ } = this.state;
 
  return (
  <Fragment>
- {fileParsingProgress ? <EuiProgress size="xs" color="accent" position="absolute" /> : null}
+ {percentageProcessed
+ ? <EuiProgress
+ value={percentageProcessed}
+ max={100}
+ size="xs"
+ color="accent"
+ position="absolute"
+ />
+ : null
+ }
  <EuiFormRow
  label={
  <FormattedMessage
@@ -172,8 +236,14 @@ export class JsonIndexFilePicker extends Component {
  isInvalid={fileUploadError !== ''}
  error={[fileUploadError]}
  helpText={
- fileParsingProgress ? (
- fileParsingProgress
+ percentageProcessed ? (
+ i18n.translate(
+ 'xpack.fileUpload.jsonIndexFilePicker.parsingFile', {
+ defaultMessage: '{featuresProcessed} features parsed...',
+ values: {
+ featuresProcessed
+ }
+ })
  ) : (
  <span>
  {i18n.translate('xpack.fileUpload.jsonIndexFilePicker.formatsAccepted', {

x-pack/legacy/plugins/file_upload/public/util/file_parser.js

@@ -7,39 +7,127 @@
 import _ from 'lodash';
 import { geoJsonCleanAndValidate } from './geo_json_clean_and_validate';
 import { i18n } from '@kbn/i18n';
+import { PatternReader } from './pattern_reader';
 
-export async function readFile(file) {
- const readPromise = new Promise((resolve, reject) => {
- if (!file) {
- reject(new Error(i18n.translate(
- 'xpack.fileUpload.fileParser.noFileProvided', {
+// In local testing, performance improvements leveled off around this size
+export const FILE_BUFFER = 1024000;
+
+const readSlice = (fileReader, file, start, stop) => {
+ const blob = file.slice(start, stop);
+ fileReader.readAsBinaryString(blob);
+};
+
+let prevFileReader;
+let prevPatternReader;
+export const fileHandler = async ({
+ file,
+ setFileProgress,
+ cleanAndValidate,
+ getFileParseActive,
+ fileReader = new FileReader()
+}) => {
+
+ if (!file) {
+ return Promise.reject(
+ new Error(
+ i18n.translate('xpack.fileUpload.fileParser.noFileProvided', {
  defaultMessage: 'Error, no file provided',
- })));
- }
- const fr = new window.FileReader();
- fr.onload = e => resolve(e.target.result);
- fr.onerror = () => {
- fr.abort();
+ })
+ )
+ );
+ }
+
+
+ // Halt any previous file reading & pattern checking activity
+ if (prevFileReader) {
+ prevFileReader.abort();
+ }
+ if (prevPatternReader) {
+ prevPatternReader.abortStream();
+ }
+
+ // Set up feature tracking
+ let featuresProcessed = 0;
+ const onFeatureRead = feature => {
+ // TODO: Add handling and tracking for cleanAndValidate fails
+ featuresProcessed++;
+ return cleanAndValidate(feature);
+ };
+
+ let start;
+ let stop = FILE_BUFFER;
+
+ prevFileReader = fileReader;
+
+ const filePromise = new Promise((resolve, reject) => {
+ const onStreamComplete = fileResults => {
+ if (!featuresProcessed) {
+ reject(
+ new Error(
+ i18n.translate('xpack.fileUpload.fileParser.noFeaturesDetected', {
+ defaultMessage: 'Error, no features detected',
+ })
+ )
+ );
+ } else {
+ resolve(fileResults);
+ }
+ };
+ const patternReader = new PatternReader({ onFeatureDetect: onFeatureRead, onStreamComplete });
+ prevPatternReader = patternReader;
+
+ fileReader.onloadend = ({ target: { readyState, result } }) => {
+ if (readyState === FileReader.DONE) {
+ if (!getFileParseActive() || !result) {
+ fileReader.abort();
+ patternReader.abortStream();
+ resolve(null);
+ return;
+ }
+ setFileProgress({
+ featuresProcessed,
+ bytesProcessed: stop || file.size,
+ totalBytes: file.size
+ });
+ patternReader.writeDataToPatternStream(result);
+ if (!stop) {
+ return;
+ }
+
+ start = stop;
+ const newStop = stop + FILE_BUFFER;
+ // Check EOF
+ stop = newStop > file.size ? undefined : newStop;
+ readSlice(fileReader, file, start, stop);
+ }
+ };
+ fileReader.onerror = () => {
+ fileReader.abort();
+ patternReader.abortStream();
  reject(new Error(i18n.translate(
  'xpack.fileUpload.fileParser.errorReadingFile', {
  defaultMessage: 'Error reading file',
  })));
  };
- fr.readAsText(file);
  });
+ readSlice(fileReader, file, start, stop);
+ return filePromise;
+};
 
- return await readPromise;
-}
-
-export function jsonPreview(json, previewFunction) {
- // Call preview (if any)
- if (json && previewFunction) {
- const defaultName = _.get(json, 'name', 'Import File');
- previewFunction(json, defaultName);
+export function jsonPreview(fileResults, previewFunction) {
+ if (fileResults && fileResults.parsedGeojson && previewFunction) {
+ const defaultName = _.get(fileResults.parsedGeojson, 'name', 'Import File');
+ previewFunction(fileResults.parsedGeojson, defaultName);
  }
 }
 
-export async function parseFile(file, transformDetails, previewCallback = null) {
+export async function parseFile({
+ file,
+ transformDetails,
+ onFileUpload: previewCallback = null,
+ setFileProgress,
+ getFileParseActive
+}) {
  let cleanAndValidate;
  if (typeof transformDetails === 'object') {
  cleanAndValidate = transformDetails.cleanAndValidate;
@@ -56,15 +144,15 @@ export async function parseFile(file, transformDetails, previewCallback = null)
  values: { transformDetails }
  })
  );
- return;
  }
  }
 
- const rawResults = await readFile(file);
- const parsedJson = JSON.parse(rawResults);
- const jsonResult = cleanAndValidate(parsedJson);
- jsonPreview(jsonResult, previewCallback);
-
- return jsonResult;
+ const fileResults = await fileHandler({
+ file,
+ setFileProgress,
+ cleanAndValidate,
+ getFileParseActive
+ });
+ jsonPreview(fileResults, previewCallback);
+ return fileResults;
 }
-