Detect and remove unusual line terminators (fixes #96142)

src/vs/base/common/strings.ts

@@ -728,6 +728,14 @@ export function isBasicASCII(str: string): boolean {
  return IS_BASIC_ASCII.test(str);
 }
 
+export const UNUSUAL_LINE_TERMINATORS = /[\u2028\u2029\u0085]/; // LINE SEPARATOR (LS), PARAGRAPH SEPARATOR (PS), NEXT LINE (NEL)
+/**
+ * Returns true if `str` contains unusual line terminators, like LS, PS or NEL
+ */
+export function containsUnusualLineTerminators(str: string): boolean {
+ return UNUSUAL_LINE_TERMINATORS.test(str);
+}
+
 export function containsFullWidthCharacter(str: string): boolean {
  for (let i = 0, len = str.length; i < len; i++) {
  if (isFullWidthCharacter(str.charCodeAt(i))) {

src/vs/editor/common/config/editorOptions.ts

@@ -93,6 +93,11 @@ export interface IEditorOptions {
  * Defaults to true.
  */
  renderFinalNewline?: boolean;
+ /**
+ * Remove unusual line terminators like LINE SEPARATOR (LS), PARAGRAPH SEPARATOR (PS), NEXT LINE (NEL).
+ * Defaults to true.
+ */
+ removeUnusualLineTerminators?: boolean;
  /**
  * Should the corresponding line be selected when clicking on the line number?
  * Defaults to true.
@@ -3547,6 +3552,7 @@ export const enum EditorOption {
  quickSuggestions,
  quickSuggestionsDelay,
  readOnly,
+ removeUnusualLineTerminators,
  renameOnType,
  renderControlCharacters,
  renderIndentGuides,
@@ -3965,6 +3971,10 @@ export const EditorOptions = {
  readOnly: register(new EditorBooleanOption(
  EditorOption.readOnly, 'readOnly', false,
  )),
+ removeUnusualLineTerminators: register(new EditorBooleanOption(
+ EditorOption.removeUnusualLineTerminators, 'removeUnusualLineTerminators', true,
+ { description: nls.localize('removeUnusualLineTerminators', "Remove unusual line terminators that might cause problems.") }
+ )),
  renameOnType: register(new EditorBooleanOption(
  EditorOption.renameOnType, 'renameOnType', false,
  { description: nls.localize('renameOnType', "Controls whether the editor auto renames on type.") }

src/vs/editor/common/model.ts

@@ -551,6 +551,18 @@ export interface ITextModel {
  */
  mightContainRTL(): boolean;
 
+ /**
+ * If true, the text model might contain LINE SEPARATOR (LS), PARAGRAPH SEPARATOR (PS), NEXT LINE (NEL).
+ * If false, the text model definitely does not contain these.
+ * @internal
+ */
+ mightContainUnusualLineTerminators(): boolean;
+
+ /**
+ * @internal
+ */
+ removeUnusualLineTerminators(selections?: Selection[]): void;
+
  /**
  * If true, the text model might contain non basic ASCII.
  * If false, the text model **contains only** basic ASCII.
@@ -1281,6 +1293,8 @@ export interface IReadonlyTextBuffer {
  onDidChangeContent: Event<void>;
  equals(other: ITextBuffer): boolean;
  mightContainRTL(): boolean;
+ mightContainUnusualLineTerminators(): boolean;
+ resetMightContainUnusualLineTerminators(): void;
  mightContainNonBasicASCII(): boolean;
  getBOM(): string;
  getEOL(): string;

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts

@@ -36,15 +36,17 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
  private readonly _pieceTree: PieceTreeBase;
  private readonly _BOM: string;
  private _mightContainRTL: boolean;
+ private _mightContainUnusualLineTerminators: boolean;
  private _mightContainNonBasicASCII: boolean;
 
  private readonly _onDidChangeContent: Emitter<void> = new Emitter<void>();
  public readonly onDidChangeContent: Event<void> = this._onDidChangeContent.event;
 
- constructor(chunks: StringBuffer[], BOM: string, eol: '\r\n' | '\n', containsRTL: boolean, isBasicASCII: boolean, eolNormalized: boolean) {
+ constructor(chunks: StringBuffer[], BOM: string, eol: '\r\n' | '\n', containsRTL: boolean, containsUnusualLineTerminators: boolean, isBasicASCII: boolean, eolNormalized: boolean) {
  this._BOM = BOM;
  this._mightContainNonBasicASCII = !isBasicASCII;
  this._mightContainRTL = containsRTL;
+ this._mightContainUnusualLineTerminators = containsUnusualLineTerminators;
  this._pieceTree = new PieceTreeBase(chunks, eol, eolNormalized);
  }
  dispose(): void {
@@ -67,6 +69,12 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
  public mightContainRTL(): boolean {
  return this._mightContainRTL;
  }
+ public mightContainUnusualLineTerminators(): boolean {
+ return this._mightContainUnusualLineTerminators;
+ }
+ public resetMightContainUnusualLineTerminators(): void {
+ this._mightContainUnusualLineTerminators = false;
+ }
  public mightContainNonBasicASCII(): boolean {
  return this._mightContainNonBasicASCII;
  }
@@ -216,6 +224,7 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
 
  public applyEdits(rawOperations: ValidAnnotatedEditOperation[], recordTrimAutoWhitespace: boolean, computeUndoEdits: boolean): ApplyEditsResult {
  let mightContainRTL = this._mightContainRTL;
+ let mightContainUnusualLineTerminators = this._mightContainUnusualLineTerminators;
  let mightContainNonBasicASCII = this._mightContainNonBasicASCII;
  let canReduceOperations = true;
 
@@ -226,12 +235,20 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
  canReduceOperations = false;
  }
  let validatedRange = op.range;
- if (!mightContainRTL && op.text) {
- // check if the new inserted text contains RTL
- mightContainRTL = strings.containsRTL(op.text);
- }
- if (!mightContainNonBasicASCII && op.text) {
- mightContainNonBasicASCII = !strings.isBasicASCII(op.text);
+ if (op.text) {
+ let textMightContainNonBasicASCII = true;
+ if (!mightContainNonBasicASCII) {
+ textMightContainNonBasicASCII = !strings.isBasicASCII(op.text);
+ mightContainNonBasicASCII = textMightContainNonBasicASCII;
+ }
+ if (!mightContainRTL && textMightContainNonBasicASCII) {
+ // check if the new inserted text contains RTL
+ mightContainRTL = strings.containsRTL(op.text);
+ }
+ if (!mightContainUnusualLineTerminators && textMightContainNonBasicASCII) {
+ // check if the new inserted text contains unusual line terminators
+ mightContainUnusualLineTerminators = strings.containsUnusualLineTerminators(op.text);
+ }
  }
 
  let validText = '';
@@ -340,6 +357,7 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
 
 
  this._mightContainRTL = mightContainRTL;
+ this._mightContainUnusualLineTerminators = mightContainUnusualLineTerminators;
  this._mightContainNonBasicASCII = mightContainNonBasicASCII;
 
  const contentChanges = this._doApplyEdits(operations);

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder.ts

@@ -18,6 +18,7 @@ export class PieceTreeTextBufferFactory implements ITextBufferFactory {
  private readonly _lf: number,
  private readonly _crlf: number,
  private readonly _containsRTL: boolean,
+ private readonly _containsUnusualLineTerminators: boolean,
  private readonly _isBasicASCII: boolean,
  private readonly _normalizeEOL: boolean
  ) { }
@@ -53,7 +54,7 @@ export class PieceTreeTextBufferFactory implements ITextBufferFactory {
  }
  }
 
- return new PieceTreeTextBuffer(chunks, this._bom, eol, this._containsRTL, this._isBasicASCII, this._normalizeEOL);
+ return new PieceTreeTextBuffer(chunks, this._bom, eol, this._containsRTL, this._containsUnusualLineTerminators, this._isBasicASCII, this._normalizeEOL);
  }
 
  public getFirstLineText(lengthLimit: number): string {
@@ -73,6 +74,7 @@ export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
  private lf: number;
  private crlf: number;
  private containsRTL: boolean;
+ private containsUnusualLineTerminators: boolean;
  private isBasicASCII: boolean;
 
  constructor() {
@@ -87,6 +89,7 @@ export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
  this.lf = 0;
  this.crlf = 0;
  this.containsRTL = false;
+ this.containsUnusualLineTerminators = false;
  this.isBasicASCII = true;
  }
 
@@ -140,9 +143,13 @@ export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
  this.isBasicASCII = lineStarts.isBasicASCII;
  }
  if (!this.isBasicASCII && !this.containsRTL) {
- // No need to check if is basic ASCII
+ // No need to check if it is basic ASCII
  this.containsRTL = strings.containsRTL(chunk);
  }
+ if (!this.isBasicASCII && !this.containsUnusualLineTerminators) {
+ // No need to check if it is basic ASCII
+ this.containsUnusualLineTerminators = strings.containsUnusualLineTerminators(chunk);
+ }
  }
 
  public finish(normalizeEOL: boolean = true): PieceTreeTextBufferFactory {
@@ -154,6 +161,7 @@ export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
  this.lf,
  this.crlf,
  this.containsRTL,
+ this.containsUnusualLineTerminators,
  this.isBasicASCII,
  normalizeEOL
  );

src/vs/editor/common/model/textModel.ts

@@ -37,6 +37,7 @@ import { Color } from 'vs/base/common/color';
 import { EditorTheme } from 'vs/editor/common/view/viewContext';
 import { IUndoRedoService } from 'vs/platform/undoRedo/common/undoRedo';
 import { TextChange } from 'vs/editor/common/model/textChange';
+import { Constants } from 'vs/base/common/uint';
 
 function createTextBufferBuilder() {
  return new PieceTreeTextBufferBuilder();
@@ -699,6 +700,17 @@ export class TextModel extends Disposable implements model.ITextModel {
  return this._buffer.mightContainRTL();
  }
 
+ public mightContainUnusualLineTerminators(): boolean {
+ return this._buffer.mightContainUnusualLineTerminators();
+ }
+
+ public removeUnusualLineTerminators(selections: Selection[] | null = null): void {
+ const matches = this.findMatches(strings.UNUSUAL_LINE_TERMINATORS.source, false, true, false, null, false, Constants.MAX_SAFE_SMALL_INTEGER);
+ const eol = this.getEOL();
+ this._buffer.resetMightContainUnusualLineTerminators();
+ this.pushEditOperations(selections, matches.map(m => ({ range: m.range, text: eol })), () => null);
+ }
+
  public mightContainNonBasicASCII(): boolean {
  return this._buffer.mightContainNonBasicASCII();
  }
@@ -1097,7 +1109,7 @@ export class TextModel extends Disposable implements model.ITextModel {
  return this._buffer.findMatchesLineByLine(searchRange, searchData, captureMatches, limitResultCount);
  }
 
- public findMatches(searchString: string, rawSearchScope: any, isRegex: boolean, matchCase: boolean, wordSeparators: string, captureMatches: boolean, limitResultCount: number = LIMIT_FIND_COUNT): model.FindMatch[] {
+ public findMatches(searchString: string, rawSearchScope: any, isRegex: boolean, matchCase: boolean, wordSeparators: string | null, captureMatches: boolean, limitResultCount: number = LIMIT_FIND_COUNT): model.FindMatch[] {
  this._assertNotDisposed();
 
  let searchRange: Range;

src/vs/editor/common/standalone/standaloneEnums.ts

@@ -240,49 +240,50 @@ export enum EditorOption {
  quickSuggestions = 70,
  quickSuggestionsDelay = 71,
  readOnly = 72,
- renameOnType = 73,
- renderControlCharacters = 74,
- renderIndentGuides = 75,
- renderFinalNewline = 76,
- renderLineHighlight = 77,
- renderLineHighlightOnlyWhenFocus = 78,
- renderValidationDecorations = 79,
- renderWhitespace = 80,
- revealHorizontalRightPadding = 81,
- roundedSelection = 82,
- rulers = 83,
- scrollbar = 84,
- scrollBeyondLastColumn = 85,
- scrollBeyondLastLine = 86,
- scrollPredominantAxis = 87,
- selectionClipboard = 88,
- selectionHighlight = 89,
- selectOnLineNumbers = 90,
- showFoldingControls = 91,
- showUnused = 92,
- snippetSuggestions = 93,
- smoothScrolling = 94,
- stopRenderingLineAfter = 95,
- suggest = 96,
- suggestFontSize = 97,
- suggestLineHeight = 98,
- suggestOnTriggerCharacters = 99,
- suggestSelection = 100,
- tabCompletion = 101,
- useTabStops = 102,
- wordSeparators = 103,
- wordWrap = 104,
- wordWrapBreakAfterCharacters = 105,
- wordWrapBreakBeforeCharacters = 106,
- wordWrapColumn = 107,
- wordWrapMinified = 108,
- wrappingIndent = 109,
- wrappingStrategy = 110,
- editorClassName = 111,
- pixelRatio = 112,
- tabFocusMode = 113,
- layoutInfo = 114,
- wrappingInfo = 115
+ removeUnusualLineTerminators = 73,
+ renameOnType = 74,
+ renderControlCharacters = 75,
+ renderIndentGuides = 76,
+ renderFinalNewline = 77,
+ renderLineHighlight = 78,
+ renderLineHighlightOnlyWhenFocus = 79,
+ renderValidationDecorations = 80,
+ renderWhitespace = 81,
+ revealHorizontalRightPadding = 82,
+ roundedSelection = 83,
+ rulers = 84,
+ scrollbar = 85,
+ scrollBeyondLastColumn = 86,
+ scrollBeyondLastLine = 87,
+ scrollPredominantAxis = 88,
+ selectionClipboard = 89,
+ selectionHighlight = 90,
+ selectOnLineNumbers = 91,
+ showFoldingControls = 92,
+ showUnused = 93,
+ snippetSuggestions = 94,
+ smoothScrolling = 95,
+ stopRenderingLineAfter = 96,
+ suggest = 97,
+ suggestFontSize = 98,
+ suggestLineHeight = 99,
+ suggestOnTriggerCharacters = 100,
+ suggestSelection = 101,
+ tabCompletion = 102,
+ useTabStops = 103,
+ wordSeparators = 104,
+ wordWrap = 105,
+ wordWrapBreakAfterCharacters = 106,
+ wordWrapBreakBeforeCharacters = 107,
+ wordWrapColumn = 108,
+ wordWrapMinified = 109,
+ wrappingIndent = 110,
+ wrappingStrategy = 111,
+ editorClassName = 112,
+ pixelRatio = 113,
+ tabFocusMode = 114,
+ layoutInfo = 115,
+ wrappingInfo = 116
 }
 
 /**