fix(cleanup): ensure usage of unicode elipsis when it makes sense (#274)

common-voice · Jan 19, 2020 · 1c380e0 · 1c380e0
1 parent 598c05b
commit 1c380e0
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 0 deletions.
diff --git a/shared/cleanup/languages/fr.js b/shared/cleanup/languages/fr.js
@@ -23,6 +23,9 @@ export function clean(sentences) {
  .replace(/^,+/g, '') // we do not want commas at the beginning of the sentence
  .replace(/,+$/g, '') // we do not want commas at the end of the sentence
 
+ // Normalize three consecutive dots into unicode elipsis
+ .replace(/\.{3}/g, '…')
+
  // In fr-FR, those should have a no space before
  .replace(/\s+,/g, ',')
  .replace(/\s+\./g, '.')

diff --git a/test/shared/cleanup/languages/fr.js b/test/shared/cleanup/languages/fr.js
@@ -149,6 +149,22 @@ test('[FR] space before/space after double punctuation', t => {
  t.deepEqual(computed, expected);
 });
 
+test('[FR] merge consecutive dots into unicode elipsis', t => {
+ const orig = [
+ 'this is .. wrong',
+ 'this is ... wrong',
+ ];
+
+ const expected = [
+ 'this is. . wrong',
+ 'this is… wrong',
+ ];
+
+ const computed = fr.clean(orig);
+
+ t.deepEqual(computed, expected);
+});
+
 test('[FR] do not keep multiple spaces', t => {
  const orig = [
  'this is wrong',