Add nori_number token filter in analysis-nori and discard_punctuation to filter

src/Nest/Analysis/Analyzers/NoriAnalyzer.cs

@@ -8,6 +8,7 @@ namespace Nest
  /// <para> - nori_tokenizer</para>
  /// <para> - nori_part_of_speech token filter</para>
  /// <para> - nori_readingform token filter</para>
+ /// <para> - nori_number token filter</para>
  /// <para> - lowercase token filter</para>
  /// </summary>
  public interface INoriAnalyzer : IAnalyzer

src/Nest/Analysis/Tokenizers/NoriTokenizer.cs

@@ -30,6 +30,12 @@ public interface INoriTokenizer : ITokenizer
  [DataMember(Name = "decompound_mode")]
  NoriDecompoundMode? DecompoundMode { get; set; }
 
+ /// <summary>
+ /// Whether punctuation should be discarded from the output. Defaults to `true`.
+ /// </summary>
+ [DataMember(Name = "discard_punctuation")]
+ bool? DiscardPunctuation { get; set; }
+
  /// <summary>
  /// The Nori tokenizer uses the mecab-ko-dic dictionary by default. A user_dictionary with custom nouns (NNG) may be
  /// appended to
@@ -57,6 +63,9 @@ public class NoriTokenizer : TokenizerBase, INoriTokenizer
  /// <inheritdoc cref="INoriTokenizer.DecompoundMode" />
  public NoriDecompoundMode? DecompoundMode { get; set; }
 
+ /// <inheritdoc cref="INoriTokenizer.DiscardPunctuation" />
+ public bool? DiscardPunctuation { get; set; }
+
  /// <inheritdoc cref="INoriTokenizer.UserDictionary" />
  public string UserDictionary { get; set; }
 
@@ -73,6 +82,7 @@ public class NoriTokenizerDescriptor
  NoriDecompoundMode? INoriTokenizer.DecompoundMode { get; set; }
  string INoriTokenizer.UserDictionary { get; set; }
  IEnumerable<string> INoriTokenizer.UserDictionaryRules { get; set; }
+ bool? INoriTokenizer.DiscardPunctuation { get; set; }
 
  /// <inheritdoc cref="INoriTokenizer.DecompoundMode" />
  public NoriTokenizerDescriptor DecompoundMode(NoriDecompoundMode? mode) => Assign(mode, (a, v) => a.DecompoundMode = v);
@@ -85,5 +95,8 @@ public class NoriTokenizerDescriptor
 
  /// <inheritdoc cref="INoriTokenizer.UserDictionaryRules" />
  public NoriTokenizerDescriptor UserDictionaryRules(IEnumerable<string> rules) => Assign(rules, (a, v) => a.UserDictionaryRules = v);
+
+ /// <inheritdoc cref="INoriTokenizer.DiscardPunctuation" />
+ public NoriTokenizerDescriptor DiscardPunctuation(bool? discard = true) => Assign(discard, (a, v) => a.DiscardPunctuation = v);
  }
 }

tests/Tests/Analysis/Tokenizers/TokenizerTests.cs

@@ -332,5 +332,21 @@ public class CharGroupTests : TokenizerAssertionBase<CharGroupTests>
 
  public override string Name => "char_group";
  }
+
+ [SkipVersion("<7.7.0", "discard_punctuation introduced in 7.7.0")]
+ public class DiscardPunctuationTests : TokenizerAssertionBase<DiscardPunctuationTests>
+ {
+ public override FuncTokenizer Fluent => (n, t) => t.Nori(n, e => e
+ .DiscardPunctuation()
+ );
+
+ public override ITokenizer Initializer => new NoriTokenizer
+ {
+ DiscardPunctuation = true
+ };
+
+ public override object Json => new { type = "nori_tokenizer", discard_punctuation = true };
+ public override string Name => "nori";
+ }
  }
 }