[Tooling] Add --exclude flag to Generator to support field removal testing

CHANGELOG.next.md

@@ -22,6 +22,7 @@ Thanks, you're awesome :-) -->
 #### Improvements
 
 * Fix ecs GitHub repo link source branch #1393
+* Add --exclude flag to Generator to support field removal testing #1411
 
 #### Deprecated
 

rfcs/text/0017/exclude-set.yml

@@ -0,0 +1,5 @@
+---
+- name: log
+ fields:
+ - name: original
+

scripts/generator.py

@@ -15,6 +15,7 @@
 from schema import cleaner
 from schema import finalizer
 from schema import subset_filter
+from schema import exclude_filter
 
 
 def main():
@@ -48,6 +49,7 @@ def main():
  cleaner.clean(fields, strict=args.strict)
  finalizer.finalize(fields)
  fields = subset_filter.filter(fields, args.subset, out_dir)
+ fields = exclude_filter.exclude(fields, args.exclude, out_dir)
  nested, flat = intermediate_files.generate(fields, os.path.join(out_dir, 'ecs'), default_dirs)
 
  if args.intermediate_only:
@@ -70,6 +72,8 @@ def argument_parser():
  Note that "--include experimental/schemas" will also respect this git ref.')
  parser.add_argument('--include', nargs='+',
  help='include user specified directory of custom field definitions')
+ parser.add_argument('--exclude', nargs='+',
+ help='exclude user specified subset of the schema')
  parser.add_argument('--subset', nargs='+',
  help='render a subset of the schema')
  parser.add_argument('--out', action='store', help='directory to output the generated files')

scripts/schema/exclude_filter.py

@@ -0,0 +1,62 @@
+import glob
+import yaml
+import os
+from schema import loader
+
+# This script should be run downstream of the subset filters - it takes
+# all ECS and custom fields already loaded by the latter and explicitly
+# removes a subset, for example, to simulate impact of future removals
+
+
+def exclude(fields, exclude_file_globs, out_dir):
+ excludes = load_exclude_definitions(exclude_file_globs)
+
+ if excludes:
+ fields = exclude_fields(fields, excludes)
+
+ return fields
+
+
+def pop_field(fields, node_path, path):
+ """pops a field from yaml derived dict using path derived from ordered list of nodes"""
+ if node_path[0] in fields:
+ if len(node_path) == 1:
+ fields.pop(node_path[0])
+ else:
+ inner_field = node_path.pop(0)
+ if 'fields' in fields[inner_field]:
+ pop_field(fields[inner_field]['fields'], node_path, path)
+ else:
+ raise ValueError(
+ '--exclude specified, but no path to field {} found'.format('.'.join([e for e in path])))
+ else:
+ raise ValueError('--exclude specified, but no field {} found'.format('.'.join([e for e in path])))
+
+
+def exclude_trace_path(fields, item, path):
+ """traverses paths to one or more nodes in a yaml derived dict"""
+ for list_item in item:
+ node_path = path.copy()
+ node_path.append(list_item['name'])
+ if not 'fields' in list_item:
+ pop_field(fields, node_path, node_path.copy())
+ else:
+ exclude_trace_path(fields, list_item['fields'], node_path)
+
+
+def exclude_fields(fields, excludes):
+ """Traverses fields and eliminates any field which matches the excludes"""
+ if excludes:
+ for ex_list in excludes:
+ for item in ex_list:
+ exclude_trace_path(fields, item['fields'], [item['name']])
+ return fields
+
+
+def load_exclude_definitions(file_globs):
+ if not file_globs:
+ return []
+ excludes = loader.load_definitions(file_globs)
+ if not excludes:
+ raise ValueError('--exclude specified, but no exclusions found in {}'.format(file_globs))
+ return excludes

scripts/schema/loader.py

@@ -251,3 +251,33 @@ def merge_fields(a, b):
  a[key].setdefault('fields', {})
  a[key]['fields'] = merge_fields(a[key]['fields'], b[key]['fields'])
  return a
+
+
+def load_yaml_file(file_name):
+ with open(file_name) as f:
+ return yaml.safe_load(f.read())
+
+
+# You know, for silent tests
+def warn(message):
+ print(message)
+
+
+def eval_globs(globs):
+ '''Accepts an array of glob patterns or file names, returns the array of actual files'''
+ all_files = []
+ for g in globs:
+ new_files = glob.glob(g)
+ if len(new_files) == 0:
+ warn("{} did not match any files".format(g))
+ else:
+ all_files.extend(new_files)
+ return all_files
+
+
+def load_definitions(file_globs):
+ sets = []
+ for f in eval_globs(file_globs):
+ raw = load_yaml_file(f)
+ sets.append(raw)
+ return sets

scripts/schema/subset_filter.py

@@ -2,7 +2,7 @@
 import yaml
 import os
 from generators import intermediate_files
-from schema import cleaner
+from schema import cleaner, loader
 
 # This script takes all ECS and custom fields already loaded, and lets users
 # filter out the ones they don't need.
@@ -33,37 +33,12 @@ def combine_all_subsets(subsets):
 def load_subset_definitions(file_globs):
  if not file_globs:
  return []
- subsets = []
- for f in eval_globs(file_globs):
- raw = load_yaml_file(f)
- subsets.append(raw)
+ subsets = loader.load_definitions(file_globs)
  if not subsets:
  raise ValueError('--subset specified, but no subsets found in {}'.format(file_globs))
  return subsets
 
 
-def load_yaml_file(file_name):
- with open(file_name) as f:
- return yaml.safe_load(f.read())
-
-
-def eval_globs(globs):
- '''Accepts an array of glob patterns or file names, returns the array of actual files'''
- all_files = []
- for g in globs:
- new_files = glob.glob(g)
- if len(new_files) == 0:
- warn("{} did not match any files".format(g))
- else:
- all_files.extend(new_files)
- return all_files
-
-
-# You know, for silent tests
-def warn(message):
- print(message)
-
-
 ecs_options = ['fields', 'enabled', 'index']
 
 

scripts/tests/unit/test_schema_exclude_filter.py

@@ -0,0 +1,97 @@
+from unittest import result
+from unittest.case import TestCase
+from schema import exclude_filter
+import mock
+import os
+import pprint
+import sys
+import unittest
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
+
+
+class TestSchemaExcludeFilter(unittest.TestCase):
+
+ def setUp(self):
+ self.maxDiff = None
+
+ @mock.patch('schema.loader.warn')
+ def test_load_exclude_definitions_raises_when_no_exclude_found(self, mock_warn):
+ with self.assertRaisesRegex(ValueError,
+ "--exclude specified, but no exclusions found in \['foo\*.yml'\]"):
+ exclude_filter.load_exclude_definitions(['foo*.yml'])
+
+ def test_exclude_field(self):
+ fields = {'my_field_set': {'fields': {
+ 'my_field_exclude': {'field_details': {'flat_name': 'my_field_set.my_field_exclude'}},
+ 'my_field_persist': {'field_details': {'flat_name': 'my_field_set.my_field_persist'}}}}}
+ excludes = [
+ [{'name': 'my_field_set', 'fields': [{'name': 'my_field_exclude'}]}]]
+ fields = exclude_filter.exclude_fields(fields, excludes)
+ expect_persisted = {'my_field_set': {'fields': {
+ 'my_field_persist': {'field_details': {'flat_name': 'my_field_set.my_field_persist'}}}}}
+ self.assertEqual(fields, expect_persisted)
+
+ def test_exclude_field_deep_path(self):
+ fields = {'d0': {'fields': {
+ 'd1': {'field_details': {'flat_name': 'd0.d1'}, 'fields': {
+ 'd2': {'field_details': {'flat_name': 'd0.d1.d2'}, 'fields': {
+ 'd3': {'field_details': {'flat_name': 'd0.d1.d2.d3'}, 'fields': {
+ 'd4': {'field_details': {'flat_name': 'd0.d1.d2.d3.d4'}, 'fields': {
+ 'd5': {'field_details': {'flat_name': 'd0.d1.d2.d3.d4.d5'}}}}}}}}}}}}}
+ excludes = [[{'name': 'd0', 'fields': [{
+ 'name': 'd1', 'fields': [{
+ 'name': 'd2', 'fields': [{
+ 'name': 'd3', 'fields': [{
+ 'name': 'd4', 'fields': [{
+ 'name': 'd5'}]}]}]}]}]}]]
+ fields = exclude_filter.exclude_fields(fields, excludes)
+ expect_persisted = {'d0': {'fields': {
+ 'd1': {'field_details': {'flat_name': 'd0.d1'}, 'fields': {
+ 'd2': {'field_details': {'flat_name': 'd0.d1.d2'}, 'fields': {
+ 'd3': {'field_details': {'flat_name': 'd0.d1.d2.d3'}, 'fields': {
+ 'd4': {'field_details': {'flat_name': 'd0.d1.d2.d3.d4'}, 'fields': {}}}}}}}}}}}
+ self.assertEqual(fields, expect_persisted)
+
+ def test_exclude_fields(self):
+ fields = {'my_field_set': {'fields': {
+ 'my_field_exclude_1': {'field_details': {'flat_name': 'my_field_set.my_field_exclude_1'}},
+ 'my_field_exclude_2': {'field_details': {'flat_name': 'my_field_set.my_field_exclude_2'}}}}}
+ excludes = [[{'name': 'my_field_set', 'fields': [
+ {'name': 'my_field_exclude_1'}, {'name': 'my_field_exclude_2'}]}]]
+ fields = exclude_filter.exclude_fields(fields, excludes)
+ expect_persisted = {'my_field_set': {'fields': {}}}
+ self.assertEqual(fields, expect_persisted)
+
+ def test_exclude_non_existing_field_set(self):
+ fields = {'my_field_set': {'fields': {
+ 'my_field': {'field_details': {'flat_name': 'my_field_set.my_field'}}}}}
+ excludes = [[{'name': 'my_non_existing_field_set', 'fields': [
+ {'name': 'my_field_exclude'}]}]]
+ with self.assertRaisesRegex(ValueError,
+ "--exclude specified, but no field my_non_existing_field_set.my_field_exclude found"):
+ exclude_filter.exclude_fields(fields, excludes)
+
+ def test_exclude_non_existing_field(self):
+ fields = {'my_field_set': {'fields': {
+ 'my_field': {'field_details': {'flat_name': 'my_field_set.my_field'}}}}}
+ excludes = [[{'name': 'my_field_set', 'fields': [
+ {'name': 'my_non_existing_field'}]}]]
+ with self.assertRaisesRegex(ValueError,
+ "--exclude specified, but no field my_field_set.my_non_existing_field found"):
+ exclude_filter.exclude_fields(fields, excludes)
+
+ def test_exclude_non_existing_field_deep_path(self):
+ fields = {'d0': {'fields': {
+ 'd1': {'field_details': {'flat_name': 'd0.d1'}}, 'fields': {
+ 'd2': {'field_details': {'flat_name': 'd0.d1.d2'}}, 'fields': {
+ 'd3': {'field_details': {'flat_name': 'd0.d1.d2.d3'}}}}}}}
+ excludes = [[{'name': 'd0', 'fields': [{
+ 'name': 'd1', 'fields': [{
+ 'name': 'd2', 'fields': [{
+ 'name': 'd3', 'fields': [{
+ 'name': 'd4', 'fields': [{
+ 'name': 'd5'}]}]}]}]}]}]]
+ with self.assertRaisesRegex(ValueError,
+ "--exclude specified, but no path to field d0.d1.d2.d3.d4.d5 found"):
+ exclude_filter.exclude_fields(fields, excludes)

scripts/tests/unit/test_schema_loader.py

@@ -14,6 +14,14 @@ class TestSchemaLoader(unittest.TestCase):
  def setUp(self):
  self.maxDiff = None
 
+ @mock.patch('schema.loader.warn')
+ def test_eval_globs(self, mock_warn):
+ files = loader.eval_globs(['schemas/*.yml', 'missing*'])
+ self.assertTrue(mock_warn.called, "a warning should have been printed for missing*")
+ self.assertIn('schemas/base.yml', files)
+ self.assertEqual(list(filter(lambda f: f.startswith('missing'), files)), [],
+ "The 'missing*' pattern should not show up in the resulting files")
+
  # Pseudo-fixtures
 
  def schema_base(self):

scripts/tests/unit/test_schema_subset_filter.py

@@ -14,15 +14,7 @@ class TestSchemaSubsetFilter(unittest.TestCase):
  def setUp(self):
  self.maxDiff = None
 
- @mock.patch('schema.subset_filter.warn')
- def test_eval_globs(self, mock_warn):
- files = subset_filter.eval_globs(['schemas/*.yml', 'missing*'])
- self.assertTrue(mock_warn.called, "a warning should have been printed for missing*")
- self.assertIn('schemas/base.yml', files)
- self.assertEqual(list(filter(lambda f: f.startswith('missing'), files)), [],
- "The 'missing*' pattern should not show up in the resulting files")
-
- @mock.patch('schema.subset_filter.warn')
+ @mock.patch('schema.loader.warn')
  def test_load_subset_definitions_raises_when_no_subset_found(self, mock_warn):
  with self.assertRaisesRegex(ValueError,
  "--subset specified, but no subsets found in \['foo\*.yml'\]"):