-
Notifications
You must be signed in to change notification settings - Fork 4.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Improve fields.yml generator of modules #7533
Changes from all commits
29b013d
e97c177
19660e3
afb2d5f
2bd678e
07b59df
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -33,6 +33,10 @@ import ( | |
const ( | ||
pipelinePath = "%s/module/%s/%s/ingest/pipeline.json" | ||
fieldsYmlPath = "%s/module/%s/%s/_meta/fields.yml" | ||
|
||
typeIdx = 0 | ||
elementsIdx = 1 | ||
hintIdx = 2 | ||
) | ||
|
||
var ( | ||
|
@@ -64,8 +68,9 @@ type pipeline struct { | |
} | ||
|
||
type field struct { | ||
Type string | ||
Elements []string | ||
Syntax string | ||
SemanticElements []string | ||
Type string | ||
} | ||
|
||
type fieldYml struct { | ||
|
@@ -92,23 +97,41 @@ func newFieldYml(name, typeName string, noDoc bool) *fieldYml { | |
} | ||
} | ||
|
||
func newField(lp string) field { | ||
lp = lp[1 : len(lp)-1] | ||
ee := strings.Split(lp, ":") | ||
if len(ee) != 2 { | ||
return field{ | ||
Type: ee[0], | ||
Elements: nil, | ||
} | ||
func newField(pattern string) field { | ||
if len(pattern) <= 2 { | ||
return field{} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we log errors instead of return empty fields? This would make debugging much easier in the future. |
||
} | ||
pattern = pattern[1 : len(pattern)-1] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Alternative here would |
||
|
||
elements := strings.Split(pattern, ":") | ||
if !isValidFormat(elements) { | ||
return field{} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Return error? |
||
} | ||
|
||
hint := "" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What about having a function |
||
if containsType(elements) { | ||
hint = elements[hintIdx] | ||
} | ||
|
||
e := strings.Split(ee[1], ".") | ||
return field{ | ||
Type: ee[0], | ||
Elements: e, | ||
Syntax: elements[typeIdx], | ||
SemanticElements: strings.Split(elements[elementsIdx], "."), | ||
Type: hint, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The variable should be called |
||
} | ||
} | ||
|
||
// isValidFormat checks if the input can be split correctly | ||
// 1. if lenght is 2, the format is {type}:{field.elements} | ||
// 2. if the lenght is 3, the format is {type}:{field.elements}:{hint} | ||
func isValidFormat(ee []string) bool { | ||
return len(ee) == 2 || len(ee) == 3 | ||
} | ||
|
||
// the last element is the type hint | ||
func containsType(ee []string) bool { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should this be |
||
return len(ee) == 3 | ||
} | ||
|
||
func readPipeline(beatsPath, module, fileset string) (*pipeline, error) { | ||
pp := fmt.Sprintf(pipelinePath, beatsPath, module, fileset) | ||
r, err := ioutil.ReadFile(pp) | ||
|
@@ -134,7 +157,7 @@ func addNewField(fs []field, f field) []field { | |
return append(fs, f) | ||
} | ||
|
||
func getElementsFromPatterns(patterns []string) ([]field, error) { | ||
func getSemanticElementsFromPatterns(patterns []string) ([]field, error) { | ||
r, err := regexp.Compile("{[\\.\\w\\:]*}") | ||
if err != nil { | ||
return nil, err | ||
|
@@ -145,7 +168,7 @@ func getElementsFromPatterns(patterns []string) ([]field, error) { | |
pp := r.FindAllString(lp, -1) | ||
for _, p := range pp { | ||
f := newField(p) | ||
if f.Elements == nil { | ||
if f.SemanticElements == nil { | ||
continue | ||
} | ||
fs = addNewField(fs, f) | ||
|
@@ -214,16 +237,16 @@ type processors struct { | |
} | ||
|
||
func (p *processors) processFields() ([]field, error) { | ||
f, err := getElementsFromPatterns(p.patterns) | ||
f, err := getSemanticElementsFromPatterns(p.patterns) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
for i, ff := range f { | ||
fs := strings.Join(ff.Elements, ".") | ||
fs := strings.Join(ff.SemanticElements, ".") | ||
for k, mv := range p.rename { | ||
if k == fs { | ||
ff.Elements = strings.Split(mv, ".") | ||
ff.SemanticElements = strings.Split(mv, ".") | ||
} | ||
} | ||
for _, rm := range p.remove { | ||
|
@@ -275,33 +298,37 @@ func getFieldByName(f []*fieldYml, name string) *fieldYml { | |
return nil | ||
} | ||
|
||
func insertLastField(f []*fieldYml, name, typeName string, noDoc bool) []*fieldYml { | ||
func insertLastField(f []*fieldYml, name string, field field, noDoc bool) []*fieldYml { | ||
ff := getFieldByName(f, name) | ||
if ff != nil { | ||
return f | ||
} | ||
|
||
nf := newFieldYml(name, types[typeName], noDoc) | ||
fieldType := field.Type | ||
if fieldType == "" { | ||
fieldType = types[field.Syntax] | ||
} | ||
nf := newFieldYml(name, fieldType, noDoc) | ||
return append(f, nf) | ||
} | ||
|
||
func insertGroup(out []*fieldYml, field field, index, count int, noDoc bool) []*fieldYml { | ||
g := getFieldByName(out, field.Elements[index]) | ||
g := getFieldByName(out, field.SemanticElements[index]) | ||
if g != nil { | ||
g.Fields = generateField(g.Fields, field, index+1, count, noDoc) | ||
return out | ||
} | ||
|
||
var groupFields []*fieldYml | ||
groupFields = generateField(groupFields, field, index+1, count, noDoc) | ||
group := newFieldYml(field.Elements[index], "group", noDoc) | ||
group := newFieldYml(field.SemanticElements[index], "group", noDoc) | ||
group.Fields = groupFields | ||
return append(out, group) | ||
} | ||
|
||
func generateField(out []*fieldYml, field field, index, count int, noDoc bool) []*fieldYml { | ||
if index+1 == count { | ||
return insertLastField(out, field.Elements[index], field.Type, noDoc) | ||
return insertLastField(out, field.SemanticElements[index], field, noDoc) | ||
} | ||
return insertGroup(out, field, index, count, noDoc) | ||
} | ||
|
@@ -310,10 +337,10 @@ func generateFields(f []field, noDoc bool) []*fieldYml { | |
var out []*fieldYml | ||
for _, ff := range f { | ||
index := 1 | ||
if len(ff.Elements) == 1 { | ||
if len(ff.SemanticElements) == 1 { | ||
index = 0 | ||
} | ||
out = generateField(out, ff, index, len(ff.Elements), noDoc) | ||
out = generateField(out, ff, index, len(ff.SemanticElements), noDoc) | ||
} | ||
return out | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Isn't that the syntax index?