Skip to content

Commit

Permalink
Simplify fields.yml generator (elastic#7713)
Browse files Browse the repository at this point in the history
This PR simplifies the process of generating a new `fields.yml` file for a Beat.
* `libbeat/fields.yml` is not a dependency anymore
* only `fields.yml` is written to disk

Changes in the asset files are due to different number of newlines in generated `fields.yml` files.

### Interface changes of generator tools
Both `dev-tools/cmd/asset/asset.go` and `libbeat/scripts/cmd/global_fields/main.go` is changed. The first one has two new flags: `-in`, and `-out`. By default it reads from stdin and writes to stdout. Using the flags it's possible to read and write from other files. The second one has one new flag `-out` which specifies the output file. If it's not set, the output is written to stdout.

This makes it possible to generate asset files for Metricbeat without writing `libbeat/fields.yml` to disk. It will be useful in the future when more Beats have modular assets.

Closes elastic#7671
(cherry picked from commit 5797005)
  • Loading branch information
kvch committed Sep 25, 2018
1 parent ed7bb7b commit 5c25fbb
Show file tree
Hide file tree
Showing 25 changed files with 157 additions and 2,246 deletions.
2 changes: 1 addition & 1 deletion auditbeat/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ include ${ES_BEATS}/libbeat/scripts/Makefile

# Collects all dependencies and then calls update
.PHONY: collect
collect: fields collect-docs configs kibana
collect: collect-docs configs kibana

# Collects all module configs
.PHONY: configs
Expand Down
2 changes: 1 addition & 1 deletion auditbeat/include/fields.go

Large diffs are not rendered by default.

58 changes: 42 additions & 16 deletions dev-tools/cmd/asset/asset.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package main

import (
"bufio"
"bytes"
"flag"
"fmt"
Expand All @@ -30,33 +31,54 @@ import (
"github.com/elastic/beats/libbeat/asset"
)

var pkg *string
var (
pkg string
input string
output string
)

func init() {
pkg = flag.String("pkg", "", "Package name")
flag.StringVar(&pkg, "pkg", "", "Package name")
flag.StringVar(&input, "in", "-", "Source of input. \"-\" means reading from stdin")
flag.StringVar(&output, "out", "-", "Output path. \"-\" means writing to stdout")
}

func main() {
flag.Parse()

args := flag.Args()
if len(args) != 2 {
fmt.Fprintln(os.Stderr, "File path must be set")
os.Exit(1)
}

file := args[0]
beatName := args[1]
var (
file, beatName string
data []byte
err error
)
if input == "-" {
if len(args) != 2 {
fmt.Fprintln(os.Stderr, "File path must be set")
os.Exit(1)
}
file = args[0]
beatName = args[1]

data, err := ioutil.ReadFile(file)
if err != nil {
fmt.Fprintln(os.Stderr, "Invalid file path: %s", args[0])
os.Exit(1)
r := bufio.NewReader(os.Stdin)
data, err = ioutil.ReadAll(r)
if err != nil {
fmt.Fprintf(os.Stderr, "Error while reading from stdin: %v\n", err)
os.Exit(1)
}
} else {
file = input
beatName = args[0]
data, err = ioutil.ReadFile(input)
if err != nil {
fmt.Fprintf(os.Stderr, "Invalid file path: %s\n", input)
os.Exit(1)
}
}

encData, err := asset.EncodeData(string(data))
if err != nil {
fmt.Fprintln(os.Stderr, "Error encoding the data: %s", err)
fmt.Fprintf(os.Stderr, "Error encoding the data: %s\n", err)
os.Exit(1)
}

Expand All @@ -65,13 +87,17 @@ func main() {
Beat: beatName,
Name: file,
Data: encData,
Package: *pkg,
Package: pkg,
})

bs, err := format.Source(buf.Bytes())
if err != nil {
panic(err)
}

os.Stdout.Write(bs)
if output == "-" {
os.Stdout.Write(bs)
} else {
ioutil.WriteFile(output, bs, 0640)
}
}
5 changes: 0 additions & 5 deletions dev-tools/jenkins_ci.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,6 @@ $env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile"
exec { go install github.com/elastic/beats/vendor/github.com/magefile/mage }
exec { go get -u github.com/jstemmer/go-junit-report }

echo "Building libbeat fields.yml"
cd libbeat
exec { mage fields }
cd ..

if (Test-Path "$env:beat") {
cd "$env:beat"
} else {
Expand Down
1 change: 1 addition & 0 deletions dev-tools/mage/fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func GenerateFieldsYAML(fieldsFiles ...string) error {
filepath.Join(beatsDir, globalFieldsCmdPath),
"-es_beats_path", beatsDir,
"-beat_path", CWD(),
"-out", "fields.yml",
)

return globalFieldsCmd(fieldsFiles...)
Expand Down
2 changes: 1 addition & 1 deletion filebeat/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ imports: python-env

# Runs all collection steps and updates afterwards
.PHONY: collect
collect: fields kibana configs collect-docs imports
collect: kibana configs collect-docs imports

# Creates a new module. Requires the params MODULE
.PHONY: create-module
Expand Down
2 changes: 1 addition & 1 deletion filebeat/include/fields.go

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion generator/beat/{beat}/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,4 @@ git-add:

# Collects all dependencies and then calls update
.PHONY: collect
collect: fields
collect:
3 changes: 1 addition & 2 deletions generator/metricbeat/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,4 @@ prepare-test:: python-env

# Collects all dependencies and then calls update
.PHONY: collect
collect: fields

collect:
2 changes: 1 addition & 1 deletion heartbeat/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ TEST_ENVIRONMENT=false

# Collects all dependencies and then calls update
.PHONY: collect
collect: fields imports kibana
collect: imports kibana

# Generate imports for all monitors
.PHONY: imports
Expand Down
2 changes: 1 addition & 1 deletion heartbeat/include/fields.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion libbeat/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ include scripts/Makefile

# Collects all dependencies and then calls update
.PHONY: collect
collect: libbeat_fields
collect:
142 changes: 49 additions & 93 deletions libbeat/generator/fields/fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,9 @@ package fields
import (
"bufio"
"bytes"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"

"github.com/pkg/errors"
)

var (
generatedFieldsYml = filepath.Join("_meta", "fields.generated.yml")
)

// YmlFile holds the info on files and how to write them into the global fields.yml
Expand All @@ -39,37 +31,67 @@ type YmlFile struct {
Indent int
}

func collectBeatFiles(beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
commonFields := filepath.Join(beatPath, "_meta", "fields.common.yml")
_, err := os.Stat(commonFields)
if os.IsNotExist(err) {
return fieldFiles, nil
} else if err != nil {
return nil, err
func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
commonFields := []string{
filepath.Join(beatPath, "_meta/fields.common.yml"),
}

var libbeatFieldFiles []*YmlFile
var err error
if !isLibbeat(beatPath) {
commonFields = append(commonFields,
filepath.Join(esBeatsPath, "libbeat/_meta/fields.common.yml"),
)

libbeatModulesPath := filepath.Join(esBeatsPath, "libbeat/processors")
libbeatFieldFiles, err = CollectModuleFiles(libbeatModulesPath)
if err != nil {
return nil, err
}
}

files := []*YmlFile{
{
Path: commonFields,
var files []*YmlFile
for _, cf := range commonFields {
_, err := os.Stat(cf)
if os.IsNotExist(err) {
continue
} else if err != nil {
return nil, err
}
files = append(files, &YmlFile{
Path: cf,
Indent: 0,
},
})
}

files = append(files, libbeatFieldFiles...)

return append(files, fieldFiles...), nil
}

func writeGeneratedFieldsYml(beatsPath string, fieldFiles []*YmlFile) error {
outPath := path.Join(beatsPath, generatedFieldsYml)
f, err := os.Create(outPath)
func isLibbeat(beatPath string) bool {
return filepath.Base(beatPath) == "libbeat"
}

func writeGeneratedFieldsYml(beatPath string, fieldFiles []*YmlFile, output string) error {
data, err := GenerateFieldsYml(fieldFiles)
if err != nil {
return err
}
defer f.Close()

data, err := GenerateFieldsYml(fieldFiles)
if output == "-" {
fw := bufio.NewWriter(os.Stdout)
fw.Write(data)
return fw.Flush()
}

outPath := filepath.Join(beatPath, output)
f, err := os.Create(outPath)
if err != nil {
return err
}
defer f.Close()

fw := bufio.NewWriter(f)
fw.Write(data)
return fw.Flush()
Expand Down Expand Up @@ -106,77 +128,11 @@ func writeIndentedLine(buf *bytes.Buffer, line string, indent int) error {
}

// Generate collects fields.yml files and concatenates them into one global file.
func Generate(esBeatsPath, beatPath string, files []*YmlFile) error {
files, err := collectBeatFiles(beatPath, files)
if err != nil {
return err
}

err = writeGeneratedFieldsYml(beatPath, files)
if err != nil {
return err
}

return AppendFromLibbeat(esBeatsPath, beatPath)
}

// AppendFromLibbeat appends fields.yml of libbeat to the fields.yml
func AppendFromLibbeat(esBeatsPath, beatPath string) error {
fieldsMetaPath := path.Join(beatPath, "_meta", "fields.yml")
generatedPath := path.Join(beatPath, generatedFieldsYml)

err := createIfNotExists(fieldsMetaPath, generatedPath)
if err != nil {
return err
}

if isLibbeat(beatPath) {
out := filepath.Join(esBeatsPath, "libbeat", "fields.yml")
return copyFileWithFlag(generatedPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
}

libbeatPath := filepath.Join(esBeatsPath, "libbeat", generatedFieldsYml)
out := filepath.Join(beatPath, "fields.yml")
err = copyFileWithFlag(libbeatPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
func Generate(esBeatsPath, beatPath string, files []*YmlFile, output string) error {
files, err := collectCommonFiles(esBeatsPath, beatPath, files)
if err != nil {
return err
}
return copyFileWithFlag(generatedPath, out, os.O_WRONLY|os.O_APPEND)
}

func isLibbeat(beatPath string) bool {
return filepath.Base(beatPath) == "libbeat"
}

func createIfNotExists(inPath, outPath string) error {
_, err := os.Stat(outPath)
if os.IsNotExist(err) {
err := copyFileWithFlag(inPath, outPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
if err != nil {
return err
}
return nil
}
return err
}

func copyFileWithFlag(in, out string, flag int) error {
input, err := ioutil.ReadFile(in)
if err != nil {
return errors.Wrap(err, "failed to read source in copy")
}

if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
return errors.Wrapf(err, "failed to create destination dir for copy "+
"at %v", filepath.Dir(out))
}

output, err := os.OpenFile(out, flag, 0644)
if err != nil {
return errors.Wrap(err, "failed to open destination file for copy")
}
defer output.Close()

_, err = output.Write(input)
return err
return writeGeneratedFieldsYml(beatPath, files, output)
}
4 changes: 2 additions & 2 deletions libbeat/generator/fields/module_fields_collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func CollectModuleFiles(modulesDir string) ([]*YmlFile, error) {
func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {

var files []*YmlFile
fieldsYmlPath := filepath.Join(modulesPath, module, "_meta", "fields.yml")
fieldsYmlPath := filepath.Join(modulesPath, module, "_meta/fields.yml")
if _, err := os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
files = append(files, &YmlFile{
Path: fieldsYmlPath,
Expand All @@ -91,7 +91,7 @@ func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {
if !s.IsDir() {
continue
}
fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta", "fields.yml")
fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta/fields.yml")
if _, err = os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
files = append(files, &YmlFile{
Path: fieldsYmlPath,
Expand Down
2 changes: 1 addition & 1 deletion libbeat/kibana/fields_transformer.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
}

if t.keys[f.Path] != nil {
msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.", f.Path)
msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.\n", f.Path)
panic(errors.New(msg))
}

Expand Down
Loading

0 comments on commit 5c25fbb

Please sign in to comment.