Skip to content

Commit

Permalink
Merge branch 'main' of github.com:harnish-elastic/integrations into a…
Browse files Browse the repository at this point in the history
…ws-ecs-migration

Conflicts:
	packages/aws/changelog.yml
	packages/aws/docs/elb.md
  • Loading branch information
harnish-elastic committed Jul 2, 2024
2 parents 9efd781 + 647b31d commit 9c7d408
Show file tree
Hide file tree
Showing 4,005 changed files with 69,272 additions and 361,679 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
1 change: 1 addition & 0 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ steps:
memory: "4G"

- label: ":sonarqube: Continuous Code Inspection"
soft_fail: true # FIXME: Coverage is failing, remove this after solving the issue
env:
VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/integrations/sonar-analyze-token"
agents:
Expand Down
29 changes: 0 additions & 29 deletions .buildkite/scripts/merge_xml.sh

This file was deleted.

8 changes: 7 additions & 1 deletion .buildkite/scripts/run_sonar_scanner.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
#!/bin/bash

source .buildkite/scripts/common.sh

set -euo pipefail

add_bin_path
with_mage

run_sonar_scanner() {
local message=""
echo "--- Download coverage reports and merge them"
Expand All @@ -15,7 +21,7 @@ run_sonar_scanner() {
fi

echo "Merge all coverage reports"
.buildkite/scripts/merge_xml.sh
mage mergeCoverage

echo "--- Execute sonar scanner CLI"
/scan-source-code.sh
Expand Down
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,7 @@
/packages/system_audit @elastic/sec-linux-platform
/packages/tanium @elastic/security-service-integrations
/packages/tcp @elastic/sec-deployment-and-devices
/packages/teleport @elastic/security-service-integrations
/packages/tenable_io @elastic/security-service-integrations
/packages/tenable_sc @elastic/security-service-integrations
/packages/thycotic_ss @elastic/security-service-integrations
Expand Down
143 changes: 143 additions & 0 deletions dev/coverage/coverage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

// File partially copied from elastic-package.

package coverage

import (
"bytes"
"encoding/xml"
"fmt"
"os"
)

// GenericCoverage is the root element for a Cobertura XML report.
type GenericCoverage struct {
XMLName xml.Name `xml:"coverage"`
Version int64 `xml:"version,attr"`
Files []*GenericFile `xml:"file"`
Timestamp int64 `xml:"-"`
TestType string `xml:",comment"`
}

type GenericFile struct {
Path string `xml:"path,attr"`
Lines []*GenericLine `xml:"lineToCover"`
}

type GenericLine struct {
LineNumber int64 `xml:"lineNumber,attr"`
Covered bool `xml:"covered,attr"`
}

func (c *GenericCoverage) Bytes() ([]byte, error) {
out, err := xml.MarshalIndent(&c, "", " ")
if err != nil {
return nil, fmt.Errorf("unable to format test results as Coverage: %w", err)
}

var buffer bytes.Buffer
buffer.WriteString(xml.Header)
buffer.WriteString("\n")
buffer.Write(out)
return buffer.Bytes(), nil
}

func (c *GenericFile) merge(b *GenericFile) error {
// Merge files
for _, coverageLine := range b.Lines {
found := false
foundId := 0
for idx, existingLine := range c.Lines {
if existingLine.LineNumber == coverageLine.LineNumber {
found = true
foundId = idx
break
}
}
if !found {
c.Lines = append(c.Lines, coverageLine)
} else {
c.Lines[foundId].Covered = c.Lines[foundId].Covered || coverageLine.Covered
}
}
return nil
}

// merge merges two coverage reports.
func (c *GenericCoverage) Merge(other *GenericCoverage) error {
// Merge files
for _, coverageFile := range other.Files {
var target *GenericFile
for _, existingFile := range c.Files {
if existingFile.Path == coverageFile.Path {
target = existingFile
break
}
}
if target != nil {
if err := target.merge(coverageFile); err != nil {
return err
}
} else {
c.Files = append(c.Files, coverageFile)
}
}
return nil
}

func ReadGenericCoverage(path string) (*GenericCoverage, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("open failed: %w", err)
}
defer f.Close()

dec := xml.NewDecoder(f)

var coverage GenericCoverage
err = dec.Decode(&coverage)
if err != nil {
return nil, fmt.Errorf("xml decode failed: %w", err)
}

return &coverage, nil
}

func MergeGenericCoverageFiles(paths []string, output string) error {
f, err := os.Create(output)
if err != nil {
return fmt.Errorf("cannot open file %s to write merged coverage: %w", output, err)
}
defer f.Close()

var coverage *GenericCoverage
for _, path := range paths {
c, err := ReadGenericCoverage(path)
if err != nil {
return fmt.Errorf("failed to read coverage from %s: %w", path, err)
}
if coverage == nil {
coverage = c
continue
}
err = coverage.Merge(c)
if err != nil {
return fmt.Errorf("failed to merge coverage from %s: %w", path, err)
}
}

d, err := coverage.Bytes()
if err != nil {
return fmt.Errorf("failed to encode merged coverage: %w", err)
}

_, err = f.Write(d)
if err != nil {
return fmt.Errorf("cannot write merged coverage to %s: %w", output, err)
}

return nil
}
31 changes: 31 additions & 0 deletions dev/coverage/coverage_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package coverage

import (
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestMergeCoverage(t *testing.T) {
coverageFiles, err := filepath.Glob("testdata/test-coverage-*.xml")
require.NoError(t, err)

expectedCoverage, err := ReadGenericCoverage("testdata/expected-test-coverage.xml")
require.NoError(t, err)

output := filepath.Join(t.TempDir(), "coverage-merged.xml")

err = MergeGenericCoverageFiles(coverageFiles, output)
require.NoError(t, err)

mergedCoverage, err := ReadGenericCoverage(output)
require.NoError(t, err)

assert.EqualValues(t, expectedCoverage, mergedCoverage)
}
23 changes: 23 additions & 0 deletions dev/coverage/testdata/expected-test-coverage.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>

<coverage version="1">
<file path="packages/elastic_package_registry/data_stream/metrics/fields/base-fields.yml">
<lineToCover lineNumber="1" covered="false"></lineToCover>
<lineToCover lineNumber="2" covered="false"></lineToCover>
<lineToCover lineNumber="3" covered="false"></lineToCover>
<lineToCover lineNumber="4" covered="false"></lineToCover>
<lineToCover lineNumber="5" covered="true"></lineToCover>
<lineToCover lineNumber="6" covered="false"></lineToCover>
<lineToCover lineNumber="7" covered="false"></lineToCover>
<lineToCover lineNumber="8" covered="true"></lineToCover>
<lineToCover lineNumber="9" covered="false"></lineToCover>
<lineToCover lineNumber="10" covered="false"></lineToCover>
<lineToCover lineNumber="11" covered="false"></lineToCover>
<lineToCover lineNumber="12" covered="false"></lineToCover>
</file>
<file path="packages/elastic_package_registry/data_stream/metrics/fields/ecs.yml">
<lineToCover lineNumber="1" covered="true"></lineToCover>
<lineToCover lineNumber="2" covered="true"></lineToCover>
<lineToCover lineNumber="3" covered="true"></lineToCover>
</file>
</coverage>
23 changes: 23 additions & 0 deletions dev/coverage/testdata/test-coverage-1.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>

<coverage version="1">
<file path="packages/elastic_package_registry/data_stream/metrics/fields/base-fields.yml">
<lineToCover lineNumber="1" covered="false"></lineToCover>
<lineToCover lineNumber="2" covered="false"></lineToCover>
<lineToCover lineNumber="3" covered="false"></lineToCover>
<lineToCover lineNumber="4" covered="false"></lineToCover>
<lineToCover lineNumber="5" covered="true"></lineToCover>
<lineToCover lineNumber="6" covered="false"></lineToCover>
<lineToCover lineNumber="7" covered="false"></lineToCover>
<lineToCover lineNumber="8" covered="false"></lineToCover>
<lineToCover lineNumber="9" covered="false"></lineToCover>
<lineToCover lineNumber="10" covered="false"></lineToCover>
<lineToCover lineNumber="11" covered="false"></lineToCover>
<lineToCover lineNumber="12" covered="false"></lineToCover>
</file>
<file path="packages/elastic_package_registry/data_stream/metrics/fields/ecs.yml">
<lineToCover lineNumber="1" covered="false"></lineToCover>
<lineToCover lineNumber="2" covered="false"></lineToCover>
<lineToCover lineNumber="3" covered="false"></lineToCover>
</file>
</coverage>
9 changes: 9 additions & 0 deletions dev/coverage/testdata/test-coverage-2.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>

<coverage version="1">
<file path="packages/elastic_package_registry/data_stream/metrics/fields/ecs.yml">
<lineToCover lineNumber="1" covered="false"></lineToCover>
<lineToCover lineNumber="2" covered="false"></lineToCover>
<lineToCover lineNumber="3" covered="false"></lineToCover>
</file>
</coverage>
23 changes: 23 additions & 0 deletions dev/coverage/testdata/test-coverage-3.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>

<coverage version="1">
<file path="packages/elastic_package_registry/data_stream/metrics/fields/base-fields.yml">
<lineToCover lineNumber="1" covered="false"></lineToCover>
<lineToCover lineNumber="2" covered="false"></lineToCover>
<lineToCover lineNumber="3" covered="false"></lineToCover>
<lineToCover lineNumber="4" covered="false"></lineToCover>
<lineToCover lineNumber="5" covered="false"></lineToCover>
<lineToCover lineNumber="6" covered="false"></lineToCover>
<lineToCover lineNumber="7" covered="false"></lineToCover>
<lineToCover lineNumber="8" covered="true"></lineToCover>
<lineToCover lineNumber="9" covered="false"></lineToCover>
<lineToCover lineNumber="10" covered="false"></lineToCover>
<lineToCover lineNumber="11" covered="false"></lineToCover>
<lineToCover lineNumber="12" covered="false"></lineToCover>
</file>
<file path="packages/elastic_package_registry/data_stream/metrics/fields/ecs.yml">
<lineToCover lineNumber="1" covered="true"></lineToCover>
<lineToCover lineNumber="2" covered="true"></lineToCover>
<lineToCover lineNumber="3" covered="true"></lineToCover>
</file>
</coverage>
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
Expand Down
9 changes: 4 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -241,13 +241,12 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
Expand Down
10 changes: 10 additions & 0 deletions magefile.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
package main

import (
"fmt"
"io"
"os"
"path/filepath"
Expand All @@ -16,6 +17,7 @@ import (
"github.com/pkg/errors"

"github.com/elastic/integrations/dev/codeowners"
"github.com/elastic/integrations/dev/coverage"
)

var (
Expand Down Expand Up @@ -51,6 +53,14 @@ func ImportBeats() error {
return sh.Run("go", args...)
}

func MergeCoverage() error {
coverageFiles, err := filepath.Glob("build/test-coverage/coverage-*.xml")
if err != nil {
return fmt.Errorf("glob failed: %w", err)
}
return coverage.MergeGenericCoverageFiles(coverageFiles, "build/test-coverage/coverage_merged.xml")
}

func build() error {
mg.Deps(buildImportBeats)
return nil
Expand Down
5 changes: 5 additions & 0 deletions packages/auditd_manager/changelog.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.16.4"
changes:
- description: Fix `audit_rule_files`
type: bugfix
link: https:/elastic/integrations/issues/10000
- version: "1.16.3"
changes:
- description: capture root requirement
Expand Down
Loading

0 comments on commit 9c7d408

Please sign in to comment.