Add vendoring to Google Cloud Functions again

CHANGELOG.next.asciidoc

@@ -357,6 +357,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Fix timeout option of GCP functions. {issue}16282[16282] {pull}16287[16287]
 - Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
+- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]
 
 ==== Added
 

dev-tools/mage/gotool/go.go

@@ -85,6 +85,25 @@ func ListDeps(pkg string) ([]string, error) {
  return getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg))
 }
 
+// ListDepsLocation calls `go list -dep` for every package spec given.
+func ListDepsLocation(pkg string) (map[string]string, error) {
+ const tmpl = `{{if not .Standard}}{{.ImportPath}};{{.Dir}}{{end}}`
+
+ lines, err := getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg))
+ if err != nil {
+ return nil, err
+ }
+ deps := make(map[string]string, len(lines))
+ for _, l := range lines {
+ parts := strings.Split(l, ";")
+ if len(parts) != 2 {
+ return nil, fmt.Errorf("invalid number of parts")
+ }
+ deps[parts[0]] = parts[1]
+ }
+ return deps, nil
+}
+
 // ListTestFiles lists all go and cgo test files available in a package.
 func ListTestFiles(pkg string) ([]string, error) {
  const tmpl = `{{ range .TestGoFiles }}{{ printf "%s\n" . }}{{ end }}` +

x-pack/functionbeat/dev-tools/packaging/packages.yml

@@ -65,6 +65,12 @@ shared:
  pkg/functionbeat-aws:
  source: 'provider/aws/build/golang-crossbuild/aws-linux-amd64'
  mode: 0755
+ pkg/pubsub/vendor:
+ source: 'provider/gcp/build/pubsub/vendor'
+ mode: 0644
+ pkg/storage/vendor:
+ source: 'provider/gcp/build/storage/vendor'
+ mode: 0644
  pkg/pubsub/pubsub.go:
  source: 'provider/gcp/pubsub/pubsub.go'
  mode: 0655

x-pack/functionbeat/magefile.go

@@ -159,9 +159,11 @@ func BuildPkgForFunctions() error {
  err := os.RemoveAll("pkg")
 
  filesToCopy := map[string]string{
- filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"),
- filepath.Join("provider", "gcp", "pubsub", "pubsub.go"): filepath.Join("pkg", "pubsub", "pubsub.go"),
- filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"),
+ filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"),
+ filepath.Join("provider", "gcp", "pubsub", "pubsub.go"): filepath.Join("pkg", "pubsub", "pubsub.go"),
+ filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"),
+ filepath.Join("provider", "gcp", "build", "pubsub", "vendor"): filepath.Join("pkg", "pubsub", "vendor"),
+ filepath.Join("provider", "gcp", "build", "storage", "vendor"): filepath.Join("pkg", "storage", "vendor"),
  }
  for src, dest := range filesToCopy {
  c := &devtools.CopyTask{

x-pack/functionbeat/manager/gcp/template_builder.go

@@ -190,5 +190,6 @@ func zipResources() map[string][]bundle.Resource {
 
 func zipResourcesOfFunc(typeName string) []bundle.Resource {
  root := filepath.Join("pkg", typeName)
- return []bundle.Resource{&bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755}}
+ vendor := bundle.Folder(filepath.Join("pkg", typeName, "vendor"), filepath.Join("pkg", typeName), 0644)
+ return append(vendor, &bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755})
 }

x-pack/functionbeat/scripts/mage/update.go

@@ -5,9 +5,13 @@
 package mage
 
 import (
+ "os"
+ "path/filepath"
+
  "github.com/magefile/mage/mg"
 
  devtools "github.com/elastic/beats/v7/dev-tools/mage"
+ "github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 // Update target namespace.
@@ -20,7 +24,7 @@ var Aliases = map[string]interface{}{
 
 // All updates all generated content.
 func (Update) All() {
- mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs)
+ mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs, Update.VendorBeats)
 }
 
 // Config generates both the short and reference configs.
@@ -46,3 +50,39 @@ func (Update) IncludeFields() error {
 
  return devtools.GenerateAllInOneFieldsGo()
 }
+
+// VendorBeats collects the vendor folder required to deploy the function for GCP.
+func (Update) VendorBeats() error {
+ for _, f := range []string{"pubsub", "storage"} {
+ gcpVendorPath := filepath.Join("provider", "gcp", "build", f, "vendor")
+ err := os.RemoveAll(gcpVendorPath)
+ if err != nil {
+ return err
+ }
+
+ deps, err := gotool.ListDepsLocation("github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/" + f)
+ if err != nil {
+ return err
+ }
+
+ for importPath, location := range deps {
+ cp := &devtools.CopyTask{
+ Source: location,
+ Dest: filepath.Join(gcpVendorPath, importPath),
+ Mode: 0600,
+ DirMode: os.ModeDir | 0750,
+ Exclude: []string{
+ ".*_test.go$",
+ ".*.yml",
+ },
+ }
+ err = cp.Execute()
+ if err != nil {
+ return err
+ }
+ }
+
+ }
+
+ return nil
+}