Merge branch 'main' into fix-commented-aws-tests

CHANGELOG.next.asciidoc

@@ -351,6 +351,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Add device support for Azure AD entity analytics. {pull}35807[35807]
 - Improve CEL input performance. {pull}35915[35915]
 - Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044]
+- Added support for min/max template functions in httpjson input. {issue}36094[36094] {pull}36036[36036]
 - Add `clean_session` configuration setting for MQTT input. {pull}35806[16204]
 - Add fingerprint mode for the filestream scanner and new file identity based on it {issue}34419[34419] {pull}35734[35734]
 - Add file system metadata to events ingested via filestream {issue}35801[35801] {pull}36065[36065]

x-pack/filebeat/docs/inputs/input-httpjson.asciidoc

@@ -214,6 +214,8 @@ Some built-in helper functions are provided to work with the input state inside
 - `hmacBase64`: calculates the hmac signature of a list of strings concatenated together. Returns a base64 encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`
 - `hmac`: calculates the hmac signature of a list of strings concatenated together. Returns a hex encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`
 - `join`: joins a list using the specified separator. Example: `[[join .body.arr ","]]`
+- `max`: returns the maximum of two values.
+- `min`: returns the minimum of two values.
 - `mul`: multiplies two integers.
 - `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `[[now (parseDuration "-1h")]]` returns the time at 1 hour before now.
 - `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`.

x-pack/filebeat/input/httpjson/texttemplate.go

@@ -0,0 +1,107 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package httpjson
+
+import (
+ "errors"
+ "reflect"
+)
+
+// These functions come from Go's text/template/funcs.go (1.19).
+//
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+var (
+ errBadComparisonType = errors.New("invalid type for comparison")
+ errBadComparison = errors.New("incompatible types for comparison")
+)
+
+type kind int
+
+const (
+ invalidKind kind = iota
+ boolKind
+ complexKind
+ intKind
+ floatKind
+ stringKind
+ uintKind
+)
+
+func basicKind(v reflect.Value) (kind, error) {
+ switch v.Kind() {
+ case reflect.Bool:
+ return boolKind, nil
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ return intKind, nil
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ return uintKind, nil
+ case reflect.Float32, reflect.Float64:
+ return floatKind, nil
+ case reflect.Complex64, reflect.Complex128:
+ return complexKind, nil
+ case reflect.String:
+ return stringKind, nil
+ }
+ return invalidKind, errBadComparisonType
+}
+
+// indirectInterface returns the concrete value in an interface value,
+// or else the zero reflect.Value.
+// That is, if v represents the interface value x, the result is the same as reflect.ValueOf(x):
+// the fact that x was an interface value is forgotten.
+func indirectInterface(v reflect.Value) reflect.Value {
+ if v.Kind() != reflect.Interface {
+ return v
+ }
+ if v.IsNil() {
+ return reflect.Value{}
+ }
+ return v.Elem()
+}
+
+// lt evaluates the comparison a < b.
+func lt(arg1, arg2 reflect.Value) (bool, error) {
+ arg1 = indirectInterface(arg1)
+ k1, err := basicKind(arg1)
+ if err != nil {
+ return false, err
+ }
+ arg2 = indirectInterface(arg2)
+ k2, err := basicKind(arg2)
+ if err != nil {
+ return false, err
+ }
+ truth := false
+ if k1 != k2 {
+ // Special case: Can compare integer values regardless of type's sign.
+ switch {
+ case k1 == intKind && k2 == uintKind:
+ truth = arg1.Int() < 0 || uint64(arg1.Int()) < arg2.Uint()
+ case k1 == uintKind && k2 == intKind:
+ truth = arg2.Int() >= 0 && arg1.Uint() < uint64(arg2.Int())
+ default:
+ return false, errBadComparison
+ }
+ } else {
+ switch k1 {
+ case boolKind, complexKind:
+ return false, errBadComparisonType
+ case floatKind:
+ truth = arg1.Float() < arg2.Float()
+ case intKind:
+ truth = arg1.Int() < arg2.Int()
+ case stringKind:
+ truth = arg1.String() < arg2.String()
+ case uintKind:
+ truth = arg1.Uint() < arg2.Uint()
+ default:
+ panic("invalid kind")
+ }
+ }
+ return truth, nil
+}

x-pack/filebeat/input/httpjson/value_tpl.go

@@ -66,6 +66,8 @@ func (t *valueTpl) Unpack(in string) error {
  "hmacBase64": hmacStringBase64,
  "join": join,
  "toJSON": toJSON,
+ "max": max,
+ "min": min,
  "mul": mul,
  "now": now,
  "parseDate": parseDate,
@@ -295,6 +297,32 @@ func div(a, b int64) int64 {
  return a / b
 }
 
+func min(arg1, arg2 reflect.Value) (interface{}, error) {
+ lessThan, err := lt(arg1, arg2)
+ if err != nil {
+ return nil, err
+ }
+
+ // arg1 is < arg2.
+ if lessThan {
+ return arg1.Interface(), nil
+ }
+ return arg2.Interface(), nil
+}
+
+func max(arg1, arg2 reflect.Value) (interface{}, error) {
+ lessThan, err := lt(arg1, arg2)
+ if err != nil {
+ return nil, err
+ }
+
+ // arg1 is < arg2.
+ if lessThan {
+ return arg2.Interface(), nil
+ }
+ return arg1.Interface(), nil
+}
+
 func base64Encode(values ...string) string {
  data := strings.Join(values, "")
  if data == "" {

x-pack/filebeat/input/httpjson/value_tpl_test.go

@@ -356,6 +356,76 @@ func TestValueTpl(t *testing.T) {
  paramTr: transformable{},
  expectedVal: "4",
  },
+ {
+ name: "func min int",
+ value: `[[min 4 1]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "1",
+ },
+ {
+ name: "func max int",
+ value: `[[max 4 1]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "4",
+ },
+ {
+ name: "func max float",
+ value: `[[max 1.23 4.666]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "4.666",
+ },
+ {
+ name: "func min float",
+ value: `[[min 1.23 4.666]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "1.23",
+ },
+ {
+ name: "func min string",
+ value: `[[min "a" "b"]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "a",
+ },
+ {
+ name: "func max string",
+ value: `[[max "a" "b"]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "b",
+ },
+ {
+ name: "func min int64 unix seconds",
+ value: `[[ min (now.Unix) 1689771139 ]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "1689771139",
+ },
+ {
+ name: "func min int year",
+ value: `[[ min (now.Year) 2020 ]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "2020",
+ },
+ {
+ name: "func max duration",
+ value: `[[ max (parseDuration "59m") (parseDuration "1h") ]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "1h0m0s",
+ },
+ {
+ name: "func min int ",
+ value: `[[ min (now.Year) 2020 ]]`,
+ paramCtx: emptyTransformContext(),
+ paramTr: transformable{},
+ expectedVal: "2020",
+ },
  {
  name: "func sha1 hmac Hex",
  value: `[[hmac "sha1" "secret" "string1" "string2"]]`,