diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a56130b776b..0aacb43832a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -309,6 +309,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update indentation for azure filebeat configuration. {pull}26604[26604] - Auditd: Fix Top Exec Commands dashboard visualization. {pull}27638[27638] - Store offset in `log.offset` field of events from the filestream input. {pull}27688[27688] +- Fix `httpjson` input rate limit processing and documentation. {pull}[] *Heartbeat* @@ -518,6 +519,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - update ECS field definitions to ECS 1.11.0. {pull}27107[27107] - The disk queue is now GA. {pull}27515[27515] - Allow non-padded base64 data to be decoded by decode_base64_field {pull}27311[27311], {issue}27021[27021] +- The Kafka support library Sarama has been updated to 1.29.1. {pull}27717[27717] +- Kafka is now supported up to version 2.8.0. {pull}27720[27720] *Auditbeat* @@ -738,6 +741,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Move AWS module and filesets to GA. {pull}27428[27428] - update ecs.version to ECS 1.11.0. {pull}27107[27107] - Add base64 Encode functionality to httpjson input. {pull}27681[27681] +- Add `join` and `sprintf` functions to `httpjson` input. {pull}27735[27735] *Heartbeat* diff --git a/Jenkinsfile b/Jenkinsfile index 3d552c4a4d0..78c2301d370 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,7 +22,7 @@ pipeline { RUNBLD_DISABLE_NOTIFICATIONS = 'true' SLACK_CHANNEL = "#beats-build" SNAPSHOT = 'true' - TERRAFORM_VERSION = "0.12.30" + TERRAFORM_VERSION = "0.13.7" XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } options { @@ -277,14 +277,16 @@ def generateStages(Map args = [:]) { } def cloud(Map args = [:]) { - withNode(labels: args.label, forceWorkspace: true){ - startCloudTestEnv(name: args.directory, dirs: args.dirs) - } - withCloudTestEnv() { - try { - target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) - } finally { - terraformCleanup(name: args.directory, dir: args.directory) + withGithubNotify(context: args.context) { + withNode(labels: args.label, forceWorkspace: true){ + startCloudTestEnv(name: args.directory, dirs: args.dirs) + } + withCloudTestEnv() { + try { + target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) + } finally { + terraformCleanup(name: args.directory, dir: args.directory) + } } } } @@ -838,8 +840,10 @@ def withCloudTestEnv(Closure body) { def maskedVars = [] def testTags = "${env.TEST_TAGS}" - // AWS - if (params.allCloudTests || params.awsCloudTests) { + // Allow AWS credentials when the build was configured to do so with: + // - the cloudtests build parameters + // - the aws github label + if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws')) { testTags = "${testTags},aws" def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data if (!aws.containsKey('access_key')) { @@ -891,6 +895,7 @@ def startCloudTestEnv(Map args = [:]) { // If it failed then cleanup without failing the build sh(label: 'Terraform Cleanup', script: ".ci/scripts/terraform-cleanup.sh ${folder}", returnStatus: true) } + error('startCloudTestEnv: terraform apply failed.') } finally { // Archive terraform states in case manual cleanup is needed. archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') diff --git a/NOTICE.txt b/NOTICE.txt index b2acf7c0154..a801be6f5e7 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2246,11 +2246,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/sarama -Version: v1.19.1-0.20210120173147-5c8cb347d877 +Version: v1.19.1-0.20210823122811-11c3ef800752 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.19.1-0.20210120173147-5c8cb347d877/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.19.1-0.20210823122811-11c3ef800752/LICENSE: Copyright (c) 2013 Shopify @@ -10080,11 +10080,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/golang/snappy -Version: v0.0.1 +Version: v0.0.3 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/golang/snappy@v0.0.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/golang/snappy@v0.0.3/LICENSE: Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. @@ -10516,11 +10516,11 @@ Contents of probable licence file $GOMODCACHE/github.com/google/flatbuffers@v1.7 -------------------------------------------------------------------------------- Dependency : github.com/google/go-cmp -Version: v0.5.2 +Version: v0.5.4 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/google/go-cmp@v0.5.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/google/go-cmp@v0.5.4/LICENSE: Copyright (c) 2017 The Go Authors. All rights reserved. @@ -14760,11 +14760,11 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/prometheus@v -------------------------------------------------------------------------------- Dependency : github.com/rcrowley/go-metrics -Version: v0.0.0-20200313005456-10cdbea86bc0 +Version: v0.0.0-20201227073835-cf1acfcdf475 Licence type (autodetected): BSD-2-Clause-FreeBSD -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/rcrowley/go-metrics@v0.0.0-20200313005456-10cdbea86bc0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/LICENSE: Copyright 2012 Richard Crowley. All rights reserved. @@ -15742,11 +15742,11 @@ Contents of probable licence file $GOMODCACHE/github.com/vmware/govmomi@v0.0.0-2 -------------------------------------------------------------------------------- Dependency : github.com/xdg/scram -Version: v0.0.0-20180814205039-7eeb5667e42c +Version: v1.0.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xdg/scram@v0.0.0-20180814205039-7eeb5667e42c/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xdg/scram@v1.0.3/LICENSE: Apache License @@ -17174,11 +17174,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.0.0-20210226172049-e18ecbb05110 +Version: v0.0.0-20210614182718-04defd469f4e Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20210226172049-e18ecbb05110/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -17322,11 +17322,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.3.5 +Version: v0.3.6 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.3.5/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.3.6/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -28205,11 +28205,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/frankban/quicktest -Version: v1.10.2 +Version: v1.11.3 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.10.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.11.3/LICENSE: MIT License @@ -32047,6 +32047,80 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/gorilla/securecookie +Version: v1.1.1 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/gorilla/securecookie@v1.1.1/LICENSE: + +Copyright (c) 2012 Rodrigo Moraes. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/gorilla/sessions +Version: v1.2.1 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/gorilla/sessions@v1.2.1/LICENSE: + +Copyright (c) 2012-2018 The Gorilla Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/gorilla/websocket Version: v1.4.2 @@ -34715,6 +34789,428 @@ See the License for the specific language governing permissions and limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/jcmturner/aescts/v2 +Version: v2.0.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/jcmturner/aescts/v2@v2.0.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/jcmturner/dnsutils/v2 +Version: v2.0.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/jcmturner/dnsutils/v2@v2.0.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/jcmturner/gofork Version: v1.0.0 @@ -34752,6 +35248,639 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/jcmturner/goidentity/v6 +Version: v6.0.1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/jcmturner/goidentity/v6@v6.0.1/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/jcmturner/gokrb5/v8 +Version: v8.4.2 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/jcmturner/gokrb5/v8@v8.4.2/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/jcmturner/rpc/v2 +Version: v2.0.3 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/jcmturner/rpc/v2@v2.0.3/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/jessevdk/go-flags Version: v1.4.0 @@ -35108,11 +36237,11 @@ match.go, match_test.go: -------------------------------------------------------------------------------- Dependency : github.com/klauspost/compress -Version: v1.11.0 +Version: v1.12.2 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.11.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.12.2/LICENSE: Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2019 Klaus Post. All rights reserved. @@ -36518,35 +37647,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/niemeyer/pretty -Version: v0.0.0-20200227124842-a10e7caefd8e -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/niemeyer/pretty@v0.0.0-20200227124842-a10e7caefd8e/License: - -Copyright 2012 Keith Rarick - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/onsi/ginkgo Version: v1.11.0 @@ -37934,11 +39034,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/pierrec/lz4 -Version: v2.5.2+incompatible +Version: v2.6.0+incompatible Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.5.2+incompatible/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.6.0+incompatible/LICENSE: Copyright (c) 2015, Pierre Curto All rights reserved. @@ -39883,11 +40983,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/xdg/stringprep -Version: v1.0.0 +Version: v1.0.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xdg/stringprep@v1.0.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xdg/stringprep@v1.0.3/LICENSE: Apache License @@ -41558,11 +42658,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : gopkg.in/check.v1 -Version: v1.0.0-20200902074654-038fdea0a05b +Version: v1.0.0-20201130134442-10cb98267c6c Licence type (autodetected): BSD-2-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20200902074654-038fdea0a05b/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20201130134442-10cb98267c6c/LICENSE: Gocheck - A rich testing framework for Go @@ -42581,11 +43681,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : gopkg.in/yaml.v3 -Version: v3.0.0-20200615113413-eeeca48fe776 +Version: v3.0.0-20210107192922-496545a6307b Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.0-20200615113413-eeeca48fe776/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.0-20210107192922-496545a6307b/LICENSE: This project is covered by two different licenses: MIT and Apache. diff --git a/filebeat/docs/inputs/input-kafka.asciidoc b/filebeat/docs/inputs/input-kafka.asciidoc index e134fb86404..e4a6ad083ce 100644 --- a/filebeat/docs/inputs/input-kafka.asciidoc +++ b/filebeat/docs/inputs/input-kafka.asciidoc @@ -51,7 +51,7 @@ link:https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-for-kafka-ecos [[kafka-input-compatibility]] ==== Compatibility -This input works with all Kafka versions in between 0.11 and 2.1.0. Older versions +This input works with all Kafka versions in between 0.11 and 2.8.0. Older versions might work as well, but are not supported. [id="{beatname_lc}-input-{type}-options"] @@ -168,6 +168,71 @@ Configuration options for Kerberos authentication. See <> for more information. +[float] +===== `parsers` + +This option expects a list of parsers that the payload has to go through. + +Available parsers: + +* `ndjson` +* `multiline` + +[float] +===== `ndjson` + +These options make it possible for {beatname_uc} to decode the payload as +JSON messages. + +Example configuration: + +[source,yaml] +---- +- ndjson: + keys_under_root: true + add_error_key: true + message_key: log +---- + +*`keys_under_root`*:: By default, the decoded JSON is placed under a "json" key +in the output document. If you enable this setting, the keys are copied top +level in the output document. The default is false. + +*`overwrite_keys`*:: If `keys_under_root` and this setting are enabled, then the +values from the decoded JSON object overwrite the fields that {beatname_uc} +normally adds (type, source, offset, etc.) in case of conflicts. + +*`expand_keys`*:: If this setting is enabled, {beatname_uc} will recursively +de-dot keys in the decoded JSON, and expand them into a hierarchical object +structure. For example, `{"a.b.c": 123}` would be expanded into `{"a":{"b":{"c":123}}}`. +This setting should be enabled when the input is produced by an +https://github.com/elastic/ecs-logging[ECS logger]. + +*`add_error_key`*:: If this setting is enabled, {beatname_uc} adds an +"error.message" and "error.type: json" key in case of JSON unmarshalling errors +or when a `message_key` is defined in the configuration but cannot be used. + +*`message_key`*:: An optional configuration setting that specifies a JSON key on +which to apply the line filtering and multiline settings. If specified the key +must be at the top level in the JSON object and the value associated with the +key must be a string, otherwise no filtering or multiline aggregation will +occur. + +*`document_id`*:: Option configuration setting that specifies the JSON key to +set the document id. If configured, the field will be removed from the original +JSON document and stored in `@metadata._id` + +*`ignore_decoding_error`*:: An optional configuration setting that specifies if +JSON decoding errors should be logged or not. If set to true, errors will not +be logged. The default is false. + +[float] +===== `multiline` + +Options that control how {beatname_uc} deals with log messages that span +multiple lines. See <> for more information about +configuring multiline options. + [id="{beatname_lc}-input-{type}-common-options"] include::../inputs/input-common-options.asciidoc[] diff --git a/filebeat/include/list.go b/filebeat/include/list.go index e4c1396d973..51b04d4f92c 100644 --- a/filebeat/include/list.go +++ b/filebeat/include/list.go @@ -23,7 +23,6 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/v7/filebeat/input/container" _ "github.com/elastic/beats/v7/filebeat/input/docker" - _ "github.com/elastic/beats/v7/filebeat/input/kafka" _ "github.com/elastic/beats/v7/filebeat/input/log" _ "github.com/elastic/beats/v7/filebeat/input/mqtt" _ "github.com/elastic/beats/v7/filebeat/input/redis" diff --git a/filebeat/input/default-inputs/inputs.go b/filebeat/input/default-inputs/inputs.go index 881f3664efd..95dd18e35f6 100644 --- a/filebeat/input/default-inputs/inputs.go +++ b/filebeat/input/default-inputs/inputs.go @@ -20,6 +20,7 @@ package inputs import ( "github.com/elastic/beats/v7/filebeat/beater" "github.com/elastic/beats/v7/filebeat/input/filestream" + "github.com/elastic/beats/v7/filebeat/input/kafka" "github.com/elastic/beats/v7/filebeat/input/unix" v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" @@ -36,6 +37,7 @@ func Init(info beat.Info, log *logp.Logger, components beater.StateStore) []v2.P func genericInputs(log *logp.Logger, components beater.StateStore) []v2.Plugin { return []v2.Plugin{ filestream.Plugin(log, components), + kafka.Plugin(), unix.Plugin(), } } diff --git a/filebeat/input/filestream/environment_test.go b/filebeat/input/filestream/environment_test.go index 8e4fc3d799a..e5de5aabd05 100644 --- a/filebeat/input/filestream/environment_test.go +++ b/filebeat/input/filestream/environment_test.go @@ -39,6 +39,7 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/statestore" "github.com/elastic/beats/v7/libbeat/statestore/storetest" + "github.com/elastic/go-concert/unison" ) type inputTestingEnvironment struct { @@ -50,7 +51,8 @@ type inputTestingEnvironment struct { pluginInitOnce sync.Once plugin v2.Plugin - wg sync.WaitGroup + wg sync.WaitGroup + grp unison.TaskGroup } type registryEntry struct { @@ -70,7 +72,9 @@ func newInputTestingEnvironment(t *testing.T) *inputTestingEnvironment { } func (e *inputTestingEnvironment) mustCreateInput(config map[string]interface{}) v2.Input { + e.grp = unison.TaskGroup{} manager := e.getManager() + manager.Init(&e.grp, v2.ModeRun) c := common.MustNewConfigFrom(config) inp, err := manager.Create(c) if err != nil { @@ -88,12 +92,13 @@ func (e *inputTestingEnvironment) getManager() v2.InputManager { func (e *inputTestingEnvironment) startInput(ctx context.Context, inp v2.Input) { e.wg.Add(1) - go func(wg *sync.WaitGroup) { + go func(wg *sync.WaitGroup, grp *unison.TaskGroup) { defer wg.Done() + defer grp.Stop() inputCtx := input.Context{Logger: logp.L(), Cancelation: ctx} inp.Run(inputCtx, e.pipeline) - }(&e.wg) + }(&e.wg, &e.grp) } func (e *inputTestingEnvironment) waitUntilInputStops() { diff --git a/filebeat/input/filestream/input_integration_test.go b/filebeat/input/filestream/input_integration_test.go index cfbbb8085a8..431c6c6458d 100644 --- a/filebeat/input/filestream/input_integration_test.go +++ b/filebeat/input/filestream/input_integration_test.go @@ -196,7 +196,6 @@ func TestFilestreamCloseEOF(t *testing.T) { // test_empty_lines from test_harvester.py func TestFilestreamEmptyLine(t *testing.T) { - t.Skip("Flaky test https://github.com/elastic/beats/issues/27585") env := newInputTestingEnvironment(t) testlogName := "test.log" diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index c69b2522a4f..338182e460a 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -28,9 +28,9 @@ import ( "github.com/elastic/beats/v7/libbeat/common/kafka" "github.com/elastic/beats/v7/libbeat/common/transport/kerberos" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/monitoring/adapter" + "github.com/elastic/beats/v7/libbeat/reader/parser" ) type kafkaInputConfig struct { @@ -53,6 +53,7 @@ type kafkaInputConfig struct { Username string `config:"username"` Password string `config:"password"` ExpandEventListFromField string `config:"expand_event_list_from_field"` + Parsers parser.Config `config:",inline"` } type kafkaFetch struct { @@ -215,7 +216,6 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { ) if err := k.Validate(); err != nil { - logp.Err("Invalid kafka configuration: %v", err) return nil, err } return k, nil diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index add3c664224..9c872f1a14b 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -21,96 +21,181 @@ import ( "context" "encoding/json" "fmt" + "io" "strings" "sync" "time" + "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/Shopify/sarama" + "github.com/pkg/errors" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" + input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/acker" "github.com/elastic/beats/v7/libbeat/common/backoff" "github.com/elastic/beats/v7/libbeat/common/kafka" + "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/libbeat/logp" - - "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/reader" + "github.com/elastic/beats/v7/libbeat/reader/parser" ) -func init() { - err := input.Register("kafka", NewInput) +const pluginName = "kafka" + +// Plugin creates a new filestream input plugin for creating a stateful input. +func Plugin() input.Plugin { + return input.Plugin{ + Name: pluginName, + Stability: feature.Stable, + Deprecated: false, + Info: "Kafka input", + Doc: "The Kafka input consumes events from topics by connecting to the configured kafka brokers", + Manager: input.ConfigureWith(configure), + } +} + +func configure(cfg *common.Config) (input.Input, error) { + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, err + } + + saramaConfig, err := newSaramaConfig(config) if err != nil { - panic(err) + return nil, errors.Wrap(err, "initializing Sarama config") } + return NewInput(config, saramaConfig) +} + +func NewInput(config kafkaInputConfig, saramaConfig *sarama.Config) (*kafkaInput, error) { + return &kafkaInput{config: config, saramaConfig: saramaConfig}, nil } -// Input contains the input and its config type kafkaInput struct { config kafkaInputConfig saramaConfig *sarama.Config - context input.Context - outlet channel.Outleter saramaWaitGroup sync.WaitGroup // indicates a sarama consumer group is active - log *logp.Logger - runOnce sync.Once } -// NewInput creates a new kafka input -func NewInput( - cfg *common.Config, - connector channel.Connector, - inputContext input.Context, -) (input.Input, error) { +func (input *kafkaInput) Name() string { return pluginName } - config := defaultConfig() - if err := cfg.Unpack(&config); err != nil { - return nil, errors.Wrap(err, "reading kafka input config") +func (input *kafkaInput) Test(ctx input.TestContext) error { + client, err := sarama.NewClient(input.config.Hosts, input.saramaConfig) + if err != nil { + ctx.Logger.Error(err) + } + topics, err := client.Topics() + if err != nil { + ctx.Logger.Error(err) + } + + var missingTopics []string + for _, neededTopic := range input.config.Topics { + if !contains(topics, neededTopic) { + missingTopics = append(missingTopics, neededTopic) + } + } + + if len(missingTopics) > 0 { + return fmt.Errorf("Of configured topics %v, topics: %v are not in available topics %v", input.config.Topics, missingTopics, topics) } - out, err := connector.ConnectWith(cfg, beat.ClientConfig{ + return nil +} + +func (input *kafkaInput) Run(ctx input.Context, pipeline beat.Pipeline) error { + log := ctx.Logger.Named("kafka input").With("hosts", input.config.Hosts) + + client, err := pipeline.ConnectWith(beat.ClientConfig{ ACKHandler: acker.ConnectionOnly( acker.EventPrivateReporter(func(_ int, events []interface{}) { for _, event := range events { if meta, ok := event.(eventMeta); ok { - meta.handler.ack(meta.message) + meta.ackHandler() } } }), ), - CloseRef: doneChannelContext(inputContext.Done), - WaitClose: config.WaitClose, + CloseRef: ctx.Cancelation, + WaitClose: input.config.WaitClose, }) if err != nil { - return nil, err + return err } - saramaConfig, err := newSaramaConfig(config) - if err != nil { - return nil, errors.Wrap(err, "initializing Sarama config") + log.Info("Starting Kafka input") + defer log.Info("Kafka input stopped") + + // Sarama uses standard go contexts to control cancellation, so we need + // to wrap our input context channel in that interface. + goContext := doneChannelContext(ctx) + + // If the consumer fails to connect, we use exponential backoff with + // jitter up to 8 * the initial backoff interval. + connectDelay := backoff.NewEqualJitterBackoff( + ctx.Cancelation.Done(), + input.config.ConnectBackoff, + 8*input.config.ConnectBackoff, + ) + + for goContext.Err() == nil { + // Connect to Kafka with a new consumer group. + consumerGroup, err := sarama.NewConsumerGroup( + input.config.Hosts, + input.config.GroupID, + input.saramaConfig, + ) + if err != nil { + log.Errorw("Error initializing kafka consumer group", "error", err) + connectDelay.Wait() + continue + } + // We've successfully connected, reset the backoff timer. + connectDelay.Reset() + + // We have a connected consumer group now, try to start the main event + // loop by calling Consume (which starts an asynchronous consumer). + // In an ideal run, this function never returns until shutdown; if it + // does, it means the errors have been logged and the consumer group + // has been closed, so we try creating a new one in the next iteration. + input.runConsumerGroup(log, client, goContext, consumerGroup) } - input := &kafkaInput{ - config: config, - saramaConfig: saramaConfig, - context: inputContext, - outlet: out, - log: logp.NewLogger("kafka input").With("hosts", config.Hosts), + if ctx.Cancelation.Err() == context.Canceled { + return nil + } else { + return ctx.Cancelation.Err() } +} - return input, nil +// Stop doesn't need to do anything because the kafka consumer group and the +// input's outlet both have a context based on input.context.Done and will +// shut themselves down, since the done channel is already closed as part of +// the shutdown process in Runner.Stop(). +func (input *kafkaInput) Stop() { +} + +// Wait should shut down the input and wait for it to complete, however (see +// Stop above) we don't need to take actions to shut down as long as the +// input.config.Done channel is closed, so we just make a (currently no-op) +// call to Stop() and then wait for sarama to signal completion. +func (input *kafkaInput) Wait() { + input.Stop() + // Wait for sarama to shut down + input.saramaWaitGroup.Wait() } -func (input *kafkaInput) runConsumerGroup( - context context.Context, consumerGroup sarama.ConsumerGroup, -) { +func (input *kafkaInput) runConsumerGroup(log *logp.Logger, client beat.Client, context context.Context, consumerGroup sarama.ConsumerGroup) { handler := &groupHandler{ version: input.config.Version, - outlet: input.outlet, + client: client, + parsers: input.config.Parsers, // expandEventListFromField will be assigned the configuration option expand_event_list_from_field expandEventListFromField: input.config.ExpandEventListFromField, - log: input.log, + log: log, } input.saramaWaitGroup.Add(1) @@ -122,70 +207,20 @@ func (input *kafkaInput) runConsumerGroup( // Listen asynchronously to any errors during the consume process go func() { for err := range consumerGroup.Errors() { - input.log.Errorw("Error reading from kafka", "error", err) + log.Errorw("Error reading from kafka", "error", err) } }() err := consumerGroup.Consume(context, input.config.Topics, handler) if err != nil { - input.log.Errorw("Kafka consume error", "error", err) + log.Errorw("Kafka consume error", "error", err) } } -// Run starts the input by scanning for incoming messages and errors. -func (input *kafkaInput) Run() { - input.runOnce.Do(func() { - go func() { - // Sarama uses standard go contexts to control cancellation, so we need - // to wrap our input context channel in that interface. - context := doneChannelContext(input.context.Done) - - // If the consumer fails to connect, we use exponential backoff with - // jitter up to 8 * the initial backoff interval. - backoff := backoff.NewEqualJitterBackoff( - input.context.Done, - input.config.ConnectBackoff, - 8*input.config.ConnectBackoff) - - for context.Err() == nil { - // Connect to Kafka with a new consumer group. - consumerGroup, err := sarama.NewConsumerGroup( - input.config.Hosts, input.config.GroupID, input.saramaConfig) - if err != nil { - input.log.Errorw( - "Error initializing kafka consumer group", "error", err) - backoff.Wait() - continue - } - // We've successfully connected, reset the backoff timer. - backoff.Reset() - - // We have a connected consumer group now, try to start the main event - // loop by calling Consume (which starts an asynchronous consumer). - // In an ideal run, this function never returns until shutdown; if it - // does, it means the errors have been logged and the consumer group - // has been closed, so we try creating a new one in the next iteration. - input.runConsumerGroup(context, consumerGroup) - } - }() - }) -} - -// Stop doesn't need to do anything because the kafka consumer group and the -// input's outlet both have a context based on input.context.Done and will -// shut themselves down, since the done channel is already closed as part of -// the shutdown process in Runner.Stop(). -func (input *kafkaInput) Stop() { -} - -// Wait should shut down the input and wait for it to complete, however (see -// Stop above) we don't need to take actions to shut down as long as the -// input.config.Done channel is closed, so we just make a (currently no-op) -// call to Stop() and then wait for sarama to signal completion. -func (input *kafkaInput) Wait() { - input.Stop() - // Wait for sarama to shut down - input.saramaWaitGroup.Wait() +// The metadata attached to incoming events, so they can be ACKed once they've +// been successfully sent. +type eventMeta struct { + ackHandler func() } func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []string { @@ -209,25 +244,23 @@ func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []string { // channels that are more common in the beats codebase. // TODO(faec): Generalize this to a common utility in a shared library // (https://github.com/elastic/beats/issues/13125). -type channelCtx <-chan struct{} - -func doneChannelContext(ch <-chan struct{}) context.Context { - return channelCtx(ch) +type channelCtx struct { + ctx input.Context } -func (c channelCtx) Deadline() (deadline time.Time, ok bool) { return } +func doneChannelContext(ctx input.Context) context.Context { + return channelCtx{ctx} +} +func (c channelCtx) Deadline() (deadline time.Time, ok bool) { + return +} func (c channelCtx) Done() <-chan struct{} { - return (<-chan struct{})(c) + return c.ctx.Cancelation.Done() } func (c channelCtx) Err() error { - select { - case <-c: - return context.Canceled - default: - return nil - } + return c.ctx.Cancelation.Err() } -func (c channelCtx) Value(key interface{}) interface{} { return nil } +func (c channelCtx) Value(_ interface{}) interface{} { return nil } // The group handler for the sarama consumer group interface. In addition to // providing the basic consumption callbacks needed by sarama, groupHandler is @@ -237,68 +270,13 @@ type groupHandler struct { sync.Mutex version kafka.Version session sarama.ConsumerGroupSession - outlet channel.Outleter + client beat.Client + parsers parser.Config // if the fileset using this input expects to receive multiple messages bundled under a specific field then this value is assigned // ex. in this case are the azure fielsets where the events are found under the json object "records" - expandEventListFromField string + expandEventListFromField string // TODO log *logp.Logger -} - -// The metadata attached to incoming events so they can be ACKed once they've -// been successfully sent. -type eventMeta struct { - handler *groupHandler - message *sarama.ConsumerMessage -} - -func (h *groupHandler) createEvents( - sess sarama.ConsumerGroupSession, - claim sarama.ConsumerGroupClaim, - message *sarama.ConsumerMessage, -) []beat.Event { - timestamp := time.Now() - kafkaFields := common.MapStr{ - "topic": claim.Topic(), - "partition": claim.Partition(), - "offset": message.Offset, - "key": string(message.Key), - } - - version, versionOk := h.version.Get() - if versionOk && version.IsAtLeast(sarama.V0_10_0_0) { - timestamp = message.Timestamp - if !message.BlockTimestamp.IsZero() { - kafkaFields["block_timestamp"] = message.BlockTimestamp - } - } - if versionOk && version.IsAtLeast(sarama.V0_11_0_0) { - kafkaFields["headers"] = arrayForKafkaHeaders(message.Headers) - } - - // if expandEventListFromField has been set, then a check for the actual json object will be done and a return for multiple messages is executed - var events []beat.Event - var messages []string - if h.expandEventListFromField == "" { - messages = []string{string(message.Value)} - } else { - messages = h.parseMultipleMessages(message.Value) - } - for _, msg := range messages { - event := beat.Event{ - Timestamp: timestamp, - Fields: common.MapStr{ - "message": msg, - "kafka": kafkaFields, - }, - Private: eventMeta{ - handler: h, - message: message, - }, - } - events = append(events, event) - - } - return events + reader reader.Reader } func (h *groupHandler) Setup(session sarama.ConsumerGroupSession) error { @@ -325,34 +303,172 @@ func (h *groupHandler) ack(message *sarama.ConsumerMessage) { } } -func (h *groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { - for msg := range claim.Messages() { - events := h.createEvents(sess, claim, msg) - for _, event := range events { - h.outlet.OnEvent(event) +func (h *groupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + reader := h.createReader(claim) + parser := h.parsers.Create(reader) + for h.session.Context().Err() == nil { + message, err := parser.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + h.client.Publish(beat.Event{ + Timestamp: message.Ts, + Meta: message.Meta, + Fields: message.Fields, + }) + } + return nil +} + +func (h *groupHandler) createReader(claim sarama.ConsumerGroupClaim) reader.Reader { + if h.expandEventListFromField != "" { + return &listFromFieldReader{ + claim: claim, + groupHandler: h, + field: h.expandEventListFromField, + log: h.log, } } + return &recordReader{ + claim: claim, + groupHandler: h, + log: h.log, + } +} + +type recordReader struct { + claim sarama.ConsumerGroupClaim + groupHandler *groupHandler + log *logp.Logger +} + +func (m *recordReader) Close() error { + return nil +} + +func (m *recordReader) Next() (reader.Message, error) { + msg, ok := <-m.claim.Messages() + if !ok { + return reader.Message{}, io.EOF + } + + timestamp, kafkaFields := composeEventMetadata(m.claim, m.groupHandler, msg) + ackHandler := func() { + m.groupHandler.ack(msg) + } + return composeMessage(timestamp, msg.Value, kafkaFields, ackHandler), nil +} + +type listFromFieldReader struct { + claim sarama.ConsumerGroupClaim + groupHandler *groupHandler + buffer []reader.Message + field string + log *logp.Logger +} + +func (l *listFromFieldReader) Close() error { return nil } +func (l *listFromFieldReader) Next() (reader.Message, error) { + if len(l.buffer) != 0 { + return l.returnFromBuffer() + } + + msg, ok := <-l.claim.Messages() + if !ok { + return reader.Message{}, io.EOF + } + + timestamp, kafkaFields := composeEventMetadata(l.claim, l.groupHandler, msg) + messages := l.parseMultipleMessages(msg.Value) + + neededAcks := atomic.MakeInt(len(messages)) + ackHandler := func() { + if neededAcks.Dec() == 0 { + l.groupHandler.ack(msg) + } + } + for _, message := range messages { + newBuffer := append(l.buffer, composeMessage(timestamp, []byte(message), kafkaFields, ackHandler)) + l.buffer = newBuffer + } + + return l.returnFromBuffer() +} + +func (l *listFromFieldReader) returnFromBuffer() (reader.Message, error) { + next := l.buffer[0] + newBuffer := l.buffer[1:] + l.buffer = newBuffer + return next, nil +} + // parseMultipleMessages will try to split the message into multiple ones based on the group field provided by the configuration -func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { +func (l *listFromFieldReader) parseMultipleMessages(bMessage []byte) []string { var obj map[string][]interface{} err := json.Unmarshal(bMessage, &obj) if err != nil { - h.log.Errorw(fmt.Sprintf("Kafka desirializing multiple messages using the group object %s", h.expandEventListFromField), "error", err) + l.log.Errorw(fmt.Sprintf("Kafka desirializing multiple messages using the group object %s", l.field), "error", err) return []string{} } var messages []string - if len(obj[h.expandEventListFromField]) > 0 { - for _, ms := range obj[h.expandEventListFromField] { - js, err := json.Marshal(ms) - if err == nil { - messages = append(messages, string(js)) - } else { - h.log.Errorw(fmt.Sprintf("Kafka serializing message %s", ms), "error", err) - } + for _, ms := range obj[l.field] { + js, err := json.Marshal(ms) + if err == nil { + messages = append(messages, string(js)) + } else { + l.log.Errorw(fmt.Sprintf("Kafka serializing message %s", ms), "error", err) } } return messages } + +func composeEventMetadata(claim sarama.ConsumerGroupClaim, handler *groupHandler, msg *sarama.ConsumerMessage) (time.Time, common.MapStr) { + timestamp := time.Now() + kafkaFields := common.MapStr{ + "topic": claim.Topic(), + "partition": claim.Partition(), + "offset": msg.Offset, + "key": string(msg.Key), + } + + version, versionOk := handler.version.Get() + if versionOk && version.IsAtLeast(sarama.V0_10_0_0) { + timestamp = msg.Timestamp + if !msg.BlockTimestamp.IsZero() { + kafkaFields["block_timestamp"] = msg.BlockTimestamp + } + } + if versionOk && version.IsAtLeast(sarama.V0_11_0_0) { + kafkaFields["headers"] = arrayForKafkaHeaders(msg.Headers) + } + return timestamp, kafkaFields +} + +func composeMessage(timestamp time.Time, content []byte, kafkaFields common.MapStr, ackHandler func()) reader.Message { + return reader.Message{ + Ts: timestamp, + Content: content, + Fields: common.MapStr{ + "kafka": kafkaFields, + "message": string(content), + }, + Meta: common.MapStr{ + "ackHandler": ackHandler, + }, + } +} + +func contains(elements []string, element string) bool { + for _, e := range elements { + if e == element { + return true + } + } + return false +} diff --git a/filebeat/input/kafka/input_test.go b/filebeat/input/kafka/input_test.go index e83c0e908a8..a239149cb48 100644 --- a/filebeat/input/kafka/input_test.go +++ b/filebeat/input/kafka/input_test.go @@ -22,15 +22,32 @@ package kafka import ( "testing" - "github.com/elastic/beats/v7/filebeat/input/inputtest" + "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/tests/resources" ) func TestNewInputDone(t *testing.T) { - config := common.MapStr{ + config := common.MustNewConfigFrom(common.MapStr{ "hosts": "localhost:9092", "topics": "messages", "group_id": "filebeat", - } - inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) + }) + + AssertNotStartedInputCanBeDone(t, config) +} + +// AssertNotStartedInputCanBeDone checks that the context of an input can be +// done before starting the input, and it doesn't leak goroutines. This is +// important to confirm that leaks don't happen with CheckConfig. +func AssertNotStartedInputCanBeDone(t *testing.T, configMap *common.Config) { + goroutines := resources.NewGoroutinesChecker() + defer goroutines.Check(t) + + config, err := common.NewConfigFrom(configMap) + require.NoError(t, err) + + _, err = Plugin().Manager.Create(config) + require.NoError(t, err) } diff --git a/filebeat/input/kafka/kafka_integration_test.go b/filebeat/input/kafka/kafka_integration_test.go index c7ad9fc999c..2f168eb6daf 100644 --- a/filebeat/input/kafka/kafka_integration_test.go +++ b/filebeat/input/kafka/kafka_integration_test.go @@ -20,20 +20,22 @@ package kafka import ( + "context" "fmt" "math/rand" "os" "strconv" "strings" - "sync" "testing" "time" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/logp" + beattest "github.com/elastic/beats/v7/libbeat/publisher/testing" + "github.com/Shopify/sarama" "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" _ "github.com/elastic/beats/v7/libbeat/outputs/codec/format" @@ -45,41 +47,6 @@ const ( kafkaDefaultPort = "9092" ) -type eventInfo struct { - events []beat.Event -} - -type eventCapturer struct { - closed bool - c chan struct{} - closeOnce sync.Once - events chan beat.Event -} - -func NewEventCapturer(events chan beat.Event) channel.Outleter { - return &eventCapturer{ - c: make(chan struct{}), - events: events, - } -} - -func (o *eventCapturer) OnEvent(event beat.Event) bool { - o.events <- event - return true -} - -func (o *eventCapturer) Close() error { - o.closeOnce.Do(func() { - o.closed = true - close(o.c) - }) - return nil -} - -func (o *eventCapturer) Done() <-chan struct{} { - return o.c -} - type testMessage struct { message string headers []sarama.RecordHeader @@ -93,12 +60,7 @@ func recordHeader(key, value string) sarama.RecordHeader { } func TestInput(t *testing.T) { - id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int()) - testTopic := fmt.Sprintf("Filebeat-TestInput-%s", id) - context := input.Context{ - Done: make(chan struct{}), - BeatDone: make(chan struct{}), - } + testTopic := createTestTopicName() // Send test messages to the topic for the input to read. messages := []testMessage{ @@ -129,22 +91,10 @@ func TestInput(t *testing.T) { "wait_close": 0, }) - // Route input events through our capturer instead of sending through ES. - events := make(chan beat.Event, 100) - defer close(events) - capturer := NewEventCapturer(events) - defer capturer.Close() - connector := channel.ConnectorFunc(func(_ *common.Config, _ beat.ClientConfig) (channel.Outleter, error) { - return channel.SubOutlet(capturer), nil - }) - - input, err := NewInput(config, connector, context) - if err != nil { - t.Fatal(err) - } - - // Run the input and wait for finalization - input.Run() + client := beattest.NewChanClient(100) + defer client.Close() + events := client.Channel + input, cancel := run(t, config, client) timeout := time.After(30 * time.Second) for range messages { @@ -169,7 +119,7 @@ func TestInput(t *testing.T) { // Close the done channel and make sure the beat shuts down in a reasonable // amount of time. - close(context.Done) + cancel() didClose := make(chan struct{}) go func() { input.Wait() @@ -184,12 +134,7 @@ func TestInput(t *testing.T) { } func TestInputWithMultipleEvents(t *testing.T) { - id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int()) - testTopic := fmt.Sprintf("Filebeat-TestInput-%s", id) - context := input.Context{ - Done: make(chan struct{}), - BeatDone: make(chan struct{}), - } + testTopic := createTestTopicName() // Send test messages to the topic for the input to read. message := testMessage{ @@ -209,40 +154,155 @@ func TestInputWithMultipleEvents(t *testing.T) { "expand_event_list_from_field": "records", }) - // Route input events through our capturer instead of sending through ES. - events := make(chan beat.Event, 100) - defer close(events) - capturer := NewEventCapturer(events) - defer capturer.Close() - connector := channel.ConnectorFunc(func(_ *common.Config, _ beat.ClientConfig) (channel.Outleter, error) { - return channel.SubOutlet(capturer), nil - }) + client := beattest.NewChanClient(100) + defer client.Close() + events := client.Channel + input, cancel := run(t, config, client) - input, err := NewInput(config, connector, context) - if err != nil { - t.Fatal(err) + timeout := time.After(30 * time.Second) + select { + case event := <-events: + text, err := event.Fields.GetValue("message") + if err != nil { + t.Fatal(err) + } + msgs := []string{"{\"val\":\"val1\"}", "{\"val\":\"val2\"}"} + assert.Contains(t, msgs, text) + checkMatchingHeaders(t, event, message.headers) + case <-timeout: + t.Fatal("timeout waiting for incoming events") } - // Run the input and wait for finalization - input.Run() + cancel() + // Close the done channel and make sure the beat shuts down in a reasonable + // amount of time. + didClose := make(chan struct{}) + go func() { + input.Wait() + close(didClose) + }() + + select { + case <-time.After(30 * time.Second): + t.Fatal("timeout waiting for beat to shut down") + case <-didClose: + } +} + +func TestInputWithJsonPayload(t *testing.T) { + testTopic := createTestTopicName() + + // Send test message to the topic for the input to read. + message := testMessage{ + message: "{\"val\":\"val1\"}", + headers: []sarama.RecordHeader{ + recordHeader("X-Test-Header", "test header value"), + }, + } + writeToKafkaTopic(t, testTopic, message.message, message.headers, time.Second*20) + + // Setup the input config + config := common.MustNewConfigFrom(common.MapStr{ + "hosts": getTestKafkaHost(), + "topics": []string{testTopic}, + "group_id": "filebeat", + "wait_close": 0, + "parsers": []common.MapStr{ + { + "ndjson": common.MapStr{ + "target": "", + }, + }, + }, + }) + + client := beattest.NewChanClient(100) + defer client.Close() + events := client.Channel + input, cancel := run(t, config, client) timeout := time.After(30 * time.Second) select { case event := <-events: - text, err := event.Fields.GetValue("message") + text, err := event.Fields.GetValue("val") if err != nil { t.Fatal(err) } - msgs := []string{"{\"val\":\"val1\"}", "{\"val\":\"val2\"}"} + msgs := []string{"val1"} assert.Contains(t, msgs, text) checkMatchingHeaders(t, event, message.headers) case <-timeout: t.Fatal("timeout waiting for incoming events") } + cancel() + // Close the done channel and make sure the beat shuts down in a reasonable + // amount of time. + didClose := make(chan struct{}) + go func() { + input.Wait() + close(didClose) + }() + + select { + case <-time.After(30 * time.Second): + t.Fatal("timeout waiting for beat to shut down") + case <-didClose: + } +} + +func TestInputWithJsonPayloadAndMultipleEvents(t *testing.T) { + testTopic := createTestTopicName() + + // Send test messages to the topic for the input to read. + message := testMessage{ + message: "{\"records\": [{\"val\":\"val1\"}, {\"val\":\"val2\"}]}", + headers: []sarama.RecordHeader{ + recordHeader("X-Test-Header", "test header value"), + }, + } + writeToKafkaTopic(t, testTopic, message.message, message.headers, time.Second*20) + + // Setup the input config + config := common.MustNewConfigFrom(common.MapStr{ + "hosts": getTestKafkaHost(), + "topics": []string{testTopic}, + "group_id": "filebeat", + "wait_close": 0, + "expand_event_list_from_field": "records", + "parsers": []common.MapStr{ + { + "ndjson": common.MapStr{ + "target": "", + }, + }, + }, + }) + + client := beattest.NewChanClient(100) + defer client.Close() + events := client.Channel + input, cancel := run(t, config, client) + + timeout := time.After(30 * time.Second) + for i := 0; i < 2; i++ { + select { + case event := <-events: + text, err := event.Fields.GetValue("val") + if err != nil { + t.Fatal(err) + } + msgs := []string{"val1", "val2"} + assert.Contains(t, msgs, text) + checkMatchingHeaders(t, event, message.headers) + case <-timeout: + t.Fatal("timeout waiting for incoming events") + } + } + + cancel() // Close the done channel and make sure the beat shuts down in a reasonable // amount of time. - close(context.Done) didClose := make(chan struct{}) go func() { input.Wait() @@ -256,6 +316,44 @@ func TestInputWithMultipleEvents(t *testing.T) { } } +func TestTest(t *testing.T) { + testTopic := createTestTopicName() + + // Send test messages to the topic for the input to read. + message := testMessage{ + message: "{\"records\": [{\"val\":\"val1\"}, {\"val\":\"val2\"}]}", + headers: []sarama.RecordHeader{ + recordHeader("X-Test-Header", "test header value"), + }, + } + writeToKafkaTopic(t, testTopic, message.message, message.headers, time.Second*20) + + // Setup the input config + config := common.MustNewConfigFrom(common.MapStr{ + "hosts": getTestKafkaHost(), + "topics": []string{testTopic}, + "group_id": "filebeat", + }) + + inp, err := Plugin().Manager.Create(config) + if err != nil { + t.Fatal(err) + } + + err = inp.Test(v2.TestContext{ + Logger: logp.NewLogger("kafka_test"), + }) + if err != nil { + t.Fatal(err) + } +} + +func createTestTopicName() string { + id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int()) + testTopic := fmt.Sprintf("Filebeat-TestInput-%s", id) + return testTopic +} + func findMessage(t *testing.T, text string, msgs []testMessage) *testMessage { var msg *testMessage for _, m := range msgs { @@ -274,23 +372,19 @@ func checkMatchingHeaders( ) { kafka, err := event.Fields.GetValue("kafka") if err != nil { - t.Error(err) - return + t.Fatal(err) } kafkaMap, ok := kafka.(common.MapStr) if !ok { - t.Error("event.Fields.kafka isn't MapStr") - return + t.Fatal("event.Fields.kafka isn't MapStr") } headers, err := kafkaMap.GetValue("headers") if err != nil { - t.Error(err) - return + t.Fatal(err) } headerArray, ok := headers.([]string) if !ok { - t.Error("event.Fields.kafka.headers isn't a []string") - return + t.Fatal("event.Fields.kafka.headers isn't a []string") } assert.Equal(t, len(expected), len(headerArray)) for i := 0; i < len(expected); i++ { @@ -357,3 +451,27 @@ func writeToKafkaTopic( t.Fatal(err) } } + +func run(t *testing.T, cfg *common.Config, client *beattest.ChanClient) (*kafkaInput, func()) { + inp, err := Plugin().Manager.Create(cfg) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := newV2Context() + t.Cleanup(cancel) + + pipeline := beattest.ConstClient(client) + input := inp.(*kafkaInput) + go input.Run(ctx, pipeline) + return input, cancel +} + +func newV2Context() (v2.Context, func()) { + ctx, cancel := context.WithCancel(context.Background()) + return v2.Context{ + Logger: logp.NewLogger("kafka_test"), + ID: "test_id", + Cancelation: ctx, + }, cancel +} diff --git a/go.mod b/go.mod index f8f51bfdcda..c0f77750bc5 100644 --- a/go.mod +++ b/go.mod @@ -90,10 +90,10 @@ require ( github.com/gogo/protobuf v1.3.1 github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.4.3 - github.com/golang/snappy v0.0.1 + github.com/golang/snappy v0.0.3 github.com/gomodule/redigo v1.8.3 github.com/google/flatbuffers v1.7.2-0.20170925184458-7a6b2bf521e9 - github.com/google/go-cmp v0.5.2 + github.com/google/go-cmp v0.5.4 github.com/google/gopacket v1.1.18-0.20191009163724-0ad7f2610e34 github.com/google/uuid v1.1.2 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 @@ -139,7 +139,7 @@ require ( github.com/prometheus/common v0.7.0 github.com/prometheus/procfs v0.0.11 github.com/prometheus/prometheus v2.5.0+incompatible - github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e // indirect github.com/samuel/go-thrift v0.0.0-20140522043831-2187045faa54 github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522 // indirect @@ -155,7 +155,7 @@ require ( github.com/ugorji/go/codec v1.1.8 github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71 github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 - github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c + github.com/xdg/scram v1.0.3 github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 // indirect go.elastic.co/apm v1.11.0 go.elastic.co/apm/module/apmelasticsearch v1.7.2 @@ -168,11 +168,11 @@ require ( go.uber.org/zap v1.14.0 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e golang.org/x/lint v0.0.0-20200130185559-910be7a94367 - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 + golang.org/x/net v0.0.0-20210614182718-04defd469f4e golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c - golang.org/x/text v0.3.5 + golang.org/x/text v0.3.6 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/tools v0.1.1 google.golang.org/api v0.15.0 @@ -180,7 +180,11 @@ require ( google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.25.0 gopkg.in/inf.v0 v0.9.1 + gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect + gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect + gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect gopkg.in/jcmturner/gokrb5.v7 v7.5.0 + gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 gopkg.in/yaml.v2 v2.3.0 gotest.tools v2.2.0+incompatible @@ -193,7 +197,7 @@ require ( replace ( github.com/Microsoft/go-winio => github.com/bi-zone/go-winio v0.4.15 - github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877 + github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752 github.com/cucumber/godog => github.com/cucumber/godog v0.8.1 github.com/docker/docker => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725 github.com/docker/go-plugins-helpers => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f diff --git a/go.sum b/go.sum index 21851604617..fef657d908f 100644 --- a/go.sum +++ b/go.sum @@ -287,8 +287,8 @@ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUt github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gosigar v0.14.1 h1:T0aQ7n/n2ZA9W7DmAnj60v+qzqKERdBgJBO1CG2W6rc= github.com/elastic/gosigar v0.14.1/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877 h1:C9LsbipColsz04JKpKoLlp0pgMJRLq2uXVTeKRDcNcY= -github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II= +github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752 h1:5/RUNg7rkIvayjPhAIoI3v8p45NfWcfWs5DZSElycis= +github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752/go.mod h1:mdtqvCSg8JOxk8PmpTNGyo6wzd4BMm4QXSfDnTXmgkE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -302,8 +302,8 @@ github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk= -github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -377,8 +377,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.3 h1:HR0kYDX2RJZvAup8CsiJwxB4dTCSC0AaUq6S4SiLwUc= github.com/gomodule/redigo v1.8.3/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -390,8 +391,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -419,6 +421,8 @@ github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVf github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -466,8 +470,18 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA= +github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -502,8 +516,8 @@ github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1q github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg= -github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8= +github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -574,8 +588,6 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -617,8 +629,8 @@ github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= -github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= +github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 h1:i5VIxp6QB8oWZ8IkK8zrDgeT6ORGIUeiN+61iETwJbI= github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0/go.mod h1:4xpMLz7RBWyB+ElzHu8Llua96TRCB3YwX+l5EP1wmHk= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -651,8 +663,8 @@ github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -738,10 +750,10 @@ github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71/go.mod h1:Wp40HwmjM59Fk github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 h1:NeNpIvfvaFOh0BH7nMEljE5Rk/VJlxhm58M41SeOD20= github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/scram v1.0.3 h1:nTadYh2Fs4BK2xdldEa2g5bbaZp0/+1nJMMPtPxS/to= +github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= @@ -791,8 +803,8 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -840,13 +852,14 @@ golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191021144547-ec77196f6094/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -902,6 +915,7 @@ golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= @@ -912,8 +926,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= @@ -978,8 +992,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1009,8 +1023,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/gotestsum v0.6.0 h1:0zIxynXq9gkAcRpboAi3qOQIkZkCt/stfQzd7ab7Czs= diff --git a/libbeat/common/kafka/version.go b/libbeat/common/kafka/version.go index f39f019441e..3df44b86216 100644 --- a/libbeat/common/kafka/version.go +++ b/libbeat/common/kafka/version.go @@ -26,78 +26,47 @@ import ( // Version is a kafka version type Version string -// TODO: remove me. -// Compat version overwrite for missing versions in sarama -// Public API is compatible between these versions. var ( - v0_10_2_1 = parseKafkaVersion("0.10.2.1") - v0_11_0_1 = parseKafkaVersion("0.11.0.1") - v0_11_0_2 = parseKafkaVersion("0.11.0.2") - v1_0_1 = parseKafkaVersion("1.0.1") - v1_0_2 = parseKafkaVersion("1.0.2") - v1_1_1 = parseKafkaVersion("1.1.1") - - kafkaVersions = map[string]sarama.KafkaVersion{ - "0.8.2.0": sarama.V0_8_2_0, - "0.8.2.1": sarama.V0_8_2_1, - "0.8.2.2": sarama.V0_8_2_2, - "0.8.2": sarama.V0_8_2_2, - "0.8": sarama.V0_8_2_2, - - "0.9.0.0": sarama.V0_9_0_0, - "0.9.0.1": sarama.V0_9_0_1, - "0.9.0": sarama.V0_9_0_1, - "0.9": sarama.V0_9_0_1, - - "0.10.0.0": sarama.V0_10_0_0, - "0.10.0.1": sarama.V0_10_0_1, - "0.10.0": sarama.V0_10_0_1, - "0.10.1.0": sarama.V0_10_1_0, - "0.10.1": sarama.V0_10_1_0, - "0.10.2.0": sarama.V0_10_2_0, - "0.10.2.1": v0_10_2_1, - "0.10.2": v0_10_2_1, - "0.10": v0_10_2_1, - - "0.11.0.0": sarama.V0_11_0_0, - "0.11.0.1": v0_11_0_1, - "0.11.0.2": v0_11_0_2, - "0.11.0": v0_11_0_2, - "0.11": v0_11_0_2, - - "1.0.0": sarama.V1_0_0_0, - "1.0.1": v1_0_1, - "1.0.2": v1_0_2, - "1.0": v1_0_2, - "1.1.0": sarama.V1_1_0_0, - "1.1.1": v1_1_1, - "1.1": v1_1_1, - "1": v1_1_1, - - "2.0.0": sarama.V2_0_0_0, - "2.0.1": sarama.V2_0_1_0, - "2.0": sarama.V2_0_1_0, - "2.1": sarama.V2_1_0_0, - "2.2": sarama.V2_2_0_0, - "2": sarama.V2_1_0_0, + // Sarama expects version strings to be fully expanded, e.g. "1.1.1". + // We also allow versions to be specified as a prefix, e.g. "1", + // understood as referencing the most recent version starting with "1". + // truncatedKafkaVersions stores a lookup of the abbreviations we accept. + truncatedKafkaVersions = map[string]sarama.KafkaVersion{ + "0.8.2": sarama.V0_8_2_2, + "0.8": sarama.V0_8_2_2, + + "0.9.0": sarama.V0_9_0_1, + "0.9": sarama.V0_9_0_1, + + "0.10.0": sarama.V0_10_0_1, + "0.10.1": sarama.V0_10_1_0, + "0.10.2": sarama.V0_10_2_1, + "0.10": sarama.V0_10_2_1, + + "0.11.0": sarama.V0_11_0_2, + "0.11": sarama.V0_11_0_2, + + "1.0": sarama.V1_0_0_0, + "1.1": sarama.V1_1_1_0, + "1": sarama.V1_1_1_0, + + "2.0": sarama.V2_0_1_0, + "2.1": sarama.V2_1_0_0, + "2.2": sarama.V2_2_0_0, + "2.3": sarama.V2_3_0_0, + "2.4": sarama.V2_4_0_0, + "2.5": sarama.V2_5_0_0, + "2.6": sarama.V2_6_0_0, + "2": sarama.V2_6_0_0, } ) -func parseKafkaVersion(s string) sarama.KafkaVersion { - v, err := sarama.ParseKafkaVersion(s) - if err != nil { - panic(err) - } - return v -} - // Validate that a kafka version is among the possible options func (v *Version) Validate() error { - if _, ok := kafkaVersions[string(*v)]; !ok { - return fmt.Errorf("unknown/unsupported kafka vesion '%v'", *v) + if _, ok := v.Get(); ok { + return nil } - - return nil + return fmt.Errorf("unknown/unsupported kafka version '%v'", *v) } // Unpack a kafka version @@ -113,6 +82,20 @@ func (v *Version) Unpack(s string) error { // Get a sarama kafka version func (v Version) Get() (sarama.KafkaVersion, bool) { - kv, ok := kafkaVersions[string(v)] - return kv, ok + // First check if it's one of the abbreviations we accept. + // If not, let sarama parse it. + s := string(v) + if version, ok := truncatedKafkaVersions[s]; ok { + return version, true + } + version, err := sarama.ParseKafkaVersion(s) + if err != nil { + return sarama.KafkaVersion{}, false + } + for _, supp := range sarama.SupportedVersions { + if version == supp { + return version, true + } + } + return sarama.KafkaVersion{}, false } diff --git a/libbeat/common/kafka/version_test.go b/libbeat/common/kafka/version_test.go new file mode 100644 index 00000000000..19d5b04ad23 --- /dev/null +++ b/libbeat/common/kafka/version_test.go @@ -0,0 +1,80 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kafka + +import ( + "testing" + + "github.com/Shopify/sarama" +) + +func TestVersionGet(t *testing.T) { + valid := map[Version]sarama.KafkaVersion{ + "0.11": sarama.V0_11_0_2, + "1": sarama.V1_1_1_0, + "2.0.0": sarama.V2_0_0_0, + "2.0.1": sarama.V2_0_1_0, + "2.0": sarama.V2_0_1_0, + "2.5": sarama.V2_5_0_0, + } + invalid := []Version{ + "1.1.2", + "1.2.3", + "1.3", + "hello", + "2.0.3", + } + for s, expect := range valid { + got, ok := s.Get() + if !ok { + t.Errorf("'%v' should parse as Kafka version %v, got nothing", + s, expect) + } else if got != expect { + t.Errorf("'%v' should parse as Kafka version %v, got %v", + s, expect, got) + } + } + for _, s := range invalid { + got, ok := s.Get() + if ok { + t.Errorf("'%v' is not a valid Kafka version but parsed as %v", + s, got) + } + } +} + +func TestSaramaUpdate(t *testing.T) { + // If any of these versions are considered valid by our parsing code, + // it means someone updated sarama without updating the parsing code + // for the new version. Gently remind them. + flagVersions := []Version{"2.8.1", "2.9.0"} + for _, v := range flagVersions { + if _, ok := v.Get(); ok { + t.Fatalf( + "Kafka version %v is now considered valid. Did you update Sarama?\n"+ + "If so, remember to:\n"+ + "- Update truncatedKafkaVersions in libbeat/common/kafka/version.go\n"+ + "- Update the documentation to list the latest version:\n"+ + " * libbeat/outputs/kafka/docs/kafka.asciidoc\n"+ + " * filebeat/docs/inputs/inputs-kafka.asciidoc\n"+ + "- Update TestSaramaUpdate in libbeat/common/kafka/version_test.go\n", + v) + + } + } +} diff --git a/libbeat/outputs/kafka/docs/kafka.asciidoc b/libbeat/outputs/kafka/docs/kafka.asciidoc index f61d4c5d985..85b467ab3f2 100644 --- a/libbeat/outputs/kafka/docs/kafka.asciidoc +++ b/libbeat/outputs/kafka/docs/kafka.asciidoc @@ -34,7 +34,7 @@ NOTE: Events bigger than <> will be [[kafka-compatibility]] ==== Compatibility -This output works with all Kafka versions in between 0.11 and 2.2.2. Older versions +This output works with all Kafka versions in between 0.11 and 2.8.0. Older versions might work as well, but are not supported. ==== Configuration options diff --git a/libbeat/service/service_unix.go b/libbeat/service/service_unix.go index 7d20b04620e..68851d7e54f 100644 --- a/libbeat/service/service_unix.go +++ b/libbeat/service/service_unix.go @@ -25,3 +25,6 @@ func ProcessWindowsControlEvents(stopCallback func()) { func notifyWindowsServiceStopped() { } + +// WaitExecutionDone is not used on non-windows platforms. +func WaitExecutionDone() {} diff --git a/libbeat/service/service_windows.go b/libbeat/service/service_windows.go index a81f4fb5a0f..027011911c2 100644 --- a/libbeat/service/service_windows.go +++ b/libbeat/service/service_windows.go @@ -29,13 +29,15 @@ import ( ) type beatService struct { - stopCallback func() - done chan struct{} + stopCallback func() + done chan struct{} + executeFinished chan struct{} } var serviceInstance = &beatService{ - stopCallback: nil, - done: make(chan struct{}, 0), + stopCallback: nil, + done: make(chan struct{}, 0), + executeFinished: make(chan struct{}, 0), } // Execute runs the beat service with the arguments and manages changes that @@ -85,6 +87,8 @@ const couldNotConnect syscall.Errno = 1063 // stopCallback function is called when the Stop/Shutdown // request is received. func ProcessWindowsControlEvents(stopCallback func()) { + defer close(serviceInstance.executeFinished) + isInteractive, err := svc.IsAnInteractiveSession() if err != nil { logp.Err("IsAnInteractiveSession: %v", err) @@ -125,3 +129,17 @@ func ProcessWindowsControlEvents(stopCallback func()) { logp.Err("Windows service setup failed: %+v", err) } + +// WaitExecutionDone returns only after stop was reported to service manager. +// If response is not retrieved within 500 millisecond wait is aborted. +func WaitExecutionDone() { + if isWinService, err := svc.IsWindowsService(); err != nil || !isWinService { + // not a service, don't wait + return + } + + select { + case <-serviceInstance.executeFinished: + case <-time.After(500 * time.Millisecond): + } +} diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index c9f02de59fb..cf24a932a91 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -59,10 +59,14 @@ func newRunCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command { } } -func run(streams *cli.IOStreams, override cfgOverrider) error { // Windows: Mark service as stopped. +func run(streams *cli.IOStreams, override cfgOverrider) error { + // Windows: Mark service as stopped. // After this is run, the service is considered by the OS to be stopped. // This must be the first deferred cleanup task (last to execute). - defer service.NotifyTermination() + defer func() { + service.NotifyTermination() + service.WaitExecutionDone() + }() locker := filelock.NewAppLocker(paths.Data(), paths.AgentLockFileName) if err := locker.TryLock(); err != nil { diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 58e555e1268..09c837bccf6 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -101,6 +101,19 @@ stages: platforms: ## override default labels in this specific stage. - "windows-7" stage: extended + cloud: + cloud: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + dirs: ## apply terraform for the given directory. + - "x-pack/filebeat/input/awss3/_meta/terraform" + when: ## Override the top-level when. + parameters: + - "awsCloudTests" + comments: + - "/test x-pack/filebeat for aws cloud" + labels: + - "aws" + stage: extended #windows-7-32: See https://github.com/elastic/beats/issues/22315 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 1e928773da2..6b5444be055 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -103,10 +103,9 @@ The state has the following elements: - `last_response.page`: A number indicating the page number of the last response. - `first_event`: A map representing the first event sent to the output (result from applying transforms to `last_response.body`). - `last_event`: A map representing the last event sent to the output (result from applying transforms to `last_response.body`). -- `url.value`: The full URL with params and fragments. -- `url.params`: A map containing the URL params. -- `header`: A map containing the headers. References request headers when used in <> or <> configuration sections, and to the last response headers when used in <>, <> or <> configuration sections. -- `body`: A map containing the body. References request body when used in <> configuration section, and to the last response body when used in <>, <> or <> configuration sections. +- `url`: The last requested URL as a raw https://pkg.go.dev/net/url#URL[`url.URL`] Go type. +- `header`: A map containing the headers. References the next request headers when used in <> or <> configuration sections, and to the last response headers when used in <>, <>, or <> configuration sections. +- `body`: A map containing the body. References the next request body when used in <> or <> configuration sections, and to the last response body when used in <> or <> configuration sections. - `cursor`: A map containing any data the user configured to be stored between restarts (See <>). All of the mentioned objects are only stored at runtime, except `cursor`, which has values that are persisted between restarts. @@ -206,6 +205,8 @@ Some built-in helper functions are provided to work with the input state inside - `hmac`: calculates the hmac signature of a list of strings concatenated together. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` - `base64Encode`: Joins and base64 encodes all supplied strings. Example `[[base64Encode "string1" "string2"]]` - `base64EncodeNoPad`: Joins and base64 encodes all supplied strings without padding. Example `[[base64EncodeNoPad "string1" "string2"]]` +- `join`: joins a list of strings using the specified separator. Example: `[[join .body.arr ","]]` +- `sprintf`: formats according to a format specifier and returns the resulting string. Refer to https://pkg.go.dev/fmt#Sprintf[the Go docs] for usage. Example: `[[sprintf "%d:%q" 34 "quote this"]]` In addition to the provided functions, any of the native functions for https://golang.org/pkg/time/#Time[`time.Time`], https://golang.org/pkg/net/http/#Header[`http.Header`], and https://golang.org/pkg/net/url/#Values[`url.Values`] types can be used on the corresponding objects. Examples: `[[(now).Day]]`, `[[.last_response.header.Get "key"]]` @@ -463,7 +464,7 @@ List of transforms to apply to the request before each execution. Available transforms for request: [`append`, `delete`, `set`]. -Can read state from: [`.last_response.*`, `.last_event.*`, `.cursor.*`]. +Can read state from: [`.last_response.*`, `.last_event.*`, `.cursor.*`, `.header.*`, `.url.*`, `.body.*`]. Can write state to: [`header.*`, `url.params.*`, `body.*`]. @@ -583,7 +584,7 @@ List of transforms that will be applied to the response to every new page reques Available transforms for pagination: [`append`, `delete`, `set`]. -Can read state from: [`.last_response.*`, `.first_event.*`, `.last_event.*`, `.cursor.*`]. +Can read state from: [`.last_response.*`, `.first_event.*`, `.last_event.*`, `.cursor.*`, `.header.*`, `.url.*`, `.body.*`]. Can write state to: [`body.*`, `header.*`, `url.*`]. diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_test.go b/x-pack/filebeat/input/httpjson/internal/v2/input_test.go index bbfdb1e5357..788a3f2d1af 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input_test.go @@ -58,9 +58,9 @@ func TestInput(t *testing.T) { baseConfig: map[string]interface{}{ "interval": 1, "http_method": "GET", - "request.rate_limit.limit": `[[.last_request.header.Get "X-Rate-Limit-Limit"]]`, - "request.rate_limit.remaining": `[[.last_request.header.Get "X-Rate-Limit-Remaining"]]`, - "request.rate_limit.reset": `[[.last_request.header.Get "X-Rate-Limit-Reset"]]`, + "request.rate_limit.limit": `[[.last_response.header.Get "X-Rate-Limit-Limit"]]`, + "request.rate_limit.remaining": `[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`, + "request.rate_limit.reset": `[[.last_response.header.Get "X-Rate-Limit-Reset"]]`, }, handler: rateLimitHandler(), expected: []string{`{"hello":"world"}`}, @@ -320,6 +320,75 @@ func TestInput(t *testing.T) { handler: oauth2Handler, expected: []string{`{"hello": "world"}`}, }, + { + name: "Test request transforms can access state from previous transforms", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerRequestTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + "/test-path" + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "POST", + "request.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "header.X-Foo", + "value": "foo", + }, + }, + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.bar", + "value": `[[.header.Get "X-Foo"]]`, + }, + }, + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.url.path", + "value": `[[.url.Path]]`, + }, + }, + }, + }, + handler: defaultHandler("POST", `{"bar":"foo","url":{"path":"/test-path"}}`), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, + { + name: "Test response transforms can't access request state from previous transforms", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerRequestTransforms() + registerResponseTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 10, + "request.method": "GET", + "request.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "header.X-Foo", + "value": "foo", + }, + }, + }, + "response.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.bar", + "value": `[[.header.Get "X-Foo"]]`, + }, + }, + }, + }, + handler: defaultHandler("GET", ""), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, } for _, testCase := range testCases { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go index 5d457dea7ae..569c0c84f18 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go @@ -102,9 +102,10 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { } tr := transformable{} - tr.setHeader(resp.Header) + ctx := emptyTransformContext() + ctx.updateLastResponse(response{header: resp.Header.Clone()}) - remaining, _ := r.remaining.Execute(emptyTransformContext(), tr, nil, r.log) + remaining, _ := r.remaining.Execute(ctx, tr, nil, r.log) if remaining == "" { return 0, errors.New("remaining value is empty") } @@ -122,7 +123,7 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { return 0, nil } - reset, _ := r.reset.Execute(emptyTransformContext(), tr, nil, r.log) + reset, _ := r.reset.Execute(ctx, tr, nil, r.log) if reset == "" { return 0, errors.New("reset value is empty") } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go index 7e41272a772..8e9e0cb0db8 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go @@ -24,9 +24,9 @@ func TestGetRateLimitReturns0IfRemainingQuota(t *testing.T) { tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} - assert.NoError(t, tplLimit.Unpack(`[[.header.Get "X-Rate-Limit-Limit"]]`)) - assert.NoError(t, tplReset.Unpack(`[[.header.Get "X-Rate-Limit-Reset"]]`)) - assert.NoError(t, tplRemaining.Unpack(`[[.header.Get "X-Rate-Limit-Remaining"]]`)) + assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) + assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) + assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) rateLimit := &rateLimiter{ limit: tplLimit, reset: tplReset, @@ -47,9 +47,9 @@ func TestGetRateLimitReturns0IfEpochInPast(t *testing.T) { tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} - assert.NoError(t, tplLimit.Unpack(`[[.header.Get "X-Rate-Limit-Limit"]]`)) - assert.NoError(t, tplReset.Unpack(`[[.header.Get "X-Rate-Limit-Reset"]]`)) - assert.NoError(t, tplRemaining.Unpack(`[[.header.Get "X-Rate-Limit-Remaining"]]`)) + assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) + assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) + assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) rateLimit := &rateLimiter{ limit: tplLimit, reset: tplReset, @@ -74,9 +74,9 @@ func TestGetRateLimitReturnsResetValue(t *testing.T) { tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} - assert.NoError(t, tplLimit.Unpack(`[[.header.Get "X-Rate-Limit-Limit"]]`)) - assert.NoError(t, tplReset.Unpack(`[[.header.Get "X-Rate-Limit-Reset"]]`)) - assert.NoError(t, tplRemaining.Unpack(`[[.header.Get "X-Rate-Limit-Remaining"]]`)) + assert.NoError(t, tplLimit.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Limit"]]`)) + assert.NoError(t, tplReset.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Reset"]]`)) + assert.NoError(t, tplRemaining.Unpack(`[[.last_response.header.Get "X-Rate-Limit-Remaining"]]`)) rateLimit := &rateLimiter{ limit: tplLimit, reset: tplReset, diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index b6c74590b9d..9f91fcdccce 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -12,6 +12,7 @@ import ( "encoding/base64" "encoding/hex" "errors" + "fmt" "hash" "reflect" "regexp" @@ -57,6 +58,8 @@ func (t *valueTpl) Unpack(in string) error { "hmac": hmacString, "base64Encode": base64Encode, "base64EncodeNoPad": base64EncodeNoPad, + "join": strings.Join, + "sprintf": fmt.Sprintf, }). Delims(leftDelim, rightDelim). Parse(in) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go index 75f291b22be..4586dec7711 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go @@ -291,6 +291,46 @@ func TestValueTpl(t *testing.T) { expectedVal: "", expectedError: errEmptyTemplateResult.Error(), }, + { + name: "func join", + value: `[[join .last_response.body.arr ","]]`, + paramCtx: &transformContext{ + firstEvent: &common.MapStr{}, + lastEvent: &common.MapStr{}, + lastResponse: newTestResponse( + common.MapStr{ + "arr": []string{ + "foo", + "bar", + }, + }, + http.Header{}, + "", + ), + }, + paramTr: transformable{}, + expectedVal: "foo,bar", + }, + { + name: "func sprintf", + value: `[[sprintf "%q:%d" (join .last_response.body.arr ",") 1]]`, + paramCtx: &transformContext{ + firstEvent: &common.MapStr{}, + lastEvent: &common.MapStr{}, + lastResponse: newTestResponse( + common.MapStr{ + "arr": []string{ + "foo", + "bar", + }, + }, + http.Header{}, + "", + ), + }, + paramTr: transformable{}, + expectedVal: `"foo,bar":1`, + }, } for _, tc := range cases { diff --git a/x-pack/metricbeat/module/aws/terraform.tf b/x-pack/metricbeat/module/aws/terraform.tf index 4f00b88df5b..93c85836fc2 100644 --- a/x-pack/metricbeat/module/aws/terraform.tf +++ b/x-pack/metricbeat/module/aws/terraform.tf @@ -50,7 +50,7 @@ resource "aws_s3_bucket_object" "test" { resource "aws_instance" "test" { ami = data.aws_ami.latest-amzn.id monitoring = true - instance_type = "t1.micro" + instance_type = "t2.micro" tags = { Name = "metricbeat-test" }