diff --git a/.backportrc.json b/.backportrc.json index 3764fa549c..4abd8f04cd 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -2,6 +2,7 @@ "upstream": "elastic/ecs", "branches": [ { "name": "master", "checked": true }, + "8.0", "1.12", "1.11", "1.10", @@ -19,7 +20,7 @@ "targetPRLabels": ["backport"], "prFilter": "label:needs_backport", "branchLabelMapping": { - "^8.0.0$": "master", + "^8.1.0$": "master", "^(\\d+).(\\d+).\\d+$": "$1.$2" } } diff --git a/.github/ISSUE_TEMPLATE/schema-changes-additions.md b/.github/ISSUE_TEMPLATE/schema-changes-additions.md index 847af99b75..ff50c1f3fe 100644 --- a/.github/ISSUE_TEMPLATE/schema-changes-additions.md +++ b/.github/ISSUE_TEMPLATE/schema-changes-additions.md @@ -7,7 +7,7 @@ labels: "enhancement" diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a3d2e4d9a8..5a03ece333 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,9 +6,9 @@ our submission, but they are here to help bring them to your attention. --> - Have you signed the [contributor license agreement](https://www.elastic.co/contributor-agreement)? -- Have you followed the [contributor guidelines](https://github.com/elastic/ecs/blob/master/CONTRIBUTING.md)? -- For proposing substantial changes or additions to the schema, have you reviewed the [RFC process](https://github.com/elastic/ecs/blob/master/rfcs/README.md)? +- Have you followed the [contributor guidelines](https://github.com/elastic/ecs/blob/main/CONTRIBUTING.md)? +- For proposing substantial changes or additions to the schema, have you reviewed the [RFC process](https://github.com/elastic/ecs/blob/main/rfcs/README.md)? - If submitting code/script changes, have you verified all tests pass locally using `make test`? - If submitting schema/fields updates, have you generated new artifacts by running `make` and committed those changes? -- Is your pull request against master? Unless there is a good reason otherwise, we prefer pull requests against master and will backport as needed. -- Have you added an entry to the [CHANGELOG.next.md](https://github.com/elastic/ecs/blob/master/CHANGELOG.next.md)? +- Is your pull request against main? Unless there is a good reason otherwise, we prefer pull requests against main and will backport as needed. +- Have you added an entry to the [CHANGELOG.next.md](https://github.com/elastic/ecs/blob/main/CHANGELOG.next.md)? diff --git a/CHANGELOG.md b/CHANGELOG.md index 821f3d5182..f467356186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,57 @@ # CHANGELOG All notable changes to this project will be documented in this file based on the [Keep a Changelog](http://keepachangelog.com/) Standard. This project adheres to [Semantic Versioning](http://semver.org/). +## [1.12.1](https://github.com/elastic/ecs/compare/v1.12.0...v1.12.1) + +### Schema Changes + +#### Bugfixes + +* Updating `x509` order to correct nesting. ##1621 + +## [1.12.0](https://github.com/elastic/ecs/compare/v1.11.0...v1.12.0) + +### Schema Changes + +#### Bugfixes + +* Updating `hash` order to correct nesting. #1603 +* Removing incorrect `hash` reuses. #1604 +* Updating `pe` order to correct nesting. #1605 +* Removing incorrect `pe` reuses. #1606 +* Correcting `enrichments` to an `array` type. #1608 + +#### Added + +* Added `file.fork_name` field. #1288 +* Added `service.address` field. #1537 +* Added `service.environment` as a beta field. #1541 +* Added `process.end` field. #1544 +* Added container metric fields into experimental schema. #1546 +* Add `code_signature.digest_algorithm` and `code_signature.timestamp` fields. #1557 +* Add `email.*` field set in the experimental fields. #1569 + +#### Improvements + +* Beta migration on some `keyword` fields to `wildcard`. #1517 +* Promote `threat.software.*` and `threat.group.*` fields to GA. #1540 +* Update `user.name` and `user.id` examples for clarity. #1566 +* Beta migration of `text` and `.text` multi-fields to `match_only_text`. #1532, #1571 + +### Tooling and Artifact Changes + +#### Added + +* Support ES 6.x type fallback for `match_only_text` field types. #1528 + +#### Bugfixes + +* Prevent failure if no files need to be deleted `find | xargs rm`. #1588 + +#### Improvements + +* Document field type family interoperability in FAQ. #1591 + ## [1.11.0](https://github.com/elastic/ecs/compare/v1.10.0...v1.11.0) ### Schema Changes diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 40a92b0a7c..aad7035d95 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -12,67 +12,68 @@ Thanks, you're awesome :-) --> #### Breaking changes -* Remove `host.user.*` field reuse. #1439 -* Remove deprecation notice on `http.request.method`. #1443 -* Migrate `log.origin.file.line` from `integer` to `long`. #1533 -* Remove `log.original` field. #1580 - #### Bugfixes #### Added #### Improvements -* Wildcard type field migration GA. #1582 -* `match_only_text` type field migration GA. #1584 -* Threat indicator fields GA from RFC 0008. #1586 - #### Deprecated -### Tooling and Artifact Changes +#### Removed -#### Breaking Changes +- Removing `process.target.*` reuses from experimental schema. #1666 -* Removing deprecated --oss from generator #1404 -* Removing use-cases directory #1405 -* Remove Go code generator. #1567 +### Tooling and Artifact Changes + +#### Breaking changes #### Bugfixes +* Add `object` as fallback for `flattened` type. #1653 + #### Added #### Improvements -* Remove remaining Go deps after removing Go code generator. #1585 +* Update refs from master to main in USAGE.md etc #1658 #### Deprecated -## 1.12.0 (Feature Freeze) +## 8.0.0 (Feature Freeze) ### Schema Changes +#### Breaking changes + +* Remove `host.user.*` field reuse. #1439 +* Remove deprecation notice on `http.request.method`. #1443 +* Migrate `log.origin.file.line` from `integer` to `long`. #1533 +* Remove `log.original` field. #1580 +* Remove `process.ppid` field. #1596 + #### Added -* Added `file.fork_name` field. #1288 -* Added `service.address` field. #1537 -* Added `service.environment` as a beta field. #1541 -* Added `process.end` field. #1544 -* Added container metric fields into experimental schema. #1546 -* Add `code_signature.digest_algorithm` and `code_signature.timestamp` fields. #1557 -* Add `email.*` field set in the experimental fields. #1569 +* Added `faas.*` field set as beta. #1628 #### Improvements -* Beta migration on some `keyword` fields to `wildcard`. #1517 -* Promote `threat.software.*` and `threat.group.*` fields to GA. #1540 -* Update `user.name` and `user.id` examples for clarity. #1566 -* Beta migration of `text` and `.text` multi-fields to `match_only_text`. #1532, #1571 +* Wildcard type field migration GA. #1582 +* `match_only_text` type field migration GA. #1584 +* Threat indicator fields GA from RFC 0008. #1586 ### Tooling and Artifact Changes -#### Added +#### Breaking Changes -* Support ES 6.x type fallback for `match_only_text` field types. #1528 +* Removing deprecated --oss from generator #1404 +* Removing use-cases directory #1405 +* Remove Go code generator. #1567 + +#### Improvements + +* Remove remaining Go deps after removing Go code generator. #1585 +* Add explicit `default_field: true` for Beats artifacts. #1633 -- Stage: **1 (draft)** -- Date: **2021-06-09** - - +- Stage: **X (abandoned)** +- Date: **2021-11-16** This RFC is to add model events that span multiple processes. There are some events for when one OS process accesses another. In Windows, this starts with a call to `OpenProcess` to gain a handle and then there are several APIs for things you can do with the handle once its open. For all of these operations, the general concept persists: one process requested access to another. @@ -14,6 +12,10 @@ The most common use cases for Windows: * attaching a debugger * reading the Process Environment Block (PEB) for other benign or nefarious purpose +## Stage X + +This RFC is not being worked on actively, and it has been marked as abandoned. If an individual wishes to advance it in the future, open a new pull request against this proposal. + ## Fields **Stage 0** @@ -169,3 +171,4 @@ The following are the people that consulted on the contents of this RFC. * Stage 0: https://github.com/elastic/ecs/pull/1286 * Stage 1: https://github.com/elastic/ecs/pull/1297 +* Stage X: https://github.com/elastic/ecs/pull/1666 diff --git a/rfcs/text/0017-remove-log-original.md b/rfcs/text/0017-remove-log-original.md index 30a4934e45..eeab5758c9 100644 --- a/rfcs/text/0017-remove-log-original.md +++ b/rfcs/text/0017-remove-log-original.md @@ -16,23 +16,23 @@ The request is to consolidate `log.original` and `event.original` by removing `l - The internal description of the field `log.original` in [`log`](0017/log.yml) should be amended by addition of a notice of deprecation and subsequently removal if/when Deprecation progresses to Removal -- The internal description of the field `event.original` in [`event`](0017/event.yml) should be updated to reflect the revised scope +- The internal description of the field `event.original` in [`event`](0017/event.yml) should be updated to reflect the revised scope - The extended description of `log.original` in the [`Log Fields documentation`](https://github.com/elastic/ecs/blob/master/docs/field-details.asciidoc#field-log-original) should be amended by addition of a notice of deprecation and subsequently removal if/when Deprecation progresses to Removal - The extended description of `event.original` in the [`Event Fields documentation`](https://github.com/elastic/ecs/blob/master/docs/field-details.asciidoc#field-event-original) should be amended to clarify the absorption of `log.original` - + ## Usage -The following examples are taken verbatim from the existing field definitions +The following examples are taken verbatim from the existing field definitions and are included for completeness. -These are the raw texts of entire events, for example a log message. They -differ from the extracted `message` field in that no processing has been -applied and the field is not indexed by default. The field can still be +These are the raw texts of entire events, for example a log message. They +differ from the extracted `message` field in that no processing has been +applied and the field is not indexed by default. The field can still be retrieved from `_source` and is well-suited to demonstration of log integrity -or in a re-index pipeline. +or in a re-index pipeline. ## Source data @@ -116,4 +116,4 @@ The following are the people that consulted on the contents of this RFC. * Stage 1: https://github.com/elastic/ecs/pull/1314 * Stage 2: https://github.com/elastic/ecs/pull/1347 * Stage 3: https://github.com/elastic/ecs/pull/1465 -* Implementation: https://github.com/elastic/ecs/pull/1469 \ No newline at end of file +* Implementation: https://github.com/elastic/ecs/pull/1469 diff --git a/rfcs/text/0017/event.yml b/rfcs/text/0017/event.yml index a26414e52e..da4a5eafda 100644 --- a/rfcs/text/0017/event.yml +++ b/rfcs/text/0017/event.yml @@ -2,12 +2,12 @@ - name: event - name: original description: > - Raw text message of entire event. Used to demonstrate log integrity - or where the full log message (before splitting it up in multiple + Raw text message of entire event. Used to demonstrate log integrity + or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. - \ No newline at end of file + diff --git a/rfcs/text/0017/log.yml b/rfcs/text/0017/log.yml index 45d81e6e72..6bddebb800 100644 --- a/rfcs/text/0017/log.yml +++ b/rfcs/text/0017/log.yml @@ -3,7 +3,7 @@ - name: original short: Deprecated Original log message with light interpretation only (encoding, newlines). description: > - Deprecated for Removal in next major version release. This field is superceeded by + Deprecated for Removal in next major version release. This field is superceeded by `event.original` This is the original log message and contains the full log message diff --git a/rfcs/text/0021-threat-enrichment.md b/rfcs/text/0021-threat-enrichment.md index 803d48a457..a7e8efd00a 100644 --- a/rfcs/text/0021-threat-enrichment.md +++ b/rfcs/text/0021-threat-enrichment.md @@ -1,7 +1,7 @@ # 0021: Threat Enrichment -- Stage: **2 (candidate)** -- Date: **2021-07-06** +- Stage: **3 (finished)** +- Date: **2021-10-20** -- Stage: **2 (candidate)** -- Date: **2021-08-05** +- Stage: **3 (finished)** +- Date: **2021-08-26** - ## Scope of impact ### Ingestion mechanisms @@ -91,7 +87,7 @@ APM, Beats, Elastic Agent, and any processors that populate `process.ppid` today ### Usage mechanisms -The security detection rules [repo](https://github.com/elastic/detection-rules) will need audited. Any usage of `process.ppid` should ideally migrate to `process.parent.pid`, but backward compatibility also remains essential. +The security detection rules [repo](https://github.com/elastic/detection-rules) will need auditing. Any usage of `process.ppid` should ideally migrate to `process.parent.pid`, but backward compatibility also remains essential. ### ECS @@ -105,7 +101,7 @@ The `process.ppid` is populated in many data producers, so migrating to `process **Resolution**: Field aliases might be of some use to alleviate some pain during the migration for any aggregations or visualizations relying on `process.ppid`: -``` +```json PUT rfc_0018/_mapping { "properties": { @@ -134,10 +130,6 @@ Removing `process.ppid` will also remove its reuse in `process.parent`: `process **Resolution**: [Discussed](https://github.com/elastic/ecs/pull/1450#issuecomment-854773783) with Protections, Endpoint, and Observability stakeholders. Not having a replacement field for the parent's parent PID didn't raise significant concerns. - - ## People The following are the people that consulted on the contents of this RFC. @@ -174,6 +166,7 @@ e.g.: * Stage 1: https://github.com/elastic/ecs/pull/1450 * Stage 1 date correction: https://github.com/elastic/ecs/pull/1555 * Stage 2: https://github.com/elastic/ecs/pull/1556 +* Stage 3: https://github.com/elastic/ecs/pull/1592 -- Stage: **1 (draft)** -- Date: **2021-08-19** +- Stage: **2 (candidate)** +- Date: **2021-09-14** +Done. ## Usage @@ -116,6 +117,160 @@ Faas functions provide meta-information in their execution environment. APM agen +The above fields will be derived by the APM agents from the AWS Lambda `context object` and the `event object` that are passed with an invocation of a Lambda function. Below is an example for the context and event object. +The mapping to the proposed fields for this example is layed out in the following table + +target ECS field | source field +--- | --- +faas.coldstart | No source field. Determined by the APM agent on the first Lambda function invocation. +faas.execution | `context.awsRequestId` +faas.trigger.type | No source field. Determined by the APM agent based on the `event object` type. Would be `http` in this example. +faas.trigger.request_id | `event.requestContext.requestId` +service.origin.name | `${event.requestContext.httpMethod} ${event.requestContext.resourcePath}/${event.requestContext.stage}` -> `GET /fetch_all/dev` +service.origin.id | `event.requestContext.apiId` +service.origin.version | No source field. Determined by the APM agent based on the `event object` type whether it's API version `1.0` or `2.0`. +cloud.origin.service.name | `api gateway` +cloud.origin.account.id | `event.requestContext.accountId` + +### AWS Lambda context object +Description [available here](https://docs.aws.amazon.com/lambda/latest/dg/nodejs-context.html). + +**context:** +```json +{ + "callbackWaitsForEmptyEventLoop": true, + "functionVersion": "$LATEST", + "functionName": "the-function-name", + "memoryLimitInMB": "128", + "logGroupName": "/aws/lambda/the-function-name", + "logStreamName": "2021/08/13/[$LATEST]08834acf4e4f463b95b7b99aa8b34aff", + "invokedFunctionArn": "arn:aws:lambda:us-west-2:XXXXXXXXXXXX:function:the-function-name", + "awsRequestId": "649bf7d0-c6ae-432d-899d-da44ccd7ee95" +} +``` +### AWS Lambda event object +Description [available here](https://docs.aws.amazon.com/lambda/latest/dg/services-apigateway.html). + +**event:** +```json +{ + "resource": "/fetch_all", + "path": "/fetch_all", + "httpMethod": "GET", + "headers": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-US,en;q=0.5", + "CloudFront-Forwarded-Proto": "https", + "CloudFront-Is-Desktop-Viewer": "true", + "CloudFront-Is-Mobile-Viewer": "false", + "CloudFront-Is-SmartTV-Viewer": "false", + "CloudFront-Is-Tablet-Viewer": "false", + "CloudFront-Viewer-Country": "US", + "Host": "02plqthge2.execute-api.us-east-1.amazonaws.com", + "upgrade-insecure-requests": "1", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0", + "Via": "2.0 969f35f01b6eddd92239a3e818fc1e0d.cloudfront.net (CloudFront)", + "X-Amz-Cf-Id": "eDbpfDwO-CRYymEFLkW6CBCsU_H_PS8R93_us53QWvXWLS45v3NvQw==", + "X-Amzn-Trace-Id": "Root=1-5e502af4-fd0c1c6fdc164e1d6361183b", + "X-Forwarded-For": "76.76.241.57, 52.46.47.139", + "X-Forwarded-Port": "443", + "X-Forwarded-Proto": "https" + }, + "multiValueHeaders": { + "Accept": [ + "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" + ], + "Accept-Encoding": [ + "gzip, deflate, br" + ], + "Accept-Language": [ + "en-US,en;q=0.5" + ], + "CloudFront-Forwarded-Proto": [ + "https" + ], + "CloudFront-Is-Desktop-Viewer": [ + "true" + ], + "CloudFront-Is-Mobile-Viewer": [ + "false" + ], + "CloudFront-Is-SmartTV-Viewer": [ + "false" + ], + "CloudFront-Is-Tablet-Viewer": [ + "false" + ], + "CloudFront-Viewer-Country": [ + "US" + ], + "Host": [ + "02plqthge2.execute-api.us-east-1.amazonaws.com" + ], + "upgrade-insecure-requests": [ + "1" + ], + "User-Agent": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0" + ], + "Via": [ + "2.0 969f35f01b6eddd92239a3e818fc1e0d.cloudfront.net (CloudFront)" + ], + "X-Amz-Cf-Id": [ + "eDbpfDwO-CRYymEFLkW6CBCsU_H_PS8R93_us53QWvXWLS45v3NvQw==" + ], + "X-Amzn-Trace-Id": [ + "Root=1-5e502af4-fd0c1c6fdc164e1d6361183b" + ], + "X-Forwarded-For": [ + "76.76.241.57, 52.46.47.139" + ], + "X-Forwarded-Port": [ + "443" + ], + "X-Forwarded-Proto": [ + "https" + ] + }, + "queryStringParameters": null, + "multiValueQueryStringParameters": null, + "pathParameters": null, + "stageVariables": null, + "requestContext": { + "resourceId": "y3tkf7", + "resourcePath": "/fetch_all", + "httpMethod": "GET", + "extendedRequestId": "IQumRELJIAMF6fQ=", + "requestTime": "21/Feb/2020:19:09:40 +0000", + "path": "/dev/fetch_all", + "accountId": "571481734049", + "protocol": "HTTP/1.1", + "stage": "dev", + "domainPrefix": "02plqthge2", + "requestTimeEpoch": 1582312180890, + "requestId": "6f3dffca-46f8-4c8b-800b-6bc1ea2554ec", + "identity": { + "cognitoIdentityPoolId": null, + "accountId": null, + "cognitoIdentityId": null, + "caller": null, + "sourceIp": "76.76.241.57", + "principalOrgId": null, + "accessKey": null, + "cognitoAuthenticationType": null, + "cognitoAuthenticationProvider": null, + "userArn": null, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0", + "user": null + }, + "domainName": "02plqthge2.execute-api.us-east-1.amazonaws.com", + "apiId": "02plqthge2" + }, + "body": null, + "isBase64Encoded": false +} +``` +- Ingestion mechanisms: + - APM server will extend the intake V2 API to accept the new fields and store them with the transaction documents + - APM server will extend OpenTelemetry field mapping to account for these new fields +- Usage mechanisms: + - APM UI may utilize the new fields to provide Lambda / serverless specific visualizations (e.g. indicating cold starts on transactions in the waterfall view, showing meta information on lambda service views) +- ECS project + - the concept of self-nesting service and cloud fields under *origin* and *target* needs clear documentation that avoids confusion around when to use which of the fields. Tried to address this with the description in the schema for those fields in this PR. ## Concerns @@ -149,6 +311,8 @@ Stage 1: Identify potential concerns, implementation challenges, or complexity. Stage 2: Document new concerns or resolutions to previously listed concerns. It's not critical that all concerns have resolutions at this point, but it would be helpful if resolutions were taking shape for the most significant concerns. --> +- extended descriptio / footnote for service and cloud fields in this PR to avoid confusion about *origin* and *target* nesting of service and cloud fields + @@ -185,3 +349,5 @@ e.g.: * Stage 0: https://github.com/elastic/ecs/pull/1518 * Stage 1: https://github.com/elastic/ecs/pull/1542 +* Stage 2: https://github.com/elastic/ecs/pull/1594 + * Stage 2 date correction: https://github.com/elastic/ecs/pull/1642 diff --git a/rfcs/text/0027/cloud.yml b/rfcs/text/0027/cloud.yml index c061488986..cc0c22fa9a 100644 --- a/rfcs/text/0027/cloud.yml +++ b/rfcs/text/0027/cloud.yml @@ -1,11 +1,22 @@ --- - name: cloud + footnote: > + Examples: If Metricbeat is running on an EC2 host and fetches data from its + host, the cloud info contains the data about this machine. If Metricbeat + runs on a remote machine outside the cloud and fetches data from a service + running in the cloud, the field contains cloud data from the machine the + service is running on. + + The cloud fields may be self-nested under cloud.origin.* and cloud.target.* to describe origin or target service's cloud information in the context of incoming or outgoing requests, respectively. + However, the fieldsets cloud.origin.* and cloud.target.* must not be confused with the root cloud fieldset that is used to describe the cloud context of the actual service under observation. + The fieldset cloud.origin.* may only be used in the context of incoming requests or events to provide the originating service's cloud information. + The fieldset cloud.target.* may only be used in the context of outgoing requests or events to describe the target service's cloud information. reusable: top_level: true expected: - at: cloud as: target - short_override: Cloud information about the invocation target. + short_override: Provides the cloud information of the target entity in case of an outgoing request or event. - at: cloud as: origin - short_override: Cloud information about the invocation origin. + short_override: Provides the cloud information of the origin entity in case of an incoming request or event. diff --git a/rfcs/text/0027/service.yml b/rfcs/text/0027/service.yml index 8e5d932b98..1217e04f14 100644 --- a/rfcs/text/0027/service.yml +++ b/rfcs/text/0027/service.yml @@ -1,11 +1,16 @@ --- - name: service + footnote: > + The service fields may be self-nested under service.origin.* and service.target.* to describe origin or target services in the context of incoming or outgoing requests, respectively. + However, the fieldsets service.origin.* and service.target.* must not be confused with the root service fieldset that is used to describe the actual service under observation. + The fieldset service.origin.* may only be used in the context of incoming requests or events to describe the originating service of the request. + The fieldset service.target.* may only be used in the context of outgoing requests or events to describe the target service of the request. reusable: top_level: true expected: - at: service as: target - short_override: Target service of an invocation. + short_override: Describes the target service in case of an outgoing request or event. - at: service as: origin - short_override: Origin service of an invocation. + short_override: Describes the origin service in case of an incoming request or event. diff --git a/rfcs/text/0028-cgroups.md b/rfcs/text/0028-cgroups.md new file mode 100644 index 0000000000..837d7e179a --- /dev/null +++ b/rfcs/text/0028-cgroups.md @@ -0,0 +1,119 @@ +# 0028: cgroup fieldset + + +- Stage: **1 (strawperson)** +- Date: **2021-11-05** + + + +This is a proposal to add a top-level `cgroup` fields to ECS. We have cgroup V2 support incoming in the system/process metricset in 7.15, and cgroups V2 reports a variety of metric fields differently compared to V1. While many OSes and platforms are currently operating in "hybrid" or V1 only mode, this will become an issue as more OSes and cgroup-based platforms like docker make use of cgroups V2. +Right now, a handful of visualizations in the observability App within Kibana use cgroup metrics, and these visualizations will only work on Cgroups V1. In order to extract the same or similar metrics under V2, they will need to access different fields. This is an excellent use case for ECS, as it allows us to standardize the placement of common metrics such as cgroup cpu/memory usage, etc, particularly as visualizations are already relying on these metrics. +In the case of Cgroups V1 versus V2, we decided to report V2 metrics "natively" as opposed to conforming to the V1 fields in order to avoid mangling V2 metrics and confusing users who were expecting V2 metrics to be reported in a similarly transparent fashion as V1. Many of the field changes are the result of cgroup controllers changing names; for example, the V1 `blkio` controller becomes `io` on V2, with many of the underlying metrics remaining the same. + +The scope of cgroup metrics may also expand over time, as other processors and data streams add cgroup metrics, thus requiring common fields for metrics reported across different data streams. This is also relevant to container monitoring, as we may want to report "raw" cgroup metrics from containers as our container monitoring expands. + + + + +## Fields + +The fields added by this RFC are _not a comprehensive list of all metrics that are shared with both cgroups V1 and V2_. Rather, it is a list of all shared cgroup metrics that are most likely to be useful for future monitoring and visualization. As it stands, this is currently limited to `cpu` and `memory` metrics. We can expand this, of course, but this current PR covers the most important metrics shared across cgroups. + + + +## Usage + +This rfc is meant to head off an oncoming problem, which is that various components in the Elastic Stack are incompatible with cgroups V2. Kibana, for example, will frequently use field names that are exclusive to cgroups V1: + +``` +export const METRIC_CGROUP_MEMORY_LIMIT_BYTES = 'system.process.cgroup.memory.mem.limit.bytes'; +``` + +While cgroups V2 adoption is fairly low, particularly among LTS-style distros and software, this is going to become a problem, and we first need an ECS standard for what these fields should be called. + +Aside from existing code, this will help with users running under a "hybrid" cgroups system, as the metricbeat cgroups code must decide if a process is under the control of cgroups V1 or V2, which means different processes on a system can report different cgroups code. A common set of fields will allow important metrics to be comparable across the system, even if that system is using two different versions of cgroups. +The same can be said for heterogeneous clusters running different versions of cgroups across multiple machines. + +## Source data + +The source data for much of cgroups comes from the cgroupfs file system on the host machine: + +``` +ls /sys/fs/cgroup/user.slice/user-1000.slice/session-23.scope +cgroup.controllers cgroup.freeze cgroup.max.descendants cgroup.stat cgroup.threads cpu.pressure io.pressure memory.events memory.high memory.max memory.numa_stat memory.pressure memory.swap.current memory.swap.high pids.current pids.max +cgroup.events cgroup.max.depth cgroup.procs cgroup.subtree_control cgroup.type cpu.stat memory.current memory.events.local memory.low memory.min memory.oom.group memory.stat memory.swap.events memory.swap.max pids.events +``` + +The structure and content of these directories varies between V1 and V2. + + + + + +## Scope of impact + + + +## Concerns + +The underlying code that reports these metrics currently exists, this RFC represents only the final step in the process to standardize metrics between two different reporting versions. Perhaps the largest potential concern here is that the problems addressed by this RFC are technically temporary. In theory, cgroups V1 will eventually get deprecated, rendering the need for a common set of fields moot. In addition, cgroups is entirely transparent +to the overwhelming majority of users, who will never interact with cgroups directly. + + + + + +## People + +The following are the people that consulted on the contents of this RFC. + +* @fearful-symmetry | author +* @jsoriano | sponsor + + + + + +## References + + + +### RFC Pull Requests + + + +* Stage 0: https://github.com/elastic/ecs/pull/1610 +* Stage 1: https://github.com/elastic/ecs/pull/1626 + * Stage 1 date correction: https://github.com/elastic/ecs/pull/1650 + diff --git a/rfcs/text/0028/cgroups.yml b/rfcs/text/0028/cgroups.yml new file mode 100644 index 0000000000..a5ac3eb343 --- /dev/null +++ b/rfcs/text/0028/cgroups.yml @@ -0,0 +1,77 @@ +--- +- name: cgroup + title: Common cgroup metrics + group: 2 + short: fields common to cgroups V1 and V2 + description: > + Fields related to cgroup metrics. Due to controller changes betweeen V1 and V2, many same or similar metrics + will often appear under different controller names. These fields are common to both cgroup versions. + type: group + fields: + - name: version + level: extended + type: long + description: > + The cgroups version linked to the metrics + - name: cpu + level: extended + type: group + short: CPU Metrics + description: > + Metrics related to CPU controllers. + fields: + - name: periods + level: extended + type: long + example: 454839343 + description: > + Number of period intervals that have elapsed. + - name: throttled + level: extended + type: group + description: > + Metrics for the time a resource has been throttled. + fields: + - name: us + level: extended + type: long + example: 15000 + description: > + Microseconds of CPU throttled time. + - name: usage + level: extended + type: scaled_float + scaling_factor: 1000 + description: > + CPU usage, normalized by the CPU count. + - name: memory + level: extended + type: group + description: > + Metrics related to memory controllers. + fields: + - name: usage + level: extended + type: long + example: 25600 + description: > + Memory usage in bytes + - name: limit + type: long + example: 256 + description: > + Memory limit within the cgroup. + - name: swap + level: extended + type: group + description: > + Cgroup swap statistics. + fields: + - name: usage + level: extended + type: long + example: 5600 + description: > + The amount of cgroup memory in swap. + + diff --git a/schemas/agent.yml b/schemas/agent.yml index a7758e90ce..8421a66eed 100644 --- a/schemas/agent.yml +++ b/schemas/agent.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: agent title: Agent diff --git a/schemas/as.yml b/schemas/as.yml index 3d1f7db031..a2c36b22ef 100644 --- a/schemas/as.yml +++ b/schemas/as.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: as title: Autonomous System diff --git a/schemas/base.yml b/schemas/base.yml index 30e345d068..34d0b60ec6 100644 --- a/schemas/base.yml +++ b/schemas/base.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: base root: true diff --git a/schemas/client.yml b/schemas/client.yml index 3011c97b46..539a7a2e18 100644 --- a/schemas/client.yml +++ b/schemas/client.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: client title: Client diff --git a/schemas/cloud.yml b/schemas/cloud.yml index ef0651ba63..0e97490b66 100644 --- a/schemas/cloud.yml +++ b/schemas/cloud.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: cloud title: Cloud @@ -12,6 +28,28 @@ runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on. + + The cloud fields may be self-nested under cloud.origin.* and cloud.target.* + to describe origin or target service's cloud information in the context of + incoming or outgoing requests, respectively. However, the fieldsets + cloud.origin.* and cloud.target.* must not be confused with the root cloud + fieldset that is used to describe the cloud context of the actual service + under observation. The fieldset cloud.origin.* may only be used in the + context of incoming requests or events to provide the originating service's + cloud information. The fieldset cloud.target.* may only be used in the + context of outgoing requests or events to describe the target service's + cloud information. + reusable: + top_level: true + expected: + - at: cloud + as: origin + beta: Reusing the `cloud` fields in this location is currently considered beta. + short_override: Provides the cloud information of the origin entity in case of an incoming request or event. + - at: cloud + as: target + beta: Reusing the `cloud` fields in this location is currently considered beta. + short_override: Provides the cloud information of the target entity in case of an outgoing request or event. type: group fields: - name: provider diff --git a/schemas/code_signature.yml b/schemas/code_signature.yml index 2804029a7f..056262370f 100644 --- a/schemas/code_signature.yml +++ b/schemas/code_signature.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: code_signature title: Code Signature diff --git a/schemas/container.yml b/schemas/container.yml index 04cc138572..047c10fa8a 100644 --- a/schemas/container.yml +++ b/schemas/container.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: container title: Container diff --git a/schemas/data_stream.yml b/schemas/data_stream.yml index a169b0a61c..9dcf02a92c 100644 --- a/schemas/data_stream.yml +++ b/schemas/data_stream.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: data_stream title: Data Stream diff --git a/schemas/destination.yml b/schemas/destination.yml index c600377b92..6b084e6bab 100644 --- a/schemas/destination.yml +++ b/schemas/destination.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: destination title: Destination diff --git a/schemas/dll.yml b/schemas/dll.yml index f4bf90a2c5..7e832dec11 100644 --- a/schemas/dll.yml +++ b/schemas/dll.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: dll title: DLL diff --git a/schemas/dns.yml b/schemas/dns.yml index afe11a190a..29826f72cd 100644 --- a/schemas/dns.yml +++ b/schemas/dns.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: dns title: DNS diff --git a/schemas/ecs.yml b/schemas/ecs.yml index de7bb39fe4..497e9d1b5d 100644 --- a/schemas/ecs.yml +++ b/schemas/ecs.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: ecs title: ECS diff --git a/schemas/elf.yml b/schemas/elf.yml index 525b155414..ca174f3a63 100644 --- a/schemas/elf.yml +++ b/schemas/elf.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: elf title: ELF Header diff --git a/schemas/error.yml b/schemas/error.yml index 7181c70799..765803440c 100644 --- a/schemas/error.yml +++ b/schemas/error.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: error title: Error diff --git a/schemas/event.yml b/schemas/event.yml index ed7ec19a3a..57299a2fa7 100644 --- a/schemas/event.yml +++ b/schemas/event.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: event title: Event @@ -64,7 +80,7 @@ `event.kind:alert` is often populated for events coming from firewalls, intrusion detection systems, endpoint detection and response systems, and so on. - + This value is not used by Elastic solutions for alert documents that are created by rules executing within the Kibana alerting framework. - name: enrichment diff --git a/schemas/faas.yml b/schemas/faas.yml new file mode 100644 index 0000000000..3d0a6d54cd --- /dev/null +++ b/schemas/faas.yml @@ -0,0 +1,65 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +- name: faas + group: 2 + title: FaaS + short: Fields describing functions as a service. + description: > + The user fields describe information about the function + as a service that is relevant to the event. + beta: > + These fields are in beta and are subject to change. + type: group + fields: + - name: coldstart + description: > + Boolean value indicating a cold start of a function. + type: boolean + level: extended + - name: execution + description: > + The execution ID of the current function execution. + type: keyword + level: extended + example: "af9d5aa4-a685-4c5f-a22b-444f80b3cc28" + - name: trigger + level: extended + type: nested + description: > + Details about the function trigger. + - name: trigger.type + level: extended + type: keyword + short: The trigger for the function execution. + description: > + The trigger for the function execution. + + Expected values are: + * http + * pubsub + * datasource + * timer + * other + example: http + - name: trigger.request_id + level: extended + type: keyword + description: > + The ID of the trigger request , message, event, etc. + example: 123456789 + diff --git a/schemas/file.yml b/schemas/file.yml index fa50793ce7..39de2ff68e 100644 --- a/schemas/file.yml +++ b/schemas/file.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: file group: 2 diff --git a/schemas/geo.yml b/schemas/geo.yml index fe1e80a3e6..874e43fa1f 100644 --- a/schemas/geo.yml +++ b/schemas/geo.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: geo title: Geo diff --git a/schemas/group.yml b/schemas/group.yml index 471e1f9a8b..47146583f4 100644 --- a/schemas/group.yml +++ b/schemas/group.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: group title: Group diff --git a/schemas/hash.yml b/schemas/hash.yml index 5305d0f3a1..4d60b52edb 100644 --- a/schemas/hash.yml +++ b/schemas/hash.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: hash title: Hash @@ -17,15 +33,11 @@ reusable: top_level: false + order: 1 expected: - file - process - dll - - at: threat.indicator - as: hash - - at: threat.enrichments.indicator - as: hash - beta: Reusing the `hash` fields in this location is currently considered beta. fields: diff --git a/schemas/host.yml b/schemas/host.yml index d46bbe6a64..8b964f2dc2 100644 --- a/schemas/host.yml +++ b/schemas/host.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: host title: Host diff --git a/schemas/http.yml b/schemas/http.yml index f49055e52a..8b86943a2c 100644 --- a/schemas/http.yml +++ b/schemas/http.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: http title: HTTP diff --git a/schemas/interface.yml b/schemas/interface.yml index ed62297394..2617f109b0 100644 --- a/schemas/interface.yml +++ b/schemas/interface.yml @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- - name: interface title: Interface group: 2 diff --git a/schemas/log.yml b/schemas/log.yml index d08b2b818e..66743520fc 100644 --- a/schemas/log.yml +++ b/schemas/log.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: log title: Log diff --git a/schemas/network.yml b/schemas/network.yml index c6fed904b7..cca7f4b563 100644 --- a/schemas/network.yml +++ b/schemas/network.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: network title: Network @@ -24,8 +40,7 @@ description: > In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc - The field value must be normalized to lowercase for querying. See - the documentation section "Implementing ECS". + The field value must be normalized to lowercase for querying. example: ipv4 - name: iana_number @@ -46,8 +61,7 @@ Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) - The field value must be normalized to lowercase for querying. See - the documentation section "Implementing ECS". + The field value must be normalized to lowercase for querying. example: tcp - name: application @@ -56,25 +70,26 @@ short: > Application level protocol name. description: > - A name given to an application level protocol. This can be arbitrarily assigned for - things like microservices, but also apply to things like skype, icq, - facebook, twitter. This would be used in situations where the vendor - or service can be decoded such as from the source/dest IP owners, - ports, or wire format. - - The field value must be normalized to lowercase for querying. See - the documentation section "Implementing ECS". + When a specific application or service is identified from network + connection details (source/dest IPs, ports, certificates, or + wire format), this field captures the application's or service's name. + + For example, the original event identifies the network connection being + from a specific web service in a `https` network connection, like + `facebook` or `twitter`. + + The field value must be normalized to lowercase for querying. example: aim - name: protocol level: core type: keyword - short: L7 Network protocol name. + short: Application protocol name. description: > - L7 Network protocol name. ex. http, lumberjack, transport protocol. + In the OSI Model this would be the Application Layer protocol. For + example, `http`, `dns`, or `ssh`. - The field value must be normalized to lowercase for querying. See - the documentation section "Implementing ECS". + The field value must be normalized to lowercase for querying. example: http - name: direction diff --git a/schemas/observer.yml b/schemas/observer.yml index 9cee51c1c6..e95cf7247d 100644 --- a/schemas/observer.yml +++ b/schemas/observer.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: observer title: Observer diff --git a/schemas/orchestrator.yml b/schemas/orchestrator.yml index e90577ca70..ddfd9c19bf 100644 --- a/schemas/orchestrator.yml +++ b/schemas/orchestrator.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: orchestrator title: Orchestrator diff --git a/schemas/organization.yml b/schemas/organization.yml index b468f2eb91..237ece1507 100644 --- a/schemas/organization.yml +++ b/schemas/organization.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: organization title: Organization diff --git a/schemas/os.yml b/schemas/os.yml index a65e2d11cd..f6ae18b0c5 100644 --- a/schemas/os.yml +++ b/schemas/os.yml @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- - name: os title: Operating System group: 2 diff --git a/schemas/package.yml b/schemas/package.yml index a1d5cbb00f..957e7d5ff8 100644 --- a/schemas/package.yml +++ b/schemas/package.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: package title: Package diff --git a/schemas/pe.yml b/schemas/pe.yml index c8601b1c9a..ef0d087161 100644 --- a/schemas/pe.yml +++ b/schemas/pe.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: pe title: PE Header @@ -6,15 +22,12 @@ type: group reusable: top_level: false + order: 1 expected: - file - dll - process - - at: threat.indicator - as: pe - - at: threat.enrichments.indicator - as: pe - beta: Reusing the `pe` fields in this location is currently considered beta. + fields: - name: original_file_name level: extended diff --git a/schemas/process.yml b/schemas/process.yml index eeced441c0..7a388aea3d 100644 --- a/schemas/process.yml +++ b/schemas/process.yml @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- - name: process title: Process group: 2 @@ -56,14 +73,6 @@ - type: match_only_text name: text - - name: ppid - format: string - level: extended - type: long - description: > - Parent process' pid. - example: 4241 - - name: pgid format: string level: extended diff --git a/schemas/registry.yml b/schemas/registry.yml index 848f848c08..0ceeabd431 100644 --- a/schemas/registry.yml +++ b/schemas/registry.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: registry title: Registry diff --git a/schemas/related.yml b/schemas/related.yml index c40e339e7e..b052fa3c00 100644 --- a/schemas/related.yml +++ b/schemas/related.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: related title: Related diff --git a/schemas/rule.yml b/schemas/rule.yml index c0daf79892..54f3d601f5 100644 --- a/schemas/rule.yml +++ b/schemas/rule.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: rule title: Rule diff --git a/schemas/server.yml b/schemas/server.yml index b5d66c63e7..1552a42964 100644 --- a/schemas/server.yml +++ b/schemas/server.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: server title: Server diff --git a/schemas/service.yml b/schemas/service.yml index 83d558c342..5eb6ea469a 100644 --- a/schemas/service.yml +++ b/schemas/service.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: service title: Service @@ -8,6 +24,27 @@ These fields help you find and correlate logs for a specific service and version. + footnote: > + The service fields may be self-nested under service.origin.* and service.target.* + to describe origin or target services in the context of incoming or outgoing requests, + respectively. + However, the fieldsets service.origin.* and service.target.* must not be confused with + the root service fieldset that is used to describe the actual service under observation. + The fieldset service.origin.* may only be used in the context of incoming requests or + events to describe the originating service of the request. The fieldset service.target.* + may only be used in the context of outgoing requests or events to describe the target + service of the request. + reusable: + top_level: true + expected: + - at: service + as: origin + beta: Reusing the `service` fields in this location is currently considered beta. + short_override: Describes the origin service in case of an incoming request or event. + - at: service + as: target + beta: Reusing the `service` fields in this location is currently considered beta. + short_override: Describes the target service in case of an outgoing request or event. type: group fields: diff --git a/schemas/source.yml b/schemas/source.yml index 8f3e5f1350..644fddcb82 100644 --- a/schemas/source.yml +++ b/schemas/source.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: source title: Source diff --git a/schemas/threat.yml b/schemas/threat.yml index 0410edfffd..e78e3a2b3b 100644 --- a/schemas/threat.yml +++ b/schemas/threat.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: threat title: Threat @@ -22,6 +38,8 @@ description: > A list of associated indicators objects enriching the event, and the context of that association/enrichment. + normalize: + - array - name: enrichments.indicator level: extended @@ -224,6 +242,14 @@ Identifies the _index of the indicator document enriching the event. example: filebeat-8.0.0-2021.05.23-000011 + - name: enrichments.matched.occurred + level: extended + type: date + short: Date of match + description: > + Indicates when the indicator match was generated + example: 2021-10-05T17:00:58.326Z + - name: enrichments.matched.type level: extended type: keyword diff --git a/schemas/tls.yml b/schemas/tls.yml index 3ecacb041a..8e7502eea3 100644 --- a/schemas/tls.yml +++ b/schemas/tls.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: tls title: TLS diff --git a/schemas/tracing.yml b/schemas/tracing.yml index 8e23514e3d..c6d6a63348 100644 --- a/schemas/tracing.yml +++ b/schemas/tracing.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: tracing title: Tracing diff --git a/schemas/url.yml b/schemas/url.yml index 1a69dec8d3..8c8209b7a4 100644 --- a/schemas/url.yml +++ b/schemas/url.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: url title: URL diff --git a/schemas/user.yml b/schemas/user.yml index ec15bc5f54..db0f994787 100644 --- a/schemas/user.yml +++ b/schemas/user.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: user title: User diff --git a/schemas/user_agent.yml b/schemas/user_agent.yml index 2facf6eb0f..d8d17d3645 100644 --- a/schemas/user_agent.yml +++ b/schemas/user_agent.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: user_agent title: User agent diff --git a/schemas/vlan.yml b/schemas/vlan.yml index 84c0457355..59536d0b1b 100644 --- a/schemas/vlan.yml +++ b/schemas/vlan.yml @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- - name: vlan title: VLAN group: 2 diff --git a/schemas/vulnerability.yml b/schemas/vulnerability.yml index c2af5f14f1..cb471086fd 100644 --- a/schemas/vulnerability.yml +++ b/schemas/vulnerability.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: vulnerability title: Vulnerability diff --git a/schemas/x509.yml b/schemas/x509.yml index 0238d4576c..34cc1a4e60 100644 --- a/schemas/x509.yml +++ b/schemas/x509.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: x509 title: x509 Certificate @@ -15,6 +31,7 @@ type: group reusable: top_level: false + order: 1 expected: - file - at: threat.indicator diff --git a/scripts/__init__.py b/scripts/__init__.py index e69de29bb2..fae77611ff 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -0,0 +1,16 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/scripts/generator.py b/scripts/generator.py index 31772f3c79..9c7fc56f15 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import glob import os diff --git a/scripts/generators/__init__.py b/scripts/generators/__init__.py index e69de29bb2..fae77611ff 100644 --- a/scripts/generators/__init__.py +++ b/scripts/generators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 840a0ad262..15ac9cc980 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from functools import wraps import os.path as path @@ -122,7 +139,7 @@ def save_asciidoc(f, text): TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates') template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR) -template_env = jinja2.Environment(loader=template_loader) +template_env = jinja2.Environment(loader=template_loader, keep_trailing_newline=True) # Rendering schemas diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index f36eebad55..cab1e50d74 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from os.path import join from collections import OrderedDict from generators import ecs_helpers @@ -23,6 +40,8 @@ def generate(ecs_nested, ecs_version, out_dir): continue beats_field = ecs_helpers.dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) + if 'default_field' not in beats_field: + beats_field['default_field'] = True beats_field['fields'] = fieldset_field_array(fieldset['fields'], df_allowlist, fieldset['prefix']) beats_fields.append(beats_field) diff --git a/scripts/generators/beats_default_fields_allowlist.yml b/scripts/generators/beats_default_fields_allowlist.yml index 6629ce6ca5..7eec6130a1 100644 --- a/scripts/generators/beats_default_fields_allowlist.yml +++ b/scripts/generators/beats_default_fields_allowlist.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- !!set # Note: other fields can be inserted as needed diff --git a/scripts/generators/csv_generator.py b/scripts/generators/csv_generator.py index e1f6bad89e..39e3ecae54 100644 --- a/scripts/generators/csv_generator.py +++ b/scripts/generators/csv_generator.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import csv import sys diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index fbf7f4a2a1..19197e56cb 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import glob import os import yaml diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index ceb631cb65..19cbf57bb4 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import copy import json import sys @@ -12,7 +29,8 @@ 'constant_keyword': 'keyword', 'wildcard': 'keyword', 'version': 'keyword', - 'match_only_text': 'text' + 'match_only_text': 'text', + 'flattened': 'object' } # Composable Template @@ -231,7 +249,8 @@ def save_json(file, data): if sys.version_info >= (3, 0): open_mode = "w" with open(file, open_mode) as jsonfile: - jsonfile.write(json.dumps(data, indent=2, sort_keys=True)) + json.dump(data, jsonfile, indent=2, sort_keys=True) + jsonfile.write('\n') def default_template_settings(ecs_version): diff --git a/scripts/generators/intermediate_files.py b/scripts/generators/intermediate_files.py index c085039b62..10c9a2466d 100644 --- a/scripts/generators/intermediate_files.py +++ b/scripts/generators/intermediate_files.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import copy from schema import visitor diff --git a/scripts/schema/__init__.py b/scripts/schema/__init__.py index e69de29bb2..fae77611ff 100644 --- a/scripts/schema/__init__.py +++ b/scripts/schema/__init__.py @@ -0,0 +1,16 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/scripts/schema/cleaner.py b/scripts/schema/cleaner.py index ca892b9f20..cc85ad5b4e 100644 --- a/scripts/schema/cleaner.py +++ b/scripts/schema/cleaner.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import copy from generators import ecs_helpers diff --git a/scripts/schema/exclude_filter.py b/scripts/schema/exclude_filter.py index 5717ecfb6f..9ccbf25351 100644 --- a/scripts/schema/exclude_filter.py +++ b/scripts/schema/exclude_filter.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from schema import loader # This script should be run downstream of the subset filters - it takes diff --git a/scripts/schema/finalizer.py b/scripts/schema/finalizer.py index 86fe5b760f..298b0d7dfa 100644 --- a/scripts/schema/finalizer.py +++ b/scripts/schema/finalizer.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import copy import re diff --git a/scripts/schema/loader.py b/scripts/schema/loader.py index a662622274..00fbf721e5 100644 --- a/scripts/schema/loader.py +++ b/scripts/schema/loader.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import copy import glob import yaml diff --git a/scripts/schema/subset_filter.py b/scripts/schema/subset_filter.py index 8c91929f0d..f00dc11e09 100644 --- a/scripts/schema/subset_filter.py +++ b/scripts/schema/subset_filter.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os from generators import intermediate_files from schema import cleaner, loader diff --git a/scripts/schema/visitor.py b/scripts/schema/visitor.py index 3c3d762bad..baf22f0ee5 100644 --- a/scripts/schema/visitor.py +++ b/scripts/schema/visitor.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + def visit_fields(fields, fieldset_func=None, field_func=None): """ This function navigates the deeply nested tree structure and runs provided diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 index 6a099f0a46..41d9a9cd11 100644 --- a/scripts/templates/field_details.j2 +++ b/scripts/templates/field_details.j2 @@ -1,4 +1,5 @@ -{# Title & Description -#} +{# Title & Description +-#} [[ecs-{{ fieldset['name'] }}]] === {{ fieldset['title'] }} Fields @@ -104,11 +105,11 @@ Note also that the `{{ fieldset['name'] }}` fields may be used directly at the r Note also that the `{{ fieldset['name'] }}` fields are not expected to be used directly at the root of the events. -{% endif %}{# if 'top_level' -#} +{%- endif %}{# if 'top_level' -#} {% endif %}{# if 'reusable' #} -{% if 'nestings' in fieldset -%} +{%- if 'nestings' in fieldset -%} [[ecs-{{ fieldset['name'] }}-nestings]] [discrete] @@ -143,7 +144,7 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d |===== {% endif %}{# if 'nestings' #} -{%- endif -%}{# if 'nestings' or 'reusable' in fieldset #} +{%- endif %}{# if 'nestings' or 'reusable' in fieldset #} {%- if usage_doc %} {# Field Usage Table Header -#} diff --git a/scripts/templates/field_values.j2 b/scripts/templates/field_values.j2 index 86576c6930..89d36782f2 100644 --- a/scripts/templates/field_values.j2 +++ b/scripts/templates/field_values.j2 @@ -1,4 +1,3 @@ - [[ecs-category-field-values-reference]] == {ecs} Categorization Fields diff --git a/scripts/templates/fields.j2 b/scripts/templates/fields.j2 index 1f5ab65472..9c9154cafd 100644 --- a/scripts/templates/fields.j2 +++ b/scripts/templates/fields.j2 @@ -1,4 +1,3 @@ - [[ecs-field-reference]] == {ecs} Field Reference @@ -11,7 +10,7 @@ at the root of the event. All other field sets are defined as objects in {es}, under which all fields are defined. -For a single page representation of all fields, please see the +For a single page representation of all fields, please see the {ecs_github_repo_link}/generated/csv/fields.csv[generated CSV of fields]. [float] @@ -26,4 +25,3 @@ For a single page representation of all fields, please see the |===== include::field-details.asciidoc[] - diff --git a/scripts/tests/__init__.py b/scripts/tests/__init__.py index e69de29bb2..fae77611ff 100644 --- a/scripts/tests/__init__.py +++ b/scripts/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py index 1851fc95fe..cdd7b93c29 100644 --- a/scripts/tests/test_asciidoc_fields.py +++ b/scripts/tests/test_asciidoc_fields.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import sys import unittest diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py index 79b554ad95..9d14677eed 100644 --- a/scripts/tests/test_ecs_helpers.py +++ b/scripts/tests/test_ecs_helpers.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import sys import unittest diff --git a/scripts/tests/test_ecs_spec.py b/scripts/tests/test_ecs_spec.py index 6ff6372579..a7a3d591c7 100644 --- a/scripts/tests/test_ecs_spec.py +++ b/scripts/tests/test_ecs_spec.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import sys import unittest diff --git a/scripts/tests/test_es_template.py b/scripts/tests/test_es_template.py index cf9ba7da3e..127a0cc170 100644 --- a/scripts/tests/test_es_template.py +++ b/scripts/tests/test_es_template.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import sys import unittest diff --git a/scripts/tests/unit/__init__.py b/scripts/tests/unit/__init__.py index e69de29bb2..fae77611ff 100644 --- a/scripts/tests/unit/__init__.py +++ b/scripts/tests/unit/__init__.py @@ -0,0 +1,16 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/scripts/tests/unit/test_beats_generator.py b/scripts/tests/unit/test_beats_generator.py index a433442557..bb421b2ac7 100644 --- a/scripts/tests/unit/test_beats_generator.py +++ b/scripts/tests/unit/test_beats_generator.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import sys from typing import OrderedDict diff --git a/scripts/tests/unit/test_schema_cleaner.py b/scripts/tests/unit/test_schema_cleaner.py index f0384d4f53..ca47f9874b 100644 --- a/scripts/tests/unit/test_schema_cleaner.py +++ b/scripts/tests/unit/test_schema_cleaner.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import pprint import sys diff --git a/scripts/tests/unit/test_schema_exclude_filter.py b/scripts/tests/unit/test_schema_exclude_filter.py index 5b6cb5d6ad..9e714a197b 100644 --- a/scripts/tests/unit/test_schema_exclude_filter.py +++ b/scripts/tests/unit/test_schema_exclude_filter.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from schema import exclude_filter import mock import os diff --git a/scripts/tests/unit/test_schema_finalizer.py b/scripts/tests/unit/test_schema_finalizer.py index fd0c8d6118..707c7a22f6 100644 --- a/scripts/tests/unit/test_schema_finalizer.py +++ b/scripts/tests/unit/test_schema_finalizer.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import pprint import sys diff --git a/scripts/tests/unit/test_schema_loader.py b/scripts/tests/unit/test_schema_loader.py index b9b263f5df..f74e50eb4c 100644 --- a/scripts/tests/unit/test_schema_loader.py +++ b/scripts/tests/unit/test_schema_loader.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mock import os import pprint diff --git a/scripts/tests/unit/test_schema_subset_filter.py b/scripts/tests/unit/test_schema_subset_filter.py index e7ae5fd211..0f1f7975bc 100644 --- a/scripts/tests/unit/test_schema_subset_filter.py +++ b/scripts/tests/unit/test_schema_subset_filter.py @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mock import os import pprint diff --git a/stages.html b/stages.html index 2269bcd9dc..03d3844e2d 100644 --- a/stages.html +++ b/stages.html @@ -26,7 +26,7 @@

ECS Proposal Stages

-

These are the stages that an individual RFC advances through before being released for general availability in the Elastic Common Schema (ECS). See the Contributing Guide for broader details about contributing changes to ECS through the RFC process. +

These are the stages that an individual RFC advances through before being released for general availability in the Elastic Common Schema (ECS). See the Contributing Guide for broader details about contributing changes to ECS through the RFC process. diff --git a/usage-example/generated/elasticsearch/6/template.json b/usage-example/generated/elasticsearch/6/template.json index 2f598e8f9b..7501b7f94e 100644 --- a/usage-example/generated/elasticsearch/6/template.json +++ b/usage-example/generated/elasticsearch/6/template.json @@ -1155,4 +1155,4 @@ "refresh_interval": "2s" } } -} \ No newline at end of file +} diff --git a/usage-example/generated/elasticsearch/7/template.json b/usage-example/generated/elasticsearch/7/template.json index c632cfda07..ee4a84b7f6 100644 --- a/usage-example/generated/elasticsearch/7/template.json +++ b/usage-example/generated/elasticsearch/7/template.json @@ -1153,4 +1153,4 @@ "refresh_interval": "2s" } } -} \ No newline at end of file +} diff --git a/version b/version index e4be07b065..761c136f26 100644 --- a/version +++ b/version @@ -1 +1 @@ -8.0.0-dev +8.1.0-dev