diff --git a/.backportrc.json b/.backportrc.json index 8ad8491534906..ef4aa68e9c441 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,9 +1,9 @@ { "upstream" : "elastic/elasticsearch", - "targetBranchChoices" : [ "main", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], + "targetBranchChoices" : [ "main", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], "targetPRLabels" : [ "backport" ], "branchLabelMapping" : { - "^v8.10.0$" : "main", + "^v8.11.0$" : "main", "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2" } } \ No newline at end of file diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 0000000000000..3a8f3e09e847e --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,60 @@ +#!/bin/bash + +# On some distros, this directory ends up not readable by the `elasticsearch` user that gets created during tests +# This fixes that +chmod 755 ~ + +WORKSPACE="$(pwd)" +export WORKSPACE + +BUILD_NUMBER="$BUILDKITE_BUILD_NUMBER" +export BUILD_NUMBER + +COMPOSE_HTTP_TIMEOUT="120" +export COMPOSE_HTTP_TIMEOUT + +JOB_BRANCH="$BUILDKITE_BRANCH" +export JOB_BRANCH + +GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/" +export GRADLEW + +GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/" +export GRADLEW_BAT + +export $(cat .ci/java-versions.properties | grep '=' | xargs) + +JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA" +export JAVA_HOME + +JAVA11_HOME="$HOME/.java/java11" +export JAVA11_HOME + +JAVA16_HOME="$HOME/.java/openjdk16" +export JAVA16_HOME + +if [[ "${ES_RUNTIME_JAVA:-}" ]]; then + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + export RUNTIME_JAVA_HOME +fi + +GRADLE_BUILD_CACHE_USERNAME=$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache) +export GRADLE_BUILD_CACHE_USERNAME + +GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache) +export GRADLE_BUILD_CACHE_PASSWORD + +BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token) +export BUILDKITE_API_TOKEN + +if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then + data=$(.buildkite/scripts/lucene-snapshot/get-credentials.sh) + + AWS_ACCESS_KEY_ID=$(echo "$data" | jq -r .data.access_key) + export AWS_ACCESS_KEY_ID + + AWS_SECRET_ACCESS_KEY=$(echo "$data" | jq -r .data.secret_key) + export AWS_SECRET_ACCESS_KEY + + unset data +fi diff --git a/.buildkite/hooks/pre-command.bat b/.buildkite/hooks/pre-command.bat new file mode 100644 index 0000000000000..ddf938a0dd0b9 --- /dev/null +++ b/.buildkite/hooks/pre-command.bat @@ -0,0 +1,21 @@ +@ECHO OFF + +FOR /F "tokens=* eol=#" %%i in ('type .ci\java-versions.properties') do set %%i + +SET JAVA_HOME=%USERPROFILE%\.java\%ES_BUILD_JAVA% +SET JAVA16_HOME=%USERPROFILE%\.java\openjdk16 + +SET GRADLEW=./gradlew --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ +SET GRADLEW_BAT=./gradlew.bat --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ + +(if not exist "%USERPROFILE%/.gradle" mkdir "%USERPROFILE%/.gradle") && (echo. >> "%USERPROFILE%/.gradle/gradle.properties" && echo org.gradle.daemon=false >> "%USERPROFILE%/.gradle/gradle.properties") + +set WORKSPACE=%cd% +set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER% +set COMPOSE_HTTP_TIMEOUT=120 +set JOB_BRANCH=%BUILDKITE_BRANCH% + +set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache +set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache + +exit /b 0 diff --git a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml new file mode 100644 index 0000000000000..8cf2a8aacbece --- /dev/null +++ b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml @@ -0,0 +1,22 @@ +steps: + - trigger: apache-lucene-build-snapshot + label: Trigger pipeline to build lucene snapshot + key: lucene-build + if: build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == "" + - wait + - label: Upload and update lucene snapshot + command: .buildkite/scripts/lucene-snapshot/upload-snapshot.sh + timeout_in_minutes: 15 + env: + USE_LUCENE_SNAPSHOT_CREDS: "true" + UPDATE_ES_LUCENE_SNAPSHOT: "true" + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - wait + - trigger: "elasticsearch-lucene-snapshot-tests" + build: + branch: "${BUILDKITE_BRANCH}" + async: true diff --git a/.buildkite/pipelines/lucene-snapshot/run-tests.yml b/.buildkite/pipelines/lucene-snapshot/run-tests.yml new file mode 100644 index 0000000000000..15d78f8495ca8 --- /dev/null +++ b/.buildkite/pipelines/lucene-snapshot/run-tests.yml @@ -0,0 +1,60 @@ +steps: + - label: sanity-check + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - wait: null + - label: part1 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - label: part2 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - label: part3 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - group: bwc-snapshots + steps: + - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest + timeout_in_minutes: 300 + matrix: + setup: + BWC_VERSION: + - 7.17.13 + - 8.9.1 + - 8.10.0 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: "{{matrix.BWC_VERSION}}" + - label: rest-compat + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/lucene-snapshot/update-branch.yml b/.buildkite/pipelines/lucene-snapshot/update-branch.yml new file mode 100644 index 0000000000000..112bfaef5570b --- /dev/null +++ b/.buildkite/pipelines/lucene-snapshot/update-branch.yml @@ -0,0 +1,9 @@ +steps: + - label: Update lucene-snapshot branch + command: .buildkite/scripts/lucene-snapshot/update-branch.sh + timeout_in_minutes: 15 + - wait + - trigger: "elasticsearch-lucene-snapshot-tests" + build: + branch: "${BUILDKITE_BRANCH}" + async: true diff --git a/.buildkite/pipelines/periodic.trigger.yml b/.buildkite/pipelines/periodic.trigger.yml new file mode 100644 index 0000000000000..5d5d592448b5d --- /dev/null +++ b/.buildkite/pipelines/periodic.trigger.yml @@ -0,0 +1,16 @@ +steps: + - trigger: elasticsearch-periodic + label: Trigger periodic pipeline for main + async: true + build: + branch: main + - trigger: elasticsearch-periodic + label: Trigger periodic pipeline for 8.9 + async: true + build: + branch: "8.9" + - trigger: elasticsearch-periodic + label: Trigger periodic pipeline for 7.17 + async: true + build: + branch: "7.17" diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml new file mode 100644 index 0000000000000..95dba6e1d44f3 --- /dev/null +++ b/.buildkite/pipelines/periodic.yml @@ -0,0 +1,99 @@ +steps: + - group: java-fips-matrix + steps: + - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.fips.enabled=true $$GRADLE_TASK + timeout_in_minutes: 180 + matrix: + setup: + ES_RUNTIME_JAVA: + - openjdk17 + GRADLE_TASK: + - checkPart1 + - checkPart2 + - checkPart3 + - bwcTestSnapshots + - checkRestCompat + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" + GRADLE_TASK: "{{matrix.GRADLE_TASK}}" + - group: java-matrix + steps: + - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $$GRADLE_TASK + timeout_in_minutes: 180 + matrix: + setup: + ES_RUNTIME_JAVA: + - graalvm-ce17 + - openjdk17 + - openjdk18 + - openjdk19 + - openjdk20 + - openjdk21 + GRADLE_TASK: + - checkPart1 + - checkPart2 + - checkPart3 + - bwcTestSnapshots + - checkRestCompat + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" + GRADLE_TASK: "{{matrix.GRADLE_TASK}}" + - group: packaging-tests-windows + steps: + - label: "{{matrix.image}} / packaging-tests-windows" + command: | + .\.buildkite\scripts\run-script.ps1 .\.ci\scripts\packaging-test.ps1 + timeout_in_minutes: 180 + matrix: + setup: + image: + - windows-2016 + - windows-2019 + - windows-2022 + agents: + provider: gcp + image: family/brian-elasticsearch-{{matrix.image}} + imageProject: elastic-images-qa + machineType: custom-32-98304 + diskType: pd-ssd + diskSizeGb: 350 + env: {} + - group: platform-support-windows + steps: + - label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-windows" + command: | + .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh + timeout_in_minutes: 360 + matrix: + setup: + image: + - windows-2016 + - windows-2019 + - windows-2022 + GRADLE_TASK: + - checkPart1 + - checkPart2 + - checkPart3 + - bwcTestSnapshots + - checkRestCompat + agents: + provider: gcp + image: family/brian-elasticsearch-{{matrix.image}} + imageProject: elastic-images-qa + machineType: custom-32-98304 + diskType: pd-ssd + diskSizeGb: 350 + env: + GRADLE_TASK: "{{matrix.GRADLE_TASK}}" diff --git a/.buildkite/scripts/lucene-snapshot/get-credentials.sh b/.buildkite/scripts/lucene-snapshot/get-credentials.sh new file mode 100755 index 0000000000000..042c664384a7f --- /dev/null +++ b/.buildkite/scripts/lucene-snapshot/get-credentials.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable. +# It should really only be used inside .buildkite/hooks/pre-command + +VAULT_ROLE_ID=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials) +VAULT_SECRET_ID=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials) +VAULT_ADDR=https://secrets.elastic.co:8200 + +unset VAULT_TOKEN +VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) +export VAULT_TOKEN + +vault read -format=json aws-elastic/creds/lucene-snapshots diff --git a/.buildkite/scripts/lucene-snapshot/remove-verification-metadata.py b/.buildkite/scripts/lucene-snapshot/remove-verification-metadata.py new file mode 100644 index 0000000000000..082e734b4b790 --- /dev/null +++ b/.buildkite/scripts/lucene-snapshot/remove-verification-metadata.py @@ -0,0 +1,13 @@ +import re + +regex = re.compile( + r'` string in version.properties and maintain the same indentation +sed -E "s/^(lucene *= *[^ ]* *).*\$/\1$LUCENE_SNAPSHOT_VERSION/" build-tools-internal/version.properties > new-version.properties +mv new-version.properties build-tools-internal/version.properties + +# Remove stale verification metadata, because generating them just appends new ones +python3 .buildkite/scripts/lucene-snapshot/remove-verification-metadata.py +./gradlew --write-verification-metadata sha256 + +# 9.8.0-snapshot-1f25c68 -> 9.8.0 +VERSION=$(echo "$LUCENE_SNAPSHOT_VERSION" | cut -f 1 -d '-') +VERSION_SNAKE=$(echo "$VERSION" | sed -E 's/\./_/g') + +sed -E "s/^(:lucene_version: *).*\$/\1$VERSION/" docs/Versions.asciidoc > docs/Versions.asciidoc.new +sed -E "s/^(:lucene_version_path: *).*\$/\1$VERSION_SNAKE/" docs/Versions.asciidoc.new > docs/Versions.asciidoc +rm -f docs/Versions.asciidoc.new + +if git diff-index --quiet HEAD --; then + echo 'No changes to commit.' +else + git config --global user.name elasticsearchmachine + git config --global user.email 'infra-root+elasticsearchmachine@elastic.co' + + git add build-tools-internal/version.properties + git add gradle/verification-metadata.xml + git add docs/Versions.asciidoc + + git commit -m "[Automated] Update Lucene snapshot to $LUCENE_SNAPSHOT_VERSION" + git push origin "$BUILDKITE_BRANCH" +fi diff --git a/.buildkite/scripts/lucene-snapshot/upload-snapshot.sh b/.buildkite/scripts/lucene-snapshot/upload-snapshot.sh new file mode 100755 index 0000000000000..2f276db50739c --- /dev/null +++ b/.buildkite/scripts/lucene-snapshot/upload-snapshot.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -euo pipefail + +LUCENE_BUILD_ID=${LUCENE_BUILD_ID:-} + +if [[ -z "$LUCENE_BUILD_ID" ]]; then + build_json=$(curl -sH "Authorization: Bearer $BUILDKITE_API_TOKEN" "https://api.buildkite.com/v2/organizations/elastic/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER") + LUCENE_BUILD_ID=$(jq -r '.jobs[] | select(.step_key == "lucene-build").triggered_build.id' <<< "$build_json") +fi + +export LUCENE_BUILD_ID + +LUCENE_SHA=$(buildkite-agent meta-data get --build "$LUCENE_BUILD_ID" lucene-snapshot-sha) +export LUCENE_SHA + +LUCENE_SNAPSHOT_VERSION=$(buildkite-agent meta-data get --build "$LUCENE_BUILD_ID" lucene-snapshot-version) +export LUCENE_SNAPSHOT_VERSION + +echo --- Downloading lucene snapshot + +mkdir lucene-snapshot +cd lucene-snapshot +buildkite-agent artifact download --build "$LUCENE_BUILD_ID" lucene-snapshot.tar.gz . +tar -xvf lucene-snapshot.tar.gz +cd - + +echo --- Upload lucene snapshot to S3 + +if ! which aws; then + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install + rm -rf awscliv2.zip aws +fi + +aws s3 sync lucene-snapshot/ "s3://download.elasticsearch.org/lucenesnapshots/$LUCENE_SHA/" --acl public-read + +if [[ "${UPDATE_ES_LUCENE_SNAPSHOT:-}" ]]; then + .buildkite/scripts/lucene-snapshot/update-es-snapshot.sh +fi diff --git a/.buildkite/scripts/run-script.ps1 b/.buildkite/scripts/run-script.ps1 new file mode 100644 index 0000000000000..95174817ea408 --- /dev/null +++ b/.buildkite/scripts/run-script.ps1 @@ -0,0 +1,154 @@ +# Usage: .buildkite/scripts/run-script.ps1 +# Example: .buildkite/scripts/run-script.ps1 bash .buildkite/scripts/tests.sh +# Example: .buildkite/scripts/run-script.ps1 .buildkite/scripts/other-tests.ps1 +# +# NOTE: Apparently passing arguments in powershell is a nightmare, so you shouldn't do it unless it's really simple. Just use the wrapper to call a script instead. +# See: https://stackoverflow.com/questions/6714165/powershell-stripping-double-quotes-from-command-line-arguments +# and: https://github.com/PowerShell/PowerShell/issues/3223#issuecomment-487975049 +# +# See here: https://github.com/buildkite/agent/issues/2202 +# Background processes after the buildkite step script finishes causes the job to hang. +# So, until this is fixed/changed in buildkite-agent (if ever), we can use this wrapper. + +# This wrapper: +# - Creates a Windows job object (which is like a process group) +# - Sets JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, which means that when the job object is closed, all processes in the job object are killed +# - Starts running your given script, and assigns it to the job object +# - Now, any child processes created by your script will also end up in the job object +# - Waits for your script (and only your script, not child processes) to finish +# - Closes the job object, which kills all processes in the job object (including any leftover child processes) +# - Exits with the exit status from your script + +Add-Type -TypeDefinition @' +using Microsoft.Win32.SafeHandles; +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; + +public class NativeMethods +{ + public enum JOBOBJECTINFOCLASS + { + AssociateCompletionPortInformation = 7, + BasicLimitInformation = 2, + BasicUIRestrictions = 4, + EndOfJobTimeInformation = 6, + ExtendedLimitInformation = 9, + SecurityLimitInformation = 5, + GroupInformation = 11 + } + + [StructLayout(LayoutKind.Sequential)] + struct JOBOBJECT_BASIC_LIMIT_INFORMATION + { + public Int64 PerProcessUserTimeLimit; + public Int64 PerJobUserTimeLimit; + public UInt32 LimitFlags; + public UIntPtr MinimumWorkingSetSize; + public UIntPtr MaximumWorkingSetSize; + public UInt32 ActiveProcessLimit; + public Int64 Affinity; + public UInt32 PriorityClass; + public UInt32 SchedulingClass; + } + + [StructLayout(LayoutKind.Sequential)] + struct IO_COUNTERS + { + public UInt64 ReadOperationCount; + public UInt64 WriteOperationCount; + public UInt64 OtherOperationCount; + public UInt64 ReadTransferCount; + public UInt64 WriteTransferCount; + public UInt64 OtherTransferCount; + } + + [StructLayout(LayoutKind.Sequential)] + struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION + { + public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation; + public IO_COUNTERS IoInfo; + public UIntPtr ProcessMemoryLimit; + public UIntPtr JobMemoryLimit; + public UIntPtr PeakProcessMemoryUsed; + public UIntPtr PeakJobMemoryUsed; + } + + [DllImport("Kernel32.dll", EntryPoint = "AssignProcessToJobObject", SetLastError = true)] + private static extern bool NativeAssignProcessToJobObject(SafeHandle hJob, SafeHandle hProcess); + + public static void AssignProcessToJobObject(SafeHandle job, SafeHandle process) + { + if (!NativeAssignProcessToJobObject(job, process)) + throw new Win32Exception(); + } + + [DllImport( + "Kernel32.dll", + CharSet = CharSet.Unicode, + EntryPoint = "CreateJobObjectW", + SetLastError = true + )] + private static extern SafeFileHandle NativeCreateJobObjectW( + IntPtr lpJobAttributes, + string lpName + ); + + [DllImport("Kernel32.dll", EntryPoint = "CloseHandle", SetLastError = true)] + private static extern bool NativeCloseHandle(SafeHandle hJob); + + [DllImport("kernel32.dll")] + public static extern bool SetInformationJobObject( + SafeHandle hJob, + JOBOBJECTINFOCLASS JobObjectInfoClass, + IntPtr lpJobObjectInfo, + uint cbJobObjectInfoLength + ); + + private const UInt32 JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000; + + public static SafeHandle CreateJobObjectW(string name) + { + SafeHandle job = NativeCreateJobObjectW(IntPtr.Zero, name); + JOBOBJECT_BASIC_LIMIT_INFORMATION info = new JOBOBJECT_BASIC_LIMIT_INFORMATION(); + info.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + JOBOBJECT_EXTENDED_LIMIT_INFORMATION extendedInfo = + new JOBOBJECT_EXTENDED_LIMIT_INFORMATION(); + extendedInfo.BasicLimitInformation = info; + int length = Marshal.SizeOf(typeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION)); + IntPtr extendedInfoPtr = Marshal.AllocHGlobal(length); + Marshal.StructureToPtr(extendedInfo, extendedInfoPtr, false); + SetInformationJobObject( + job, + JOBOBJECTINFOCLASS.ExtendedLimitInformation, + extendedInfoPtr, + (uint)length + ); + if (job.IsInvalid) + throw new Win32Exception(); + return job; + } + + public static void CloseHandle(SafeHandle job) + { + if (!NativeCloseHandle(job)) + throw new Win32Exception(); + } +} + +'@ + +$guid = [guid]::NewGuid().Guid +Write-Output "Creating job object with name $guid" +$job = [NativeMethods]::CreateJobObjectW($guid) +$process = Start-Process -PassThru -NoNewWindow powershell -ArgumentList "$args" +[NativeMethods]::AssignProcessToJobObject($job, $process.SafeHandle) + +try { + Write-Output "Waiting for process $($process.Id) to complete..." + $process | Wait-Process + Write-Output "Process finished with exit code $($process.ExitCode), terminating job and exiting..." +} finally { + [NativeMethods]::CloseHandle($job) + exit $process.ExitCode +} diff --git a/.buildkite/scripts/windows-run-gradle.sh b/.buildkite/scripts/windows-run-gradle.sh new file mode 100755 index 0000000000000..75999e105c873 --- /dev/null +++ b/.buildkite/scripts/windows-run-gradle.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK diff --git a/.ci/bwcVersions b/.ci/bwcVersions index c5414fa994330..65b7f54bca8db 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -61,6 +61,7 @@ BWC_VERSION: - "7.17.10" - "7.17.11" - "7.17.12" + - "7.17.13" - "8.0.0" - "8.0.1" - "8.1.0" @@ -91,6 +92,7 @@ BWC_VERSION: - "8.8.0" - "8.8.1" - "8.8.2" - - "8.8.3" - "8.9.0" + - "8.9.1" - "8.10.0" + - "8.11.0" diff --git a/.ci/init.gradle b/.ci/init.gradle index ce5ee20f8fbd8..4b2cbd1907ca0 100644 --- a/.ci/init.gradle +++ b/.ci/init.gradle @@ -13,14 +13,26 @@ initscript { boolean USE_ARTIFACTORY = false if (System.getenv('VAULT_ADDR') == null) { + // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up + if (System.getenv('CI') == null) { + return + } + throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.") } if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) { + // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up + if (System.getenv('CI') == null) { + return + } + throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " + "or the VAULT_TOKEN environment variable to use this init script.") } +final String vaultPathPrefix = System.getenv('VAULT_ADDR') ==~ /.+vault-ci.+\.dev.*/ ? "secret/ci/elastic-elasticsearch/migrated" : "secret/elasticsearch-ci" + final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault( new VaultConfig() .address(System.env.VAULT_ADDR) @@ -44,7 +56,7 @@ final Vault vault = new Vault( if (USE_ARTIFACTORY) { final Map artifactoryCredentials = vault.logical() - .read("secret/elasticsearch-ci/artifactory.elstc.co") + .read("${vaultPathPrefix}/artifactory.elstc.co") .getData() logger.info("Using elastic artifactory repos") Closure configCache = { @@ -78,10 +90,10 @@ if (USE_ARTIFACTORY) { } } -projectsLoaded { - rootProject { - project.pluginManager.withPlugin('com.gradle.build-scan') { - buildScan.server = 'https://gradle-enterprise.elastic.co' +gradle.settingsEvaluated { settings -> + settings.pluginManager.withPlugin("com.gradle.enterprise") { + settings.gradleEnterprise { + server = 'https://gradle-enterprise.elastic.co' } } } @@ -91,8 +103,8 @@ final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.u final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false')) if (buildCacheUrl) { - final Map buildCacheCredentials = vault.logical() - .read("secret/elasticsearch-ci/gradle-build-cache") + final Map buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical() + .read("${vaultPathPrefix}/gradle-build-cache") .getData() gradle.settingsEvaluated { settings -> settings.buildCache { @@ -104,8 +116,8 @@ if (buildCacheUrl) { url = buildCacheUrl push = buildCachePush credentials { - username = buildCacheCredentials.get("username") - password = buildCacheCredentials.get("password") + username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username") + password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password") } } } diff --git a/.ci/scripts/packaging-test.ps1 b/.ci/scripts/packaging-test.ps1 index 67c9008bde076..8cc59d60e0d26 100644 --- a/.ci/scripts/packaging-test.ps1 +++ b/.ci/scripts/packaging-test.ps1 @@ -12,7 +12,7 @@ $env:ES_BUILD_JAVA=$AppProps.ES_BUILD_JAVA $env:JAVA_TOOL_OPTIONS='' $ErrorActionPreference="Stop" -$gradleInit = "C:\Users\$env:username\.gradle\init.d\" +$gradleInit = "$env:USERPROFILE\.gradle\init.d\" echo "Remove $gradleInit" Remove-Item -Recurse -Force $gradleInit -ErrorAction Ignore New-Item -ItemType directory -Path $gradleInit @@ -20,13 +20,13 @@ echo "Copy .ci/init.gradle to $gradleInit" Copy-Item .ci/init.gradle -Destination $gradleInit [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine") -$env:PATH="C:\Users\jenkins\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH" +$env:PATH="$env:USERPROFILE\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH" $env:JAVA_HOME=$null -$env:SYSTEM_JAVA_HOME="C:\Users\jenkins\.java\$env:ES_BUILD_JAVA" +$env:SYSTEM_JAVA_HOME="$env:USERPROFILE\.java\$env:ES_BUILD_JAVA" Remove-Item -Recurse -Force \tmp -ErrorAction Ignore New-Item -ItemType directory -Path \tmp $ErrorActionPreference="Continue" -& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks +& .\gradlew.bat -g "$env:USERPROFILE\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks exit $LastExitCode diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh index d1e17d08bf762..afe162b23e564 100755 --- a/.ci/scripts/packaging-test.sh +++ b/.ci/scripts/packaging-test.sh @@ -39,6 +39,7 @@ if [ -f "/etc/os-release" ] ; then # Work around incorrect lintian version # https://github.com/elastic/elasticsearch/issues/48573 if [ $VERSION_ID == 10 ] ; then + sudo apt-get update -y sudo apt-get install -y --allow-downgrades lintian=2.15.0 fi fi diff --git a/.ci/scripts/run-gradle.sh b/.ci/scripts/run-gradle.sh index a63895465d52f..ea68ab7718579 100755 --- a/.ci/scripts/run-gradle.sh +++ b/.ci/scripts/run-gradle.sh @@ -1,27 +1,35 @@ #!/bin/bash -# drop page cache and kernel slab objects on linux -[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches + rm -Rfv ~/.gradle/init.d mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d -if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then - MAX_WORKERS=16 -elif [ -f /proc/cpuinfo ]; then - MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` -else - if [[ "$OSTYPE" == "darwin"* ]]; then - MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` - else - echo "Unsupported OS Type: $OSTYPE" - exit 1 - fi -fi -if pwd | grep -v -q ^/dev/shm ; then - echo "Not running on a ramdisk, reducing number of workers" - MAX_WORKERS=$(($MAX_WORKERS*2/3)) -fi -# Export glibc version as environment variable since some BWC tests are incompatible with later versions -export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/') +MAX_WORKERS=4 + +# Don't run this stuff on Windows +if ! uname -a | grep -q MING; then + # drop page cache and kernel slab objects on linux + [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches + + if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then + MAX_WORKERS=16 + elif [ -f /proc/cpuinfo ]; then + MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` + else + if [[ "$OSTYPE" == "darwin"* ]]; then + MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` + else + echo "Unsupported OS Type: $OSTYPE" + exit 1 + fi + fi + if pwd | grep -v -q ^/dev/shm ; then + echo "Not running on a ramdisk, reducing number of workers" + MAX_WORKERS=$(($MAX_WORKERS*2/3)) + fi + + # Export glibc version as environment variable since some BWC tests are incompatible with later versions + export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/') +fi set -e $GRADLEW -S --max-workers=$MAX_WORKERS $@ diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index a59023ae923da..f25f0ae5619d1 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - - "7.17.12" - - "8.8.3" - - "8.9.0" + - "7.17.13" + - "8.9.1" - "8.10.0" + - "8.11.0" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6a3d07df917ea..c29ed92f39547 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -19,3 +19,6 @@ x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monito # Fleet x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet @elastic/fleet x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet + +# Kibana Security +x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security diff --git a/BUILDING.md b/BUILDING.md index 692a7a559de25..814a9fb60ded8 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -244,7 +244,7 @@ For third party libraries that are not built with maven (e.g. ant) or provided a a flat directory repository that resolves artifacts from a flat directory on your filesystem. 1. Put the jar artifact with the format `artifactName-version.jar` into a directory named `localRepo` (you have to create this manually) -2. Declare a flatDir repository in your root build.gradle file +2. Declare a flatDir repository in your root build.gradle file (That ensures all projects have the flatDir repository declared and also the projects consuming the project you tweaked can resolve that local dependency) ``` allprojects { @@ -257,14 +257,14 @@ allprojects { ``` 3. Update the dependency declaration of the artifact in question to match the custom build version. For a file named e.g. `jmxri-1.2.1.jar` the - dependency definition would be `:jmxri:1.2.1` as it comes with no group information: + dependency definition would be `x:jmxri:1.2.1` as it the group information is ignored on flatdir repositories you can replace the `x` in group name: ``` dependencies { - implementation ':jmxri:1.2.1' + implementation 'x:jmxri:1.2.1' } ``` -4. Run the gradle build as needed. +4. Run the gradle build as needed with `--write-verification-metadata` to ensure the gradle dependency verification does not fail on your custom dependency. --- **NOTE** diff --git a/TRACING.md b/TRACING.md index 181d18e7ff760..3935a94eff160 100644 --- a/TRACING.md +++ b/TRACING.md @@ -1,16 +1,15 @@ # Tracing in Elasticsearch Elasticsearch is instrumented using the [OpenTelemetry][otel] API, which allows -us to gather traces and analyze what Elasticsearch is doing. - +ES developers to gather traces and analyze what Elasticsearch is doing. ## How is tracing implemented? -The Elasticsearch server code contains a [`tracing`][tracing] package, which is +The Elasticsearch server code contains a [tracing][tracing] package, which is an abstraction over the OpenTelemetry API. All locations in the code that perform instrumentation and tracing must use these abstractions. -Separately, there is the [`apm`](./modules/apm/) module, which works with the +Separately, there is the [apm](./modules/apm) module, which works with the OpenTelemetry API directly to record trace data. Underneath the OTel API, we use Elastic's [APM agent for Java][agent], which attaches at runtime to the Elasticsearch JVM and removes the need for Elasticsearch to hard-code the use of @@ -18,12 +17,21 @@ an OTel implementation. Note that while it is possible to programmatically start the APM agent, the Security Manager permissions required make this essentially impossible. - ## How is tracing configured? You must supply configuration and credentials for the APM server (see below). -You must also set `tracing.apm.enabled` to `true`, but this can be toggled at -runtime. +In your `elasticsearch.yml` add the following configuration: + +``` +tracing.apm.enabled: true +tracing.apm.agent.server_url: https://:443 +``` + +When using a secret token to authenticate with the APM server, you must add it to the Elasticsearch keystore under `tracing.apm.secret_token`. For example, execute: + + bin/elasticsearch-keystore add tracing.apm.secret_token + +then enter the token when prompted. If you are using API keys, change the keystore key name to `tracing.apm.api_key`. All APM settings live under `tracing.apm`. All settings related to the Java agent go under `tracing.apm.agent`. Anything you set under there will be propagated to @@ -91,7 +99,7 @@ tasks are long-lived and are not suitable candidates for APM tracing. When a span is started, Elasticsearch tracks information about that span in the current [thread context][thread-context]. If a new thread context is created, -then the current span information must not propagated but instead renamed, so +then the current span information must not be propagated but instead renamed, so that (1) it doesn't interfere when new trace information is set in the context, and (2) the previous trace information is available to establish a parent / child span relationship. This is done with `ThreadContext#newTraceContext()`. @@ -117,7 +125,7 @@ of new trace contexts when child spans need to be created. That's up to you. Be careful not to capture anything that could leak sensitive or personal information. -## What is "scope" and when should I used it? +## What is "scope" and when should I use it? Usually you won't need to. @@ -148,9 +156,8 @@ explicitly opening a scope via the `Tracer`. [otel]: https://opentelemetry.io/ -[thread-context]: ./server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java). +[thread-context]: ./server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java [w3c]: https://www.w3.org/TR/trace-context/ -[tracing]: ./server/src/main/java/org/elasticsearch/tracing/ -[config]: ./modules/apm/src/main/config/elasticapm.properties +[tracing]: ./server/src/main/java/org/elasticsearch/tracing [agent-config]: https://www.elastic.co/guide/en/apm/agent/java/master/configuration.html [agent]: https://www.elastic.co/guide/en/apm/agent/java/current/index.html diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index d5e4497fcf727..cd20a68ae5c1b 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -21,6 +21,7 @@ base { } tasks.named("test").configure { enabled = false } +tasks.named("javadoc").configure { enabled = false } configurations { expression @@ -47,6 +48,9 @@ dependencies { // needs to be added separately otherwise Gradle will quote it and javac will fail tasks.named("compileJava").configure { options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"]) + // org.elasticsearch.plugins.internal is used in signatures classes used in benchmarks but we don't want to expose it publicly + // adding an export to allow compilation with gradle. This does not solve a problem in intellij as it does not use compileJava task + options.compilerArgs.addAll(["--add-exports", "org.elasticsearch.server/org.elasticsearch.plugins.internal=ALL-UNNAMED"]) } tasks.register('copyExpression', Copy) { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java index 8d1a0f8199997..96c99648e356d 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.internal.DocumentParsingObserver; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptContext; @@ -71,7 +72,8 @@ public static MapperService create(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - } + }, + () -> DocumentParsingObserver.EMPTY_INSTANCE ); try { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index ca0b72fb13507..6a82f2722dea4 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.plugins.internal.DocumentParsingObserver; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptContext; @@ -188,7 +189,8 @@ protected final MapperService createMapperService(String mappings) { public T compile(Script script, ScriptContext scriptContext) { throw new UnsupportedOperationException(); } - } + }, + () -> DocumentParsingObserver.EMPTY_INSTANCE ); try { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/StringTemplatePluginTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/StringTemplatePluginTest.groovy index 962f0ba2afa70..6c94bdd3e86c5 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/StringTemplatePluginTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/StringTemplatePluginTest.groovy @@ -35,12 +35,12 @@ class StringTemplatePluginTest extends AbstractGradleFuncTest { tasks.named("stringTemplates").configure { template { it.properties = ["Type" : "Int", "type" : "int"] - it.inputFile = new File("${projectDir}/src/main/p/X-Box.java.st") + it.inputFile = file("src/main/p/X-Box.java.st") it.outputFile = "p/IntBox.java" } template { it.properties = ["Type" : "Long", "type" : "long"] - it.inputFile = new File("${projectDir}/src/main/p/X-Box.java.st") + it.inputFile = file("src/main/p/X-Box.java.st") it.outputFile = "p/LongBox.java" } } @@ -90,7 +90,7 @@ class StringTemplatePluginTest extends AbstractGradleFuncTest { tasks.named("stringTemplates").configure { template { it.properties = ["Red" : "true", "Blue" : "", "Green" : "true"] - it.inputFile = new File("${projectDir}/src/main/Color.txt.st") + it.inputFile = file("src/main/Color.txt.st") it.outputFile = "Color.txt" } } @@ -126,7 +126,7 @@ class StringTemplatePluginTest extends AbstractGradleFuncTest { tasks.named("stringTemplates").configure { template { it.properties = [:] // no properties - it.inputFile = new File("${projectDir}/src/main/Token.txt.st") + it.inputFile = file("src/main/Token.txt.st") it.outputFile = "Token.txt" } } @@ -162,7 +162,7 @@ class StringTemplatePluginTest extends AbstractGradleFuncTest { tasks.named("stringTemplates").configure { template { it.properties = [:] // no properties - it.inputFile = new File("${projectDir}/src/main/UpToDate.txt.st") + it.inputFile = file("src/main/UpToDate.txt.st") it.outputFile = "UpToDate.txt" } } @@ -198,7 +198,7 @@ class StringTemplatePluginTest extends AbstractGradleFuncTest { tasks.named("stringTemplates").configure { template { it.properties = [:] // no properties - it.inputFile = new File("${projectDir}/src/main/Message.txt.st") + it.inputFile = file("src/main/Message.txt.st") it.outputFile = "Message.txt" } } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index 85fb6e8acd969..0e0c4cb25a759 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -35,18 +35,22 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInterna version = "$version" repositories { - mavenCentral() + mavenCentral() } - + dependencies { implementation 'org.apache.lucene:lucene-monitor:9.2.0' } + + tasks.named('generateSnykDependencyGraph').configure { + remoteUrl = "http://acme.org" + } """ when: def build = gradleRunner("generateSnykDependencyGraph").build() then: build.task(":generateSnykDependencyGraph").outcome == TaskOutcome.SUCCESS - JSONAssert.assertEquals(file("build/snyk/dependencies.json").text, """{ + JSONAssert.assertEquals("""{ "meta": { "method": "custom gradle", "id": "gradle", @@ -101,7 +105,7 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInterna { "nodeId": "org.apache.lucene:lucene-core@9.2.0", "deps": [ - + ], "pkgId": "org.apache.lucene:lucene-core@9.2.0" }, @@ -155,7 +159,7 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInterna ] }, "target": { - "remoteUrl": "http://github.com/elastic/elasticsearch.git", + "remoteUrl": "http://acme.org", "branch": "unknown" }, "targetReference": "$version", @@ -164,7 +168,7 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInterna "$expectedLifecycle" ] } - }""", true) + }""", file("build/snyk/dependencies.json").text, true) where: version | expectedLifecycle diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index 575d514126765..7f9e258abd65a 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -16,7 +16,7 @@ buildScan { String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null // Automatically publish scans from Elasticsearch CI - if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev')) { + if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') { publishAlways() buildScan.server = 'https://gradle-enterprise.elastic.co' } @@ -121,12 +121,16 @@ buildScan { buildScanPublished { scan -> // Attach build scan link as build metadata // See: https://buildkite.com/docs/pipelines/build-meta-data - ['buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}"].execute() + new ProcessBuilder('buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}") + .start() + .waitFor() // Add a build annotation // See: https://buildkite.com/docs/agent/v3/cli-annotate def body = """
${System.getenv('BUILDKITE_LABEL')} :gradle: build ran: gradle ${gradle.startParameter.taskNames.join(' ')}
""" - ['buildkite-agent', 'annotate', '--context', 'gradle-build-scans', '--append', '--style', 'info', body].execute() + new ProcessBuilder('buildkite-agent', 'annotate', '--context', 'gradle-build-scans', '--append', '--style', 'info', body) + .start() + .waitFor() } } else { tag 'LOCAL' diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy index 1c1cf6d2a878c..1580ec891ed2b 100644 --- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy +++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy @@ -304,7 +304,7 @@ class SnippetsTask extends DefaultTask { } static Source matchSource(String line) { - def matcher = line =~ /\["?source"?,\s*"?([-\w]+)"?(,((?!id=).)*(id="?([-\w]+)"?)?(.*))?].*/ + def matcher = line =~ /\["?source"?(?:\.[^,]+)?,\s*"?([-\w]+)"?(,((?!id=).)*(id="?([-\w]+)"?)?(.*))?].*/ if(matcher.matches()){ return new Source(matches: true, language: matcher.group(1), name: matcher.group(5)) } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java index 0afa675c9dfc3..f0f718e15f05f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java @@ -13,6 +13,7 @@ import org.gradle.api.file.FileCollection; import org.gradle.api.model.ObjectFactory; import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.TaskAction; import java.util.Collection; @@ -27,7 +28,7 @@ public class ResolveAllDependencies extends DefaultTask { private final ObjectFactory objectFactory; - Collection configs; + private Collection configs; @Inject public ResolveAllDependencies(ObjectFactory objectFactory) { @@ -45,6 +46,15 @@ void resolveAll() { // do nothing, dependencies are resolved when snapshotting task inputs } + @Internal + public Collection getConfigs() { + return configs; + } + + public void setConfigs(Collection configs) { + this.configs = configs; + } + private static boolean canBeResolved(Configuration configuration) { if (configuration.isCanBeResolved() == false) { return false; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java index d76d715977bde..8dfea22ae2f91 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java @@ -171,8 +171,8 @@ private String formatJavaVendorDetails(JvmInstallationMetadata runtimeJdkMetaDat * logic in VersionUtils.java. */ private static BwcVersions resolveBwcVersions(File root) { File versionsFile = new File(root, DEFAULT_VERSION_JAVA_FILE_PATH); - try { - List versionLines = IOUtils.readLines(new FileInputStream(versionsFile), "UTF-8"); + try (var is = new FileInputStream(versionsFile)) { + List versionLines = IOUtils.readLines(is, "UTF-8"); return new BwcVersions(versionLines); } catch (IOException e) { throw new IllegalStateException("Unable to resolve to resolve bwc versions from versionsFile.", e); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java index cee10cf70b19c..d031113550295 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java @@ -116,8 +116,9 @@ private void checkModuleVersion(ModuleReference mref) { private void checkModuleNamePrefix(ModuleReference mref) { getLogger().info("{} checking module name prefix for {}", this, mref.descriptor().name()); - if (mref.descriptor().name().startsWith("org.elasticsearch.") == false) { - throw new GradleException("Expected name starting with \"org.elasticsearch.\", in " + mref.descriptor()); + if (mref.descriptor().name().startsWith("org.elasticsearch.") == false + && mref.descriptor().name().startsWith("co.elastic.") == false) { + throw new GradleException("Expected name starting with \"org.elasticsearch.\" or \"co.elastic\" in " + mref.descriptor()); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditPrecommitPlugin.java index d2abc00b18faa..90195db2ec296 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditPrecommitPlugin.java @@ -30,6 +30,7 @@ public TaskProvider createTask(Project project) { t.setProjectBuildDirs(getProjectBuildDirs(project)); t.setClasspath(project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME)); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); + t.dependsOn(mainSourceSet.getJava().getSourceDirectories()); t.getSrcDirs().set(project.provider(() -> mainSourceSet.getAllSource().getSrcDirs())); }); return task; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java index fc4b27042a369..9eb57eb636aa1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java @@ -57,6 +57,7 @@ public class GenerateSnykDependencyGraph extends DefaultTask { private final Property projectPath; private final Property targetReference; private final Property version; + private final Property remoteUrl; @Inject public GenerateSnykDependencyGraph(ObjectFactory objectFactory) { @@ -66,6 +67,7 @@ public GenerateSnykDependencyGraph(ObjectFactory objectFactory) { projectName = objectFactory.property(String.class); projectPath = objectFactory.property(String.class); version = objectFactory.property(String.class); + remoteUrl = objectFactory.property(String.class); targetReference = objectFactory.property(String.class); } @@ -115,7 +117,7 @@ private Map> projectAttributesData() { } private Object buildTargetData() { - return Map.of("remoteUrl", "http://github.com/elastic/elasticsearch.git", "branch", BuildParams.getGitRevision()); + return Map.of("remoteUrl", remoteUrl.get(), "branch", BuildParams.getGitRevision()); } @InputFiles @@ -148,6 +150,11 @@ public Property getGradleVersion() { return gradleVersion; } + @Input + public Property getRemoteUrl() { + return remoteUrl; + } + @Input public Property getTargetReference() { return targetReference; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java index 60c278948398a..a22de5aae5a94 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java @@ -8,6 +8,7 @@ package org.elasticsearch.gradle.internal.snyk; +import org.elasticsearch.gradle.internal.conventions.info.GitInfo; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; @@ -42,14 +43,16 @@ public void apply(Project project) { generateSnykDependencyGraph.getGradleVersion().set(project.getGradle().getGradleVersion()); generateSnykDependencyGraph.getTargetReference() .set(providerFactory.gradleProperty("snykTargetReference").orElse(projectVersion)); + generateSnykDependencyGraph.getRemoteUrl() + .convention(providerFactory.provider(() -> GitInfo.gitInfo(project.getRootDir()).urlFromOrigin())); generateSnykDependencyGraph.getOutputFile().set(projectLayout.getBuildDirectory().file("snyk/dependencies.json")); }); project.getTasks().register(UPLOAD_TASK_NAME, UploadSnykDependenciesGraph.class, t -> { t.getInputFile().set(generateTaskProvider.get().getOutputFile()); t.getToken().set(providerFactory.environmentVariable("SNYK_TOKEN")); - // the elasticsearch snyk project id - t.getProjectId().set(providerFactory.gradleProperty("snykProjectId")); + // the snyk org to target + t.getSnykOrganisation().set(providerFactory.gradleProperty("snykOrganisation")); }); project.getPlugins().withType(JavaPlugin.class, javaPlugin -> generateTaskProvider.configure(generateSnykDependencyGraph -> { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index ecfd9afb6b02d..445496f47eb7d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -42,12 +42,12 @@ public class UploadSnykDependenciesGraph extends DefaultTask { private final RegularFileProperty inputFile; private final Property token; private final Property url; - private final Property projectId; + private final Property snykOrganisation; @Inject public UploadSnykDependenciesGraph(ObjectFactory objectFactory) { url = objectFactory.property(String.class).convention(DEFAULT_SERVER + GRADLE_GRAPH_ENDPOINT); - projectId = objectFactory.property(String.class); + snykOrganisation = objectFactory.property(String.class); token = objectFactory.property(String.class); inputFile = objectFactory.fileProperty(); } @@ -76,7 +76,7 @@ void upload() { private String calculateEffectiveEndpoint() { String url = this.url.get(); - return url.endsWith(GRADLE_GRAPH_ENDPOINT) ? url : projectId.map(id -> url + "?org=" + id).getOrElse(url); + return snykOrganisation.map(id -> url + "?org=" + id).getOrElse(url); } @Input @@ -91,8 +91,8 @@ public Property getUrl() { @Input @Optional - public Property getProjectId() { - return projectId; + public Property getSnykOrganisation() { + return snykOrganisation; } @InputFile diff --git a/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt index ca1e6d85be984..c49082af9363b 100644 --- a/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt +++ b/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt @@ -17,11 +17,6 @@ java.util.concurrent.ThreadLocalRandom java.security.MessageDigest#clone() @ use org.elasticsearch.common.hash.MessageDigests -@defaultMessage Don't use MethodHandles in slow ways, don't be lenient in tests. -java.lang.invoke.MethodHandle#invoke(java.lang.Object[]) -java.lang.invoke.MethodHandle#invokeWithArguments(java.lang.Object[]) -java.lang.invoke.MethodHandle#invokeWithArguments(java.util.List) - @defaultMessage Don't open socket connections java.net.URL#openStream() java.net.URLConnection#connect() diff --git a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt index a8f380195e16c..c22339be5332f 100644 --- a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt +++ b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt @@ -38,6 +38,11 @@ java.lang.Math#abs(long) @defaultMessage Please do not try to stop the world java.lang.System#gc() +@defaultMessage Don't use MethodHandles in slow ways +java.lang.invoke.MethodHandle#invoke(java.lang.Object[]) +java.lang.invoke.MethodHandle#invokeWithArguments(java.lang.Object[]) +java.lang.invoke.MethodHandle#invokeWithArguments(java.util.List) + @defaultMessage Use Channels.* methods to write to channels. Do not write directly. java.nio.channels.WritableByteChannel#write(java.nio.ByteBuffer) java.nio.channels.FileChannel#write(java.nio.ByteBuffer, long) diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc b/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc index be4074591d720..855a443a8081f 100644 --- a/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc @@ -16,32 +16,21 @@ coming::[${majorDotMinorDotRevision}] [[breaking-changes-${majorDotMinor}]] === Breaking changes <% if (breakingByNotabilityByArea.isEmpty()) { %> -// tag::notable-breaking-changes[] There are no breaking changes in {es} ${majorDotMinor}. -// end::notable-breaking-changes[] <% } else { %> The following changes in {es} ${majorDotMinor} might affect your applications and prevent them from operating normally. Before upgrading to ${majorDotMinor}, review these changes and take the described steps to mitigate the impact. - <% if (breakingByNotabilityByArea.getOrDefault(true, []).isEmpty()) { %> -// tag::notable-breaking-changes[] + There are no notable breaking changes in {es} ${majorDotMinor}. -// end::notable-breaking-changes[] But there are some less critical breaking changes. <% } [true, false].each { isNotable -> def breakingByArea = breakingByNotabilityByArea.getOrDefault(isNotable, []) if (breakingByArea.isEmpty() == false) { - if (isNotable) { - /* No newline here, one will be added below */ - print "// NOTE: The notable-breaking-changes tagged regions are re-used in the\n" - print "// Installation and Upgrade Guide\n" - print "// tag::notable-breaking-changes[]" - } - breakingByArea.eachWithIndex { area, breakingChanges, i -> print "\n[discrete]\n" print "[[breaking_${majorMinor}_${ area.toLowerCase().replaceAll("[^a-z0-9]+", "_") }_changes]]\n" @@ -62,9 +51,6 @@ ${breaking.impact.trim()} } } - if (isNotable) { - print "// end::notable-breaking-changes[]\n" - } } } } @@ -83,16 +69,10 @@ after upgrading to ${majorDotMinor}. To find out if you are using any deprecated functionality, enable <>. - <% [true, false].each { isNotable -> def deprecationsByArea = deprecationsByNotabilityByArea.getOrDefault(isNotable, []) if (deprecationsByArea.isEmpty() == false) { - if (isNotable) { - /* No newline here, one will be added below */ - print "// tag::notable-breaking-changes[]" - } - deprecationsByArea.eachWithIndex { area, deprecations, i -> print "\n[discrete]\n" print "[[deprecations_${majorMinor}_${ area.toLowerCase().replaceAll("[^a-z0-9]+", "_") }]]\n" @@ -113,9 +93,6 @@ ${deprecation.impact.trim()} } } - if (isNotable) { - print "// end::notable-breaking-changes[]\n" - } } } } %> diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/doc/SnippetsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/doc/SnippetsTaskTests.java index 846835e918e0c..0acae6ca03297 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/doc/SnippetsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/doc/SnippetsTaskTests.java @@ -51,5 +51,13 @@ public void testMatchSource() { assertTrue(source.getMatches()); assertEquals("console", source.getLanguage()); assertEquals("snippet-name-1", source.getName()); + + source = SnippetsTask.matchSource("[source.merge.styled,esql]"); + assertTrue(source.getMatches()); + assertEquals("esql", source.getLanguage()); + + source = SnippetsTask.matchSource("[source.merge.styled,foo-bar]"); + assertTrue(source.getMatches()); + assertEquals("foo-bar", source.getLanguage()); } } diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc index d8be431302601..ee78d10fd4b07 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc @@ -21,9 +21,6 @@ and prevent them from operating normally. Before upgrading to 8.4, review these changes and take the described steps to mitigate the impact. -// NOTE: The notable-breaking-changes tagged regions are re-used in the -// Installation and Upgrade Guide -// tag::notable-breaking-changes[] [discrete] [[breaking_84_api_changes]] ==== API changes @@ -64,7 +61,6 @@ Breaking change details 4 *Impact* + Breaking change impact description 4 ==== -// end::notable-breaking-changes[] [discrete] [[breaking_84_transform_changes]] @@ -95,7 +91,6 @@ after upgrading to 8.4. To find out if you are using any deprecated functionality, enable <>. -// tag::notable-breaking-changes[] [discrete] [[deprecations_84_cluster_and_node_setting]] ==== Cluster and node setting deprecations @@ -121,7 +116,6 @@ Deprecation change details 6 *Impact* + Deprecation change impact description 6 ==== -// end::notable-breaking-changes[] [discrete] [[deprecations_84_cluster_and_node_setting]] diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 10e771cc1f4e0..dc4d6f0dc6d47 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,8 +1,8 @@ -elasticsearch = 8.10.0 +elasticsearch = 8.11.0 lucene = 9.7.0 bundled_jdk_vendor = openjdk -bundled_jdk = 20.0.1+9@b4887098932d415489976708ad6d1a4b +bundled_jdk = 20.0.2+9@6e380f22cbe7469fa75fb448bd903d8e # optional dependencies spatial4j = 0.7 diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java index d63efbe3e55cb..afd8b13c839f8 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java @@ -39,7 +39,7 @@ public abstract class ReaperService implements BuildServiceconditional() - .onWindows(() -> new String[] { "Taskkill", "/F", "/PID", String.valueOf(pid) }) + .onWindows(() -> new String[] { "Taskkill", "/F", "/T", "/PID", String.valueOf(pid) }) .onUnix(() -> new String[] { "kill", "-9", String.valueOf(pid) }) .supply(); registerCommand(serviceId, killPidCommand); diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 1a34eb750692c..f0ab67fe51a34 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -1213,12 +1213,12 @@ private void waitForProcessToExit(ProcessHandle processHandle) { try { processHandle.onExit().get(ES_DESTROY_TIMEOUT, ES_DESTROY_TIMEOUT_UNIT); } catch (InterruptedException e) { - LOGGER.info("Interrupted while waiting for ES process", e); + LOGGER.info("[{}] Interrupted while waiting for ES process", name, e); Thread.currentThread().interrupt(); } catch (ExecutionException e) { - LOGGER.info("Failure while waiting for process to exist", e); + LOGGER.info("[{}] Failure while waiting for process to exist", name, e); } catch (TimeoutException e) { - LOGGER.info("Timed out waiting for process to exit", e); + LOGGER.info("[{}] Timed out waiting for process to exit", name, e); } } diff --git a/catalog-info.yaml b/catalog-info.yaml index 38b7752a893d8..4f5d06357b312 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -16,13 +16,14 @@ spec: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - description: ':elasticsearch: Update elasticsearch submodule in elasticsearch-serverless' + description: ":elasticsearch: Update elasticsearch submodule in elasticsearch-serverless" name: elasticsearch / update serverless submodule spec: repository: elastic/elasticsearch pipeline_file: .buildkite/update-es-serverless.yml teams: elasticsearch-team: {} + ml-core: {} everyone: access_level: READ_ONLY provider_settings: @@ -30,7 +31,7 @@ spec: schedules: daily promotion: branch: main - cronline: '@daily' + cronline: "@daily" --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json apiVersion: backstage.io/v1alpha1 @@ -49,7 +50,7 @@ spec: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - description: ':elasticsearch: Validate elasticsearch changes against serverless' + description: ":elasticsearch: Validate elasticsearch changes against serverless" name: elasticsearch / check serverless submodule spec: repository: elastic/elasticsearch @@ -57,8 +58,202 @@ spec: branch_configuration: main teams: elasticsearch-team: {} + ml-core: {} everyone: access_level: READ_ONLY provider_settings: build_pull_requests: false publish_commit_status: false +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-elasticsearch-periodic + description: Elasticsearch tests and checks that are run a few times daily + links: + - title: Pipeline + url: https://buildkite.com/elastic/elasticsearch-periodic +spec: + type: buildkite-pipeline + system: buildkite + owner: group:elasticsearch-team + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + description: ":elasticsearch: Tests and checks that are run a few times daily" + name: elasticsearch / periodic + spec: + repository: elastic/elasticsearch + pipeline_file: .buildkite/pipelines/periodic.yml + branch_configuration: main + teams: + elasticsearch-team: {} + ml-core: {} + everyone: + access_level: BUILD_AND_READ + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-elasticsearch-periodic-trigger + description: Triggers periodic pipeline for all required branches + links: + - title: Pipeline + url: https://buildkite.com/elastic/elasticsearch-periodic-trigger +spec: + type: buildkite-pipeline + system: buildkite + owner: group:elasticsearch-team + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + description: ":elasticsearch: Triggers periodic pipeline for all required branches" + name: elasticsearch / periodic / trigger + spec: + repository: elastic/elasticsearch + pipeline_file: .buildkite/pipelines/periodic.trigger.yml + branch_configuration: main + teams: + elasticsearch-team: {} + ml-core: {} + everyone: + access_level: BUILD_AND_READ + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + schedules: + Periodically on main: + branch: main + cronline: "0 0,8,16 * * * America/New_York" + message: "Triggers pipelines 3x daily" +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-elasticsearch-lucene-snapshot-build + description: Builds a new lucene snapshot, uploads, updates the lucene_snapshot branch in ES, runs tests + links: + - title: Pipeline + url: https://buildkite.com/elastic/elasticsearch-lucene-snapshot-build +spec: + type: buildkite-pipeline + system: buildkite + owner: group:elasticsearch-team + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + description: ":elasticsearch: Builds a new lucene snapshot and tests it" + name: elasticsearch / lucene-snapshot / build-and-update + spec: + repository: elastic/elasticsearch + pipeline_file: .buildkite/pipelines/lucene-snapshot/build-snapshot.yml + branch_configuration: lucene_snapshot + default_branch: lucene_snapshot + teams: + elasticsearch-team: {} + ml-core: {} + everyone: + access_level: BUILD_AND_READ + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + schedules: + Periodically on lucene_snapshot: + branch: lucene_snapshot + cronline: "0 2 * * * America/New_York" + message: "Builds a new lucene snapshot 1x per day" +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-elasticsearch-lucene-snapshot-update-branch + description: Merge main into the lucene_snapshot branch, and run tests + links: + - title: Pipeline + url: https://buildkite.com/elastic/elasticsearch-lucene-snapshot-update-branch +spec: + type: buildkite-pipeline + system: buildkite + owner: group:elasticsearch-team + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + description: ":elasticsearch: Merges main into lucene_snapshot branch and runs tests" + name: elasticsearch / lucene-snapshot / update-branch + spec: + repository: elastic/elasticsearch + pipeline_file: .buildkite/pipelines/lucene-snapshot/update-branch.yml + branch_configuration: lucene_snapshot + default_branch: lucene_snapshot + teams: + elasticsearch-team: {} + ml-core: {} + everyone: + access_level: BUILD_AND_READ + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + schedules: + Periodically on lucene_snapshot: + branch: lucene_snapshot + cronline: "0 6 * * * America/New_York" + message: "Merges main into lucene_snapshot branch 1x per day" +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-elasticsearch-lucene-snapshot-tests + description: Runs tests against lucene_snapshot branch + links: + - title: Pipeline + url: https://buildkite.com/elastic/elasticsearch-lucene-snapshot-tests +spec: + type: buildkite-pipeline + system: buildkite + owner: group:elasticsearch-team + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + description: ":elasticsearch: Runs tests against lucene_snapshot branch" + name: elasticsearch / lucene-snapshot / tests + spec: + repository: elastic/elasticsearch + pipeline_file: .buildkite/pipelines/lucene-snapshot/run-tests.yml + branch_configuration: lucene_snapshot + default_branch: lucene_snapshot + teams: + elasticsearch-team: {} + ml-core: {} + everyone: + access_level: BUILD_AND_READ + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + schedules: + Periodically on lucene_snapshot: + branch: lucene_snapshot + cronline: "0 9,12,15,18 * * * America/New_York" + message: "Runs tests against lucene_snapshot branch several times per day" diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index c5c6528272621..c99a9af302c40 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -308,6 +308,7 @@ Closure commonDebConfig(String architecture) { into('/usr/share/lintian/overrides') { from('src/deb/lintian/elasticsearch') + fileMode 0644 } } } diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java index d18c1463e48c5..7d900155488b2 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.plugins.PluginTestUtil; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.VersionUtils; import org.junit.Before; import java.io.BufferedReader; @@ -141,17 +140,8 @@ public void testRemoveMultiple() throws Exception { assertRemoveCleaned(env); } - public void testRemoveOldVersion() throws Exception { - Version previous = VersionUtils.getPreviousVersion(); - if (previous.before(Version.CURRENT.minimumIndexCompatibilityVersion())) { - // Can happen when bumping majors: 8.0 is only compat back to 7.0, but that's not released yet - // In this case, ignore what's released and just find that latest version before current - previous = VersionUtils.allVersions().stream().filter(v -> v.before(Version.CURRENT)).max(Version::compareTo).get(); - } - createPlugin("fake", VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumIndexCompatibilityVersion(), previous)); - removePlugin("fake", home, randomBoolean()); - assertThat(Files.exists(env.pluginsFile().resolve("fake")), equalTo(false)); - assertRemoveCleaned(env); + private static Version minimumCompatibleVersion(Version v) { + return Version.fromString((v.major - 1) + ".0.0"); } public void testBin() throws Exception { diff --git a/docs/build.gradle b/docs/build.gradle index 5f30043f4fd25..4cb5a0e9962c1 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -127,15 +127,6 @@ tasks.named("yamlRestTest").configure { } } -// TODO: Remove the following when RCS feature is released -// The get-builtin-privileges doc test does not include the new cluster privilege for RCS -// So we disable the test if the build is a snapshot where unreleased feature is enabled by default -tasks.named("yamlRestTest").configure { - if (BuildParams.isSnapshotBuild()) { - systemProperty 'tests.rest.blacklist', 'reference/rest-api/usage/*' - } -} - tasks.named("forbiddenPatterns").configure { exclude '**/*.mmdb' } diff --git a/docs/changelog/94132.yaml b/docs/changelog/94132.yaml new file mode 100644 index 0000000000000..e53bc9bd5fabb --- /dev/null +++ b/docs/changelog/94132.yaml @@ -0,0 +1,5 @@ +pr: 94132 +summary: HDFS plugin add replication_factor param +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/96068.yaml b/docs/changelog/96068.yaml new file mode 100644 index 0000000000000..9faf74414d36e --- /dev/null +++ b/docs/changelog/96068.yaml @@ -0,0 +1,5 @@ +pr: 96068 +summary: Use the Weight#matches mode for highlighting by default +area: Search +type: enhancement +issues: [] \ No newline at end of file diff --git a/docs/changelog/96082.yaml b/docs/changelog/96082.yaml new file mode 100644 index 0000000000000..225ac9ffa0b5d --- /dev/null +++ b/docs/changelog/96082.yaml @@ -0,0 +1,5 @@ +pr: 96082 +summary: Support minimum_should_match field for terms_set query +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/96515.yaml b/docs/changelog/96515.yaml new file mode 100644 index 0000000000000..bf6403f6990ce --- /dev/null +++ b/docs/changelog/96515.yaml @@ -0,0 +1,5 @@ +pr: 96515 +summary: Support boxplot aggregation in transform +area: Transform +type: enhancement +issues: [] diff --git a/docs/changelog/97092.yaml b/docs/changelog/97092.yaml new file mode 100644 index 0000000000000..d065a7291b5ac --- /dev/null +++ b/docs/changelog/97092.yaml @@ -0,0 +1,6 @@ +pr: 97092 +summary: "Fix `fields` API with `subobjects: false`" +area: Mapping +type: bug +issues: + - 96700 diff --git a/docs/changelog/97110.yaml b/docs/changelog/97110.yaml new file mode 100644 index 0000000000000..a1e65266c865a --- /dev/null +++ b/docs/changelog/97110.yaml @@ -0,0 +1,6 @@ +pr: 97110 +summary: Copy "index.lifecycle.name" for ILM managed indices +area: Downsampling +type: bug +issues: + - 96732 diff --git a/docs/changelog/97395.yaml b/docs/changelog/97395.yaml new file mode 100644 index 0000000000000..3e32858aa340d --- /dev/null +++ b/docs/changelog/97395.yaml @@ -0,0 +1,5 @@ +pr: 97395 +summary: Refresh tokens without search +area: Authentication +type: enhancement +issues: [] diff --git a/docs/changelog/97410.yaml b/docs/changelog/97410.yaml new file mode 100644 index 0000000000000..cf2536df59e7e --- /dev/null +++ b/docs/changelog/97410.yaml @@ -0,0 +1,5 @@ +pr: 97410 +summary: Introduce a collector manager for `QueryPhaseCollector` +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/97416.yaml b/docs/changelog/97416.yaml new file mode 100644 index 0000000000000..079e93ee99b5f --- /dev/null +++ b/docs/changelog/97416.yaml @@ -0,0 +1,5 @@ +pr: 97416 +summary: Enable parallel collection in Dfs phase +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/97460.yaml b/docs/changelog/97460.yaml new file mode 100644 index 0000000000000..55bcacaaded11 --- /dev/null +++ b/docs/changelog/97460.yaml @@ -0,0 +1,6 @@ +pr: 97460 +summary: Set new providers before building `FetchSubPhaseProcessors` +area: "Search" +type: bug +issues: + - 96284 diff --git a/docs/changelog/97463.yaml b/docs/changelog/97463.yaml new file mode 100644 index 0000000000000..df2ce0e474011 --- /dev/null +++ b/docs/changelog/97463.yaml @@ -0,0 +1,5 @@ +pr: 97463 +summary: "[Enterprise Search] Add connectors indices and ent-search pipeline" +area: Application +type: feature +issues: [] diff --git a/docs/changelog/97466.yaml b/docs/changelog/97466.yaml new file mode 100644 index 0000000000000..5f9e72430bd6d --- /dev/null +++ b/docs/changelog/97466.yaml @@ -0,0 +1,5 @@ +pr: 97466 +summary: Enable Query Rules as technical preview +area: Application +type: feature +issues: [] diff --git a/docs/changelog/97488.yaml b/docs/changelog/97488.yaml new file mode 100644 index 0000000000000..2094338d63896 --- /dev/null +++ b/docs/changelog/97488.yaml @@ -0,0 +1,6 @@ +pr: 97488 +summary: Limit `_terms_enum` prefix size +area: Search +type: enhancement +issues: + - 96572 diff --git a/docs/changelog/97509.yaml b/docs/changelog/97509.yaml new file mode 100644 index 0000000000000..dd371f65fd658 --- /dev/null +++ b/docs/changelog/97509.yaml @@ -0,0 +1,6 @@ +pr: 97509 +summary: Fix bug when creating empty `geo_lines` +area: Geo +type: bug +issues: + - 97311 diff --git a/docs/changelog/97539.yaml b/docs/changelog/97539.yaml new file mode 100644 index 0000000000000..b7e3ffd4eeb9e --- /dev/null +++ b/docs/changelog/97539.yaml @@ -0,0 +1,5 @@ +pr: 97539 +summary: "Give all acces to .slo-observability.* indice to kibana user" +area: Authentication +type: enhancement +issues: [] \ No newline at end of file diff --git a/docs/changelog/97540.yaml b/docs/changelog/97540.yaml new file mode 100644 index 0000000000000..471400689a1ec --- /dev/null +++ b/docs/changelog/97540.yaml @@ -0,0 +1,6 @@ +pr: 97540 +summary: Make `terminate_after` early termination friendly +area: Search +type: bug +issues: + - 97269 diff --git a/docs/changelog/97550.yaml b/docs/changelog/97550.yaml new file mode 100644 index 0000000000000..b7e6097f70264 --- /dev/null +++ b/docs/changelog/97550.yaml @@ -0,0 +1,5 @@ +pr: 97550 +summary: Introduce a collector manager for `PartialHitCountCollector` +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/97557.yaml b/docs/changelog/97557.yaml new file mode 100644 index 0000000000000..28e0b9e413964 --- /dev/null +++ b/docs/changelog/97557.yaml @@ -0,0 +1,6 @@ +pr: 97557 +summary: Run downsampling using persistent tasks +area: Downsampling +type: enhancement +issues: + - 93582 diff --git a/docs/changelog/97581.yaml b/docs/changelog/97581.yaml new file mode 100644 index 0000000000000..ada73b7962631 --- /dev/null +++ b/docs/changelog/97581.yaml @@ -0,0 +1,5 @@ +pr: 97581 +summary: Add Setting to optionally use mmap for shared cache IO +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/97587.yaml b/docs/changelog/97587.yaml new file mode 100644 index 0000000000000..39906bc4a4412 --- /dev/null +++ b/docs/changelog/97587.yaml @@ -0,0 +1,5 @@ +pr: 97587 +summary: Fix `sub_searches` serialization bug +area: Ranking +type: bug +issues: [] diff --git a/docs/changelog/97588.yaml b/docs/changelog/97588.yaml new file mode 100644 index 0000000000000..e714cb14f77c5 --- /dev/null +++ b/docs/changelog/97588.yaml @@ -0,0 +1,5 @@ +pr: 97588 +summary: Fix weird `change_point` bug where all data values are equivalent +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/97591.yaml b/docs/changelog/97591.yaml new file mode 100644 index 0000000000000..bd188761c6202 --- /dev/null +++ b/docs/changelog/97591.yaml @@ -0,0 +1,5 @@ +pr: 97591 +summary: Changing watcher to disable cookies in shared http client +area: Watcher +type: bug +issues: [] diff --git a/docs/changelog/97594.yaml b/docs/changelog/97594.yaml new file mode 100644 index 0000000000000..61641fcaf86f2 --- /dev/null +++ b/docs/changelog/97594.yaml @@ -0,0 +1,5 @@ +pr: 97594 +summary: Improve wildcard query and terms query rewrite +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/97602.yaml b/docs/changelog/97602.yaml new file mode 100644 index 0000000000000..0af420167e5ed --- /dev/null +++ b/docs/changelog/97602.yaml @@ -0,0 +1,5 @@ +pr: 97602 +summary: Add tsdb metrics builtin component template +area: "TSDB" +type: enhancement +issues: [] diff --git a/docs/changelog/97619.yaml b/docs/changelog/97619.yaml new file mode 100644 index 0000000000000..1c17e4e513691 --- /dev/null +++ b/docs/changelog/97619.yaml @@ -0,0 +1,6 @@ +pr: 97619 +summary: Support for Byte and Short as vector tiles features +area: Geo +type: bug +issues: + - 97612 diff --git a/docs/changelog/97630.yaml b/docs/changelog/97630.yaml new file mode 100644 index 0000000000000..edceb0c51c6b1 --- /dev/null +++ b/docs/changelog/97630.yaml @@ -0,0 +1,5 @@ +pr: 97630 +summary: Add an API for managing the settings of Security system indices +area: Security +type: enhancement +issues: [] diff --git a/docs/changelog/97679.yaml b/docs/changelog/97679.yaml new file mode 100644 index 0000000000000..d8c96fcba4acb --- /dev/null +++ b/docs/changelog/97679.yaml @@ -0,0 +1,5 @@ +pr: 97679 +summary: Add Configuration to `PatternLayout` +area: Infra/Logging +type: bug +issues: [] diff --git a/docs/changelog/97683.yaml b/docs/changelog/97683.yaml new file mode 100644 index 0000000000000..8b843eb7e9cf6 --- /dev/null +++ b/docs/changelog/97683.yaml @@ -0,0 +1,5 @@ +pr: 97683 +summary: Refactor nested field handling in `FieldFetcher` +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/97700.yaml b/docs/changelog/97700.yaml new file mode 100644 index 0000000000000..0e144a22ab39b --- /dev/null +++ b/docs/changelog/97700.yaml @@ -0,0 +1,6 @@ +pr: 97700 +summary: Add `completion_time` time field to `async_search` get and status response +area: Search +type: enhancement +issues: + - 88640 diff --git a/docs/changelog/97703.yaml b/docs/changelog/97703.yaml new file mode 100644 index 0000000000000..cb9efb0e87c94 --- /dev/null +++ b/docs/changelog/97703.yaml @@ -0,0 +1,6 @@ +pr: 97703 +summary: Track `max_score` in collapse when requested +area: Search +type: bug +issues: + - 97653 diff --git a/docs/changelog/97705.yaml b/docs/changelog/97705.yaml new file mode 100644 index 0000000000000..a9b6c3f79cffb --- /dev/null +++ b/docs/changelog/97705.yaml @@ -0,0 +1,7 @@ +pr: 97705 +summary: "Fix to stop aggregatable subobjects from being considered multi-fields, to support \ + `\"subobjects\": false` in data frame analytics" +area: Machine Learning +type: bug +issues: + - 88605 diff --git a/docs/changelog/97711.yaml b/docs/changelog/97711.yaml new file mode 100644 index 0000000000000..e1dc8612091f3 --- /dev/null +++ b/docs/changelog/97711.yaml @@ -0,0 +1,5 @@ +pr: 97711 +summary: Release master service task on timeout +area: Cluster Coordination +type: bug +issues: [] diff --git a/docs/changelog/97718.yaml b/docs/changelog/97718.yaml new file mode 100644 index 0000000000000..f864aa70ae384 --- /dev/null +++ b/docs/changelog/97718.yaml @@ -0,0 +1,6 @@ +pr: 97718 +summary: Fix async missing events +area: EQL +type: bug +issues: + - 97644 diff --git a/docs/changelog/97724.yaml b/docs/changelog/97724.yaml new file mode 100644 index 0000000000000..78f3812e36cd7 --- /dev/null +++ b/docs/changelog/97724.yaml @@ -0,0 +1,5 @@ +pr: 97724 +summary: Mapped field types searchable with doc values +area: TSDB +type: bug +issues: [] diff --git a/docs/changelog/97726.yaml b/docs/changelog/97726.yaml new file mode 100644 index 0000000000000..37f8e966829cd --- /dev/null +++ b/docs/changelog/97726.yaml @@ -0,0 +1,5 @@ +pr: 97726 +summary: Wire `QueryPhaseCollectorManager` into the query phase +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/97732.yaml b/docs/changelog/97732.yaml new file mode 100644 index 0000000000000..d2b9d7cb67fb9 --- /dev/null +++ b/docs/changelog/97732.yaml @@ -0,0 +1,5 @@ +pr: 97732 +summary: "[Fleet] Allow `kibana_system` to put datastream lifecycle" +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/97741.yaml b/docs/changelog/97741.yaml new file mode 100644 index 0000000000000..926e933e7c838 --- /dev/null +++ b/docs/changelog/97741.yaml @@ -0,0 +1,5 @@ +pr: 97741 +summary: Upgrade xmlsec to 2.1.8 +area: Authentication +type: enhancement +issues: [] diff --git a/docs/changelog/97755.yaml b/docs/changelog/97755.yaml new file mode 100644 index 0000000000000..a2b50ba2f0fc8 --- /dev/null +++ b/docs/changelog/97755.yaml @@ -0,0 +1,5 @@ +pr: 97755 +summary: Wire concurrent top docs collector managers when size is 0 +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/97765.yaml b/docs/changelog/97765.yaml new file mode 100644 index 0000000000000..67a5c8f72cc70 --- /dev/null +++ b/docs/changelog/97765.yaml @@ -0,0 +1,5 @@ +pr: 97765 +summary: Install data stream template for Kibana reporting +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/97773.yaml b/docs/changelog/97773.yaml new file mode 100644 index 0000000000000..3106dc2621742 --- /dev/null +++ b/docs/changelog/97773.yaml @@ -0,0 +1,5 @@ +pr: 97773 +summary: "[Profiling] Support index migrations" +area: Application +type: enhancement +issues: [] diff --git a/docs/changelog/97775.yaml b/docs/changelog/97775.yaml new file mode 100644 index 0000000000000..9d6eee2569c31 --- /dev/null +++ b/docs/changelog/97775.yaml @@ -0,0 +1,5 @@ +pr: 97775 +summary: Fix NPE in Desired Balance API +area: Allocation +type: bug +issues: [] diff --git a/docs/changelog/97822.yaml b/docs/changelog/97822.yaml new file mode 100644 index 0000000000000..1457fb4fad12f --- /dev/null +++ b/docs/changelog/97822.yaml @@ -0,0 +1,5 @@ +pr: 97822 +summary: Allow Lucene directory implementations to estimate their size +area: Store +type: enhancement +issues: [] diff --git a/docs/changelog/97840.yaml b/docs/changelog/97840.yaml new file mode 100644 index 0000000000000..3bcf62328c873 --- /dev/null +++ b/docs/changelog/97840.yaml @@ -0,0 +1,6 @@ +pr: 97840 +summary: Improve exception handling in Coordinator#publish +area: Cluster Coordination +type: bug +issues: + - 97798 diff --git a/docs/changelog/97850.yaml b/docs/changelog/97850.yaml new file mode 100644 index 0000000000000..f798c3f5cd404 --- /dev/null +++ b/docs/changelog/97850.yaml @@ -0,0 +1,5 @@ +pr: 97850 +summary: Allow custom geo ip database files to be downloaded +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/changelog/97869.yaml b/docs/changelog/97869.yaml new file mode 100644 index 0000000000000..067ee5c93ba3e --- /dev/null +++ b/docs/changelog/97869.yaml @@ -0,0 +1,5 @@ +pr: 97869 +summary: Add missing sync on `indicesThatCannotBeCreated` +area: CRUD +type: bug +issues: [] diff --git a/docs/changelog/97890.yaml b/docs/changelog/97890.yaml new file mode 100644 index 0000000000000..9d2339d04194f --- /dev/null +++ b/docs/changelog/97890.yaml @@ -0,0 +1,5 @@ +pr: 97890 +summary: "[Profiling] Consider static settings in status" +area: Application +type: bug +issues: [] diff --git a/docs/changelog/97899.yaml b/docs/changelog/97899.yaml new file mode 100644 index 0000000000000..c6ea51c11461a --- /dev/null +++ b/docs/changelog/97899.yaml @@ -0,0 +1,5 @@ +pr: 97899 +summary: Fork response reading in `TransportNodesAction` +area: Distributed +type: bug +issues: [] diff --git a/docs/changelog/97920.yaml b/docs/changelog/97920.yaml new file mode 100644 index 0000000000000..34cfa8c445424 --- /dev/null +++ b/docs/changelog/97920.yaml @@ -0,0 +1,6 @@ +pr: 97920 +summary: Avoid `transport_worker` thread in `TransportBroadcastByNodeAction` +area: Distributed +type: bug +issues: + - 97914 diff --git a/docs/changelog/97922.yaml b/docs/changelog/97922.yaml new file mode 100644 index 0000000000000..e8fb279f8d291 --- /dev/null +++ b/docs/changelog/97922.yaml @@ -0,0 +1,5 @@ +pr: 97922 +summary: Fork remote-cluster response handling +area: Network +type: bug +issues: [] diff --git a/docs/changelog/97936.yaml b/docs/changelog/97936.yaml new file mode 100644 index 0000000000000..2cf0c3e381b94 --- /dev/null +++ b/docs/changelog/97936.yaml @@ -0,0 +1,6 @@ +pr: 97936 +summary: Migrate to data tiers routing configures correct default for mounted indices +area: ILM+SLM +type: bug +issues: + - 97898 diff --git a/docs/changelog/97961.yaml b/docs/changelog/97961.yaml new file mode 100644 index 0000000000000..94bfb3f9bc4d0 --- /dev/null +++ b/docs/changelog/97961.yaml @@ -0,0 +1,5 @@ +pr: 97961 +summary: Infrastructure to report upon document parsing +area: Infra/Core +type: enhancement +issues: [] diff --git a/docs/changelog/97962.yaml b/docs/changelog/97962.yaml new file mode 100644 index 0000000000000..d4987245bf352 --- /dev/null +++ b/docs/changelog/97962.yaml @@ -0,0 +1,6 @@ +pr: 97962 +summary: Introduce Synonyms Management API used for synonym and synonym_graph filters +area: Search +type: enhancement +issues: + - 38523 diff --git a/docs/changelog/97967.yaml b/docs/changelog/97967.yaml new file mode 100644 index 0000000000000..d14fbf788864e --- /dev/null +++ b/docs/changelog/97967.yaml @@ -0,0 +1,5 @@ +pr: 97967 +summary: Ensure frozen indices have correct tier preference +area: "Indices APIs" +type: bug +issues: [] diff --git a/docs/changelog/97979.yaml b/docs/changelog/97979.yaml new file mode 100644 index 0000000000000..e7907a761ebf7 --- /dev/null +++ b/docs/changelog/97979.yaml @@ -0,0 +1,5 @@ +pr: 97979 +summary: Ignore the `total_shards_per_node` setting on searchable snapshots in frozen +area: ILM+SLM +type: bug +issues: [] diff --git a/docs/changelog/97985.yaml b/docs/changelog/97985.yaml new file mode 100644 index 0000000000000..6389f6cfc110f --- /dev/null +++ b/docs/changelog/97985.yaml @@ -0,0 +1,5 @@ +pr: 97985 +summary: Adding `ApiFilteringActionFilter` +area: Infra/Plugins +type: enhancement +issues: [] diff --git a/docs/changelog/97987.yaml b/docs/changelog/97987.yaml new file mode 100644 index 0000000000000..a44d219e37769 --- /dev/null +++ b/docs/changelog/97987.yaml @@ -0,0 +1,6 @@ +pr: 97987 +summary: '`UnmappedFieldFetcher` should ignore nested fields' +area: Search +type: bug +issues: + - 97684 diff --git a/docs/changelog/97992.yaml b/docs/changelog/97992.yaml new file mode 100644 index 0000000000000..6f5746c04b852 --- /dev/null +++ b/docs/changelog/97992.yaml @@ -0,0 +1,6 @@ +pr: 97401 +summary: Fix transform incorrectly calculating date bucket on updating old data +area: Transform +type: bug +issues: + - 97101 diff --git a/docs/changelog/98001.yaml b/docs/changelog/98001.yaml new file mode 100644 index 0000000000000..2a5a1c13d9741 --- /dev/null +++ b/docs/changelog/98001.yaml @@ -0,0 +1,5 @@ +pr: 98001 +summary: Avoid `transport_worker` thread in `TransportBroadcastAction` +area: Distributed +type: bug +issues: [] diff --git a/docs/changelog/98051.yaml b/docs/changelog/98051.yaml new file mode 100644 index 0000000000000..3c6d6fa974efd --- /dev/null +++ b/docs/changelog/98051.yaml @@ -0,0 +1,5 @@ +pr: 98051 +summary: Mark customer settings for serverless +area: Infra/Settings +type: enhancement +issues: [] diff --git a/docs/changelog/98067.yaml b/docs/changelog/98067.yaml new file mode 100644 index 0000000000000..18227a151cf51 --- /dev/null +++ b/docs/changelog/98067.yaml @@ -0,0 +1,6 @@ +pr: 98067 +summary: Avoid double get +area: Authentication +type: enhancement +issues: + - 97928 diff --git a/docs/changelog/98083.yaml b/docs/changelog/98083.yaml new file mode 100644 index 0000000000000..28f8e0698ff92 --- /dev/null +++ b/docs/changelog/98083.yaml @@ -0,0 +1,5 @@ +pr: 98083 +summary: Collect additional object store stats for S3 +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/98085.yaml b/docs/changelog/98085.yaml new file mode 100644 index 0000000000000..b079ea4b10c07 --- /dev/null +++ b/docs/changelog/98085.yaml @@ -0,0 +1,5 @@ +pr: 98085 +summary: Allow `ByteSizeDirectory` to expose their data set sizes +area: Store +type: enhancement +issues: [] diff --git a/docs/changelog/98091.yaml b/docs/changelog/98091.yaml new file mode 100644 index 0000000000000..2042babd3b090 --- /dev/null +++ b/docs/changelog/98091.yaml @@ -0,0 +1,6 @@ +pr: 98091 +summary: '`GlobalAggregator` should call rewrite() before `createWeight()`' +area: Aggregations +type: bug +issues: + - 98076 diff --git a/docs/changelog/98113.yaml b/docs/changelog/98113.yaml new file mode 100644 index 0000000000000..ceb081ea5b11d --- /dev/null +++ b/docs/changelog/98113.yaml @@ -0,0 +1,5 @@ +pr: 98113 +summary: Fix APM trace start time +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/98124.yaml b/docs/changelog/98124.yaml new file mode 100644 index 0000000000000..01d7e313fff90 --- /dev/null +++ b/docs/changelog/98124.yaml @@ -0,0 +1,6 @@ +pr: 98124 +summary: Fork CCS remote-cluster responses +area: Search +type: bug +issues: + - 97997 diff --git a/docs/changelog/98167.yaml b/docs/changelog/98167.yaml new file mode 100644 index 0000000000000..4622c9fc2b037 --- /dev/null +++ b/docs/changelog/98167.yaml @@ -0,0 +1,6 @@ +pr: 98167 +summary: Fix failure processing Question Answering model output where the input has been spanned over multiple sequences +area: Machine Learning +type: bug +issues: + - 97917 diff --git a/docs/changelog/98176.yaml b/docs/changelog/98176.yaml new file mode 100644 index 0000000000000..938166b5c2d7d --- /dev/null +++ b/docs/changelog/98176.yaml @@ -0,0 +1,5 @@ +pr: 98176 +summary: Enhance regex performance with duplicate wildcards +area: Infra/Core +type: enhancement +issues: [] diff --git a/docs/changelog/98184.yaml b/docs/changelog/98184.yaml new file mode 100644 index 0000000000000..6191d92cb5153 --- /dev/null +++ b/docs/changelog/98184.yaml @@ -0,0 +1,5 @@ +pr: 98184 +summary: Separating SLM from ILM +area: ILM+SLM +type: enhancement +issues: [] diff --git a/docs/changelog/98187.yaml b/docs/changelog/98187.yaml new file mode 100644 index 0000000000000..8163d3a215ad4 --- /dev/null +++ b/docs/changelog/98187.yaml @@ -0,0 +1,5 @@ +pr: 98187 +summary: Make `RestController` pluggable +area: Infra/REST API +type: enhancement +issues: [] diff --git a/docs/changelog/98204.yaml b/docs/changelog/98204.yaml new file mode 100644 index 0000000000000..417d4ec28d5a3 --- /dev/null +++ b/docs/changelog/98204.yaml @@ -0,0 +1,5 @@ +pr: 98204 +summary: Introduce executor for concurrent search +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/98206.yaml b/docs/changelog/98206.yaml new file mode 100644 index 0000000000000..452f645987e58 --- /dev/null +++ b/docs/changelog/98206.yaml @@ -0,0 +1,5 @@ +pr: 98206 +summary: Detect infinite loop in the WordPiece tokenizer +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/98209.yaml b/docs/changelog/98209.yaml new file mode 100644 index 0000000000000..7939137429d14 --- /dev/null +++ b/docs/changelog/98209.yaml @@ -0,0 +1,5 @@ +pr: 98209 +summary: Fork CCS search-shards handling +area: Search +type: bug +issues: [] diff --git a/docs/changelog/98246.yaml b/docs/changelog/98246.yaml new file mode 100644 index 0000000000000..595e97838cbc7 --- /dev/null +++ b/docs/changelog/98246.yaml @@ -0,0 +1,5 @@ +pr: 98246 +summary: Read operator privs enabled from Env settings +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/98259.yaml b/docs/changelog/98259.yaml new file mode 100644 index 0000000000000..359ec0c6c390c --- /dev/null +++ b/docs/changelog/98259.yaml @@ -0,0 +1,6 @@ +pr: 98259 +summary: Support getting active-only API keys via Get API keys API +area: Security +type: enhancement +issues: + - 97995 diff --git a/docs/changelog/98260.yaml b/docs/changelog/98260.yaml new file mode 100644 index 0000000000000..b33c5cb3ff707 --- /dev/null +++ b/docs/changelog/98260.yaml @@ -0,0 +1,5 @@ +pr: 98260 +summary: Avoid lifecycle NPE in the data stream lifecycle usage API +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/98295.yaml b/docs/changelog/98295.yaml new file mode 100644 index 0000000000000..43682555bf3f0 --- /dev/null +++ b/docs/changelog/98295.yaml @@ -0,0 +1,6 @@ +pr: 98295 +summary: Skip segment for `MatchNoDocsQuery` filters +area: Aggregations +type: bug +issues: + - 94637 diff --git a/docs/changelog/98296.yaml b/docs/changelog/98296.yaml new file mode 100644 index 0000000000000..94c5fb0e8ab4b --- /dev/null +++ b/docs/changelog/98296.yaml @@ -0,0 +1,5 @@ +pr: 98296 +summary: Add setting to scale the processor count used in the model assignment planner +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/98307.yaml b/docs/changelog/98307.yaml new file mode 100644 index 0000000000000..f04520c3ec024 --- /dev/null +++ b/docs/changelog/98307.yaml @@ -0,0 +1,13 @@ +pr: 98307 +summary: Beta release for API key based cross-cluster access +area: Security +type: feature +issues: [] +highlight: + title: API key based cross-cluster search and replication (Beta) + body: |- + This <> uses a cross-cluster API key to authenticate + and authorize cross-cluster operations to a remote cluster. It completely separates + network security controls between the local and remote cluster as well as offers + administrators of both the local and the remote cluster fine-grained access controls. + notable: false diff --git a/docs/changelog/98324.yaml b/docs/changelog/98324.yaml new file mode 100644 index 0000000000000..c91dc57bbb9fa --- /dev/null +++ b/docs/changelog/98324.yaml @@ -0,0 +1,5 @@ +pr: 98324 +summary: Avoid risk of OOM in datafeeds when memory is constrained +area: Machine Learning +type: bug +issues: [89769] diff --git a/docs/changelog/98354.yaml b/docs/changelog/98354.yaml new file mode 100644 index 0000000000000..f97e3f02b8541 --- /dev/null +++ b/docs/changelog/98354.yaml @@ -0,0 +1,6 @@ +pr: 98354 +summary: Improve reliability of elections with message delays +area: Cluster Coordination +type: bug +issues: + - 97909 diff --git a/docs/changelog/98367.yaml b/docs/changelog/98367.yaml new file mode 100644 index 0000000000000..2e8240b80745f --- /dev/null +++ b/docs/changelog/98367.yaml @@ -0,0 +1,6 @@ +pr: 98367 +summary: Check ILM status before reporting node migration STALLED +area: Infra/Node Lifecycle +type: enhancement +issues: + - 89486 diff --git a/docs/changelog/98386.yaml b/docs/changelog/98386.yaml new file mode 100644 index 0000000000000..14ead4ba721dd --- /dev/null +++ b/docs/changelog/98386.yaml @@ -0,0 +1,5 @@ +pr: 98386 +summary: Make `TransportAddVotingConfigExclusionsAction` retryable +area: Cluster Coordination +type: bug +issues: [] diff --git a/docs/changelog/98414.yaml b/docs/changelog/98414.yaml new file mode 100644 index 0000000000000..0a245893676e0 --- /dev/null +++ b/docs/changelog/98414.yaml @@ -0,0 +1,5 @@ +pr: 98414 +summary: '`TimeSeriesIndexSearcher` to offload to the provided executor' +area: TSDB +type: enhancement +issues: [] diff --git a/docs/changelog/98420.yaml b/docs/changelog/98420.yaml new file mode 100644 index 0000000000000..7f62e500a6fed --- /dev/null +++ b/docs/changelog/98420.yaml @@ -0,0 +1,5 @@ +pr: 98420 +summary: Fix election scheduling after discovery outage +area: Cluster Coordination +type: bug +issues: [] diff --git a/docs/changelog/98425.yaml b/docs/changelog/98425.yaml new file mode 100644 index 0000000000000..bdd34a3ec8de6 --- /dev/null +++ b/docs/changelog/98425.yaml @@ -0,0 +1,5 @@ +pr: 98425 +summary: Add support for concurrent collection when size is greater than zero +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/98455.yaml b/docs/changelog/98455.yaml new file mode 100644 index 0000000000000..542706173b378 --- /dev/null +++ b/docs/changelog/98455.yaml @@ -0,0 +1,5 @@ +pr: 98455 +summary: Add setting for search parallelism +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/98459.yaml b/docs/changelog/98459.yaml new file mode 100644 index 0000000000000..2bd3969324604 --- /dev/null +++ b/docs/changelog/98459.yaml @@ -0,0 +1,5 @@ +pr: 98459 +summary: Unwrap IOException in `ContextIndexSearcher` concurrent code-path +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/98508.yaml b/docs/changelog/98508.yaml new file mode 100644 index 0000000000000..60290bca6dbe9 --- /dev/null +++ b/docs/changelog/98508.yaml @@ -0,0 +1,5 @@ +pr: 98508 +summary: Adding special logic to the disk health check for search-only nodes +area: Health +type: enhancement +issues: [] diff --git a/docs/painless/painless-guide/painless-execute-script.asciidoc b/docs/painless/painless-guide/painless-execute-script.asciidoc index 681b852746281..1c7ec61ffa75b 100644 --- a/docs/painless/painless-guide/painless-execute-script.asciidoc +++ b/docs/painless/painless-guide/painless-execute-script.asciidoc @@ -118,6 +118,11 @@ Document that's temporarily indexed in-memory and accessible from the script. `index`:: (Required, string) Index containing a mapping that's compatible with the indexed document. +You may specify a remote index by prefixing the index with the remote cluster +alias. For example, `remote1:my_index` indicates that you want to execute +the painless script against the "my_index" index on the "remote1" cluster. This +request will be forwarded to the "remote1" cluster if you have +{ref}/remote-clusters-connect.html[configured a connection] to that remote cluster. ==== `params`:: (`Map`, read-only) diff --git a/docs/plugins/repository-hdfs.asciidoc b/docs/plugins/repository-hdfs.asciidoc index 94aecddbf0bc5..0bebc3b071436 100644 --- a/docs/plugins/repository-hdfs.asciidoc +++ b/docs/plugins/repository-hdfs.asciidoc @@ -77,6 +77,12 @@ include::repository-shared-settings.asciidoc[] the pattern with the hostname of the node at runtime (see link:repository-hdfs-security-runtime[Creating the Secure Repository]). +`replication_factor`:: + + The replication factor for all new HDFS files created by this repository. + Must be greater or equal to `dfs.replication.min` and less or equal to `dfs.replication.max` HDFS option. + Defaults to using HDFS cluster setting. + [[repository-hdfs-availability]] [discrete] ===== A note on HDFS availability diff --git a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc index 04afd004e67da..39a36cfdf9f26 100644 --- a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc @@ -288,7 +288,10 @@ image:images/spatial/geoshape_grid.png[] ==== Options [horizontal] -field:: Mandatory. The name of the field indexed with GeoPoints. +field:: Mandatory. Field containing indexed geo-point or geo-shape + values. Must be explicitly mapped as a <> + or a <> field. If the field contains an + array, `geohash_grid` aggregates all array values. precision:: Optional. The string length of the geohashes used to define cells/buckets in the results. Defaults to 5. diff --git a/docs/reference/aggregations/bucket/range-aggregation.asciidoc b/docs/reference/aggregations/bucket/range-aggregation.asciidoc index e3c955e10e093..07095882c2f04 100644 --- a/docs/reference/aggregations/bucket/range-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/range-aggregation.asciidoc @@ -338,7 +338,25 @@ with latency metrics (in milliseconds) for different networks: [source,console] ---- -PUT metrics_index/_doc/1 +PUT metrics_index +{ + "mappings": { + "properties": { + "network": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "latency_histo": { + "type": "histogram" + } + } + } +} + +PUT metrics_index/_doc/1?refresh { "network.name" : "net-1", "latency_histo" : { @@ -347,7 +365,7 @@ PUT metrics_index/_doc/1 } } -PUT metrics_index/_doc/2 +PUT metrics_index/_doc/2?refresh { "network.name" : "net-2", "latency_histo" : { @@ -385,24 +403,24 @@ return the following output: "buckets": [ { "key": "*-2.0", - "to": 2, + "to": 2.0, "doc_count": 11 }, { "key": "2.0-3.0", - "from": 2, - "to": 3, + "from": 2.0, + "to": 3.0, "doc_count": 0 }, { "key": "3.0-10.0", - "from": 3, - "to": 10, + "from": 3.0, + "to": 10.0, "doc_count": 55 }, { "key": "10.0-*", - "from": 10, + "from": 10.0, "doc_count": 31 } ] @@ -410,7 +428,7 @@ return the following output: } } ---- -// TESTRESPONSE[skip:test not setup] +// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] [IMPORTANT] ======== diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc index 48c45c574ae99..e384fb6d42c4c 100644 --- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc @@ -77,6 +77,8 @@ https://static.googleusercontent.com/media/research.google.com/fr//pubs/archive/ algorithm, which counts based on the hashes of the values with some interesting properties: +// tag::explanation[] + * configurable precision, which decides on how to trade memory for accuracy, * excellent accuracy on low-cardinality sets, * fixed memory usage: no matter if there are tens or billions of unique values, @@ -157,9 +159,11 @@ accuracy. Also note that even with a threshold as low as 100, the error remains very low (1-6% as seen in the above graph) even when counting millions of items. The HyperLogLog++ algorithm depends on the leading zeros of hashed -values, the exact distributions of hashes in a dataset can affect the +values, the exact distributions of hashes in a dataset can affect the accuracy of the cardinality. +// end::explanation[] + ==== Pre-computed hashes On string fields that have a high cardinality, it might be faster to store the @@ -237,9 +241,10 @@ POST /sales/_search?size=0 -------------------------------------------------- // TEST[setup:sales] <1> Documents without a value in the `tag` field will fall into the same bucket as documents that have the value `N/A`. -==== Execution Hint -There are different mechanisms by which cardinality aggregations can be executed: +==== Execution hint + +You can run cardinality aggregations using different mechanisms: - by using field values directly (`direct`) - by using global ordinals of the field and resolving those values after @@ -248,13 +253,14 @@ There are different mechanisms by which cardinality aggregations can be executed segment (`segment_ordinals`) Additionally, there are two "heuristic based" modes. These modes will cause -Elasticsearch to use some data about the state of the index to choose an -appropriate execution method. The two heuristics are: - - `save_time_heuristic` - this is the default in Elasticsearch 8.4 and later. - - `save_memory_heuristic` - this was the default in Elasticsearch 8.3 and +{es} to use some data about the state of the index to choose an +appropriate execution method. The two heuristics are: + + - `save_time_heuristic` - this is the default in {es} 8.4 and later. + - `save_memory_heuristic` - this was the default in {es} 8.3 and earlier -When not specified, Elasticsearch will apply a heuristic to chose the -appropriate mode. Also note that some data (i.e. non-ordinal fields), `direct` +When not specified, {es} will apply a heuristic to choose the +appropriate mode. Also note that for some data (non-ordinal fields), `direct` is the only option, and the hint will be ignored in these cases. Generally speaking, it should not be necessary to set this value. diff --git a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc index 4ed684aa090d3..aabe8d172e4a0 100644 --- a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc @@ -1,8 +1,8 @@ [role="xpack"] [[search-aggregations-metrics-geo-line]] -=== Geo-Line Aggregation +=== Geo-line aggregation ++++ -Geo-Line +Geo-line ++++ The `geo_line` aggregation aggregates all `geo_point` values within a bucket into a `LineString` ordered @@ -77,13 +77,12 @@ Which returns: The resulting https://tools.ietf.org/html/rfc7946#section-3.2[GeoJSON Feature] contains both a `LineString` geometry for the path generated by the aggregation, as well as a map of `properties`. The property `complete` informs of whether all documents matched were used to generate the geometry. -The `size` option described below can be used to limit the number of documents included in the aggregation, +The <> can be used to limit the number of documents included in the aggregation, leading to results with `complete: false`. -Exactly which documents are dropped from results depends on whether the aggregation is based -on `time_series` or not, and this is discussed in -<>. +Exactly which documents are dropped from results <>. -The above result could be displayed in a map user interface: +This result could be displayed in a map user interface: image:images/spatial/geo_line.png[Kibana map with museum tour of Amsterdam] @@ -132,18 +131,19 @@ feature properties. The line is sorted in ascending order by the sort key when set to "ASC", and in descending with "DESC". +[[search-aggregations-metrics-geo-line-size]] `size`:: (Optional, integer, default: `10000`) The maximum length of the line represented in the aggregation. Valid sizes are between one and 10000. Within <> the aggregation uses line simplification to constrain the size, otherwise it uses truncation. -See <> +Refer to <> for a discussion on the subtleties involved. [[search-aggregations-metrics-geo-line-grouping]] ==== Grouping -The simple example above will produce a single track for all the data selected by the query. However, it is far more +This simple example produces a single track for all the data selected by the query. However, it is far more common to need to group the data into multiple tracks. For example, grouping flight transponder measurements by flight call-sign before sorting each flight by timestamp and producing a separate track for each. @@ -210,7 +210,7 @@ POST /tour/_bulk?refresh [[search-aggregations-metrics-geo-line-grouping-terms]] ==== Grouping with terms -Using the above data, for a non-time-series use case, the grouping can be done using a +Using this data, for a non-time-series use case, the grouping can be done using a <> based on city name. This would work whether or not we had defined the `tour` index as a time series index. @@ -294,17 +294,19 @@ Which returns: ---- // TESTRESPONSE -The above results contain an array of buckets, where each bucket is a JSON object with the `key` showing the name +These results contain an array of buckets, where each bucket is a JSON object with the `key` showing the name of the `city` field, and an inner aggregation result called `museum_tour` containing a https://tools.ietf.org/html/rfc7946#section-3.2[GeoJSON Feature] describing the actual route between the various attractions in that city. Each result also includes a `properties` object with a `complete` value which will be `false` if the geometry was truncated to the limits specified in the `size` parameter. -Note that when we use `time_series` in the example below, we will get the same results structured a little differently. +Note that when we use `time_series` in the next example, we will get the same results structured a little differently. [[search-aggregations-metrics-geo-line-grouping-time-series]] ==== Grouping with time-series +preview::[] + Using the same data as before, we can also perform the grouping with a <>. This will group by TSID, which is defined as the combinations of all fields with `time_series_dimension: true`, @@ -337,7 +339,7 @@ NOTE: The `geo_line` aggregation no longer requires the `sort` field when nested This is because the sort field is set to `@timestamp`, which all time-series indexes are pre-sorted by. If you do set this parameter, and set it to something other than `@timestamp` you will get an error. -The above query will result in: +This query will result in: [source,js] ---- @@ -400,7 +402,7 @@ The above query will result in: ---- // TESTRESPONSE -The above results are essentially the same as with the previous `terms` aggregation example, but structured differently. +These results are essentially the same as with the previous `terms` aggregation example, but structured differently. Here we see the buckets returned as a map, where the key is an internal description of the TSID. This TSID is unique for each unique combination of fields with `time_series_dimension: true`. Each bucket contains a `key` field which is also a map of all dimension values for the TSID, in this case only the city @@ -414,7 +416,7 @@ was simplified to the limits specified in the `size` parameter. [[search-aggregations-metrics-geo-line-grouping-time-series-advantages]] ==== Why group with time-series? -When reviewing the above examples, you might think that there is little difference between using +When reviewing these examples, you might think that there is little difference between using <> or <> to group the geo-lines. However, there are some important differences in behaviour between the two cases. diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc index 20c0266b8b539..c4397748debfd 100644 --- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc @@ -234,6 +234,7 @@ GET latency/_search [[search-aggregations-metrics-percentile-aggregation-approximation]] ==== Percentiles are (usually) approximate +// tag::approximate[] There are many different algorithms to calculate percentiles. The naive implementation simply stores all the values in a sorted array. To find the 50th percentile, you simply find the value that is at `my_array[count(my_array) * 0.5]`. @@ -268,6 +269,8 @@ for large number of values is that the law of large numbers makes the distributi values more and more uniform and the t-digest tree can do a better job at summarizing it. It would not be the case on more skewed distributions. +// end::approximate[] + [WARNING] ==== Percentile aggregations are also @@ -352,7 +355,7 @@ GET latency/_search This option can lead to improved accuracy (relative error close to 0.01% for millions of samples in some cases) but then percentile queries take 2x-10x longer to complete. -==== HDR Histogram +==== HDR histogram https://github.com/HdrHistogram/HdrHistogram[HDR Histogram] (High Dynamic Range Histogram) is an alternative implementation that can be useful when calculating percentiles for latency measurements as it can be faster than the t-digest implementation @@ -361,7 +364,7 @@ as a number of significant digits). This means that if data is recorded with val (3,600,000,000 microseconds) in a histogram set to 3 significant digits, it will maintain a value resolution of 1 microsecond for values up to 1 millisecond and 3.6 seconds (or better) for the maximum tracked value (1 hour). -The HDR Histogram can be used by specifying the `method` parameter in the request: +The HDR Histogram can be used by specifying the `hdr` parameter in the request: [source,console] -------------------------------------------------- diff --git a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc index 5f822428e66ac..a9bb5b72ea7c3 100644 --- a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc @@ -4,7 +4,7 @@ Synonym graph ++++ -The `synonym_graph` token filter allows to easily handle synonyms, +The `synonym_graph` token filter allows to easily handle <>, including multi-word synonyms correctly during the analysis process. In order to properly handle multi-word synonyms this token filter @@ -19,37 +19,58 @@ only. If you want to apply synonyms during indexing please use the standard <>. =============================== -Synonyms are configured using a configuration file. -Here is an example: +[discrete] +[[analysis-synonym-graph-define-synonyms]] +==== Define synonyms sets -[source,console] --------------------------------------------------- -PUT /test_index -{ - "settings": { - "index": { - "analysis": { - "analyzer": { - "search_synonyms": { - "tokenizer": "whitespace", - "filter": [ "graph_synonyms" ] - } - }, - "filter": { - "graph_synonyms": { - "type": "synonym_graph", - "synonyms_path": "analysis/synonym.txt" - } - } - } +include::synonyms-format.asciidoc[] + +[discrete] +[[analysis-synonym-graph-configure-sets]] +==== Configure synonyms sets + +Synonyms can be configured using the <>, a <>, or directly <> in the token filter configuration. +See <> for more details on each option. + +Use `synonyms_set` configuration option to provide a synonym set created via Synonyms Management APIs: + +[source,JSON] +---- + "filter": { + "synonyms_filter": { + "type": "synonym", + "synonyms_set": "my-synonym-set", + "updateable": true } } -} --------------------------------------------------- +---- + +Use `synonyms_path` to provide a synonym file : + +[source,JSON] +---- + "filter": { + "synonyms_filter": { + "type": "synonym", + "synonyms_path": "analysis/synonym-set.txt" + } + } +---- -The above configures a `search_synonyms` filter, with a path of -`analysis/synonym.txt` (relative to the `config` location). The -`search_synonyms` analyzer is then configured with the filter. +The above configures a `synonym` filter, with a path of +`analysis/synonym-set.txt` (relative to the `config` location). + +Use `synonyms` to define inline synonyms: + +[source,JSON] +---- + "filter": { + "synonyms_filter": { + "type": "synonym", + "synonyms": ["pc => personal computer", "computer, pc, laptop"] + } + } +---- Additional settings are: @@ -99,103 +120,45 @@ stop word. [discrete] [[synonym-graph-tokenizer-ignore_case-deprecated]] -==== `tokenizer` and `ignore_case` are deprecated +===== `tokenizer` and `ignore_case` are deprecated The `tokenizer` parameter controls the tokenizers that will be used to tokenize the synonym, this parameter is for backwards compatibility for indices that created before 6.0. The `ignore_case` parameter works with `tokenizer` parameter only. -Two synonym formats are supported: Solr, WordNet. - [discrete] -==== Solr synonyms - -The following is a sample format of the file: - -[source,synonyms] --------------------------------------------------- -include::{es-test-dir}/cluster/config/analysis/synonym.txt[] --------------------------------------------------- - -You can also define synonyms for the filter directly in the -configuration file (note use of `synonyms` instead of `synonyms_path`): - -[source,console] --------------------------------------------------- -PUT /test_index -{ - "settings": { - "index": { - "analysis": { - "filter": { - "synonym": { - "type": "synonym_graph", - "synonyms": [ - "lol, laughing out loud", - "universe, cosmos" - ] - } +[[analysis-synonym-graph-analizers-configure]] +==== Configure analyzers with synonym graph token filters + +To apply synonyms, you will need to include a synonym graph token filter into an analyzer: + +[source,JSON] +---- + "analyzer": { + "my_analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "synonym_graph"] } } - } - } -} --------------------------------------------------- - -However, it is recommended to define large synonyms set in a file using -`synonyms_path`, because specifying them inline increases cluster size unnecessarily. +---- [discrete] -==== WordNet synonyms +[[analysis-synonym-graph-token-order]] +===== Token filters ordering -Synonyms based on https://wordnet.princeton.edu/[WordNet] format can be -declared using `format`: +Order is important for your token filters. +Text will be processed first through filters preceding the synonym filter before being processed by the synonym filter. -[source,console] --------------------------------------------------- -PUT /test_index -{ - "settings": { - "index": { - "analysis": { - "filter": { - "synonym": { - "type": "synonym_graph", - "format": "wordnet", - "synonyms": [ - "s(100000001,1,'abstain',v,1,0).", - "s(100000001,2,'refrain',v,1,0).", - "s(100000001,3,'desist',v,1,0)." - ] - } - } - } - } - } -} --------------------------------------------------- +In the above example, text will be lowercased by the `lowercase` filter before being processed by the `synonyms_filter`. +This means that all the synonyms defined there needs to be in lowercase, or they won't be found by the synonyms filter. -Using `synonyms_path` to define WordNet synonyms in a file is supported -as well. +The synonym rules should not contain words that are removed by a filter that appears later in the chain (like a `stop` filter). +Removing a term from a synonym rule means there will be no matching for it at query time. -[discrete] -==== Parsing synonym files - -Elasticsearch will use the token filters preceding the synonym filter -in a tokenizer chain to parse the entries in a synonym file. So, for example, if a -synonym filter is placed after a stemmer, then the stemmer will also be applied -to the synonym entries. Because entries in the synonym map cannot have stacked -positions, some token filters may cause issues here. Token filters that produce -multiple versions of a token may choose which version of the token to emit when -parsing synonyms, e.g. `asciifolding` will only produce the folded version of the -token. Others, e.g. `multiplexer`, `word_delimiter_graph` or `ngram` will throw an -error. - -If you need to build analyzers that include both multi-token filters and synonym -filters, consider using the <> filter, -with the multi-token filters in one branch and the synonym filter in the other. - -WARNING: The synonym rules should not contain words that are removed by -a filter that appears after in the chain (a `stop` filter for instance). -Removing a term from a synonym rule breaks the matching at query time. +Because entries in the synonym map cannot have stacked positions, some token filters may cause issues here. +Token filters that produce multiple versions of a token may choose which version of the token to emit when parsing synonyms. +For example, `asciifolding` will only produce the folded version of the token. +Others, like `multiplexer`, `word_delimiter_graph` or `ngram` will throw an error. +If you need to build analyzers that include both multi-token filters and synonym filters, consider using the <> filter, with the multi-token filters in one branch and the synonym filter in the other. diff --git a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc index ea78e31b3f564..91c0a49f41066 100644 --- a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc @@ -4,41 +4,61 @@ Synonym ++++ -The `synonym` token filter allows to easily handle synonyms during the -analysis process. Synonyms are configured using a configuration file. -Here is an example: +The `synonym` token filter allows to easily handle <> during the +analysis process. -[source,console] --------------------------------------------------- -PUT /test_index -{ - "settings": { - "index": { - "analysis": { - "analyzer": { - "synonym": { - "tokenizer": "whitespace", - "filter": [ "synonym" ] - } - }, - "filter": { - "synonym": { - "type": "synonym", - "synonyms_path": "analysis/synonym.txt" - } - } - } +[discrete] +[[analysis-synonym-define-synonyms]] +==== Define synonyms sets + +include::synonyms-format.asciidoc[] + +[discrete] +[[analysis-synonym-configure-sets]] +==== Configure synonyms sets + +Synonyms can be configured using the <>, a <>, or directly <> in the token filter configuration. +See <> for more details on each option. + +Use `synonyms_set` configuration option to provide a synonym set created via Synonyms Management APIs: + +[source,JSON] +---- + "filter": { + "synonyms_filter": { + "type": "synonym", + "synonyms_set": "my-synonym-set", + "updateable": true } } -} --------------------------------------------------- +---- + +Use `synonyms_path` to provide a synonym file : + +[source,JSON] +---- + "filter": { + "synonyms_filter": { + "type": "synonym", + "synonyms_path": "analysis/synonym-set.txt" + } + } +---- The above configures a `synonym` filter, with a path of -`analysis/synonym.txt` (relative to the `config` location). The -`synonym` analyzer is then configured with the filter. +`analysis/synonym-set.txt` (relative to the `config` location). -This filter tokenizes synonyms with whatever tokenizer and token filters -appear before it in the chain. +Use `synonyms` to define inline synonyms: + +[source,JSON] +---- + "filter": { + "synonyms_filter": { + "type": "synonym", + "synonyms": ["pc => personal computer", "computer, pc, laptop"] + } + } +---- Additional settings are: @@ -90,98 +110,45 @@ stop word. [discrete] [[synonym-tokenizer-ignore_case-deprecated]] -==== `tokenizer` and `ignore_case` are deprecated +===== `tokenizer` and `ignore_case` are deprecated The `tokenizer` parameter controls the tokenizers that will be used to tokenize the synonym, this parameter is for backwards compatibility for indices that created before 6.0. The `ignore_case` parameter works with `tokenizer` parameter only. -Two synonym formats are supported: Solr, WordNet. - [discrete] -==== Solr synonyms - -The following is a sample format of the file: - -[source,synonyms] --------------------------------------------------- -include::{es-test-dir}/cluster/config/analysis/synonym.txt[] --------------------------------------------------- - -You can also define synonyms for the filter directly in the -configuration file (note use of `synonyms` instead of `synonyms_path`): - -[source,console] --------------------------------------------------- -PUT /test_index -{ - "settings": { - "index": { - "analysis": { - "filter": { - "synonym": { - "type": "synonym", - "synonyms": [ - "i-pod, i pod => ipod", - "universe, cosmos" - ] - } +[[analysis-synonym-analizers-configure]] +==== Configure analyzers with synonym token filters + +To apply synonyms, you will need to include a synonym token filters into an analyzer: + +[source,JSON] +---- + "analyzer": { + "my_analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "synonym"] } } - } - } -} --------------------------------------------------- - -However, it is recommended to define large synonyms set in a file using -`synonyms_path`, because specifying them inline increases cluster size unnecessarily. +---- [discrete] -==== WordNet synonyms +[[analysis-synonym-token-order]] +===== Token filters ordering -Synonyms based on https://wordnet.princeton.edu/[WordNet] format can be -declared using `format`: +Order is important for your token filters. +Text will be processed first through filters preceding the synonym filter before being processed by the synonym filter. -[source,console] --------------------------------------------------- -PUT /test_index -{ - "settings": { - "index": { - "analysis": { - "filter": { - "synonym": { - "type": "synonym", - "format": "wordnet", - "synonyms": [ - "s(100000001,1,'abstain',v,1,0).", - "s(100000001,2,'refrain',v,1,0).", - "s(100000001,3,'desist',v,1,0)." - ] - } - } - } - } - } -} --------------------------------------------------- +In the above example, text will be lowercased by the `lowercase` filter before being processed by the `synonyms_filter`. +This means that all the synonyms defined there needs to be in lowercase, or they won't be found by the synonyms filter. -Using `synonyms_path` to define WordNet synonyms in a file is supported -as well. +The synonym rules should not contain words that are removed by a filter that appears later in the chain (like a `stop` filter). +Removing a term from a synonym rule means there will be no matching for it at query time. -[discrete] -=== Parsing synonym files - -Elasticsearch will use the token filters preceding the synonym filter -in a tokenizer chain to parse the entries in a synonym file. So, for example, if a -synonym filter is placed after a stemmer, then the stemmer will also be applied -to the synonym entries. Because entries in the synonym map cannot have stacked -positions, some token filters may cause issues here. Token filters that produce -multiple versions of a token may choose which version of the token to emit when -parsing synonyms, e.g. `asciifolding` will only produce the folded version of the -token. Others, e.g. `multiplexer`, `word_delimiter_graph` or `ngram` will throw an -error. - -If you need to build analyzers that include both multi-token filters and synonym -filters, consider using the <> filter, -with the multi-token filters in one branch and the synonym filter in the other. +Because entries in the synonym map cannot have stacked positions, some token filters may cause issues here. +Token filters that produce multiple versions of a token may choose which version of the token to emit when parsing synonyms. +For example, `asciifolding` will only produce the folded version of the token. +Others, like `multiplexer`, `word_delimiter_graph` or `ngram` will throw an error. + +If you need to build analyzers that include both multi-token filters and synonym filters, consider using the <> filter, with the multi-token filters in one branch and the synonym filter in the other. diff --git a/docs/reference/analysis/tokenfilters/synonyms-format.asciidoc b/docs/reference/analysis/tokenfilters/synonyms-format.asciidoc new file mode 100644 index 0000000000000..63dd72dade8d0 --- /dev/null +++ b/docs/reference/analysis/tokenfilters/synonyms-format.asciidoc @@ -0,0 +1,44 @@ +Synonyms in a synonyms set are defined using *synonym rules*. +Each synonym rule contains words that are synonyms. + +You can use two formats to define synonym rules: Solr and WordNet. + +[discrete] +===== Solr format + +This format uses two different definitions: + +* Equivalent synonyms: Define groups of words that are equivalent. Words are separated by commas. Example: ++ +[source,synonyms] +---- +ipod, i-pod, i pod +computer, pc, laptop +---- +* Explicit mappings: Matches a group of words to other words. Words on the left hand side of the rule definition are expanded into all the possibilities described on the right hand side. Example: ++ +[source,synonyms] +---- +personal computer => pc +sea biscuit, sea biscit => seabiscuit +---- + +[discrete] +===== WordNet format + +https://wordnet.princeton.edu/[WordNet] defines synonyms sets spanning multiple lines. Each line contains the following information: + +* Synonyms set numeric identifier +* Ordinal of the synonym in the synonyms set +* Synonym word +* Word type identifier: Noun (n), verb (v), adjective (a) or adverb (b). +* Depth of the word in the synonym net + +The following example defines a synonym set for the words "come", "advance" and "approach": + +[source,synonyms] +---- +s(100000002,1,'come',v,1,0). +s(100000002,2,'advance',v,1,0). +s(100000002,3,'approach',v,1,0)."""; +---- diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 0e231056e8daf..64cb499a9cd4e 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -195,6 +195,13 @@ request that targets an excluded alias. For example, if `test3` is an index alias, the pattern `test*,-test3` still targets the indices for `test3`. To avoid this, exclude the concrete indices for the alias instead. +You can also exclude clusters from a list of clusters to search using the `-` character: +`remote*:*,-remote1:*,-remote4:*` will search all clusters with an alias that starts +with "remote" except for "remote1" and "remote4". Note that to exclude a cluster +with this notation you must exclude all of its indexes. Excluding a subset of indexes +on a remote cluster is currently not supported. For example, this will throw an exception: +`remote*:*,-remote1:logs*`. + Multi-target APIs that can target indices support the following query string parameters: diff --git a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc index 38610efea15f0..9b15bcca3fc85 100644 --- a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc @@ -8,6 +8,15 @@ beta::[] Delete Analytics Collection ++++ +//// +[source,console] +---- +PUT _application/analytics/my_analytics_collection +---- +// TESTSETUP + +//// + Removes an Analytics Collection and its associated data stream. [[delete-analytics-collection-request]] @@ -44,4 +53,3 @@ The following example deletes the Analytics Collection named `my_analytics_colle ---- DELETE _application/analytics/my_analytics_collection/ ---- -// TEST[skip:TBD] diff --git a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc index fddc8c7a9a9fc..8d2491ff8a6ee 100644 --- a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc @@ -8,6 +8,22 @@ beta::[] List Analytics Collections ++++ +//// +[source,console] +---- +PUT _application/analytics/my_analytics_collection +PUT _application/analytics/my_analytics_collection2 +---- +// TESTSETUP + +[source,console] +---- +DELETE _application/analytics/my_analytics_collection +DELETE _application/analytics/my_analytics_collection2 +---- +// TEARDOWN +//// + Returns information about Analytics Collections. [[list-analytics-collection-request]] @@ -44,7 +60,6 @@ The following example lists all configured Analytics Collections: ---- GET _application/analytics/ ---- -// TEST[skip:TBD] A sample response: @@ -70,7 +85,6 @@ The following example returns the Analytics Collection that matches `my_analytic ---- GET _application/analytics/my_analytics_collection ---- -// TEST[skip:TBD] A sample response: @@ -91,7 +105,6 @@ The following example returns all Analytics Collections prefixed with `my`: ---- GET _application/analytics/my* ---- -// TEST[skip:TBD] A sample response: @@ -110,4 +123,3 @@ A sample response: } } ---- -// TEST[skip:TBD] diff --git a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc index ba2c88de52e8a..d20134644dde4 100644 --- a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc +++ b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc @@ -8,6 +8,20 @@ beta::[] Post Analytics Collection Event ++++ +//// +[source,console] +---- +PUT _application/analytics/my_analytics_collection +---- +// TESTSETUP + +[source,console] +---- +DELETE _application/analytics/my_analytics_collection +---- +// TEARDOWN +//// + Post an event to an Analytics Collection. [[post-analytics-collection-event-request]] @@ -22,7 +36,12 @@ Post an event to an Analytics Collection. (Required, string) Analytics collection name you want to ingest event in. ``:: -(Required, string) Analytics event type. Can be one of `page_view`, `search`, `click` +(Required, string) Analytics event type. Can be one of `page_view`, `search`, `search_click`. + +[[post-analytics-collection-event-request-body]] +==== {api-request-body-title} +Full request body parameters can be found in {enterprise-search-ref}/analytics-events-reference.html[Enterprise Analytics Events^]. + [[post-analytics-collection-event-prereqs]] ==== {api-prereq-title} @@ -44,11 +63,11 @@ Occurs either when the event type is unknown or when event payload contains inva [[post-analytics-collection-event-example]] ==== {api-examples-title} -The following example send a `page_view` event to an Analytics Collection called `my_analytics_collection`: +The following example send a `search_click` event to an Analytics Collection called `my_analytics_collection`: [source,console] ---- -POST _application/analytics/my_analytics_collection/event/page_view +POST _application/analytics/my_analytics_collection/event/search_click { "session": { "id": "1797ca95-91c9-4e2e-b1bd-9c38e6f386a9" @@ -56,9 +75,27 @@ POST _application/analytics/my_analytics_collection/event/page_view "user": { "id": "5f26f01a-bbee-4202-9298-81261067abbd" }, - "page": { - "url": "https://www.elastic.co/" + "search":{ + "query": "search term", + "results": { + "items": [ + { + "document": { + "id": "123", + "index": "products" + } + } + ], + "total_results": 10 + }, + "sort": { + "name": "relevance" + }, + "search_application": "website" + }, + "document":{ + "id": "123", + "index": "products" } } ---- -// TEST[skip:TBD] diff --git a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc index c7c32b0d800c8..48273fb3906c4 100644 --- a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc @@ -8,6 +8,14 @@ beta::[] Put Analytics Collection ++++ +//// +[source,console] +---- +DELETE _application/analytics/my_analytics_collection +---- +// TEARDOWN +//// + Creates an Analytics Collection. [[put-analytics-collection-request]] @@ -40,6 +48,4 @@ The following example creates a new Analytics Collection called `my_analytics_co [source,console] ---- PUT _application/analytics/my_analytics_collection - ---- -// TEST[skip:TBD] diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index 79202733bfcfd..b670ee26a20a9 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -149,6 +149,9 @@ Node uptime, such as `17.3m`. `completion.size`, `cs`, `completionSize`:: Size of completion, such as `0b`. +`dense_vector.value_count`, `dvc`, `denseVectorCount`:: +Number of indexed dense vector. + `fielddata.memory_size`, `fm`, `fielddataMemory`:: Used fielddata cache memory, such as `0b`. @@ -315,6 +318,9 @@ Time spent in suggest, such as `0`. `suggest.total`, `suto`, `suggestTotal`:: Number of suggest operations, such as `0`. +`shard_stats.total_count`, `sstc`, `shardStatsTotalCount`:: +Number of shards assigned. + `mappings.total_count`, `mtc`, `mappingsTotalCount`:: Number of mappings, including <> and <> fields. diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc index 0cfe0c5cec41f..aee602e12bb3b 100644 --- a/docs/reference/cat/shards.asciidoc +++ b/docs/reference/cat/shards.asciidoc @@ -92,6 +92,9 @@ Valid columns are: `completion.size`, `cs`, `completionSize`:: Size of completion, such as `0b`. +`dense_vector.value_count`, `dvc`, `denseVectorCount`:: +Number of indexed dense vector. + `fielddata.memory_size`, `fm`, `fielddataMemory`:: Used fielddata cache memory, such as `0b`. diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index e7cad42014072..caab04cb564ee 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -2,6 +2,7 @@ (object) Settings to override from the leader index. Note that certain settings can not be overrode (e.g., `index.number_of_shards`). +// tag::ccr-resume-follow-request-body[] `max_read_request_operation_count`:: (integer) The maximum number of operations to pull per read from the remote cluster. @@ -101,3 +102,4 @@ values for the above described index follow request parameters: } -------------------------------------------------- +// end::ccr-resume-follow-request-body[] \ No newline at end of file diff --git a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc index ee1030b4f3301..3d72fa0bff3e8 100644 --- a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc @@ -68,7 +68,7 @@ returns, the follower index will resume fetching operations from the leader inde [[ccr-post-resume-follow-request-body]] ==== {api-request-body-title} -include::../follow-request-body.asciidoc[] +include::../follow-request-body.asciidoc[tag=ccr-resume-follow-request-body] [[ccr-post-resume-follow-examples]] ==== {api-examples-title} diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc index 4c6d0585986a8..3b079a5a4764c 100644 --- a/docs/reference/cluster/allocation-explain.asciidoc +++ b/docs/reference/cluster/allocation-explain.asciidoc @@ -60,8 +60,9 @@ you might expect otherwise. ==== {api-request-body-title} `current_node`:: - (Optional, string) Specifies the node ID or the name of the node to only - explain a shard that is currently located on the specified node. + (Optional, string) Specifies the node ID or the name of the node currently + holding the shard to explain. To explain an unassigned shard, omit this + parameter. `index`:: (Optional, string) Specifies the name of the index that you would like an diff --git a/docs/reference/cluster/nodes-hot-threads.asciidoc b/docs/reference/cluster/nodes-hot-threads.asciidoc index e8bdef825999a..552d41629e1e2 100644 --- a/docs/reference/cluster/nodes-hot-threads.asciidoc +++ b/docs/reference/cluster/nodes-hot-threads.asciidoc @@ -6,7 +6,6 @@ Returns the hot threads on each selected node in the cluster. - [[cluster-nodes-hot-threads-api-request]] ==== {api-request-title} @@ -53,7 +52,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] `threads`:: (Optional, integer) Specifies the number of hot threads to provide - information for. Defaults to `3`. + information for. Defaults to `3`. If you are using this API for + troubleshooting, set this parameter to a large number (e.g. + `9999`) to get information about all the threads in the system. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index c871d069287a6..eacbabb99f045 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -121,6 +121,7 @@ using metrics. * `store` * `translog` * `warmer` + * `dense_vector` -- include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] @@ -682,12 +683,11 @@ Number of query cache misses. `cache_size`:: (integer) -Size, in bytes, of the query cache. +Current number of cached queries. `cache_count`:: (integer) -Count of queries -in the query cache. +Total number of all queries that have been cached. `evictions`:: (integer) @@ -1004,6 +1004,18 @@ Estimated heap overhead of mappings on this node, which allows for 1kiB of heap Estimated heap overhead, in bytes, of mappings on this node, which allows for 1kiB of heap for every mapped field. ======= +`dense_vector`:: +(object) +Contains statistics about dense_vector across all shards assigned to the node. ++ +.Properties of `dense_vector` +[%collapsible%open] +======= +`value_count`:: +(integer) +Total number of dense vector indexed across all shards assigned +to the node. +======= ====== [[cluster-nodes-stats-api-response-body-os]] diff --git a/docs/reference/cluster/remote-info.asciidoc b/docs/reference/cluster/remote-info.asciidoc index 60e6882fb922d..0c1cb1f3adbb7 100644 --- a/docs/reference/cluster/remote-info.asciidoc +++ b/docs/reference/cluster/remote-info.asciidoc @@ -67,3 +67,9 @@ Refer to <>. `max_proxy_socket_connections`:: The maximum number of socket connections to the remote cluster when proxy mode is configured. + +`cluster_credentials`:: +// TODO: fix the link to new page of API key based remote clusters +beta:[] +This field presents and has value of `::es_redacted::` only when the remote cluster +is configured with the API key based model. Otherwise, the field is not present. diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 56e65c567de62..dad6b46686ea2 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -840,6 +840,70 @@ Occurrences of the built-in analyzer in selected nodes. `index_count`:: (integer) Number of indices using the built-in analyzer in selected nodes. +====== + +`synonyms`:: +(object) +Contains statistics about synonyms defined in <> and <> token filters configuration. + ++ +.Properties of `synonyms` objects +[%collapsible%open] +====== +`inline`:: +(object) +Inline synonyms defined using `synonyms` configuration in synonym or synonym graph token filters. + ++ +.Properties of `inline` objects +[%collapsible%open] +======= + +`count`:: +(integer) +Occurrences of inline synonyms configuration in selected nodes. +Each inline synonyms configuration will be counted separately, regardless of the synonyms defined. +Two synonyms configurations with the same synonyms will count as separate ocurrences. + +`index_count`:: +(integer) +Number of indices that use inline synonyms configuration for synonyms token filters. +======= + +`paths`:: +(object) +Contains statistics about synonym files defined as `synonyms_path` in <> and <> token filters configuration. + ++ +.Properties of `paths` objects +[%collapsible%open] +======= +`count`:: +(integer) +Occurrences of unique synonym paths in selected nodes. + +`index_count`:: +(integer) +Number of indices that use `synonyms_path` configuration for synonyms token filters. +======= + +`sets`:: +(object) +Contains statistics about synonyms sets configured as `synonyms_set` in <> and <> token filters configuration. + ++ +.Properties of `sets` objects +[%collapsible%open] +======= +`count`:: +(integer) +Occurrences of unique synonyms sets in selected nodes. + +`index_count`:: +(integer) +Number of indices that use `synonyms_set` configuration for synonyms token filters. +======= + ====== ===== @@ -873,6 +937,18 @@ rather than by the number of times it's been used in that individual search requ Search sections used in selected nodes. For each section, name and number of times it's been used is reported. +===== + +`dense_vector`:: +(object) +Contains statistics about indexed dense vector used in selected nodes. ++ +.Properties of `dense_vector` +[%collapsible%open] +===== +`value_count`:: +(integer) +Total number of dense vector indexed in selected nodes. ===== ==== @@ -1595,7 +1671,8 @@ The API returns the following response: "built_in_char_filters": [], "built_in_tokenizers": [], "built_in_filters": [], - "built_in_analyzers": [] + "built_in_analyzers": [], + "synonyms": {} }, "versions": [ { @@ -1605,7 +1682,10 @@ The API returns the following response: "total_primary_size": "7.4kb", "total_primary_bytes": 7632 } - ] + ], + "dense_vector": { + "value_count": 0 + } }, "nodes": { "count": { diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc index 9c762787a7299..75a6352196b29 100644 --- a/docs/reference/cluster/tasks.asciidoc +++ b/docs/reference/cluster/tasks.asciidoc @@ -245,11 +245,16 @@ POST _tasks/_cancel?nodes=nodeId1,nodeId2&actions=*reindex -------------------------------------------------- A task may continue to run for some time after it has been cancelled because it -may not be able to safely stop its current activity straight away. The list -tasks API will continue to list these cancelled tasks until they complete. The -`cancelled` flag in the response to the list tasks API indicates that the -cancellation command has been processed and the task will stop as soon as -possible. +may not be able to safely stop its current activity straight away, or because +{es} must complete its work on other tasks before it can process the +cancellation. The list tasks API will continue to list these cancelled tasks +until they complete. The `cancelled` flag in the response to the list tasks API +indicates that the cancellation command has been processed and the task will +stop as soon as possible. To troubleshoot why a cancelled task does not +complete promptly, use the list tasks API with the `?detailed` parameter to +identify the other tasks the system is running and also use the +<> API to obtain detailed information about the work +the system is doing instead of completing the cancelled task. ===== Task Grouping diff --git a/docs/reference/data-management.asciidoc b/docs/reference/data-management.asciidoc index f189ef4e4e965..5848f01f77936 100644 --- a/docs/reference/data-management.asciidoc +++ b/docs/reference/data-management.asciidoc @@ -20,6 +20,7 @@ so you can move it to less expensive, less performant hardware. For your oldest data, what matters is that you have access to the data. It's ok if queries take longer to complete. +ifeval::["{release-state}"=="unreleased"] To help you manage your data, {es} offers you: * <> ({ilm-init}) to manage both indices and data streams and it is fully customisable, and @@ -27,6 +28,11 @@ To help you manage your data, {es} offers you: common lifecycle management needs. preview::["The built-in data stream lifecycle is in technical preview and may be changed or removed in a future release. Elastic will apply best effort to fix any issues, but this feature is not subject to the support SLA of official GA features."] +endif::[] +ifeval::["{release-state}"=="released"] +To help you manage your data, {es} offers you <> ({ilm-init}) to manage both indices and data +streams and it is fully customisable. +endif::[] **{ilm-init}** can be used to manage both indices and data streams and it allows you to: @@ -38,12 +44,14 @@ Data older than this period can be deleted by {es}. for your older indices while reducing operating costs and maintaining search performance. * Perform <> of data stored on less-performant hardware. +ifeval::["{release-state}"=="unreleased"] **Data stream lifecycle** is less feature rich but is focused on simplicity, so it allows you to easily: * Define the retention period of your data. The retention period is the minimum time your data will be stored in {es}. Data older than this period can be deleted by {es} at a later time. * Improve the performance of your data stream by performing background operations that will optimise the way your data stream is stored. +endif::[] -- include::ilm/index.asciidoc[] diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index 580ea5102c8ae..461addf65c53c 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -69,14 +69,16 @@ PUT /_data_stream/my-data-stream POST /my-data-stream/_rollover/ PUT /_data_stream/new-data-stream + +DELETE /_data_stream/*/_lifecycle ---- // TESTSETUP [source,console] ---- -DELETE /_data_stream/* +DELETE /_data_stream/my-data-stream*,new-data-stream* -DELETE /_index_template/* +DELETE /_index_template/my-data-stream-template,new-data-stream-template DELETE /_ilm/policy/my-data-stream-policy ---- diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc index b21a08aa853b2..fba2e696e33a5 100644 --- a/docs/reference/data-streams/data-stream-apis.asciidoc +++ b/docs/reference/data-streams/data-stream-apis.asciidoc @@ -12,6 +12,7 @@ The following APIs are available for managing <>: * <> * <> +ifeval::["{release-state}"=="unreleased"] [[data-stream-lifecycle-api]] The following APIs are available for managing the built-in lifecycle of data streams: @@ -19,6 +20,7 @@ The following APIs are available for managing the built-in lifecycle of data str * <> preview:[] * <> preview:[] * <> preview:[] +endif::[] The following API is available for <>: @@ -41,6 +43,7 @@ include::{es-repo-dir}/data-streams/promote-data-stream-api.asciidoc[] include::{es-repo-dir}/data-streams/modify-data-streams-api.asciidoc[] +ifeval::["{release-state}"=="unreleased"] include::{es-repo-dir}/data-streams/lifecycle/apis/put-lifecycle.asciidoc[] include::{es-repo-dir}/data-streams/lifecycle/apis/get-lifecycle.asciidoc[] @@ -48,5 +51,6 @@ include::{es-repo-dir}/data-streams/lifecycle/apis/get-lifecycle.asciidoc[] include::{es-repo-dir}/data-streams/lifecycle/apis/delete-lifecycle.asciidoc[] include::{es-repo-dir}/data-streams/lifecycle/apis/explain-lifecycle.asciidoc[] +endif::[] include::{es-repo-dir}/indices/downsample-data-stream.asciidoc[] diff --git a/docs/reference/data-streams/downsampling-manual.asciidoc b/docs/reference/data-streams/downsampling-manual.asciidoc index 856bf173635b5..6b98816c2cf56 100644 --- a/docs/reference/data-streams/downsampling-manual.asciidoc +++ b/docs/reference/data-streams/downsampling-manual.asciidoc @@ -4,10 +4,22 @@ Run downsampling manually ++++ -This is a simplified example that allows you to see quickly how -<> works to reduce the storage size of a time series -index. The example uses typical Kubernetes cluster monitoring data. To test out -downsampling, follow these steps: +//// +[source,console] +---- +DELETE _data_stream/my-data-stream +DELETE _index_template/my-data-stream-template +DELETE _ingest/pipeline/my-timestamp-pipeline +---- +// TEARDOWN +//// + +The recommended way to downsample a time series data stream (TSDS) is +<>. However, if +you're not using ILM, you can downsample a TSDS manually. This guide shows you +how, using typical Kubernetes cluster monitoring data. + +To test out manual downsampling, follow these steps: . Check the <>. . <>. @@ -19,28 +31,25 @@ downsampling, follow these steps: [[downsampling-manual-prereqs]] ==== Prerequisites -Refer to <>. - -For the example you need a sample data file. Download the file from link: -https://static-www.elastic.co/v3/assets/bltefdd0b53724fa2ce/bltf2fe7a300c3c59f7/631b4bc5cc56115de2f58e8c/sample-k8s-metrics.json[here] -and save it in the local directory where you're running {es}. +* Refer to the <>. +* It is not possible to downsample a data stream directly, nor +multiple indices at once. It's only possible to downsample one time series index +(TSDS backing index). +* In order to downsample an index, it needs to be read-only. For a TSDS write +index, this means it needs to be rolled over and made read-only first. +* Downsampling uses UTC timestamps. +* Downsampling needs at least one metric field to exist in the time series +index. [discrete] [[downsampling-manual-create-index]] -==== Create a time series index +==== Create a time series data stream -This creates an index for a basic data stream. The available parameters for an -index are described in detail in <>. - -The time series boundaries are set so that sampling data for the index begins at -`2022-06-10T00:00:00Z` and ends at `2022-06-30T23:59:59Z`. - -For simplicity, in the time series mapping all `time_series_metric` parameters -are set to type `gauge`, but <> such as -`counter` and `histogram` may also be used. The `time_series_metric` values -determine the kind of statistical representations that are used during -downsampling. +First, you'll create a TSDS. For simplicity, in the time series mapping all +`time_series_metric` parameters are set to type `gauge`, but +<> such as `counter` and `histogram` may also +be used. The `time_series_metric` values determine the kind of statistical +representations that are used during downsampling. The index template includes a set of static <>: `host`, `namespace`, @@ -49,258 +58,241 @@ downsampling process. [source,console] ---- -PUT /sample-01 +PUT _index_template/my-data-stream-template { + "index_patterns": [ + "my-data-stream*" + ], + "data_stream": {}, + "template": { "settings": { - "index": { - "mode": "time_series", - "time_series": { - "start_time": "2022-06-10T00:00:00Z", - "end_time": "2022-06-30T23:59:59Z" - }, - "routing_path": [ - "kubernetes.namespace", - "kubernetes.host", - "kubernetes.node", - "kubernetes.pod" - ], - "number_of_replicas": 0, - "number_of_shards": 2 - } + "index": { + "mode": "time_series", + "routing_path": [ + "kubernetes.namespace", + "kubernetes.host", + "kubernetes.node", + "kubernetes.pod" + ], + "number_of_replicas": 0, + "number_of_shards": 2 + } }, "mappings": { - "properties": { - "@timestamp": { - "type": "date" - }, - "kubernetes": { - "properties": { - "container": { - "properties": { - "cpu": { - "properties": { - "usage": { - "properties": { - "core": { - "properties": { - "ns": { - "type": "long" - } - } - }, - "limit": { - "properties": { - "pct": { - "type": "float" - } - } - }, - "nanocores": { - "type": "long", - "time_series_metric": "gauge" - }, - "node": { - "properties": { - "pct": { - "type": "float" - } - } - } - } - } - } - }, - "memory": { - "properties": { - "available": { - "properties": { - "bytes": { - "type": "long", - "time_series_metric": "gauge" - } - } - }, - "majorpagefaults": { - "type": "long" - }, - "pagefaults": { - "type": "long", - "time_series_metric": "gauge" - }, - "rss": { - "properties": { - "bytes": { - "type": "long", - "time_series_metric": "gauge" - } - } - }, - "usage": { - "properties": { - "bytes": { - "type": "long", - "time_series_metric": "gauge" - }, - "limit": { - "properties": { - "pct": { - "type": "float" - } - } - }, - "node": { - "properties": { - "pct": { - "type": "float" - } - } - } - } - }, - "workingset": { - "properties": { - "bytes": { - "type": "long", - "time_series_metric": "gauge" - } - } - } - } - }, - "name": { - "type": "keyword" - }, - "start_time": { - "type": "date" + "properties": { + "@timestamp": { + "type": "date" + }, + "kubernetes": { + "properties": { + "container": { + "properties": { + "cpu": { + "properties": { + "usage": { + "properties": { + "core": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "limit": { + "properties": { + "pct": { + "type": "float" } + } + }, + "nanocores": { + "type": "long", + "time_series_metric": "gauge" + }, + "node": { + "properties": { + "pct": { + "type": "float" + } + } + } + } + } + } + }, + "memory": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long", + "time_series_metric": "gauge" } + } }, - "host": { - "type": "keyword", - "time_series_dimension": true + "majorpagefaults": { + "type": "long" }, - "namespace": { - "type": "keyword", - "time_series_dimension": true + "pagefaults": { + "type": "long", + "time_series_metric": "gauge" }, - "node": { - "type": "keyword", - "time_series_dimension": true + "rss": { + "properties": { + "bytes": { + "type": "long", + "time_series_metric": "gauge" + } + } }, - "pod": { - "type": "keyword", - "time_series_dimension": true + "usage": { + "properties": { + "bytes": { + "type": "long", + "time_series_metric": "gauge" + }, + "limit": { + "properties": { + "pct": { + "type": "float" + } + } + }, + "node": { + "properties": { + "pct": { + "type": "float" + } + } + } + } + }, + "workingset": { + "properties": { + "bytes": { + "type": "long", + "time_series_metric": "gauge" + } + } } + } + }, + "name": { + "type": "keyword" + }, + "start_time": { + "type": "date" } + } + }, + "host": { + "type": "keyword", + "time_series_dimension": true + }, + "namespace": { + "type": "keyword", + "time_series_dimension": true + }, + "node": { + "type": "keyword", + "time_series_dimension": true + }, + "pod": { + "type": "keyword", + "time_series_dimension": true } + } } + } } + } } - ---- [discrete] [[downsampling-manual-ingest-data]] ==== Ingest time series data -In a terminal window with {es} running, run the following curl command to load -the documents from the downloaded sample data file: +Because time series data streams have been designed to +<>, in this example, you'll +use an ingest pipeline to time-shift the data as it gets indexed. As a result, +the indexed data will have an `@timestamp` from the last 15 minutes. + +Create the pipeline with this request: -[source,sh] +[source,console] ---- -curl -s -H "Content-Type: application/json" \ - -XPOST http:///sample-01/_bulk?pretty \ - --data-binary @sample-k8s-metrics.json +PUT _ingest/pipeline/my-timestamp-pipeline +{ + "description": "Shifts the @timestamp to the last 15 minutes", + "processors": [ + { + "set": { + "field": "ingest_time", + "value": "{{_ingest.timestamp}}" + } + }, + { + "script": { + "lang": "painless", + "source": """ + def delta = ChronoUnit.SECONDS.between( + ZonedDateTime.parse("2022-06-21T15:49:00Z"), + ZonedDateTime.parse(ctx["ingest_time"]) + ); + ctx["@timestamp"] = ZonedDateTime.parse(ctx["@timestamp"]).plus(delta,ChronoUnit.SECONDS).toString(); + """ + } + } + ] +} ---- -// NOTCONSOLE +// TEST[continued] -Approximately 18,000 documents are added. Check the search results for the newly -ingested data: +Next, use a bulk API request to automatically create your TSDS and index a set +of ten documents: [source,console] ---- -GET /sample-01*/_search +PUT /my-data-stream/_bulk?refresh&pipeline=my-timestamp-pipeline +{"create": {}} +{"@timestamp":"2022-06-21T15:49:00Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":91153,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":463314616},"usage":{"bytes":307007078,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":585236},"rss":{"bytes":102728},"pagefaults":120901,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} +{"create": {}} +{"@timestamp":"2022-06-21T15:45:50Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":124501,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":982546514},"usage":{"bytes":360035574,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1339884},"rss":{"bytes":381174},"pagefaults":178473,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} +{"create": {}} +{"@timestamp":"2022-06-21T15:44:50Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":38907,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":862723768},"usage":{"bytes":379572388,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":431227},"rss":{"bytes":386580},"pagefaults":233166,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} +{"create": {}} +{"@timestamp":"2022-06-21T15:44:40Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":86706,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":567160996},"usage":{"bytes":103266017,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1724908},"rss":{"bytes":105431},"pagefaults":233166,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} +{"create": {}} +{"@timestamp":"2022-06-21T15:44:00Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":150069,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":639054643},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1786511},"rss":{"bytes":189235},"pagefaults":138172,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} +{"create": {}} +{"@timestamp":"2022-06-21T15:42:40Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":82260,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":854735585},"usage":{"bytes":309798052,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":924058},"rss":{"bytes":110838},"pagefaults":259073,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} +{"create": {}} +{"@timestamp":"2022-06-21T15:42:10Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":153404,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":279586406},"usage":{"bytes":214904955,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1047265},"rss":{"bytes":91914},"pagefaults":302252,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} +{"create": {}} +{"@timestamp":"2022-06-21T15:40:20Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":125613,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":822782853},"usage":{"bytes":100475044,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2109932},"rss":{"bytes":278446},"pagefaults":74843,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} +{"create": {}} +{"@timestamp":"2022-06-21T15:40:10Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":100046,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":567160996},"usage":{"bytes":362826547,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1986724},"rss":{"bytes":402801},"pagefaults":296495,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} +{"create": {}} +{"@timestamp":"2022-06-21T15:38:30Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":40018,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":1062428344},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2294743},"rss":{"bytes":340623},"pagefaults":224530,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} ---- // TEST[continued] -The query has at least 10,000 hits and returns the first 10. In each document -you can see the time series dimensions (`host`, `node`, `pod` and `container`) -as well as the various CPU and memory time series metrics. +You can use the search API to check if the documents have been indexed +correctly: -[source,console-result] +[source,console] ---- - "hits": { - "total": { - "value": 10000, - "relation": "gte" - }, - "max_score": 1, - "hits": [ - { - "_index": "sample-01", - "_id": "WyHN6N6AwdaJByQWAAABgYOOweA", - "_score": 1, - "_source": { - "@timestamp": "2022-06-20T23:59:40Z", - "kubernetes": { - "host": "gke-apps-0", - "node": "gke-apps-0-1", - "pod": "gke-apps-0-1-0", - "container": { - "cpu": { - "usage": { - "nanocores": 80037, - "core": { - "ns": 12828317850 - }, - "node": { - "pct": 0.0000277905 - }, - "limit": { - "pct": 0.0000277905 - } - } - }, - "memory": { - "available": { - "bytes": 790830121 - }, - "usage": { - "bytes": 139548672, - "node": { - "pct": 0.01770037710617187 - }, - "limit": { - "pct": 0.00009923134671484496 - } - }, - "workingset": { - "bytes": 2248540 - }, - "rss": { - "bytes": 289260 - }, - "pagefaults": 74843, - "majorpagefaults": 0 - }, - "start_time": "2021-03-30T07:59:06Z", - "name": "container-name-44" - }, - "namespace": "namespace26" - } - } - } -... +GET /my-data-stream/_search ---- -// TEST[skip:todo] // TEST[continued] -Next, you can run a terms aggregation on the set of time series dimensions (`_tsid`) to -view a date histogram on a fixed interval of one day. +Run the following aggregation on the data to calculate some interesting +statistics: [source,console] ---- -GET /sample-01*/_search +GET /my-data-stream/_search { "size": 0, "aggs": { @@ -341,122 +333,260 @@ GET /sample-01*/_search [discrete] [[downsampling-manual-run]] -==== Run downsampling for the index +==== Downsample the TSDS -Before running downsampling, the index needs to be set to read only mode: +A TSDS can't be downsampled directly. You need to downsample its backing indices +instead. You can see the backing index for your data stream by running: [source,console] ---- -PUT /sample-01/_block/write +GET /_data_stream/my-data-stream ---- // TEST[continued] -And now, you can use the <> to -downsample the index, setting the time series interval to one hour: +This returns: + +[source,console-result] +---- +{ + "data_streams": [ + { + "name": "my-data-stream", + "timestamp_field": { + "name": "@timestamp" + }, + "indices": [ + { + "index_name": ".ds-my-data-stream-2023.07.26-000001", <1> + "index_uuid": "ltOJGmqgTVm4T-Buoe7Acg" + } + ], + "generation": 1, + "status": "GREEN", + "template": "my-data-stream-template", + "hidden": false, + "system": false, + "lifecycle": { + "enabled": true + }, + "allow_custom_routing": false, + "replicated": false, + "time_series": { + "temporal_ranges": [ + { + "start": "2023-07-26T09:26:42.000Z", + "end": "2023-07-26T13:26:42.000Z" + } + ] + } + } + ] +} +---- +// TESTRESPONSE[s/".ds-my-data-stream-2023.07.26-000001"/$body.data_streams.0.indices.0.index_name/] +// TESTRESPONSE[s/"ltOJGmqgTVm4T-Buoe7Acg"/$body.data_streams.0.indices.0.index_uuid/] +// TESTRESPONSE[s/"2023-07-26T09:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.start/] +// TESTRESPONSE[s/"2023-07-26T13:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.end/] +<1> The backing index for this data stream. + +Before a backing index can be downsampled, the TSDS needs to be rolled over and +the old index needs to be made read-only. + +Roll over the TSDS using the <>: [source,console] ---- -POST /sample-01/_downsample/sample-01-downsample +POST /my-data-stream/_rollover/ +---- +// TEST[continued] + +Copy the name of the `old_index` from the response. In the following steps, +replace the index name with that of your `old_index`. + +The old index needs to be set to read-only mode. Run the following request: + +[source,console] +---- +PUT /.ds-my-data-stream-2023.07.26-000001/_block/write +---- +// TEST[skip:We don't know the index name at test time] + +Next, use the <> to downsample +the index, setting the time series interval to one hour: + +[source,console] +---- +POST /.ds-my-data-stream-2023.07.26-000001/_downsample/.ds-my-data-stream-2023.07.26-000001-downsample { "fixed_interval": "1h" } ---- -// TEST[continued] +// TEST[skip:We don't know the index name at test time] -Finally, delete the original index: +Now you can <>, and replace the +original index with the downsampled one: [source,console] ---- -DELETE /sample-01 +POST _data_stream/_modify +{ + "actions": [ + { + "remove_backing_index": { + "data_stream": "my-data-stream", + "index": ".ds-my-data-stream-2023.07.26-000001" + } + }, + { + "add_backing_index": { + "data_stream": "my-data-stream", + "index": ".ds-my-data-stream-2023.07.26-000001-downsample" + } + } + ] +} ---- -// TEST[continued] +// TEST[skip:We don't know the index name at test time] + +You can now delete the old backing index. But be aware this will delete the +original data. Don't delete the index if you may need the original data in the +future. [discrete] [[downsampling-manual-view-results]] ==== View the results - -Re-run your search query (note that when querying downsampled indices there are <>): +Re-run the earlier search query (note that when querying downsampled indices +there are <>): [source,console] ---- -GET /sample-01*/_search +GET /my-data-stream/_search ---- -// TEST[continued] +// TEST[skip:Because we've skipped the previous steps] -In the query results, notice that the number of hits has been reduced to only 288 -documents. As well, for each time series metric statistical representations have -been calculated: `min`, `max`, `sum`, and `value_count`. +The TSDS with the new downsampled backing index contains just one document. For +counters, this document would only have the last value. For gauges, the field +type is now `aggregate_metric_double`. You see the `min`, `max`, `sum`, and +`value_count` statistics based off of the original sampled metrics: [source,console-result] ---- +{ + "took": 2, + "timed_out": false, + "_shards": { + "total": 4, + "successful": 4, + "skipped": 0, + "failed": 0 + }, "hits": { "total": { - "value": 288, + "value": 1, "relation": "eq" }, "max_score": 1, "hits": [ { - "_index": "sample-01-downsample", - "_id": "WyHN6N6AwdaJByQWAAABgYNYIYA", + "_index": ".ds-my-data-stream-2023.07.26-000001-downsample", + "_id": "0eL0wC_4-45SnTNFAAABiZHbD4A", "_score": 1, "_source": { - "@timestamp": "2022-06-20T23:00:00.000Z", - "_doc_count": 81, - "kubernetes.host": "gke-apps-0", - "kubernetes.namespace": "namespace26", - "kubernetes.node": "gke-apps-0-1", - "kubernetes.pod": "gke-apps-0-1-0", - "kubernetes.container.cpu.usage.nanocores": { - "min": 23344, - "max": 163408, - "sum": 7488985, - "value_count": 81 - }, - "kubernetes.container.memory.available.bytes": { - "min": 167751844, - "max": 1182251090, - "sum": 58169948901, - "value_count": 81 - }, - "kubernetes.container.memory.rss.bytes": { - "min": 54067, - "max": 391987, - "sum": 17550215, - "value_count": 81 - }, - "kubernetes.container.memory.pagefaults": { - "min": 69086, - "max": 428910, - "sum": 20239365, - "value_count": 81 - }, - "kubernetes.container.memory.workingset.bytes": { - "min": 323420, - "max": 2279342, - "sum": 104233700, - "value_count": 81 - }, - "kubernetes.container.memory.usage.bytes": { - "min": 61401416, - "max": 413064069, - "sum": 18557182404, - "value_count": 81 + "@timestamp": "2023-07-26T11:00:00.000Z", + "_doc_count": 10, + "ingest_time": "2023-07-26T11:26:42.715Z", + "kubernetes": { + "container": { + "cpu": { + "usage": { + "core": { + "ns": 12828317850 + }, + "limit": { + "pct": 0.0000277905 + }, + "nanocores": { + "min": 38907, + "max": 153404, + "sum": 992677, + "value_count": 10 + }, + "node": { + "pct": 0.0000277905 + } + } + }, + "memory": { + "available": { + "bytes": { + "min": 279586406, + "max": 1062428344, + "sum": 7101494721, + "value_count": 10 + } + }, + "majorpagefaults": 0, + "pagefaults": { + "min": 74843, + "max": 302252, + "sum": 2061071, + "value_count": 10 + }, + "rss": { + "bytes": { + "min": 91914, + "max": 402801, + "sum": 2389770, + "value_count": 10 + } + }, + "usage": { + "bytes": { + "min": 100475044, + "max": 379572388, + "sum": 2668170609, + "value_count": 10 + }, + "limit": { + "pct": 0.00009923134 + }, + "node": { + "pct": 0.017700378 + } + }, + "workingset": { + "bytes": { + "min": 431227, + "max": 2294743, + "sum": 14230488, + "value_count": 10 + } + } + }, + "name": "container-name-44", + "start_time": "2021-03-30T07:59:06.000Z" + }, + "host": "gke-apps-0", + "namespace": "namespace26", + "node": "gke-apps-0-0", + "pod": "gke-apps-0-0-0" } } - }, -... + } + ] + } +} ---- -// TEST[skip:todo] +// TEST[skip:Because we've skipped the previous step] -You can now re-run the earlier aggregation. Even though the aggregation runs on -the downsampled data stream that only contains 288 documents, it returns the -same results as the earlier aggregation on the original data stream. +Re-run the earlier aggregation. Even though the aggregation runs on the +downsampled TSDS that only contains 1 document, it returns the same results as +the earlier aggregation on the original TSDS. [source,console] ---- -GET /sample-01*/_search +GET /my-data-stream/_search { "size": 0, "aggs": { @@ -493,13 +623,13 @@ GET /sample-01*/_search } } ---- -// TEST[continued] +// TEST[skip:Because we've skipped the previous steps] This example demonstrates how downsampling can dramatically reduce the number of -records stored for time series data, within whatever time boundaries you choose. -It's also possible to perform downsampling on already downsampled data, to -further reduce storage and associated costs, as the time series data ages and +documents stored for time series data, within whatever time boundaries you +choose. It's also possible to perform downsampling on already downsampled data, +to further reduce storage and associated costs, as the time series data ages and the data resolution becomes less critical. -Downsampling is very easily integrated within an ILM policy. To learn more, try -the <> example. +The recommended way to downsample a TSDS is with ILM. To learn more, try the +<> example. diff --git a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc index 7359f4a030456..f0afa9da2e00e 100644 --- a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc @@ -105,7 +105,9 @@ index: "managed_by_lifecycle" : true, "index_creation_date_millis" : 1679475563571, "time_since_index_creation" : "843ms", - "lifecycle" : { }, + "lifecycle" : { + "enabled": true + }, "error": "{\"type\":\"validation_exception\",\"reason\":\"Validation Failed: 1: this action would add [2] shards, but this cluster currently has [4]/[3] maximum normal shards open;\"}" <1> } diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc index 64c8dab90ef61..f20a3393c191c 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc @@ -129,12 +129,14 @@ The response will look like the following: { "name": "my-data-stream-1", "lifecycle": { + "enabled": true, "data_retention": "7d" } }, { "name": "my-data-stream-2", "lifecycle": { + "enabled": true, "data_retention": "7d" } } diff --git a/docs/reference/data-streams/lifecycle/index.asciidoc b/docs/reference/data-streams/lifecycle/index.asciidoc index 3ffdb1390cef0..d1e61d02593d9 100644 --- a/docs/reference/data-streams/lifecycle/index.asciidoc +++ b/docs/reference/data-streams/lifecycle/index.asciidoc @@ -1,5 +1,6 @@ [role="xpack"] [[data-stream-lifecycle]] +ifeval::["{release-state}"=="unreleased"] == Data stream lifecycle preview::[] @@ -61,4 +62,5 @@ because it is applied on the data stream level and not on the individual backing include::tutorial-manage-new-data-stream.asciidoc[] -include::tutorial-manage-existing-data-stream.asciidoc[] \ No newline at end of file +include::tutorial-manage-existing-data-stream.asciidoc[] +endif::[] diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc index 31d6fce2c5295..5670faaade3ce 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc @@ -60,6 +60,7 @@ The response will look like: "index_creation_date_millis": 1681919221417, "time_since_index_creation": "6.85s", <3> "lifecycle": { + "enabled": true, "data_retention": "30d" <4> } }, @@ -72,6 +73,7 @@ The response will look like: "time_since_rollover": "6.84s", <8> "generation_time": "6.84s", <9> "lifecycle": { + "enabled": true, "data_retention": "30d" <10> } } diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc index 628ff1aec58a8..13472d282333d 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc @@ -95,14 +95,16 @@ The result will look like this: { "name": "my-data-stream",<1> "lifecycle": { - "data_retention": "7d" <2> + "enabled": true, <2> + "data_retention": "7d" <3> } } ] } -------------------------------------------------- <1> The name of your data stream. -<2> The retention period of the data indexed in this data stream, this means that the data in this data stream will +<2> Shows if the data stream lifecycle is enabled for this data stream. +<3> The retention period of the data indexed in this data stream, this means that the data in this data stream will be kept at least for 7 days. After that {es} can delete it at its own discretion. If you want to see more information about how the data stream lifecycle is applied on individual backing indices use the @@ -125,6 +127,7 @@ The result will look like this: "index_creation_date_millis": 1681918009501, "time_since_index_creation": "1.6m", <3> "lifecycle": { <4> + "enabled": true, "data_retention": "7d" } } diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc index 781b4737be0ba..3f49a7ab8c700 100644 --- a/docs/reference/data-streams/tsds.asciidoc +++ b/docs/reference/data-streams/tsds.asciidoc @@ -6,7 +6,7 @@ more time series. You can use a TSDS to store metrics data more efficiently. In our benchmarks, metrics data stored in a TSDS used 70% less disk space than a regular data -stream. +stream. The exact impact will vary per data set. [discrete] [[when-to-use-tsds]] @@ -53,6 +53,9 @@ shard segments by `_tsid` and `@timestamp`. documents, the document `_id` is a hash of the document's dimensions and `@timestamp`. A TSDS doesn't support custom document `_id` values. +* A TSDS uses <>, and as a result is +subject to a number of <>. + [discrete] [[time-series]] === What is a time series? @@ -142,17 +145,20 @@ Accepted metric types vary based on the field type: [%collapsible%open] ==== // tag::time-series-metric-counter[] -`counter`:: A number that only increases or resets to `0` (zero). For +`counter`:: A cumulative metric that only monotonically increases or resets to `0` (zero). For example, a count of errors or completed tasks. // end::time-series-metric-counter[] + -Only numeric and `aggregate_metric_double` fields support the `counter` metric -type. +A counter field has additional semantic meaning, because it represents a cumulative counter. This works well with +the `rate` aggregation, since a rate can be derived from a cumulative monotonically increasing counter. However a number +of aggregations (for example `sum`) compute results that don't make sense for a counter field, because of its cumulative nature. ++ +Only numeric and `aggregate_metric_double` fields support the `counter` metric type. -NOTE: Counter fields do come with a limitation in aggregations. Only the following aggregations are supported with the `counter` field: `rate`, `histogram`, `range`, `min`, `max`, `top_metrics` and `variable_width_histogram`. +NOTE: Due to the cumulative nature of counter fields, the following aggregations are supported and expected to provide meaningful results with the `counter` field: `rate`, `histogram`, `range`, `min`, `max`, `top_metrics` and `variable_width_histogram`. In order to prevent issues with existing integrations and custom dashboards, we also allow the following aggregations, even if the result might be meaningless on counters: `avg`, `box plot`, `cardinality`, `extended stats`, `median absolute deviation`, `percentile ranks`, `percentiles`, `stats`, `sum` and `value count`. // tag::time-series-metric-gauge[] -`gauge`:: A number that can increase or decrease. For example, a temperature or +`gauge`:: A metric that represents a single numeric that can arbitrarily increase or decrease. For example, a temperature or available disk space. // end::time-series-metric-gauge[] + @@ -316,5 +322,5 @@ Now that you know the basics, you're ready to <> or include::set-up-tsds.asciidoc[] include::tsds-index-settings.asciidoc[] include::downsampling.asciidoc[] -include::downsampling-manual.asciidoc[] include::downsampling-ilm.asciidoc[] +include::downsampling-manual.asciidoc[] diff --git a/docs/reference/data-streams/use-a-data-stream.asciidoc b/docs/reference/data-streams/use-a-data-stream.asciidoc index eb75fb19156b6..7f5dda6b8a948 100644 --- a/docs/reference/data-streams/use-a-data-stream.asciidoc +++ b/docs/reference/data-streams/use-a-data-stream.asciidoc @@ -43,9 +43,9 @@ PUT /my-data-stream/_create/bfspvnIBr7VVZlfp2lqX?refresh=wait_for [source,console] ---- -DELETE /_data_stream/* +DELETE /_data_stream/my-data-stream* -DELETE /_index_template/* +DELETE /_index_template/my-data-stream-template ---- // TEARDOWN //// diff --git a/docs/reference/eql/eql.asciidoc b/docs/reference/eql/eql.asciidoc index f2191b8402c18..2ede5e0fc737e 100644 --- a/docs/reference/eql/eql.asciidoc +++ b/docs/reference/eql/eql.asciidoc @@ -32,7 +32,7 @@ describe activity that goes beyond IOCs. With the exception of sample queries, EQL searches require that the searched data stream or index contains a _timestamp_ field. By default, EQL uses the -`@timestamp` field from the {ecs-ref}[Elastic Common Schema (ECS)]. +`@timestamp` field from the {ecs-ref}[Elastic Common Schema (ECS)]. EQL searches also require an _event category_ field, unless you use the <> to search for documents @@ -249,6 +249,90 @@ GET /my-data-stream/_eql/search ---- // TEST[setup:sec_logs] +Use `!` to match <>: events in a sequence +that do not meet a condition within a given timespan: + +[source,console] +---- +GET /my-data-stream/_eql/search +{ + "query": """ + sequence with maxspan=1d + [ process where process.name == "cmd.exe" ] + ![ process where stringContains(process.command_line, "ocx") ] + [ file where stringContains(file.name, "scrobj.dll") ] + """ +} +---- +// TEST[setup:sec_logs] + +Missing events are indicated in the response as `missing": true`: + +[source,console-result] +---- +{ + ... + "hits": { + "total": ..., + "sequences": [ + { + "events": [ + { + "_index": ".ds-my-data-stream-2023.07.04-000001", + "_id": "AnpTIYkBrVQ2QEgsWg94", + "_source": { + "@timestamp": "2099-12-07T11:06:07.000Z", + "event": { + "category": "process", + "id": "cMyt5SZ2", + "sequence": 3 + }, + "process": { + "pid": 2012, + "name": "cmd.exe", + "executable": "C:\\Windows\\System32\\cmd.exe" + } + } + }, + { + "_index": "", + "_id": "", + "_source": {}, + "missing": true + }, + { + "_index": ".ds-my-data-stream-2023.07.04-000001", + "_id": "BHpTIYkBrVQ2QEgsWg94", + "_source": { + "@timestamp": "2099-12-07T11:07:10.000Z", + "event": { + "category": "file", + "id": "tZ1NWVOs", + "sequence": 5 + }, + "process": { + "pid": 2012, + "name": "regsvr32.exe", + "executable": "C:\\Windows\\System32\\regsvr32.exe" + }, + "file": { + "path": "C:\\Windows\\System32\\scrobj.dll", + "name": "scrobj.dll" + } + } + } + ] + } + ] + } +} +---- +// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[s/"total": \.\.\.,/"total": { "value": 1, "relation": "eq" },/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2023.07.04-000001"/"_index": $body.hits.sequences.0.events.0._index/] +// TESTRESPONSE[s/"_id": "AnpTIYkBrVQ2QEgsWg94"/"_id": $body.hits.sequences.0.events.0._id/] +// TESTRESPONSE[s/"_id": "BHpTIYkBrVQ2QEgsWg94"/"_id": $body.hits.sequences.0.events.2._id/] + Use the <> to match events that share the same field values: @@ -555,9 +639,9 @@ GET /my-index*/_eql/search { "query": """ sample by host - [any where uptime > 0] - [any where port > 100] - [any where bool == true] + [any where uptime > 0] + [any where port > 100] + [any where bool == true] """ } ---- @@ -691,7 +775,7 @@ each of the filters. Events are returned in the order of the filters they match: } } ---- -// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[skip:Response is illustrative only] <1> The events in the first sample have a value of `doom` for `host`. <2> This event matches the first filter. @@ -845,7 +929,7 @@ for `os` or `op_sys`, as well as for `host`. For example: } } ---- -// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[skip:Response is illustrative only] <1> The events in this sample have a value of `doom` for `host` and a value of `redhat` for `os` or `op_sys`. @@ -854,7 +938,7 @@ By default, the response of a sample query contains up to 10 samples, with one sample per unique set of join keys. Use the `size` parameter to get a smaller or larger set of samples. To retrieve more than one sample per set of join keys, use the `max_samples_per_key` parameter. Pipes are not supported for sample -queries. +queries. [source,console] ---- diff --git a/docs/reference/eql/syntax.asciidoc b/docs/reference/eql/syntax.asciidoc index 1893f6d82e6c2..f592610f487c9 100644 --- a/docs/reference/eql/syntax.asciidoc +++ b/docs/reference/eql/syntax.asciidoc @@ -574,6 +574,40 @@ sequence with maxspan=15m [ process where true ] ---- +[discrete] +[[eql-missing-events]] +==== Missing events + +Use `!` to match missing events: events in a timespan-constrained sequence that +do not meet a given condition. + +[source,eql] +---- +sequence with maxspan=1h + [ event_category_1 where condition_1 ] + ![ event_category_2 where condition_2 ] + [ event_category_3 where condition_3 ] + ... +---- + +Missing event clauses can be used at the beginning, at the end, and/or in the +middle of a sequence, in any combination with positive event clauses. A sequence +can have multiple missing event clauses, but needs to have at least one positive +clause. <> is mandatory when missing +event clauses are present. + + +*Example* + +The following sequence query finds logon events that are not followed within 5 +seconds by a logoff event. + +[source,eql] +---- +sequence by host.name, user.name with maxspan=5s + [ authentication where event.code : "4624" ] + ![ authentication where event.code : "4647" ] +---- + [discrete] [[eql-by-keyword]] ==== `by` keyword diff --git a/docs/reference/fleet/fleet-multi-search.asciidoc b/docs/reference/fleet/fleet-multi-search.asciidoc index 036e2f3cadbb3..3ee6b67b06ba4 100644 --- a/docs/reference/fleet/fleet-multi-search.asciidoc +++ b/docs/reference/fleet/fleet-multi-search.asciidoc @@ -2,7 +2,7 @@ [[fleet-multi-search]] === Fleet multi search API ++++ -Fleet search +Fleet multi search ++++ Executes several <> with a single API request. diff --git a/docs/reference/fleet/index.asciidoc b/docs/reference/fleet/index.asciidoc index f550542589e0c..b22609aff32e7 100644 --- a/docs/reference/fleet/index.asciidoc +++ b/docs/reference/fleet/index.asciidoc @@ -11,6 +11,7 @@ agent and action data. These APIs are experimental and for internal use by * <> * <> +* <> // top-level include::get-global-checkpoints.asciidoc[] diff --git a/docs/reference/how-to/fix-common-cluster-issues.asciidoc b/docs/reference/how-to/fix-common-cluster-issues.asciidoc index 3b9bdf8f6fc3b..531ae44cc3be2 100644 --- a/docs/reference/how-to/fix-common-cluster-issues.asciidoc +++ b/docs/reference/how-to/fix-common-cluster-issues.asciidoc @@ -443,13 +443,10 @@ GET _cluster/allocation/explain?filter_path=index,node_allocation_decisions.node { "index": "my-index", "shard": 0, - "primary": false, - "current_node": "my-node" + "primary": false } ---- // TEST[s/^/PUT my-index\n/] -// TEST[s/"primary": false,/"primary": false/] -// TEST[s/"current_node": "my-node"//] [discrete] [[fix-red-yellow-cluster-status]] diff --git a/docs/reference/how-to/size-your-shards.asciidoc b/docs/reference/how-to/size-your-shards.asciidoc index a94d0d0161a01..8b631dbbaa5ce 100644 --- a/docs/reference/how-to/size-your-shards.asciidoc +++ b/docs/reference/how-to/size-your-shards.asciidoc @@ -140,20 +140,21 @@ Every new backing index is an opportunity to further tune your strategy. [discrete] [[shard-size-recommendation]] -==== Aim for shard sizes between 10GB and 50GB - -Larger shards take longer to recover after a failure. When a node fails, {es} -rebalances the node's shards across the data tier's remaining nodes. This -recovery process typically involves copying the shard contents across the -network, so a 100GB shard will take twice as long to recover than a 50GB shard. -In contrast, small shards carry proportionally more overhead and are less -efficient to search. Searching fifty 1GB shards will take substantially more -resources than searching a single 50GB shard containing the same data. - -There are no hard limits on shard size, but experience shows that shards -between 10GB and 50GB typically work well for logs and time series data. You -may be able to use larger shards depending on your network and use case. -Smaller shards may be appropriate for +==== Aim for shards of up to 200M documents, or with sizes between 10GB and 50GB + +There is some overhead associated with each shard, both in terms of cluster +management and search performance. Searching a thousand 50MB shards will be +substantially more expensive than searching a single 50GB shard containing the +same data. However, very large shards can also cause slower searches and will +take longer to recover after a failure. + +There is no hard limit on the physical size of a shard, and each shard can in +theory contain up to just over two billion documents. However, experience shows +that shards between 10GB and 50GB typically work well for many use cases, as +long as the per-shard document count is kept below 200 million. + +You may be able to use larger shards depending on your network and use case, +and smaller shards may be appropriate for {enterprise-search-ref}/index.html[Enterprise Search] and similar use cases. If you use {ilm-init}, set the <>'s diff --git a/docs/reference/ilm/actions/ilm-forcemerge.asciidoc b/docs/reference/ilm/actions/ilm-forcemerge.asciidoc index 0e5c5263c6688..38b5f30db76f0 100644 --- a/docs/reference/ilm/actions/ilm-forcemerge.asciidoc +++ b/docs/reference/ilm/actions/ilm-forcemerge.asciidoc @@ -6,7 +6,6 @@ Phases allowed: hot, warm. <> the index into the specified maximum number of <>. -This action makes the index <>. [NOTE] The `forcemerge` action is best effort. It might happen that some of the diff --git a/docs/reference/ilm/actions/ilm-rollover.asciidoc b/docs/reference/ilm/actions/ilm-rollover.asciidoc index bf026440f4172..9c4489bac1a94 100644 --- a/docs/reference/ilm/actions/ilm-rollover.asciidoc +++ b/docs/reference/ilm/actions/ilm-rollover.asciidoc @@ -129,6 +129,11 @@ opt in to rolling over empty indices, by adding a `"min_docs": 0` condition. Thi disabled on a cluster-wide basis by setting `indices.lifecycle.rollover.only_if_has_documents` to `false`. +NOTE: The rollover action implicitly always rolls over a data stream or alias if one or more shards contain + 200000000 or more documents. Normally a shard will reach 50GB long before it reaches 200M documents, + but this isn't the case for space efficient data sets. Search performance will very likely suffer + if a shard contains more than 200M documents. This is the reason of the builtin limit. + [[ilm-rollover-ex]] ==== Example diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 27257c739f3e3..0496605cfb856 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -17,6 +17,11 @@ the frozen phase, the action mounts a <> prefixed with `partial-` to the frozen tier. In other phases, the action mounts a <> prefixed with `restored-` to the corresponding data tier. +In the frozen tier, the action will ignore the setting +<>, if it was present in the original index, +to account for the difference in the number of nodes between the frozen and the other tiers. + + WARNING: Don't include the `searchable_snapshot` action in both the hot and cold phases. This can result in indices failing to automatically migrate to the cold tier during the cold phase. diff --git a/docs/reference/ilm/error-handling.asciidoc b/docs/reference/ilm/error-handling.asciidoc index 9b9ccdfc7dff7..da76e475c9d14 100644 --- a/docs/reference/ilm/error-handling.asciidoc +++ b/docs/reference/ilm/error-handling.asciidoc @@ -149,6 +149,20 @@ POST /my-index-000001/_ilm/retry {ilm-init} subsequently attempts to re-run the step that failed. You can use the <> to monitor the progress. + +[discrete] +=== Common {ilm-init} setting issues + +[discrete] +==== How `min_age` is calculated + +When setting up an <> or <>, be aware that`min_age` can be relative to either the rollover time or the index creation time. + +If you use <>, `min_age` is calculated relative to the time the index was rolled over. This is because the <> generates a new index. The `creation_date` of the new index (retrievable via <>) is used in the calculation. If you do not use rollover in the {ilm-init} policy, `min_age` is calculated relative to the `creation_date` of the original index. + +You can override how `min_age` is calculated using the `index.lifecycle.origination_date` and `index.lifecycle.parse_origination_date` <>. + + [discrete] === Common {ilm-init} errors diff --git a/docs/reference/ilm/ilm-actions.asciidoc b/docs/reference/ilm/ilm-actions.asciidoc index 0d6dee1d0697b..24bdf73b9a3ca 100644 --- a/docs/reference/ilm/ilm-actions.asciidoc +++ b/docs/reference/ilm/ilm-actions.asciidoc @@ -11,7 +11,6 @@ Permanently remove the index. <>:: Reduce the number of index segments and purge deleted documents. -Makes the index read-only. <>:: Move the index shards to the <> that corresponds diff --git a/docs/reference/images/index-mgmt/compassicon.png b/docs/reference/images/index-mgmt/compassicon.png new file mode 100644 index 0000000000000..f4bb2bf477d7d Binary files /dev/null and b/docs/reference/images/index-mgmt/compassicon.png differ diff --git a/docs/reference/images/index-mgmt/management_index_data_stream_stats.png b/docs/reference/images/index-mgmt/management_index_data_stream_stats.png index d22e3a8bd1bb1..a474839fc653b 100644 Binary files a/docs/reference/images/index-mgmt/management_index_data_stream_stats.png and b/docs/reference/images/index-mgmt/management_index_data_stream_stats.png differ diff --git a/docs/reference/images/index-mgmt/management_index_details.png b/docs/reference/images/index-mgmt/management_index_details.png index 79568818aa4ec..edf1a6517f6a2 100644 Binary files a/docs/reference/images/index-mgmt/management_index_details.png and b/docs/reference/images/index-mgmt/management_index_details.png differ diff --git a/docs/reference/images/search/vector-search-oversimplification.png b/docs/reference/images/search/vector-search-oversimplification.png new file mode 100644 index 0000000000000..f7c519fb10517 Binary files /dev/null and b/docs/reference/images/search/vector-search-oversimplification.png differ diff --git a/docs/reference/index-modules/allocation/filtering.asciidoc b/docs/reference/index-modules/allocation/filtering.asciidoc index f2c43e36bf14f..07a2455ca1eff 100644 --- a/docs/reference/index-modules/allocation/filtering.asciidoc +++ b/docs/reference/index-modules/allocation/filtering.asciidoc @@ -12,8 +12,8 @@ attributes. <> uses filt on custom node attributes to determine how to reallocate shards when moving between phases. -The `cluster.routing.allocation` settings are dynamic, enabling live indices to -be moved from one set of nodes to another. Shards are only relocated if it is +The `cluster.routing.allocation` settings are dynamic, enabling existing indices to +be moved immediately from one set of nodes to another. Shards are only relocated if it is possible to do so without breaking another routing constraint, such as never allocating a primary and replica shard on the same node. diff --git a/docs/reference/indices/data-stream-stats.asciidoc b/docs/reference/indices/data-stream-stats.asciidoc index 1d6d1fd459377..3d27eacf830da 100644 --- a/docs/reference/indices/data-stream-stats.asciidoc +++ b/docs/reference/indices/data-stream-stats.asciidoc @@ -41,8 +41,8 @@ POST /my-data-stream-two/_rollover/ //// [source,console] ---- -DELETE /_data_stream/* -DELETE /_index_template/* +DELETE /_data_stream/my-data-stream* +DELETE /_index_template/template ---- // TEARDOWN //// diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index d84ca797fdf4b..ef2cf7eeee946 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -51,6 +51,8 @@ PUT /_data_stream/my-data-stream POST /my-data-stream/_rollover PUT /_data_stream/my-data-stream-two + +DELETE /_data_stream/my-data-stream*/_lifecycle ---- // TESTSETUP //// diff --git a/docs/reference/indices/index-mgmt.asciidoc b/docs/reference/indices/index-mgmt.asciidoc index 68d0730ee59c4..571f1c813fbfa 100644 --- a/docs/reference/indices/index-mgmt.asciidoc +++ b/docs/reference/indices/index-mgmt.asciidoc @@ -51,7 +51,9 @@ You can also filter indices using the search bar. You can drill down into each index to investigate the index <>, <>, and statistics. -From this view, you can also edit the index settings. +From this view, you can also edit the index settings. + +To view and explore the documents within an index, click the compass icon image:compassicon.png[width=3%] next to the index name to open {kibana-ref}/discover.html[Discover]. [role="screenshot"] image::images/index-mgmt/management_index_details.png[Index Management UI] @@ -80,6 +82,8 @@ them. To view more information about a data stream, such as its generation or its current index lifecycle policy, click the stream's name. +To view and explore the data within a data stream, click the compass icon image:compassicon.png[width=3%] next to the data stream name to open {kibana-ref}/discover.html[Discover]. + [role="screenshot"] image::images/index-mgmt/management_index_data_stream_stats.png[Data stream details] diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index 15516bad5808d..35b42f94640be 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -95,6 +95,7 @@ Name of the component template to create. - `logs-settings` - `metrics-mappings` - `metrics-settings` +- `metrics-tsdb-settings` - `synthetics-mapping` - `synthetics-settings` // end::built-in-component-templates[] diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 63cd20058e7b9..77572f707f4cb 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -386,8 +386,8 @@ databases. http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site]. + You can also use custom city, country, and ASN `.mmdb` files. These files must -be uncompressed and use the respective `-City.mmdb`, `-Country.mmdb`, or -`-ASN.mmdb` extensions. +be uncompressed. The type (city, country, or ASN) will be pulled from the file +metadata, so the filename does not matter. . On {ess} deployments upload database using a {cloud}/ec-custom-bundles.html[custom bundle]. diff --git a/docs/reference/mapping/dynamic/field-mapping.asciidoc b/docs/reference/mapping/dynamic/field-mapping.asciidoc index 45486b70334d4..4ce4b56d1ba8c 100644 --- a/docs/reference/mapping/dynamic/field-mapping.asciidoc +++ b/docs/reference/mapping/dynamic/field-mapping.asciidoc @@ -216,6 +216,11 @@ GET my-index-000001/_mapping -------------------------------------------------- ==== +[NOTE] +==== +Epoch formats (`epoch_millis` and `epoch_second`) are not supported as dynamic date formats. +==== + [[numeric-detection]] ==== Numeric detection diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index 596b6dee191d5..5da334b883721 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -28,12 +28,21 @@ PUT idx While this on the fly reconstruction is *generally* slower than saving the source documents verbatim and loading them at query time, it saves a lot of storage -space. There are a couple of restrictions to be aware of: +space. + +[[synthetic-source-restrictions]] +===== Synthetic `_source` restrictions + +There are a couple of restrictions to be aware of: * When you retrieve synthetic `_source` content it undergoes minor <> compared to the original JSON. * The `params._source` is unavailable in scripts. Instead use the -{painless}/painless-field-context.html[`doc`] API or the <>. +{painless}/painless-field-context.html[`doc`] API or the <>. +* Runtime fields <>, and runtime +fields that access `_source` are currently not supported for indices that use +synthetic `_source`. Use a scripted runtime field that accesses fields <> or the +<> instead. * Synthetic `_source` can be used with indices that contain only these field types: @@ -59,8 +68,6 @@ types: ** <> ** <> -Runtime fields cannot, at this stage, use synthetic `_source`. - [[synthetic-source-modifications]] ===== Synthetic `_source` modifications diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 165ff74825113..664f43a1b4104 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -112,12 +112,7 @@ integer values between -128 to 127, inclusive for both indexing and searching. `dims`:: (Required, integer) -Number of vector dimensions. Can't exceed `1024` for indexed vectors -(`"index": true`), or `2048` for non-indexed vectors. - -+ -experimental:[] -The number of dimensions for indexed vectors can be extended up to `2048`. +Number of vector dimensions. Can't exceed `2048`. `index`:: (Optional, Boolean) diff --git a/docs/reference/migration/apis/feature-migration.asciidoc b/docs/reference/migration/apis/feature-migration.asciidoc index 87903fbb7758e..9a6306dc2f596 100644 --- a/docs/reference/migration/apis/feature-migration.asciidoc +++ b/docs/reference/migration/apis/feature-migration.asciidoc @@ -142,7 +142,7 @@ Example response: "migration_status" : "NO_MIGRATION_NEEDED" } -------------------------------------------------- -// TESTRESPONSE[s/"minimum_index_version" : "8100099"/"minimum_index_version" : $body.$_path/] +// TESTRESPONSE[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/97780] When you submit a POST request to the `_migration/system_features` endpoint to start the migration process, the response indicates what features will be diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index 1213ca840b309..119eb2600af5f 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -1,5 +1,6 @@ include::migration_intro.asciidoc[] +* <> * <> * <> * <> @@ -12,6 +13,7 @@ include::migration_intro.asciidoc[] * <> * <> +include::migrate_8_11.asciidoc[] include::migrate_8_10.asciidoc[] include::migrate_8_9.asciidoc[] include::migrate_8_8.asciidoc[] diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc index 9c7ba87a4759f..09433904f2ea8 100644 --- a/docs/reference/migration/migrate_8_0.asciidoc +++ b/docs/reference/migration/migrate_8_0.asciidoc @@ -46,9 +46,6 @@ after upgrading to 8.0. To find out if you are using any deprecated functionality, enable <>. -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide -//tag::notable-breaking-changes[] [discrete] [[breaking_80_cluster_node_setting_deprecations]] ==== Cluster and node setting deprecations @@ -92,7 +89,6 @@ Passwords are generated automatically for the `elastic` user when you start {es} starting {es}, it will fail because the `elastic` user password is already configured. ==== -//end::notable-breaking-changes[] include::migrate_8_0/migrate_to_java_time.asciidoc[] include::transient-settings-migration-guide.asciidoc[] diff --git a/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc b/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc index 455f67aed38fd..75aa2676e7ed7 100644 --- a/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_cluster_node_setting_changes]] ==== Cluster and node setting changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] TIP: {ess-setting-change} .`action.destructive_requires_name` now defaults to `true`. {ess-icon} @@ -884,7 +880,6 @@ repository-specific `compress` setting to enable compression instead. Refer to {ref}/snapshots-filesystem-repository.html#filesystem-repository-settings[Shared file system repository settings]. ==== -//end::notable-breaking-changes[] // This change is not notable because it should not have any impact on upgrades // However we document it here out of an abundance of caution @@ -911,7 +906,6 @@ Any node that did not have an explicitly configured password hashing algorithm i {es} 6.x or {es} 7.x would have failed to start. ==== -//tag::notable-breaking-changes[] .The `xpack.monitoring.history.duration` will not delete indices created by metricbeat or elastic agent [%collapsible] ==== @@ -935,4 +929,3 @@ metricbeat or agent to collect monitoring data, you can also remove any custom ` settings. ==== -// end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/command-line-tool-changes.asciidoc b/docs/reference/migration/migrate_8_0/command-line-tool-changes.asciidoc index 94da5c9b6215a..7af28a1ae95cc 100644 --- a/docs/reference/migration/migrate_8_0/command-line-tool-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/command-line-tool-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_command_line_tool_changes]] ==== Command line tool changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] TIP: {ess-skip-section} [[migrate-tool-removed]] @@ -22,4 +18,3 @@ realm directly. Discontinue use of the `elasticsearch-migrate` tool. Attempts to use the `elasticsearch-migrate` tool will result in an error. ==== -//end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/index-setting-changes.asciidoc b/docs/reference/migration/migrate_8_0/index-setting-changes.asciidoc index 35d054560267b..60e5588a187a7 100644 --- a/docs/reference/migration/migrate_8_0/index-setting-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/index-setting-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_index_setting_changes]] ==== Index setting changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] [[deprecation-system-indices]] .Direct access to system indices is deprecated. [%collapsible] @@ -124,4 +120,3 @@ Discontinue use of the `index.translog.retention.age` and `index.translog.retention.size` index settings. Requests that include these settings will return an error. ==== -//end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/java-api-changes.asciidoc b/docs/reference/migration/migrate_8_0/java-api-changes.asciidoc index 5727f5ef92eca..2d1cc1be7f3a7 100644 --- a/docs/reference/migration/migrate_8_0/java-api-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/java-api-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_java_api_changes]] ==== Java API changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] [[ilm-hlrc-rename]] .The `indexlifecycle` package has been renamed `ilm` in the Java High Level REST Client. [%collapsible] @@ -52,4 +48,3 @@ testability. *Impact* + No action needed. ==== -//end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/jvm-option-changes.asciidoc b/docs/reference/migration/migrate_8_0/jvm-option-changes.asciidoc index 8d9c602666f24..bdcffe4667ca4 100644 --- a/docs/reference/migration/migrate_8_0/jvm-option-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/jvm-option-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_jvm_option_changes]] ==== JVM option changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] TIP: {ess-skip-section} [[breaking_80_allocation_change_flood_stage_block_always_removed]] @@ -56,4 +52,3 @@ system property, and ensure that all nodes of the same version are running exactly the same build. Setting this system property will result in an error on startup. ==== -//end::notable-breaking-changes[] \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_0/logging-changes.asciidoc b/docs/reference/migration/migrate_8_0/logging-changes.asciidoc index da28aae5f007a..63c025746a64c 100644 --- a/docs/reference/migration/migrate_8_0/logging-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/logging-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_logging_changes]] ==== Logging changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] .{es} JSON logs now comply with ECS. [%collapsible] ==== @@ -55,4 +51,3 @@ to use the new names and to possibly account for `gzip` archives instead of plain text. The Docker build of {es} is not affected because it logs on `stdout`, where rollover is not performed. ==== -//end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/mapping-changes.asciidoc b/docs/reference/migration/migrate_8_0/mapping-changes.asciidoc index 404460bb6b957..7b3922cf0a5dd 100644 --- a/docs/reference/migration/migrate_8_0/mapping-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/mapping-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_mapping_changes]] ==== Mapping changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] .Indices created in {es} 6.x and earlier versions are not supported. [%collapsible] ==== @@ -135,4 +131,3 @@ GitHub or the 'discuss' forums. Discontinue use of the `sparse_vector` field data type. Requests containing a mapping for this field data type will return an error. ==== -//end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/packaging-changes.asciidoc b/docs/reference/migration/migrate_8_0/packaging-changes.asciidoc index 3247d0b723c83..7e0c2c72ee6d7 100644 --- a/docs/reference/migration/migrate_8_0/packaging-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/packaging-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_packaging_changes]] ==== Packaging changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] TIP: {ess-skip-section} .The layout of the data folder has changed. @@ -62,4 +58,3 @@ After the geoip downloader has completed downloading the most up to data databas then the geoip processor will function as normal. The window of time that the geoip processor can't do geoip lookups after cluster startup should be very small. ==== -//end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/painless-changes.asciidoc b/docs/reference/migration/migrate_8_0/painless-changes.asciidoc index 96a6f819e10ed..601866cb8995d 100644 --- a/docs/reference/migration/migrate_8_0/painless-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/painless-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_painless_changes]] ==== Painless changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] .The `JodaCompatibleZonedDateTime` class has been removed. [%collapsible] ==== @@ -44,4 +40,3 @@ The following `JodaCompatibleZonedDateTime` methods must be replaced using * `toString(String)` -> a DateTimeFormatter * `toString(String, Locale)` -> a DateTimeFormatter ==== -//end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/plugin-changes.asciidoc b/docs/reference/migration/migrate_8_0/plugin-changes.asciidoc index 63a625c89026d..42baf8f7f2a69 100644 --- a/docs/reference/migration/migrate_8_0/plugin-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/plugin-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_plugin_changes]] ==== Plugin changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] TIP: {ess-skip-section} .The S3, GCS and Azure repository plugins are now included in Elasticsearch @@ -66,5 +62,3 @@ check with the plugin author and ensure that the plugin is available for your target version of {es} before upgrading. ==== - -//end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc b/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc index 3203a8c310d76..87e7041109710 100644 --- a/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_rest_api_changes]] ==== REST API changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] .REST API endpoints containing `_xpack` have been removed. [%collapsible] ==== @@ -1140,4 +1136,3 @@ for both cases. *Impact* + To detect a server timeout, check the `timed_out` field of the JSON response. ==== -//end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/sql-jdbc-changes.asciidoc b/docs/reference/migration/migrate_8_0/sql-jdbc-changes.asciidoc index 0e14d5b4ec0af..71efa21e032f5 100644 --- a/docs/reference/migration/migrate_8_0/sql-jdbc-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/sql-jdbc-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_jdbc_changes]] ==== SQL JDBC changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] .JDBC driver returns geometry objects as well-known-text string instead of `org.elasticsearch.geo` objects. [%collapsible] ==== @@ -24,4 +20,3 @@ Elasticsearch JDBC driver with their WKT representation by simply calling `toStr This change does NOT impact users that do not use geometry classes. ==== -// end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_0/system-req-changes.asciidoc b/docs/reference/migration/migrate_8_0/system-req-changes.asciidoc index 13564b4cb4c75..df698eaeb2dfd 100644 --- a/docs/reference/migration/migrate_8_0/system-req-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/system-req-changes.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_system_req_changes]] ==== System requirement changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] TIP: {ess-skip-section} .Several EOL operating systems are no longer supported. @@ -61,4 +57,3 @@ the bundled JDK (preferable), or set `ES_JAVA_HOME`. Use the bundled JDK (preferable), or set `ES_JAVA_HOME`. `JAVA_HOME` will be ignored. ==== -//end::notable-breaking-changes[] \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_0/transform.asciidoc b/docs/reference/migration/migrate_8_0/transform.asciidoc index 839b5dda73047..aa47e28d83750 100644 --- a/docs/reference/migration/migrate_8_0/transform.asciidoc +++ b/docs/reference/migration/migrate_8_0/transform.asciidoc @@ -2,10 +2,6 @@ [[breaking_80_transform_changes]] ==== Transform changes -//NOTE: The notable-breaking-changes tagged regions are re-used in the -//Installation and Upgrade Guide - -//tag::notable-breaking-changes[] .{transforms-cap} created in 7.4 or earlier versions must be upgraded. [%collapsible] ==== @@ -18,4 +14,3 @@ that is no longer supported. Use the {ref}/upgrade-transforms.html[upgrade {transforms} API] to fix your {transforms}. This upgrade does not affect the source or destination indices. ==== -// end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_1.asciidoc b/docs/reference/migration/migrate_8_1.asciidoc index af3fe36197987..692559205f735 100644 --- a/docs/reference/migration/migrate_8_1.asciidoc +++ b/docs/reference/migration/migrate_8_1.asciidoc @@ -19,9 +19,6 @@ and prevent them from operating normally. Before upgrading to 8.1, review these changes and take the described steps to mitigate the impact. -// NOTE: The notable-breaking-changes tagged regions are re-used in the -// Installation and Upgrade Guide -// tag::notable-breaking-changes[] [discrete] [[breaking_81_rest_api_changes]] ==== REST API changes @@ -40,7 +37,6 @@ still retrieve original, unnormalized geometry objects from `_source`. If your application requires unnormalized geometry objects, retrieve them from `_source` rather than using the `fields` parameter. ==== -// end::notable-breaking-changes[] [discrete] @@ -56,7 +52,6 @@ after upgrading to 8.1. To find out if you are using any deprecated functionality, enable <>. -// tag::notable-breaking-changes[] [discrete] [[deprecations_81_cluster_and_node_setting]] ==== Cluster and node setting deprecations @@ -94,7 +89,6 @@ actions. Ensure that bulk actions are well-formed JSON objects containing a single entry with the correct key. ==== -// end::notable-breaking-changes[] [[deprecate_index_include_frozen_request_parameter_in_sql_api]] .Deprecate `index_include_frozen` request parameter in `_sql` API diff --git a/docs/reference/migration/migrate_8_10.asciidoc b/docs/reference/migration/migrate_8_10.asciidoc index b6460de502dca..84600fd96bd69 100644 --- a/docs/reference/migration/migrate_8_10.asciidoc +++ b/docs/reference/migration/migrate_8_10.asciidoc @@ -16,7 +16,4 @@ coming::[8.10.0] [[breaking-changes-8.10]] === Breaking changes -// tag::notable-breaking-changes[] There are no breaking changes in {es} 8.10. -// end::notable-breaking-changes[] - diff --git a/docs/reference/migration/migrate_8_11.asciidoc b/docs/reference/migration/migrate_8_11.asciidoc new file mode 100644 index 0000000000000..a353d1a6a87fa --- /dev/null +++ b/docs/reference/migration/migrate_8_11.asciidoc @@ -0,0 +1,20 @@ +[[migrating-8.11]] +== Migrating to 8.11 +++++ +8.11 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.11. + +See also <> and <>. + +coming::[8.11.0] + + +[discrete] +[[breaking-changes-8.11]] +=== Breaking changes + +There are no breaking changes in {es} 8.11. + diff --git a/docs/reference/migration/migrate_8_2.asciidoc b/docs/reference/migration/migrate_8_2.asciidoc index 16308321e3fdb..3630456aed6fd 100644 --- a/docs/reference/migration/migrate_8_2.asciidoc +++ b/docs/reference/migration/migrate_8_2.asciidoc @@ -9,13 +9,8 @@ your application to {es} 8.2. See also <> and <>. -// NOTE: The notable-breaking-changes tagged regions are re-used in the -// Installation and Upgrade Guide -// tag::notable-breaking-changes[] [discrete] [[breaking-changes-8.2]] === Breaking changes There are no breaking changes in {es} 8.2. - -// end::notable-breaking-changes[] diff --git a/docs/reference/migration/migrate_8_4.asciidoc b/docs/reference/migration/migrate_8_4.asciidoc index e3e1bee164a9a..d9aab317d70f7 100644 --- a/docs/reference/migration/migrate_8_4.asciidoc +++ b/docs/reference/migration/migrate_8_4.asciidoc @@ -13,10 +13,7 @@ See also <> and <>. [[breaking-changes-8.4]] === Breaking changes -// tag::notable-breaking-changes[] There are no breaking changes in {es} 8.4. -// end::notable-breaking-changes[] - [discrete] [[deprecated-8.4]] diff --git a/docs/reference/migration/migrate_8_5.asciidoc b/docs/reference/migration/migrate_8_5.asciidoc index 4a91c68afb1dd..1f040946670e1 100644 --- a/docs/reference/migration/migrate_8_5.asciidoc +++ b/docs/reference/migration/migrate_8_5.asciidoc @@ -17,9 +17,6 @@ The following changes in {es} 8.5 might affect your applications and prevent them from operating normally. Before upgrading to 8.5, review these changes and take the described steps to mitigate the impact. -// NOTE: The notable-breaking-changes tagged regions are re-used in the -// Installation and Upgrade Guide -// tag::notable-breaking-changes[] [discrete] [[breaking_85_rest_api_changes]] ==== REST API changes @@ -50,7 +47,6 @@ send unrecognized actions to {es}. Ensure your application only sends items with type `create`, `update`, `index` or `delete` to the bulk API. ==== -// end::notable-breaking-changes[] [discrete] [[deprecated-8.5]] diff --git a/docs/reference/migration/migrate_8_6.asciidoc b/docs/reference/migration/migrate_8_6.asciidoc index 60d2af03cbf66..80c0ece8c1e37 100644 --- a/docs/reference/migration/migrate_8_6.asciidoc +++ b/docs/reference/migration/migrate_8_6.asciidoc @@ -13,10 +13,7 @@ See also <> and <>. [[breaking-changes-8.6]] === Breaking changes -// tag::notable-breaking-changes[] There are no breaking changes in {es} 8.6. -// end::notable-breaking-changes[] - [discrete] [[deprecated-8.6]] diff --git a/docs/reference/migration/migrate_8_7.asciidoc b/docs/reference/migration/migrate_8_7.asciidoc index 0545a458d2abb..2061743e1be4a 100644 --- a/docs/reference/migration/migrate_8_7.asciidoc +++ b/docs/reference/migration/migrate_8_7.asciidoc @@ -18,9 +18,7 @@ and prevent them from operating normally. Before upgrading to 8.7, review these changes and take the described steps to mitigate the impact. -// tag::notable-breaking-changes[] There are no notable breaking changes in {es} 8.7. -// end::notable-breaking-changes[] But there are some less critical breaking changes. [discrete] diff --git a/docs/reference/migration/migrate_8_8.asciidoc b/docs/reference/migration/migrate_8_8.asciidoc index 85f934111ed79..22c5ae2a33750 100644 --- a/docs/reference/migration/migrate_8_8.asciidoc +++ b/docs/reference/migration/migrate_8_8.asciidoc @@ -14,10 +14,7 @@ See also <> and <>. [[breaking-changes-8.8]] === Breaking changes -// tag::notable-breaking-changes[] There are no breaking changes in {es} 8.8. -// end::notable-breaking-changes[] - [discrete] [[deprecated-8.8]] diff --git a/docs/reference/migration/migrate_8_9.asciidoc b/docs/reference/migration/migrate_8_9.asciidoc index 446ede6276110..e2f54dc58bfe5 100644 --- a/docs/reference/migration/migrate_8_9.asciidoc +++ b/docs/reference/migration/migrate_8_9.asciidoc @@ -9,14 +9,27 @@ your application to {es} 8.9. See also <> and <>. -coming::[8.9.0] - - [discrete] [[breaking-changes-8.9]] === Breaking changes -// tag::notable-breaking-changes[] -There are no breaking changes in {es} 8.9. -// end::notable-breaking-changes[] +The following changes in {es} 8.9 might affect your applications +and prevent them from operating normally. +Before upgrading to 8.9, review these changes and take the described steps +to mitigate the impact. + +[discrete] +[[breaking_89_rest_api_changes]] +==== REST API changes + +[[switch_tdigeststate_to_use_hybriddigest_by_default]] +.Switch TDigestState to use `HybridDigest` by default +[%collapsible] +==== +*Details* + +The default implementation for TDigest in percentile calculations switches to a new internal implementation offering superior performance (2x-10x speedup), at a very small accuracy penalty for very large sample populations. + +*Impact* + +This change leads to generating slightly different results in percentile calculations. If the highest possible accuracy is desired, or it's crucial to produce exactly the same results as in previous versions, one can either set `execution_hint` to `high_accuracy` in the `tdigest` spec of a given percentile calculation, or set `search.aggs.tdigest_execution_hint` to `high_accuracy` in cluster settings to apply to all percentile queries. +==== diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 4101dd42852bc..ac8746d761e06 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -209,7 +209,12 @@ value. (string) Reserved for future use, currently set to `anomaly_detector`. `job_version`:: -(string) The version of {es} that existed on the node when the job was created. +(string) The {ml} configuration version number at which the the job was created. + +NOTE: From {es} 8.10.0, a new version number is used to +track the configuration and state changes in the {ml} plugin. This new +version number is decoupled from the product version and will increment +independently. The `job_version` value represents the new version number. `model_snapshot_id`:: (string) diff --git a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc index 5f9eef088ee04..0132f57d412ef 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc @@ -18,7 +18,7 @@ Retrieves information about model snapshots. [[ml-get-snapshot-prereqs]] == {api-prereq-title} -Requires the `monitor_ml` cluster privilege. This privilege is included in the +Requires the `monitor_ml` cluster privilege. This privilege is included in the `machine_learning_user` built-in role. [[ml-get-snapshot-path-parms]] @@ -42,7 +42,7 @@ specifying `*` as the snapshot ID, or by omitting the snapshot ID. == {api-query-parms-title} `desc`:: - (Optional, Boolean) If true, the results are sorted in descending order. + (Optional, Boolean) If true, the results are sorted in descending order. Defaults to `false`. `end`:: @@ -80,7 +80,7 @@ You can also specify the query parameters in the request body; the exception are (Optional, integer) Skips the specified number of snapshots. Defaults to `0`. `size`::: -(Optional, integer) Specifies the maximum number of snapshots to obtain. +(Optional, integer) Specifies the maximum number of snapshots to obtain. Defaults to `100`. ==== @@ -105,7 +105,13 @@ properties: (date) The timestamp of the latest bucket result. `min_version`:: -(string) The minimum version required to be able to restore the model snapshot. +(string) The minimum {ml} configuration version number required to be +able to restore the model snapshot. + +NOTE: From {es} 8.10.0, a new version number is used to +track the configuration and state changes in the {ml} plugin. This new +version number is decoupled from the product version and will increment +independently. The `min_version` value represents the new version number. //Begin model_size_stats `model_size_stats`:: diff --git a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc index 78b366565fbdd..3f09b292dc2d2 100644 --- a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc @@ -7,6 +7,11 @@ Upgrades an {anomaly-detect} model snapshot to the latest major version. +NOTE: From {es} 8.10.0, a new version number is used to +track the configuration and state changes in the {ml} plugin. This new +version number is decoupled from the product version and will increment +independently. + [[ml-upgrade-job-model-snapshot-request]] == {api-request-title} @@ -15,7 +20,7 @@ Upgrades an {anomaly-detect} model snapshot to the latest major version. [[ml-upgrade-job-model-snapshot-prereqs]] == {api-prereq-title} -* Requires the `manage_ml` cluster privilege. This privilege is included in the +* Requires the `manage_ml` cluster privilege. This privilege is included in the `machine_learning_admin` built-in role. * The upgraded snapshot must have a version matching the previous major version. * The upgraded snapshot must NOT be the current {anomaly-job} snapshot. diff --git a/docs/reference/ml/anomaly-detection/ml-delayed-data-detection.asciidoc b/docs/reference/ml/anomaly-detection/ml-delayed-data-detection.asciidoc index 60b3b5b163667..a5a7b01f095ac 100644 --- a/docs/reference/ml/anomaly-detection/ml-delayed-data-detection.asciidoc +++ b/docs/reference/ml/anomaly-detection/ml-delayed-data-detection.asciidoc @@ -16,6 +16,12 @@ if it is set too high, analysis drifts farther away from real-time. The balance that is struck depends upon each use case and the environmental factors of the cluster. +IMPORTANT: If you get an error that says +`Datafeed missed XXXX documents due to ingest latency`, consider increasing +the value of `query_delay'. If it doesn't help, investigate the ingest latency and its +cause. You can do this by comparing event and ingest timestamps. High latency +is often caused by bursts of ingested documents, misconfiguration of the ingest +pipeline, or misalignment of system clocks. == Why worry about delayed data? If data are delayed randomly (and consequently are missing from analysis), the diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 329a55fdbb88c..cda2a52818147 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -419,6 +419,10 @@ of the most recent snapshot for this job. Valid values range from `0` to jobs created before version 7.8.0, the default value matches `model_snapshot_retention_days`. For more information, refer to {ml-docs}/ml-ad-run-jobs.html#ml-ad-model-snapshots[Model snapshots]. +NOTE: From {es} 8.10.0, a new version number is used to +track the configuration and state changes in the {ml} plugin. This new +version number is decoupled from the product version and will increment +independently. end::daily-model-snapshot-retention-after-days[] tag::data-description[] @@ -955,7 +959,7 @@ BERT-style tokenization is to be performed with the enclosed settings. end::inference-config-nlp-tokenization-bert[] tag::inference-config-nlp-tokenization-bert-ja[] -experimental:[] BERT-style tokenization for Japanese text is to be performed +experimental:[] BERT-style tokenization for Japanese text is to be performed with the enclosed settings. end::inference-config-nlp-tokenization-bert-ja[] @@ -1125,6 +1129,10 @@ The field that is added to incoming documents to contain the inference prediction. Defaults to `predicted_value`. end::inference-config-results-field[] +tag::inference-config-mask-token[] +The string/token which will be removed from incoming documents and replaced with the inference prediction(s). In a response, this field contains the mask token for the specified model/tokenizer. Each model and tokenizer has a predefined mask token which cannot be changed. Thus, it is recommended not to set this value in requests. However, if this field is present in a request, its value must match the predefined value for that model/tokenizer, otherwise the request will fail. +end::inference-config-mask-token[] + tag::inference-config-results-field-processor[] The field that is added to incoming documents to contain the inference prediction. Defaults to the `results_field` value of the {dfanalytics-job} that was @@ -1407,6 +1415,13 @@ ensure that high resource usage is a conscious decision. If you have jobs that are expected to analyze high cardinality fields, you will likely need to use a higher value. + +-- +NOTE: From {es} 8.10.0, a new version number is used to +track the configuration and state changes in the {ml} plugin. This new +version number is decoupled from the product version and will increment +independently. +-- ++ If you specify a number instead of a string, the units are assumed to be MiB. Specifying a string is recommended for clarity. If you specify a byte size unit of `b` or `kb` and the number does not equate to a discrete number of megabytes, @@ -1852,6 +1867,11 @@ tokenizer in old versions of the product (before 6.2). `ml_classic` was the default categorization tokenizer in versions 6.2 to 7.13, so if you need categorization identical to the default for jobs created in these versions, specify `"tokenizer": "ml_classic"` in your `categorization_analyzer`. + +NOTE: From {es} 8.10.0, a new version number is used to +track the configuration and state changes in the {ml} plugin. This new +version number is decoupled from the product version and will increment +independently. end::tokenizer[] tag::total-by-field-count[] diff --git a/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc index c64423fe0d3c4..648595a83e8f9 100644 --- a/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc @@ -166,7 +166,6 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] (string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-top-classes-results-field] ====== - `fill_mask`:::: (Optional, object) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-fill-mask] @@ -174,6 +173,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-fill-mask] .Properties of fill_mask inference [%collapsible%open] ====== +`mask_token`:::: +(Optional, string) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-mask-token] + `tokenization`:::: (Optional, object) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] @@ -1426,7 +1429,12 @@ A comma delimited string of tags. A trained model can have many tags, or none. `version`:: (string) -The {es} version number in which the trained model was created. +The {ml} configuration version number at which the trained model was created. + +NOTE: From {es} 8.10.0, a new version number is used to +track the configuration and state changes in the {ml} plugin. This new +version number is decoupled from the product version and will increment +independently. The `version` value represents the new version number. ==== diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc index d517d215ab177..85de1fb7d4105 100644 --- a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc +++ b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc @@ -61,14 +61,15 @@ NLP models, the field name is `text_field`. //Begin inference_config `inference_config`:: -(Required, object) +(Optional, object) The default configuration for inference. This can be: `regression`, `classification`, `fill_mask`, `ner`, `question_answering`, `text_classification`, `text_embedding` or `zero_shot_classification`. If `regression` or `classification`, it must match the `target_type` of the underlying `definition.trained_model`. If `fill_mask`, `ner`, `question_answering`, `text_classification`, or `text_embedding`; the -`model_type` must be `pytorch`. +`model_type` must be `pytorch`. If not specified, the `inference_config` +from when the model was created is used. + .Properties of `inference_config` [%collapsible%open] diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc index 098da98420e5f..acaf2dea056fd 100644 --- a/docs/reference/modules/cluster/misc.asciidoc +++ b/docs/reference/modules/cluster/misc.asciidoc @@ -74,6 +74,10 @@ example, a cluster with a `cluster.max_shards_per_node` setting of `100` and three data nodes has a shard limit of 300. If the cluster already contains 296 shards, {es} rejects any request that adds five or more shards to the cluster. +Note that if `cluster.max_shards_per_node` is set to a higher value than the +default, the limits for <> and +<> might also require adjustment. + Notice that frozen shards have their own independent limit. -- diff --git a/docs/reference/modules/cluster/remote-clusters-settings.asciidoc b/docs/reference/modules/cluster/remote-clusters-settings.asciidoc index 5a699d79025bd..ff16666d52172 100644 --- a/docs/reference/modules/cluster/remote-clusters-settings.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-settings.asciidoc @@ -64,6 +64,20 @@ mode are described separately. are `deflate` and `lz4`. If unset, the global `transport.compression_scheme` is used as the fallback setting. + +// TODO: fix the link to new page of API key based remote clusters + +`cluster.remote..credentials` (<>):: + +beta:[] + Per cluster setting for configuring remote clusters with the API Key based model. + This setting takes the encoded value of a + <> and must be set + in the <> on each node in the cluster. + The presence (or not) of this setting determines which model a remote cluster uses. + If present, the remote cluster uses the API key based model. + Otherwise, it uses the certificate based model. + [[remote-cluster-sniff-settings]] ==== Sniff mode remote cluster settings @@ -103,4 +117,4 @@ mode are described separately. the TLS Server Name Indication extension if <>. The TLS transport will fail to open remote connections if this field is not a valid hostname as defined by the - TLS SNI specification. \ No newline at end of file + TLS SNI specification. diff --git a/docs/reference/modules/discovery/bootstrapping.asciidoc b/docs/reference/modules/discovery/bootstrapping.asciidoc index ee854f1976678..fc43177bd74b4 100644 --- a/docs/reference/modules/discovery/bootstrapping.asciidoc +++ b/docs/reference/modules/discovery/bootstrapping.asciidoc @@ -27,15 +27,18 @@ node: if it is not possible to use the `node.name` of the node and there are multiple nodes sharing a single IP address. -IMPORTANT: After the cluster has formed, remove the `cluster.initial_master_nodes` -setting from each node's configuration. It should not be set for -master-ineligible nodes, master-eligible nodes joining an existing cluster, or -nodes which are restarting. -+ +[IMPORTANT] +==== +After the cluster has formed, remove the `cluster.initial_master_nodes` setting +from each node's configuration. It should not be set for master-ineligible +nodes, master-eligible nodes joining an existing cluster, or nodes which are +restarting. + If you leave `cluster.initial_master_nodes` in place once the cluster has formed then there is a risk that a future misconfiguration may result in bootstrapping a new cluster alongside your existing cluster. It may not be possible to recover from this situation without losing data. +==== The simplest way to create a new cluster is for you to select one of your master-eligible nodes that will bootstrap itself into a single-node cluster, diff --git a/docs/reference/modules/discovery/fault-detection.asciidoc b/docs/reference/modules/discovery/fault-detection.asciidoc index 32dfc601c330b..001763430cf4b 100644 --- a/docs/reference/modules/discovery/fault-detection.asciidoc +++ b/docs/reference/modules/discovery/fault-detection.asciidoc @@ -26,7 +26,8 @@ its data path is unhealthy then it is removed from the cluster until the data path recovers. You can control this behavior with the <>. -[[cluster-fault-detection-cluster-state-publishing]] The elected master node +[[cluster-fault-detection-cluster-state-publishing]] +The elected master node will also remove nodes from the cluster if nodes are unable to apply an updated cluster state within a reasonable time. The timeout defaults to 2 minutes starting from the beginning of the cluster state update. Refer to @@ -120,6 +121,9 @@ When it rejoins, the `NodeJoinExecutor` will log that it processed a is unexpectedly restarting, look at the node's logs to see why it is shutting down. +The <> API on the affected node will also provide some useful +information about the situation. + If the node did not restart then you should look at the reason for its departure more closely. Each reason has different troubleshooting steps, described below. There are three possible reasons: @@ -244,141 +248,17 @@ a possible cause for this kind of instability. Log messages containing If the last check failed with an exception then the exception is reported, and typically indicates the problem that needs to be addressed. If any of the -checks timed out, it may be necessary to understand the detailed sequence of -steps involved in a successful check. Here is an example of such a sequence: - -. The master's `FollowerChecker`, running on thread -`elasticsearch[master][scheduler][T#1]`, tells the `TransportService` to send -the check request message to a follower node. - -. The master's `TransportService` running on thread -`elasticsearch[master][transport_worker][T#2]` passes the check request message -onto the operating system. - -. The operating system on the master converts the message into one or more -packets and sends them out over the network. - -. Miscellaneous routers, firewalls, and other devices between the master node -and the follower node forward the packets, possibly fragmenting or -defragmenting them on the way. - -. The operating system on the follower node receives the packets and notifies -{es} that they've been received. - -. The follower's `TransportService`, running on thread -`elasticsearch[follower][transport_worker][T#3]`, reads the incoming packets. -It then reconstructs and processes the check request. Usually, the check -quickly succeeds. If so, the same thread immediately constructs a response and -passes it back to the operating system. - -. If the check doesn't immediately succeed (for example, an election started -recently) then: - -.. The follower's `FollowerChecker`, running on thread -`elasticsearch[follower][cluster_coordination][T#4]`, processes the request. It -constructs a response and tells the `TransportService` to send the response -back to the master. - -.. The follower's `TransportService`, running on thread -`elasticsearch[follower][transport_worker][T#3]`, passes the response to the -operating system. - -. The operating system on the follower converts the response into one or more -packets and sends them out over the network. - -. Miscellaneous routers, firewalls, and other devices between master and -follower forward the packets, possibly fragmenting or defragmenting them on the -way. - -. The operating system on the master receives the packets and notifies {es} -that they've been received. - -. The master's `TransportService`, running on thread -`elasticsearch[master][transport_worker][T#2]`, reads the incoming packets, -reconstructs the check response, and processes it as long as the check didn't -already time out. - -There are a lot of different things that can delay the completion of a check -and cause it to time out. Here are some examples for each step: - -. There may be a long garbage collection (GC) or virtual machine (VM) pause -after passing the check request to the `TransportService`. - -. There may be a long wait for the specific `transport_worker` thread to become -available, or there may be a long GC or VM pause before passing the check -request onto the operating system. - -. A system fault (for example, a broken network card) on the master may delay -sending the message over the network, possibly indefinitely. - -. Intermediate devices may delay, drop, or corrupt packets along the way. The -operating system for the master will wait and retransmit any unacknowledged or -corrupted packets up to `net.ipv4.tcp_retries2` times. We recommend -<> since the default represents a -very long delay. - -. A system fault (for example, a broken network card) on the follower may delay -receiving the message from the network. - -. There may be a long wait for the specific `transport_worker` thread to become -available, or there may be a long GC or VM pause during the processing of the -request on the follower. - -. There may be a long wait for the `cluster_coordination` thread to become -available, or for the specific `transport_worker` thread to become available -again. There may also be a long GC or VM pause during the processing of the -request. - -. A system fault (for example, a broken network card) on the follower may delay -sending the response from the network. - -. Intermediate devices may delay, drop, or corrupt packets along the way again, -causing retransmissions. - -. A system fault (for example, a broken network card) on the master may delay -receiving the message from the network. - -. There may be a long wait for the specific `transport_worker` thread to become -available to process the response, or a long GC or VM pause. - -To determine why follower checks are timing out, we can narrow down the reason -for the delay as follows: +checks timed out then narrow down the problem as follows. -* GC pauses are recorded in the GC logs that {es} emits by default, and also -usually by the `JvmMonitorService` in the main node logs. Use these logs to -confirm whether or not GC is resulting in delays. +include::../../troubleshooting/network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-gc-vm] -* VM pauses also affect other processes on the same host. A VM pause also -typically causes a discontinuity in the system clock, which {es} will report in -its logs. +include::../../troubleshooting/network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-packet-capture-fault-detection] -* Packet captures will reveal system-level and network-level faults, especially -if you capture the network traffic simultaneously at the elected master and the -faulty node. The connection used for follower checks is not used for any other -traffic so it can be easily identified from the flow pattern alone, even if TLS -is in use: almost exactly every second there will be a few hundred bytes sent -each way, first the request by the master and then the response by the -follower. You should be able to observe any retransmissions, packet loss, or -other delays on such a connection. +include::../../troubleshooting/network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-threads] -* Long waits for particular threads to be available can be identified by taking -stack dumps (for example, using `jstack`) or a profiling trace (for example, -using Java Flight Recorder) in the few seconds leading up to a node departure. -+ By default the follower checks will time out after 30s, so if node departures are unpredictable then capture stack dumps every 15s to be sure that at least one stack dump was taken at the right time. -+ -The <> API sometimes yields useful information, but -bear in mind that this API also requires a number of `transport_worker` and -`generic` threads across all the nodes in the cluster. The API may be affected -by the very problem you're trying to diagnose. `jstack` is much more reliable -since it doesn't require any JVM threads. -+ -The threads involved in the follower checks are `transport_worker` and -`cluster_coordination` threads, for which there should never be a long wait. -There may also be evidence of long waits for threads in the {es} logs. See -<> for more information. ===== Diagnosing `ShardLockObtainFailedException` failures diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index 72eb21b1f5443..1fc95025a1eb3 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -62,6 +62,13 @@ master-eligible node. + Defaults to `9300-9400`. +`remote_cluster.port`:: +(<>, integer) +beta:[] +The port to bind for remote cluster client communication. Accepts a single value. ++ +Defaults to `9443`. + [[network-interface-values]] ==== Special values for network addresses @@ -257,6 +264,8 @@ include::http.asciidoc[] include::transport.asciidoc[] +include::remote-cluster-network.asciidoc[] + include::network/tracers.asciidoc[] include::network/threading.asciidoc[] diff --git a/docs/reference/modules/remote-cluster-network.asciidoc b/docs/reference/modules/remote-cluster-network.asciidoc new file mode 100644 index 0000000000000..ac2b4cbf65d0a --- /dev/null +++ b/docs/reference/modules/remote-cluster-network.asciidoc @@ -0,0 +1,103 @@ +[[remote-cluster-network-settings]] +==== Advanced remote cluster (API key based model) settings + +beta::[] + +Use the following advanced settings to configure the remote cluster interface (API key based model) +independently of the <>. You can also +configure both interfaces together using the <>. + +`remote_cluster_server.enabled`:: +(<>, boolean) +Determines whether the remote cluster server should be enabled. This setting must +be `true` for `remote_cluster.port` and all following remote cluster settings to +take effect. Enabling it allows the cluster to serve cross-cluster requests using +the API key based model. Defaults to `false`. + +`remote_cluster.host`:: +(<>, string) +Sets the address of this node for remote cluster server traffic. The node will bind to this +address and will also use it as its remote cluster server publish address. Accepts an IP +address, a hostname, or a <>. +Use this setting only if you require different configurations for the +remote cluster server and transport interfaces. ++ +Defaults to the address given by `transport.bind_host`. + +`remote_cluster.bind_host`:: +(<>, string) +The network address(es) to which the node should bind in order to listen for +incoming remote cluster connections. Accepts a list of IP addresses, hostnames, and +<>. Defaults to the address given by +`remote_cluster.host`. Use this setting only if you require +to bind to multiple addresses or to use different addresses for publishing and +binding, and you also require different binding configurations for the +remote cluster server and transport interfaces. + +`remote_cluster.publish_host`:: +(<>, string) +The network address at which the node can be contacted by other nodes. Accepts +an IP address, a hostname, or a <>. +Defaults to the address given by `remote_cluster.host`. +Use this setting only if you require to bind to multiple addresses or to use +different addresses for publishing and binding, and you also require different +binding configurations for the remote cluster server and transport interfaces. + +`remote_cluster.publish_port`:: +(<>, integer) +The port of the <>. Set this parameter only if you need the publish port to be +different from `remote_cluster.port`. Defaults to the port assigned via +`remote_cluster.port`. + +`remote_cluster.tcp.keep_alive`:: +(<>, boolean) +Configures the `SO_KEEPALIVE` option for remote cluster sockets, which determines +whether they send TCP keepalive probes. Defaults to `transport.tcp.keep_alive`. + +`remote_cluster.tcp.keep_idle`:: +(<>, integer) +Configures the `TCP_KEEPIDLE` option for transport sockets, which determines +the time in seconds that a connection must be idle before starting to send TCP +keepalive probes. Defaults to `transport.tcp.keep_idle` if set, or the system +default otherwise. This value cannot exceed `300` seconds. In cases where the +system default is higher than `300`, the value is automatically lowered to +`300`. Only applicable on Linux and macOS. + +`remote_cluster.tcp.keep_interval`:: +(<>, integer) +Configures the `TCP_KEEPINTVL` option for transport sockets, which determines +the time in seconds between sending TCP keepalive probes. Defaults to +`transport.tcp.keep_interval` if set, or the system default otherwise. This value +cannot exceed `300` seconds. In cases where the system default is higher than +`300`, the value is automatically lowered to `300`. Only applicable on Linux +and macOS. + +`remote_cluster.tcp.keep_count`:: +(<>, integer) +Configures the `TCP_KEEPCNT` option for transport sockets, which determines the +number of unacknowledged TCP keepalive probes that may be sent on a connection +before it is dropped. Defaults to `transport.tcp.keep_count` if set, or the +system default otherwise. Only applicable on Linux and macOS. + +`remote_cluster.tcp.no_delay`:: +(<>, boolean) +Configures the `TCP_NODELAY` option on transport sockets, which determines +whether {wikipedia}/Nagle%27s_algorithm[TCP no delay] is enabled. Defaults to +`transport.tcp.no_delay`. + +`remote_cluster.tcp.reuse_address`:: +(<>, boolean) +Configures the `SO_REUSEADDR` option for network sockets, which determines +whether the address can be reused or not. Defaults to +`transport.tcp.reuse_address`. + +`remote_cluster.tcp.send_buffer_size`:: +(<>, <>) +The size of the TCP send buffer for transport traffic. Defaults to +`transport.tcp.send_buffer_size`. + +`remote_cluster.tcp.receive_buffer_size`:: +(<>, <>) +The size of the TCP receive buffer for transport traffic. Defaults to +`transport.tcp.receive_buffer_size`. diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc index 4047bb98c721a..4ca0b1339ac40 100644 --- a/docs/reference/modules/threadpool.asciidoc +++ b/docs/reference/modules/threadpool.asciidoc @@ -13,18 +13,24 @@ There are several thread pools, but the important ones include: [[search-threadpool]] `search`:: - For count/search/suggest operations. Thread pool type is - `fixed` with a size of `int((`<>`pass:[ * ]3) / 2) + 1`, and queue_size of `1000`. +`search_worker`:: + For the heavy workload of count/search operations that may be executed concurrently + across segments within the same shard when possible. Thread pool type is `fixed` + with a size of `int((`<>`pass:[ * ]3) / 2) + 1`, and unbounded queue_size . + [[search-throttled]]`search_throttled`:: For count/search/suggest/get operations on `search_throttled indices`. Thread pool type is `fixed` with a size of `1`, and queue_size of `100`. `search_coordination`:: For lightweight search-related coordination operations. Thread pool type is - `fixed` with a size of a max of `min(5, (`<>`) / 2)`, and queue_size of `1000`. + `fixed` with a size of `(`<>`) / 2`, + and queue_size of `1000`. `get`:: For get operations. Thread pool type is diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc index cccf399358a32..eb840508aba39 100644 --- a/docs/reference/query-dsl/match-query.asciidoc +++ b/docs/reference/query-dsl/match-query.asciidoc @@ -65,6 +65,17 @@ See <> for an example. -- +`boost`:: ++ +-- +(Optional, float) Floating point number used to decrease or increase the +<> of the query. Defaults to `1.0`. + +Boost values are relative to the default value of `1.0`. A boost value between +`0` and `1.0` decreases the relevance score. A value greater than `1.0` +increases the relevance score. +-- + `fuzziness`:: (Optional, string) Maximum edit distance allowed for matching. See <> for valid values and more information. See <> diff --git a/docs/reference/query-dsl/rule-query.asciidoc b/docs/reference/query-dsl/rule-query.asciidoc new file mode 100644 index 0000000000000..cf79a564f81a3 --- /dev/null +++ b/docs/reference/query-dsl/rule-query.asciidoc @@ -0,0 +1,87 @@ +[role="xpack"] +[[query-dsl-rule-query]] +=== Rule query +++++ +Rule +++++ + +preview::[] + +Applies <> to the query before returning results. +This feature is used to promote documents in the manner of a <> based on matching defined rules. +If no matching query rules are defined, the "organic" matches for the query are returned. + +[NOTE] +==== +To use the rule query, you first need a defined set of query rules. +Use the <> to create and manage query rules. +For more information and examples see <>. +==== + +==== Example request + +//// + +[source,console] +-------------------------------------------------- +PUT _query_rules/my-ruleset +{ + "rules": [ + { + "rule_id": "my-rule1", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "user_query", + "values": ["puggles"] + } + ], + "actions": { + "ids": [ "id1" ] + } + } + ] +} +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _query_rules/my-ruleset +-------------------------------------------------- +// TEARDOWN + +//// + +[source,console] +-------------------------------------------------- +GET /_search +{ + "query": { + "rule_query": { + "match_criteria": { + "user_query": "pugs" + }, + "ruleset_id": "my-ruleset", + "organic": { + "match": { + "description": "puggles" + } + } + } + } +} +-------------------------------------------------- + +[[rule-query-top-level-parameters]] +==== Top-level parameters for `rule_query` + +`ruleset_id`:: +(Required, string) A unique <> ID with query-based rules to match and apply as applicable. +`match_criteria`:: +(Required, object) Defines the match criteria to apply to rules in the given query ruleset. +Match criteria should match the keys defined in the `criteria.metadata` field of the rule. +`organic`:: +(Required, object) Any choice of <> used to return results, that may be modified by matching query rules. +If no query rules are matched and applied, this query will be executed with no modification. diff --git a/docs/reference/query-dsl/special-queries.asciidoc b/docs/reference/query-dsl/special-queries.asciidoc index ca3afd94092d7..a6d35d4f9b707 100644 --- a/docs/reference/query-dsl/special-queries.asciidoc +++ b/docs/reference/query-dsl/special-queries.asciidoc @@ -34,6 +34,9 @@ A query that accepts other queries as json or yaml string. <>:: A query that promotes selected documents over others matching a given query. +<>:: +A query that supports applying query-based contextual rules, defined using the <>, to a given query. + include::distance-feature-query.asciidoc[] include::mlt-query.asciidoc[] @@ -48,4 +51,6 @@ include::script-score-query.asciidoc[] include::wrapper-query.asciidoc[] -include::pinned-query.asciidoc[] \ No newline at end of file +include::pinned-query.asciidoc[] + +include::rule-query.asciidoc[] diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index 91d597b79aced..74ee80ba821a1 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -54,8 +54,8 @@ The query text you want to use for search. [discrete] -[[text-expansion-query-notes]] -=== Notes +[[text-expansion-query-example]] +=== Example The following is an example of the `text_expansion` query that references the ELSER model to perform semantic search. For a more detailed description of how @@ -76,4 +76,31 @@ GET my-index/_search } } ---- -// TEST[skip: TBD] \ No newline at end of file +// TEST[skip: TBD] + +[discrete] +[[optimizing-text-expansion]] +=== Optimizing the search performance of the text_expansion query + +https://www.elastic.co/blog/faster-retrieval-of-top-hits-in-elasticsearch-with-block-max-wand[Max WAND] +is an optimization technique used by {es} to skip documents that cannot score +competitively against the current best matching documents. However, the tokens +generated by the ELSER model don't work well with the Max WAND optimization. +Consequently, enabling Max WAND can actually increase query latency for +`text_expansion`. For datasets of a significant size, disabling Max +WAND leads to lower query latencies. + +Max WAND is controlled by the +<> query parameter. Setting track_total_hits +to true forces {es} to consider all documents, resulting in lower query +latencies for the `text_expansion` query. However, other {es} queries run slower +when Max WAND is disabled. + +If you are combining the `text_expansion` query with standard text queries in a +compound search, it is recommended to measure the query performance before +deciding which setting to use. + +NOTE: The `track_total_hits` option applies to all queries in the search request +and may be optimal for some queries but not for others. Take into account the +characteristics of all your queries to determine the most suitable +configuration. diff --git a/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc b/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc new file mode 100644 index 0000000000000..dc18e2b486eb1 --- /dev/null +++ b/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc @@ -0,0 +1,73 @@ +[role="xpack"] +[[delete-query-ruleset]] +=== Delete query ruleset + +preview::[] + +++++ +Delete query ruleset +++++ + +Removes a query ruleset and its associated data. +This is a destructive action that is not recoverable. + +[[delete-query-ruleset-request]] +==== {api-request-title} + +`DELETE _query_rules/` + +[[delete-query-ruleset-prereq]] +==== {api-prereq-title} + +Requires the `manage_search_query_rules` privilege. + +[[delete-query_ruleset-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[delete-query-ruleset-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `ruleset_id` was not provided. + +`404` (Missing resources):: +No query ruleset matching `ruleset_id` could be found. + +[[delete-query-ruleset-example]] +==== {api-examples-title} + +The following example deletes the query ruleset named `my-ruleset`: + +//// +[source,console] +---- +PUT _query_rules/my-ruleset +{ + "rules": [ + { + "rule_id": "my-rule1", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "query_string", + "values": [ "marvel" ] + } + ], + "actions": { + "ids": ["id1"] + } + } + ] +} +---- +// TESTSETUP +//// + +[source,console] +---- +DELETE _query_rules/my-ruleset/ +---- diff --git a/docs/reference/query-rules/apis/get-query-ruleset.asciidoc b/docs/reference/query-rules/apis/get-query-ruleset.asciidoc new file mode 100644 index 0000000000000..e253ccda42376 --- /dev/null +++ b/docs/reference/query-rules/apis/get-query-ruleset.asciidoc @@ -0,0 +1,157 @@ +[role="xpack"] +[[get-query-ruleset]] +=== Get query ruleset + +preview::[] + +++++ +Get query ruleset +++++ + +Retrieves information about a query ruleset. + +[[get-query-ruleset-request]] +==== {api-request-title} + +`GET _query_rules/` + +[[get-query-ruleset-prereq]] +==== {api-prereq-title} + +Requires the `manage_search_query_rules` privilege. + +[[get-query-ruleset-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[get-query-ruleset-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `ruleset_id` was not provided. + +`404` (Missing resources):: +No query ruleset matching `ruleset_id` could be found. + +[[get-query-ruleset-example]] +==== {api-examples-title} + +The following example gets the query ruleset named `my-ruleset`: + +//// + +[source,console] +-------------------------------------------------- +PUT _query_rules/my-ruleset +{ + "rules": [ + { + "rule_id": "my-rule1", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "query_string", + "values": [ "marvel" ] + } + ], + "actions": { + "ids": [ + "id1", + "id2" + ] + } + }, + { + "rule_id": "my-rule2", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "query_string", + "values": [ "dc" ] + } + ], + "actions": { + "docs": [ + { + "_index": "index1", + "_id": "id3" + }, + { + "_index": "index2", + "_id": "id4" + } + ] + } + } + ] +} +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _query_rules/my-ruleset +-------------------------------------------------- +// TEARDOWN + +//// + +[source,console] +---- +GET _query_rules/my-ruleset/ +---- + +A sample response: + +[source,console-result] +---- +{ + "ruleset_id": "my-ruleset", + "rules": [ + { + "rule_id": "my-rule1", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "query_string", + "values": [ "marvel" ] + } + ], + "actions": { + "ids": [ + "id1", + "id2" + ] + } + }, + { + "rule_id": "my-rule2", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "query_string", + "values": [ "dc" ] + } + ], + "actions": { + "docs": [ + { + "_index": "index1", + "_id": "id3" + }, + { + "_index": "index2", + "_id": "id4" + } + ] + } + } + ] +} +---- diff --git a/docs/reference/query-rules/apis/index.asciidoc b/docs/reference/query-rules/apis/index.asciidoc new file mode 100644 index 0000000000000..2ab5be069a6d5 --- /dev/null +++ b/docs/reference/query-rules/apis/index.asciidoc @@ -0,0 +1,30 @@ +[[query-rules-apis]] +== Query rules APIs + +preview::[] + +++++ +Query rules APIs +++++ + +--- + +<> allow you to configure per-query rules that are applied at query time to queries that match the specific rule. +Query rules are organized into _rulesets_, collections of query rules that are matched against incoming queries. +Query rules are applied using the <>. + +If a query matches one or more rules in the ruleset, the query is re-written to apply the rules before searching. +This allows pinning documents for only queries that match a specific term. + +Use the following APIs to manage query rulesets: + +* <> +* <> +* <> +* <> + +include::put-query-ruleset.asciidoc[] +include::get-query-ruleset.asciidoc[] +include::list-query-rulesets.asciidoc[] +include::delete-query-ruleset.asciidoc[] + diff --git a/docs/reference/query-rules/apis/list-query-rulesets.asciidoc b/docs/reference/query-rules/apis/list-query-rulesets.asciidoc new file mode 100644 index 0000000000000..4eba65dceeea4 --- /dev/null +++ b/docs/reference/query-rules/apis/list-query-rulesets.asciidoc @@ -0,0 +1,207 @@ +[role="xpack"] +[[list-query-rulesets]] +=== List query rulesets + +preview::[] + +++++ +List query rulesets +++++ + +Returns information about all stored query rulesets. +Summary information on the number of rules per ruleset will be returned, and full details can be returned with the <> command. + +[[list-query-rules-request]] +==== {api-request-title} + +`GET _query_rules/` + +[[list-query-rules-prereq]] +==== {api-prereq-title} + +Requires the `manage_search_query_rules` privilege. + +[[list-query-rules-path-params]] +==== {api-path-parms-title} + +`size`:: +(Optional, integer) Maximum number of results to retrieve. + +`from`:: +(Optional, integer) The offset from the first result to fetch. + +[[list-query-rules-example]] +==== {api-examples-title} + +The following example lists all configured query rulesets: + +//// + +[source,console] +-------------------------------------------------- +PUT _query_rules/ruleset-1 +{ + "rules": [ + { + "rule_id": "rule-1", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "query_string", + "values": [ "puggles" ] + } + ], + "actions": { + "ids": ["id1"] + } + } + ] +} + +PUT _query_rules/ruleset-2 +{ + "rules": [ + { + "rule_id": "rule-1", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "query_string", + "values": [ "puggles" ] + } + ], + "actions": { + "ids": ["id1"] + } + }, + { + "rule_id": "rule-2", + "type": "pinned", + "criteria": [ + { + "type": "fuzzy", + "metadata": "query_string", + "values": [ "pugs" ] + } + ], + "actions": { + "ids": ["id2"] + } + } + ] +} + +PUT _query_rules/ruleset-3 +{ + "rules": [ + { + "rule_id": "rule-1", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "query_string", + "values": [ "puggles" ] + } + ], + "actions": { + "ids": ["id1"] + } + }, + { + "rule_id": "rule-2", + "type": "pinned", + "criteria": [ + { + "type": "fuzzy", + "metadata": "query_string", + "values": [ "pugs" ] + } + ], + "actions": { + "ids": ["id2"] + } + }, + { + "rule_id": "rule-3", + "type": "pinned", + "criteria": [ + { + "type": "fuzzy", + "metadata": "query_string", + "values": [ "beagles" ] + } + ], + "actions": { + "ids": ["id2"] + } + } + ] +} +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _query_rules/ruleset-1 + +DELETE _query_rules/ruleset-2 + +DELETE _query_rules/ruleset-3 + +-------------------------------------------------- +// TEARDOWN + +//// + +[source,console] +---- +GET _query_rules/ +---- + +The following example lists the first three query rulesets: + +[source,console] +---- +GET _query_rules/?from=0&size=3 +---- + +A sample response: + +[source,console-result] +---- +{ + "count": 3, + "results": [ + { + "ruleset_id": "ruleset-1", + "rule_total_count": 1, + "rule_criteria_types_counts": { + "exact": 1 + } + }, + { + "ruleset_id": "ruleset-2", + "rule_total_count": 2, + "rule_criteria_types_counts": { + "exact": 1, + "fuzzy": 1 + } + }, + { + "ruleset_id": "ruleset-3", + "rule_total_count": 3, + "rule_criteria_types_counts": { + "exact": 1, + "fuzzy": 2 + } + } + ] +} +---- +// TEST[continued] + +[NOTE] +The counts in `rule_criteria_types_counts` may be larger than the value of `rule_total_count`, because a rule may have multiple criteria. diff --git a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc new file mode 100644 index 0000000000000..e1a26a49cc525 --- /dev/null +++ b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc @@ -0,0 +1,178 @@ +[role="xpack"] +[[put-query-ruleset]] +=== Create or update query ruleset + +preview::[] + +++++ +Create or update query ruleset +++++ + +Creates or updates a query ruleset. + +[[put-query-ruleset-request]] +==== {api-request-title} + +`PUT _query_rules/` + +[[put-query-ruleset-prereqs]] +==== {api-prereq-title} + +Requires the `manage_search_query_rules` privilege. + +[role="child_attributes"] +[[put-query-ruleset-request-body]] +(Required, object) +Contains parameters for a query ruleset: + +==== {api-request-body-title} +`rules`:: +(Required, array of objects) +The specific rules included in this query ruleset. + +Each rule must have the following information: + +- `rule_id` (Required, string) + A unique identifier for this rule. +- `type` (Required, string) + The type of rule. At this time only `pinned` query rule types are allowed. +- `criteria` (Required, array of objects) + The criteria that must be met for the rule to be applied. If multiple criteria are specified for a rule, all criteria must be met for the rule to be applied. +- `actions` (Required, object) + The actions to take when the rule is matched. The format of this action depends on the rule type. + +Criteria must have the following information: + +- `type` (Required, string) + The type of criteria. The following criteria types are supported: ++ +-- +- `exact` +Only exact matches meet the criteria defined by the rule. +Applicable for string or numerical values. +- `fuzzy` +Exact matches or matches within the allowed {wikipedia}/Levenshtein_distance[Levenshtein Edit Distance] meet the criteria defined by the rule. +Only applicable for string values. +- `prefix` +Matches that start with this value meet the criteria defined by the rule. +Only applicable for string values. +- `suffix` +Matches that end with this value meet the criteria defined by the rule. +Only applicable for string values. +- `contains` +Matches that contain this value anywhere in the field meet the criteria defined by the rule. +Only applicable for string values. +- `lt` +Matches with a value less than this value meet the criteria defined by the rule. +Only applicable for numerical values. +- `lte` +Matches with a value less than or equal to this value meet the criteria defined by the rule. +Only applicable for numerical values. +- `gt` +Matches with a value greater than this value meet the criteria defined by the rule. +Only applicable for numerical values. +- `gte` +Matches with a value greater than or equal to this value meet the criteria defined by the rule. +Only applicable for numerical values. +- `always` +Matches all queries, regardless of input. +-- +- `metadata` (Optional, string) + The metadata field to match against. Required for all criteria types except `global`. +- `values` (Optional, array of strings) + The values to match against the metadata field. Only one value must match for the criteria to be met. Required for all criteria types except `global`. + +Actions depend on the rule type. +For `pinned` rules, actions follow the format specified by the <>. +The following actions are allowed: + +- `ids` (Optional, array of strings) + The The unique <> of the documents to pin. + Only one of `ids` or `docs` may be specified, and at least one must be specified. +- `docs` (Optional, array of objects) + The documents to pin. Only one of `ids` or `docs` may be specified, and at least one must be specified. + You can specify the following attributes for each document: ++ +-- +- `_index` (Required, string) + The index of the document to pin. +- `_id` (Required, string) + The unique <>. +-- + +IMPORTANT: Due to limitations within <>, you can only pin documents using `ids` or `docs`, but cannot use both in single rule. It is advised to use one or the other in query rulesets, to avoid errors. Additionally, pinned queries have a maximum limit of 100 pinned hits. If multiple matching rules pin more than 100 documents, only the first 100 documents are pinned in the order they are specified in the ruleset. + +[[put-query-ruleset-example]] +==== {api-examples-title} + +The following example creates a new query ruleset called `my-ruleset`. + +Two rules are associated with `my-ruleset`: + +- `my-rule1` will pin documents with IDs `id1` and `id2` when `user.query` exactly matches `marvel` _or_ `dc` **and** `user.country` exactly matches `us`. +- `my-rule2` will pin documents from different, specified indices with IDs `id3` and `id4` when the `query_string` fuzzily matches `comic`. + +[source,console] +---- +PUT _query_rules/my-ruleset +{ + "rules": [ + { + "rule_id": "my-rule1", + "type": "pinned", + "criteria": [ + { + "type": "contains", + "metadata": "user.query", + "values": [ "marvel", "dc" ] + }, + { + "type": "exact", + "metadata": "user.country", + "values": [ "us" ] + } + ], + "actions": { + "ids": [ + "id1", + "id2" + ] + } + }, + { + "rule_id": "my-rule2", + "type": "pinned", + "criteria": [ + { + "type": "fuzzy", + "metadata": "query_string", + "values": [ "comic" ] + } + ], + "actions": { + "docs": [ + { + "_index": "index1", + "_id": "id3" + }, + { + "_index": "index2", + "_id": "id4" + } + ] + } + } + ] +} +---- +// TESTSETUP + +////////////////////////// + +[source,console] +-------------------------------------------------- +DELETE _query_rules/my-ruleset +-------------------------------------------------- +// TEARDOWN + +////////////////////////// diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 47140e93ad980..0fd1cfdf7dfb7 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -1885,49 +1885,68 @@ Refer to <>. [role="exclude",id="dlm-delete-lifecycle"] === Delete the lifecycle of a data stream +ifeval::["{release-state}"=="unreleased"] Refer to <>. +endif::[] +ifeval::["{release-state}"=="unreleased"] +This feature has not been released yet +endif::[] + [role="exclude",id="dlm-explain-lifecycle"] === Explain the lifecycle of a data stream +ifeval::["{release-state}"=="unreleased"] Refer to <>. +endif::[] +ifeval::["{release-state}"=="unreleased"] +This feature has not been released yet +endif::[] + [role="exclude",id="dlm-get-lifecycle"] === Get the lifecycle of a data stream +ifeval::["{release-state}"=="unreleased"] Refer to <>. +endif::[] +ifeval::["{release-state}"=="unreleased"] +This feature has not been released yet +endif::[] + [role="exclude",id="dlm-put-lifecycle"] === Update the lifecycle of a data stream +ifeval::["{release-state}"=="unreleased"] Refer to <>. -[role="exclude",id="get-synonym-rule"] -=== Get synonym rule API - -coming::[8.10.0] +endif::[] +ifeval::["{release-state}"=="unreleased"] +This feature has not been released yet +endif::[] [role="exclude",id="get-synonyms"] -=== Get synonym set API +=== Get synonyms set API -coming::[8.10.0] +See <> -[role="exclude",id="list-synonyms"] -=== List synonym sets API +[role="exclude",id="delete-synonyms"] +=== Delete synonyms sets API -coming::[8.10.0] +See <> -[role="exclude",id="put-synonym-rule"] -=== Create or update synonym rule API +[role="exclude",id="put-synonyms"] +=== Create or delete synonyms sets API -coming::[8.10.0] +See <> -[role="exclude",id="delete-synonyms"] -=== Delete synonym sets API +[role="exclude",id="remote-clusters-api-key"] +=== Add remote clusters using API key authentication -coming::[8.10.0] +coming::[8.10] -[role="exclude",id="put-synonyms"] -=== Create or update synonym sets API +[role="exclude",id="remote-clusters-cert"] +=== Add remote clusters using TLS certificate authentication -coming::[8.10.0] \ No newline at end of file +coming::[8.10] diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index bbdf48928307b..6f39135db2d2b 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -6,8 +6,10 @@ This section summarizes the changes in each release. +* <> * <> * <> +* <> * <> * <> * <> @@ -45,8 +47,10 @@ This section summarizes the changes in each release. -- +include::release-notes/8.11.0.asciidoc[] include::release-notes/8.10.0.asciidoc[] include::release-notes/8.9.0.asciidoc[] +include::release-notes/8.8.2.asciidoc[] include::release-notes/8.8.1.asciidoc[] include::release-notes/8.8.0.asciidoc[] include::release-notes/8.7.1.asciidoc[] diff --git a/docs/reference/release-notes/8.11.0.asciidoc b/docs/reference/release-notes/8.11.0.asciidoc new file mode 100644 index 0000000000000..a5c8c8c73128e --- /dev/null +++ b/docs/reference/release-notes/8.11.0.asciidoc @@ -0,0 +1,8 @@ +[[release-notes-8.11.0]] +== {es} version 8.11.0 + +coming[8.11.0] + +Also see <>. + + diff --git a/docs/reference/release-notes/8.8.2.asciidoc b/docs/reference/release-notes/8.8.2.asciidoc new file mode 100644 index 0000000000000..d05f837743b52 --- /dev/null +++ b/docs/reference/release-notes/8.8.2.asciidoc @@ -0,0 +1,43 @@ +[[release-notes-8.8.2]] +== {es} version 8.8.2 + +coming[8.8.2] + +Also see <>. + +[[bug-8.8.2]] +[float] +=== Bug fixes + +Aggregations:: +* Fix iteration of empty percentiles throwing Null Pointer Exception {es-pull}96668[#96668] (issue: {es-issue}96626[#96626]) + +Health:: +* Uses `ClusterSettings` instead of Node `Settings` in `HealthMetadataService` {es-pull}96843[#96843] (issue: {es-issue}96219[#96219]) + +Ingest Node:: +* Support dotted field notations in the reroute processor {es-pull}96243[#96243] + +Machine Learning:: +* Ensure NLP model inference queue is always cleared after shutdown or failure {es-pull}96738[#96738] + +SQL:: +* Fix translation of queries involving Version vals {es-pull}96540[#96540] (issue: {es-issue}96509[#96509]) + +Search:: +* Increase concurrent request of opening point-in-time {es-pull}96782[#96782] + +TSDB:: +* The get data stream api incorrectly prints warning log for upgraded tsdb data streams {es-pull}96606[#96606] + +[[enhancement-8.8.2]] +[float] +=== Enhancements + +TSDB:: +* Change rollup thread pool settings {es-pull}96821[#96821] (issue: {es-issue}96758[#96758]) + +Transform:: +* Adding null check to fix potential NPE {es-pull}96785[#96785] (issue: {es-issue}96781[#96781]) + + diff --git a/docs/reference/release-notes/8.9.0.asciidoc b/docs/reference/release-notes/8.9.0.asciidoc index fd2dd31fb321e..2b7b143c268dc 100644 --- a/docs/reference/release-notes/8.9.0.asciidoc +++ b/docs/reference/release-notes/8.9.0.asciidoc @@ -1,8 +1,284 @@ [[release-notes-8.9.0]] == {es} version 8.9.0 -coming[8.9.0] - Also see <>. +[[known-issues-8.9.0]] +[float] +=== Known issues + +* Question Answering fails on long input text. If the context supplied to the +task is longer than the model's max_sequence_length and truncate is set to none +then inference fails with the message `question answering result has +invalid dimension`. (issue: {es-issue}97917[#97917]) + +[[breaking-8.9.0]] +[float] +=== Breaking changes + +Aggregations:: +* Switch TDigestState to use `HybridDigest` by default {es-pull}96904[#96904] + +[[bug-8.9.0]] +[float] +=== Bug fixes + +Allocation:: +* Attempt to fix delay allocation {es-pull}95921[#95921] +* Fix NPE in Desired Balance API {es-pull}97775[#97775] +* Fix autoexpand during node replace {es-pull}96281[#96281] + +Authorization:: +* Resolving wildcard application names without prefix query {es-pull}96479[#96479] (issue: {es-issue}96465[#96465]) + +CRUD:: +* Fix `retry_on_conflict` parameter in update API to not retry indefinitely {es-pull}96262[#96262] +* Handle failure in `TransportUpdateAction#handleUpdateFailureWithRetry` {es-pull}97290[#97290] (issue: {es-issue}97286[#97286]) + +Cluster Coordination:: +* Avoid `getStateForMasterService` where possible {es-pull}97304[#97304] +* Become candidate on publication failure {es-pull}96490[#96490] (issue: {es-issue}96273[#96273]) +* Fix cluster settings update task acknowledgment {es-pull}97111[#97111] + +Data streams:: +* Accept timestamp as object at root level {es-pull}97401[#97401] + +Geo:: +* Fix bug when creating empty `geo_lines` {es-pull}97509[#97509] (issue: {es-issue}97311[#97311]) +* Fix time-series geo_line to include reduce phase in MergedGeoLines {es-pull}96953[#96953] (issue: {es-issue}96983[#96983]) +* Support for Byte and Short as vector tiles features {es-pull}97619[#97619] (issue: {es-issue}97612[#97612]) + +ILM+SLM:: +* Limit the details field length we store for each SLM invocation {es-pull}97038[#97038] (issue: {es-issue}96918[#96918]) + +Infra/CLI:: +* Initialise ES logging in CLI {es-pull}97353[#97353] (issue: {es-issue}97350[#97350]) + +Infra/Core:: +* Capture max processors in static init {es-pull}97119[#97119] (issue: {es-issue}97088[#97088]) +* Interpret microseconds cpu stats from cgroups2 properly as nanos {es-pull}96924[#96924] (issue: {es-issue}96089[#96089]) + +Infra/Logging:: +* Add slf4j-nop in order to prevent startup warnings {es-pull}95459[#95459] + +Infra/REST API:: +* Fix tchar pattern in `RestRequest` {es-pull}96406[#96406] + +Infra/Scripting:: +* Fix Painless method lookup over unknown super interfaces {es-pull}97062[#97062] (issue: {es-issue}97022[#97022]) + +Infra/Settings:: +* Enable validation for `versionSettings` {es-pull}95874[#95874] (issue: {es-issue}95873[#95873]) + +Ingest Node:: +* Fixing `DateProcessor` when the format is `epoch_millis` {es-pull}95996[#95996] +* Fixing `GeoIpDownloaderStatsAction$NodeResponse` serialization by defensively copying inputs {es-pull}96777[#96777] (issue: {es-issue}96438[#96438]) +* Trim field references in reroute processor {es-pull}96941[#96941] (issue: {es-issue}96939[#96939]) + +Machine Learning:: +* Catch exceptions thrown during inference and report as errors {ml-pull}2542[#2542] +* Fix `WordPiece` tokenization where stripping accents results in an empty string {es-pull}97354[#97354] +* Improve model downloader robustness {es-pull}97274[#97274] +* Prevent high memory usage by evaluating batch inference singularly {ml-pull}2538[#2538] + +Mapping:: +* Avoid stack overflow while parsing mapping {es-pull}95705[#95705] (issue: {es-issue}52098[#52098]) +* Fix mapping parsing logic to determine synthetic source is active {es-pull}97355[#97355] (issue: {es-issue}97320[#97320]) + +Ranking:: +* Fix `sub_searches` serialization bug {es-pull}97587[#97587] + +Recovery:: +* Promptly fail recovery from snapshot {es-pull}96421[#96421] (issue: {es-issue}95525[#95525]) + +Search:: +* Prevent instantiation of `top_metrics` when sub-aggregations are present {es-pull}96180[#96180] (issue: {es-issue}95663[#95663]) +* Set new providers before building `FetchSubPhaseProcessors` {es-pull}97460[#97460] (issue: {es-issue}96284[#96284]) + +Snapshot/Restore:: +* Fix blob cache races/assertion errors {es-pull}96458[#96458] +* Fix reused/recovered bytes for files that are only partially recovered from cache {es-pull}95987[#95987] (issues: {es-issue}95970[#95970], {es-issue}95994[#95994]) +* Fix reused/recovered bytes for files that are recovered from cache {es-pull}97278[#97278] (issue: {es-issue}95994[#95994]) +* Refactor `RestoreClusterStateListener` to use `ClusterStateObserver` {es-pull}96662[#96662] (issue: {es-issue}96425[#96425]) + +TSDB:: +* Error message for misconfigured TSDB index {es-pull}96956[#96956] (issue: {es-issue}96445[#96445]) +* Min score for time series {es-pull}96878[#96878] + +Task Management:: +* Improve cancellability in `TransportTasksAction` {es-pull}96279[#96279] + +Transform:: +* Improve reporting status of the transform that is about to finish {es-pull}95672[#95672] + +[[enhancement-8.9.0]] +[float] +=== Enhancements + +Aggregations:: +* Add cluster setting to `SearchExecutionContext` to configure `TDigestExecutionHint` {es-pull}96943[#96943] +* Add support for dynamic pruning to cardinality aggregations on low-cardinality keyword fields {es-pull}92060[#92060] +* Make TDigestState configurable {es-pull}96794[#96794] +* Skip `SortingDigest` when merging a large digest in `HybridDigest` {es-pull}97099[#97099] +* Support value retrieval in `top_hits` {es-pull}95828[#95828] + +Allocation:: +* Take into account `expectedShardSize` when initializing shard in simulation {es-pull}95734[#95734] + +Analysis:: +* Create `.synonyms` system index {es-pull}95548[#95548] + +Application:: +* Add template parameters to Search Applications {es-pull}95674[#95674] +* Chunk profiling stacktrace response {es-pull}96340[#96340] +* [Profiling] Add status API {es-pull}96272[#96272] +* [Profiling] Allow to upgrade managed ILM policy {es-pull}96550[#96550] +* [Profiling] Introduce ILM for K/V indices {es-pull}96268[#96268] +* [Profiling] Require POST to retrieve stacktraces {es-pull}96790[#96790] +* [Profiling] Tweak default ILM policy {es-pull}96516[#96516] +* [Search Applications] Support arrays in stored mustache templates {es-pull}96197[#96197] + +Authentication:: +* Header validator with Security {es-pull}95112[#95112] + +Authorization:: +* Add Search ALC filter index prefix to the enterprise search user {es-pull}96885[#96885] +* Ensure checking application privileges work with nested-limited roles {es-pull}96970[#96970] + +Autoscaling:: +* Add shard explain info to `ReactiveReason` about unassigned shards {es-pull}88590[#88590] (issue: {es-issue}85243[#85243]) + +DLM:: +* Add auto force merge functionality to DLM {es-pull}95204[#95204] +* Adding `data_lifecycle` to the _xpack/usage API {es-pull}96177[#96177] +* Adding `manage_data_stream_lifecycle` index privilege and expanding `view_index_metadata` for access to data stream lifecycle APIs {es-pull}95512[#95512] +* Allow for the data lifecycle and the retention to be explicitly nullified {es-pull}95979[#95979] + +Data streams:: +* Add support for `logs@custom` component template for `logs-*-* data streams {es-pull}95481[#95481] (issue: {es-issue}95469[#95469]) +* Adding ECS dynamic mappings component and applying it to logs data streams by default {es-pull}96171[#96171] (issue: {es-issue}95538[#95538]) +* Adjust ECS dynamic templates to support `subobjects: false` {es-pull}96712[#96712] +* Automatically parse log events in logs data streams, if their `message` field contains JSON content {es-pull}96083[#96083] (issue: {es-issue}95522[#95522]) +* Change default of `ignore_malformed` to `true` in `logs-*-*` data streams {es-pull}95329[#95329] (issue: {es-issue}95224[#95224]) +* Set `@timestamp` for documents in logs data streams if missing and add support for custom pipeline {es-pull}95971[#95971] (issues: {es-issue}95537[#95537], {es-issue}95551[#95551]) +* Update data streams implicit timestamp `ignore_malformed` settings {es-pull}96051[#96051] + +Engine:: +* Cache modification time of translog writer file {es-pull}95107[#95107] +* Trigger refresh when shard becomes search active {es-pull}96321[#96321] (issue: {es-issue}95544[#95544]) + +Geo:: +* Add brute force approach to `GeoHashGridTiler` {es-pull}96863[#96863] +* Asset tracking - geo_line in time-series aggregations {es-pull}94954[#94954] + +ILM+SLM:: +* Chunk the GET _ilm/policy response {es-pull}97251[#97251] (issue: {es-issue}96569[#96569]) +* Move get lifecycle API to Management thread pool and make cancellable {es-pull}97248[#97248] (issue: {es-issue}96568[#96568]) +* Reduce WaitForNoFollowersStep requests indices shard stats {es-pull}94510[#94510] + +Indices APIs:: +* Bootstrap profiling indices at startup {es-pull}95666[#95666] + +Infra/Node Lifecycle:: +* SIGTERM node shutdown type {es-pull}95430[#95430] + +Ingest Node:: +* Add mappings for enrich fields {es-pull}96056[#96056] +* Ingest: expose reroute inquiry/reset via Elastic-internal API bridge {es-pull}96958[#96958] + +Machine Learning:: +* Improved compliance with memory limitations {ml-pull}2469[#2469] +* Improve detection of calendar cyclic components with long bucket lengths {ml-pull}2493[#2493] +* Improve detection of time shifts, for example for daylight saving {ml-pull}2479[#2479] + +Mapping:: +* Allow unsigned long field to use decay functions {es-pull}96394[#96394] (issue: {es-issue}89603[#89603]) + +Ranking:: +* Add multiple queries for ranking to the search endpoint {es-pull}96224[#96224] + +Recovery:: +* Implement `StartRecoveryRequest#getDescription` {es-pull}95731[#95731] + +Search:: +* Add search shards endpoint {es-pull}94534[#94534] +* Don't generate stacktrace in `EarlyTerminationException` and `TimeExceededException` {es-pull}95910[#95910] +* Feature/speed up binary vector decoding {es-pull}96716[#96716] +* Improve brute force vector search speed by using Lucene functions {es-pull}96617[#96617] +* Include search idle info to shard stats {es-pull}95740[#95740] (issue: {es-issue}95727[#95727]) +* Integrate CCS with new `search_shards` API {es-pull}95894[#95894] (issue: {es-issue}93730[#93730]) +* Introduce a filtered collector manager {es-pull}96824[#96824] +* Introduce minimum score collector manager {es-pull}96834[#96834] +* Skip shards when querying constant keyword fields {es-pull}96161[#96161] (issue: {es-issue}95541[#95541]) +* Support CCS minimize round trips in async search {es-pull}96012[#96012] +* Support for patter_replace filter in keyword normalizer {es-pull}96588[#96588] +* Support null_value for rank_feature field type {es-pull}95811[#95811] + +Security:: +* Add "_storage" internal user {es-pull}95694[#95694] + +Snapshot/Restore:: +* Reduce overhead in blob cache service get {es-pull}96399[#96399] + +Stats:: +* Add `ingest` information to the cluster info endpoint {es-pull}96328[#96328] (issue: {es-issue}95392[#95392]) +* Add `script` information to the cluster info endpoint {es-pull}96613[#96613] (issue: {es-issue}95394[#95394]) +* Add `thread_pool` information to the cluster info endpoint {es-pull}96407[#96407] (issue: {es-issue}95393[#95393]) + +TSDB:: +* Feature: include unit support for time series rate aggregation {es-pull}96605[#96605] (issue: {es-issue}94630[#94630]) + +Vector Search:: +* Leverage SIMD hardware instructions in Vector Search {es-pull}96453[#96453] (issue: {es-issue}96370[#96370]) + +[[feature-8.9.0]] +[float] +=== New features + +Application:: +* Enable analytics geoip in behavioral analytics {es-pull}96624[#96624] + +Authorization:: +* Support restricting access of API keys to only certain workflows {es-pull}96744[#96744] + +Data streams:: +* Adding ability to auto-install ingest pipelines and refer to them from index templates {es-pull}95782[#95782] + +Geo:: +* Geometry simplifier {es-pull}94859[#94859] + +ILM+SLM:: +* Enhance ILM Health Indicator {es-pull}96092[#96092] + +Infra/Node Lifecycle:: +* Gracefully shutdown elasticsearch {es-pull}96363[#96363] + +Infra/Plugins:: +* [Fleet] Add `.fleet-secrets` system index {es-pull}95625[#95625] (issue: {es-issue}95143[#95143]) + +Machine Learning:: +* Add support for `xlm_roberta` tokenized models {es-pull}94089[#94089] +* Removes the technical preview admonition from query_vector_builder docs {es-pull}96735[#96735] + +Snapshot/Restore:: +* Add repo throttle metrics to node stats api response {es-pull}96678[#96678] (issue: {es-issue}89385[#89385]) + +Stats:: +* New HTTP info endpoint {es-pull}96198[#96198] (issue: {es-issue}95391[#95391]) + +[[upgrade-8.9.0]] +[float] +=== Upgrades + +Infra/Transport API:: +* Bump `TransportVersion` to the first non-release version number. Transport protocol is now versioned independently of release version. {es-pull}95286[#95286] + +Network:: +* Upgrade Netty to 4.1.92 {es-pull}95575[#95575] +* Upgrade Netty to 4.1.94.Final {es-pull}97112[#97112] + +Search:: +* Upgrade Lucene to a 9.7.0 snapshot {es-pull}96433[#96433] +* Upgrade to new lucene snapshot 9.7.0-snapshot-a8602d6ef88 {es-pull}96741[#96741] + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 49a6fade5fd1b..23e6be1440af4 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -11,7 +11,8 @@ For detailed information about this release, see the <> and // Add previous release to the list Other versions: -{ref-bare}/8.9/release-highlights.html[8.9] +{ref-bare}/8.10/release-highlights.html[8.10] +| {ref-bare}/8.9/release-highlights.html[8.9] | {ref-bare}/8.8/release-highlights.html[8.8] | {ref-bare}/8.7/release-highlights.html[8.7] | {ref-bare}/8.6/release-highlights.html[8.6] diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 6c33b9db4c597..afba5e18c1631 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -767,6 +767,7 @@ currently supported: + -- * <> +* <> * <> * <> * <> diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc index 6245df668665a..59555517e4971 100644 --- a/docs/reference/rest-api/index.asciidoc +++ b/docs/reference/rest-api/index.asciidoc @@ -38,6 +38,7 @@ not be included yet. * <> * <> * <> +* <> * <> * <> * <> @@ -49,6 +50,7 @@ not be included yet. * <> * <> * <> +* <> * <> * <> * <> @@ -82,6 +84,7 @@ include::{es-repo-dir}/ml/df-analytics/apis/index.asciidoc[] include::{es-repo-dir}/ml/trained-models/apis/index.asciidoc[] include::{es-repo-dir}/migration/migration.asciidoc[] include::{es-repo-dir}/shutdown/apis/shutdown-api.asciidoc[] +include::{es-repo-dir}/query-rules/apis/index.asciidoc[] include::{es-repo-dir}/indices/apis/reload-analyzers.asciidoc[] include::{es-repo-dir}/repositories-metering-api/repositories-metering-apis.asciidoc[] include::{es-repo-dir}/rollup/rollup-apis.asciidoc[] @@ -93,6 +96,7 @@ include::{xes-repo-dir}/rest-api/security.asciidoc[] include::{es-repo-dir}/snapshot-restore/apis/snapshot-restore-apis.asciidoc[] include::{es-repo-dir}/slm/apis/slm-api.asciidoc[] include::{es-repo-dir}/sql/apis/sql-apis.asciidoc[] +include::{es-repo-dir}/synonyms/apis/synonyms-apis.asciidoc[] include::{es-repo-dir}/transform/apis/index.asciidoc[] include::usage.asciidoc[] include::{xes-repo-dir}/rest-api/watcher.asciidoc[] diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 7515b69a7bc0c..2cbc2ce3fc747 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -419,6 +419,17 @@ GET /_xpack/usage "total": 0 } }, + "remote_clusters": { + "size": 0, + "mode": { + "proxy": 0, + "sniff": 0 + }, + "security": { + "cert": 0, + "api_key": 0 + } + }, "enterprise_search" : { "available": true, "enabled": true, @@ -427,6 +438,12 @@ GET /_xpack/usage }, "analytics_collections": { "count": 0 + }, + "query_rulesets": { + "total_rule_count": 0, + "total_count": 0, + "min_rule_count": 0, + "max_rule_count": 0 } } } diff --git a/docs/reference/scripting/security.asciidoc b/docs/reference/scripting/security.asciidoc index 2726938cb179d..0f322d08726b9 100644 --- a/docs/reference/scripting/security.asciidoc +++ b/docs/reference/scripting/security.asciidoc @@ -36,7 +36,7 @@ configured to run both types of scripts. To limit what type of scripts are run, set `script.allowed_types` to `inline` or `stored`. To prevent any scripts from running, set `script.allowed_types` to `none`. -IMPORTANT: If you use {kib}, set `script.allowed_types` to `both` or `inline`. +IMPORTANT: If you use {kib}, set `script.allowed_types` to both or just `inline`. Some {kib} features rely on inline scripts and do not function as expected if {es} does not allow inline scripts. diff --git a/docs/reference/search-application/apis/delete-search-application.asciidoc b/docs/reference/search-application/apis/delete-search-application.asciidoc index 80b84020669ca..4043942b09503 100644 --- a/docs/reference/search-application/apis/delete-search-application.asciidoc +++ b/docs/reference/search-application/apis/delete-search-application.asciidoc @@ -42,8 +42,31 @@ No Search Application matching `name` could be found. The following example deletes the Search Application named `my-app`: +//// +[source,console] +---- +PUT /index1 + +PUT _application/search_application/my-app +{ + "indices": [ "index1" ], + "template": { + "script": { + "source": { + "query": { + "query_string": { + "query": "{{query_string}}" + } + } + } + } + } +} +---- +// TESTSETUP +//// + [source,console] ---- DELETE _application/search_application/my-app/ ---- -// TEST[skip:TBD] diff --git a/docs/reference/search-application/apis/get-search-application.asciidoc b/docs/reference/search-application/apis/get-search-application.asciidoc index 0feba6145640e..5beafd96c2e6b 100644 --- a/docs/reference/search-application/apis/get-search-application.asciidoc +++ b/docs/reference/search-application/apis/get-search-application.asciidoc @@ -10,9 +10,6 @@ beta::[] Retrieves information about a search application. -If the search application has an inconsistent state between its alias and configured indices, a warning header will be returned with the response. -To resolve this inconsistent state, issue an updated <> command. - [[get-search-application-request]] ==== {api-request-title} @@ -43,19 +40,17 @@ No Search Application matching `name` could be found. The following example gets the Search Application named `my-app`: +//// + [source,console] ----- -GET _application/search_application/my-app/ ----- -// TEST[skip:TBD] +-------------------------------------------------- +PUT index1 -A sample response: +PUT index2 -[source,console-result] ----- +PUT _application/search_application/my-app { - "name": "my-app", - "indices": [ "index1", "index2" ], + "indices": ["index1", "index2"], "updated_at_millis": 1682105622204, "template": { "script": { @@ -74,4 +69,60 @@ A sample response: } } } +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _application/search_application/my-app + +DELETE index1 + +DELETE index2 +-------------------------------------------------- +// TEARDOWN + +//// + +[source,console] +---- +GET _application/search_application/my-app/ +---- + +A sample response: + +[source,console-result] +---- +{ + "name": "my-app", + "updated_at_millis": 1682105622204, + "template": { + "script": { + "source": { + "query": { + "query_string": { + "query": "{{query_string}}", + "default_field": "{{default_field}}" + } + } + }, + "lang": "mustache", + "options": { + "content_type": "application/json;charset=utf-8" + }, + "params": { + "query_string": "*", + "default_field": "*" + } + } + } +} ---- +// TESTRESPONSE[s/"updated_at_millis": 1682105622204/"updated_at_millis": $body.$_path/] + +[NOTE] +==== +The indices associated with a search application are not returned with the GET response. +To view the indices, use the <> API. +The alias name will match the search application name, for example `my-app` in the example above. +==== diff --git a/docs/reference/search-application/apis/list-search-applications.asciidoc b/docs/reference/search-application/apis/list-search-applications.asciidoc index bc7eb3579ef39..3cc077bf682d6 100644 --- a/docs/reference/search-application/apis/list-search-applications.asciidoc +++ b/docs/reference/search-application/apis/list-search-applications.asciidoc @@ -40,19 +40,67 @@ Requires the `manage_search_application` cluster privilege. The following example lists all configured search applications: +//// + +[source,console] +-------------------------------------------------- +PUT index1 + +PUT _application/search_application/app-1 +{ + "indices": [ "index1" ], + "template": { + "script": { + "source": { + "query": { + "query_string": { + "query": "{{query_string}}" + } + } + } + } + } +} + +PUT _application/search_application/app-2 +{ + "indices": [ "index1" ], + "template": { + "script": { + "source": { + "query": { + "query_string": { + "query": "{{query_string}}" + } + } + } + } + } +} +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _application/search_application/app-1 + +DELETE _application/search_application/app-2 +-------------------------------------------------- +// TEARDOWN + +//// + [source,console] ---- GET _application/search_application/ ---- -// TEST[skip:TBD] -The following example lists the first three search applications whose names match the query string `app`: +The following example queries the first three search applications whose names start with `app`: [source,console] ---- -GET _application/search_application/?from=0&size=3&q=app +GET _application/search_application?from=0&size=3&q=app* ---- -// TEST[skip:TBD] A sample response: @@ -63,13 +111,14 @@ A sample response: "results": [ { "name": "app-1", - "indices": [ "index1", "index2" ] + "updated_at_millis": 1690981129366 }, { "name": "app-2", - "indices": [ "index3", "index4" ] + "updated_at_millis": 1691501823939 } - ], - "updated_at_millis": 1682105622204 + ] } ---- +// TESTRESPONSE[s/"updated_at_millis": 1690981129366/"updated_at_millis": $body.$_path/] +// TESTRESPONSE[s/"updated_at_millis": 1691501823939/"updated_at_millis": $body.$_path/] diff --git a/docs/reference/search-application/apis/put-search-application.asciidoc b/docs/reference/search-application/apis/put-search-application.asciidoc index 9e77b2c98cd6b..9cd7ee37aaa02 100644 --- a/docs/reference/search-application/apis/put-search-application.asciidoc +++ b/docs/reference/search-application/apis/put-search-application.asciidoc @@ -49,6 +49,21 @@ The <> associated with this search application. - The template may be modified with subsequent <> requests. - If no template is specified when creating a search application, or if a template is removed from a search application, we use the <> defined in the template examples as a default. - This template will be used by the <> API to execute searches. +- The template accepts an optional `dictionary` parameter which defines a https://json-schema.org[JSON schema] used for validating parameters sent to the <> API. + +.Properties of `