diff --git a/.eslintignore b/.eslintignore index 6ea478bc2bf7..f5df2d394adf 100644 --- a/.eslintignore +++ b/.eslintignore @@ -18,8 +18,7 @@ target # plugin overrides /src/core/lib/osd_internal_native_observable /src/legacy/plugin_discovery/plugin_pack/__tests__/fixtures/plugins/broken -/src/plugins/data/common/opensearch_query/kuery/ast/_generated_/** -/src/plugins/vis_type_timeline/public/_generated_/** +/src/plugins/**/_generated_/** # package overrides /packages/opensearch-eslint-config-opensearch-dashboards diff --git a/CHANGELOG.md b/CHANGELOG.md index 074a51ccae5e..a140148fc7d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Deprecations +- Rename `withLongNumerals` to `withLongNumeralsSupport` in `HttpFetchOptions` [#5592](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5592) + ### 🛡 Security - Add support for TLS v1.3 ([#5133](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5133)) @@ -17,6 +19,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - [Workspace] Add core workspace service module to enable the implementation of workspace features within OSD plugins ([#5092](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5092)) - [Workspace] Setup workspace skeleton and implement basic CRUD API ([#5075](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5075)) - [Decouple] Add new cross compatibility check core service which export functionality for plugins to verify if their OpenSearch plugin counterpart is installed on the cluster or has incompatible version to configure the plugin behavior([#4710](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4710)) +- [Discover] Add long numerals support [#5592](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5592) - [Discover] Display inner properties in the left navigation bar [#5429](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5429) ### 🐛 Bug Fixes diff --git a/package.json b/package.json index ed996e0dbf59..97ace6a1ef43 100644 --- a/package.json +++ b/package.json @@ -131,7 +131,7 @@ "@elastic/datemath": "5.0.3", "@elastic/eui": "npm:@opensearch-project/oui@1.4.0", "@elastic/good": "^9.0.1-kibana3", - "@elastic/numeral": "^2.5.0", + "@elastic/numeral": "npm:@amoo-miki/numeral@2.6.0", "@elastic/request-crypto": "2.0.0", "@elastic/safer-lodash-set": "0.0.0", "@hapi/accept": "^5.0.2", diff --git a/packages/osd-std/src/json.ts b/packages/osd-std/src/json.ts index 7c619dcd1656..4761eadea2dd 100644 --- a/packages/osd-std/src/json.ts +++ b/packages/osd-std/src/json.ts @@ -285,6 +285,7 @@ export const parse = ( if ( numeralsAreNumbers && typeof val === 'number' && + isFinite(val) && (val < Number.MAX_SAFE_INTEGER || val > Number.MAX_SAFE_INTEGER) ) { numeralsAreNumbers = false; diff --git a/packages/osd-ui-shared-deps/package.json b/packages/osd-ui-shared-deps/package.json index 250d500fd94c..1c2b532a7227 100644 --- a/packages/osd-ui-shared-deps/package.json +++ b/packages/osd-ui-shared-deps/package.json @@ -11,7 +11,7 @@ "dependencies": { "@elastic/charts": "31.1.0", "@elastic/eui": "npm:@opensearch-project/oui@1.4.0", - "@elastic/numeral": "^2.5.0", + "@elastic/numeral": "npm:@amoo-miki/numeral@2.6.0", "@opensearch/datemath": "5.0.3", "@osd/i18n": "1.0.0", "@osd/monaco": "1.0.0", @@ -52,3 +52,4 @@ "webpack": "npm:@amoo-miki/webpack@4.46.0-rc.2" } } + diff --git a/scripts/postinstall.js b/scripts/postinstall.js index ce13dee9f0dd..7865473ee494 100644 --- a/scripts/postinstall.js +++ b/scripts/postinstall.js @@ -71,6 +71,19 @@ const run = async () => { }, ]) ); + promises.push( + patchFile('node_modules/rison-node/js/rison.js', [ + { + from: 'return Number(s)', + to: + 'return isFinite(s) && (s > Number.MAX_SAFE_INTEGER || s < Number.MIN_SAFE_INTEGER) ? BigInt(s) : Number(s)', + }, + { + from: 's = {', + to: 's = {\n bigint: x => x.toString(),', + }, + ]) + ); await Promise.all(promises); }; diff --git a/src/core/public/http/fetch.test.ts b/src/core/public/http/fetch.test.ts index 20f070dbba80..b97a281b8663 100644 --- a/src/core/public/http/fetch.test.ts +++ b/src/core/public/http/fetch.test.ts @@ -839,7 +839,9 @@ describe('Fetch', () => { }, }); - await expect(fetchInstance.fetch('/my/path', { withLongNumerals: true })).resolves.toEqual({ + await expect( + fetchInstance.fetch('/my/path', { withLongNumeralsSupport: true }) + ).resolves.toEqual({ 'long-max': longPositive, 'long-min': longNegative, }); @@ -854,7 +856,9 @@ describe('Fetch', () => { }, }); - await expect(fetchInstance.fetch('/my/path', { withLongNumerals: true })).resolves.toEqual({ + await expect( + fetchInstance.fetch('/my/path', { withLongNumeralsSupport: true }) + ).resolves.toEqual({ 'long-max': longPositive, 'long-min': longNegative, }); diff --git a/src/core/public/http/fetch.ts b/src/core/public/http/fetch.ts index 767d58643003..9a25ecc5ea65 100644 --- a/src/core/public/http/fetch.ts +++ b/src/core/public/http/fetch.ts @@ -190,12 +190,20 @@ export class Fetch { if (NDJSON_CONTENT.test(contentType)) { body = await response.blob(); } else if (JSON_CONTENT.test(contentType)) { - body = fetchOptions.withLongNumerals ? parse(await response.text()) : await response.json(); + // ToDo: [3.x] Remove withLongNumerals + body = + fetchOptions.withLongNumeralsSupport || fetchOptions.withLongNumerals + ? parse(await response.text()) + : await response.json(); } else { const text = await response.text(); try { - body = fetchOptions.withLongNumerals ? parse(text) : JSON.parse(text); + // ToDo: [3.x] Remove withLongNumerals + body = + fetchOptions.withLongNumeralsSupport || fetchOptions.withLongNumerals + ? parse(text) + : JSON.parse(text); } catch (err) { body = text; } diff --git a/src/core/public/http/types.ts b/src/core/public/http/types.ts index 3b7dff71c811..f2573a6badd5 100644 --- a/src/core/public/http/types.ts +++ b/src/core/public/http/types.ts @@ -262,6 +262,9 @@ export interface HttpFetchOptions extends HttpRequestInit { * When `true`, if the response has a JSON mime type, the {@link HttpResponse} will use an alternate JSON parser * that converts long numerals to BigInts. Defaults to `false`. */ + withLongNumeralsSupport?: boolean; + + /** @deprecated use {@link withLongNumeralsSupport} instead */ withLongNumerals?: boolean; } diff --git a/src/core/public/ui_settings/__snapshots__/ui_settings_client.test.ts.snap b/src/core/public/ui_settings/__snapshots__/ui_settings_client.test.ts.snap index bf9a6f115722..68948ef0d4af 100644 --- a/src/core/public/ui_settings/__snapshots__/ui_settings_client.test.ts.snap +++ b/src/core/public/ui_settings/__snapshots__/ui_settings_client.test.ts.snap @@ -62,6 +62,10 @@ exports[`#overrideLocalDefault key has no user value synchronously modifies the exports[`#overrideLocalDefault key has no user value synchronously modifies the value returned by getAll(): getAll after override 1`] = ` Object { + "aLongNumeral": Object { + "type": "number", + "value": "9007199254741002", + }, "dateFormat": Object { "value": "bar", }, @@ -70,6 +74,10 @@ Object { exports[`#overrideLocalDefault key has no user value synchronously modifies the value returned by getAll(): getAll before override 1`] = ` Object { + "aLongNumeral": Object { + "type": "number", + "value": "9007199254741002", + }, "dateFormat": Object { "value": "Browser", }, @@ -82,6 +90,10 @@ exports[`#overrideLocalDefault key with user value does not modify the return va exports[`#overrideLocalDefault key with user value is included in the return value of getAll: getAll after override 1`] = ` Object { + "aLongNumeral": Object { + "type": "number", + "value": "9007199254741002", + }, "dateFormat": Object { "userValue": "foo", "value": "bar", @@ -91,6 +103,10 @@ Object { exports[`#overrideLocalDefault key with user value is included in the return value of getAll: getAll before override 1`] = ` Object { + "aLongNumeral": Object { + "type": "number", + "value": "9007199254741002", + }, "dateFormat": Object { "userValue": "foo", "value": "Browser", @@ -104,6 +120,10 @@ exports[`#overrideLocalDefault key with user value returns default override when exports[`#overrideLocalDefault key with user value returns default override when setting removed: getAll after override 1`] = ` Object { + "aLongNumeral": Object { + "type": "number", + "value": "9007199254741002", + }, "dateFormat": Object { "value": "bar", }, @@ -112,6 +132,10 @@ Object { exports[`#overrideLocalDefault key with user value returns default override when setting removed: getAll before override 1`] = ` Object { + "aLongNumeral": Object { + "type": "number", + "value": "9007199254741002", + }, "dateFormat": Object { "userValue": "foo", "value": "bar", diff --git a/src/core/public/ui_settings/ui_settings_client.test.ts b/src/core/public/ui_settings/ui_settings_client.test.ts index b297167439cc..9060b0d6db4e 100644 --- a/src/core/public/ui_settings/ui_settings_client.test.ts +++ b/src/core/public/ui_settings/ui_settings_client.test.ts @@ -36,7 +36,13 @@ import { UiSettingsClient } from './ui_settings_client'; let done$: Subject; function setup(options: { defaults?: any; initialSettings?: any } = {}) { - const { defaults = { dateFormat: { value: 'Browser' } }, initialSettings = {} } = options; + const { + defaults = { + dateFormat: { value: 'Browser' }, + aLongNumeral: { value: `${BigInt(Number.MAX_SAFE_INTEGER) + 11n}`, type: 'number' }, + }, + initialSettings = {}, + } = options; const batchSet = jest.fn(() => ({ settings: {}, @@ -62,6 +68,7 @@ describe('#get', () => { it('gives access to uiSettings values', () => { const { client } = setup(); expect(client.get('dateFormat')).toMatchSnapshot(); + expect(client.get('aLongNumeral')).toBe(BigInt(Number.MAX_SAFE_INTEGER) + 11n); }); it('supports the default value overload', () => { @@ -82,13 +89,19 @@ describe('#get', () => { const { client } = setup(); // if you are hitting this error, then a test is setting this client value globally and not unsetting it! expect(client.isDefault('dateFormat')).toBe(true); + expect(client.isDefault('aLongNumeral')).toBe(true); const defaultDateFormat = client.get('dateFormat'); + const defaultLongNumeral = client.get('aLongNumeral'); expect(client.get('dateFormat', 'xyz')).toBe('xyz'); + expect(client.get('aLongNumeral', 1n)).toBe(1n); + // shouldn't change other usages expect(client.get('dateFormat')).toBe(defaultDateFormat); expect(client.get('dataFormat', defaultDateFormat)).toBe(defaultDateFormat); + expect(client.get('aLongNumeral')).toBe(defaultLongNumeral); + expect(client.get('aLongNumeral', defaultLongNumeral)).toBe(defaultLongNumeral); }); it("throws on unknown properties that don't have a value yet.", () => { diff --git a/src/core/public/ui_settings/ui_settings_client.ts b/src/core/public/ui_settings/ui_settings_client.ts index d2015468befa..8a5701de6b39 100644 --- a/src/core/public/ui_settings/ui_settings_client.ts +++ b/src/core/public/ui_settings/ui_settings_client.ts @@ -97,11 +97,11 @@ You can use \`IUiSettingsClient.get("${key}", defaultValue)\`, which will just r return JSON.parse(value); } - if (type === 'number') { - return parseFloat(value); - } - - return value; + return type === 'number' && typeof value !== 'bigint' + ? isFinite(value) && (value > Number.MAX_SAFE_INTEGER || value < Number.MIN_SAFE_INTEGER) + ? BigInt(value) + : parseFloat(value) + : value; } get$(key: string, defaultOverride?: T) { diff --git a/src/core/server/http/router/response.ts b/src/core/server/http/router/response.ts index c38c58f3ba1c..ddf1c4f9481e 100644 --- a/src/core/server/http/router/response.ts +++ b/src/core/server/http/router/response.ts @@ -87,6 +87,8 @@ export interface HttpResponseOptions { body?: HttpResponsePayload; /** HTTP Headers with additional information about response */ headers?: ResponseHeaders; + /** Indicates if alternate serialization should be employed */ + withLongNumeralsSupport?: boolean; } /** diff --git a/src/core/server/http/router/response_adapter.ts b/src/core/server/http/router/response_adapter.ts index 08dcaf1e9c60..ff5ff5ca84bc 100644 --- a/src/core/server/http/router/response_adapter.ts +++ b/src/core/server/http/router/response_adapter.ts @@ -35,6 +35,7 @@ import { import typeDetect from 'type-detect'; import Boom from '@hapi/boom'; import * as stream from 'stream'; +import { stringify } from '@osd/std'; import { HttpResponsePayload, @@ -108,9 +109,18 @@ export class HapiResponseAdapter { opensearchDashboardsResponse: OpenSearchDashboardsResponse ) { const response = this.responseToolkit - .response(opensearchDashboardsResponse.payload) + .response( + opensearchDashboardsResponse.options.withLongNumeralsSupport + ? stringify(opensearchDashboardsResponse.payload) + : opensearchDashboardsResponse.payload + ) .code(opensearchDashboardsResponse.status); setHeaders(response, opensearchDashboardsResponse.options.headers); + if (opensearchDashboardsResponse.options.withLongNumeralsSupport) { + setHeaders(response, { + 'content-type': 'application/json', + }); + } return response; } diff --git a/src/plugins/console/public/lib/opensearch/opensearch.ts b/src/plugins/console/public/lib/opensearch/opensearch.ts index d1ba8797e474..907323611358 100644 --- a/src/plugins/console/public/lib/opensearch/opensearch.ts +++ b/src/plugins/console/public/lib/opensearch/opensearch.ts @@ -57,7 +57,7 @@ export async function send( body: data, prependBasePath: true, asResponse: true, - withLongNumerals: true, + withLongNumeralsSupport: true, }); } diff --git a/src/plugins/data/common/field_formats/content_types/html_content_type.ts b/src/plugins/data/common/field_formats/content_types/html_content_type.ts index 1e3195f256c1..904e6a09c6e8 100644 --- a/src/plugins/data/common/field_formats/content_types/html_content_type.ts +++ b/src/plugins/data/common/field_formats/content_types/html_content_type.ts @@ -72,7 +72,7 @@ export const setup = ( }; const wrap: HtmlContextTypeConvert = (value, options) => { - return `${recurse(value, options)}`; + return `${recurse(value, options)}`; }; return wrap; diff --git a/src/plugins/data/common/field_formats/converters/color.test.ts b/src/plugins/data/common/field_formats/converters/color.test.ts index 689abf0e3daa..ac1b0c9bc4e9 100644 --- a/src/plugins/data/common/field_formats/converters/color.test.ts +++ b/src/plugins/data/common/field_formats/converters/color.test.ts @@ -48,14 +48,14 @@ describe('Color Format', () => { jest.fn() ); - expect(colorer.convert(99, HTML_CONTEXT_TYPE)).toBe('99'); + expect(colorer.convert(99, HTML_CONTEXT_TYPE)).toBe('99'); expect(colorer.convert(100, HTML_CONTEXT_TYPE)).toBe( - '100' + '100' ); expect(colorer.convert(150, HTML_CONTEXT_TYPE)).toBe( - '150' + '150' ); - expect(colorer.convert(151, HTML_CONTEXT_TYPE)).toBe('151'); + expect(colorer.convert(151, HTML_CONTEXT_TYPE)).toBe('151'); }); test('should not convert invalid ranges', () => { @@ -73,7 +73,7 @@ describe('Color Format', () => { jest.fn() ); - expect(colorer.convert(99, HTML_CONTEXT_TYPE)).toBe('99'); + expect(colorer.convert(99, HTML_CONTEXT_TYPE)).toBe('99'); }); }); @@ -94,26 +94,26 @@ describe('Color Format', () => { ); const converter = colorer.getConverterFor(HTML_CONTEXT_TYPE) as Function; - expect(converter('B', HTML_CONTEXT_TYPE)).toBe('B'); + expect(converter('B', HTML_CONTEXT_TYPE)).toBe('B'); expect(converter('AAA', HTML_CONTEXT_TYPE)).toBe( - 'AAA' + 'AAA' ); expect(converter('AB', HTML_CONTEXT_TYPE)).toBe( - 'AB' + 'AB' ); - expect(converter('a', HTML_CONTEXT_TYPE)).toBe('a'); + expect(converter('a', HTML_CONTEXT_TYPE)).toBe('a'); - expect(converter('B', HTML_CONTEXT_TYPE)).toBe('B'); + expect(converter('B', HTML_CONTEXT_TYPE)).toBe('B'); expect(converter('AAA', HTML_CONTEXT_TYPE)).toBe( - 'AAA' + 'AAA' ); expect(converter('AB', HTML_CONTEXT_TYPE)).toBe( - 'AB' + 'AB' ); expect(converter('AB <', HTML_CONTEXT_TYPE)).toBe( - 'AB <' + 'AB <' ); - expect(converter('a', HTML_CONTEXT_TYPE)).toBe('a'); + expect(converter('a', HTML_CONTEXT_TYPE)).toBe('a'); }); test('returns original value (escaped) when regex is invalid', () => { @@ -132,7 +132,7 @@ describe('Color Format', () => { ); const converter = colorer.getConverterFor(HTML_CONTEXT_TYPE) as Function; - expect(converter('<', HTML_CONTEXT_TYPE)).toBe('<'); + expect(converter('<', HTML_CONTEXT_TYPE)).toBe('<'); }); }); }); diff --git a/src/plugins/data/common/field_formats/converters/numeral.ts b/src/plugins/data/common/field_formats/converters/numeral.ts index ad0b9773823f..7219b2e6ab40 100644 --- a/src/plugins/data/common/field_formats/converters/numeral.ts +++ b/src/plugins/data/common/field_formats/converters/numeral.ts @@ -56,11 +56,14 @@ export abstract class NumeralFormat extends FieldFormat { protected getConvertedValue(val: any): string { if (val === -Infinity) return '-∞'; if (val === +Infinity) return '+∞'; - if (typeof val !== 'number') { + + const isBigInt = typeof val === 'bigint'; + + if (typeof val !== 'number' && !isBigInt) { val = parseFloat(val); } - if (isNaN(val)) return ''; + if (!isBigInt && isNaN(val)) return ''; const previousLocale = numeral.language(); const defaultLocale = diff --git a/src/plugins/data/common/field_formats/converters/source.test.ts b/src/plugins/data/common/field_formats/converters/source.test.ts index 2b70e4f4f6a3..763987f05054 100644 --- a/src/plugins/data/common/field_formats/converters/source.test.ts +++ b/src/plugins/data/common/field_formats/converters/source.test.ts @@ -50,7 +50,7 @@ describe('Source Format', () => { }; expect(convertHtml(hit)).toBe( - '{"foo":"bar","number":42,"hello":"<h1>World</h1>","also":"with \\"quotes\\" or 'single quotes'"}' + '{"foo":"bar","number":42,"hello":"<h1>World</h1>","also":"with \\"quotes\\" or 'single quotes'"}' ); }); }); diff --git a/src/plugins/data/common/field_formats/converters/url.test.ts b/src/plugins/data/common/field_formats/converters/url.test.ts index f80dc4fd11b9..91acf522b89d 100644 --- a/src/plugins/data/common/field_formats/converters/url.test.ts +++ b/src/plugins/data/common/field_formats/converters/url.test.ts @@ -36,7 +36,7 @@ describe('UrlFormat', () => { const url = new UrlFormat({}); expect(url.convert('http://opensearch.org', HTML_CONTEXT_TYPE)).toBe( - 'http://opensearch.org' + 'http://opensearch.org' ); }); @@ -44,7 +44,7 @@ describe('UrlFormat', () => { const url = new UrlFormat({ type: 'audio' }); expect(url.convert('http://opensearch.org', HTML_CONTEXT_TYPE)).toBe( - '' + '' ); }); @@ -53,7 +53,7 @@ describe('UrlFormat', () => { const url = new UrlFormat({ type: 'img' }); expect(url.convert('http://opensearch.org', HTML_CONTEXT_TYPE)).toBe( - 'A dynamically-specified image located at http://opensearch.orgA dynamically-specified image located at http://opensearch.org' ); }); @@ -62,7 +62,7 @@ describe('UrlFormat', () => { const url = new UrlFormat({ type: 'img', width: '12', height: '55' }); expect(url.convert('http://opensearch.org', HTML_CONTEXT_TYPE)).toBe( - 'A dynamically-specified image located at http://opensearch.orgA dynamically-specified image located at http://opensearch.org' ); }); @@ -71,7 +71,7 @@ describe('UrlFormat', () => { const url = new UrlFormat({ type: 'img', height: '55' }); expect(url.convert('http://opensearch.org', HTML_CONTEXT_TYPE)).toBe( - 'A dynamically-specified image located at http://opensearch.orgA dynamically-specified image located at http://opensearch.org' ); }); @@ -80,7 +80,7 @@ describe('UrlFormat', () => { const url = new UrlFormat({ type: 'img', width: '22' }); expect(url.convert('http://opensearch.org', HTML_CONTEXT_TYPE)).toBe( - 'A dynamically-specified image located at http://opensearch.orgA dynamically-specified image located at http://opensearch.org' ); }); @@ -89,7 +89,7 @@ describe('UrlFormat', () => { const url = new UrlFormat({ type: 'img', width: 'not a number' }); expect(url.convert('http://opensearch.org', HTML_CONTEXT_TYPE)).toBe( - 'A dynamically-specified image located at http://opensearch.orgA dynamically-specified image located at http://opensearch.org' ); }); @@ -98,7 +98,7 @@ describe('UrlFormat', () => { const url = new UrlFormat({ type: 'img', height: 'not a number' }); expect(url.convert('http://opensearch.org', HTML_CONTEXT_TYPE)).toBe( - 'A dynamically-specified image located at http://opensearch.orgA dynamically-specified image located at http://opensearch.org' ); }); @@ -109,7 +109,7 @@ describe('UrlFormat', () => { const url = new UrlFormat({ urlTemplate: 'http://{{ value }}' }); expect(url.convert('url', HTML_CONTEXT_TYPE)).toBe( - 'http://url' + 'http://url' ); }); @@ -128,7 +128,7 @@ describe('UrlFormat', () => { }); expect(url.convert('php', HTML_CONTEXT_TYPE)).toBe( - 'extension: php' + 'extension: php' ); }); @@ -188,19 +188,19 @@ describe('UrlFormat', () => { const converter = url.getConverterFor(HTML_CONTEXT_TYPE) as Function; expect(converter('www.opensearch.org')).toBe( - 'www.opensearch.org' + 'www.opensearch.org' ); expect(converter('opensearch.org')).toBe( - 'opensearch.org' + 'opensearch.org' ); expect(converter('opensearch')).toBe( - 'opensearch' + 'opensearch' ); expect(converter('ftp://opensearch.org')).toBe( - 'ftp://opensearch.org' + 'ftp://opensearch.org' ); }); @@ -213,19 +213,19 @@ describe('UrlFormat', () => { const converter = url.getConverterFor(HTML_CONTEXT_TYPE) as Function; expect(converter('www.opensearch.org')).toBe( - 'www.opensearch.org' + 'www.opensearch.org' ); expect(converter('opensearch.org')).toBe( - 'opensearch.org' + 'opensearch.org' ); expect(converter('opensearch')).toBe( - 'opensearch' + 'opensearch' ); expect(converter('ftp://opensearch.org')).toBe( - 'ftp://opensearch.org' + 'ftp://opensearch.org' ); }); @@ -238,7 +238,7 @@ describe('UrlFormat', () => { const converter = url.getConverterFor(HTML_CONTEXT_TYPE) as Function; expect(converter('../app/opensearch-dashboards')).toBe( - '../app/opensearch-dashboards' + '../app/opensearch-dashboards' ); }); @@ -246,11 +246,11 @@ describe('UrlFormat', () => { const url = new UrlFormat({}); expect(url.convert('../app/opensearch-dashboards', HTML_CONTEXT_TYPE)).toBe( - '../app/opensearch-dashboards' + '../app/opensearch-dashboards' ); expect(url.convert('http://www.opensearch.org', HTML_CONTEXT_TYPE)).toBe( - 'http://www.opensearch.org' + 'http://www.opensearch.org' ); }); @@ -264,15 +264,15 @@ describe('UrlFormat', () => { const converter = url.getConverterFor(HTML_CONTEXT_TYPE) as Function; expect(converter('#/foo')).toBe( - '#/foo' + '#/foo' ); expect(converter('/nbc/app/discover#/')).toBe( - '/nbc/app/discover#/' + '/nbc/app/discover#/' ); expect(converter('../foo/bar')).toBe( - '../foo/bar' + '../foo/bar' ); }); @@ -285,23 +285,23 @@ describe('UrlFormat', () => { const converter = url.getConverterFor(HTML_CONTEXT_TYPE) as Function; expect(converter('10.22.55.66')).toBe( - '10.22.55.66' + '10.22.55.66' ); expect(converter('http://www.domain.name/app/opensearch-dashboards#/dashboard/')).toBe( - 'http://www.domain.name/app/opensearch-dashboards#/dashboard/' + 'http://www.domain.name/app/opensearch-dashboards#/dashboard/' ); expect(converter('/app/opensearch-dashboards')).toBe( - '/app/opensearch-dashboards' + '/app/opensearch-dashboards' ); expect(converter('opensearch-dashboards#/dashboard/')).toBe( - 'opensearch-dashboards#/dashboard/' + 'opensearch-dashboards#/dashboard/' ); expect(converter('#/dashboard/')).toBe( - '#/dashboard/' + '#/dashboard/' ); }); @@ -314,23 +314,23 @@ describe('UrlFormat', () => { const converter = url.getConverterFor(HTML_CONTEXT_TYPE) as Function; expect(converter('10.22.55.66')).toBe( - '10.22.55.66' + '10.22.55.66' ); expect(converter('http://www.domain.name/app/kibana#/dashboard/')).toBe( - 'http://www.domain.name/app/kibana#/dashboard/' + 'http://www.domain.name/app/kibana#/dashboard/' ); expect(converter('/app/kibana')).toBe( - '/app/kibana' + '/app/kibana' ); expect(converter('kibana#/dashboard/')).toBe( - 'kibana#/dashboard/' + 'kibana#/dashboard/' ); expect(converter('#/dashboard/')).toBe( - '#/dashboard/' + '#/dashboard/' ); }); }); diff --git a/src/plugins/data/common/field_formats/field_format.test.ts b/src/plugins/data/common/field_formats/field_format.test.ts index 4dadca841120..7d3d83a9c49f 100644 --- a/src/plugins/data/common/field_formats/field_format.test.ts +++ b/src/plugins/data/common/field_formats/field_format.test.ts @@ -109,7 +109,7 @@ describe('FieldFormat class', () => { expect(text).not.toBe(html); expect(text && text('formatted')).toBe('formatted'); - expect(html && html('formatted')).toBe('formatted'); + expect(html && html('formatted')).toBe('formatted'); }); test('can be an object, with separate text and html converter', () => { @@ -119,7 +119,7 @@ describe('FieldFormat class', () => { expect(text).not.toBe(html); expect(text && text('formatted text')).toBe('formatted text'); - expect(html && html('formatted html')).toBe('formatted html'); + expect(html && html('formatted html')).toBe('formatted html'); }); test('does not escape the output of the text converter', () => { @@ -131,10 +131,7 @@ describe('FieldFormat class', () => { test('does escape the output of the text converter if used in an html context', () => { const f = getTestFormat(undefined, constant('')); - const expected = trimEnd( - trimStart(f.convert('', 'html'), ''), - '' - ); + const expected = trimEnd(trimStart(f.convert('', 'html'), ''), ''); expect(expected).not.toContain('<'); }); @@ -143,7 +140,7 @@ describe('FieldFormat class', () => { const f = getTestFormat(undefined, constant(''), constant('')); expect(f.convert('', 'text')).toBe(''); - expect(f.convert('', 'html')).toBe(''); + expect(f.convert('', 'html')).toBe(''); }); }); @@ -157,7 +154,7 @@ describe('FieldFormat class', () => { test('formats a value as html, when specified via second param', () => { const f = getTestFormat(undefined, constant('text'), constant('html')); - expect(f.convert('val', 'html')).toBe('html'); + expect(f.convert('val', 'html')).toBe('html'); }); test('formats a value as " - " when no value is specified', () => { diff --git a/src/plugins/data/common/index_patterns/index_patterns/format_hit.ts b/src/plugins/data/common/index_patterns/index_patterns/format_hit.ts index 204fea175728..7f30b7ddf3e0 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/format_hit.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/format_hit.ts @@ -52,8 +52,6 @@ export function formatHitProvider(indexPattern: IndexPattern, defaultFormat: any function formatHit(hit: Record, type: string = 'html') { if (type === 'text') { - // formatHit of type text is for react components to get rid of - // since it's currently just used at the discover's doc view table, caching is not necessary const flattened = indexPattern.flattenHit(hit); const result: Record = {}; for (const [key, value] of Object.entries(flattened)) { diff --git a/src/plugins/data/common/opensearch_query/filters/range_filter.test.ts b/src/plugins/data/common/opensearch_query/filters/range_filter.test.ts index c4c64de72aa7..e57471101976 100644 --- a/src/plugins/data/common/opensearch_query/filters/range_filter.test.ts +++ b/src/plugins/data/common/opensearch_query/filters/range_filter.test.ts @@ -63,6 +63,44 @@ describe('Range filter builder', () => { }); }); + it('should return a range filter when passed a standard field of type BigInt', () => { + const field = getField('bytes'); + + const lowerBigInt = BigInt(Number.MIN_SAFE_INTEGER) - 11n; + const upperBigInt = BigInt(Number.MAX_SAFE_INTEGER) + 11n; + + expect( + buildRangeFilter(field, { gte: `${lowerBigInt}`, lte: `${upperBigInt}` }, indexPattern) + ).toEqual({ + meta: { + index: 'id', + params: {}, + }, + range: { + bytes: { + gte: lowerBigInt, + lte: upperBigInt, + }, + }, + }); + }); + + it('should return a range filter when passed a scripted field using BigInt', () => { + const field = getField('script number'); + + const lowerBigInt = BigInt(Number.MIN_SAFE_INTEGER) - 11n; + const upperBigInt = BigInt(Number.MAX_SAFE_INTEGER) + 11n; + + const filter = buildRangeFilter( + field, + { gte: `${lowerBigInt}`, lt: `${upperBigInt}` }, + indexPattern + ); + + expect(filter.script!.script.params).toHaveProperty('gte', lowerBigInt); + expect(filter.script!.script.params).toHaveProperty('lt', upperBigInt); + }); + it('should return a script filter when passed a scripted field', () => { const field = getField('script number'); diff --git a/src/plugins/data/common/opensearch_query/filters/range_filter.ts b/src/plugins/data/common/opensearch_query/filters/range_filter.ts index 12d49f5cc71c..e7a6565cbc39 100644 --- a/src/plugins/data/common/opensearch_query/filters/range_filter.ts +++ b/src/plugins/data/common/opensearch_query/filters/range_filter.ts @@ -124,19 +124,24 @@ export const buildRangeFilter = ( filter.meta.formattedValue = formattedValue; } - params = mapValues(params, (value: any) => (field.type === 'number' ? parseFloat(value) : value)); + params = mapValues(params, (value: any) => + field.type === 'number' && typeof value !== 'bigint' + ? isFinite(value) && (value > Number.MAX_SAFE_INTEGER || value < Number.MIN_SAFE_INTEGER) + ? BigInt(value) + : parseFloat(value) + : value + ); if ('gte' in params && 'gt' in params) throw new Error('gte and gt are mutually exclusive'); if ('lte' in params && 'lt' in params) throw new Error('lte and lt are mutually exclusive'); const totalInfinite = ['gt', 'lt'].reduce((acc: number, op: any) => { - const key = op in params ? op : `${op}e`; - const isInfinite = Math.abs(get(params, key)) === Infinity; + const key: keyof RangeFilterParams = op in params ? op : `${op}e`; + const isInfinite = + typeof params[key] !== 'bigint' && Math.abs(get(params, key) as number) === Infinity; if (isInfinite) { acc++; - - // @ts-ignore delete params[key]; } diff --git a/src/plugins/data/common/opensearch_query/kuery/ast/_generated_/kuery.js b/src/plugins/data/common/opensearch_query/kuery/ast/_generated_/kuery.js index 7faf78fc757d..7396607e55c7 100644 --- a/src/plugins/data/common/opensearch_query/kuery/ast/_generated_/kuery.js +++ b/src/plugins/data/common/opensearch_query/kuery/ast/_generated_/kuery.js @@ -233,8 +233,14 @@ module.exports = (function() { if (sequence === 'true') return buildLiteralNode(true); if (sequence === 'false') return buildLiteralNode(false); if (chars.includes(wildcardSymbol)) return buildWildcardNode(sequence); - const isNumberPattern = /^(-?[1-9]+\d*([.]\d+)?)$|^(-?0[.]\d*[1-9]+)$|^0$|^0.0$|^[.]\d{1,}$/ - return buildLiteralNode(isNumberPattern.test(sequence) ? Number(sequence) : sequence); + const isNumberPattern = /^(-?[1-9]+\d*([.]\d+)?)$|^(-?0[.]\d*[1-9]+)$|^0$|^0.0$|^[.]\d{1,}$/; + return buildLiteralNode( + isNumberPattern.test(sequence) + ? isFinite(sequence) && (sequence > Number.MAX_SAFE_INTEGER || sequence < Number.MIN_SAFE_INTEGER) + ? BigInt(sequence) + : Number(sequence) + : sequence + ); }, peg$c50 = { type: "any", description: "any character" }, peg$c51 = "*", diff --git a/src/plugins/data/common/opensearch_query/kuery/ast/ast.test.ts b/src/plugins/data/common/opensearch_query/kuery/ast/ast.test.ts index 3e5744ed72cd..2841e07c9adc 100644 --- a/src/plugins/data/common/opensearch_query/kuery/ast/ast.test.ts +++ b/src/plugins/data/common/opensearch_query/kuery/ast/ast.test.ts @@ -182,6 +182,21 @@ describe('kuery AST API', () => { expect(actual).toEqual(expected); }); + test('should support long numerals', () => { + const lowerBigInt = BigInt(Number.MIN_SAFE_INTEGER) - 11n; + const upperBigInt = BigInt(Number.MAX_SAFE_INTEGER) + 11n; + const expected = nodeTypes.function.buildNode('and', [ + nodeTypes.function.buildNode('range', 'bytes', { + gt: lowerBigInt, + }), + nodeTypes.function.buildNode('range', 'bytes', { + lt: upperBigInt, + }), + ]); + const actual = fromKueryExpression(`bytes > ${lowerBigInt} and bytes < ${upperBigInt}`); + expect(actual).toEqual(expected); + }); + test('should support inclusive range operators', () => { const expected = nodeTypes.function.buildNode('and', [ nodeTypes.function.buildNode('range', 'bytes', { @@ -284,6 +299,7 @@ describe('kuery AST API', () => { const booleanFalseLiteral = nodeTypes.literal.buildNode(false); const booleanTrueLiteral = nodeTypes.literal.buildNode(true); const numberLiteral = nodeTypes.literal.buildNode(42); + const upperBigInt = BigInt(Number.MAX_SAFE_INTEGER) + 11n; expect(fromLiteralExpression('foo')).toEqual(stringLiteral); expect(fromLiteralExpression('true')).toEqual(booleanTrueLiteral); @@ -302,6 +318,7 @@ describe('kuery AST API', () => { expect(fromLiteralExpression('790.9').value).toEqual(790.9); expect(fromLiteralExpression('0.0001').value).toEqual(0.0001); expect(fromLiteralExpression('96565646732345').value).toEqual(96565646732345); + expect(fromLiteralExpression(upperBigInt.toString()).value).toEqual(upperBigInt); expect(fromLiteralExpression('..4').value).toEqual('..4'); expect(fromLiteralExpression('.3text').value).toEqual('.3text'); @@ -316,6 +333,7 @@ describe('kuery AST API', () => { expect(fromLiteralExpression('-.4').value).toEqual('-.4'); expect(fromLiteralExpression('-0').value).toEqual('-0'); expect(fromLiteralExpression('00949').value).toEqual('00949'); + expect(fromLiteralExpression(`00${upperBigInt}`).value).toEqual(`00${upperBigInt}`); }); test('should allow escaping of special characters with a backslash', () => { diff --git a/src/plugins/data/common/opensearch_query/kuery/ast/kuery.peg b/src/plugins/data/common/opensearch_query/kuery/ast/kuery.peg index 625c5069f936..bc5e4a77ff19 100644 --- a/src/plugins/data/common/opensearch_query/kuery/ast/kuery.peg +++ b/src/plugins/data/common/opensearch_query/kuery/ast/kuery.peg @@ -247,8 +247,14 @@ UnquotedLiteral if (sequence === 'true') return buildLiteralNode(true); if (sequence === 'false') return buildLiteralNode(false); if (chars.includes(wildcardSymbol)) return buildWildcardNode(sequence); - const isNumberPattern = /^(-?[1-9]+\d*([.]\d+)?)$|^(-?0[.]\d*[1-9]+)$|^0$|^0.0$|^[.]\d{1,}$/ - return buildLiteralNode(isNumberPattern.test(sequence) ? Number(sequence) : sequence); + const isNumberPattern = /^(-?[1-9]+\d*([.]\d+)?)$|^(-?0[.]\d*[1-9]+)$|^0$|^0.0$|^[.]\d{1,}$/; + return buildLiteralNode( + isNumberPattern.test(sequence) + ? isFinite(sequence) && (sequence > Number.MAX_SAFE_INTEGER || sequence < Number.MIN_SAFE_INTEGER) + ? BigInt(sequence) + : Number(sequence) + : sequence + ); } UnquotedCharacter diff --git a/src/plugins/data/common/search/opensearch_search/types.ts b/src/plugins/data/common/search/opensearch_search/types.ts index a45d4d0660ce..3b93177bf201 100644 --- a/src/plugins/data/common/search/opensearch_search/types.ts +++ b/src/plugins/data/common/search/opensearch_search/types.ts @@ -33,6 +33,7 @@ import { Search } from '@opensearch-project/opensearch/api/requestParams'; import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types'; export const OPENSEARCH_SEARCH_STRATEGY = 'opensearch'; +export const OPENSEARCH_SEARCH_WITH_LONG_NUMERALS_STRATEGY = 'opensearch-with-long-numerals'; export interface ISearchOptions { /** @@ -43,6 +44,10 @@ export interface ISearchOptions { * Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. */ strategy?: string; + /** + * Use this option to enable support for long numerals. + */ + withLongNumeralsSupport?: boolean; } export type ISearchRequestParams> = { diff --git a/src/plugins/data/common/search/search_source/parse_json.ts b/src/plugins/data/common/search/search_source/parse_json.ts index 80a58d62ff7f..5e03234ce355 100644 --- a/src/plugins/data/common/search/search_source/parse_json.ts +++ b/src/plugins/data/common/search/search_source/parse_json.ts @@ -28,6 +28,7 @@ * under the License. */ +import { parse } from '@osd/std'; import { SearchSourceFields } from './types'; import { InvalidJSONProperty } from '../../../../opensearch_dashboards_utils/common'; @@ -35,7 +36,7 @@ export const parseSearchSourceJSON = (searchSourceJSON: string) => { // if we have a searchSource, set its values based on the searchSourceJson field let searchSourceValues: SearchSourceFields; try { - searchSourceValues = JSON.parse(searchSourceJSON); + searchSourceValues = parse(searchSourceJSON); } catch (e) { throw new InvalidJSONProperty( `Invalid JSON in search source. ${e.message} JSON: ${searchSourceJSON}` diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 9a85ec85ce87..abe6fa1b5cb4 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -81,6 +81,7 @@ */ import { setWith } from '@elastic/safer-lodash-set'; +import { stringify } from '@osd/std'; import { uniqueId, uniq, extend, pick, difference, omit, isObject, keys, isFunction } from 'lodash'; import { normalizeSortRequest } from './normalize_sort_request'; import { filterDocvalueFields } from './filter_docvalue_fields'; @@ -561,7 +562,7 @@ export class SearchSource { * @public */ public serialize() { const [searchSourceFields, references] = extractReferences(this.getSerializedFields()); - return { searchSourceJSON: JSON.stringify(searchSourceFields), references }; + return { searchSourceJSON: stringify(searchSourceFields), references }; } private getFilters(filterField: SearchSourceFields['filter']): Filter[] { diff --git a/src/plugins/data/common/search/types.ts b/src/plugins/data/common/search/types.ts index 5c4de4b0e426..e05c0adb46f0 100644 --- a/src/plugins/data/common/search/types.ts +++ b/src/plugins/data/common/search/types.ts @@ -76,6 +76,11 @@ export interface IOpenSearchDashboardsSearchResponse { */ isPartial?: boolean; + /** + * Indicates whether the results returned need long numerals treatment + */ + withLongNumeralsSupport?: boolean; + rawResponse: RawResponse; } diff --git a/src/plugins/data/public/search/search_interceptor.test.ts b/src/plugins/data/public/search/search_interceptor.test.ts index c19e92e3872e..3c75b74d2359 100644 --- a/src/plugins/data/public/search/search_interceptor.test.ts +++ b/src/plugins/data/public/search/search_interceptor.test.ts @@ -32,7 +32,11 @@ import { setImmediate } from 'timers'; import { CoreSetup } from '../../../../core/public'; import { coreMock } from '../../../../core/public/mocks'; -import { IOpenSearchSearchRequest } from '../../common/search'; +import { + IOpenSearchSearchRequest, + OPENSEARCH_SEARCH_STRATEGY, + OPENSEARCH_SEARCH_WITH_LONG_NUMERALS_STRATEGY, +} from '../../common'; import { SearchInterceptor } from './search_interceptor'; import { AbortError } from '../../common'; import { SearchTimeoutError, PainlessError } from './errors'; @@ -204,5 +208,82 @@ describe('SearchInterceptor', () => { }; response.subscribe({ error }); }); + + test('Should use the default strategy when no strategy or long-numerals support is requested', async () => { + const mockResponse: any = { result: 200 }; + mockCoreSetup.http.fetch.mockResolvedValueOnce(mockResponse); + const mockRequest: IOpenSearchSearchRequest = { + params: {}, + }; + const response = searchInterceptor.search(mockRequest); + + await response.toPromise(); + + expect(mockCoreSetup.http.fetch).toBeCalledWith( + expect.objectContaining({ + path: `/internal/search/${OPENSEARCH_SEARCH_STRATEGY}`, + }) + ); + }); + + test('Should use the correct strategy when long-numerals support with no specific strategy is requested', async () => { + const mockResponse: any = { result: 200 }; + mockCoreSetup.http.fetch.mockResolvedValueOnce(mockResponse); + const mockRequest: IOpenSearchSearchRequest = { + params: {}, + }; + const response = searchInterceptor.search(mockRequest, { + withLongNumeralsSupport: true, + }); + + await response.toPromise(); + + expect(mockCoreSetup.http.fetch).toBeCalledWith( + expect.objectContaining({ + path: `/internal/search/${OPENSEARCH_SEARCH_WITH_LONG_NUMERALS_STRATEGY}`, + }) + ); + }); + + test('Should use the requested strategy when no long-numerals support is requested', async () => { + const mockResponse: any = { result: 200 }; + mockCoreSetup.http.fetch.mockResolvedValueOnce(mockResponse); + const mockRequest: IOpenSearchSearchRequest = { + params: {}, + }; + + const strategy = 'unregistered-strategy'; + const response = searchInterceptor.search(mockRequest, { strategy }); + + await response.toPromise(); + + expect(mockCoreSetup.http.fetch).toBeCalledWith( + expect.objectContaining({ + path: `/internal/search/${strategy}`, + }) + ); + }); + + test('Should use the requested strategy even when long-numerals support is requested', async () => { + const mockResponse: any = { result: 200 }; + mockCoreSetup.http.fetch.mockResolvedValueOnce(mockResponse); + const mockRequest: IOpenSearchSearchRequest = { + params: {}, + }; + + const strategy = 'unregistered-strategy'; + const response = searchInterceptor.search(mockRequest, { + strategy, + withLongNumeralsSupport: true, + }); + + await response.toPromise(); + + expect(mockCoreSetup.http.fetch).toBeCalledWith( + expect.objectContaining({ + path: `/internal/search/${strategy}`, + }) + ); + }); }); }); diff --git a/src/plugins/data/public/search/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor.ts index 1c2fdb9c371c..153ac80a249c 100644 --- a/src/plugins/data/public/search/search_interceptor.ts +++ b/src/plugins/data/public/search/search_interceptor.ts @@ -32,6 +32,7 @@ import { get, trimEnd, debounce } from 'lodash'; import { BehaviorSubject, throwError, timer, defer, from, Observable, NEVER } from 'rxjs'; import { catchError, finalize } from 'rxjs/operators'; import { CoreStart, CoreSetup, ToastsSetup } from 'opensearch-dashboards/public'; +import { stringify } from '@osd/std'; import { getCombinedSignal, AbortError, @@ -39,6 +40,7 @@ import { IOpenSearchDashboardsSearchResponse, ISearchOptions, OPENSEARCH_SEARCH_STRATEGY, + OPENSEARCH_SEARCH_WITH_LONG_NUMERALS_STRATEGY, } from '../../common'; import { SearchUsageCollector } from './collectors'; import { SearchTimeoutError, PainlessError, isPainlessError } from './errors'; @@ -113,20 +115,40 @@ export class SearchInterceptor { protected runSearch( request: IOpenSearchDashboardsSearchRequest, signal: AbortSignal, - strategy?: string + strategy?: string, + withLongNumeralsSupport?: boolean ): Observable { const { id, ...searchRequest } = request; const path = trimEnd( - `/internal/search/${strategy || OPENSEARCH_SEARCH_STRATEGY}/${id || ''}`, + `/internal/search/${ + strategy || + (withLongNumeralsSupport + ? OPENSEARCH_SEARCH_WITH_LONG_NUMERALS_STRATEGY + : OPENSEARCH_SEARCH_STRATEGY) + }/${id || ''}`, '/' ); - const body = JSON.stringify(searchRequest); + /* When long-numerals support is not explicitly requested, it indicates that the response handler + * is not capable of handling BigInts. Ideally, because this is the outward request, that wouldn't + * matter and `body = stringify(searchRequest)` would be the best option. However, to maintain the + * existing behavior, we use @osd/std/json.stringify only when long-numerals support is explicitly + * requested. Otherwise, JSON.stringify is used with imprecise numbers for BigInts. + * + * ToDo: With OSD v3, change to `body = stringify(searchRequest)`. + */ + const body = withLongNumeralsSupport + ? stringify(searchRequest) + : JSON.stringify(searchRequest, (key: string, value: any) => + typeof value === 'bigint' ? Number(value) : value + ); + return from( this.deps.http.fetch({ method: 'POST', path, body, signal, + withLongNumeralsSupport, }) ); } @@ -214,7 +236,12 @@ export class SearchInterceptor { }); this.pendingCount$.next(this.pendingCount$.getValue() + 1); - return this.runSearch(request, combinedSignal, options?.strategy).pipe( + return this.runSearch( + request, + combinedSignal, + options?.strategy, + options?.withLongNumeralsSupport + ).pipe( catchError((e: any) => { return throwError( this.handleSearchError(e, request, timeoutSignal, options?.abortSignal) diff --git a/src/plugins/data/public/ui/filter_bar/filter_bar.tsx b/src/plugins/data/public/ui/filter_bar/filter_bar.tsx index 94f9c6590b91..c44ba583c3b4 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_bar.tsx +++ b/src/plugins/data/public/ui/filter_bar/filter_bar.tsx @@ -38,6 +38,7 @@ import { import { FormattedMessage, InjectedIntl, injectI18n } from '@osd/i18n/react'; import classNames from 'classnames'; import React, { useState } from 'react'; +import { stringify } from '@osd/std'; import { FilterEditor } from './filter_editor'; import { FilterItem } from './filter_item'; @@ -143,7 +144,7 @@ function FilterBarUI(props: Props) { indexPatterns={props.indexPatterns} onSubmit={onAdd} onCancel={() => setIsAddFilterPopoverOpen(false)} - key={JSON.stringify(newFilter)} + key={stringify(newFilter)} /> diff --git a/src/plugins/data/public/ui/filter_bar/filter_editor/index.tsx b/src/plugins/data/public/ui/filter_bar/filter_editor/index.tsx index 7a4bb433bd10..b73eeb84df35 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_editor/index.tsx +++ b/src/plugins/data/public/ui/filter_bar/filter_editor/index.tsx @@ -47,6 +47,7 @@ import { i18n } from '@osd/i18n'; import { FormattedMessage, InjectedIntl, injectI18n } from '@osd/i18n/react'; import { get } from 'lodash'; import React, { Component } from 'react'; +import { parse, stringify } from '@osd/std'; import { GenericComboBox, GenericComboBoxProps } from './generic_combo_box'; import { getFieldFromFilter, @@ -99,7 +100,7 @@ class FilterEditorUI extends Component { params: getFilterParams(props.filter), useCustomLabel: props.filter.meta.alias !== null, customLabel: props.filter.meta.alias, - queryDsl: JSON.stringify(cleanFilter(props.filter), null, 2), + queryDsl: stringify(cleanFilter(props.filter), null, 2), isCustomEditorOpen: this.isUnknownFilterType(), }; } @@ -430,7 +431,7 @@ class FilterEditorUI extends Component { if (isCustomEditorOpen) { try { - return Boolean(JSON.parse(queryDsl)); + return Boolean(parse(queryDsl)); } catch (e) { return false; } @@ -501,7 +502,7 @@ class FilterEditorUI extends Component { if (isCustomEditorOpen) { const { index, disabled, negate } = this.props.filter.meta; const newIndex = index || this.props.indexPatterns[0].id!; - const body = JSON.parse(queryDsl); + const body = parse(queryDsl); const filter = buildCustomFilter(newIndex, body, disabled, negate, alias, $state.store); this.props.onSubmit(filter); } else if (indexPattern && field && operator) { diff --git a/src/plugins/data/public/ui/filter_bar/filter_editor/lib/filter_label.tsx b/src/plugins/data/public/ui/filter_bar/filter_editor/lib/filter_label.tsx index 3861c8dbce4a..529053ffd042 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_editor/lib/filter_label.tsx +++ b/src/plugins/data/public/ui/filter_bar/filter_editor/lib/filter_label.tsx @@ -31,6 +31,7 @@ import React, { Fragment } from 'react'; import { EuiTextColor } from '@elastic/eui'; import { i18n } from '@osd/i18n'; +import { stringify } from '@osd/std'; import { existsOperator, isOneOfOperator } from './filter_operators'; import { Filter, FILTERS } from '../../../../../common'; import type { FilterLabelStatus } from '../../filter_item'; @@ -119,7 +120,7 @@ export default function FilterLabel({ filter, valueLabel, filterLabelStatus }: F return ( {prefix} - {getValue(`${JSON.stringify(filter.query) || filter.meta.value}`)} + {getValue(`${stringify(filter.query) || filter.meta.value}`)} ); } diff --git a/src/plugins/data/public/ui/filter_bar/filter_editor/value_input_type.tsx b/src/plugins/data/public/ui/filter_bar/filter_editor/value_input_type.tsx index 360eecc91674..1bd1a653e027 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_editor/value_input_type.tsx +++ b/src/plugins/data/public/ui/filter_bar/filter_editor/value_input_type.tsx @@ -68,7 +68,13 @@ class ValueInputTypeUI extends Component { => { // if data source feature is disabled, return default opensearch client of current user const client = request.dataSourceId && context.dataSource ? await context.dataSource.opensearch.getClient(request.dataSourceId) + : withLongNumeralsSupport + ? context.core.opensearch.client.asCurrentUserWithLongNumeralsSupport : context.core.opensearch.client.asCurrentUser; return client; }; diff --git a/src/plugins/data/server/search/opensearch_search/opensearch_search_strategy.ts b/src/plugins/data/server/search/opensearch_search/opensearch_search_strategy.ts index ba50740e6baf..5eb290517792 100644 --- a/src/plugins/data/server/search/opensearch_search/opensearch_search_strategy.ts +++ b/src/plugins/data/server/search/opensearch_search/opensearch_search_strategy.ts @@ -49,11 +49,11 @@ export const opensearchSearchStrategyProvider = ( config$: Observable, logger: Logger, usage?: SearchUsage, - dataSource?: DataSourcePluginSetup + dataSource?: DataSourcePluginSetup, + withLongNumeralsSupport?: boolean ): ISearchStrategy => { return { search: async (context, request, options) => { - logger.debug(`search ${request.params?.index}`); const config = await config$.pipe(first()).toPromise(); const uiSettingsClient = await context.core.uiSettings.client; @@ -73,7 +73,7 @@ export const opensearchSearchStrategyProvider = ( }); try { - const client = await decideClient(context, request); + const client = await decideClient(context, request, withLongNumeralsSupport); const promise = shimAbortSignal(client.search(params), options?.abortSignal); const { body: rawResponse } = (await promise) as ApiResponse>; @@ -87,6 +87,7 @@ export const opensearchSearchStrategyProvider = ( isRunning: false, rawResponse, ...getTotalLoaded(rawResponse._shards), + withLongNumeralsSupport, }; } catch (e) { if (usage) usage.trackError(); diff --git a/src/plugins/data/server/search/routes/search.ts b/src/plugins/data/server/search/routes/search.ts index 1fa4c00e2463..d569de539f39 100644 --- a/src/plugins/data/server/search/routes/search.ts +++ b/src/plugins/data/server/search/routes/search.ts @@ -60,7 +60,7 @@ export function registerSearchRoute( const [, , selfStart] = await getStartServices(); try { - const response = await selfStart.search.search( + const { withLongNumeralsSupport, ...response } = await selfStart.search.search( context, { ...searchRequest, id }, { @@ -76,6 +76,7 @@ export function registerSearchRoute( rawResponse: shimHitsTotal(response.rawResponse), }, }, + withLongNumeralsSupport, }); } catch (err) { return res.customError({ diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index 6620b88a0fe3..feb1a3157794 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -65,6 +65,7 @@ import { SearchSourceDependencies, SearchSourceService, searchSourceRequiredUiSettings, + OPENSEARCH_SEARCH_WITH_LONG_NUMERALS_STRATEGY, } from '../../common/search'; import { getShardDelayBucketAgg, @@ -133,6 +134,17 @@ export class SearchService implements Plugin { ) ); + this.registerSearchStrategy( + OPENSEARCH_SEARCH_WITH_LONG_NUMERALS_STRATEGY, + opensearchSearchStrategyProvider( + this.initializerContext.config.legacy.globalConfig$, + this.logger, + usage, + dataSource, + true + ) + ); + core.savedObjects.registerType(searchTelemetry); if (usageCollection) { registerUsageCollector(usageCollection, this.initializerContext); @@ -242,7 +254,6 @@ export class SearchService implements Plugin { name: string, strategy: ISearchStrategy ) => { - this.logger.debug(`Register strategy ${name}`); this.searchStrategies[name] = strategy; }; @@ -265,7 +276,6 @@ export class SearchService implements Plugin { >( name: string ): ISearchStrategy => { - this.logger.debug(`Get strategy ${name}`); const strategy = this.searchStrategies[name]; if (!strategy) { throw new Error(`Search strategy ${name} not found`); diff --git a/src/plugins/discover/public/application/components/data_grid/data_grid_table_cell_value.tsx b/src/plugins/discover/public/application/components/data_grid/data_grid_table_cell_value.tsx index 4089c90ede15..69e84e0b5e5a 100644 --- a/src/plugins/discover/public/application/components/data_grid/data_grid_table_cell_value.tsx +++ b/src/plugins/discover/public/application/components/data_grid/data_grid_table_cell_value.tsx @@ -13,6 +13,7 @@ import { EuiDescriptionListTitle, EuiDescriptionListDescription, } from '@elastic/eui'; +import { stringify } from '@osd/std'; import { IndexPattern } from '../../../opensearch_dashboards_services'; import { OpenSearchSearchHit } from '../../doc_views/doc_views_types'; @@ -23,7 +24,7 @@ function fetchSourceTypeDataCell( isDetails: boolean ) { if (isDetails) { - return {JSON.stringify(row[columnId], null, 2)}; + return {stringify(row[columnId], null, 2)}; } const formattedRow = idxPattern.formatHit(row); @@ -58,12 +59,12 @@ export const fetchTableDataCell = ( return -; } - if (!fieldInfo?.type && flattenedRow && typeof flattenedRow[columnId] === 'object') { + if (!fieldInfo?.type && typeof flattenedRow?.[columnId] === 'object') { if (isDetails) { - return {JSON.stringify(flattenedRow[columnId], null, 2)}; + return {stringify(flattenedRow[columnId], null, 2)}; } - return {JSON.stringify(flattenedRow[columnId])}; + return {stringify(flattenedRow[columnId])}; } if (fieldInfo?.type === '_source') { @@ -74,7 +75,7 @@ export const fetchTableDataCell = ( if (typeof formattedValue === 'undefined') { return -; } else { - const sanitizedCellValue = dompurify.sanitize(idxPattern.formatField(singleRow, columnId)); + const sanitizedCellValue = dompurify.sanitize(formattedValue); return ( // eslint-disable-next-line react/no-danger diff --git a/src/plugins/discover/public/application/components/doc/use_opensearch_doc_search.ts b/src/plugins/discover/public/application/components/doc/use_opensearch_doc_search.ts index b5ca9fec1c2f..4389485a3a77 100644 --- a/src/plugins/discover/public/application/components/doc/use_opensearch_doc_search.ts +++ b/src/plugins/discover/public/application/components/doc/use_opensearch_doc_search.ts @@ -81,13 +81,18 @@ export function useOpenSearchDocSearch({ setIndexPattern(indexPatternEntity); const { rawResponse } = await getServices() - .data.search.search({ - dataSourceId: indexPatternEntity.dataSourceRef?.id, - params: { - index, - body: buildSearchBody(id, indexPatternEntity), + .data.search.search( + { + dataSourceId: indexPatternEntity.dataSourceRef?.id, + params: { + index, + body: buildSearchBody(id, indexPatternEntity), + }, }, - }) + { + withLongNumeralsSupport: true, + } + ) .toPromise(); const hits = rawResponse.hits; diff --git a/src/plugins/discover/public/application/components/json_code_block/json_code_block.tsx b/src/plugins/discover/public/application/components/json_code_block/json_code_block.tsx index f33cae438cb2..7b9dbc450c30 100644 --- a/src/plugins/discover/public/application/components/json_code_block/json_code_block.tsx +++ b/src/plugins/discover/public/application/components/json_code_block/json_code_block.tsx @@ -31,6 +31,7 @@ import React from 'react'; import { EuiCodeBlock } from '@elastic/eui'; import { i18n } from '@osd/i18n'; +import { stringify } from '@osd/std'; import { DocViewRenderProps } from '../../doc_views/doc_views_types'; export function JsonCodeBlock({ hit }: DocViewRenderProps) { @@ -39,7 +40,7 @@ export function JsonCodeBlock({ hit }: DocViewRenderProps) { }); return ( - {JSON.stringify(hit, null, 2)} + {stringify(hit, null, 2)} ); } diff --git a/src/plugins/discover/public/application/view_components/utils/use_search.ts b/src/plugins/discover/public/application/view_components/utils/use_search.ts index 7e284ef6f443..06eabb1e139f 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_search.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_search.ts @@ -156,6 +156,7 @@ export const useSearch = (services: DiscoverViewServices) => { // Execute the search const fetchResp = await searchSource.fetch({ abortSignal: fetchStateRef.current.abortController.signal, + withLongNumeralsSupport: true, }); inspectorRequest diff --git a/src/plugins/inspector/public/views/requests/components/details/req_details_request.tsx b/src/plugins/inspector/public/views/requests/components/details/req_details_request.tsx index fe758e128322..adc00588307a 100644 --- a/src/plugins/inspector/public/views/requests/components/details/req_details_request.tsx +++ b/src/plugins/inspector/public/views/requests/components/details/req_details_request.tsx @@ -31,6 +31,7 @@ import React, { Component } from 'react'; import PropTypes from 'prop-types'; import { EuiCodeBlock } from '@elastic/eui'; +import { stringify } from '@osd/std'; import { Request } from '../../../../../common/adapters/request/types'; import { RequestDetailsProps } from '../types'; @@ -55,7 +56,7 @@ export class RequestDetailsRequest extends Component { isCopyable data-test-subj="inspectorRequestBody" > - {JSON.stringify(json, null, 2)} + {stringify(json, null, 2)} ); } diff --git a/src/plugins/inspector/public/views/requests/components/details/req_details_response.tsx b/src/plugins/inspector/public/views/requests/components/details/req_details_response.tsx index d5cd80ec35e2..4693b0dbfba5 100644 --- a/src/plugins/inspector/public/views/requests/components/details/req_details_response.tsx +++ b/src/plugins/inspector/public/views/requests/components/details/req_details_response.tsx @@ -31,6 +31,7 @@ import React, { Component } from 'react'; import PropTypes from 'prop-types'; import { EuiCodeBlock } from '@elastic/eui'; +import { stringify } from '@osd/std'; import { Request } from '../../../../../common/adapters/request/types'; import { RequestDetailsProps } from '../types'; @@ -58,7 +59,7 @@ export class RequestDetailsResponse extends Component { isCopyable data-test-subj="inspectorResponseBody" > - {JSON.stringify(responseJSON, null, 2)} + {stringify(responseJSON, null, 2)} ); } diff --git a/src/plugins/saved_objects/public/saved_object/helpers/apply_opensearch_resp.ts b/src/plugins/saved_objects/public/saved_object/helpers/apply_opensearch_resp.ts index 754ac59e445c..a7e864a49751 100644 --- a/src/plugins/saved_objects/public/saved_object/helpers/apply_opensearch_resp.ts +++ b/src/plugins/saved_objects/public/saved_object/helpers/apply_opensearch_resp.ts @@ -29,6 +29,7 @@ */ import _ from 'lodash'; +import { parse } from '@osd/std'; import { OpenSearchResponse, SavedObject, @@ -109,7 +110,7 @@ export async function applyOpenSearchResp( // remember the reference - this is required for error handling on legacy imports savedObject.unresolvedIndexPatternReference = { name: 'kibanaSavedObjectMeta.searchSourceJSON.index', - id: JSON.parse(meta.searchSourceJSON).index, + id: parse(meta.searchSourceJSON).index, type: 'index-pattern', }; } diff --git a/src/plugins/saved_objects/public/saved_object/helpers/serialize_saved_object.ts b/src/plugins/saved_objects/public/saved_object/helpers/serialize_saved_object.ts index 7717574d47d2..4a00ec28f5f2 100644 --- a/src/plugins/saved_objects/public/saved_object/helpers/serialize_saved_object.ts +++ b/src/plugins/saved_objects/public/saved_object/helpers/serialize_saved_object.ts @@ -29,6 +29,7 @@ */ import _ from 'lodash'; +import { stringify } from '@osd/std'; import { SavedObject, SavedObjectConfig } from '../../types'; import { extractSearchSourceReferences, expandShorthand } from '../../../../data/public'; @@ -64,7 +65,7 @@ export function serializeSavedObject(savedObject: SavedObject, config: SavedObje const [searchSourceFields, searchSourceReferences] = extractSearchSourceReferences( savedObject.searchSourceFields ); - const searchSourceJSON = JSON.stringify(searchSourceFields); + const searchSourceJSON = stringify(searchSourceFields); attributes.kibanaSavedObjectMeta = { searchSourceJSON }; references.push(...searchSourceReferences); } diff --git a/src/plugins/vis_type_timeline/public/_generated_/chain.js b/src/plugins/vis_type_timeline/common/_generated_/chain.js similarity index 100% rename from src/plugins/vis_type_timeline/public/_generated_/chain.js rename to src/plugins/vis_type_timeline/common/_generated_/chain.js diff --git a/src/plugins/vis_type_timeline/public/components/timeline_expression_input_helpers.ts b/src/plugins/vis_type_timeline/public/components/timeline_expression_input_helpers.ts index c4c5b5a03d94..fc94037a1618 100644 --- a/src/plugins/vis_type_timeline/public/components/timeline_expression_input_helpers.ts +++ b/src/plugins/vis_type_timeline/public/components/timeline_expression_input_helpers.ts @@ -35,7 +35,7 @@ import { monaco } from '@osd/monaco'; import { Parser } from 'pegjs'; // @ts-ignore -import { parse } from '../_generated_/chain'; +import { parse } from '../../common/_generated_/chain'; import { ArgValueSuggestions, FunctionArg, Location } from '../helpers/arg_value_suggestions'; import { ITimelineFunction, TimelineFunctionArgs } from '../../common/types'; diff --git a/tasks/config/peg.js b/tasks/config/peg.js index 26cbe51eba6f..9acb7b3d3a6c 100644 --- a/tasks/config/peg.js +++ b/tasks/config/peg.js @@ -34,10 +34,28 @@ module.exports = { dest: 'src/plugins/data/common/opensearch_query/kuery/ast/_generated_/kuery.js', options: { allowedStartRules: ['start', 'Literal'], + wrapper: function (src, parser) { + return ( + '/*\n * SPDX-License-Identifier: Apache-2.0\n *\n * The OpenSearch Contributors require contributions made to\n * this file be licensed under the Apache-2.0 license or a\n * compatible open source license.\n *\n * Any modifications Copyright OpenSearch Contributors. See\n * GitHub history for details.\n */\n\n' + + 'module.exports = ' + + parser + + ';\n' + ); + }, }, }, timeline_chain: { - src: 'src/plugins/vis_type_timeline/public/chain.peg', - dest: 'src/plugins/vis_type_timeline/public/_generated_/chain.js', + src: 'src/plugins/vis_type_timeline/common/chain.peg', + dest: 'src/plugins/vis_type_timeline/common/_generated_/chain.js', + options: { + wrapper: function (src, parser) { + return ( + '/*\n * SPDX-License-Identifier: Apache-2.0\n *\n * The OpenSearch Contributors require contributions made to\n * this file be licensed under the Apache-2.0 license or a\n * compatible open source license.\n *\n * Any modifications Copyright OpenSearch Contributors. See\n * GitHub history for details.\n */\n\n' + + 'module.exports = ' + + parser + + ';\n' + ); + }, + }, }, }; diff --git a/tasks/config/run.js b/tasks/config/run.js index f0e4fb7fcd66..c02d75b2afc0 100644 --- a/tasks/config/run.js +++ b/tasks/config/run.js @@ -28,8 +28,6 @@ * under the License. */ -import { getFunctionalTestGroupRunConfigs } from '../function_test_groups'; - const { version } = require('../../package.json'); const OPENSEARCH_DASHBOARDS_INSTALL_DIR = process.env.OPENSEARCH_DASHBOARDS_INSTALL_DIR || @@ -232,9 +230,5 @@ module.exports = function () { cmd: YARN, args: ['run', 'grunt', 'test:projects'], }, - - ...getFunctionalTestGroupRunConfigs({ - opensearchDashboardsInstallDir: OPENSEARCH_DASHBOARDS_INSTALL_DIR, - }), }; }; diff --git a/yarn.lock b/yarn.lock index 0c6ae9ebba0f..6e79ec173ae4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1383,10 +1383,10 @@ resolved "https://registry.yarnpkg.com/@elastic/node-crypto/-/node-crypto-1.1.1.tgz#619b70322c9cce4a7ee5fbf8f678b1baa7f06095" integrity sha512-F6tIk8Txdqjg8Siv60iAvXzO9ZdQI87K3sS/fh5xd2XaWK+T5ZfqeTvsT7srwG6fr6uCBfuQEJV1KBBl+JpLZA== -"@elastic/numeral@^2.5.0": - version "2.5.1" - resolved "https://registry.yarnpkg.com/@elastic/numeral/-/numeral-2.5.1.tgz#96acf39c3d599950646ef8ccfd24a3f057cf4932" - integrity sha512-Tby6TKjixRFY+atVNeYUdGr9m0iaOq8230KTwn8BbUhkh7LwozfgKq0U98HRX7n63ZL62szl+cDKTYzh5WPCFQ== +"@elastic/numeral@npm:@amoo-miki/numeral@2.6.0": + version "2.6.0" + resolved "https://registry.yarnpkg.com/@amoo-miki/numeral/-/numeral-2.6.0.tgz#3a114ef81cd36ab8207dc771751e47a1323f3a6f" + integrity sha512-P2w5/ufeYdMuvY6Y1BiI3Gzj4MQ+87NAvGTQ0Qvx1wJbTh8q1b+ZWUGJjT5h9xl9BgYQ6sF4X8UZgIwW5T/ljg== "@elastic/request-crypto@2.0.0": version "2.0.0"