From e3e15cf46998120b017ac620afad0dbba2d6dd7b Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 29 Sep 2020 12:09:47 -0500 Subject: [PATCH 01/13] Scripting: enable regular expressions by default * Setting `script.painless.regex.enabled` has a new option, `use-factor`, the default. This defaults to using regular expressions but limiting the complexity of the regular expressions. In addition to `use-factor`, the setting can be `true`, as before, which enables regular expressions without limiting them. `false` totally disables regular expressions, which was the old default. * New setting `script.painless.regex.limit-factor`. This limits regular expression complexity by limiting the number characters a regular expression can consider based on input length. The default is `6`, so a regular expression can consider `6` * input length number of characters. With input `foobarbaz` (length `9`), for example, the regular expression can consider `54` (`6 * 9`) characters. This reduces the impact of exponential backtracking in Java's regular expression engine. * add `@inject_constant` annotation to whitelist. This annotation signals that a compiler settings will be injected at the beginning of a whitelisted method. The format is `argnum=settingname`: `1=foo_setting 2=bar_setting`. Argument numbers must start at one and must be sequential. * Augment `Pattern.split(CharSequence)` `Pattern.split(CharSequence, int)`, `Pattern.splitAsStream(CharSequence)` `Pattern.matcher(CharSequence)` to take the value of `script.painless.regex.limit-factor` as a an injected parameter, limiting as explained above when this setting is in use. Fixes: #49873 --- .../annotation/InjectConstantAnnotation.java | 31 +++ .../InjectConstantAnnotationParser.java | 49 ++++ .../annotation/WhitelistAnnotationParser.java | 3 +- .../org/elasticsearch/painless/Compiler.java | 4 +- .../painless/CompilerSettings.java | 87 ++++++- .../java/org/elasticsearch/painless/Def.java | 70 ++++-- .../elasticsearch/painless/DefBootstrap.java | 17 +- .../elasticsearch/painless/FunctionRef.java | 26 ++- .../painless/LambdaBootstrap.java | 34 ++- .../elasticsearch/painless/MethodWriter.java | 17 +- .../painless/PainlessPlugin.java | 2 +- .../painless/PainlessScriptEngine.java | 8 + .../painless/WriterConstants.java | 5 +- .../painless/api/Augmentation.java | 163 +++++++------ .../painless/api/LimitedCharSequence.java | 134 +++++++++++ .../lookup/PainlessLookupBuilder.java | 14 +- .../lookup/PainlessLookupUtility.java | 33 ++- .../phase/DefaultSemanticAnalysisPhase.java | 15 +- .../phase/DefaultUserTreeToIRTreePhase.java | 47 +++- .../phase/PainlessUserTreeToIRTreePhase.java | 2 +- .../painless/symbol/ScriptScope.java | 2 + .../painless/spi/java.util.regex.txt | 9 +- .../painless/AugmentationTests.java | 213 +++++++++++++++++ .../painless/DefBootstrapTests.java | 10 + .../FeatureTestAugmentationObject.java | 24 ++ .../painless/FeatureTestObject.java | 24 ++ .../painless/FeatureTestObject2.java | 31 +++ .../painless/InjectionTests.java | 217 ++++++++++++++++++ .../spi/org.elasticsearch.painless.test | 19 +- .../script/ScriptContextInfo.java | 2 +- 30 files changed, 1165 insertions(+), 147 deletions(-) create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject2.java create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/InjectionTests.java diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java new file mode 100644 index 0000000000000..fde52ed7aef2c --- /dev/null +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.spi.annotation; + +import java.util.Collections; +import java.util.List; + +public class InjectConstantAnnotation { + public static final String NAME = "inject_constant"; + public final List injects; + public InjectConstantAnnotation(List injects) { + this.injects = Collections.unmodifiableList(injects); + } +} diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java new file mode 100644 index 0000000000000..3c88ae3f529fd --- /dev/null +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.spi.annotation; + +import java.util.ArrayList; +import java.util.Map; + +public class InjectConstantAnnotationParser implements WhitelistAnnotationParser { + + public static final InjectConstantAnnotationParser INSTANCE = new InjectConstantAnnotationParser(); + + private InjectConstantAnnotationParser() {} + + @Override + public Object parse(Map arguments) { + if (arguments.isEmpty()) { + throw new IllegalArgumentException("[@inject_constant] requires at least one name to inject"); + } + ArrayList argList = new ArrayList<>(arguments.size()); + for (int i = 1; i <= arguments.size(); i++) { + String argNum = Integer.toString(i); + if (arguments.containsKey(argNum) == false) { + throw new IllegalArgumentException("[@inject_constant] missing argument number [" + argNum + "]"); + } + // TODO(stu): Jack, how do I verify against CompilerSettings. + // answer: do validation in PainlessLookupBuilder + argList.add(arguments.get(argNum)); + } + + return new InjectConstantAnnotation(argList); + } +} diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java index ecf1c45c7602f..43acf061e062c 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java @@ -35,7 +35,8 @@ public interface WhitelistAnnotationParser { Stream.of( new AbstractMap.SimpleEntry<>(NoImportAnnotation.NAME, NoImportAnnotationParser.INSTANCE), new AbstractMap.SimpleEntry<>(DeprecatedAnnotation.NAME, DeprecatedAnnotationParser.INSTANCE), - new AbstractMap.SimpleEntry<>(NonDeterministicAnnotation.NAME, NonDeterministicAnnotationParser.INSTANCE) + new AbstractMap.SimpleEntry<>(NonDeterministicAnnotation.NAME, NonDeterministicAnnotationParser.INSTANCE), + new AbstractMap.SimpleEntry<>(InjectConstantAnnotation.NAME, InjectConstantAnnotationParser.INSTANCE) ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) ); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java index dc2ffb220720b..890b2f22d13ae 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java @@ -220,7 +220,7 @@ ScriptScope compile(Loader loader, String name, String source, CompilerSettings ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1); new PainlessSemanticHeaderPhase().visitClass(root, scriptScope); new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope); - // TODO(stu): Make this phase optional #60156 + // TODO: Make this phase optional #60156 new DocFieldsPhase().visitClass(root, scriptScope); new PainlessUserTreeToIRTreePhase().visitClass(root, scriptScope); ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode(); @@ -255,7 +255,7 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1); new PainlessSemanticHeaderPhase().visitClass(root, scriptScope); new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope); - // TODO(stu): Make this phase optional #60156 + // TODO: Make this phase optional #60156 new DocFieldsPhase().visitClass(root, scriptScope); new PainlessUserTreeToIRTreePhase().visitClass(root, scriptScope); ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode(); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java index e723081e36c0c..57270e8792658 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java @@ -21,16 +21,26 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.painless.api.Augmentation; + +import java.util.HashMap; +import java.util.Map; /** * Settings to use when compiling a script. */ public final class CompilerSettings { /** - * Are regexes enabled? This is a node level setting because regexes break out of painless's lovely sandbox and can cause stack - * overflows and we can't analyze the regex to be sure it won't. + * Are regexes enabled? If */ - public static final Setting REGEX_ENABLED = Setting.boolSetting("script.painless.regex.enabled", false, Property.NodeScope); + public static final Setting REGEX_ENABLED = + new Setting<>("script.painless.regex.enabled", RegexEnabled.USE_FACTOR.value, RegexEnabled::parse, Property.NodeScope); + + /** + * How complex can a regex be? This is the number of characters that can be considered expressed as a multiple of string length. + */ + public static final Setting REGEX_LIMIT_FACTOR = + Setting.intSetting("script.painless.regex.limit-factor", 6, 1, Property.NodeScope); /** * Constant to be used when specifying the maximum loop counter when compiling a script. @@ -65,12 +75,20 @@ public final class CompilerSettings { * For testing. Do not use. */ private int initialCallSiteDepth = 0; + private int testInject0 = 2; + private int testInject1 = 4; + private int testInject2 = 6; /** - * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple - * looking regexes can cause stack overflows. + * Are regexes enabled? Defaults to using the factor setting. + */ + private RegexEnabled regexesEnabled = RegexEnabled.USE_FACTOR; + + + /** + * How complex can regexes be? Expressed as a multiple of the input string. */ - private boolean regexesEnabled = false; + private int regexLimitFactor = 0; /** * Returns the value for the cumulative total number of statements that can be made in all loops @@ -123,10 +141,9 @@ public void setInitialCallSiteDepth(int depth) { } /** - * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple - * looking regexes can cause stack overflows. + * Are regexes enabled? */ - public boolean areRegexesEnabled() { + public RegexEnabled areRegexesEnabled() { return regexesEnabled; } @@ -134,7 +151,57 @@ public boolean areRegexesEnabled() { * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple * looking regexes can cause stack overflows. */ - public void setRegexesEnabled(boolean regexesEnabled) { + public void setRegexesEnabled(RegexEnabled regexesEnabled) { this.regexesEnabled = regexesEnabled; } + + public void setRegexLimitFactor(int regexLimitFactor) { + this.regexLimitFactor = regexLimitFactor; + } + + public int getRegexLimitFactor() { + return regexLimitFactor; + } + + public Map asMap() { + int regexLimitFactor = this.regexLimitFactor; + if (regexesEnabled == RegexEnabled.TRUE) { + regexLimitFactor = Augmentation.UNLIMITED_PATTERN_FACTOR; + } else if (regexesEnabled == RegexEnabled.FALSE) { + regexLimitFactor = Augmentation.DISABLED_PATTERN_FACTOR; + } + Map map = new HashMap<>(); + map.put("regex_limit_factor", regexLimitFactor); + + // for testing only + map.put("testInject0", testInject0); + map.put("testInject1", testInject1); + map.put("testInject2", testInject2); + + return map; + } + + public enum RegexEnabled { + TRUE("true"), + FALSE("false"), + USE_FACTOR("use-factor"); + final String value; + + RegexEnabled(String value) { + this.value = value; + } + + public static RegexEnabled parse(String value) { + if (TRUE.value.equals(value)) { + return TRUE; + } else if (FALSE.value.equals(value)) { + return FALSE; + } else if (USE_FACTOR.value.equals(value)) { + return USE_FACTOR; + } + throw new IllegalArgumentException( + "invalid value [" + value + "] must be one of [" + TRUE.value + "," + FALSE.value + "," + USE_FACTOR.value + "]" + ); + } + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java index dd7e00a6c8a2b..9f83c0b434a38 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java @@ -191,8 +191,8 @@ static MethodHandle arrayLengthGetter(Class arrayType) { * @throws IllegalArgumentException if no matching whitelisted method was found. * @throws Throwable if a method reference cannot be converted to an functional interface */ - static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable functions, - MethodHandles.Lookup methodHandlesLookup, MethodType callSiteType, Class receiverClass, String name, Object args[]) + static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable functions, Map constants, + MethodHandles.Lookup methodHandlesLookup, MethodType callSiteType, Class receiverClass, String name, Object[] args) throws Throwable { String recipeString = (String) args[0]; @@ -206,7 +206,15 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable fu "[" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + (numArguments - 1) + "] not found"); } - return painlessMethod.methodHandle; + MethodHandle handle = painlessMethod.methodHandle; + Object[] injections = PainlessLookupUtility.buildInjections(painlessMethod, constants); + + if (injections.length > 0) { + // method handle contains the "this" pointer so start injections at 1 + handle = MethodHandles.insertArguments(handle, 1, injections); + } + + return handle; } // convert recipe string to a bitset for convenience (the code below should be refactored...) @@ -236,7 +244,13 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable fu "dynamic method [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + arity + "] not found"); } - MethodHandle handle = method.methodHandle; + MethodHandle handle = method.methodHandle; + Object[] injections = PainlessLookupUtility.buildInjections(method, constants); + + if (injections.length > 0) { + // method handle contains the "this" pointer so start injections at 1 + handle = MethodHandles.insertArguments(handle, 1, injections); + } int replaced = 0; upTo = 1; @@ -257,22 +271,25 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable fu // we have everything. filter = lookupReferenceInternal(painlessLookup, functions, + constants, methodHandlesLookup, interfaceType, type, call, - numCaptures); + numCaptures + ); } else if (signature.charAt(0) == 'D') { // the interface type is now known, but we need to get the implementation. // this is dynamically based on the receiver type (and cached separately, underneath // this cache). It won't blow up since we never nest here (just references) - Class captures[] = new Class[numCaptures]; + Class[] captures = new Class[numCaptures]; for (int capture = 0; capture < captures.length; capture++) { captures[capture] = callSiteType.parameterType(i + 1 + capture); } MethodType nestedType = MethodType.methodType(interfaceType, captures); CallSite nested = DefBootstrap.bootstrap(painlessLookup, functions, + constants, methodHandlesLookup, call, nestedType, @@ -300,8 +317,10 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable fu * This is just like LambdaMetaFactory, only with a dynamic type. The interface type is known, * so we simply need to lookup the matching implementation method based on receiver type. */ - static MethodHandle lookupReference(PainlessLookup painlessLookup, FunctionTable functions, - MethodHandles.Lookup methodHandlesLookup, String interfaceClass, Class receiverClass, String name) throws Throwable { + static MethodHandle lookupReference(PainlessLookup painlessLookup, FunctionTable functions, Map constants, + MethodHandles.Lookup methodHandlesLookup, String interfaceClass, Class receiverClass, String name) + throws Throwable { + Class interfaceType = painlessLookup.canonicalTypeNameToType(interfaceClass); if (interfaceType == null) { throw new IllegalArgumentException("type [" + interfaceClass + "] not found"); @@ -317,25 +336,30 @@ static MethodHandle lookupReference(PainlessLookup painlessLookup, FunctionTable "dynamic method [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + arity + "] not found"); } - return lookupReferenceInternal(painlessLookup, functions, methodHandlesLookup, - interfaceType, PainlessLookupUtility.typeToCanonicalTypeName(implMethod.targetClass), - implMethod.javaMethod.getName(), 1); + return lookupReferenceInternal(painlessLookup, functions, constants, + methodHandlesLookup, interfaceType, PainlessLookupUtility.typeToCanonicalTypeName(implMethod.targetClass), + implMethod.javaMethod.getName(), 1); } /** Returns a method handle to an implementation of clazz, given method reference signature. */ - private static MethodHandle lookupReferenceInternal(PainlessLookup painlessLookup, FunctionTable functions, - MethodHandles.Lookup methodHandlesLookup, Class clazz, String type, String call, int captures) throws Throwable { - final FunctionRef ref = FunctionRef.create(painlessLookup, functions, null, clazz, type, call, captures); + private static MethodHandle lookupReferenceInternal( + PainlessLookup painlessLookup, FunctionTable functions, Map constants, + MethodHandles.Lookup methodHandlesLookup, Class clazz, String type, String call, int captures + ) throws Throwable { + + final FunctionRef ref = FunctionRef.create(painlessLookup, functions, null, clazz, type, call, captures, constants); final CallSite callSite = LambdaBootstrap.lambdaBootstrap( - methodHandlesLookup, - ref.interfaceMethodName, - ref.factoryMethodType, - ref.interfaceMethodType, - ref.delegateClassName, - ref.delegateInvokeType, - ref.delegateMethodName, - ref.delegateMethodType, - ref.isDelegateInterface ? 1 : 0 + methodHandlesLookup, + ref.interfaceMethodName, + ref.factoryMethodType, + ref.interfaceMethodType, + ref.delegateClassName, + ref.delegateInvokeType, + ref.delegateMethodName, + ref.delegateMethodType, + ref.isDelegateInterface ? 1 : 0, + ref.isDelegateAugmented ? 1 : 0, + ref.delegateInjections ); return callSite.dynamicInvoker().asType(MethodType.methodType(clazz, ref.factoryMethodType.parameterArray())); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java index 9bee5afeb5894..f67f8a5334825 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java @@ -29,6 +29,7 @@ import java.lang.invoke.MethodType; import java.lang.invoke.MutableCallSite; import java.lang.invoke.WrongMethodTypeException; +import java.util.Map; /** * Painless invokedynamic bootstrap for the call site. @@ -107,13 +108,14 @@ static final class PIC extends MutableCallSite { private final PainlessLookup painlessLookup; private final FunctionTable functions; + private final Map constants; private final MethodHandles.Lookup methodHandlesLookup; private final String name; private final int flavor; private final Object[] args; int depth; // pkg-protected for testing - PIC(PainlessLookup painlessLookup, FunctionTable functions, + PIC(PainlessLookup painlessLookup, FunctionTable functions, Map constants, MethodHandles.Lookup methodHandlesLookup, String name, MethodType type, int initialDepth, int flavor, Object[] args) { super(type); if (type.parameterType(0) != Object.class) { @@ -121,6 +123,7 @@ static final class PIC extends MutableCallSite { } this.painlessLookup = painlessLookup; this.functions = functions; + this.constants = constants; this.methodHandlesLookup = methodHandlesLookup; this.name = name; this.flavor = flavor; @@ -148,7 +151,7 @@ static boolean checkClass(Class clazz, Object receiver) { private MethodHandle lookup(int flavor, String name, Class receiver) throws Throwable { switch(flavor) { case METHOD_CALL: - return Def.lookupMethod(painlessLookup, functions, methodHandlesLookup, type(), receiver, name, args); + return Def.lookupMethod(painlessLookup, functions, constants, methodHandlesLookup, type(), receiver, name, args); case LOAD: return Def.lookupGetter(painlessLookup, receiver, name); case STORE: @@ -160,7 +163,7 @@ private MethodHandle lookup(int flavor, String name, Class receiver) throws T case ITERATOR: return Def.lookupIterator(receiver); case REFERENCE: - return Def.lookupReference(painlessLookup, functions, methodHandlesLookup, (String) args[0], receiver, name); + return Def.lookupReference(painlessLookup, functions, constants, methodHandlesLookup, (String) args[0], receiver, name); case INDEX_NORMALIZE: return Def.lookupIndexNormalize(receiver); default: throw new AssertionError(); @@ -436,7 +439,7 @@ static boolean checkBoth(Class left, Class right, Object leftObject, Objec * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic */ @SuppressWarnings("unchecked") - public static CallSite bootstrap(PainlessLookup painlessLookup, FunctionTable functions, + public static CallSite bootstrap(PainlessLookup painlessLookup, FunctionTable functions, Map constants, MethodHandles.Lookup methodHandlesLookup, String name, MethodType type, int initialDepth, int flavor, Object... args) { // validate arguments switch(flavor) { @@ -456,7 +459,7 @@ public static CallSite bootstrap(PainlessLookup painlessLookup, FunctionTable fu if (args.length != numLambdas + 1) { throw new BootstrapMethodError("Illegal number of parameters: expected " + numLambdas + " references"); } - return new PIC(painlessLookup, functions, methodHandlesLookup, name, type, initialDepth, flavor, args); + return new PIC(painlessLookup, functions, constants, methodHandlesLookup, name, type, initialDepth, flavor, args); case LOAD: case STORE: case ARRAY_LOAD: @@ -466,7 +469,7 @@ public static CallSite bootstrap(PainlessLookup painlessLookup, FunctionTable fu if (args.length > 0) { throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor); } - return new PIC(painlessLookup, functions, methodHandlesLookup, name, type, initialDepth, flavor, args); + return new PIC(painlessLookup, functions, constants, methodHandlesLookup, name, type, initialDepth, flavor, args); case REFERENCE: if (args.length != 1) { throw new BootstrapMethodError("Invalid number of parameters for reference call"); @@ -474,7 +477,7 @@ public static CallSite bootstrap(PainlessLookup painlessLookup, FunctionTable fu if (args[0] instanceof String == false) { throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]); } - return new PIC(painlessLookup, functions, methodHandlesLookup, name, type, initialDepth, flavor, args); + return new PIC(painlessLookup, functions, constants, methodHandlesLookup, name, type, initialDepth, flavor, args); // operators get monomorphic cache, with a generic impl for a fallback case UNARY_OPERATOR: diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java index 0e8f9ef81e062..ed8402b74a025 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java @@ -30,6 +30,7 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.painless.WriterConstants.CLASS_NAME; @@ -44,7 +45,6 @@ * lambda function. */ public class FunctionRef { - /** * Creates a new FunctionRef which will resolve {@code type::call} from the whitelist. * @param painlessLookup the whitelist against which this script is being compiled @@ -54,9 +54,10 @@ public class FunctionRef { * @param typeName the left hand side of a method reference expression * @param methodName the right hand side of a method reference expression * @param numberOfCaptures number of captured arguments + * @param constants constants used for injection when necessary */ public static FunctionRef create(PainlessLookup painlessLookup, FunctionTable functionTable, Location location, - Class targetClass, String typeName, String methodName, int numberOfCaptures) { + Class targetClass, String typeName, String methodName, int numberOfCaptures, Map constants) { Objects.requireNonNull(painlessLookup); Objects.requireNonNull(targetClass); @@ -78,9 +79,11 @@ public static FunctionRef create(PainlessLookup painlessLookup, FunctionTable fu MethodType interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1); String delegateClassName; boolean isDelegateInterface; + boolean isDelegateAugmented; int delegateInvokeType; String delegateMethodName; MethodType delegateMethodType; + Object[] delegateInjections; Class delegateMethodReturnType; List> delegateMethodParameters; @@ -105,9 +108,11 @@ public static FunctionRef create(PainlessLookup painlessLookup, FunctionTable fu delegateClassName = CLASS_NAME; isDelegateInterface = false; + isDelegateAugmented = false; delegateInvokeType = H_INVOKESTATIC; delegateMethodName = localFunction.getFunctionName(); delegateMethodType = localFunction.getMethodType(); + delegateInjections = new Object[0]; delegateMethodReturnType = localFunction.getReturnType(); delegateMethodParameters = localFunction.getTypeParameters(); @@ -126,9 +131,11 @@ public static FunctionRef create(PainlessLookup painlessLookup, FunctionTable fu delegateClassName = painlessConstructor.javaConstructor.getDeclaringClass().getName(); isDelegateInterface = false; + isDelegateAugmented = false; delegateInvokeType = H_NEWINVOKESPECIAL; delegateMethodName = PainlessLookupUtility.CONSTRUCTOR_NAME; delegateMethodType = painlessConstructor.methodType; + delegateInjections = new Object[0]; delegateMethodReturnType = painlessConstructor.javaConstructor.getDeclaringClass(); delegateMethodParameters = painlessConstructor.typeParameters; @@ -157,6 +164,7 @@ public static FunctionRef create(PainlessLookup painlessLookup, FunctionTable fu delegateClassName = painlessMethod.javaMethod.getDeclaringClass().getName(); isDelegateInterface = painlessMethod.javaMethod.getDeclaringClass().isInterface(); + isDelegateAugmented = painlessMethod.javaMethod.getDeclaringClass() != painlessMethod.targetClass; if (Modifier.isStatic(painlessMethod.javaMethod.getModifiers())) { delegateInvokeType = H_INVOKESTATIC; @@ -168,6 +176,7 @@ public static FunctionRef create(PainlessLookup painlessLookup, FunctionTable fu delegateMethodName = painlessMethod.javaMethod.getName(); delegateMethodType = painlessMethod.methodType; + delegateInjections = PainlessLookupUtility.buildInjections(painlessMethod, constants); delegateMethodReturnType = painlessMethod.returnType; @@ -196,7 +205,8 @@ public static FunctionRef create(PainlessLookup painlessLookup, FunctionTable fu delegateMethodType = delegateMethodType.dropParameterTypes(0, numberOfCaptures); return new FunctionRef(interfaceMethodName, interfaceMethodType, - delegateClassName, isDelegateInterface, delegateInvokeType, delegateMethodName, delegateMethodType, + delegateClassName, isDelegateInterface, isDelegateAugmented, + delegateInvokeType, delegateMethodName, delegateMethodType, delegateInjections, factoryMethodType ); } catch (IllegalArgumentException iae) { @@ -216,28 +226,34 @@ public static FunctionRef create(PainlessLookup painlessLookup, FunctionTable fu public final String delegateClassName; /** whether a call is made on a delegate interface */ public final boolean isDelegateInterface; + /** if delegate method is augmented */ + public final boolean isDelegateAugmented; /** the invocation type of the delegate method */ public final int delegateInvokeType; /** the name of the delegate method */ public final String delegateMethodName; /** delegate method signature */ public final MethodType delegateMethodType; + /** injected constants */ + public final Object[] delegateInjections; /** factory (CallSite) method signature */ public final MethodType factoryMethodType; private FunctionRef( String interfaceMethodName, MethodType interfaceMethodType, - String delegateClassName, boolean isDelegateInterface, - int delegateInvokeType, String delegateMethodName, MethodType delegateMethodType, + String delegateClassName, boolean isDelegateInterface, boolean isDelegateAugmented, + int delegateInvokeType, String delegateMethodName, MethodType delegateMethodType, Object[] delegateInjections, MethodType factoryMethodType) { this.interfaceMethodName = interfaceMethodName; this.interfaceMethodType = interfaceMethodType; this.delegateClassName = delegateClassName; this.isDelegateInterface = isDelegateInterface; + this.isDelegateAugmented = isDelegateAugmented; this.delegateInvokeType = delegateInvokeType; this.delegateMethodName = delegateMethodName; this.delegateMethodType = delegateMethodType; + this.delegateInjections = delegateInjections; this.factoryMethodType = factoryMethodType; } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java index 9db9011f0590e..12a9ccc34bdff 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java @@ -194,6 +194,7 @@ private Capture(int count, Class type) { * if the value is '1' if the delegate is an interface and '0' * otherwise; note this is an int because the bootstrap method * cannot convert constants to boolean + * @param injections Optionally add injectable constants into a method reference * @return A {@link CallSite} linked to a factory method for creating a lambda class * that implements the expected functional interface * @throws LambdaConversionException Thrown when an illegal type conversion occurs at link time @@ -207,7 +208,9 @@ public static CallSite lambdaBootstrap( int delegateInvokeType, String delegateMethodName, MethodType delegateMethodType, - int isDelegateInterface) + int isDelegateInterface, + int isDelegateAugmented, + Object... injections) throws LambdaConversionException { Compiler.Loader loader = (Compiler.Loader)lookup.lookupClass().getClassLoader(); String lambdaClassName = Type.getInternalName(lookup.lookupClass()) + "$$Lambda" + loader.newLambdaIdentifier(); @@ -232,7 +235,7 @@ public static CallSite lambdaBootstrap( generateInterfaceMethod(cw, factoryMethodType, lambdaClassType, interfaceMethodName, interfaceMethodType, delegateClassType, delegateInvokeType, - delegateMethodName, delegateMethodType, isDelegateInterface == 1, captures); + delegateMethodName, delegateMethodType, isDelegateInterface == 1, isDelegateAugmented == 1, captures, injections); endLambdaClass(cw); @@ -377,7 +380,9 @@ private static void generateInterfaceMethod( String delegateMethodName, MethodType delegateMethodType, boolean isDelegateInterface, - Capture[] captures) + boolean isDelegateAugmented, + Capture[] captures, + Object... injections) throws LambdaConversionException { String lamDesc = interfaceMethodType.toMethodDescriptorString(); @@ -443,9 +448,17 @@ private static void generateInterfaceMethod( new Handle(delegateInvokeType, delegateClassType.getInternalName(), delegateMethodName, delegateMethodType.toMethodDescriptorString(), isDelegateInterface); - iface.invokeDynamic(delegateMethodName, Type.getMethodType(interfaceMethodType - .toMethodDescriptorString()).getDescriptor(), DELEGATE_BOOTSTRAP_HANDLE, - delegateHandle); + // Fill in args for indy. Always add the delegate handle and + // whether it's static or not then injections as necessary. + Object[] args = new Object[2 + injections.length]; + args[0] = delegateHandle; + args[1] = delegateInvokeType == H_INVOKESTATIC && isDelegateAugmented == false ? 0 : 1; + System.arraycopy(injections, 0, args, 2, injections.length); + iface.invokeDynamic( + delegateMethodName, + Type.getMethodType(interfaceMethodType.toMethodDescriptorString()).getDescriptor(), + DELEGATE_BOOTSTRAP_HANDLE, + args); iface.returnValue(); iface.endMethod(); @@ -517,7 +530,14 @@ private static CallSite createCaptureCallSite( public static CallSite delegateBootstrap(Lookup lookup, String delegateMethodName, MethodType interfaceMethodType, - MethodHandle delegateMethodHandle) { + MethodHandle delegateMethodHandle, + int isVirtual, + Object... injections) { + + if (injections.length > 0) { + delegateMethodHandle = MethodHandles.insertArguments(delegateMethodHandle, isVirtual, injections); + } + return new ConstantCallSite(delegateMethodHandle.asType(interfaceMethodType)); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java index 524b60d42fa9a..692d9aa065517 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java @@ -515,16 +515,21 @@ public void invokeMethodCall(PainlessMethod painlessMethod) { } public void invokeLambdaCall(FunctionRef functionRef) { + Object[] args = new Object[7 + functionRef.delegateInjections.length]; + args[0] = Type.getMethodType(functionRef.interfaceMethodType.toMethodDescriptorString()); + args[1] = functionRef.delegateClassName; + args[2] = functionRef.delegateInvokeType; + args[3] = functionRef.delegateMethodName; + args[4] = Type.getMethodType(functionRef.delegateMethodType.toMethodDescriptorString()); + args[5] = functionRef.isDelegateInterface ? 1 : 0; + args[6] = functionRef.isDelegateAugmented ? 1 : 0; + System.arraycopy(functionRef.delegateInjections, 0, args, 7, functionRef.delegateInjections.length); + invokeDynamic( functionRef.interfaceMethodName, functionRef.factoryMethodType.toMethodDescriptorString(), LAMBDA_BOOTSTRAP_HANDLE, - Type.getMethodType(functionRef.interfaceMethodType.toMethodDescriptorString()), - functionRef.delegateClassName, - functionRef.delegateInvokeType, - functionRef.delegateMethodName, - Type.getMethodType(functionRef.delegateMethodType.toMethodDescriptorString()), - functionRef.isDelegateInterface ? 1 : 0 + args ); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java index 358c4aa6376e6..f6d31d0b911e7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java @@ -130,7 +130,7 @@ public Collection createComponents(Client client, ClusterService cluster @Override public List> getSettings() { - return Arrays.asList(CompilerSettings.REGEX_ENABLED); + return Arrays.asList(CompilerSettings.REGEX_ENABLED, CompilerSettings.REGEX_LIMIT_FACTOR); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java index a81d7c99e11ff..8706b96549d12 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java @@ -92,6 +92,7 @@ public final class PainlessScriptEngine implements ScriptEngine { */ public PainlessScriptEngine(Settings settings, Map, List> contexts) { defaultCompilerSettings.setRegexesEnabled(CompilerSettings.REGEX_ENABLED.get(settings)); + defaultCompilerSettings.setRegexLimitFactor(CompilerSettings.REGEX_LIMIT_FACTOR.get(settings)); Map, Compiler> contextsToCompilers = new HashMap<>(); Map, PainlessLookup> contextsToLookups = new HashMap<>(); @@ -429,6 +430,8 @@ private CompilerSettings buildCompilerSettings(Map params) { // Except regexes enabled - this is a node level setting and can't be changed in the request. compilerSettings.setRegexesEnabled(defaultCompilerSettings.areRegexesEnabled()); + compilerSettings.setRegexLimitFactor(defaultCompilerSettings.getRegexLimitFactor()); + Map copy = new HashMap<>(params); String value = copy.remove(CompilerSettings.MAX_LOOP_COUNTER); @@ -451,6 +454,11 @@ private CompilerSettings buildCompilerSettings(Map params) { throw new IllegalArgumentException("[painless.regex.enabled] can only be set on node startup."); } + value = copy.remove(CompilerSettings.REGEX_LIMIT_FACTOR.getKey()); + if (value != null) { + throw new IllegalArgumentException("[painless.regex.limit-factor] can only be set on node startup."); + } + if (!copy.isEmpty()) { throw new IllegalArgumentException("Unrecognized compile-time parameter(s): " + copy); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java index 0deb154cbff1b..3c70e7d72fa77 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java @@ -134,12 +134,13 @@ public final class WriterConstants { /** invokedynamic bootstrap for lambda expression/method references */ public static final MethodType LAMBDA_BOOTSTRAP_TYPE = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, - MethodType.class, String.class, int.class, String.class, MethodType.class, int.class); + MethodType.class, String.class, int.class, String.class, MethodType.class, int.class, int.class, Object[].class); public static final Handle LAMBDA_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaBootstrap.class), "lambdaBootstrap", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false); public static final MethodType DELEGATE_BOOTSTRAP_TYPE = - MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, MethodHandle.class); + MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, MethodHandle.class, + int.class, Object[].class); public static final Handle DELEGATE_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaBootstrap.class), "delegateBootstrap", DELEGATE_BOOTSTRAP_TYPE.toMethodDescriptorString(), false); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java index 74e14571aa35f..cb1525b392886 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java @@ -41,10 +41,11 @@ import java.util.function.ToDoubleFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; /** Additional methods added to classes. These must be static methods with receiver as first argument */ public class Augmentation { - + // static methods only! private Augmentation() {} @@ -57,10 +58,10 @@ public static int getLength(List receiver) { public static String namedGroup(Matcher receiver, String name) { return receiver.group(name); } - + // some groovy methods on iterable // see http://docs.groovy-lang.org/latest/html/groovy-jdk/java/lang/Iterable.html - + /** Iterates over the contents of an iterable, and checks whether a predicate is valid for at least one element. */ public static boolean any(Iterable receiver, Predicate predicate) { for (T t : receiver) { @@ -70,7 +71,7 @@ public static boolean any(Iterable receiver, Predicate predicate) { } return false; } - + /** Converts this Iterable to a Collection. Returns the original Iterable if it is already a Collection. */ public static Collection asCollection(Iterable receiver) { if (receiver instanceof Collection) { @@ -82,7 +83,7 @@ public static Collection asCollection(Iterable receiver) { } return list; } - + /** Converts this Iterable to a List. Returns the original Iterable if it is already a List. */ public static List asList(Iterable receiver) { if (receiver instanceof List) { @@ -94,8 +95,8 @@ public static List asList(Iterable receiver) { } return list; } - - /** Counts the number of occurrences which satisfy the given predicate from inside this Iterable. */ + + /** Counts the number of occurrences which satisfy the given predicate from inside this Iterable. */ public static int count(Iterable receiver, Predicate predicate) { int count = 0; for (T t : receiver) { @@ -105,7 +106,7 @@ public static int count(Iterable receiver, Predicate predicate) { } return count; } - + // instead of covariant overrides for every possibility, we just return receiver as 'def' for now // that way if someone chains the calls, everything works. @@ -114,9 +115,9 @@ public static Object each(Iterable receiver, Consumer consumer) { receiver.forEach(consumer); return receiver; } - - /** - * Iterates through an iterable type, passing each item and the item's index + + /** + * Iterates through an iterable type, passing each item and the item's index * (a counter starting at zero) to the given consumer. */ public static Object eachWithIndex(Iterable receiver, ObjIntConsumer consumer) { @@ -126,7 +127,7 @@ public static Object eachWithIndex(Iterable receiver, ObjIntConsumer c } return receiver; } - + /** * Used to determine if the given predicate is valid (i.e. returns true for all items in this iterable). */ @@ -138,10 +139,10 @@ public static boolean every(Iterable receiver, Predicate predicate) { } return true; } - + /** - * Iterates through the Iterable transforming items using the supplied function and - * collecting any non-null results. + * Iterates through the Iterable transforming items using the supplied function and + * collecting any non-null results. */ public static List findResults(Iterable receiver, Function filter) { List list = new ArrayList<>(); @@ -153,9 +154,9 @@ public static List findResults(Iterable receiver, Function filt } return list; } - + /** - * Sorts all Iterable members into groups determined by the supplied mapping function. + * Sorts all Iterable members into groups determined by the supplied mapping function. */ public static Map> groupBy(Iterable receiver, Function mapper) { Map> map = new LinkedHashMap<>(); @@ -170,10 +171,10 @@ public static Map> groupBy(Iterable receiver, Function m } return map; } - + /** - * Concatenates the toString() representation of each item in this Iterable, - * with the given String as a separator between each item. + * Concatenates the toString() representation of each item in this Iterable, + * with the given String as a separator between each item. */ public static String join(Iterable receiver, String separator) { StringBuilder sb = new StringBuilder(); @@ -185,7 +186,7 @@ public static String join(Iterable receiver, String separator) { } return sb.toString(); } - + /** * Sums the result of an Iterable */ @@ -196,9 +197,9 @@ public static double sum(Iterable receiver) { } return sum; } - + /** - * Sums the result of applying a function to each item of an Iterable. + * Sums the result of applying a function to each item of an Iterable. */ public static double sum(Iterable receiver, ToDoubleFunction function) { double sum = 0; @@ -207,13 +208,13 @@ public static double sum(Iterable receiver, ToDoubleFunction function) } return sum; } - + // some groovy methods on collection // see http://docs.groovy-lang.org/latest/html/groovy-jdk/java/util/Collection.html - + /** - * Iterates through this collection transforming each entry into a new value using - * the function, returning a list of transformed values. + * Iterates through this collection transforming each entry into a new value using + * the function, returning a list of transformed values. */ public static List collect(Collection receiver, Function function) { List list = new ArrayList<>(); @@ -222,9 +223,9 @@ public static List collect(Collection receiver, Function functi } return list; } - + /** - * Iterates through this collection transforming each entry into a new value using + * Iterates through this collection transforming each entry into a new value using * the function, adding the values to the specified collection. */ public static Object collect(Collection receiver, Collection collection, Function function) { @@ -233,7 +234,7 @@ public static Object collect(Collection receiver, Collection collect } return collection; } - + /** * Finds the first value matching the predicate, or returns null. */ @@ -245,7 +246,7 @@ public static T find(Collection receiver, Predicate predicate) { } return null; } - + /** * Finds all values matching the predicate, returns as a list */ @@ -258,19 +259,19 @@ public static List findAll(Collection receiver, Predicate predicate } return list; } - + /** - * Iterates through the collection calling the given function for each item - * but stopping once the first non-null result is found and returning that result. - * If all results are null, null is returned. + * Iterates through the collection calling the given function for each item + * but stopping once the first non-null result is found and returning that result. + * If all results are null, null is returned. */ public static Object findResult(Collection receiver, Function function) { return findResult(receiver, null, function); } - + /** - * Iterates through the collection calling the given function for each item - * but stopping once the first non-null result is found and returning that result. + * Iterates through the collection calling the given function for each item + * but stopping once the first non-null result is found and returning that result. * If all results are null, defaultResult is returned. */ public static Object findResult(Collection receiver, Object defaultResult, Function function) { @@ -282,10 +283,10 @@ public static Object findResult(Collection receiver, Object defaultResu } return defaultResult; } - + /** - * Splits all items into two collections based on the predicate. - * The first list contains all items which match the closure expression. The second list all those that don't. + * Splits all items into two collections based on the predicate. + * The first list contains all items which match the closure expression. The second list all those that don't. */ public static List> split(Collection receiver, Predicate predicate) { List matched = new ArrayList<>(); @@ -302,13 +303,13 @@ public static List> split(Collection receiver, Predicate predi } return result; } - + // some groovy methods on map // see http://docs.groovy-lang.org/latest/html/groovy-jdk/java/util/Map.html - + /** - * Iterates through this map transforming each entry into a new value using - * the function, returning a list of transformed values. + * Iterates through this map transforming each entry into a new value using + * the function, returning a list of transformed values. */ public static List collect(Map receiver, BiFunction function) { List list = new ArrayList<>(); @@ -317,9 +318,9 @@ public static List collect(Map receiver, BiFunction funct } return list; } - + /** - * Iterates through this map transforming each entry into a new value using + * Iterates through this map transforming each entry into a new value using * the function, adding the values to the specified collection. */ public static Object collect(Map receiver, Collection collection, BiFunction function) { @@ -328,8 +329,8 @@ public static Object collect(Map receiver, Collection collection } return collection; } - - /** Counts the number of occurrences which satisfy the given predicate from inside this Map */ + + /** Counts the number of occurrences which satisfy the given predicate from inside this Map */ public static int count(Map receiver, BiPredicate predicate) { int count = 0; for (Map.Entry kvPair : receiver.entrySet()) { @@ -339,13 +340,13 @@ public static int count(Map receiver, BiPredicate predicate) { } return count; } - + /** Iterates through a Map, passing each item to the given consumer. */ public static Object each(Map receiver, BiConsumer consumer) { receiver.forEach(consumer); return receiver; } - + /** * Used to determine if the given predicate is valid (i.e. returns true for all items in this map). */ @@ -357,7 +358,7 @@ public static boolean every(Map receiver, BiPredicate predicate) } return true; } - + /** * Finds the first entry matching the predicate, or returns null. */ @@ -369,7 +370,7 @@ public static Map.Entry find(Map receiver, BiPredicate pred } return null; } - + /** * Finds all values matching the predicate, returns as a map. */ @@ -388,19 +389,19 @@ public static Map findAll(Map receiver, BiPredicate predica } return map; } - + /** - * Iterates through the map calling the given function for each item - * but stopping once the first non-null result is found and returning that result. - * If all results are null, null is returned. + * Iterates through the map calling the given function for each item + * but stopping once the first non-null result is found and returning that result. + * If all results are null, null is returned. */ public static Object findResult(Map receiver, BiFunction function) { return findResult(receiver, null, function); } - + /** - * Iterates through the map calling the given function for each item - * but stopping once the first non-null result is found and returning that result. + * Iterates through the map calling the given function for each item + * but stopping once the first non-null result is found and returning that result. * If all results are null, defaultResult is returned. */ public static Object findResult(Map receiver, Object defaultResult, BiFunction function) { @@ -412,10 +413,10 @@ public static Object findResult(Map receiver, Object defaultResult, } return defaultResult; } - + /** - * Iterates through the map transforming items using the supplied function and - * collecting any non-null results. + * Iterates through the map transforming items using the supplied function and + * collecting any non-null results. */ public static List findResults(Map receiver, BiFunction filter) { List list = new ArrayList<>(); @@ -427,9 +428,9 @@ public static List findResults(Map receiver, BiFunction f } return list; } - + /** - * Sorts all Map members into groups determined by the supplied mapping function. + * Sorts all Map members into groups determined by the supplied mapping function. */ public static Map> groupBy(Map receiver, BiFunction mapper) { Map> map = new LinkedHashMap<>(); @@ -679,4 +680,36 @@ public static String sha256(String source) { MessageDigests.sha256().digest(source.getBytes(StandardCharsets.UTF_8)) ); } + + public static final int UNLIMITED_PATTERN_FACTOR = 0; + public static final int DISABLED_PATTERN_FACTOR = -1; + + // Regular Expression Pattern augmentations with limit factor injected + public static String[] split(Pattern receiver, int limitFactor, CharSequence input) { + if (limitFactor == UNLIMITED_PATTERN_FACTOR) { + return receiver.split(input); + } + return receiver.split(LimitedCharSequence.limitedCharSequence(input, receiver, limitFactor)); + } + + public static String[] split​(Pattern receiver, int limitFactor, CharSequence input, int limit) { + if (limitFactor == UNLIMITED_PATTERN_FACTOR) { + return receiver.split(input, limit); + } + return receiver.split(LimitedCharSequence.limitedCharSequence(input, receiver, limitFactor), limit); + } + + public static Stream splitAsStream​(Pattern receiver, int limitFactor, CharSequence input) { + if (limitFactor == UNLIMITED_PATTERN_FACTOR) { + return receiver.splitAsStream(input); + } + return receiver.splitAsStream(LimitedCharSequence.limitedCharSequence(input, receiver, limitFactor)); + } + + public static Matcher matcher(Pattern receiver, int limitFactor, CharSequence input) { + if (limitFactor == UNLIMITED_PATTERN_FACTOR) { + return receiver.matcher(input); + } + return receiver.matcher(LimitedCharSequence.limitedCharSequence(input, receiver, limitFactor)); + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java new file mode 100644 index 0000000000000..0a39169174f3d --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.api; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.painless.CompilerSettings; + +import java.util.regex.Pattern; + +/* + * CharSequence that wraps another sequence and limits the number of times charAt can be + */ +public class LimitedCharSequence implements CharSequence { + private final CharSequence wrapped; + private final Counter counter; + + // for errors + private final boolean isSubSequence; + private final int offset; // if this is a subSequence, the start of the subSequence + private final CharSequence original; + private final Pattern pattern; + private final int limitFactor; + + private static final int SNIPPET_LIMIT = 64; + + private LimitedCharSequence(CharSequence wrap, Pattern pattern, int limitFactor) { + this.wrapped = wrap; + this.counter = new Counter(limitFactor * wrapped.length()); + + this.isSubSequence = false; + this.offset = 0; + this.original = wrap; + this.pattern = pattern; + this.limitFactor = limitFactor; + } + + // subSequence constructor + private LimitedCharSequence(LimitedCharSequence superSequence, int start, int end) { + this.wrapped = superSequence.wrapped.subSequence(start, end); + this.counter = superSequence.counter; + + this.isSubSequence = true; + this.offset = superSequence.offset + start; + this.original = superSequence.original; + this.pattern = superSequence.pattern; + this.limitFactor = superSequence.limitFactor; + } + + public static CharSequence limitedCharSequence(CharSequence wrapped, Pattern pattern, int limitFactor) { + if (limitFactor <= 0) { + throw new IllegalArgumentException("limitFactor must be positive"); + } + if (wrapped instanceof LimitedCharSequence) { + return wrapped; + } + return new LimitedCharSequence(wrapped, pattern, limitFactor); + } + + public String details(int index) { + return "pattern: [" + pattern.pattern() + "], " + + "limit factor: [" + limitFactor + "], " + + "char limit: [" + counter.charAtLimit + "], " + + "snippet: [" + sequenceSnippet(index) + "], " + + "count: [" + counter.count + "], " + + // TODO(stu): remove these when sequenceSnippet is implemented + "isSubSequence: [" + isSubSequence + "], " + + "offset: [" + offset + "], " + + "wrapped: [" + wrapped.toString() + "]"; + } + + String sequenceSnippet(int index) { + // TODO(stu): consider isSubSequence and snippetLimit + return original.toString(); + } + + @Override + public int length() { + return wrapped.length(); + } + + @Override + public char charAt(int index) { + counter.count++; + if (counter.hitLimit()) { + throw new CircuitBreakingException("[scripting] Regular expression considered too many characters, " + details(index) + + ", this limit can be changed by changed by the [" + CompilerSettings.REGEX_LIMIT_FACTOR.getKey() + "] setting", + CircuitBreaker.Durability.TRANSIENT); + } + return wrapped.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return new LimitedCharSequence(this, start, end); + } + + @Override + public String toString() { + return wrapped.toString(); + } + + // Counter object to keep track of charAts for original sequence and all subsequences + private static class Counter { + public final int charAtLimit; + public int count; + + Counter(int charAtLimit) { + this.charAtLimit = charAtLimit; + this.count = 0; + } + + boolean hitLimit() { + return count > charAtLimit; + } + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index 204b54f60fa86..db2d4d7831dac 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -30,6 +30,7 @@ import org.elasticsearch.painless.spi.WhitelistField; import org.elasticsearch.painless.spi.WhitelistInstanceBinding; import org.elasticsearch.painless.spi.WhitelistMethod; +import org.elasticsearch.painless.spi.annotation.InjectConstantAnnotation; import org.elasticsearch.painless.spi.annotation.NoImportAnnotation; import org.objectweb.asm.ClassWriter; import org.objectweb.asm.Opcodes; @@ -518,6 +519,7 @@ public void addPainlessMethod(Class targetClass, Class augmentedClass, "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", nsme); } } else { + // TODO(stu): fix try { javaMethod = augmentedClass.getMethod(methodName, javaTypeParameters.toArray(new Class[typeParametersSize])); @@ -533,6 +535,17 @@ public void addPainlessMethod(Class targetClass, Class augmentedClass, } } + // injections alter the type parameters required for the user to call this method, since some are injected by compiler + if (annotations.containsKey(InjectConstantAnnotation.class)) { + int numInjections = ((InjectConstantAnnotation)annotations.get(InjectConstantAnnotation.class)).injects.size(); + + if (numInjections > 0) { + typeParameters.subList(0, numInjections).clear(); + } + + typeParametersSize = typeParameters.size(); + } + if (javaMethod.getReturnType() != typeToJavaType(returnType)) { throw new IllegalArgumentException("return type [" + typeToCanonicalTypeName(javaMethod.getReturnType()) + "] " + "does not match the specified returned type [" + typeToCanonicalTypeName(returnType) + "] " + @@ -562,7 +575,6 @@ public void addPainlessMethod(Class targetClass, Class augmentedClass, } MethodType methodType = methodHandle.type(); - boolean isStatic = augmentedClass == null && Modifier.isStatic(javaMethod.getModifiers()); String painlessMethodKey = buildPainlessMethodKey(methodName, typeParametersSize); PainlessMethod existingPainlessMethod = isStatic ? diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java index c3bfa17a16e51..99de139347454 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java @@ -19,6 +19,8 @@ package org.elasticsearch.painless.lookup; +import org.elasticsearch.painless.spi.annotation.InjectConstantAnnotation; + import java.util.Arrays; import java.util.List; import java.util.Map; @@ -72,7 +74,7 @@ * */ public final class PainlessLookupUtility { - + /** * The name for an anonymous class. */ @@ -359,7 +361,34 @@ public static String buildPainlessMethodKey(String methodName, int methodArity) public static String buildPainlessFieldKey(String fieldName) { return fieldName; } - + + /** + * Constructs an array of injectable constants for a specific {@link PainlessMethod} + * derived from an {@link org.elasticsearch.painless.spi.annotation.InjectConstantAnnotation}. + */ + public static Object[] buildInjections(PainlessMethod painlessMethod, Map constants) { + if (painlessMethod.annotations.containsKey(InjectConstantAnnotation.class) == false) { + return new Object[0]; + } + + List names = ((InjectConstantAnnotation)painlessMethod.annotations.get(InjectConstantAnnotation.class)).injects; + Object[] injections = new Object[names.size()]; + + for (int i = 0; i < names.size(); i++) { + String name = names.get(i); + Object constant = constants.get(name); + + if (constant == null) { + throw new IllegalStateException("constant [" + name + "] not found for injection into method " + + "[" + buildPainlessMethodKey(painlessMethod.javaMethod.getName(), painlessMethod.typeParameters.size()) + "]"); + } + + injections[i] = constant; + } + + return injections; + } + private PainlessLookupUtility() { } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java index 47fe9e5630b43..55f1bfb69d1f3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java @@ -20,6 +20,7 @@ package org.elasticsearch.painless.phase; import org.elasticsearch.painless.AnalyzerCaster; +import org.elasticsearch.painless.CompilerSettings; import org.elasticsearch.painless.FunctionRef; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Operation; @@ -2049,7 +2050,7 @@ public void visitRegex(ERegex userRegexNode, SemanticScope semanticScope) { "not a statement: regex constant [" + pattern + "] with flags [" + flags + "] not used")); } - if (semanticScope.getScriptScope().getCompilerSettings().areRegexesEnabled() == false) { + if (semanticScope.getScriptScope().getCompilerSettings().areRegexesEnabled() == CompilerSettings.RegexEnabled.FALSE) { throw userRegexNode.createError(new IllegalStateException("Regexes are disabled. Set [script.painless.regex.enabled] to [true] " + "in elasticsearch.yaml to allow them. Be careful though, regexes break out of Painless's protection against deep " + "recursion and long loops.")); @@ -2228,7 +2229,8 @@ public void visitLambda(ELambda userLambdaNode, SemanticScope semanticScope) { semanticScope.putDecoration(userLambdaNode, new EncodingDecoration(defReferenceEncoding)); } else { FunctionRef ref = FunctionRef.create(scriptScope.getPainlessLookup(), scriptScope.getFunctionTable(), - location, targetType.getTargetType(), "this", name, capturedVariables.size()); + location, targetType.getTargetType(), "this", name, capturedVariables.size(), + scriptScope.getCompilerSettings().asMap()); valueType = targetType.getTargetType(); semanticScope.putDecoration(userLambdaNode, new ReferenceDecoration(ref)); } @@ -2276,7 +2278,8 @@ public void visitFunctionRef(EFunctionRef userFunctionRefNode, SemanticScope sem semanticScope.putDecoration(userFunctionRefNode, new EncodingDecoration(defReferenceEncoding)); } else { FunctionRef ref = FunctionRef.create(scriptScope.getPainlessLookup(), scriptScope.getFunctionTable(), - location, targetType.getTargetType(), symbol, methodName, 0); + location, targetType.getTargetType(), symbol, methodName, 0, + scriptScope.getCompilerSettings().asMap()); valueType = targetType.getTargetType(); semanticScope.putDecoration(userFunctionRefNode, new ReferenceDecoration(ref)); } @@ -2309,7 +2312,8 @@ public void visitFunctionRef(EFunctionRef userFunctionRefNode, SemanticScope sem // static case if (captured.getType() != def.class) { FunctionRef ref = FunctionRef.create(scriptScope.getPainlessLookup(), scriptScope.getFunctionTable(), location, - targetType.getTargetType(), captured.getCanonicalTypeName(), methodName, 1); + targetType.getTargetType(), captured.getCanonicalTypeName(), methodName, 1, + scriptScope.getCompilerSettings().asMap()); semanticScope.putDecoration(userFunctionRefNode, new ReferenceDecoration(ref)); } } @@ -2358,7 +2362,8 @@ public void visitNewArrayFunctionRef(ENewArrayFunctionRef userNewArrayFunctionRe scriptScope.putDecoration(userNewArrayFunctionRefNode, new EncodingDecoration(defReferenceEncoding)); } else { FunctionRef ref = FunctionRef.create(scriptScope.getPainlessLookup(), scriptScope.getFunctionTable(), - userNewArrayFunctionRefNode.getLocation(), targetType.getTargetType(), "this", name, 0); + userNewArrayFunctionRefNode.getLocation(), targetType.getTargetType(), "this", name, 0, + scriptScope.getCompilerSettings().asMap()); valueType = targetType.getTargetType(); semanticScope.putDecoration(userNewArrayFunctionRefNode, new ReferenceDecoration(ref)); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java index 32c34992cadb0..3d95ea8738842 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java @@ -210,6 +210,7 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.regex.Pattern; public class DefaultUserTreeToIRTreePhase implements UserTreeVisitor { @@ -219,7 +220,7 @@ public class DefaultUserTreeToIRTreePhase implements UserTreeVisitor[] parameterTypes = method.javaMethod.getParameterTypes(); + int augmentedOffset = method.javaMethod.getDeclaringClass() == method.targetClass ? 0 : 1; - for (AExpression userArgumentNode : userCallNode.getArgumentNodes()) { - irInvokeCallNode.addArgumentNode(injectCast(userArgumentNode, scriptScope)); + for (int i = 0; i < injections.length; i++) { + Object injection = injections[i]; + Class parameterType = parameterTypes[i + augmentedOffset]; + + if (parameterType != PainlessLookupUtility.typeToUnboxedType(injection.getClass())) { + throw new IllegalStateException("illegal tree structure"); + } + + ConstantNode constantNode = new ConstantNode(); + constantNode.setLocation(userCallNode.getLocation()); + constantNode.setExpressionType(parameterType); + constantNode.setConstant(injection); + irInvokeCallNode.addArgumentNode(constantNode); + } + + for (AExpression userCallArgumentNode : userCallNode.getArgumentNodes()) { + irInvokeCallNode.addArgumentNode(injectCast(userCallArgumentNode, scriptScope)); } irInvokeCallNode.setLocation(userCallNode.getLocation()); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java index 9274f53d96c41..4929f9b20138b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java @@ -211,7 +211,7 @@ protected void injectStaticFieldsAndGetters() { irLoadFieldMemberNode.setLocation(internalLocation); irLoadFieldMemberNode.setExpressionType(String.class); irLoadFieldMemberNode.setName("$NAME"); - irLoadFieldMemberNode.setStatic(true); + irLoadFieldMemberNode.setStatic(true); // TODO(stu): add $COMPILER_INJECTS, add hash map and set it irReturnNode.setExpressionNode(irLoadFieldMemberNode); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java index a2a36a2374799..3bfccc60c5810 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java @@ -66,6 +66,8 @@ public ScriptScope(PainlessLookup painlessLookup, CompilerSettings compilerSetti staticConstants.put("$SOURCE", scriptSource); staticConstants.put("$DEFINITION", painlessLookup); staticConstants.put("$FUNCTIONS", functionTable); + // TODO(stu): inject compiler settings here + staticConstants.put("$COMPILERSETTINGS", compilerSettings.asMap()); } public PainlessLookup getPainlessLookup() { diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.regex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.regex.txt index f062d2f688563..521d092efb97b 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.regex.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.regex.txt @@ -27,12 +27,12 @@ class java.util.regex.Pattern { # the script is run which is super slow. LRegex generates code that calls this method but it skips these checks. Predicate asPredicate() int flags() - Matcher matcher(CharSequence) + Matcher org.elasticsearch.painless.api.Augmentation matcher(int, CharSequence) @inject_constant[1="regex_limit_factor"] String pattern() String quote(String) - String[] split(CharSequence) - String[] split(CharSequence,int) - Stream splitAsStream(CharSequence) + String[] org.elasticsearch.painless.api.Augmentation split(int, CharSequence) @inject_constant[1="regex_limit_factor"] + String[] org.elasticsearch.painless.api.Augmentation split(int, CharSequence,int) @inject_constant[1="regex_limit_factor"] + Stream org.elasticsearch.painless.api.Augmentation splitAsStream(int, CharSequence) @inject_constant[1="regex_limit_factor"] } class java.util.regex.Matcher { @@ -58,6 +58,7 @@ class java.util.regex.Matcher { String replaceFirst(String) boolean requireEnd() Matcher reset() + # Whitelisting Matcher.reset(String) works around the regex limiting int start() int start(int) Matcher useAnchoringBounds(boolean) diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java index 8fe095beeb33a..54d8ff1fb1e88 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.painless; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; @@ -285,4 +287,215 @@ public void testSha256() { assertEquals("fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", execDigest("'bar'.sha256()")); assertEquals("97df3588b5a3f24babc3851b372f0ba71a9dcdded43b14b9d06961bfc1707d9d", execDigest("'foobarbaz'.sha256()")); } + + // This regex has backtracking due to .*? + private final String pattern = "/abc.*?def/"; + private final String charSequence = "'abcdodef'"; + private final String splitCharSequence = "'0-abc-1-def-X-abc-2-def-Y-abc-3-def-Z-abc'"; + private final String regexCircuitMessage = "[scripting] Regular expression considered too many characters"; + + public void testRegexInject_Matcher() { + String[] scripts = new String[]{pattern + ".matcher(" + charSequence + ").matches()", + "Matcher m = " + pattern + ".matcher(" + charSequence + "); m.matches()"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertEquals(Boolean.TRUE, exec(script)); + + // Backtracking means the regular expression will fail with limit factor 1 (don't consider more than each char once) + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testRegexInject_Def_Matcher() { + String[] scripts = new String[]{"def p = " + pattern + "; p.matcher(" + charSequence + ").matches()", + "def p = " + pattern + "; def m = p.matcher(" + charSequence + "); m.matches()"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertEquals(Boolean.TRUE, exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testMethodRegexInject_Ref_Matcher() { + String script = + "boolean isMatch(Function func) { func.apply(" + charSequence +").matches(); } " + + "Pattern pattern = " + pattern + ";" + + "isMatch(pattern::matcher)"; + setRegexLimitFactor(2); + assertEquals(Boolean.TRUE, exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_DefMethodRef_Matcher() { + String script = + "boolean isMatch(Function func) { func.apply(" + charSequence +").matches(); } " + + "def pattern = " + pattern + ";" + + "isMatch(pattern::matcher)"; + setRegexLimitFactor(2); + assertEquals(Boolean.TRUE, exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_SplitLimit() { + String[] scripts = new String[]{pattern + ".split(" + splitCharSequence + ", 2)", + "Pattern p = " + pattern + "; p.split(" + splitCharSequence + ", 2)"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testRegexInject_Def_SplitLimit() { + String script = "def p = " + pattern + "; p.split(" + splitCharSequence + ", 2)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_Ref_SplitLimit() { + String script = + "String[] splitLimit(BiFunction func) { func.apply(" + splitCharSequence + ", 2); } " + + "Pattern pattern = " + pattern + ";" + + "splitLimit(pattern::split)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_DefMethodRef_SplitLimit() { + String script = + "String[] splitLimit(BiFunction func) { func.apply(" + splitCharSequence + ", 2); } " + + "def pattern = " + pattern + ";" + + "splitLimit(pattern::split)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_Split() { + String[] scripts = new String[]{pattern + ".split(" + splitCharSequence + ")", + "Pattern p = " + pattern + "; p.split(" + splitCharSequence + ")"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testRegexInject_Def_Split() { + String script = "def p = " + pattern + "; p.split(" + splitCharSequence + ")"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_Ref_Split() { + String script = + "String[] split(Function func) { func.apply(" + splitCharSequence + "); } " + + "Pattern pattern = " + pattern + ";" + + "split(pattern::split)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_DefMethodRef_Split() { + String script = + "String[] split(Function func) { func.apply(" + splitCharSequence +"); } " + + "def pattern = " + pattern + ";" + + "split(pattern::split)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_SplitAsStream() { + String[] scripts = new String[]{pattern + ".splitAsStream(" + splitCharSequence + ").toArray(String[]::new)", + "Pattern p = " + pattern + "; p.splitAsStream(" + splitCharSequence + ").toArray(String[]::new)"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testRegexInject_Def_SplitAsStream() { + String script = "def p = " + pattern + "; p.splitAsStream(" + splitCharSequence + ").toArray(String[]::new)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_Ref_SplitAsStream() { + String script = + "Stream splitStream(Function func) { func.apply(" + splitCharSequence +"); } " + + "Pattern pattern = " + pattern + ";" + + "splitStream(pattern::splitAsStream).toArray(String[]::new)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_DefMethodRef_SplitAsStream() { + String script = + "Stream splitStream(Function func) { func.apply(" + splitCharSequence +"); } " + + "def pattern = " + pattern + ";" + + "splitStream(pattern::splitAsStream).toArray(String[]::new)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + private void setRegexLimitFactor(int factor) { + Settings settings = Settings.builder().put(CompilerSettings.REGEX_LIMIT_FACTOR.getKey(), factor).build(); + scriptEngine = new PainlessScriptEngine(settings, scriptContexts()); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java index c9e77080cfcb9..a640e2b5c6a57 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java @@ -40,6 +40,7 @@ public class DefBootstrapTests extends ESTestCase { public void testOneType() throws Throwable { CallSite site = DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "toString", MethodType.methodType(String.class, Object.class), @@ -61,6 +62,7 @@ public void testOneType() throws Throwable { public void testTwoTypes() throws Throwable { CallSite site = DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "toString", MethodType.methodType(String.class, Object.class), @@ -87,6 +89,7 @@ public void testTooManyTypes() throws Throwable { assertEquals(5, DefBootstrap.PIC.MAX_DEPTH); CallSite site = DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "toString", MethodType.methodType(String.class, Object.class), @@ -114,6 +117,7 @@ public void testTooManyTypes() throws Throwable { public void testMegamorphic() throws Throwable { DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "size", MethodType.methodType(int.class, Object.class), @@ -147,6 +151,7 @@ public void testMegamorphic() throws Throwable { public void testNullGuardAdd() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, Object.class, Object.class), @@ -160,6 +165,7 @@ public void testNullGuardAdd() throws Throwable { public void testNullGuardAddWhenCached() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, Object.class, Object.class), @@ -174,6 +180,7 @@ public void testNullGuardAddWhenCached() throws Throwable { public void testNullGuardEq() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "eq", MethodType.methodType(boolean.class, Object.class, Object.class), @@ -188,6 +195,7 @@ public void testNullGuardEq() throws Throwable { public void testNullGuardEqWhenCached() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "eq", MethodType.methodType(boolean.class, Object.class, Object.class), @@ -207,6 +215,7 @@ public void testNullGuardEqWhenCached() throws Throwable { public void testNoNullGuardAdd() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, int.class, Object.class), @@ -222,6 +231,7 @@ public void testNoNullGuardAdd() throws Throwable { public void testNoNullGuardAddWhenCached() throws Throwable { DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, new FunctionTable(), + Collections.emptyMap(), MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, int.class, Object.class), diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestAugmentationObject.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestAugmentationObject.java index ca9fef97df297..b6e1c5b743c14 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestAugmentationObject.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestAugmentationObject.java @@ -19,7 +19,10 @@ package org.elasticsearch.painless; +import java.util.function.Function; + public class FeatureTestAugmentationObject { + public static int getTotal(FeatureTestObject ft) { return ft.getX() + ft.getY(); } @@ -28,5 +31,26 @@ public static int addToTotal(FeatureTestObject ft, int add) { return getTotal(ft) + add; } + public static int augmentInjectTimesX(FeatureTestObject ft, int injected, short user) { + return ft.getX() * injected * user; + } + + public static int augmentTimesSupplier(FeatureTestObject ft, Function fn, short fnArg, int userArg) { + return fn.apply(fnArg) * userArg; + } + + public static int augmentInjectWithLambda(FeatureTestObject ft, int injected, Function fn, short arg) { + return ft.getX()*fn.apply(arg)*injected; + } + + public static int augmentInjectMultiTimesX(FeatureTestObject ft, int inject1, int inject2, short user) { + return ft.getX() * (inject1 + inject2) * user; + } + + public static int augmentInjectMultiWithLambda(FeatureTestObject ft, + int inject1, int inject2, int inject3, int inject4, Function fn, short arg) { + return ft.getX()*fn.apply(arg)*(inject1 + inject2 + inject3 + inject4); + } + private FeatureTestAugmentationObject() {} } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject.java index 59a1a62d7b8b5..43c0e6808eba3 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject.java @@ -44,6 +44,10 @@ public static int staticNumberTest(Number number) { return number.intValue(); } + public static int staticNumberArgument(int injected, int userArgument) { + return injected * userArgument; + } + private int x; private int y; public int z; @@ -90,6 +94,26 @@ public void setI(Integer i) { this.i = i; } + public int injectTimesX(int injected, short user) { + return this.x * injected * user; + } + + public int timesSupplier(Function fn, short fnArg, int userArg) { + return fn.apply(fnArg) * userArg; + } + + public int injectWithLambda(int injected, Function fn, short arg) { + return this.x*fn.apply(arg)*injected; + } + + public int injectMultiTimesX(int inject1, int inject2, int inject3, short user) { + return this.x * (inject1 + inject2 + inject3) * user; + } + + public int injectMultiWithLambda(int inject1, int inject2, int inject3, Function fn, short arg) { + return this.x*fn.apply(arg)*(inject1 + inject2 + inject3); + } + public Double mixedAdd(int i, Byte b, char c, Float f) { return (double)(i + b + c + f); } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject2.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject2.java new file mode 100644 index 0000000000000..a1fe4c5fda445 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject2.java @@ -0,0 +1,31 @@ +package org.elasticsearch.painless; + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** Currently just a dummy class for testing a few features not yet exposed by whitelist! */ +public class FeatureTestObject2 { + public FeatureTestObject2() {super();} + public static int staticNumberArgument(int injected, int userArgument) { + return injected * userArgument; + } + public static int staticNumberArgument2(int userArgument1, int userArgument2) { + return userArgument1 * userArgument2; + } +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InjectionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InjectionTests.java new file mode 100644 index 0000000000000..e4447a3a9aab0 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InjectionTests.java @@ -0,0 +1,217 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless; + +public class InjectionTests extends ScriptTestCase { + + public void testInjection() { + assertEquals(16, + exec("org.elasticsearch.painless.FeatureTestObject.staticNumberArgument(8);")); + } + + public void testInstanceInjection() { + assertEquals(1000, + exec("org.elasticsearch.painless.FeatureTestObject f = new org.elasticsearch.painless.FeatureTestObject(100, 0); " + + "f.injectTimesX(5)")); + } + + public void testInstanceInjectWithLambda() { + assertEquals(2000, + exec("org.elasticsearch.painless.FeatureTestObject f = new org.elasticsearch.painless.FeatureTestObject(100, 0); " + + "f.injectWithLambda(x -> 2*x, 5)")); + } + + public void testInstanceInjectWithDefLambda() { + assertEquals(2000, + exec("def f = new org.elasticsearch.painless.FeatureTestObject(100, 0); f.injectWithLambda(x -> 2*x, (short)5)")); + } + + public void testInjectionOnDefNoInject() { + assertEquals(1000, + exec("def d = new org.elasticsearch.painless.FeatureTestObject(100, 0); d.injectTimesX((short)5)")); + } + + public void testInjectionOnMethodReference() { + assertEquals(60, + exec( + "def ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "org.elasticsearch.painless.FeatureTestObject ft1 = " + + " new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.timesSupplier(ft0::injectTimesX, (short)3, 5)")); + } + + public void testInjectionOnMethodReference2() { + assertEquals(60, + exec( + "org.elasticsearch.painless.FeatureTestObject ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "def ft1 = new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.timesSupplier(ft0::injectTimesX, (short)3, 5)")); + } + + public void testInjectionOnMethodReference3() { + assertEquals(60, + exec( + "def ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "def ft1 = new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.timesSupplier(ft0::injectTimesX, (short)3, 5)")); + } + + public void testAugmentedInstanceInjection() { + assertEquals(1000, + exec("org.elasticsearch.painless.FeatureTestObject f = new org.elasticsearch.painless.FeatureTestObject(100, 0); " + + "f.augmentInjectTimesX(5)")); + } + + public void testAugmentedInstanceInjectWithLambda() { + assertEquals(2000, + exec("org.elasticsearch.painless.FeatureTestObject f = new org.elasticsearch.painless.FeatureTestObject(100, 0); " + + "f.augmentInjectWithLambda(x -> 2*x, 5)")); + } + + public void testAugmentedInstanceInjectWithDefLambda() { + assertEquals(2000, + exec("def f = new org.elasticsearch.painless.FeatureTestObject(100, 0); f.augmentInjectWithLambda(x -> 2*x, (short)5)")); + } + + public void testAugmentedInjectionOnDefNoInject() { + assertEquals(1000, + exec("def d = new org.elasticsearch.painless.FeatureTestObject(100, 0); d.augmentInjectTimesX((short)5)")); + } + + public void testAugmentedInjectionOnMethodReference() { + assertEquals(60, + exec( + "def ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "org.elasticsearch.painless.FeatureTestObject ft1 = " + + " new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.augmentTimesSupplier(ft0::augmentInjectTimesX, (short)3, 5)")); + } + + public void testAugmentedInjectionOnMethodReference2() { + assertEquals(60, + exec( + "org.elasticsearch.painless.FeatureTestObject ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "def ft1 = new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.augmentTimesSupplier(ft0::augmentInjectTimesX, (short)3, 5)")); + } + + public void testAugmentedInjectionOnMethodReference3() { + assertEquals(60, + exec( + "def ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "def ft1 = new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.augmentTimesSupplier(ft0::augmentInjectTimesX, (short)3, 5)")); + } + + public void testInstanceMultiInjection() { + assertEquals(6000, + exec("org.elasticsearch.painless.FeatureTestObject f = new org.elasticsearch.painless.FeatureTestObject(100, 0); " + + "f.injectMultiTimesX(5)")); + } + + public void testInstanceMultiInjectWithLambda() { + assertEquals(8000, + exec("org.elasticsearch.painless.FeatureTestObject f = new org.elasticsearch.painless.FeatureTestObject(100, 0); " + + "f.injectMultiWithLambda(x -> 2*x, 5)")); + } + + public void testInstanceMultiInjectWithDefLambda() { + assertEquals(2000, + exec("def f = new org.elasticsearch.painless.FeatureTestObject(100, 0); f.injectWithLambda(x -> 2*x, (short)5)")); + } + + public void testMultiInjectionOnDefNoMultiInject() { + assertEquals(6000, + exec("def d = new org.elasticsearch.painless.FeatureTestObject(100, 0); d.injectMultiTimesX((short)5)")); + } + + public void testMultiInjectionOnMethodReference() { + assertEquals(60, + exec( + "def ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "org.elasticsearch.painless.FeatureTestObject ft1 = " + + " new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.timesSupplier(ft0::injectTimesX, (short)3, 5)")); + } + + public void testMultiInjectionOnMethodReference2() { + assertEquals(60, + exec( + "org.elasticsearch.painless.FeatureTestObject ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "def ft1 = new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.timesSupplier(ft0::injectTimesX, (short)3, 5)")); + } + + public void testMultiInjectionOnMethodReference3() { + assertEquals(60, + exec( + "def ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "def ft1 = new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.timesSupplier(ft0::injectTimesX, (short)3, 5)")); + } + + public void testAugmentedInstanceMultiInjection() { + assertEquals(5000, + exec("org.elasticsearch.painless.FeatureTestObject f = new org.elasticsearch.painless.FeatureTestObject(100, 0); " + + "f.augmentInjectMultiTimesX(5)")); + } + + public void testAugmentedInstanceMultiInjectWithLambda() { + assertEquals(20000, + exec("org.elasticsearch.painless.FeatureTestObject f = new org.elasticsearch.painless.FeatureTestObject(100, 0); " + + "f.augmentInjectMultiWithLambda(x -> 2*x, 5)")); + } + + public void testAugmentedInstanceMultiInjectWithDefLambda() { + assertEquals(20000, + exec("def f = new org.elasticsearch.painless.FeatureTestObject(100, 0); " + + "f.augmentInjectMultiWithLambda(x -> 2*x, (short)5)")); + } + + public void testAugmentedMultiInjectionOnDefNoMultiInject() { + assertEquals(5000, + exec("def d = new org.elasticsearch.painless.FeatureTestObject(100, 0); d.augmentInjectMultiTimesX((short)5)")); + } + + public void testAugmentedMultiInjectionOnMethodReference() { + assertEquals(300, + exec( + "def ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "org.elasticsearch.painless.FeatureTestObject ft1 = " + + " new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.augmentTimesSupplier(ft0::augmentInjectMultiTimesX, (short)3, 5)")); + } + + public void testAugmentedMultiInjectionOnMethodReference2() { + assertEquals(300, + exec( + "org.elasticsearch.painless.FeatureTestObject ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "def ft1 = new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.augmentTimesSupplier(ft0::augmentInjectMultiTimesX, (short)3, 5)")); + } + + public void testAugmentedMultiInjectionOnMethodReference3() { + assertEquals(300, + exec( + "def ft0 = new org.elasticsearch.painless.FeatureTestObject(2, 0); " + + "def ft1 = new org.elasticsearch.painless.FeatureTestObject(1000, 0); " + + "ft1.augmentTimesSupplier(ft0::augmentInjectMultiTimesX, (short)3, 5)")); + } +} diff --git a/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test index 1d27b7e44313a..dbea835edeb03 100644 --- a/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test +++ b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test @@ -8,6 +8,12 @@ class org.elasticsearch.script.JodaCompatibleZonedDateTime { class org.elasticsearch.painless.BindingsTests$BindingsTestScript { } +class org.elasticsearch.painless.FeatureTestObject2 { + () + int staticNumberArgument(int, int) @inject_constant[1="foo"] + int staticNumberArgument2(int, int) +} + class org.elasticsearch.painless.FeatureTestObject @no_import { int z () @@ -21,9 +27,20 @@ class org.elasticsearch.painless.FeatureTestObject @no_import { boolean overloadedStatic() boolean overloadedStatic(boolean) int staticNumberTest(Number) + int staticNumberArgument(int, int) @inject_constant[1="testInject0"] Double mixedAdd(int, Byte, char, Float) Object twoFunctionsOfX(Function,Function) void listInput(List) + int injectTimesX(int, short) @inject_constant[1="testInject0"] + int timesSupplier(Function, short, int) + int injectWithLambda(int, Function, short) @inject_constant[1="testInject0"] + int org.elasticsearch.painless.FeatureTestAugmentationObject augmentInjectTimesX(int, short) @inject_constant[1="testInject0"] + int org.elasticsearch.painless.FeatureTestAugmentationObject augmentTimesSupplier(Function, short, int) + int org.elasticsearch.painless.FeatureTestAugmentationObject augmentInjectWithLambda(int, Function, short) @inject_constant[1="testInject0"] + int injectMultiTimesX(int, int, int, short) @inject_constant[1="testInject0", 2="testInject1", 3="testInject2"] + int injectMultiWithLambda(int, int, int, Function, short) @inject_constant[1="testInject0", 2="testInject1", 3="testInject0"] + int org.elasticsearch.painless.FeatureTestAugmentationObject augmentInjectMultiTimesX(int, int, short) @inject_constant[1="testInject1", 2="testInject2"] + int org.elasticsearch.painless.FeatureTestAugmentationObject augmentInjectMultiWithLambda(int, int, int, int, Function, short) @inject_constant[1="testInject2", 2="testInject1", 3="testInject1", 4="testInject2"] int org.elasticsearch.painless.FeatureTestAugmentationObject getTotal() int org.elasticsearch.painless.FeatureTestAugmentationObject addToTotal(int) } @@ -34,4 +51,4 @@ static_import { int addWithState(int, int, int, double) bound_to org.elasticsearch.painless.BindingsTests$BindingTestClass int addThisWithState(BindingsTests.BindingsTestScript, int, int, int, double) bound_to org.elasticsearch.painless.BindingsTests$ThisBindingTestClass int addEmptyThisWithState(BindingsTests.BindingsTestScript, int) bound_to org.elasticsearch.painless.BindingsTests$EmptyThisBindingTestClass -} \ No newline at end of file +} diff --git a/server/src/main/java/org/elasticsearch/script/ScriptContextInfo.java b/server/src/main/java/org/elasticsearch/script/ScriptContextInfo.java index 65bbcf2cded82..a97d8c677eae2 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptContextInfo.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptContextInfo.java @@ -329,7 +329,7 @@ static ScriptMethodInfo executeFromContext(Class clazz) { Class[] parameterTypes = execute.getParameterTypes(); List parameters = new ArrayList<>(); if (parameterTypes.length > 0) { - // TODO(stu): ensure empty/no PARAMETERS if parameterTypes.length == 0? + // TODO: ensure empty/no PARAMETERS if parameterTypes.length == 0? String parametersFieldName = "PARAMETERS"; // See ScriptClassInfo.readArgumentNamesConstant From 6d58686cfb46b976007f235e9c04641881684a29 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 29 Sep 2020 12:31:33 -0500 Subject: [PATCH 02/13] Add javadoc, remove done TODOS --- .../InjectConstantAnnotationParser.java | 2 -- .../painless/CompilerSettings.java | 23 ++++++++++++++++--- .../java/org/elasticsearch/painless/Def.java | 2 ++ .../lookup/PainlessLookupBuilder.java | 1 - .../phase/DefaultUserTreeToIRTreePhase.java | 3 --- .../phase/PainlessUserTreeToIRTreePhase.java | 2 +- .../painless/symbol/ScriptScope.java | 1 - 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java index 3c88ae3f529fd..765c1b22245b5 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java @@ -39,8 +39,6 @@ public Object parse(Map arguments) { if (arguments.containsKey(argNum) == false) { throw new IllegalArgumentException("[@inject_constant] missing argument number [" + argNum + "]"); } - // TODO(stu): Jack, how do I verify against CompilerSettings. - // answer: do validation in PainlessLookupBuilder argList.add(arguments.get(argNum)); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java index 57270e8792658..c57bf8fde7a0b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java @@ -31,7 +31,9 @@ */ public final class CompilerSettings { /** - * Are regexes enabled? If + * Are regexes enabled? If {@code true}, regexes are enabled and unlimited by the limit factor. If {@code false}, they are completely + * disabled. If {@code use-limit}, the default, regexes are enabled but limited in complexity according to the + * {@code script.painless.regex.limit-factor} setting. */ public static final Setting REGEX_ENABLED = new Setting<>("script.painless.regex.enabled", RegexEnabled.USE_FACTOR.value, RegexEnabled::parse, Property.NodeScope); @@ -148,21 +150,30 @@ public RegexEnabled areRegexesEnabled() { } /** - * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple - * looking regexes can cause stack overflows. + * Are regexes enabled or limited? */ public void setRegexesEnabled(RegexEnabled regexesEnabled) { this.regexesEnabled = regexesEnabled; } + /** + * What is the limitation on regex complexity? How many multiples of input length can a regular expression consider? + */ public void setRegexLimitFactor(int regexLimitFactor) { this.regexLimitFactor = regexLimitFactor; } + /** + * What is the limit factor for regexes? + */ public int getRegexLimitFactor() { return regexLimitFactor; } + /** + * Get compiler settings as a map. This is used to inject compiler settings into augmented methods with the {@code @inject_constant} + * annotation. + */ public Map asMap() { int regexLimitFactor = this.regexLimitFactor; if (regexesEnabled == RegexEnabled.TRUE) { @@ -181,6 +192,9 @@ public Map asMap() { return map; } + /** + * Options for {@code script.painless.regex.enabled} setting. + */ public enum RegexEnabled { TRUE("true"), FALSE("false"), @@ -191,6 +205,9 @@ public enum RegexEnabled { this.value = value; } + /** + * Parse string value, necessary because `valueOf` would require strings to be upper case. + */ public static RegexEnabled parse(String value) { if (TRUE.value.equals(value)) { return TRUE; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java index 9f83c0b434a38..4e2ff07b5b22f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java @@ -182,6 +182,8 @@ static MethodHandle arrayLengthGetter(Class arrayType) { * Otherwise it returns a handle to the matching method. *

* @param painlessLookup the whitelist + * @param functions user defined functions and lambdas + * @param constants available constants to be used if the method has the {@code InjectConstantAnnotation} * @param methodHandlesLookup caller's lookup * @param callSiteType callsite's type * @param receiverClass Class of the object to invoke the method on. diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index db2d4d7831dac..d6b9fd63a3ba2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -519,7 +519,6 @@ public void addPainlessMethod(Class targetClass, Class augmentedClass, "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", nsme); } } else { - // TODO(stu): fix try { javaMethod = augmentedClass.getMethod(methodName, javaTypeParameters.toArray(new Class[typeParametersSize])); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java index 3d95ea8738842..9d576966f7633 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java @@ -244,7 +244,6 @@ protected void injectBootstrapMethod(ScriptScope scriptScope) { irClassNode.addFieldNode(irFieldNode); - // TODO(stu): add compiler settings here irFieldNode = new FieldNode(); irFieldNode.setLocation(internalLocation); irFieldNode.setModifiers(modifiers); @@ -346,7 +345,6 @@ protected void injectBootstrapMethod(ScriptScope scriptScope) { invokeCallNode.addArgumentNode(irLoadFieldMemberNode); - // TODO(stu): copy for compiler settings irLoadFieldMemberNode = new LoadFieldMemberNode(); irLoadFieldMemberNode.setLocation(internalLocation); irLoadFieldMemberNode.setExpressionType(Map.class); @@ -1314,7 +1312,6 @@ public void visitRegex(ERegex userRegexNode, ScriptScope scriptScope) { invokeCallNode.setLocation(userRegexNode.getLocation()); invokeCallNode.setExpressionType(Pattern.class); invokeCallNode.setBox(Pattern.class); - // scriptScope.getCompilerSettings() invokeCallNode.setMethod(new PainlessMethod( Pattern.class.getMethod("compile", String.class, int.class), Pattern.class, diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java index 4929f9b20138b..9274f53d96c41 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java @@ -211,7 +211,7 @@ protected void injectStaticFieldsAndGetters() { irLoadFieldMemberNode.setLocation(internalLocation); irLoadFieldMemberNode.setExpressionType(String.class); irLoadFieldMemberNode.setName("$NAME"); - irLoadFieldMemberNode.setStatic(true); // TODO(stu): add $COMPILER_INJECTS, add hash map and set it + irLoadFieldMemberNode.setStatic(true); irReturnNode.setExpressionNode(irLoadFieldMemberNode); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java index 3bfccc60c5810..4edd2837982dc 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java @@ -66,7 +66,6 @@ public ScriptScope(PainlessLookup painlessLookup, CompilerSettings compilerSetti staticConstants.put("$SOURCE", scriptSource); staticConstants.put("$DEFINITION", painlessLookup); staticConstants.put("$FUNCTIONS", functionTable); - // TODO(stu): inject compiler settings here staticConstants.put("$COMPILERSETTINGS", compilerSettings.asMap()); } From 3a139fc831d4c555307ea84b896a6a0ea5d2073a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 30 Sep 2020 09:47:24 -0500 Subject: [PATCH 03/13] Regexes are enabled by default --- .../elasticsearch/painless/WhenThingsGoWrongTests.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java index a6c79a5ba1389..e13cdacfda48f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java @@ -262,12 +262,6 @@ public void testStackOverflowError() { }); } - public void testRegexDisabledByDefault() { - IllegalStateException e = expectScriptThrows(IllegalStateException.class, () -> exec("return 'foo' ==~ /foo/")); - assertEquals("Regexes are disabled. Set [script.painless.regex.enabled] to [true] in elasticsearch.yaml to allow them. " - + "Be careful though, regexes break out of Painless's protection against deep recursion and long loops.", e.getMessage()); - } - public void testCanNotOverrideRegexEnabled() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> exec("", null, singletonMap(CompilerSettings.REGEX_ENABLED.getKey(), "true"), false)); @@ -540,7 +534,7 @@ public void testCannotResolveSymbol() { iae = expectScriptThrows(IllegalArgumentException.class, () -> exec("while (test0) {int x = 1;}")); assertEquals(iae.getMessage(), "cannot resolve symbol [test0]"); } - + public void testPartialType() { int dots = randomIntBetween(1, 5); StringBuilder builder = new StringBuilder("test0"); From b89bea1bde0a7eafbe0111b3ecd3e604c347f06d Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 30 Sep 2020 09:55:44 -0500 Subject: [PATCH 04/13] use_factor -> limited, InjectConstantAnnotation javadocs --- .../spi/annotation/InjectConstantAnnotation.java | 5 +++++ .../org/elasticsearch/painless/CompilerSettings.java | 12 ++++++------ .../elasticsearch/painless/spi/java.util.regex.txt | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java index fde52ed7aef2c..b4b810ffc9607 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java @@ -22,6 +22,11 @@ import java.util.Collections; import java.util.List; +/** + * Inject compiler setting constants. + * Format: {@code inject_constant["1=foo_compiler_setting", 2="bar_compiler_setting"]} injects "foo_compiler_setting and + * "bar_compiler_setting" as the first two arguments (other than receiver reference for instance methods) to the annotated method. + */ public class InjectConstantAnnotation { public static final String NAME = "inject_constant"; public final List injects; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java index c57bf8fde7a0b..8909e11cf3ce0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java @@ -36,7 +36,7 @@ public final class CompilerSettings { * {@code script.painless.regex.limit-factor} setting. */ public static final Setting REGEX_ENABLED = - new Setting<>("script.painless.regex.enabled", RegexEnabled.USE_FACTOR.value, RegexEnabled::parse, Property.NodeScope); + new Setting<>("script.painless.regex.enabled", RegexEnabled.LIMITED.value, RegexEnabled::parse, Property.NodeScope); /** * How complex can a regex be? This is the number of characters that can be considered expressed as a multiple of string length. @@ -84,7 +84,7 @@ public final class CompilerSettings { /** * Are regexes enabled? Defaults to using the factor setting. */ - private RegexEnabled regexesEnabled = RegexEnabled.USE_FACTOR; + private RegexEnabled regexesEnabled = RegexEnabled.LIMITED; /** @@ -198,7 +198,7 @@ public Map asMap() { public enum RegexEnabled { TRUE("true"), FALSE("false"), - USE_FACTOR("use-factor"); + LIMITED("limited"); final String value; RegexEnabled(String value) { @@ -213,11 +213,11 @@ public static RegexEnabled parse(String value) { return TRUE; } else if (FALSE.value.equals(value)) { return FALSE; - } else if (USE_FACTOR.value.equals(value)) { - return USE_FACTOR; + } else if (LIMITED.value.equals(value)) { + return LIMITED; } throw new IllegalArgumentException( - "invalid value [" + value + "] must be one of [" + TRUE.value + "," + FALSE.value + "," + USE_FACTOR.value + "]" + "invalid value [" + value + "] must be one of [" + TRUE.value + "," + FALSE.value + "," + LIMITED.value + "]" ); } } diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.regex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.regex.txt index 521d092efb97b..bf8be701b7d93 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.regex.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.regex.txt @@ -58,7 +58,7 @@ class java.util.regex.Matcher { String replaceFirst(String) boolean requireEnd() Matcher reset() - # Whitelisting Matcher.reset(String) works around the regex limiting + # Note: Do not whitelist Matcher.reset(String), it subverts regex limiting int start() int start(int) Matcher useAnchoringBounds(boolean) From 8e17ec86bbd7c8dcff340237f27b784d3e2fa978 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 1 Oct 2020 14:30:46 -0500 Subject: [PATCH 05/13] find and match operator --- .../elasticsearch/painless/WriterConstants.java | 2 +- .../painless/api/LimitedCharSequence.java | 10 +++------- .../elasticsearch/painless/ir/BinaryMathNode.java | 14 +++++++++++++- .../painless/lookup/PainlessLookupBuilder.java | 2 ++ .../phase/DefaultUserTreeToIRTreePhase.java | 9 +++++---- .../elasticsearch/painless/AugmentationTests.java | 2 ++ .../painless/spi/org.elasticsearch.painless.test | 6 ------ 7 files changed, 26 insertions(+), 19 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java index 3c70e7d72fa77..5398579b091df 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java @@ -80,7 +80,7 @@ public final class WriterConstants { * regex per time it is run. */ public static final Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class, int.class); - public static final Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class); + public static final Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", Pattern.class, int.class, CharSequence.class); public static final Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches"); public static final Method MATCHER_FIND = getAsmMethod(boolean.class, "find"); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java index 0a39169174f3d..b2911fac7b56c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java @@ -75,22 +75,18 @@ public static CharSequence limitedCharSequence(CharSequence wrapped, Pattern pat } public String details(int index) { + // TODO(stu): pattern may be null return "pattern: [" + pattern.pattern() + "], " + "limit factor: [" + limitFactor + "], " + "char limit: [" + counter.charAtLimit + "], " + - "snippet: [" + sequenceSnippet(index) + "], " + + // TODO(stu): add ... for long fields + // "snippet: [" + sequenceSnippet(index) + "], " + "count: [" + counter.count + "], " + - // TODO(stu): remove these when sequenceSnippet is implemented "isSubSequence: [" + isSubSequence + "], " + "offset: [" + offset + "], " + "wrapped: [" + wrapped.toString() + "]"; } - String sequenceSnippet(int index) { - // TODO(stu): consider isSubSequence and snippetLimit - return original.toString(); - } - @Override public int length() { return wrapped.length(); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java index 441752e78e073..ab565d8452502 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java @@ -24,6 +24,7 @@ import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.Operation; import org.elasticsearch.painless.WriterConstants; +import org.elasticsearch.painless.api.Augmentation; import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.elasticsearch.painless.lookup.def; import org.elasticsearch.painless.phase.IRTreeVisitor; @@ -40,6 +41,8 @@ public class BinaryMathNode extends BinaryNode { private Class binaryType; private Class shiftType; private int flags; + // TODO(stu): DefaultUserTreeToIRTree -> visitRegex should have compiler settings in script set. set it + private int regexLimit; public void setOperation(Operation operation) { this.operation = operation; @@ -81,6 +84,14 @@ public int getFlags() { return flags; } + public void setRegexLimit(int regexLimit) { + this.regexLimit = regexLimit; + } + + public int getRegexLimit() { + return regexLimit; + } + /* ---- end node data, begin visitor ---- */ @Override @@ -106,8 +117,9 @@ protected void write(ClassWriter classWriter, MethodWriter methodWriter, WriteSc if (operation == Operation.FIND || operation == Operation.MATCH) { getRightNode().write(classWriter, methodWriter, writeScope); + methodWriter.push(regexLimit); getLeftNode().write(classWriter, methodWriter, writeScope); - methodWriter.invokeVirtual(org.objectweb.asm.Type.getType(Pattern.class), WriterConstants.PATTERN_MATCHER); + methodWriter.invokeStatic(org.objectweb.asm.Type.getType(Augmentation.class), WriterConstants.PATTERN_MATCHER); if (operation == Operation.FIND) { methodWriter.invokeVirtual(org.objectweb.asm.Type.getType(Matcher.class), WriterConstants.MATCHER_FIND); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index d6b9fd63a3ba2..90f4276e30c10 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -56,6 +56,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.elasticsearch.painless.WriterConstants.DEF_TO_B_BYTE_IMPLICIT; @@ -250,6 +251,7 @@ public void addPainlessClass(ClassLoader classLoader, String javaClassName, bool public void addPainlessClass(Class clazz, boolean importClassName) { Objects.requireNonNull(clazz); + //Matcher m = new Matcher(); if (clazz == def.class) { throw new IllegalArgumentException("cannot add reserved class [" + DEF_CLASS_NAME + "]"); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java index 7ac0b1057092c..325dd16b27b3c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java @@ -242,8 +242,7 @@ protected void injectBootstrapMethod(ScriptScope scriptScope) { irClassNode.addFieldNode(irFieldNode); - irFieldNode = new FieldNode(); - irFieldNode.setLocation(internalLocation); + irFieldNode = new FieldNode(internalLocation); irFieldNode.setModifiers(modifiers); irFieldNode.setFieldType(Map.class); irFieldNode.setName("$COMPILERSETTINGS"); @@ -957,6 +956,9 @@ public void visitBinary(EBinary userBinaryNode, ScriptScope scriptScope) { BinaryMathNode irBinaryMathNode = new BinaryMathNode(userBinaryNode.getLocation()); + if (operation == Operation.MATCH) { + irBinaryMathNode.setRegexLimit(scriptScope.getCompilerSettings().getRegexLimitFactor()); + } irBinaryMathNode.setBinaryType(scriptScope.getDecoration(userBinaryNode, BinaryType.class).getBinaryType()); irBinaryMathNode.setShiftType(shiftType); irBinaryMathNode.setOperation(operation); @@ -1738,8 +1740,7 @@ public void visitCall(ECall userCallNode, ScriptScope scriptScope) { throw new IllegalStateException("illegal tree structure"); } - ConstantNode constantNode = new ConstantNode(); - constantNode.setLocation(userCallNode.getLocation()); + ConstantNode constantNode = new ConstantNode(userCallNode.getLocation()); constantNode.setExpressionType(parameterType); constantNode.setConstant(injection); irInvokeCallNode.addArgumentNode(constantNode); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java index 54d8ff1fb1e88..375ef7eec9dfc 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java @@ -494,6 +494,8 @@ public void testRegexInject_DefMethodRef_SplitAsStream() { assertTrue(cbe.getMessage().contains(regexCircuitMessage)); } + //qpublic void testRegexInject + private void setRegexLimitFactor(int factor) { Settings settings = Settings.builder().put(CompilerSettings.REGEX_LIMIT_FACTOR.getKey(), factor).build(); scriptEngine = new PainlessScriptEngine(settings, scriptContexts()); diff --git a/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test index dbea835edeb03..d4913ce5344fc 100644 --- a/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test +++ b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test @@ -8,12 +8,6 @@ class org.elasticsearch.script.JodaCompatibleZonedDateTime { class org.elasticsearch.painless.BindingsTests$BindingsTestScript { } -class org.elasticsearch.painless.FeatureTestObject2 { - () - int staticNumberArgument(int, int) @inject_constant[1="foo"] - int staticNumberArgument2(int, int) -} - class org.elasticsearch.painless.FeatureTestObject @no_import { int z () From 51bda05efc560152f98f4f06d1f3707e454b8047 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 1 Oct 2020 14:41:16 -0500 Subject: [PATCH 06/13] Set limit factor for Find, add tests --- .../phase/DefaultUserTreeToIRTreePhase.java | 2 +- .../painless/AugmentationTests.java | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java index 325dd16b27b3c..254faccb32671 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java @@ -956,7 +956,7 @@ public void visitBinary(EBinary userBinaryNode, ScriptScope scriptScope) { BinaryMathNode irBinaryMathNode = new BinaryMathNode(userBinaryNode.getLocation()); - if (operation == Operation.MATCH) { + if (operation == Operation.MATCH || operation == Operation.FIND) { irBinaryMathNode.setRegexLimit(scriptScope.getCompilerSettings().getRegexLimitFactor()); } irBinaryMathNode.setBinaryType(scriptScope.getDecoration(userBinaryNode, BinaryType.class).getBinaryType()); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java index 375ef7eec9dfc..ec852f02ae5d0 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java @@ -494,7 +494,25 @@ public void testRegexInject_DefMethodRef_SplitAsStream() { assertTrue(cbe.getMessage().contains(regexCircuitMessage)); } - //qpublic void testRegexInject + public void testRegexInjectFindOperator() { + String script = "if (" + charSequence + " =~ " + pattern + ") { return 100; } return 200"; + setRegexLimitFactor(2); + assertEquals(Integer.valueOf(100), (Integer) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInjectMatchOperator() { + String script = "if (" + charSequence + " ==~ " + pattern + ") { return 100; } return 200"; + setRegexLimitFactor(2); + assertEquals(Integer.valueOf(100), (Integer) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } private void setRegexLimitFactor(int factor) { Settings settings = Settings.builder().put(CompilerSettings.REGEX_LIMIT_FACTOR.getKey(), factor).build(); From a8c838dfaf1a3ac32e93a48a2f1b22bcd357beb5 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 1 Oct 2020 15:14:00 -0500 Subject: [PATCH 07/13] Add pattern --- .../painless/api/LimitedCharSequence.java | 4 +- .../painless/api/LimitedCharSequenceTest.java | 79 +++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java index b2911fac7b56c..9f5ad3805471b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java @@ -39,8 +39,6 @@ public class LimitedCharSequence implements CharSequence { private final Pattern pattern; private final int limitFactor; - private static final int SNIPPET_LIMIT = 64; - private LimitedCharSequence(CharSequence wrap, Pattern pattern, int limitFactor) { this.wrapped = wrap; this.counter = new Counter(limitFactor * wrapped.length()); @@ -76,7 +74,7 @@ public static CharSequence limitedCharSequence(CharSequence wrapped, Pattern pat public String details(int index) { // TODO(stu): pattern may be null - return "pattern: [" + pattern.pattern() + "], " + + return pattern != null ? "pattern: [" + pattern.pattern() + "], " : "" + "limit factor: [" + limitFactor + "], " + "char limit: [" + counter.charAtLimit + "], " + // TODO(stu): add ... for long fields diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java new file mode 100644 index 0000000000000..f9d100faa9bb8 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.api; + +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.test.ESTestCase; + +public class LimitedCharSequenceTest extends ESTestCase { + public void testBadFactor() { + IllegalArgumentException badArg = expectThrows(IllegalArgumentException.class, + () -> LimitedCharSequence.limitedCharSequence("abc", null, -1) + ); + assertEquals("limitFactor must be positive", badArg.getMessage()); + + badArg = expectThrows(IllegalArgumentException.class, + () -> LimitedCharSequence.limitedCharSequence("abc", null, 0) + ); + assertEquals("limitFactor must be positive", badArg.getMessage()); + } + + public void testLength() { + String str = "abc"; + assertEquals(str.length(), LimitedCharSequence.limitedCharSequence("abc", null, 1).length()); + } + + public void testCharAtEqualLimit() { + String str = "abc"; + for (int limitFactor=1; limitFactor < 4; limitFactor++){ + CharSequence seq = LimitedCharSequence.limitedCharSequence(str, null, limitFactor); + for (int i=0; i seq.charAt(0)); + assertEquals( + "[scripting] Regular expression considered too many characters, " + + "limit factor: [" + limitFactor + "], " + + "char limit: [" + limit + "], " + + "count: [" + (limit + 1) + "], " + + "isSubSequence: [false], offset: [0], " + + "wrapped: [" + str + "], " + + "this limit can be changed by changed by the [script.painless.regex.limit-factor] setting", + circuitBreakingException.getMessage()); + } + } + + public void testToString() { + String str = "abc"; + assertEquals(str, LimitedCharSequence.limitedCharSequence(str, null, 1).toString()); + } +} From f6e0363343b155992604d5928ac5e49679f0a89a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 1 Oct 2020 15:56:30 -0500 Subject: [PATCH 08/13] Add tests for unlimited --- .../painless/api/Augmentation.java | 8 +- .../painless/api/LimitedCharSequence.java | 40 +-- .../painless/AugmentationTests.java | 231 -------------- .../painless/RegexLimitTests.java | 297 ++++++++++++++++++ .../painless/api/LimitedCharSequenceTest.java | 62 ++-- 5 files changed, 347 insertions(+), 291 deletions(-) create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java index cb1525b392886..41e79883034cf 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java @@ -689,27 +689,27 @@ public static String[] split(Pattern receiver, int limitFactor, CharSequence inp if (limitFactor == UNLIMITED_PATTERN_FACTOR) { return receiver.split(input); } - return receiver.split(LimitedCharSequence.limitedCharSequence(input, receiver, limitFactor)); + return receiver.split(new LimitedCharSequence(input, receiver, limitFactor)); } public static String[] split​(Pattern receiver, int limitFactor, CharSequence input, int limit) { if (limitFactor == UNLIMITED_PATTERN_FACTOR) { return receiver.split(input, limit); } - return receiver.split(LimitedCharSequence.limitedCharSequence(input, receiver, limitFactor), limit); + return receiver.split(new LimitedCharSequence(input, receiver, limitFactor), limit); } public static Stream splitAsStream​(Pattern receiver, int limitFactor, CharSequence input) { if (limitFactor == UNLIMITED_PATTERN_FACTOR) { return receiver.splitAsStream(input); } - return receiver.splitAsStream(LimitedCharSequence.limitedCharSequence(input, receiver, limitFactor)); + return receiver.splitAsStream(new LimitedCharSequence(input, receiver, limitFactor)); } public static Matcher matcher(Pattern receiver, int limitFactor, CharSequence input) { if (limitFactor == UNLIMITED_PATTERN_FACTOR) { return receiver.matcher(input); } - return receiver.matcher(LimitedCharSequence.limitedCharSequence(input, receiver, limitFactor)); + return receiver.matcher(new LimitedCharSequence(input, receiver, limitFactor)); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java index 9f5ad3805471b..8adc225cdf5c3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java @@ -33,55 +33,27 @@ public class LimitedCharSequence implements CharSequence { private final Counter counter; // for errors - private final boolean isSubSequence; - private final int offset; // if this is a subSequence, the start of the subSequence - private final CharSequence original; private final Pattern pattern; private final int limitFactor; - private LimitedCharSequence(CharSequence wrap, Pattern pattern, int limitFactor) { + public LimitedCharSequence(CharSequence wrap, Pattern pattern, int limitFactor) { + if (limitFactor <= 0) { + throw new IllegalArgumentException("limitFactor must be positive"); + } this.wrapped = wrap; this.counter = new Counter(limitFactor * wrapped.length()); - this.isSubSequence = false; - this.offset = 0; - this.original = wrap; this.pattern = pattern; this.limitFactor = limitFactor; } - // subSequence constructor - private LimitedCharSequence(LimitedCharSequence superSequence, int start, int end) { - this.wrapped = superSequence.wrapped.subSequence(start, end); - this.counter = superSequence.counter; - - this.isSubSequence = true; - this.offset = superSequence.offset + start; - this.original = superSequence.original; - this.pattern = superSequence.pattern; - this.limitFactor = superSequence.limitFactor; - } - - public static CharSequence limitedCharSequence(CharSequence wrapped, Pattern pattern, int limitFactor) { - if (limitFactor <= 0) { - throw new IllegalArgumentException("limitFactor must be positive"); - } - if (wrapped instanceof LimitedCharSequence) { - return wrapped; - } - return new LimitedCharSequence(wrapped, pattern, limitFactor); - } - public String details(int index) { - // TODO(stu): pattern may be null - return pattern != null ? "pattern: [" + pattern.pattern() + "], " : "" + + return (pattern != null ? "pattern: [" + pattern.pattern() + "], " : "") + "limit factor: [" + limitFactor + "], " + "char limit: [" + counter.charAtLimit + "], " + // TODO(stu): add ... for long fields // "snippet: [" + sequenceSnippet(index) + "], " + "count: [" + counter.count + "], " + - "isSubSequence: [" + isSubSequence + "], " + - "offset: [" + offset + "], " + "wrapped: [" + wrapped.toString() + "]"; } @@ -103,7 +75,7 @@ public char charAt(int index) { @Override public CharSequence subSequence(int start, int end) { - return new LimitedCharSequence(this, start, end); + return wrapped.subSequence(start, end); } @Override diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java index ec852f02ae5d0..689cc3700c061 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java @@ -287,235 +287,4 @@ public void testSha256() { assertEquals("fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", execDigest("'bar'.sha256()")); assertEquals("97df3588b5a3f24babc3851b372f0ba71a9dcdded43b14b9d06961bfc1707d9d", execDigest("'foobarbaz'.sha256()")); } - - // This regex has backtracking due to .*? - private final String pattern = "/abc.*?def/"; - private final String charSequence = "'abcdodef'"; - private final String splitCharSequence = "'0-abc-1-def-X-abc-2-def-Y-abc-3-def-Z-abc'"; - private final String regexCircuitMessage = "[scripting] Regular expression considered too many characters"; - - public void testRegexInject_Matcher() { - String[] scripts = new String[]{pattern + ".matcher(" + charSequence + ").matches()", - "Matcher m = " + pattern + ".matcher(" + charSequence + "); m.matches()"}; - for (String script : scripts) { - setRegexLimitFactor(2); - assertEquals(Boolean.TRUE, exec(script)); - - // Backtracking means the regular expression will fail with limit factor 1 (don't consider more than each char once) - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - } - - public void testRegexInject_Def_Matcher() { - String[] scripts = new String[]{"def p = " + pattern + "; p.matcher(" + charSequence + ").matches()", - "def p = " + pattern + "; def m = p.matcher(" + charSequence + "); m.matches()"}; - for (String script : scripts) { - setRegexLimitFactor(2); - assertEquals(Boolean.TRUE, exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - } - - public void testMethodRegexInject_Ref_Matcher() { - String script = - "boolean isMatch(Function func) { func.apply(" + charSequence +").matches(); } " + - "Pattern pattern = " + pattern + ";" + - "isMatch(pattern::matcher)"; - setRegexLimitFactor(2); - assertEquals(Boolean.TRUE, exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_DefMethodRef_Matcher() { - String script = - "boolean isMatch(Function func) { func.apply(" + charSequence +").matches(); } " + - "def pattern = " + pattern + ";" + - "isMatch(pattern::matcher)"; - setRegexLimitFactor(2); - assertEquals(Boolean.TRUE, exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_SplitLimit() { - String[] scripts = new String[]{pattern + ".split(" + splitCharSequence + ", 2)", - "Pattern p = " + pattern + "; p.split(" + splitCharSequence + ", 2)"}; - for (String script : scripts) { - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - } - - public void testRegexInject_Def_SplitLimit() { - String script = "def p = " + pattern + "; p.split(" + splitCharSequence + ", 2)"; - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_Ref_SplitLimit() { - String script = - "String[] splitLimit(BiFunction func) { func.apply(" + splitCharSequence + ", 2); } " + - "Pattern pattern = " + pattern + ";" + - "splitLimit(pattern::split)"; - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_DefMethodRef_SplitLimit() { - String script = - "String[] splitLimit(BiFunction func) { func.apply(" + splitCharSequence + ", 2); } " + - "def pattern = " + pattern + ";" + - "splitLimit(pattern::split)"; - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_Split() { - String[] scripts = new String[]{pattern + ".split(" + splitCharSequence + ")", - "Pattern p = " + pattern + "; p.split(" + splitCharSequence + ")"}; - for (String script : scripts) { - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - } - - public void testRegexInject_Def_Split() { - String script = "def p = " + pattern + "; p.split(" + splitCharSequence + ")"; - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_Ref_Split() { - String script = - "String[] split(Function func) { func.apply(" + splitCharSequence + "); } " + - "Pattern pattern = " + pattern + ";" + - "split(pattern::split)"; - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_DefMethodRef_Split() { - String script = - "String[] split(Function func) { func.apply(" + splitCharSequence +"); } " + - "def pattern = " + pattern + ";" + - "split(pattern::split)"; - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_SplitAsStream() { - String[] scripts = new String[]{pattern + ".splitAsStream(" + splitCharSequence + ").toArray(String[]::new)", - "Pattern p = " + pattern + "; p.splitAsStream(" + splitCharSequence + ").toArray(String[]::new)"}; - for (String script : scripts) { - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - } - - public void testRegexInject_Def_SplitAsStream() { - String script = "def p = " + pattern + "; p.splitAsStream(" + splitCharSequence + ").toArray(String[]::new)"; - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_Ref_SplitAsStream() { - String script = - "Stream splitStream(Function func) { func.apply(" + splitCharSequence +"); } " + - "Pattern pattern = " + pattern + ";" + - "splitStream(pattern::splitAsStream).toArray(String[]::new)"; - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInject_DefMethodRef_SplitAsStream() { - String script = - "Stream splitStream(Function func) { func.apply(" + splitCharSequence +"); } " + - "def pattern = " + pattern + ";" + - "splitStream(pattern::splitAsStream).toArray(String[]::new)"; - setRegexLimitFactor(2); - assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInjectFindOperator() { - String script = "if (" + charSequence + " =~ " + pattern + ") { return 100; } return 200"; - setRegexLimitFactor(2); - assertEquals(Integer.valueOf(100), (Integer) exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - public void testRegexInjectMatchOperator() { - String script = "if (" + charSequence + " ==~ " + pattern + ") { return 100; } return 200"; - setRegexLimitFactor(2); - assertEquals(Integer.valueOf(100), (Integer) exec(script)); - - setRegexLimitFactor(1); - CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); - assertTrue(cbe.getMessage().contains(regexCircuitMessage)); - } - - private void setRegexLimitFactor(int factor) { - Settings settings = Settings.builder().put(CompilerSettings.REGEX_LIMIT_FACTOR.getKey(), factor).build(); - scriptEngine = new PainlessScriptEngine(settings, scriptContexts()); - } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java new file mode 100644 index 0000000000000..fd99ceb9f3b12 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java @@ -0,0 +1,297 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless; + +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.settings.Settings; + +public class RegexLimitTests extends ScriptTestCase { + // This regex has backtracking due to .*? + private final String pattern = "/abc.*?def/"; + private final String charSequence = "'abcdodef'"; + private final String splitCharSequence = "'0-abc-1-def-X-abc-2-def-Y-abc-3-def-Z-abc'"; + private final String regexCircuitMessage = "[scripting] Regular expression considered too many characters"; + + public void testRegexInject_Matcher() { + String[] scripts = new String[]{pattern + ".matcher(" + charSequence + ").matches()", + "Matcher m = " + pattern + ".matcher(" + charSequence + "); m.matches()"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertEquals(Boolean.TRUE, exec(script)); + + // Backtracking means the regular expression will fail with limit factor 1 (don't consider more than each char once) + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testRegexInjectUnlimited_Matcher() { + String[] scripts = new String[]{pattern + ".matcher(" + charSequence + ").matches()", + "Matcher m = " + pattern + ".matcher(" + charSequence + "); m.matches()"}; + for (String script : scripts) { + setRegexEnabled(); + assertEquals(Boolean.TRUE, exec(script)); + } + } + + public void testRegexInject_Def_Matcher() { + String[] scripts = new String[]{"def p = " + pattern + "; p.matcher(" + charSequence + ").matches()", + "def p = " + pattern + "; def m = p.matcher(" + charSequence + "); m.matches()"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertEquals(Boolean.TRUE, exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testMethodRegexInject_Ref_Matcher() { + String script = + "boolean isMatch(Function func) { func.apply(" + charSequence +").matches(); } " + + "Pattern pattern = " + pattern + ";" + + "isMatch(pattern::matcher)"; + setRegexLimitFactor(2); + assertEquals(Boolean.TRUE, exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_DefMethodRef_Matcher() { + String script = + "boolean isMatch(Function func) { func.apply(" + charSequence +").matches(); } " + + "def pattern = " + pattern + ";" + + "isMatch(pattern::matcher)"; + setRegexLimitFactor(2); + assertEquals(Boolean.TRUE, exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_SplitLimit() { + String[] scripts = new String[]{pattern + ".split(" + splitCharSequence + ", 2)", + "Pattern p = " + pattern + "; p.split(" + splitCharSequence + ", 2)"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testRegexInjectUnlimited_SplitLimit() { + String[] scripts = new String[]{pattern + ".split(" + splitCharSequence + ", 2)", + "Pattern p = " + pattern + "; p.split(" + splitCharSequence + ", 2)"}; + for (String script : scripts) { + setRegexEnabled(); + assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); + } + } + + public void testRegexInject_Def_SplitLimit() { + String script = "def p = " + pattern + "; p.split(" + splitCharSequence + ", 2)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_Ref_SplitLimit() { + String script = + "String[] splitLimit(BiFunction func) { func.apply(" + splitCharSequence + ", 2); } " + + "Pattern pattern = " + pattern + ";" + + "splitLimit(pattern::split)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_DefMethodRef_SplitLimit() { + String script = + "String[] splitLimit(BiFunction func) { func.apply(" + splitCharSequence + ", 2); } " + + "def pattern = " + pattern + ";" + + "splitLimit(pattern::split)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-abc-2-def-Y-abc-3-def-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_Split() { + String[] scripts = new String[]{pattern + ".split(" + splitCharSequence + ")", + "Pattern p = " + pattern + "; p.split(" + splitCharSequence + ")"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testRegexInjectUnlimited_Split() { + String[] scripts = new String[]{pattern + ".split(" + splitCharSequence + ")", + "Pattern p = " + pattern + "; p.split(" + splitCharSequence + ")"}; + for (String script : scripts) { + setRegexEnabled(); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); + } + } + + public void testRegexInject_Def_Split() { + String script = "def p = " + pattern + "; p.split(" + splitCharSequence + ")"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_Ref_Split() { + String script = + "String[] split(Function func) { func.apply(" + splitCharSequence + "); } " + + "Pattern pattern = " + pattern + ";" + + "split(pattern::split)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_DefMethodRef_Split() { + String script = + "String[] split(Function func) { func.apply(" + splitCharSequence +"); } " + + "def pattern = " + pattern + ";" + + "split(pattern::split)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[])exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_SplitAsStream() { + String[] scripts = new String[]{pattern + ".splitAsStream(" + splitCharSequence + ").toArray(String[]::new)", + "Pattern p = " + pattern + "; p.splitAsStream(" + splitCharSequence + ").toArray(String[]::new)"}; + for (String script : scripts) { + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + } + + public void testRegexInjectUnlimited_SplitAsStream() { + String[] scripts = new String[]{pattern + ".splitAsStream(" + splitCharSequence + ").toArray(String[]::new)", + "Pattern p = " + pattern + "; p.splitAsStream(" + splitCharSequence + ").toArray(String[]::new)"}; + for (String script : scripts) { + setRegexEnabled(); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); + } + } + + public void testRegexInject_Def_SplitAsStream() { + String script = "def p = " + pattern + "; p.splitAsStream(" + splitCharSequence + ").toArray(String[]::new)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_Ref_SplitAsStream() { + String script = + "Stream splitStream(Function func) { func.apply(" + splitCharSequence +"); } " + + "Pattern pattern = " + pattern + ";" + + "splitStream(pattern::splitAsStream).toArray(String[]::new)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInject_DefMethodRef_SplitAsStream() { + String script = + "Stream splitStream(Function func) { func.apply(" + splitCharSequence +"); } " + + "def pattern = " + pattern + ";" + + "splitStream(pattern::splitAsStream).toArray(String[]::new)"; + setRegexLimitFactor(2); + assertArrayEquals(new String[]{"0-", "-X-", "-Y-", "-Z-abc"}, (String[]) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInjectFindOperator() { + String script = "if (" + charSequence + " =~ " + pattern + ") { return 100; } return 200"; + setRegexLimitFactor(2); + assertEquals(Integer.valueOf(100), (Integer) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + public void testRegexInjectMatchOperator() { + String script = "if (" + charSequence + " ==~ " + pattern + ") { return 100; } return 200"; + setRegexLimitFactor(2); + assertEquals(Integer.valueOf(100), (Integer) exec(script)); + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + } + + private void setRegexLimitFactor(int factor) { + Settings settings = Settings.builder().put(CompilerSettings.REGEX_LIMIT_FACTOR.getKey(), factor).build(); + scriptEngine = new PainlessScriptEngine(settings, scriptContexts()); + } + + private void setRegexEnabled() { + Settings settings = Settings.builder().put(CompilerSettings.REGEX_ENABLED.getKey(), "true").build(); + scriptEngine = new PainlessScriptEngine(settings, scriptContexts()); + } +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java index f9d100faa9bb8..55d7d4c3522b7 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java @@ -22,58 +22,76 @@ import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.test.ESTestCase; +import java.util.regex.Pattern; + public class LimitedCharSequenceTest extends ESTestCase { public void testBadFactor() { IllegalArgumentException badArg = expectThrows(IllegalArgumentException.class, - () -> LimitedCharSequence.limitedCharSequence("abc", null, -1) + () -> new LimitedCharSequence("abc", null, -1) ); assertEquals("limitFactor must be positive", badArg.getMessage()); badArg = expectThrows(IllegalArgumentException.class, - () -> LimitedCharSequence.limitedCharSequence("abc", null, 0) + () -> new LimitedCharSequence("abc", null, 0) ); assertEquals("limitFactor must be positive", badArg.getMessage()); } public void testLength() { String str = "abc"; - assertEquals(str.length(), LimitedCharSequence.limitedCharSequence("abc", null, 1).length()); + assertEquals(str.length(), new LimitedCharSequence("abc", null, 1).length()); } public void testCharAtEqualLimit() { String str = "abc"; for (int limitFactor=1; limitFactor < 4; limitFactor++){ - CharSequence seq = LimitedCharSequence.limitedCharSequence(str, null, limitFactor); + CharSequence seq = new LimitedCharSequence(str, null, limitFactor); for (int i=0; i seq.charAt(0)); - assertEquals( - "[scripting] Regular expression considered too many characters, " + - "limit factor: [" + limitFactor + "], " + - "char limit: [" + limit + "], " + - "count: [" + (limit + 1) + "], " + - "isSubSequence: [false], offset: [0], " + - "wrapped: [" + str + "], " + - "this limit can be changed by changed by the [script.painless.regex.limit-factor] setting", - circuitBreakingException.getMessage()); + String patternStr = "a.*bc"; + Pattern p = Pattern.compile(patternStr); + final CharSequence seq = new LimitedCharSequence(str, p, 2); + for (int i = 0; i < 6; i++) { + seq.charAt(0); } + CircuitBreakingException circuitBreakingException = expectThrows(CircuitBreakingException.class, () -> seq.charAt(0)); + assertEquals( + "[scripting] Regular expression considered too many characters, " + + "pattern: [a.*bc], " + + "limit factor: [2], " + + "char limit: [6], " + + "count: [7], " + + "wrapped: [abc], " + + "this limit can be changed by changed by the [script.painless.regex.limit-factor] setting", + circuitBreakingException.getMessage()); + + final CharSequence seqNullPattern = new LimitedCharSequence(str, null, 2); + for (int i = 0; i < 6; i++) { + seqNullPattern.charAt(0); + } + circuitBreakingException = expectThrows(CircuitBreakingException.class, () -> seqNullPattern.charAt(0)); + assertEquals( + "[scripting] Regular expression considered too many characters, " + + "limit factor: [2], " + + "char limit: [6], " + + "count: [7], " + + "wrapped: [abc], " + + "this limit can be changed by changed by the [script.painless.regex.limit-factor] setting", + circuitBreakingException.getMessage()); + } + + public void testSubSequence() { + assertEquals("def", (new LimitedCharSequence("abcdef", null, 1)).subSequence(3, 6)); } public void testToString() { String str = "abc"; - assertEquals(str, LimitedCharSequence.limitedCharSequence(str, null, 1).toString()); + assertEquals(str, new LimitedCharSequence(str, null, 1).toString()); } } From 821639163de34db8b16386d8a9735a93edec4fb7 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 1 Oct 2020 16:04:46 -0500 Subject: [PATCH 09/13] fix style problems --- .../main/java/org/elasticsearch/painless/ir/BinaryMathNode.java | 1 - .../elasticsearch/painless/lookup/PainlessLookupBuilder.java | 1 - .../test/java/org/elasticsearch/painless/AugmentationTests.java | 2 -- ...mitedCharSequenceTest.java => LimitedCharSequenceTests.java} | 2 +- 4 files changed, 1 insertion(+), 5 deletions(-) rename modules/lang-painless/src/test/java/org/elasticsearch/painless/api/{LimitedCharSequenceTest.java => LimitedCharSequenceTests.java} (98%) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java index ab565d8452502..5f8ef3e123b83 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java @@ -31,7 +31,6 @@ import org.elasticsearch.painless.symbol.WriteScope; import java.util.regex.Matcher; -import java.util.regex.Pattern; public class BinaryMathNode extends BinaryNode { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index 90f4276e30c10..517c742b3d6ea 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -56,7 +56,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.elasticsearch.painless.WriterConstants.DEF_TO_B_BYTE_IMPLICIT; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java index 689cc3700c061..8fe095beeb33a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java @@ -19,8 +19,6 @@ package org.elasticsearch.painless; -import org.elasticsearch.common.breaker.CircuitBreakingException; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTests.java similarity index 98% rename from modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java rename to modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTests.java index 55d7d4c3522b7..8c3fae4fafba2 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTest.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTests.java @@ -24,7 +24,7 @@ import java.util.regex.Pattern; -public class LimitedCharSequenceTest extends ESTestCase { +public class LimitedCharSequenceTests extends ESTestCase { public void testBadFactor() { IllegalArgumentException badArg = expectThrows(IllegalArgumentException.class, () -> new LimitedCharSequence("abc", null, -1) From e8ebf4ddea4f30fa44f7cce204496ec1bb2cab78 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 1 Oct 2020 16:15:36 -0500 Subject: [PATCH 10/13] Skip todo --- .../org/elasticsearch/painless/api/LimitedCharSequence.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java index 8adc225cdf5c3..f7bab937114c6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java @@ -36,6 +36,9 @@ public class LimitedCharSequence implements CharSequence { private final Pattern pattern; private final int limitFactor; + private static final int MAX_STR_LENGTH = 64; + private static final String SNIPPET = "..."; + public LimitedCharSequence(CharSequence wrap, Pattern pattern, int limitFactor) { if (limitFactor <= 0) { throw new IllegalArgumentException("limitFactor must be positive"); @@ -51,8 +54,6 @@ public String details(int index) { return (pattern != null ? "pattern: [" + pattern.pattern() + "], " : "") + "limit factor: [" + limitFactor + "], " + "char limit: [" + counter.charAtLimit + "], " + - // TODO(stu): add ... for long fields - // "snippet: [" + sequenceSnippet(index) + "], " + "count: [" + counter.count + "], " + "wrapped: [" + wrapped.toString() + "]"; } From 6a2ed98ea3465855d82213165b7fa99a0c8f5e55 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 1 Oct 2020 16:32:10 -0500 Subject: [PATCH 11/13] add snippet --- .../painless/api/LimitedCharSequence.java | 24 +++++++++++++++---- .../painless/RegexLimitTests.java | 11 +++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java index f7bab937114c6..b3ded2fdca337 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java @@ -36,7 +36,7 @@ public class LimitedCharSequence implements CharSequence { private final Pattern pattern; private final int limitFactor; - private static final int MAX_STR_LENGTH = 64; + public static final int MAX_STR_LENGTH = 64; private static final String SNIPPET = "..."; public LimitedCharSequence(CharSequence wrap, Pattern pattern, int limitFactor) { @@ -50,12 +50,28 @@ public LimitedCharSequence(CharSequence wrap, Pattern pattern, int limitFactor) this.limitFactor = limitFactor; } - public String details(int index) { + public String details() { return (pattern != null ? "pattern: [" + pattern.pattern() + "], " : "") + "limit factor: [" + limitFactor + "], " + "char limit: [" + counter.charAtLimit + "], " + "count: [" + counter.count + "], " + - "wrapped: [" + wrapped.toString() + "]"; + "wrapped: [" + snippet(MAX_STR_LENGTH) + "]"; + } + + /** + * Snip a long wrapped CharSequences for error messages + */ + String snippet(int maxStrLength) { + if (maxStrLength < SNIPPET.length() * 6) { + throw new IllegalArgumentException("max str length must be large enough to include three snippets and three context chars, " + + "at least [" + SNIPPET.length() * 6 +"], not [" + maxStrLength + "]"); + } + + if (wrapped.length() <= maxStrLength) { + return wrapped.toString(); + } + + return wrapped.subSequence(0, maxStrLength - SNIPPET.length()) + "..." ; } @Override @@ -67,7 +83,7 @@ public int length() { public char charAt(int index) { counter.count++; if (counter.hitLimit()) { - throw new CircuitBreakingException("[scripting] Regular expression considered too many characters, " + details(index) + + throw new CircuitBreakingException("[scripting] Regular expression considered too many characters, " + details() + ", this limit can be changed by changed by the [" + CompilerSettings.REGEX_LIMIT_FACTOR.getKey() + "] setting", CircuitBreaker.Durability.TRANSIENT); } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java index fd99ceb9f3b12..5c7f996d237f2 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.painless.api.LimitedCharSequenceTests; public class RegexLimitTests extends ScriptTestCase { // This regex has backtracking due to .*? @@ -285,6 +286,16 @@ public void testRegexInjectMatchOperator() { assertTrue(cbe.getMessage().contains(regexCircuitMessage)); } + public void testSnippetRegex() { + String charSequence = "abcdef123456".repeat(100); + String script = "if ('" + charSequence + "' ==~ " + pattern + ") { return 100; } return 200"; + + setRegexLimitFactor(1); + CircuitBreakingException cbe = expectScriptThrows(CircuitBreakingException.class, () -> exec(script)); + assertTrue(cbe.getMessage().contains(regexCircuitMessage)); + assertTrue(cbe.getMessage().contains(charSequence.subSequence(0, 61) + "...")); + } + private void setRegexLimitFactor(int factor) { Settings settings = Settings.builder().put(CompilerSettings.REGEX_LIMIT_FACTOR.getKey(), factor).build(); scriptEngine = new PainlessScriptEngine(settings, scriptContexts()); From 60bc11a96b7f0ce6861c3259d86bc4386366575e Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 1 Oct 2020 16:34:28 -0500 Subject: [PATCH 12/13] unused import --- .../test/java/org/elasticsearch/painless/RegexLimitTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java index 5c7f996d237f2..3b5a88fe8d2df 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.painless.api.LimitedCharSequenceTests; public class RegexLimitTests extends ScriptTestCase { // This regex has backtracking due to .*? From 297d4403faf836c069ab1148f8306fff595a7a0d Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Fri, 2 Oct 2020 11:07:10 -0500 Subject: [PATCH 13/13] No longer disabled --- .../test/painless/40_disabled.yml | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_disabled.yml diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_disabled.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_disabled.yml deleted file mode 100644 index 245f14641f7a5..0000000000000 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_disabled.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -"Regex in update fails": - - - do: - index: - index: test_1 - id: 1 - body: - foo: bar - count: 1 - - - do: - catch: /Regexes are disabled. Set \[script.painless.regex.enabled\] to \[true\] in elasticsearch.yaml to allow them. Be careful though, regexes break out of Painless's protection against deep recursion and long loops./ - update: - index: test_1 - id: 1 - body: - script: - lang: painless - inline: "ctx._source.foo = params.bar ==~ /cat/" - params: { bar: 'xxx' } - ---- -"Regex enabled is not a dynamic setting": - - - do: - catch: /setting \[script.painless.regex.enabled\], not dynamically updateable/ - cluster.put_settings: - body: - transient: - script.painless.regex.enabled: true