Implement SPICE-0009 External Readers

[SPICE-0009](apple/pkl-evolution#10)

bench/src/jmh/java/org/pkl/core/ListSort.java

@@ -28,7 +28,7 @@
 import org.pkl.core.repl.ReplRequest;
 import org.pkl.core.repl.ReplResponse;
 import org.pkl.core.repl.ReplServer;
-import org.pkl.core.resource.ResourceReaders;
+import org.pkl.core.resource.ResourceReaderFactories;
 import org.pkl.core.util.IoUtils;
 
 @Warmup(iterations = 5, time = 2)
@@ -43,7 +43,7 @@ public class ListSort {
  HttpClient.dummyClient(),
  Loggers.stdErr(),
  List.of(ModuleKeyFactories.standardLibrary),
- List.of(ResourceReaders.file()),
+ List.of(ResourceReaderFactories.file()),
  Map.of(),
  Map.of(),
  null,

docs/src/test/kotlin/DocSnippetTests.kt

@@ -22,10 +22,10 @@ import org.pkl.core.parser.antlr.PklParser
 import org.pkl.core.repl.ReplRequest
 import org.pkl.core.repl.ReplResponse
 import org.pkl.core.repl.ReplServer
-import org.pkl.core.resource.ResourceReaders
 import org.pkl.core.util.IoUtils
 import org.antlr.v4.runtime.ParserRuleContext
 import org.pkl.core.http.HttpClient
+import org.pkl.core.resource.ResourceReaders
 import java.nio.file.Files
 import kotlin.io.path.isDirectory
 import kotlin.io.path.isRegularFile

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -20,6 +20,7 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.module.ProjectDependenciesManager
 import org.pkl.core.util.IoUtils
 
@@ -134,6 +135,12 @@ data class CliBaseOptions(
 
  /** Hostnames, IP addresses, or CIDR blocks to not proxy. */
  val httpNoProxy: List<String>? = null,
+
+ /** External module reader process specs */
+ val externalModuleReaders: Map<String, ExternalReader> = mapOf(),
+
+ /** External resource reader process specs */
+ val externalResourceReaders: Map<String, ExternalReader> = mapOf(),
 ) {
 
  companion object {

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -21,6 +21,7 @@ import java.util.regex.Pattern
 import kotlin.io.path.isRegularFile
 import org.pkl.core.*
 import org.pkl.core.evaluatorSettings.PklEvaluatorSettings
+import org.pkl.core.externalProcess.ExternalProcessImpl
 import org.pkl.core.http.HttpClient
 import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.module.ModuleKeyFactory
@@ -108,12 +109,16 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
  protected val allowedModules: List<Pattern> by lazy {
  cliOptions.allowedModules
- ?: evaluatorSettings?.allowedModules ?: SecurityManagers.defaultAllowedModules
+ ?: evaluatorSettings?.allowedModules
+ ?: (SecurityManagers.defaultAllowedModules +
+ externalModuleReaders.keys.map { Pattern.compile("$it:") }.toList())
  }
 
  protected val allowedResources: List<Pattern> by lazy {
  cliOptions.allowedResources
- ?: evaluatorSettings?.allowedResources ?: SecurityManagers.defaultAllowedResources
+ ?: evaluatorSettings?.allowedResources
+ ?: (SecurityManagers.defaultAllowedResources +
+ externalResourceReaders.keys.map { Pattern.compile("$it:") }.toList())
  }
 
  protected val rootDir: Path? by lazy {
@@ -169,6 +174,26 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
  ?: project?.evaluatorSettings?.http?.proxy?.noProxy ?: settings.http?.proxy?.noProxy
  }
 
+ private val externalModuleReaders by lazy {
+ (project?.evaluatorSettings?.externalModuleReaders
+ ?: emptyMap()) + cliOptions.externalModuleReaders
+ }
+
+ private val externalResourceReaders by lazy {
+ (project?.evaluatorSettings?.externalResourceReaders
+ ?: emptyMap()) + cliOptions.externalResourceReaders
+ }
+
+ private val externalProcesses by lazy {
+ // share ExternalProcessImpl instances between configured external resource/module readers with
+ // the same spec
+ // this avoids spawning multiple subprocesses if the same reader implements both reader types
+ // and/or multiple schemes
+ (externalModuleReaders + externalResourceReaders).values.associateWith {
+ ExternalProcessImpl(it)
+ }
+ }
+
  private fun HttpClient.Builder.addDefaultCliCertificates() {
  val caCertsDir = IoUtils.getPklHomeDir().resolve("cacerts")
  var certsAdded = false
@@ -213,6 +238,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
  protected fun moduleKeyFactories(modulePathResolver: ModulePathResolver): List<ModuleKeyFactory> {
  return buildList {
+ externalModuleReaders.forEach { (key, value) ->
+ add(ModuleKeyFactories.external(key, externalProcesses[value]!!))
+ }
  add(ModuleKeyFactories.standardLibrary)
  add(ModuleKeyFactories.modulePath(modulePathResolver))
  add(ModuleKeyFactories.pkg)
@@ -226,6 +254,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
 
  private fun resourceReaders(modulePathResolver: ModulePathResolver): List<ResourceReader> {
  return buildList {
+ externalResourceReaders.forEach { (key, value) ->
+ add(ResourceReaders.external(key, externalProcesses[value]!!))
+ }
  add(ResourceReaders.environmentVariable())
  add(ResourceReaders.externalProperty())
  add(ResourceReaders.modulePath(modulePathResolver))

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -28,6 +28,8 @@ import java.time.Duration
 import java.util.regex.Pattern
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
+import org.pkl.commons.shlex
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.runtime.VmUtils
 import org.pkl.core.util.IoUtils
 
@@ -207,6 +209,34 @@ class BaseOptions : OptionGroup() {
  .single()
  .split(",")
 
+ val externalModuleReaders: Map<String, ExternalReader> by
+ option(
+ names = arrayOf("--external-module"),
+ metavar = "<scheme>='<executable>[ <arguments>]'",
+ help = "External reader registrations for module URI schemes"
+ )
+ .splitPair("=")
+ .convert { // in newer clikt versions this can be done in one call using associateWith
+ val cmd = shlex(it.second)
+ Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1)))
+ }
+ .multiple()
+ .toMap()
+
+ val externalResourceReaders: Map<String, ExternalReader> by
+ option(
+ names = arrayOf("--external-resource"),
+ metavar = "<scheme>='<executable>[ <arguments>]'",
+ help = "External reader registrations for resource URI schemes"
+ )
+ .splitPair("=")
+ .convert { // in newer clikt versions this can be done in one call using associateWith
+ val cmd = shlex(it.second)
+ Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1)))
+ }
+ .multiple()
+ .toMap()
+
  // hidden option used by native tests
  private val testPort: Int by
  option(names = arrayOf("--test-port"), help = "Internal test option", hidden = true)
@@ -239,7 +269,9 @@ class BaseOptions : OptionGroup() {
  noProject = projectOptions?.noProject ?: false,
  caCertificates = caCertificates,
  httpProxy = proxy,
- httpNoProxy = noProxy ?: emptyList()
+ httpNoProxy = noProxy ?: emptyList(),
+ externalModuleReaders = externalModuleReaders,
+ externalResourceReaders = externalResourceReaders,
  )
  }
 }

pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt

@@ -18,7 +18,9 @@ package org.pkl.commons
 import java.io.File
 import java.net.URI
 import java.nio.file.Path
+import java.util.*
 import java.util.regex.Pattern
+import kotlin.collections.ArrayList
 
 fun String.toPath(): Path = Path.of(this)
 
@@ -36,3 +38,53 @@ fun String.toUri(): URI {
  }
  return URI(null, null, this, null)
 }
+
+/** Lex a string into tokens similar to how a shell would */
+fun shlex(input: String): List<String> {
+ val result = ArrayList<String>()
+ var inEscape = false
+ var quote: Char? = null
+ var lastCloseQuoteIndex = Int.MIN_VALUE
+ var current = StringBuilder()
+
+ for ((idx, char) in input.toCharArray().withIndex()) {
+ when {
+ // if in an escape always append the next character
+ inEscape -> {
+ inEscape = false
+ current.append(char)
+ }
+ // enter an escape on \ if not in a quote or in a non-single quote
+ char == '\\' && quote != '\'' -> inEscape = true
+ // if in a quote and encounter the delimiter, tentatively exit the quote
+ // this handles cases with adjoining quotes e.g. `abc'123''xyz'`
+ quote == char -> {
+ quote = null
+ lastCloseQuoteIndex = idx
+ }
+ // if not in a quote and encounter a quote charater, enter a quote
+ quote == null && (char == '\'' || char == '"') -> {
+ quote = char
+ }
+ // if not in a quote and whitespace is encountered
+ quote == null && char.isWhitespace() -> {
+ // if the current token isn't empty or if a quote has just ended, finalize the current token
+ // otherwise do nothing, which handles multiple whitespace cases e.g. `abc 123`
+ if (current.isNotEmpty() || lastCloseQuoteIndex == (idx - 1)) {
+ result.add(current.toString())
+ current = StringBuilder()
+ }
+ }
+ // in other cases, append to the current token
+ else -> current.append(char)
+ }
+ }
+ // clean up last token
+ // if the current token isn't empty or if a quote has just ended, finalize the token
+ // if this condition is false, the input likely ended in whitespace
+ if (current.isNotEmpty() || lastCloseQuoteIndex == (input.length - 1)) {
+ result.add(current.toString())
+ }
+
+ return result
+}

pkl-commons/src/test/kotlin/org/pkl/commons/ShlexTest.kt

@@ -0,0 +1,78 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons
+
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+
+class ShlexTest {
+
+ @Test
+ fun `empty input produces empty output`() {
+ assertThat(shlex("")).isEqualTo(emptyList<String>())
+ }
+
+ @Test
+ fun `whitespace input produces empty output`() {
+ assertThat(shlex(" \n \t ")).isEqualTo(emptyList<String>())
+ }
+
+ @Test
+ fun `regular token parsing`() {
+ assertThat(shlex("\nabc def\tghi ")).isEqualTo(listOf("abc", "def", "ghi"))
+ }
+
+ @Test
+ fun `single quoted token parsing`() {
+ assertThat(shlex("'this is a single token'")).isEqualTo(listOf("this is a single token"))
+ }
+
+ @Test
+ fun `double quoted token parsing`() {
+ assertThat(shlex("\"this is a single token\"")).isEqualTo(listOf("this is a single token"))
+ }
+
+ @Test
+ fun `escaping handles double quotes`() {
+ assertThat(shlex(""""\"this is a single double quoted token\"""""))
+ .isEqualTo(listOf("\"this is a single double quoted token\""))
+ }
+
+ @Test
+ fun `escaping does not apply within single quotes`() {
+ assertThat(shlex("""'this is a single \" token'"""))
+ .isEqualTo(listOf("""this is a single \" token"""))
+ }
+
+ @Test
+ fun `adjacent quoted strings are one token`() {
+ assertThat(shlex(""""single"' joined 'token""")).isEqualTo(listOf("single joined token"))
+ assertThat(shlex(""""single"' 'token""")).isEqualTo(listOf("single token"))
+ }
+
+ @Test
+ fun `space escapes do not split tokens`() {
+ assertThat(shlex("""single\ token""")).isEqualTo(listOf("single token"))
+ }
+
+ @Test
+ fun `empty quotes produce a single empty token`() {
+ assertThat(shlex("\"\"")).isEqualTo(listOf(""))
+ assertThat(shlex("''")).isEqualTo(listOf(""))
+ assertThat(shlex("'' ''")).isEqualTo(listOf("", ""))
+ assertThat(shlex("''''")).isEqualTo(listOf(""))
+ }
+}

pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java

@@ -20,6 +20,7 @@
 import java.util.*;
 import java.util.regex.Pattern;
 import org.pkl.core.SecurityManagers.StandardBuilder;
+import org.pkl.core.externalProcess.ExternalProcessImpl;
 import org.pkl.core.http.HttpClient;
 import org.pkl.core.module.ModuleKeyFactories;
 import org.pkl.core.module.ModuleKeyFactory;
@@ -474,6 +475,18 @@ public EvaluatorBuilder applyFromProject(Project project) {
  } else if (settings.moduleCacheDir() != null) {
  setModuleCacheDir(settings.moduleCacheDir());
  }
+ if (settings.externalModuleReaders() != null) {
+ for (var entry : settings.externalModuleReaders().entrySet()) {
+ var process = new ExternalProcessImpl(entry.getValue());
+ addModuleKeyFactory(ModuleKeyFactories.external(entry.getKey(), process));
+ }
+ }
+ if (settings.externalResourceReaders() != null) {
+ for (var entry : settings.externalResourceReaders().entrySet()) {
+ var process = new ExternalProcessImpl(entry.getValue());
+ addResourceReader(ResourceReaders.external(entry.getKey(), process));
+ }
+ }
  return this;
  }
 

pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java

@@ -55,6 +55,7 @@
 import org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen;
 import org.pkl.core.ast.member.*;
 import org.pkl.core.ast.type.*;
+import org.pkl.core.externalProcess.ExternalProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.module.ModuleKeys;
 import org.pkl.core.module.ResolvedModuleKey;
@@ -1837,6 +1838,12 @@ private URI resolveImport(String importUri, StringConstantContext importUriCtx)
  .withHint(e.getHint())
  .withSourceSection(createSourceSection(importUriCtx))
  .build();
+ } catch (ExternalProcessException e) {
+ throw exceptionBuilder()
+ .evalError("externalReaderFailure")
+ .withCause(e.getCause())
+ .withSourceSection(createSourceSection(importUriCtx))
+ .build();
  }
 
  if (!resolvedUri.isAbsolute()) {

pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java

@@ -23,6 +23,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import org.pkl.core.SecurityManagerException;
+import org.pkl.core.externalProcess.ExternalProcessException;
 import org.pkl.core.module.ModuleKey;
 import org.pkl.core.packages.PackageLoadError;
 import org.pkl.core.runtime.VmContext;
@@ -75,6 +76,8 @@ private URI resolveResource(ModuleKey moduleKey, String resourceUri) {
  .build();
  } catch (PackageLoadError | SecurityManagerException e) {
  throw exceptionBuilder().withCause(e).build();
+ } catch (ExternalProcessException e) {
+ throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build();
  }
 
  if (!resolvedUri.isAbsolute()) {