Automate the addition of RUN and simplify RUN lines (#2292)

This was an offshoot of the discussion about how much boilerplate we could remove. lit requires RUN lines be there, everything else is optional.

bazel/testing/lit.cfg.py

@@ -6,54 +6,71 @@
 
 import lit.formats
 import os
+from pathlib import Path
 
 
 # This is a provided variable, ignore the undefined name warning.
 config = config # noqa: F821
 
 
 def fullpath(relative_path):
- return os.path.join(os.environ["TEST_SRCDIR"], relative_path)
+ return Path(os.environ["TEST_SRCDIR"]).joinpath(relative_path)
+
+
+def add_substitution(before, after):
+ """Adds a substitution to the config. Wraps before as `%{before}`."""
+ config.substitutions.append((f"%{{{before}}}", after))
+
+
+def add_substitutions():
+ """Adds required substitutions to the config."""
+ tools = {
+ "carbon": fullpath("carbon/toolchain/driver/carbon"),
+ "explorer": fullpath("carbon/explorer/explorer"),
+ "explorer_prelude": fullpath("carbon/explorer/data/prelude.carbon"),
+ "filecheck": fullpath("llvm-project/llvm/FileCheck"),
+ "not": fullpath("llvm-project/llvm/not"),
+ "merge_output": fullpath("carbon/bazel/testing/merge_output"),
+ }
+
+ run_carbon = f"{tools['merge_output']} {tools['carbon']}"
+ run_explorer = (
+ f"{tools['merge_output']} {tools['explorer']} %s "
+ f"--prelude={tools['explorer_prelude']}"
+ )
+ filecheck_allow_unmatched = (
+ f"{tools['filecheck']} %s --match-full-lines --strict-whitespace"
+ )
+ filecheck_strict = (
+ f"{filecheck_allow_unmatched} --implicit-check-not={{{{.}}}}"
+ )
+
+ add_substitution("carbon", f"{tools['merge_output']} {tools['carbon']}")
+ add_substitution(
+ "carbon-run-parser",
+ f"{run_carbon} dump parse-tree %s | {filecheck_strict}",
+ )
+ add_substitution(
+ "carbon-run-semantics",
+ f"{run_carbon} dump semantics-ir %s | {filecheck_strict}",
+ )
+ add_substitution(
+ "carbon-run-tokens", f"{run_carbon} dump tokens %s | {filecheck_strict}"
+ )
+ add_substitution(
+ "explorer-run",
+ f"{run_explorer} | {filecheck_strict}",
+ )
+ add_substitution(
+ "explorer-run-trace",
+ f"{run_explorer} --parser_debug --trace_file=- | "
+ f"{filecheck_allow_unmatched}",
+ )
+ add_substitution("FileCheck-strict", filecheck_strict)
+ add_substitution("not", tools["not"])
 
 
 config.name = "lit"
 config.suffixes = [".carbon"]
 config.test_format = lit.formats.ShTest()
-
-_MERGE_OUTPUT = fullpath("carbon/bazel/testing/merge_output")
-
-config.substitutions.append(
- (
- "%{carbon}",
- "%s %s" % (_MERGE_OUTPUT, fullpath("carbon/toolchain/driver/carbon")),
- )
-)
-_EXPLORER = "%s %s --prelude=%s" % (
- _MERGE_OUTPUT,
- fullpath("carbon/explorer/explorer"),
- fullpath("carbon/explorer/data/prelude.carbon"),
-)
-config.substitutions.append(("%{explorer}", _EXPLORER))
-config.substitutions.append(
- ("%{explorer-trace}", _EXPLORER + " --parser_debug --trace_file=-")
-)
-
-config.substitutions.append(("%{not}", fullpath("llvm-project/llvm/not")))
-
-_FILE_CHECK = "%s --dump-input-filter=all" % fullpath(
- "llvm-project/llvm/FileCheck"
-)
-config.substitutions.append(("%{FileCheck}", _FILE_CHECK))
-config.substitutions.append(
- (
- "%{FileCheck-allow-unmatched}",
- _FILE_CHECK + " --match-full-lines --strict-whitespace",
- )
-)
-config.substitutions.append(
- (
- "%{FileCheck-strict}",
- _FILE_CHECK
- + " --implicit-check-not={{.}} --match-full-lines --strict-whitespace",
- )
-)
+add_substitutions()

bazel/testing/lit_autoupdate_base.py

@@ -11,53 +11,68 @@
 from abc import ABC, abstractmethod
 import argparse
 from concurrent import futures
-import logging
 import os
 from pathlib import Path
 import re
 import subprocess
 from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set, Tuple
 
 # A prefix followed by a command to run for autoupdating checked output.
-AUTOUPDATE_MARKER = "// AUTOUPDATE: "
+AUTOUPDATE_MARKER = "// AUTOUPDATE"
 
 # Indicates no autoupdate is requested.
 NOAUTOUPDATE_MARKER = "// NOAUTOUPDATE"
 
 # Standard replacements normally done in lit.cfg.py.
 MERGE_OUTPUT = "./bazel-bin/bazel/testing/merge_output"
-LIT_REPLACEMENTS = [
- ("%{carbon}", f"{MERGE_OUTPUT} ./bazel-bin/toolchain/driver/carbon"),
- ("%{explorer}", f"{MERGE_OUTPUT} ./bazel-bin/explorer/explorer"),
-]
+
+
+class Tool(NamedTuple):
+ build_target: str
+ autoupdate_cmd: List[str]
+
+
+tools = {
+ "carbon": Tool(
+ "//toolchain/driver:carbon",
+ [MERGE_OUTPUT, "./bazel-bin/toolchain/driver/carbon"],
+ ),
+ "explorer": Tool(
+ "//explorer",
+ [MERGE_OUTPUT, "./bazel-bin/explorer/explorer"],
+ ),
+}
 
 
 class ParsedArgs(NamedTuple):
+ autoupdate_args: List[str]
  build_mode: str
- build_target: str
  extra_check_replacements: List[Tuple[Pattern, Pattern, str]]
  line_number_format: str
  line_number_pattern: Pattern
+ lit_run: List[str]
  testdata: str
  tests: List[Path]
+ tool: str
 
 
 def parse_args() -> ParsedArgs:
  """Parses command-line arguments and flags."""
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument("tests", nargs="*")
+ parser.add_argument(
+ "--autoupdate_arg",
+ metavar="COMMAND",
+ default=[],
+ action="append",
+ help="Optional arguments to pass to the autoupdate command.",
+ )
  parser.add_argument(
  "--build_mode",
  metavar="MODE",
  default="opt",
  help="The build mode to use. Defaults to opt for faster execution.",
  )
- parser.add_argument(
- "--build_target",
- metavar="TARGET",
- required=True,
- help="The target to build.",
- )
  parser.add_argument(
  "--extra_check_replacement",
  nargs=3,
@@ -80,26 +95,42 @@ def parse_args() -> ParsedArgs:
  help="A regular expression which matches line numbers to update as its "
  "only group.",
  )
+ parser.add_argument(
+ "--lit_run",
+ metavar="COMMAND",
+ required=True,
+ action="append",
+ help="RUN lines to set.",
+ )
  parser.add_argument(
  "--testdata",
  metavar="PATH",
  required=True,
  help="The path to the testdata to update, relative to the workspace "
  "root.",
  )
+ parser.add_argument(
+ "--tool",
+ metavar="TOOL",
+ required=True,
+ choices=tools.keys(),
+ help="The tool being tested.",
+ )
  parsed_args = parser.parse_args()
  extra_check_replacements = [
  (re.compile(line_matcher), re.compile(before), after)
  for line_matcher, before, after in parsed_args.extra_check_replacement
  ]
  return ParsedArgs(
+ autoupdate_args=parsed_args.autoupdate_arg,
  build_mode=parsed_args.build_mode,
- build_target=parsed_args.build_target,
  extra_check_replacements=extra_check_replacements,
  line_number_format=parsed_args.line_number_format,
  line_number_pattern=re.compile(parsed_args.line_number_pattern),
+ lit_run=parsed_args.lit_run,
  testdata=parsed_args.testdata,
  tests=[Path(test).resolve() for test in parsed_args.tests],
+ tool=parsed_args.tool,
  )
 
 
@@ -139,6 +170,16 @@ def format(self, **kwargs: Any) -> str:
  return self.text
 
 
+class RunLine(Line):
+ """A RUN line."""
+
+ def __init__(self, text: str) -> None:
+ self.text = text
+
+ def format(self, **kwargs: Any) -> str:
+ return self.text
+
+
 class CheckLine(Line):
  """A `// CHECK:` line generated from the test output.
 
@@ -179,23 +220,17 @@ def format(
  return f"{self.indent}// CHECK:{result}\n"
 
 
-class Autoupdate(NamedTuple):
- line_number: int
- cmd: str
-
-
-def find_autoupdate(test: str, orig_lines: List[str]) -> Optional[Autoupdate]:
+def find_autoupdate(test: str, orig_lines: List[str]) -> Optional[int]:
  """Figures out whether autoupdate should occur.
 
- For AUTOUPDATE, returns the line and command. For NOAUTOUPDATE, returns
- None.
+ For AUTOUPDATE, returns the line. For NOAUTOUPDATE, returns None.
  """
  found = 0
  result = None
  for line_number, line in enumerate(orig_lines):
  if line.startswith(AUTOUPDATE_MARKER):
  found += 1
- result = Autoupdate(line_number, line[len(AUTOUPDATE_MARKER) :])
+ result = line_number
  elif line.startswith(NOAUTOUPDATE_MARKER):
  found += 1
  if found == 0:
@@ -219,23 +254,16 @@ def replace_all(s: str, replacements: List[Tuple[str, str]]) -> str:
 
 
 def get_matchable_test_output(
- parsed_args: ParsedArgs,
- test: str,
- autoupdate_cmd: str,
+ autoupdate_args: List[str],
  extra_check_replacements: List[Tuple[Pattern, Pattern, str]],
+ tool: str,
+ test: str,
 ) -> List[str]:
  """Runs the autoupdate command and returns the output lines."""
- # Mirror lit.cfg.py substitutions; bazel runs don't need --prelude.
- # Also replaces `%s` with the test file.
- autoupdate_cmd = replace_all(
- autoupdate_cmd, [("%s", test)] + LIT_REPLACEMENTS
- )
-
  # Run the autoupdate command to generate output.
  # (`bazel run` would serialize)
  p = subprocess.run(
- autoupdate_cmd,
- shell=True,
+ tools[tool].autoupdate_cmd + autoupdate_args + [test],
  stdout=subprocess.PIPE,
  stderr=subprocess.STDOUT,
  )
@@ -263,9 +291,17 @@ def get_matchable_test_output(
  return out_lines
 
 
+def is_replaced(line: str) -> bool:
+ """Returns true if autoupdate should replace the line."""
+ line = line.lstrip()
+ return line.startswith("// CHECK") or line.startswith("// RUN:")
+
+
 def merge_lines(
  line_number_format: str,
  line_number_pattern: Pattern,
+ lit_run: List[str],
+ test: str,
  autoupdate_line_number: int,
  raw_orig_lines: List[str],
  out_lines: List[str],
@@ -274,8 +310,7 @@ def merge_lines(
  orig_lines = [
  OriginalLine(i, line)
  for i, line in enumerate(raw_orig_lines)
- # Remove CHECK lines in the original output.
- if not line.lstrip().startswith("// CHECK")
+ if not is_replaced(line)
  ]
  check_lines = [
  CheckLine(out_line, line_number_format, line_number_pattern)
@@ -286,6 +321,11 @@ def merge_lines(
  # CHECK lines must go after AUTOUPDATE.
  while orig_lines and orig_lines[0].line_number <= autoupdate_line_number:
  result_lines.append(orig_lines.pop(0))
+ for line in lit_run:
+ run_not = ""
+ if Path(test).name.startswith("fail_"):
+ run_not = "%{not} "
+ result_lines.append(RunLine(f"// RUN: {run_not}{line}\n"))
  # Interleave the original lines and the CHECK: lines.
  while orig_lines and check_lines:
  # Original lines go first when the CHECK line is known and later.
@@ -315,21 +355,23 @@ def update_check(parsed_args: ParsedArgs, test: Path) -> bool:
  orig_lines = f.readlines()
 
  # Make sure we're supposed to autoupdate.
- autoupdate = find_autoupdate(str(test), orig_lines)
- if autoupdate is None:
+ autoupdate_line = find_autoupdate(str(test), orig_lines)
+ if autoupdate_line is None:
  return False
 
  # Determine the merged output lines.
  out_lines = get_matchable_test_output(
- parsed_args,
- str(test),
- autoupdate.cmd,
+ parsed_args.autoupdate_args,
  parsed_args.extra_check_replacements,
+ parsed_args.tool,
+ str(test),
  )
  result_lines = merge_lines(
  parsed_args.line_number_format,
  parsed_args.line_number_pattern,
- autoupdate.line_number,
+ parsed_args.lit_run,
+ str(test),
+ autoupdate_line,
  orig_lines,
  out_lines,
  )
@@ -369,8 +411,8 @@ def update_checks(parsed_args: ParsedArgs, tests: Set[Path]) -> None:
  def map_helper(test: Path) -> bool:
  try:
  updated = update_check(parsed_args, test)
- except Exception:
- logging.exception(f"Failed to update {test}")
+ except Exception as e:
+ raise ValueError(f"Failed to update {test}") from e
  print(".", end="", flush=True)
  return updated
 
@@ -406,7 +448,7 @@ def main() -> None:
  "-c",
  parsed_args.build_mode,
  "//bazel/testing:merge_output",
- parsed_args.build_target,
+ tools[parsed_args.tool].build_target,
  ]
  )