Fix build in postbuild substitution

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
  exclude: '^tests/.*/__snapshots__/.*.ambr$'
  - id: end-of-file-fixer
  - id: check-yaml
- exclude: '^tests/testdata/cluster8/apps/.*\.yaml$'
+ exclude: '^tests/testdata/cluster8/apps/.*\.yaml|tests/testdata/cluster/apps/prod/.*\.yaml$'
  - id: check-added-large-files
  - repo: https:/psf/black
  rev: 24.4.2

flux_local/git_repo.py

@@ -403,27 +403,54 @@ def remove(self, kustomization: Kustomization) -> None:
  del self._cache[key]
 
 
+@dataclass
+class VisitResult:
+ """Result of visiting a kustomization."""
+
+ kustomizations: list[Kustomization]
+ config_maps: list[ConfigMap]
+ secrets: list[Secret]
+
+ def __post_init__(self) -> None:
+ """Validate the object"""
+ unique = {ks.namespaced_name for ks in self.kustomizations}
+ if len(unique) != len(self.kustomizations):
+ ks_names = [ks.namespaced_name for ks in self.kustomizations]
+ dupes = list(filter(lambda x: ks_names.count(x) > 1, ks_names))
+ raise FluxException(
+ f"Detected multiple Fluxtomizations with the same name: {dupes}. "
+ "This indicates either (1) an incorrect Kustomization which needs to be fixed "
+ "or (2) a multi-cluster setup which requires flux-local to run with a more strict --path."
+ )
+
+
 async def visit_kustomization(
  selector: PathSelector,
  builder: CachableBuilder,
  path: Path,
  visit_ks: Kustomization | None,
-) -> list[Kustomization]:
+) -> VisitResult:
  """Visit a path and return a list of Kustomizations."""
 
  _LOGGER.debug("Visiting path (%s) %s", selector.path, path)
  label = visit_ks.namespaced_name if visit_ks else str(path)
+
+ kinds = [CLUSTER_KUSTOMIZE_KIND, CONFIG_MAP_KIND, SECRET_KIND]
+
  with trace_context(f"Kustomization '{label}'"):
  cmd: kustomize.Kustomize
  if visit_ks is None:
- cmd = kustomize.grep(f"kind={CLUSTER_KUSTOMIZE_KIND}", selector.root / path)
+ cmd = kustomize.filter_resources(kinds, selector.root / path)
  else:
  cmd = await builder.build(visit_ks, selector.root / path)
- cmd = cmd.grep(f"kind={CLUSTER_KUSTOMIZE_KIND}")
- cmd = cmd.grep(GREP_SOURCE_REF_KIND)
+ cmd = cmd.filter_resources(kinds)
+ cmd = await cmd.stash()
+ ks_cmd = cmd.grep(GREP_SOURCE_REF_KIND)
+ cfg_cmd = cmd.filter_resources([CONFIG_MAP_KIND, SECRET_KIND])
 
  try:
- docs = await cmd.objects()
+ ks_docs = await ks_cmd.objects()
+ cfg_docs = await cfg_cmd.objects()
  except KustomizePathException as err:
  raise FluxException(err) from err
  except FluxException as err:
@@ -436,25 +463,26 @@ async def visit_kustomization(
  f"Error building Fluxtomization '{visit_ks.namespaced_name}' "
  f"path '{path}': {ERROR_DETAIL_BAD_KS} {err}"
  ) from err
- kustomizations = list(
- filter(
- is_allowed_source(selector.sources or []),
- [
- Kustomization.parse_doc(doc)
- for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, docs)
- ],
- )
+
+ return VisitResult(
+ kustomizations=list(
+ filter(
+ is_allowed_source(selector.sources or []),
+ [
+ Kustomization.parse_doc(doc)
+ for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, ks_docs)
+ ],
+ )
+ ),
+ config_maps=[
+ ConfigMap.parse_doc(doc)
+ for doc in cfg_docs
+ if doc.get("kind") == CONFIG_MAP_KIND
+ ],
+ secrets=[
+ Secret.parse_doc(doc) for doc in cfg_docs if doc.get("kind") == SECRET_KIND
+ ],
  )
- unique = {ks.namespaced_name for ks in kustomizations}
- if len(unique) != len(kustomizations):
- ks_names = [ks.namespaced_name for ks in kustomizations]
- dupes = list(filter(lambda x: ks_names.count(x) > 1, ks_names))
- raise FluxException(
- f"Detected multiple Fluxtomizations with the same name: {dupes}. "
- "This indicates either (1) an incorrect Kustomization which needs to be fixed "
- "or (2) a multi-cluster setup which requires flux-local to run with a more strict --path."
- )
- return kustomizations
 
 
 async def kustomization_traversal(
@@ -468,6 +496,7 @@ async def kustomization_traversal(
 
  path_queue: deque[tuple[Path, Kustomization | None]] = deque()
  path_queue.append((selector.relative_path, None))
+ cluster_config = values.cluster_config([], [])
  while path_queue:
  # Fully empty the queue, running all tasks in parallel
  tasks = []
@@ -484,7 +513,10 @@ async def kustomization_traversal(
  # Find new kustomizations
  kustomizations = []
  for result in await asyncio.gather(*tasks):
- for ks in result:
+ cluster_config = values.merge_cluster_config(
+ cluster_config, result.secrets, result.config_maps
+ )
+ for ks in result.kustomizations:
  if ks.namespaced_name in visited_ks:
  continue
  kustomizations.append(ks)
@@ -502,6 +534,12 @@ async def kustomization_traversal(
  if not (ks_path := adjust_ks_path(ks, selector)):
  continue
  ks.path = str(ks_path)
+ if ks.postbuild_substitute_from:
+ values.expand_postbuild_substitute_reference(
+ ks,
+ cluster_config,
+ )
+
  path_queue.append((ks_path, ks))
  response_kustomizations.append(ks)
 
@@ -581,8 +619,7 @@ async def build_kustomization(
  if not kinds:
  return
 
- regexp = f"kind=^({'|'.join(kinds)})$"
- docs = await cmd.grep(regexp).objects(
+ docs = await cmd.filter_resources(kinds).objects(
  target_namespace=kustomization.target_namespace
  )
 

flux_local/kustomize.py

@@ -141,6 +141,13 @@ def skip_resources(self, kinds: list[str]) -> "Kustomize":
  skip_re = "|".join(kinds)
  return self.grep(f"kind=^({skip_re})$", invert=True)
 
+ def filter_resources(self, kinds: list[str]) -> "Kustomize":
+ """Skip resources kinds of the specified types."""
+ if not kinds:
+ return self
+ skip_re = "|".join(kinds)
+ return self.grep(f"kind=^({skip_re})$", invert=False)
+
  async def validate_policies(self, policies: list[manifest.ClusterPolicy]) -> None:
  """Apply kyverno policies to objects built so far."""
  if not policies:
@@ -283,6 +290,12 @@ def grep(expr: str, path: Path, invert: bool = False) -> Kustomize:
  return Kustomize([Command(args, cwd=cwd, exc=KustomizeException)])
 
 
+def filter_resources(kinds: list[str], path: Path) -> Kustomize:
+ """Filter resources in the specified path based of a specific kind."""
+ regexp = f"kind=^({'|'.join(kinds)})$"
+ return grep(regexp, path)
+
+
 def update_namespace(doc: dict[str, Any], namespace: str) -> dict[str, Any]:
  """Update the namespace of the specified document.
 

flux_local/values.py

@@ -59,6 +59,16 @@ def cluster_config(
  )
 
 
+def merge_cluster_config(
+ config: ClusterConfig, secrets: list[Secret], config_maps: list[ConfigMap]
+) -> ClusterConfig:
+ """Create a ClusterConfig from a list of secrets and configmaps."""
+ return ClusterConfig(
+ lambda: list(config.secrets) + secrets,
+ lambda: list(config.config_maps) + config_maps,
+ )
+
+
 def ks_cluster_config(kustomizations: list[Kustomization]) -> ClusterConfig:
  """Create a ClusterConfig from a list of Kustomizations."""
 
@@ -262,7 +272,9 @@ def expand_postbuild_substitute_reference(
  continue
 
  if found_data is None:
- if not ref.optional and not ref.kind == SECRET_KIND: # Secrets are commonly filtered
+ if (
+ not ref.optional and not ref.kind == SECRET_KIND
+ ): # Secrets are commonly filtered
  _LOGGER.warning(
  "Unable to find SubstituteReference for %s: %s",
  ks.namespaced_name,
@@ -271,5 +283,6 @@ def expand_postbuild_substitute_reference(
  continue
 
  values.update(found_data)
+ _LOGGER.debug("update_postbuild_substitutions=%s", values)
  ks.update_postbuild_substitutions(values)
  return ks