Combines Chenyu PR with mine

core/dbt/cli/main.py

@@ -6,11 +6,11 @@
 from dbt.adapters.factory import adapter_management
 from dbt.cli import params as p
 from dbt.cli.flags import Flags
+from dbt.config.runtime import load_project, load_profile
 from dbt.events.functions import setup_event_logger
 from dbt.profiler import profiler
-from dbt.tracking import initialize_from_flags, track_run
-from dbt.config.runtime import load_project
 from dbt.task.deps import DepsTask
+from dbt.tracking import initialize_from_flags, track_run
 
 
 def cli_runner():
@@ -46,6 +46,7 @@ def cli_runner():
 @p.printer_width
 @p.quiet
 @p.record_timing_info
+@p.single_threaded
 @p.static_parser
 @p.use_colors
 @p.use_experimental_parser
@@ -57,8 +58,9 @@ def cli(ctx, **kwargs):
  """An ELT tool for managing your SQL transformations and data models.
  For more documentation on these commands, visit: docs.getdbt.com
  """
- ctx.obj = {}
+ # Get primatives
  flags = Flags()
+
  # Logging
  # N.B. Legacy logger is not supported
  setup_event_logger(
@@ -67,6 +69,7 @@ def cli(ctx, **kwargs):
  flags.USE_COLORS,
  flags.DEBUG,
  )
+
  # Tracking
  initialize_from_flags(flags.ANONYMOUS_USAGE_STATS, flags.PROFILES_DIR)
  ctx.with_resource(track_run(run_command=ctx.invoked_subcommand))
@@ -75,20 +78,27 @@ def cli(ctx, **kwargs):
  if flags.RECORD_TIMING_INFO:
  ctx.with_resource(profiler(enable=True, outfile=flags.RECORD_TIMING_INFO))
 
- # TODO need profile to exisit
- profile = None
-
- # project need profile to render because it requires knowing Target
- ctx.obj["project"] = load_project(flags.PROJECT_DIR, flags.VERSION_CHECK, profile, flags.VARS)
  # Adapter management
  ctx.with_resource(adapter_management())
 
  # Version info
  if flags.VERSION:
  click.echo(f"`version` called\n ctx.params: {pf(ctx.params)}")
  return
- else:
- del ctx.params["version"]
+
+ # Profile
+ profile = load_profile(
+ flags.PROJECT_DIR, flags.VARS, flags.PROFILE, flags.TARGET, flags.THREADS
+ )
+
+ # Project
+ project = load_project(flags.PROJECT_DIR, flags.VERSION_CHECK, profile, flags.VARS)
+
+ # Context for downstream commands
+ ctx.obj = {}
+ ctx.obj["flags"] = flags
+ ctx.obj["profile"] = profile
+ ctx.obj["project"] = project
 
 
 # dbt build

core/dbt/cli/params.py

@@ -282,6 +282,8 @@
 # The original env var was `DBT_TEST_SINGLE_THREADED`.
 # This broke the existing naming convention.
 # This will need to be communicated as a change to the community!
+#
+# N.B. This flag is only used for testing, hence it's hidden from help text.
 single_threaded = click.option(
  "--single-threaded/--no-single-threaded",
  envvar="DBT_SINGLE_THREADED",

core/dbt/config/profile.py

@@ -399,11 +399,13 @@ def from_raw_profiles(
  )
 
  @classmethod
- def render_from_args(
+ def render(
  cls,
- args: Any,
  renderer: ProfileRenderer,
  project_profile_name: Optional[str],
+ profile_name_override: Optional[str] = None,
+ target_override: Optional[str] = None,
+ threads_override: Optional[int] = None,
  ) -> "Profile":
  """Given the raw profiles as read from disk and the name of the desired
  profile if specified, return the profile component of the runtime
@@ -419,9 +421,7 @@ def render_from_args(
  target could not be found.
  :returns Profile: The new Profile object.
  """
- threads_override = getattr(args, "threads", None)
- target_override = getattr(args, "target", None)
- profile_name_override = getattr(args, "profile", None)
+
  raw_profiles = read_profile(flags.PROFILES_DIR)
  profile_name = cls.pick_profile_name(profile_name_override, project_profile_name)
  return cls.from_raw_profiles(

core/dbt/config/runtime.py

@@ -48,6 +48,25 @@ def load_project(
  return project
 
 
+def load_profile(
+ project_root: str,
+ cli_vars: Dict[str, Any],
+ profile_name_override: Optional[str] = None,
+ target_override: Optional[str] = None,
+ threads_override: Optional[int] = None,
+) -> Profile:
+ raw_project = load_raw_project(project_root)
+ raw_profile_name = raw_project.get("profile")
+ profile_renderer = ProfileRenderer(cli_vars)
+ profile_name = profile_renderer.render_value(raw_profile_name)
+ profile = Profile.render(
+ profile_renderer, profile_name, profile_name_override, target_override, threads_override
+ )
+ # Save env_vars encountered in rendering for partial parsing
+ profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
+ return profile
+
+
 def _project_quoting_dict(proj: Project, profile: Profile) -> Dict[ComponentName, bool]:
  src: Dict[str, Any] = profile.credentials.translate_aliases(proj.quoting)
  result: Dict[ComponentName, bool] = {}
@@ -69,6 +88,21 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
  def __post_init__(self):
  self.validate()
 
+ @classmethod
+ def get_profile(
+ cls,
+ project_root: str,
+ cli_vars: Dict[str, Any],
+ args: Any,
+ ) -> Profile:
+ return load_profile(
+ project_root,
+ cli_vars,
+ args.profile,
+ args.target,
+ args.threads,
+ )
+
  # Called by 'new_project' and 'from_args'
  @classmethod
  def from_parts(
@@ -196,42 +230,17 @@ def validate(self):
  except ValidationError as e:
  raise DbtProjectError(validator_error_message(e)) from e
 
- @classmethod
- def _get_rendered_profile(
- cls,
- args: Any,
- profile_renderer: ProfileRenderer,
- profile_name: Optional[str],
- ) -> Profile:
-
- return Profile.render_from_args(args, profile_renderer, profile_name)
-
- @classmethod
- def get_profile(
- cls: Type["RuntimeConfig"], args: Any, cli_vars: Dict[str, Any], raw_profile_name: str
- ) -> Profile:
- # build the profile using the base renderer and the one fact we know
- # Note: only the named profile section is rendered. The rest of the
- # profile is ignored.
- profile_renderer = ProfileRenderer(cli_vars)
- profile_name = profile_renderer.render_value(raw_profile_name)
- profile = cls._get_rendered_profile(args, profile_renderer, profile_name)
- # Save env_vars encountered in rendering for partial parsing
- profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
- return profile
-
  @classmethod
  def collect_parts(cls: Type["RuntimeConfig"], args: Any) -> Tuple[Project, Profile]:
  # profile_name from the project
  project_root = args.project_dir if args.project_dir else os.getcwd()
- raw_project = load_raw_project(project_root)
- raw_profile_name: str = raw_project.get("profile") # type: ignore
  cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
-
- profile = cls.get_profile(args, cli_vars, raw_profile_name)
-
+ profile = cls.get_profile(
+ project_root,
+ cli_vars,
+ args,
+ )
  project = load_project(project_root, bool(flags.VERSION_CHECK), profile, cli_vars)
-
  return (project, profile)
 
  # Called in main.py, lib.py, task/base.py
@@ -582,18 +591,25 @@ def from_parts(
  )
 
  @classmethod
- def _get_rendered_profile(
+ def get_profile(
  cls,
+ project_root: str,
+ cli_vars: Dict[str, Any],
  args: Any,
- profile_renderer: ProfileRenderer,
- profile_name: Optional[str],
  ) -> Profile:
+ """
+ Moving all logic regarding constructing a complete UnsetProfile to this function
+ This way we can have a clean load_profile function to call by the new CLI and remove
+ all logic for UnsetProfile once we migrate to new click CLI
+ """
 
  profile = UnsetProfile()
  # The profile (for warehouse connection) is not needed, but we want
  # to get the UserConfig, which is also in profiles.yml
- user_config = read_user_config(flags.PROFILES_DIR)
+ user_config = read_user_config(project_root)
  profile.user_config = user_config
+ profile_renderer = ProfileRenderer(cli_vars)
+ profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
  return profile
 
  @classmethod

core/dbt/config/utils.py

@@ -64,7 +64,13 @@ def get_project_config(
  flags.set_from_args(args, user_config)
  if cli_vars is None:
  cli_vars = {}
- profile = Profile.render_from_args(args, ProfileRenderer(cli_vars), profile_name)
+ profile = Profile.render(
+ ProfileRenderer(cli_vars),
+ profile_name,
+ args.THREADS,
+ args.TARGET,
+ args.PROFILE,
+ )
  # Generate a project
  project = Project.from_project_root(
  project_path,

core/dbt/task/debug.py

@@ -249,7 +249,13 @@ def _load_profile(self):
  renderer = ProfileRenderer(self.cli_vars)
  for profile_name in profile_names:
  try:
- profile: Profile = Profile.render_from_args(self.args, renderer, profile_name)
+ profile: Profile = Profile.render(
+ renderer,
+ profile_name,
+ self.args.threads,
+ self.args.target,
+ self.args.profile,
+ )
  except dbt.exceptions.DbtConfigError as exc:
  profile_errors.append(str(exc))
  else:

test/unit/test_config.py

@@ -404,12 +404,14 @@ def from_raw_profile_info(self, raw_profile=None, profile_name='default', **kwar
 
  def from_args(self, project_profile_name='default', **kwargs):
  kw = {
- 'args': self.args,
  'project_profile_name': project_profile_name,
- 'renderer': empty_profile_renderer()
+ 'renderer': empty_profile_renderer(),
+ 'threads_override': self.args.threads,
+ 'target_override': self.args.target,
+ 'profile_name_override': self.args.profile,
  }
  kw.update(kwargs)
- return dbt.config.Profile.render_from_args(**kw)
+ return dbt.config.Profile.render(**kw)
 
  def test_profile_simple(self):
  profile = self.from_args()