From c407d7c48815d1736e078247e54f7a22e576a64e Mon Sep 17 00:00:00 2001 From: Daniel Hedlund Date: Fri, 15 Sep 2023 14:03:17 -0700 Subject: [PATCH] Improve performance of algorithm to determine project deps Introduces smarter tracking of deps as they are loaded. These changes help to avoid traversing the same dep sub-trees multiple times. For larger projects, including umbrella projects, the same can be traversed millions of times. --- lib/dialyxir/project.ex | 215 ++++++++++++++++++++++++++-------------- 1 file changed, 141 insertions(+), 74 deletions(-) diff --git a/lib/dialyxir/project.ex b/lib/dialyxir/project.ex index cabfea7..84b9dcb 100644 --- a/lib/dialyxir/project.ex +++ b/lib/dialyxir/project.ex @@ -6,6 +6,9 @@ defmodule Dialyxir.Project do alias Dialyxir.Formatter.Short alias Dialyxir.Formatter.Utils + # Maximum depth in the dependency tree to traverse before giving up. + @max_dep_traversal_depth 100 + def plts_list(deps, include_project \\ true, exclude_core \\ false) do elixir_apps = [:elixir] erlang_apps = [:erts, :kernel, :stdlib, :crypto] @@ -308,11 +311,17 @@ defmodule Dialyxir.Project do defp include_deps do method = dialyzer_config()[:plt_add_deps] - reduce_umbrella_children([], fn deps -> - deps ++ + initial_acc = { + _loaded_apps = [], + _unloaded_apps = [], + _initial_load_statuses = %{} + } + + {loaded_apps, _unloaded_apps, _final_load_statuses} = + reduce_umbrella_children(initial_acc, fn acc -> case method do false -> - [] + acc # compatibility true -> @@ -320,112 +329,118 @@ defmodule Dialyxir.Project do "Dialyxir has deprecated plt_add_deps: true in favor of apps_direct, which includes only runtime dependencies." ) - deps_project() ++ deps_app(false) + acc + |> load_project_deps() + |> load_external_deps(recursive: false) :project -> warning( "Dialyxir has deprecated plt_add_deps: :project in favor of apps_direct, which includes only runtime dependencies." ) - deps_project() ++ deps_app(false) + acc + |> load_project_deps() + |> load_external_deps(recursive: false) :apps_direct -> - deps_app(false) + load_external_deps(acc, recursive: false) :transitive -> warning( "Dialyxir has deprecated plt_add_deps: :transitive in favor of app_tree, which includes only runtime dependencies." ) - deps_transitive() ++ deps_app(true) + acc + |> load_transitive_deps() + |> load_external_deps(recursive: true) _app_tree -> - deps_app(true) + load_external_deps(acc, recursive: true) end - end) + end) + + loaded_apps end - defp deps_project do - Mix.Project.config()[:deps] - |> Enum.filter(&env_dep(&1)) - |> Enum.map(&elem(&1, 0)) + defp load_project_deps({loaded_apps, unloaded_apps, load_statuses}) do + apps = + Mix.Project.config()[:deps] + |> Enum.filter(&env_dep(&1)) + |> Enum.map(&elem(&1, 0)) + + app_load_statuses = Map.new(apps, &{elem(&1, 0), :loaded}) + + update_load_statuses({loaded_apps, unloaded_apps -- apps, load_statuses}, app_load_statuses) end - defp deps_transitive do - Mix.Project.deps_paths() - |> Map.keys() + defp load_transitive_deps({loaded_apps, unloaded_apps, load_statuses}) do + apps = Mix.Project.deps_paths() |> Map.values() + app_load_statuses = Map.new(apps, &{elem(&1, 0), :loaded}) + + update_load_statuses({loaded_apps, unloaded_apps -- apps, load_statuses}, app_load_statuses) end - @spec deps_app(boolean()) :: [atom] - defp deps_app(recursive) do + defp load_external_deps({loaded_apps, _unloaded_apps, load_statuses}, opts) do + # Non-recursive traversal of 2 tries to load the app immediate deps. + traversal_depth = + case Keyword.fetch!(opts, :recursive) do + true -> @max_dep_traversal_depth + false -> 2 + end + app = Mix.Project.config()[:app] - deps_app(app, recursive) - end - if System.version() |> Version.parse!() |> then(&(&1.major >= 1 and &1.minor >= 15)) do - @spec deps_app(atom(), boolean()) :: [atom()] - defp deps_app(app, recursive) do - case do_load_app(app) do - :ok -> - with_each = - if recursive do - &deps_app(&1, true) - else - fn _ -> [] end - end + # Even if already loaded, we'll need to traverse it again to get its deps. + load_statuses_w_app = Map.put(load_statuses, app, {:unloaded, :required}) + traverse_deps_for_apps({loaded_apps -- [app], [app], load_statuses_w_app}, traversal_depth) + end - # Identify the optional applications which can't be loaded and thus not available - missing_apps = - Application.spec(app, :optional_applications) - |> List.wrap() - |> Enum.reject(&(do_load_app(&1) == :ok)) + defp traverse_deps_for_apps({loaded_apps, [] = unloaded_deps, load_statuses}, _rem_depth), + do: {loaded_apps, unloaded_deps, load_statuses} - # Remove the optional applications which are not available from all the applications - required_apps = - Application.spec(app, :applications) - |> List.wrap() - |> Enum.reject(&(&1 in missing_apps)) + defp traverse_deps_for_apps({loaded_apps, unloaded_deps, load_statuses}, 0 = _rem_depth), + do: {loaded_apps, unloaded_deps, load_statuses} - required_apps |> Stream.flat_map(&with_each.(&1)) |> Enum.concat(required_apps) + defp traverse_deps_for_apps({loaded_apps, apps_to_load, load_statuses}, rem_depth) do + initial_acc = {loaded_apps, [], load_statuses} - {:error, err} -> - error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}") + {updated_loaded_apps, updated_unloaded_apps, updated_load_statuses} = + Enum.reduce(apps_to_load, initial_acc, fn app, acc -> + required? = Map.fetch!(load_statuses, app) == {:unloaded, :required} + {app_load_status, app_dep_statuses} = load_app(app, required?) - [] - end - end - else - @spec deps_app(atom(), boolean()) :: [atom()] - defp deps_app(app, recursive) do - with_each = - if recursive do - &deps_app(&1, true) - else - fn _ -> [] end - end + acc + |> update_load_statuses(%{app => app_load_status}) + |> update_load_statuses(app_dep_statuses) + end) - case do_load_app(app) do - :ok -> - nil + traverse_deps_for_apps( + {updated_loaded_apps, updated_unloaded_apps, updated_load_statuses}, + rem_depth - 1 + ) + end - {:error, err} -> - error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}") + defp load_app(app, required?) do + case do_load_app(app) do + :ok -> + {dependencies, optional_deps} = app_dep_specs(app) - nil - end + dep_statuses = + Map.new(dependencies, fn dep -> + case dep in optional_deps do + true -> {dep, {:unloaded, :optional}} + false -> {dep, {:unloaded, :required}} + end + end) - case Application.spec(app, :applications) do - [] -> - [] + {:loaded, dep_statuses} - nil -> - [] + {:error, err} -> + if required? do + error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}") + end - this_apps -> - Enum.map(this_apps, with_each) - |> List.flatten() - |> Enum.concat(this_apps) - end + {{:error, err}, %{}} end end @@ -443,6 +458,58 @@ defmodule Dialyxir.Project do end end + if System.version() |> Version.parse!() |> then(&(&1.major >= 1 and &1.minor >= 15)) do + defp app_dep_specs(app) do + # Values returned by :optional_applications are also in :applications. + dependencies = Application.spec(app, :applications) || [] + optional_deps = Application.spec(app, :optional_applications) || [] + + {dependencies, optional_deps} + end + else + defp app_dep_specs(app) do + {Application.spec(app, :applications) || [], []} + end + end + + defp update_load_statuses({loaded_apps, unloaded_apps, load_statuses}, new_statuses) do + initial_acc = {loaded_apps, unloaded_apps, load_statuses} + + Enum.reduce(new_statuses, initial_acc, fn {app, new_status}, acc -> + {current_loaded_apps, current_unloaded_apps, statuses} = acc + existing_status = Map.get(statuses, app, :unset) + + {new_loaded_apps, new_unloaded_apps, updated_load_statuses} = + case {existing_status, new_status} do + {:unset, {:unloaded, _} = new_status} -> + # Haven't seen this app before. + {[], [app], Map.put(statuses, app, new_status)} + + {{:unloaded, :optional}, {:unloaded, :required}} -> + # A previous app had this as optional, but another one requires it. + {[], [], Map.put(statuses, app, {:unloaded, :required})} + + {{:unloaded, _}, :loaded} -> + # Final state. Dependency successfully loaded. + {[app], [], Map.put(statuses, app, :loaded)} + + {{:unloaded, _}, {:error, err}} -> + # Final state. Dependency failed to load. + {[], [], Map.put(statuses, app, {:error, err})} + + {_prev_unloaded_or_final, _nwe_unloaded_or_final} -> + # No status change, or one that doesn't matter like final to final. + {[], [], statuses} + end + + { + new_loaded_apps ++ current_loaded_apps, + new_unloaded_apps ++ current_unloaded_apps, + updated_load_statuses + } + end) + end + defp env_dep(dep) do only_envs = dep_only(dep) only_envs == nil || Mix.env() in List.wrap(only_envs) @@ -452,7 +519,7 @@ defmodule Dialyxir.Project do defp dep_only({_, _, opts}) when is_list(opts), do: opts[:only] defp dep_only(_), do: nil - @spec reduce_umbrella_children(list(), (list() -> list())) :: list() + @spec reduce_umbrella_children(acc, (acc -> acc)) :: acc when acc: term defp reduce_umbrella_children(acc, f) do if Mix.Project.umbrella?() do children = Mix.Dep.Umbrella.loaded()