Create Metric Registry

lib/sanbase/clickhouse/metric/file_handler.ex

@@ -314,6 +314,7 @@ defmodule Sanbase.Clickhouse.MetricAdapter.FileHandler do
  """)
  end
 
+ def metrics_json(), do: @metrics_json
  def aggregations(), do: @aggregations
  def access_map(), do: @access_map |> transform()
  def table_map(), do: @table_map |> transform()

lib/sanbase/metric/registry/registry.ex

@@ -0,0 +1,90 @@
+defmodule Sanbase.Metric.Registry do
+ use Ecto.Schema
+
+ import Ecto.Changeset
+
+ alias __MODULE__.Validation
+
+ schema "metric_registry" do
+ # How the metric is exposed to external users
+ field(:metric, :string)
+ field(:human_readable_name, :string)
+ field(:aliases, {:array, :string}, default: [])
+
+ # What is the name of the metric in the DB and where to find it
+ field(:internal_metric, :string)
+ field(:table, :string)
+
+ field(:aggregation, :string)
+ field(:min_interval, :string)
+
+ # If the metric is a template metric, then the parameters need to be used
+ # to define the full set of metrics
+ field(:is_template_metric, :boolean, default: false)
+ field(:parameters, :map, default: %{})
+
+ field(:is_deprecated, :boolean, default: false)
+ field(:hard_deprecate_after, :utc_datetime, default: nil)
+
+ field(:data_type, :string)
+ field(:docs_links, {:array, :string}, default: [])
+
+ timestamps()
+ end
+
+ def changeset(%__MODULE__{} = metric_description, attrs) do
+ metric_description
+ |> cast(attrs, [
+ :metric,
+ :internal_metric,
+ :human_readable_name,
+ :aliases,
+ :table,
+ :is_template_metric,
+ :parameters,
+ :aggregation,
+ :min_interval,
+ :is_deprecated,
+ :hard_deprecate_after,
+ :data_type,
+ :docs_links
+ ])
+ |> validate_required([
+ :metric,
+ :internal_metric,
+ :human_readable_name,
+ :table,
+ :aggregation,
+ :min_interval
+ ])
+ |> validate_change(:aggregation, &Validation.validate_aggregation/2)
+ |> validate_change(:min_interval, &Validation.validate_min_interval/2)
+ |> validate_change(:data_type, &Validation.validate_data_type/2)
+ end
+
+ def populate() do
+ Sanbase.Clickhouse.MetricAdapter.FileHandler.metrics_json()
+ |> Enum.map(fn map ->
+ {:ok, captures} = Sanbase.TemplateEngine.Captures.extract_captures(map["metric"])
+ is_template_metric = captures != []
+
+ %__MODULE__{}
+ |> changeset(%{
+ metric: map["metric"],
+ internal_metric: map["internal_metric"],
+ human_readable_name: map["human_readable_name"],
+ aliases: map["aliases"],
+ table: map["table"],
+ aggregation: map["aggregation"],
+ min_interval: map["min_interval"],
+ is_template_metric: is_template_metric,
+ parameters: Map.get(map, "parameters", %{}),
+ is_deprecated: Map.get(map, "is_deprecated", false),
+ hard_deprecate_after: map["hard_deprecate_after"],
+ has_incomplete_data: Map.get(map, "has__incomplete_data", false),
+ data_type: map["data_type"],
+ docs_links: Map.get(map, "docs_links", [])
+ })
+ end)
+ end
+end

lib/sanbase/metric/registry/validation.ex

@@ -0,0 +1,35 @@
+defmodule Sanbase.Metric.Registry.Validation do
+ @valid_default_aggregation ["sum", "last", "count", "avg", "max", "min", "first"]
+ def validate_aggregation(:aggregation, aggregation) do
+ if aggregation in @valid_default_aggregation do
+ []
+ else
+ [
+ aggregation:
+ "The aggregation #{aggregation} is not a valid aggregation. Valid aggregations are #{Enum.join(@valid_default_aggregation, ", ")}"
+ ]
+ end
+ end
+
+ def validate_min_interval(:min_interval, min_interval) do
+ if Sanbase.DateTimeUtils.valid_compound_duration?(min_interval) do
+ []
+ else
+ [
+ min_interval:
+ "The provided min_interval #{min_interval} is not a valid duration - a number followed by one of: s (second), m (minute), h (hour) or d (day)"
+ ]
+ end
+ end
+
+ def validate_data_type(:data_type, data_type) do
+ if data_type in ["timeseries", "histogram"] do
+ []
+ else
+ [
+ data_type:
+ "Invalid data type #{data_type} is not one of the supported: timeseries or histogram"
+ ]
+ end
+ end
+end

priv/repo/migrations/20241014115340_create_metric_registry.exs

@@ -0,0 +1,97 @@
+defmodule Sanbase.Repo.Migrations.CreateMetricRegistry do
+ use Ecto.Migration
+
+ # We have JSON records that define single metrics:
+
+ # {
+ # "human_readable_name": "USD Price",
+ # "name": "price_usd_5m",
+ # "metric": "price_usd",
+ # "version": "2019-01-01",
+ # "access": "free",
+ # "selectors": [
+ # "slug"
+ # ],
+ # "min_plan": {
+ # "SANAPI": "free",
+ # "SANBASE": "free"
+ # },
+ # "aggregation": "last",
+ # "min_interval": "5m",
+ # "table": "intraday_metrics",
+ # "has_incomplete_data": false,
+ # "data_type": "timeseries",
+ # "docs_links": ["https://academy.santiment.net/metrics/price"]
+ # }
+
+ # But we also have metrics that define multiple metrics using templates
+
+ # {
+ # "human_readable_name": "Mean Realized USD Price for coins that moved in the past {{timebound:human_readable}}",
+ # "name": "mean_realized_price_usd_{{timebound}}",
+ # "metric": "mean_realized_price_usd_{{timebound}}",
+ # "parameters": [
+ # { "timebound": "1d" },
+ # { "timebound": "7d" },
+ # { "timebound": "30d" },
+ # { "timebound": "60d" },
+ # { "timebound": "90d" },
+ # { "timebound": "180d" },
+ # { "timebound": "365d" },
+ # { "timebound": "2y" },
+ # { "timebound": "3y" },
+ # { "timebound": "5y" },
+ # { "timebound": "10y" }
+ # ],
+ # "is_timebound": true,
+ # "version": "2019-01-01",
+ # "access": "restricted",
+ # "selectors": [ "slug" ],
+ # "min_plan": {
+ # "SANAPI": "free",
+ # "SANBASE": "free"
+ # },
+ # "aggregation": "avg",
+ # "min_interval": "1d",
+ # "table": "daily_metrics_v2",
+ # "has_incomplete_data": true,
+ # "data_type": "timeseries",
+ # "docs_links": ["https://academy.santiment.net/metrics/mean-realized-price"]
+ # }
+ def change do
+ create table(:metric_registry) do
+ add(:metric, :string, null: false)
+ add(:internal_metric, :string, null: false)
+ add(:human_readable_name, :string, null: false)
+ add(:aliases, {:array, :string}, null: false, default: [])
+ add(:table, :string, null: false)
+
+ add(:is_template_metric, :boolean, null: false, default: false)
+ add(:parameters, :jsonb, null: false, default: "{}")
+
+ add(:is_timebound, :boolean, null: false, null: false)
+ add(:is_exposed, :boolean, null: false, default: true)
+ add(:exposed_environments, :string, null: false, default: "all")
+
+ add(:version, :string)
+
+ add(:selectors, {:array, :string}, null: false, default: [])
+ add(:required_selectors, {:array, :string}, null: false, default: [])
+
+ add(:aggregation, :string, null: false)
+ add(:min_interval, :string, null: false)
+ add(:has_incomplete_data, :boolean, null: false)
+
+ add(:data_type, :string, null: false, default: "timeseries")
+ add(:docs_links, {:array, :string}, null: false, default: [])
+
+ add(:is_deprecated, :boolean, null: false, default: false)
+ add(:hard_deprecate_after, :utc_datetime, null: true, default: nil)
+
+ timestamps()
+ end
+
+ create(unique_index(:metric_registry, [:metric, :data_type, :parameters]))
+ create(index(:metric_registry, [:metric]))
+ end
+end

priv/repo/structure.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 15.1 (Homebrew)
--- Dumped by pg_dump version 15.1 (Homebrew)
+-- Dumped from database version 14.12 (Homebrew)
+-- Dumped by pg_dump version 14.12 (Homebrew)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -2227,6 +2227,56 @@ CREATE SEQUENCE public.menus_id_seq
 ALTER SEQUENCE public.menus_id_seq OWNED BY public.menus.id;
 
 
+--
+-- Name: metric_registry; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.metric_registry (
+ id bigint NOT NULL,
+ metric character varying(255) NOT NULL,
+ internal_metric character varying(255) NOT NULL,
+ human_readable_name character varying(255) NOT NULL,
+ aliases character varying(255)[] DEFAULT ARRAY[]::character varying[] NOT NULL,
+ "table" character varying(255) NOT NULL,
+ is_template_metric boolean DEFAULT false NOT NULL,
+ parameters jsonb DEFAULT '{}'::jsonb NOT NULL,
+ is_timebound boolean NOT NULL,
+ is_exposed boolean DEFAULT true NOT NULL,
+ exposed_environments character varying(255) DEFAULT 'all'::character varying NOT NULL,
+ version character varying(255),
+ selectors character varying(255)[] DEFAULT ARRAY[]::character varying[] NOT NULL,
+ required_selectors character varying(255)[] DEFAULT ARRAY[]::character varying[] NOT NULL,
+ aggregation character varying(255) NOT NULL,
+ min_interval character varying(255) NOT NULL,
+ has_incomplete_data boolean NOT NULL,
+ data_type character varying(255) DEFAULT 'timeseries'::character varying NOT NULL,
+ docs_links character varying(255)[] DEFAULT ARRAY[]::character varying[] NOT NULL,
+ is_deprecated boolean DEFAULT false NOT NULL,
+ hard_deprecate_after timestamp(0) without time zone DEFAULT NULL::timestamp without time zone,
+ inserted_at timestamp without time zone NOT NULL,
+ updated_at timestamp without time zone NOT NULL
+);
+
+
+--
+-- Name: metric_registry_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.metric_registry_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+--
+-- Name: metric_registry_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.metric_registry_id_seq OWNED BY public.metric_registry.id;
+
+
 --
 -- Name: metrics; Type: TABLE; Schema: public; Owner: -
 --
@@ -4998,6 +5048,13 @@ ALTER TABLE ONLY public.menu_items ALTER COLUMN id SET DEFAULT nextval('public.m
 ALTER TABLE ONLY public.menus ALTER COLUMN id SET DEFAULT nextval('public.menus_id_seq'::regclass);
 
 
+--
+-- Name: metric_registry id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.metric_registry ALTER COLUMN id SET DEFAULT nextval('public.metric_registry_id_seq'::regclass);
+
+
 --
 -- Name: metrics id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -5874,6 +5931,14 @@ ALTER TABLE ONLY public.menus
  ADD CONSTRAINT menus_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: metric_registry metric_registry_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.metric_registry
+ ADD CONSTRAINT metric_registry_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -6932,6 +6997,20 @@ CREATE UNIQUE INDEX market_segments_name_index ON public.market_segments USING b
 CREATE INDEX menus_user_id_index ON public.menus USING btree (user_id);
 
 
+--
+-- Name: metric_registry_metric_data_type_parameters_index; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX metric_registry_metric_data_type_parameters_index ON public.metric_registry USING btree (metric, data_type, parameters);
+
+
+--
+-- Name: metric_registry_metric_index; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX metric_registry_metric_index ON public.metric_registry USING btree (metric);
+
+
 --
 -- Name: metrics_name_index; Type: INDEX; Schema: public; Owner: -
 --
@@ -9295,4 +9374,5 @@ INSERT INTO public."schema_migrations" (version) VALUES (20240809122904);
 INSERT INTO public."schema_migrations" (version) VALUES (20240904135651);
 INSERT INTO public."schema_migrations" (version) VALUES (20240926130910);
 INSERT INTO public."schema_migrations" (version) VALUES (20240926135951);
+INSERT INTO public."schema_migrations" (version) VALUES (20241014115340);
 INSERT INTO public."schema_migrations" (version) VALUES (20241017092520);