Track resource counts

CHANGELOG.md

@@ -34,6 +34,7 @@ Contributors:
 - Bump werkzeug upper bound dependency to `<v2.0` ([#3011](https:/fishtown-analytics/dbt/pull/3011))
 - Performance fixes for many different things ([#2862](https:/fishtown-analytics/dbt/issues/2862), [#3034](https:/fishtown-analytics/dbt/pull/3034))
 - Update code to use Mashumaro 2.0 ([#3138](https:/fishtown-analytics/dbt/pull/3138))
+- Add an event to track resource counts ([#3050](https:/fishtown-analytics/dbt/issues/3050), [#3156](https:/fishtown-analytics/dbt/pull/3156))
 
 Contributors:
 - [@Bl3f](https:/Bl3f) ([#3011](https:/fishtown-analytics/dbt/pull/3011))

core/dbt/compilation.py

@@ -30,6 +30,7 @@
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.node_types import NodeType
 from dbt.utils import pluralize
+import dbt.tracking
 
 graph_file_name = 'graph.gpickle'
 
@@ -58,6 +59,11 @@ def print_compile_stats(stats):
  results = {k: 0 for k in names.keys()}
  results.update(stats)
 
+ # create tracking event for resource_counts
+ if dbt.tracking.active_user is not None:
+ resource_counts = {k.pluralize(): v for k, v in results.items()}
+ dbt.tracking.track_resource_counts(resource_counts)
+
  stat_line = ", ".join([
  pluralize(ct, names.get(t)) for t, ct in results.items()
  if t in names

core/dbt/tracking.py

@@ -29,6 +29,7 @@
 RPC_REQUEST_SPEC = 'iglu:com.dbt/rpc_request/jsonschema/1-0-1'
 DEPRECATION_WARN_SPEC = 'iglu:com.dbt/deprecation_warn/jsonschema/1-0-0'
 LOAD_ALL_TIMING_SPEC = 'iglu:com.dbt/load_all_timing/jsonschema/1-0-0'
+RESOURCE_COUNTS = 'iglu:com.dbt/resource_counts/jsonschema/1-0-0'
 
 DBT_INVOCATION_ENV = 'DBT_INVOCATION_ENV'
 
@@ -289,6 +290,20 @@ def track_project_load(options):
  )
 
 
+def track_resource_counts(resource_counts):
+ context = [SelfDescribingJson(RESOURCE_COUNTS, resource_counts)]
+ assert active_user is not None, \
+ 'Cannot track resource counts when active user is None'
+
+ track(
+ active_user,
+ category='dbt',
+ action='resource_counts',
+ label=active_user.invocation_id,
+ context=context
+ )
+
+
 def track_model_run(options):
  context = [SelfDescribingJson(RUN_MODEL_SPEC, options)]
  assert active_user is not None, \

test/integration/033_event_tracking_test/test_events.py

@@ -105,6 +105,24 @@ def populate(project_id, user_id, invocation_id, version):
  }]
  return populate
 
+ def resource_counts_context(self):
+ return [
+ {
+ 'schema': 'iglu:com.dbt/resource_counts/jsonschema/1-0-0',
+ 'data': {
+ 'models': ANY,
+ 'tests': ANY,
+ 'snapshots': ANY,
+ 'analyses': ANY,
+ 'macros': ANY,
+ 'operations': ANY,
+ 'seeds': ANY,
+ 'sources': ANY,
+ 'exposures': ANY,
+ }
+ }
+ ]
+
  def build_context(
  self,
  command,
@@ -231,6 +249,12 @@ def test__postgres_event_tracking_compile(self):
  label=ANY,
  context=ANY
  ),
+ call(
+ category='dbt',
+ action='resource_counts',
+ label=ANY,
+ context=ANY
+ ),
  call(
  category='dbt',
  action='invocation',
@@ -242,6 +266,7 @@ def test__postgres_event_tracking_compile(self):
  expected_contexts = [
  self.build_context('compile', 'start'),
  self.load_context(),
+ self.resource_counts_context(),
  self.build_context('compile', 'end', result_type='ok')
  ]
 
@@ -345,6 +370,12 @@ def seed_context(project_id, user_id, invocation_id, version):
  label=ANY,
  context=ANY
  ),
+ call(
+ category='dbt',
+ action='resource_counts',
+ label=ANY,
+ context=ANY
+ ),
  call(
  category='dbt',
  action='run_model',
@@ -362,6 +393,7 @@ def seed_context(project_id, user_id, invocation_id, version):
  expected_contexts = [
  self.build_context('seed', 'start'),
  self.load_context(),
+ self.resource_counts_context(),
  seed_context,
  self.build_context('seed', 'end', result_type='ok')
  ]
@@ -383,6 +415,12 @@ def test__postgres_event_tracking_models(self):
  label=ANY,
  context=ANY
  ),
+ call(
+ category='dbt',
+ action='resource_counts',
+ label=ANY,
+ context=ANY
+ ),
  call(
  category='dbt',
  action='run_model',
@@ -412,6 +450,7 @@ def test__postgres_event_tracking_models(self):
  expected_contexts = [
  self.build_context('run', 'start'),
  self.load_context(),
+ self.resource_counts_context(),
  self.run_context(
  hashed_contents='1e5789d34cddfbd5da47d7713aa9191c',
  model_id='4fbacae0e1b69924b22964b457148fb8',
@@ -455,6 +494,12 @@ def test__postgres_event_tracking_model_error(self):
  label=ANY,
  context=ANY
  ),
+ call(
+ category='dbt',
+ action='resource_counts',
+ label=ANY,
+ context=ANY
+ ),
  call(
  category='dbt',
  action='run_model',
@@ -472,6 +517,7 @@ def test__postgres_event_tracking_model_error(self):
  expected_contexts = [
  self.build_context('run', 'start'),
  self.load_context(),
+ self.resource_counts_context(),
  self.run_context(
  hashed_contents='4419e809ce0995d99026299e54266037',
  model_id='576c3d4489593f00fad42b97c278641e',
@@ -508,6 +554,12 @@ def test__postgres_event_tracking_tests(self):
  label=ANY,
  context=ANY
  ),
+ call(
+ category='dbt',
+ action='resource_counts',
+ label=ANY,
+ context=ANY
+ ),
  call(
  category='dbt',
  action='invocation',
@@ -519,6 +571,7 @@ def test__postgres_event_tracking_tests(self):
  expected_contexts = [
  self.build_context('test', 'start'),
  self.load_context(),
+ self.resource_counts_context(),
  self.build_context('test', 'end', result_type='ok')
  ]
 
@@ -619,6 +672,12 @@ def test__postgres_event_tracking_unable_to_connect(self):
  label=ANY,
  context=ANY
  ),
+ call(
+ category='dbt',
+ action='resource_counts',
+ label=ANY,
+ context=ANY
+ ),
  call(
  category='dbt',
  action='invocation',
@@ -630,6 +689,7 @@ def test__postgres_event_tracking_unable_to_connect(self):
  expected_contexts = [
  self.build_context('run', 'start'),
  self.load_context(),
+ self.resource_counts_context(),
  self.build_context('run', 'end', result_type='error')
  ]
 
@@ -666,6 +726,12 @@ def test__postgres_event_tracking_snapshot(self):
  label=ANY,
  context=ANY
  ),
+ call(
+ category='dbt',
+ action='resource_counts',
+ label=ANY,
+ context=ANY
+ ),
  call(
  category='dbt',
  action='run_model',
@@ -684,6 +750,7 @@ def test__postgres_event_tracking_snapshot(self):
  expected_contexts = [
  self.build_context('snapshot', 'start'),
  self.load_context(),
+ self.resource_counts_context(),
  self.run_context(
  hashed_contents=ANY,
  model_id='820793a4def8d8a38d109a9709374849',
@@ -721,6 +788,12 @@ def test__postgres_event_tracking_catalog_generate(self):
  label=ANY,
  context=ANY,
  ),
+ call(
+ category='dbt',
+ action='resource_counts',
+ label=ANY,
+ context=ANY,
+ ),
  call(
  category='dbt',
  action='invocation',
@@ -732,6 +805,7 @@ def test__postgres_event_tracking_catalog_generate(self):
  expected_contexts = [
  self.build_context('generate', 'start'),
  self.load_context(),
+ self.resource_counts_context(),
  self.build_context('generate', 'end', result_type='ok')
  ]