feat: add support for alerting on absent metric

frontend/public/locales/en-GB/alerts.json

@@ -111,5 +111,7 @@
  "exceptions_based_alert": "Exceptions-based Alert",
  "exceptions_based_alert_desc": "Send a notification when a condition occurs in the exceptions data.",
  "field_unit": "Threshold unit",
+ "text_alert_on_absent": "Send a notification if data is missing for",
+ "text_for": "minutes",
  "selected_query_placeholder": "Select query"
 }

frontend/public/locales/en/alerts.json

@@ -111,5 +111,7 @@
  "exceptions_based_alert": "Exceptions-based Alert",
  "exceptions_based_alert_desc": "Send a notification when a condition occurs in the exceptions data.",
  "field_unit": "Threshold unit",
+ "text_alert_on_absent": "Send a notification if data is missing for",
+ "text_for": "minutes",
  "selected_query_placeholder": "Select query"
 }

frontend/src/container/FormAlertRules/RuleOptions.tsx

@@ -1,4 +1,5 @@
 import {
+ Checkbox,
  Form,
  InputNumber,
  InputNumberProps,
@@ -213,28 +214,66 @@ function RuleOptions({
  ? renderPromRuleOptions()
  : renderThresholdRuleOpts()}
 
- <Space direction="horizontal" align="center">
- <Form.Item noStyle name={['condition', 'target']}>
- <InputNumber
- addonBefore={t('field_threshold')}
- value={alertDef?.condition?.target}
- onChange={onChange}
- type="number"
- onWheel={(e): void => e.currentTarget.blur()}
- />
- </Form.Item>
-
- <Form.Item noStyle>
- <Select
- getPopupContainer={popupContainer}
- allowClear
- showSearch
- options={categorySelectOptions}
- placeholder={t('field_unit')}
- value={alertDef.condition.targetUnit}
- onChange={onChangeAlertUnit}
- />
- </Form.Item>
+ <Space direction="vertical" size="large">
+ <Space direction="horizontal" align="center">
+ <Form.Item noStyle name={['condition', 'target']}>
+ <InputNumber
+ addonBefore={t('field_threshold')}
+ value={alertDef?.condition?.target}
+ onChange={onChange}
+ type="number"
+ onWheel={(e): void => e.currentTarget.blur()}
+ />
+ </Form.Item>
+
+ <Form.Item noStyle>
+ <Select
+ getPopupContainer={popupContainer}
+ allowClear
+ showSearch
+ options={categorySelectOptions}
+ placeholder={t('field_unit')}
+ value={alertDef.condition.targetUnit}
+ onChange={onChangeAlertUnit}
+ />
+ </Form.Item>
+ </Space>
+ <Space direction="horizontal" align="center">
+ <Form.Item noStyle name={['condition', 'alertOnAbsent']}>
+ <Checkbox
+ checked={alertDef?.condition?.alertOnAbsent}
+ onChange={(e): void => {
+ setAlertDef({
+ ...alertDef,
+ condition: {
+ ...alertDef.condition,
+ alertOnAbsent: e.target.checked,
+ },
+ });
+ }}
+ />
+ </Form.Item>
+ <Typography.Text>{t('text_alert_on_absent')}</Typography.Text>
+
+ <Form.Item noStyle name={['condition', 'absentFor']}>
+ <InputNumber
+ min={1}
+ value={alertDef?.condition?.absentFor}
+ onChange={(value): void => {
+ setAlertDef({
+ ...alertDef,
+ condition: {
+ ...alertDef.condition,
+ absentFor: Number(value) || 0,
+ },
+ });
+ }}
+ type="number"
+ onWheel={(e): void => e.currentTarget.blur()}
+ />
+ </Form.Item>
+ <Typography.Text>{t('text_for')}</Typography.Text>
+ </Space>
  </Space>
  </FormContainer>
  </>

frontend/src/types/api/alerts/def.ts

@@ -32,6 +32,8 @@ export interface RuleCondition {
  matchType?: string;
  targetUnit?: string;
  selectedQueryName?: string;
+ alertOnAbsent?: boolean | undefined;
+ absentFor?: number | undefined;
 }
 
 export interface Labels {

pkg/query-service/constants/constants.go

@@ -224,7 +224,8 @@ var TimeoutExcludedRoutes = map[string]bool{
 // alert related constants
 const (
  // AlertHelpPage is used in case default alert repo url is not set
- AlertHelpPage = "https://signoz.io/docs/userguide/alerts-management/#generator-url"
+ AlertHelpPage = "https://signoz.io/docs/userguide/alerts-management/#generator-url"
+ AlertTimeFormat = "2006-01-02 15:04:05"
 )
 
 func GetOrDefaultEnv(key string, fallback string) string {

pkg/query-service/rules/alerting.go

@@ -142,9 +142,11 @@ type RuleCondition struct {
  CompositeQuery *v3.CompositeQuery `json:"compositeQuery,omitempty" yaml:"compositeQuery,omitempty"`
  CompareOp CompareOp `yaml:"op,omitempty" json:"op,omitempty"`
  Target *float64 `yaml:"target,omitempty" json:"target,omitempty"`
- MatchType `json:"matchType,omitempty"`
- TargetUnit string `json:"targetUnit,omitempty"`
- SelectedQuery string `json:"selectedQueryName,omitempty"`
+ AlertOnAbsent bool `yaml:"alertOnAbsent,omitempty" json:"alertOnAbsent,omitempty"`
+ AbsentFor time.Duration `yaml:"absentFor,omitempty" json:"absentFor,omitempty"`
+ MatchType MatchType `json:"matchType,omitempty"`
+ TargetUnit string `json:"targetUnit,omitempty"`
+ SelectedQuery string `json:"selectedQueryName,omitempty"`
 }
 
 func (rc *RuleCondition) IsValid() bool {

pkg/query-service/rules/resultTypes.go

@@ -20,6 +20,8 @@ type Sample struct {
  // Label keys as-is from the result query.
  // The original labels are used to prepare the related{logs, traces} link in alert notification
  MetricOrig labels.Labels
+
+ IsMissing bool
 }
 
 func (s Sample) String() string {

pkg/query-service/rules/thresholdRule.go

@@ -71,7 +71,9 @@ type ThresholdRule struct {
  temporalityMap map[string]map[v3.Temporality]bool
 
  opts ThresholdRuleOpts
- typ string
+
+ lastTimestampWithDatapoints time.Time
+ typ string
 }
 
 type ThresholdRuleOpts struct {
@@ -531,6 +533,7 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer
  if err := rows.Scan(vars...); err != nil {
  return nil, err
  }
+ r.lastTimestampWithDatapoints = time.Now()
 
  sample := Sample{}
  // Why do we maintain two labels sets? Alertmanager requires
@@ -555,8 +558,8 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer
  if colName == "ts" || colName == "interval" {
  sample.Point.T = timval.Unix()
  } else {
- lbls.Set(colName, timval.Format("2006-01-02 15:04:05"))
- lblsOrig.Set(columnNames[i], timval.Format("2006-01-02 15:04:05"))
+ lbls.Set(colName, timval.Format(constants.AlertTimeFormat))
+ lblsOrig.Set(columnNames[i], timval.Format(constants.AlertTimeFormat))
  }
 
  case *float64:
@@ -709,6 +712,20 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer
 
  zap.S().Debugf("ruleid:", r.ID(), "\t resultmap(potential alerts):", len(resultMap))
 
+ // if the data is missing for `For` duration then we should send alert
+ if r.ruleCondition.AlertOnAbsent && r.lastTimestampWithDatapoints.Add(r.Condition().AbsentFor).Before(time.Now()) {
+ zap.S().Debugf("ruleid:", r.ID(), "\t msg: no data found for rule condition")
+ lbls := labels.NewBuilder(labels.Labels{})
+ if !r.lastTimestampWithDatapoints.IsZero() {
+ lbls.Set("lastSeen", r.lastTimestampWithDatapoints.Format(constants.AlertTimeFormat))
+ }
+ result = append(result, Sample{
+ Metric: lbls.Labels(),
+ IsMissing: true,
+ })
+ return result, nil
+ }
+
  for _, sample := range resultMap {
  // check alert rule condition before dumping results, if sendUnmatchedResults
  // is set then add results irrespective of condition
@@ -1177,6 +1194,11 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time, queriers *Querie
 
  annotations := make(labels.Labels, 0, len(r.annotations))
  for _, a := range r.annotations {
+ if smpl.IsMissing {
+ if a.Name == labels.AlertDescriptionLabel || a.Name == labels.AlertSummaryLabel {
+ a.Value = labels.AlertMissingData
+ }
+ }
  annotations = append(annotations, labels.Label{Name: normalizeLabelName(a.Name), Value: expand(a.Value)})
  }
 

pkg/query-service/utils/labels/labels.go

@@ -23,8 +23,11 @@ const (
  AlertRuleIdLabel = "ruleId"
  RuleSourceLabel = "ruleSource"
 
- RuleThresholdLabel = "threshold"
- AlertSummaryLabel = "summary"
+ RuleThresholdLabel = "threshold"
+ AlertSummaryLabel = "summary"
+ AlertDescriptionLabel = "description"
+
+ AlertMissingData = "Missing data"
 )
 
 // Label is a key/value pair of strings.