Plugin for OPL cluster_read.py to count file lines matching to various regexps

opl/cluster_read.py

@@ -68,7 +68,7 @@ def measure(self, ri, **args):
 
  def _dump_raw_data(self, name, mydata):
  """
- Dumps raw data for monitoring plagins into CSV files (first column
+ Dumps raw data for monitoring plugins into CSV files (first column
  for timestamp, second for value) into provided directory.
  """
  if self.args.monitoring_raw_data_dir is None:
@@ -374,6 +374,37 @@ def measure(self, ri, name, command, output="text"):
  return name, result
 
 
+class CountLinePlugin(BasePlugin):
+ def measure(
+ self,
+ ri,
+ config,
+ output="text",
+ ):
+ """
+ Execute command "command" and return result as per its "output" configuration
+ """
+ name = config["name"]
+ log_source_command = config["log_source_command"]
+ result = execute(log_source_command).splitlines()
+
+ output = {}
+ output["all"] = len(result)
+
+ for pattern_name, pattern_value in config.items():
+ if not pattern_name.startswith("log_regexp_"):
+ continue
+ pattern_key = pattern_name[len("log_regexp_") :]
+ pattern_regexp = re.compile(pattern_value)
+ counter = 0
+ for line in result:
+ if pattern_regexp.search(line):
+ counter += 1
+ output[pattern_key] = counter
+
+ return name, output
+
+
 class CopyFromPlugin(BasePlugin):
  def measure(self, ri, name, copy_from):
  """
@@ -399,6 +430,7 @@ def measure(self, ri, name, **kwargs):
  "env_variable": EnvironmentPlugin,
  "command": CommandPlugin,
  "copy_from": CopyFromPlugin,
+ "log_source_command": CountLinePlugin,
  "monitoring_query": PrometheusMeasurementsPlugin,
  "grafana_target": GrafanaMeasurementsPlugin,
  "metric_query": PerformanceInsightsMeasurementPlugin,
@@ -508,8 +540,12 @@ def __next__(self):
  if i < len(self.config):
  if self._find_plugin(self.config[i].keys()):
  instance = self._find_plugin(self.config[i].keys())
+ name = list(self.config[i].keys())[1]
  try:
- output = instance.measure(self, **self.config[i])
+ if name == "log_source_command":
+ output = instance.measure(self, self.config[i])
+ else:
+ output = instance.measure(self, **self.config[i])
  except Exception as e:
  logging.exception(
  f"Failed to measure {self.config[i]['name']}: {e}"

opl/cluster_read_example.yaml

@@ -22,3 +22,8 @@
 
 - name: parameters.env.JOB_NAME_ID
  env_variable: JOB_NAME_ID
+
+- name: measurements.logs.openshift-pipelines.pipelines-as-code-controller
+ log_source_command: oc -n openshift-pipelines logs --since=10h --all-containers --selector app.kubernetes.io/component=controller,app.kubernetes.io/instance=default,app.kubernetes.io/name=controller,app.kubernetes.io/part-of=pipelines-as-code,app=pipelines-as-code-controller
+ log_regexp_error: '"level":"error"'
+ log_regexp_warning: '"level":"warning"'

tests/test_cluster_read.py

@@ -28,6 +28,31 @@ def test_date(self):
  self.assertGreaterEqual(int(v), before)
  self.assertGreaterEqual(after, int(v))
 
+ def test_count(self):
+ string = """
+ - name: print.output
+ log_source_command: echo -e "error line1\\ninfo line2\\nwarning line3\\nerror line4"
+ log_regexp_error: '^error '
+ log_regexp_warning: '^warning '
+ """
+ ri = opl.cluster_read.RequestedInfo(string)
+ k, v = next(ri)
+ self.assertEqual(k, "print.output")
+ self.assertEqual(v, {"all": 4, "error": 1, "warning": 1})
+
+ def test_count_large(self):
+ string = """
+ - name: print.output
+ log_source_command: echo -e '{"level":"error", "logger":"ABC", "msg":"Oh no"}\\n{"level":"info", "logger":"XYZ", "msg":"Hello!"}\\n{"level":"error", "logger":"XYZ", "msg":"Oh no"}\\n{"level":"warning", "logger":"XYZ", "msg":"Beware"}',
+ log_regexp_error_abc: '"level":"error", "logger":"ABC"'
+ log_regexp_error_xyz: '"level":"error", "logger":"XYZ"'
+ log_regexp_warning: '"level":"warning"'
+ """
+ ri = opl.cluster_read.RequestedInfo(string)
+ k, v = next(ri)
+ self.assertEqual(k, "print.output")
+ self.assertEqual(v, {"all": 4, "error_abc": 1, "error_xyz": 1, "warning": 1})
+
  def test_json(self):
  string = """
  - name: myjson