Version 1.3.0 (#16)

* preparing v.1.2.0

* bugfix for version_display_bug

* merged automated testing framework

* updated docs to represenet no-default-filepath

* Added configureable stderr logging, fixed a bug in stderr output handling

* amended changelog

* fixed pip -> pip3 in CLI docs

* re-enable STDERR disabling for stability

* fixed a windows specifc bug

* removed .bat file windows fix

* bumped etp-cli 0.3.6

* amended changelog for etp 3.6

* merged pre v1.3.0

* fixed submodules issue

* v1.3.0 finalized

* modified bug report template

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,29 +10,24 @@ assignees: bitonio, MikeSchiessl
 **Describe the bug**
 A clear and concise description of what the bug is.
 
-**To Reproduce**
+**To Reproduce** 
 Steps to reproduce the behavior:
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
 4. See error
 
-**Expected behavior**
+**Expected behavior** 
 A clear and concise description of what you expected to happen.
 
-**Screenshots**
+**Screenshots** 
 If applicable, add screenshots to help explain your problem.
 
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+**ULS Version output** 
+please run the following and attach the output here.
+```bash
+bin/uls.py --version
+```
 
-**Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
-
-**Additional context**
+**Additional context** 
 Add any other context about the problem here.

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "test/bats/bats-assert"]
+ path = test/bats/bats-assert
+ url = https:/ztombol/bats-assert.git
+[submodule "test/bats/bats-support"]
+ path = test/bats/bats-support
+ url = https:/ztombol/bats-support.git

Dockerfile

@@ -10,8 +10,8 @@ ARG HOMEDIR="/opt/akamai-uls"
 ARG ULS_DIR="$HOMEDIR/uls"
 ARG EXT_DIR="$ULS_DIR/ext"
 
-ARG ETP_CLI_VERSION="0.3.5"
-ARG EAA_CLI_VERSION="0.4.4"
+ARG ETP_CLI_VERSION="0.3.6"
+ARG EAA_CLI_VERSION="0.4.5"
 ARG MFA_CLI_VERSION="0.0.6"
 
 # ENV VARS
@@ -42,6 +42,7 @@ RUN mkdir -p ${ULS_DIR}
 
 # Install ULS
 COPY bin/ ${ULS_DIR}/bin
+COPY var/ ${ULS_DIR}/var
 WORKDIR ${ULS_DIR}
 RUN pip3 install -r ${ULS_DIR}/bin/requirements.txt
 

bin/config/global_config.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.2.0"
+__version__ = "1.3.0"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
@@ -50,7 +50,8 @@
 input_rerun_retries = 3 # Number of rerun attempts before giving up
 input_run_delay = 1 # Time in seconds to wait for the first health check
 input_rerun_delay = 1 # Time in seconds between rerun attempts
-
+input_disable_stderr = True # Enable STDERR output disabling (see value below to specify when this should happen)
+input_disable_stderr_after = 25 # Disable stderr output after x input_cli cycles --> to prevent buffer overflow
 
 # OUTPUT Configuration
 output_reconnect_retries = 10 # Number of reconnect attempts before giving up
@@ -62,10 +63,10 @@
 output_tcp_timeout = 10.0 # TCP SEND / CONNECT Timeout (seconds)
  ## HTTP
 output_http_header = {'User-Agent': f'{__tool_name_long__}/{__version__}'} # HTTP Additional Headers to send (requests module KV pairs)
+output_http_timeout = 10 # Timeout after which a request will be considered as failed
  ## FILE
 output_file_encoding = "utf-8" # FILE Encoding setting
 output_file_handler_choices = ['SIZE', 'TIME'] # Available Choices for the file handler
-output_file_default_name = 'tmp/uls_file.output' # Default file name (path + name), only used if not set
 output_file_default_backup_count = 3 # Default number of backup files (after rotation)
 output_file_default_maxbytes = 50* 1024 * 1024 # Default maximum size of a file when rotated by the FILE - handler
 output_file_default_time_use_utc = False # Use UTC instead of local system time (Default: False)
@@ -83,4 +84,10 @@
 edgerc_eaa_legacy = ["eaa_api_host", "eaa_api_key", "eaa_api_secret"] # required for EAA - Legacy
 edgerc_mfa = ["mfa_integration_id", "mfa_signing_key"] # Required for MFA
 edgerc_documentation_url = "https:/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md"
-edgerc_mock_file = "ext/edgerc" # Required for display the version if no edgercfile was given
+edgerc_mock_file = "ext/edgerc" # Required for display the version if no edgercfile was given
+
+# Autoresume Configuration
+autoresume_checkpoint_path = "var/" # (Default) Path, where the checkpointfiles should be stored to
+autoresume_supported_inputs = ['ETP', 'EAA'] # Internal Var only, to adjust supported inputs
+autoresume_write_after = 1000 # Write checkpoint only every ${autoresume_write_every} loglines
+

bin/modules/UlsArgsParser.py

@@ -39,6 +39,7 @@ def init():
  const=True,
  help=f'Display {uls_config.__tool_name_short__} version and operational information')
 
+
  # ----------------------
  # Input GROUP
  input_group = parser.add_argument_group(title="Input",
@@ -134,7 +135,7 @@ def init():
  output_group.add_argument('--port',
  action='store',
  type=int,
- default=int(os.environ.get('ULS_OUTPUT_PORT') or '0'),
+ default=int(os.environ.get('ULS_OUTPUT_PORT') or '0' ),
  help="Port for TCP/UDP")
 
  # HTTP
@@ -176,7 +177,7 @@ def init():
  output_group.add_argument('--filehandler',
  action='store',
  type=str.upper,
- default=(os.environ.get('ULS_FILE_HANDLER') or None),
+ default=(os.environ.get('ULS_FILE_HANDLER') or "SIZE"),
  choices=uls_config.output_file_handler_choices,
  help=f"Output file handler - Decides when files are rotated -"
  f"Choices: {uls_config.output_file_handler_choices} -"
@@ -186,9 +187,9 @@ def init():
  action='store',
  type=str,
  default=(os.environ.get('ULS_FILE_NAME') or
- uls_config.output_file_default_name),
+ None),
  help=f"Output file destination (path + filename)"
- f" Default: {uls_config.output_file_default_name}")
+ f" Default: None")
 
  ## File Backup count
  output_group.add_argument('--filebackupcount',
@@ -227,6 +228,14 @@ def init():
  help=f"Specifies the file rotation interval based on `--filetime` unit value"
  f" Default: {uls_config.output_file_time_interval}")
 
+ ## File Action
+ output_group.add_argument('--fileaction',
+ action='store',
+ type=str,
+ default=(os.environ.get('ULS_FILE_ACTION') or None),
+ help=f"This enables you to specify your own action upon a file rotation. ('%%s' defines the absolute file_name e.g. /path/to/my_file.log.1)."
+ f" Default: <None>")
+
  # ----------------------
  special_group = parser.add_argument_group(title="Transformation",
  description="Define Module Settings (Output manipulation)")
@@ -254,6 +263,35 @@ def init():
  default=(os.environ.get('ULS_TRANSFORMATION_PATTERN') or None),
  help="Provide a pattern to transform the output (Required for JMESPATH)")
 
+
+
+ #-------------------------
+ resume_group = parser.add_argument_group(title="Autoresume",
+ description="Define Autoresume Settings")
+ # Autoresume / Resume Switch
+ resume_group.add_argument('--autoresume', '--resume',
+ action='store',
+ type=bool,
+ dest='autoresume',
+ default=(os.environ.get('ULS_AUTORESUME') or False),
+ nargs='?',
+ const=True,
+ help=f'Enable automated resume on based on a checkpoint (do not use alongside --starttime)')
+
+ resume_group.add_argument('--autoresumepath',
+ action='store',
+ type=str,
+ dest='autoresumepath',
+ default=(os.environ.get('ULS_AUTORESUME_PATH') or uls_config.autoresume_checkpoint_path),
+ help=f'Specify the path where checkpoint files should be written to. (Trailing /) [Default: {uls_config.autoresume_checkpoint_path}]')
+
+ resume_group.add_argument('--autoresumewriteafter',
+ action='store',
+ type=int,
+ dest='autoresumewriteafter',
+ default=(os.environ.get('ULS_AUTORESUME_WRITEAFTER') or uls_config.autoresume_write_after),
+ help=f'Specify after how many loglines a checkpoint should be written [Default: {uls_config.autoresume_write_after}]')
+
  return parser.parse_args()
 
 

bin/modules/UlsInputCli.py

@@ -38,7 +38,8 @@ def __init__(self,
  rawcmd=None,
  inproxy=None,
  starttime: int=None,
- endtime: int=None):
+ endtime: int=None,
+ root_path: str=None):
  """
  Initialzing a new UlsInput handler
  :param product: Input product
@@ -50,6 +51,7 @@ def __init__(self,
  :param inproxy: Proxy config
  :param starttime: Start time in epoch seconds
  :param endtime: End time in epoch seconds
+ :param root_path: Root path of the git repo (to avoid runtime issues)
  """""
 
  # Defaults (may vary later)
@@ -60,6 +62,7 @@ def __init__(self,
  self.rerun_counter = 1
  self.cli_proc = None
  self.run_once = False
+ self.cycle_counter = 0
 
  # Handover Parameters
  self.product = product
@@ -71,12 +74,15 @@ def __init__(self,
  self.inproxy = inproxy
  self.starttime = starttime
  self.endtime = endtime
+ self.root_path = root_path
 
  # Variables (load from uls_config)
  self.run_delay = uls_config.input_run_delay # Time in seconds to wait for the first health check
  self.rerun_retries = uls_config.input_rerun_retries # Number of rerun attempts before giving up
  self.rerun_delay = uls_config.input_rerun_delay # Time in seconds between rerun attempts
  self.bin_python = uls_config.bin_python # The python binary
+ self.disable_stderr = uls_config.input_disable_stderr #Specify if STDERR should be disabled at all after $disable_stderr_after cycles
+ self.disable_stderr_after = uls_config.input_disable_stderr_after # Disable StdErr Output after # cycles
 
  def _feed_selector(self, feed, product_feeds):
  if feed in product_feeds:
@@ -139,7 +145,7 @@ def proc_create(self):
 
  # EAA config
  if self.product == "EAA":
- product_path = uls_config.bin_eaa_cli
+ product_path = self.root_path + "/" + uls_config.bin_eaa_cli
  product_feeds = uls_config.eaa_cli_feeds
  if not self.rawcmd:
  my_feed = self._feed_selector(self.feed, product_feeds)
@@ -180,7 +186,7 @@ def proc_create(self):
 
  # ETP config
  elif self.product == "ETP":
- product_path = uls_config.bin_etp_cli
+ product_path = self.root_path + "/" + uls_config.bin_etp_cli
  product_feeds = uls_config.etp_cli_feeds
 
  if not self.cliformat == "JSON":
@@ -210,7 +216,7 @@ def proc_create(self):
 
  # MFA config
  elif self.product == "MFA":
- product_path = uls_config.bin_mfa_cli
+ product_path = self.root_path + "/" + uls_config.bin_mfa_cli
  product_feeds = uls_config.mfa_cli_feeds
  if not self.cliformat == "JSON":
  aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
@@ -243,7 +249,9 @@ def proc_create(self):
  aka_log.log.critical(f" {self.name} - No valid product selected "
  f"(--input={self.product}).")
  sys.exit(1)
+
  try:
+ self.cycle_counter = 0
  aka_log.log.info(f'{self.name} - CLI Command: {" ".join(cli_command)}')
  os.environ["PYTHONUNBUFFERED"] = "1"
  self.cli_proc = subprocess.Popen(cli_command,
@@ -253,10 +261,15 @@ def proc_create(self):
  f"{' '.join(cli_command)}")
  self.proc = self.cli_proc
  self.proc_output = self.cli_proc.stdout
- os.set_blocking(self.proc_output.fileno(), False)
+
+ # Unblocking on windows causes trouble so we're avoiding it
+ if not os.name == 'nt':
+ os.set_blocking(self.proc_output.fileno(), False)
+
  time.sleep(1)
 
  if not self.check_proc():
+ #self.rerun_counter += 1
  raise NameError(f"process [{self.cli_proc.pid}] "
  f"exited RC={self.cli_proc.returncode}, REASON: "
  f"{self.cli_proc.stderr.read().decode()}")
@@ -267,7 +280,8 @@ def proc_create(self):
  self.rerun_counter = 1
  if self.endtime:
  self.run_once = True
- self.cli_proc.stderr = subprocess.DEVNULL
+
+ #self.cli_proc.stderr = subprocess.DEVNULL
 
  except Exception as my_error:
  time.sleep(self.rerun_delay)
@@ -279,18 +293,27 @@ def proc_create(self):
  if self.running is False and self.rerun_counter > self.rerun_retries:
  aka_log.log.critical(f'{self.name} - Not able to start the CLI for '
  f'{self.product}. See above errors. '
- f'Giving up after {self.rerun_counter - 1} retries.')
+ f'Giving up after {self.rerun_counter - 2} retries.')
  sys.exit(1)
 
  def check_proc(self):
  try:
  if self.proc.poll() is None:
 
+ if self.cycle_counter == self.disable_stderr_after and self.disable_stderr:
+ aka_log.log.info(f"{self.name} - Disabling STDERR output from now on, after {self.cycle_counter} successful cycles")
+ self.cli_proc.stderr = subprocess.DEVNULL
+ aka_log.log.debug(f'{self.name} - Successful cycles for proc[{self.proc.pid}]: {self.cycle_counter}')
+ self.cycle_counter = self.cycle_counter + 1
  return True
  else:
  self.running = False
- aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}]'
- f' was found stale - {self.proc.stderr.read().decode()}')
+ self.rerun_counter += 1
+ if (self.cycle_counter <= self.disable_stderr_after and self.disable_stderr) or not self.disable_stderr:
+ aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}]'
+ f' was found stale - Reason: "{self.proc.stderr.read().decode()}" ')
+ else:
+ aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}], sadly stderr has been disabled')
  self.proc_create()
  return False
 
@@ -299,7 +322,7 @@ def check_proc(self):
  aka_log.log.critical(f"{self.name} - '--endtime' was specified - so stopping now")
  sys.exit(0)
  else:
- aka_log.log.critical(f'{self.name} - Soemthing really '
+ aka_log.log.error(f'{self.name} - Soemthing really '
  f'strange happened - message: {my_error}')
 
  self.proc_create()

bin/modules/UlsMonitoring.py

@@ -45,6 +45,7 @@ def __init__(self, stopEvent, product, feed, output):
  # Variables
  self.monitoring_enabled = uls_config.monitoring_enabled # Monitoring enable Flag
  self.monitoring_interval = uls_config.monitoring_interval # Monitoring interval
+ self._version = uls_config.__version__
 
  # Definitions
  self.name = "UlsMonitoring" # Class Human readable name
@@ -79,8 +80,10 @@ def display(self):
  'uls_product': self._product,
  'uls_feed': self._feed,
  'uls_output': self._output,
+ 'uls_version': self._version,
  'uls_runtime': self._runtime(),
  'event_count': self.overall_messages_handled,
+ 'event_count_interval': self.window_messages_handled,
  'event_rate': round(self.window_messages_handled / self.monitoring_interval, 2),
  'mon_interval': self.monitoring_interval
  }