v.1.2.0

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.6-slim-buster
+FROM python:3.10-slim-bullseye
 LABEL MAINTAINER="Mike Schiessl - [email protected]"
 LABEL APP_LONG="Akamai Universal Log Streamer"
 LABEL APP_SHORT="ULS"
@@ -11,7 +11,7 @@ ARG ULS_DIR="$HOMEDIR/uls"
 ARG EXT_DIR="$ULS_DIR/ext"
 
 ARG ETP_CLI_VERSION="0.3.5"
-ARG EAA_CLI_VERSION="0.4.2"
+ARG EAA_CLI_VERSION="0.4.4"
 ARG MFA_CLI_VERSION="0.0.6"
 
 # ENV VARS
@@ -27,7 +27,8 @@ RUN apt-get update && \
  ca-certificates \
  git \
  curl \
- telnet && \
+ telnet \
+ gcc libssl-dev libffi-dev && \
  rm -rf /var/lib/apt/lists/
 
 # USER & GROUP
@@ -42,22 +43,26 @@ RUN mkdir -p ${ULS_DIR}
 # Install ULS
 COPY bin/ ${ULS_DIR}/bin
 WORKDIR ${ULS_DIR}
+RUN pip3 install -r ${ULS_DIR}/bin/requirements.txt
 
 # Install external CLI'S
 ## ETP CLI
 ENV ETP_CLI_VERSION=$ETP_CLI_VERSION
 RUN git clone --depth 1 -b "${ETP_CLI_VERSION}" --single-branch https:/akamai/cli-etp.git ${EXT_DIR}/cli-etp && \
  pip3 install -r ${EXT_DIR}/cli-etp/requirements.txt
+
 ## EAA CLI
 ENV EAA-CLI_VERSION=$EAA_CLI_VERSION
 RUN git clone --depth 1 -b "${EAA_CLI_VERSION}" --single-branch https:/akamai/cli-eaa.git ${EXT_DIR}/cli-eaa && \
  pip3 install -r ${EXT_DIR}/cli-eaa/requirements.txt
+
+
 ## MFA CLI
 ENV MFA-CLI_VERSION=$MFA_CLI_VERSION
 RUN git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch https:/akamai/cli-mfa.git ${EXT_DIR}/cli-mfa && \
  pip3 install -r ${EXT_DIR}/cli-mfa/requirements.txt
 
 # ENTRYPOINTS / CMD
-ENTRYPOINT ["/usr/local/bin/python3","bin/uls.py"]
+ENTRYPOINT ["/usr/local/bin/python3","-u","bin/uls.py"]
 #CMD ["--help"]
 # EOF

README.md

@@ -46,11 +46,13 @@ It can be run directly as Python code, as a provided Docker container or through
 
 
 - Supported data outputs
- - TCP Socket (tcp://host:port) `--output tcp`
- - UDP Socket (udp://host:port) `--output udp`
- - HTTP(S) URL (http(s)://host:port/path) (supporting Authentication) `--output http`
- - RAW (>STDOUT) `--output raw`
-
+ - [`TCP` Socket (tcp://host:port) `--output tcp`](docs/OUTPUTS.md#tcp--udp)
+ - [`UDP` Socket (udp://host:port) `--output udp`](docs/OUTPUTS.md#tcp--udp)
+ - [`HTTP` and `HTTPS` URL (http(s)://host:port/path) (supporting Authentication) `--output http`](docs/OUTPUTS.md#http-and-https)
+ - [`RAW` (>STDOUT) `--output raw`](docs/OUTPUTS.md#raw)
+ - [`FILE` (writes log to a file on the local filesystem and rotates them) `--output file`](docs/OUTPUTS.md#file) 
+ The output documentation can be found [here](docs/OUTPUTS.md).
+
 
 - Operation types
  - [python (command line)](./docs/COMMAND_LINE_USAGE.md)
@@ -64,6 +66,7 @@ It can be run directly as Python code, as a provided Docker container or through
  - HTTP CA CERT verification skipping
  - Adoptable HTTP - POST format
  - [Output filter](docs/ADDITIONAL_FEATURES.md#filter---filter-feature) (to reduce number of log lines sent to SIEM)
+ - [Output transformation](docs/TRANSFORMATIONS.md) (to produce other log formats i.e. for 3rd party integrations)
 
 ## Documentation
 ULS can be operated in many ways. 

bin/config/global_config.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.1.0"
+__version__ = "1.2.0"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
@@ -11,6 +11,9 @@
 output_line_breaker = '\r\n' # Line breaking type (to split messages when streaming data)
 main_wait_default = 0.01 # Default wait time within the main loop
 main_wait_max = 60 # Maximum wait time for the main loop
+main_resend_attempts = 10 # Maximum number of attempts to deliver the data
+main_resend_exit_on_fail = False # Stop program, if a single logline was not able to be delivered after $main_resend_attempts
+
 
  # EAA
  # Path to the EAA CLI Executable
@@ -33,13 +36,16 @@
 input_format_choices = ['JSON', 'TEXT'] # Available input format choices (need to be supported by cli)
 
  # OUTPUT Choices
-output_choices = ['TCP', 'HTTP', 'UDP', 'RAW'] # Definition of OUTPUT Choices
+output_choices = ['TCP', 'HTTP', 'UDP', 'RAW', 'FILE'] # Definition of OUTPUT Choices
+
+ # Transformation Choices
+transformation_choices = ['MCAS', 'JMESPATH']
+
 
  # LogLevels
 log_levels_available = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
 log_level_default = 'WARNING'
 
-
 # INPUT Configuration
 input_rerun_retries = 3 # Number of rerun attempts before giving up
 input_run_delay = 1 # Time in seconds to wait for the first health check
@@ -49,13 +55,23 @@
 # OUTPUT Configuration
 output_reconnect_retries = 10 # Number of reconnect attempts before giving up
 output_reconnect_delay = 1 # Time in seconds between reconnect attempts
+ ## TCP / UDP
 output_udp_send_buffer = 262144 # UDP Send buffer in bytes
 output_udp_timeout = 10.0 # UDP SEND / CONNECT Timeout (seconds)
 output_tcp_send_buffer = 262144 # TCP Send buffer in bytes
 output_tcp_timeout = 10.0 # TCP SEND / CONNECT Timeout (seconds)
- # Additional Headers to send (requests module KV pairs)
-output_http_header = {'User-Agent': f'{__tool_name_long__}/{__version__}'}
-
+ ## HTTP
+output_http_header = {'User-Agent': f'{__tool_name_long__}/{__version__}'} # HTTP Additional Headers to send (requests module KV pairs)
+ ## FILE
+output_file_encoding = "utf-8" # FILE Encoding setting
+output_file_handler_choices = ['SIZE', 'TIME'] # Available Choices for the file handler
+output_file_default_name = 'tmp/uls_file.output' # Default file name (path + name), only used if not set
+output_file_default_backup_count = 3 # Default number of backup files (after rotation)
+output_file_default_maxbytes = 50* 1024 * 1024 # Default maximum size of a file when rotated by the FILE - handler
+output_file_default_time_use_utc = False # Use UTC instead of local system time (Default: False)
+output_file_time_choices = ['S','M','H','D','W0','W1','W2','W3','W4','W5','W6','MIDNIGHT'] # Available choices for the time unit
+output_file_time_default = 'M' # Default value for the time unit (Minutes)
+output_file_time_interval = 30 # Default value for the interval (30)
 
 # Monitoring Configuration
 monitoring_enabled = True # Set to false to disable monitoring outputs
@@ -66,4 +82,5 @@
 edgerc_openapi = ["host", "client_token", "client_secret", "access_token"] # required fields for OPENAPI
 edgerc_eaa_legacy = ["eaa_api_host", "eaa_api_key", "eaa_api_secret"] # required for EAA - Legacy
 edgerc_mfa = ["mfa_integration_id", "mfa_signing_key"] # Required for MFA
-edgerc_documentation_url = "https:/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md"
+edgerc_documentation_url = "https:/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md"
+edgerc_mock_file = "ext/edgerc" # Required for display the version if no edgercfile was given

bin/config/transformation_config.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+
+# MCAS
+ ## Global
+mcas_name_long = "Microsoft Cloud Applican Security"
+mcas_name_short = "MCAS"
+
+ ## Requirements
+mcas_required_input = ['ETP']
+mcas_required_feed = ['PROXY', 'DNS']
+mcas_required_cliformat = ['JSON']
+
+ ## MCAS INPUT LOG FORMAT
+mcas_input_format = "detection_time={0} client_ip={1} destination_ip={2} domain={3} user_name={4} bytes_uploaded={5} bytes_downloaded={6} bytes_total={7} action={8}"
+
+
+# JMESPATH
+ ## Global
+jmespath_name_long = "JMESPath https://jmespath.org/"
+jmespath_name_short = "JMESPath"
+
+ ## Requirements
+jmespath_required_cliformat = ['JSON']

bin/modules/UlsArgsParser.py

@@ -70,9 +70,9 @@ def init():
  dest='inproxy',
  type=str,
  default=(os.environ.get('ULS_INPUT_PROXY') or None),
- help=argparse.SUPPRESS)
- # We're surpressing this for now, as the param does not seem to work (mschiess-20210818 - see EME-498)
- #help="Use a proxy Server for the INPUT requests (fetching data from AKAMAI API'S)")
+ help="Use a proxy Server for the INPUT requests (fetching data from AKAMAI API'S)")
+  #help=argparse.SUPPRESS)
+
  # RAWCMD
  input_group.add_argument('--rawcmd',
  action='store',
@@ -94,6 +94,21 @@ def init():
  default=(os.environ.get('ULS_SECTION') or 'default'),
  help="Credentials file Section's name to use ('default' if not specified).")
 
+ # Log Starttime
+ input_group.add_argument('--starttime',
+ action='store',
+ type=int,
+ dest="starttime",
+ default=(os.environ.get('ULS_STARTTIME') or None),
+ help="Start time (EPOCH SECONDS) from when to start getting logs ('default': cli_default (now), example: '1631556101')")
+ # Log Endtime
+ input_group.add_argument('--endtime',
+ action='store',
+ type=int,
+ dest="endtime",
+ default=(os.environ.get('ULS_ENDTIME') or None),
+ help="End time (EPOCH SECONDS) until when to stop getting logs ('default': cli_default (never), example: '1631556101')")
+
  # ----------------------
  # Output GROUP
  output_group = parser.add_argument_group(title="Output",
@@ -107,44 +122,39 @@ def init():
  choices=uls_config.output_choices,
  help="Select the Output Destination Default: None")
 
- # Output HOST
+ # TCP / UPD
+ ## Output HOST
  output_group.add_argument('--host',
  action='store',
  type=str,
  default=(os.environ.get('ULS_OUTPUT_HOST') or None),
  help="Host for TCP/UDP")
 
- # OUTPUT PORT
+ ## OUTPUT PORT
  output_group.add_argument('--port',
  action='store',
  type=int,
  default=int(os.environ.get('ULS_OUTPUT_PORT') or '0'),
  help="Port for TCP/UDP")
 
- # Output FILTER
- output_group.add_argument('--filter',
- action='store',
- type=str,
- default=(os.environ.get('ULS_OUTPUT_FILTER') or None),
- help="Filter (regex) to reduce number of sent log files (Only send lines that match the --filter argument).")
-
- # HTTP URL
+ # HTTP
+ ## HTTP URL
  output_group.add_argument('--httpurl',
  action='store',
  type=str,
  default=(os.environ.get('ULS_HTTP_URL') or None),
  help=f'Full http(s) target url i.e. '
  f'https://my.splunk.host:9091/services/collector/event"')
 
- # HTTP AUTH HEADER
+ ## HTTP AUTH HEADER
  output_group.add_argument('--httpauthheader',
  action='store',
  type=str,
  default=(os.environ.get('ULS_HTTP_AUTH_HEADER') or None),
  help='HTTP Header for authorization. Example: '
  '\'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}\'')
 
- # Disable HTTP TLS verification
+ ## Disable HTTP TLS verification
  output_group.add_argument('--httpinsecure',
  action='store',
  type=bool,
@@ -153,14 +163,98 @@ def init():
  const=True,
  help=f'Disable TLS CA Certificate verification. Default: False')
 
- # HTTP FORMAT DEFINITION
+ ## HTTP FORMAT DEFINITION
  output_group.add_argument('--httpformat',
  action='store',
  type=str,
  default=(os.environ.get('ULS_HTTP_FORMAT') or '{"event": %s}'),
  help='HTTP Message format expected by http receiver '
  '(%%s defines the data string). Default \'{\"event\": %%s}\'')
 
+ # FILE STUFF
+ ## File Handler
+ output_group.add_argument('--filehandler',
+ action='store',
+ type=str.upper,
+ default=(os.environ.get('ULS_FILE_HANDLER') or None),
+ choices=uls_config.output_file_handler_choices,
+ help=f"Output file handler - Decides when files are rotated -"
+ f"Choices: {uls_config.output_file_handler_choices} -"
+ f" Default: None")
+ ## File Name
+ output_group.add_argument('--filename',
+ action='store',
+ type=str,
+ default=(os.environ.get('ULS_FILE_NAME') or
+ uls_config.output_file_default_name),
+ help=f"Output file destination (path + filename)"
+ f" Default: {uls_config.output_file_default_name}")
+
+ ## File Backup count
+ output_group.add_argument('--filebackupcount',
+ action='store',
+ type=int,
+ default=(os.environ.get('ULS_FILE_BACKUPCOUNT') or
+ uls_config.output_file_default_backup_count),
+ help=f"Number of rotated files to keep (backup)"
+ f" Default: {uls_config.output_file_default_backup_count}")
+
+ ## File Max bytes
+ output_group.add_argument('--filemaxbytes',
+ action='store',
+ type=int,
+ default=(os.environ.get('ULS_FILE_MAXBYTES') or
+ uls_config.output_file_default_maxbytes),
+ help=f"Number of rotated files to keep (backup)"
+ f" Default: {uls_config.output_file_default_maxbytes} bytes")
+
+ ## File Time
+ output_group.add_argument('--filetime',
+ action='store',
+ type=str.upper,
+ default=(os.environ.get('ULS_FILE_TIME') or
+ uls_config.output_file_time_default),
+ choices=uls_config.output_file_time_choices,
+ help=f"Specifies the file rotation trigger unit "
+ f" Default: {uls_config.output_file_time_default}, Valid Choices: {uls_config.output_file_time_choices}")
+
+ ## File Time Interval
+ output_group.add_argument('--fileinterval',
+ action='store',
+ type=int,
+ default=(os.environ.get('ULS_FILE_INTERVAL') or
+ uls_config.output_file_time_interval),
+ help=f"Specifies the file rotation interval based on `--filetime` unit value"
+ f" Default: {uls_config.output_file_time_interval}")
+
+ # ----------------------
+ special_group = parser.add_argument_group(title="Transformation",
+ description="Define Module Settings (Output manipulation)")
+
+ # Output FILTER
+ special_group.add_argument('--filter',
+ action='store',
+ type=str,
+ default=(os.environ.get('ULS_OUTPUT_FILTER') or None),
+ help="Filter (regex) to reduce number of sent log files (Only send lines that match the --filter argument).")
+
+ # Transformation Handler
+ special_group.add_argument('--transformation',
+ action='store',
+ dest="transformation",
+ type=str.upper,
+ default=(os.environ.get('ULS_TRANSFORMATION') or None),
+ choices=uls_config.transformation_choices,
+ help="Select a transformation to manipulate the output format (optional)")
+
+ special_group.add_argument('--transformpattern', '--transformationpattern',
+ action='store',
+ dest="transformationpattern",
+ type=str,
+ default=(os.environ.get('ULS_TRANSFORMATION_PATTERN') or None),
+ help="Provide a pattern to transform the output (Required for JMESPATH)")
+
  return parser.parse_args()
 
+
 # EOF