diff --git a/core/opl/shovel.py b/core/opl/shovel.py deleted file mode 100755 index 405d709..0000000 --- a/core/opl/shovel.py +++ /dev/null @@ -1,177 +0,0 @@ -import argparse -import logging - -from . import skelet - - -class pluginProw: - def __init__(self, args): - print("Hello from pluginProw init") - self.logger = logging.getLogger("opl.showel.pluginProw") - - def list(self): - print("Hello from pluginProw list") - - def download(self): - print("Hello from pluginProw download") - - @staticmethod - def args(parser, group_actions): - group_actions.add_argument( - "--prow-list", - dest="actions", - default=[], - action="append_const", - const=("prow", "list"), - help="List runs for specific Prow run", - ) - group_actions.add_argument( - "--prow-download", - dest="actions", - default=[], - action="append_const", - const=("prow", "download"), - help="Download file from Prow run artifacts", - ) - - group = parser.add_argument_group( - title="prow", - description="Options needed to work with Prow", - ) - group.add_argument( - "--prow-base-url", - default="https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/origin-ci-test/logs/", - help="Base URL", - ) - group.add_argument( - "--prow-job-name", - default="periodic-ci-redhat-appstudio-e2e-tests-load-test-ci-daily-10u-10t", - help="Job name as available in ci-operator/jobs/...", - ) - group.add_argument( - "--prow-test-name", - default="load-test-ci-daily-10u-10t", - help="Test name as configured in ci-operator/config/...", - ) - group.add_argument( - "--prow-artifact-path", - default="redhat-appstudio-load-test/artifacts/load-tests.json", - help="Path to the artifact", - ) - - -class pluginOpenSearch: - def __init__(self, args): - print("Hello from pluginOpenSearch init") - self.logger = logging.getLogger("opl.showel.pluginOpenSearch") - - def upload(self): - print("Hello from pluginOpenSearch upload") - - @staticmethod - def args(parser, group_actions): - group_actions.add_argument( - "--opensearch-upload", - dest="actions", - default=[], - action="append_const", - const=("opensearch", "upload"), - help="Upload file to OpenSearch if not already there", - ) - - group = parser.add_argument_group( - title="opensearch", - description="Options needed to work with OpenSearch", - ) - - -class pluginHorreum: - def __init__(self, args): - print("Hello from pluginHorreum init") - self.logger = logging.getLogger("opl.showel.pluginHorreum") - - def upload(self): - print("Hello from pluginHorreum upload") - - def result(self): - print("Hello from pluginHorreum result") - - @staticmethod - def args(parser, group_actions): - group_actions.add_argument( - "--horreum-upload", - dest="actions", - default=[], - action="append_const", - const=("horreum", "upload"), - help="Upload file to Horreum if not already there", - ) - group_actions.add_argument( - "--horreum-result", - dest="actions", - default=[], - action="append_const", - const=("horreum", "result"), - help="Get Horreum no-/change signal for a given time range", - ) - - group = parser.add_argument_group( - title="horreum", - description="Options needed to work with Horreum", - ) - - -class pluginResultsDashboard: - def __init__(self, args): - print("Hello from pluginResultsDashboard init") - self.logger = logging.getLogger("opl.showel.pluginResultsDashboard") - - def upload(self): - print("Hello from pluginResultsDashboard upload") - - @staticmethod - def args(parser, group_actions): - group_actions.add_argument( - "--resultsdashboard-upload", - dest="actions", - default=[], - action="append_const", - const=("resultsdashboard", "upload"), - help="Upload file to Results Dashboard if not already there", - ) - - group = parser.add_argument_group( - title="resultsdashboard", - description="Options needed to work with Results Dashboard", - ) - - -PLUGINS = { - "prow": pluginProw, - "opensearch": pluginOpenSearch, - "horreum": pluginHorreum, - "resultsdashboard": pluginResultsDashboard, -} - - -def main(): - parser = argparse.ArgumentParser( - description="Shovel data from A to B", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - group_actions = parser.add_argument_group( - title="actions", - description="Various high level things you can do", - ) - for name, plugin in PLUGINS.items(): - plugin.args(parser, group_actions) - - with skelet.test_setup(parser) as (args, status_data): - logger = logging.getLogger("main") - for plugin_name, function_name in args.actions: - logger.info( - f"Instantiating plugin {plugin_name} for function {function_name}" - ) - plugin_object = PLUGINS[plugin_name] - plugin_instance = plugin_object(args) - getattr(plugin_instance, function_name)() diff --git a/opl/http.py b/opl/http.py index 3e20a32..3e8ce3c 100644 --- a/opl/http.py +++ b/opl/http.py @@ -8,6 +8,12 @@ session = requests.Session() +def insecure(): + session.verify = False + logging.debug("Disabling SSL verifications for this session") + disable_insecure_request_warnings(True) + + def disable_insecure_request_warnings(disable_it): if disable_it: logging.debug("Disabling insecure request warnings") diff --git a/opl/investigator/config.py b/opl/investigator/config.py index b163441..c3c49ab 100644 --- a/opl/investigator/config.py +++ b/opl/investigator/config.py @@ -75,6 +75,15 @@ def load_config(conf, fp): assert not conf.history_es_server.endswith("/") conf.history_es_index = data["history"]["es_index"] conf.history_es_query = data["history"]["es_query"] + if "es_server_user" in data["history"]: + conf.history_es_server_user = data["history"]["es_server_user"] + conf.history_es_server_pass_env_var = data["history"][ + "es_server_pass_env_var" + ] + if "es_server_verify" in data["history"]: + conf.history_es_server_verify = data["history"]["es_server_verify"] + else: + conf.history_es_server_verify = True if conf.history_type == "sd_dir": conf.history_dir = data["history"]["dir"] @@ -88,6 +97,16 @@ def load_config(conf, fp): conf.decisions_es_server = data["decisions"]["es_server"] assert not conf.decisions_es_server.endswith("/") conf.decisions_es_index = data["decisions"]["es_index"] + if "es_server_user" in data["decisions"]: + conf.decisions_es_server_user = data["decisions"]["es_server_user"] + conf.decisions_es_server_pass_env_var = data["decisions"][ + "es_server_pass_env_var" + ] + if "es_server_verify" in data["decisions"]: + conf.decisions_es_server_verify = data["decisions"]["es_server_verify"] + else: + conf.decisions_es_server_verify = True + if conf.decisions_type == "csv": conf.decisions_filename = data["decisions"]["filename"] diff --git a/opl/investigator/elasticsearch_decisions.py b/opl/investigator/elasticsearch_decisions.py index b4bba77..a8d3e57 100644 --- a/opl/investigator/elasticsearch_decisions.py +++ b/opl/investigator/elasticsearch_decisions.py @@ -6,7 +6,9 @@ import requests -def store(server, index, decisions): +def store(server, index, decisions, **kwargs): + es_server_user = kwargs.get("es_server_user") + decisions_es_server_pass_env_var = kwargs.get("es_server_pass_env_var") # This is our workaround on how to add additional metadata about the decision job_name = os.environ.get("JOB_NAME", "") build_url = os.environ.get("BUILD_URL", "") @@ -26,7 +28,17 @@ def store(server, index, decisions): f"Storing decision to ES url={url}, headers={headers} and json={json.dumps(decision)}" ) - response = requests.post(url, headers=headers, json=decision) + if es_server_user and decisions_es_server_pass_env_var: + # fetch the password from Jenkins credentials + open_search_password = os.environ.get(decisions_es_server_pass_env_var) + response = requests.post( + url, + auth=requests.auth.HTTPBasicAuth(es_server_user, open_search_password), + headers=headers, + json=decision, + ) + else: + response = requests.post(url, headers=headers, json=decision) if not response.ok: logging.warning(f"Failed to store decision to ES: {response.text}") diff --git a/opl/investigator/elasticsearch_loader.py b/opl/investigator/elasticsearch_loader.py index 5c76c17..f2b6d60 100644 --- a/opl/investigator/elasticsearch_loader.py +++ b/opl/investigator/elasticsearch_loader.py @@ -2,11 +2,16 @@ import logging import tempfile +import os import opl.http import opl.status_data +from requests.auth import HTTPBasicAuth -def load(server, index, query, paths): +def load(server, index, query, paths, **kwargs): + es_server_user = kwargs.get("es_server_user") + es_server_pass_env_var = kwargs.get("es_server_pass_env_var") + out = {} for path in paths: @@ -21,7 +26,17 @@ def load(server, index, query, paths): f"Querying ES with url={url}, headers={headers} and json={json.dumps(data)}" ) - response = opl.http.get(url, headers=headers, json=data) + if es_server_user and es_server_pass_env_var: + # fetch the password from Jenkins credentials + open_search_password = os.environ.get(es_server_pass_env_var) + response = opl.http.get( + url, + auth=HTTPBasicAuth(es_server_user, open_search_password), + headers=headers, + json=data, + ) + else: + response = opl.http.get(url, headers=headers, json=data) for item in response["hits"]["hits"]: logging.debug( diff --git a/opl/pass_or_fail.py b/opl/pass_or_fail.py index 0be5e71..a52b116 100755 --- a/opl/pass_or_fail.py +++ b/opl/pass_or_fail.py @@ -114,12 +114,23 @@ def main(): if args.history_type == "csv": history = opl.investigator.csv_loader.load(args.history_file, args.sets) elif args.history_type == "elasticsearch": + if ( + hasattr(args, "history_es_server_verify") + and not args.history_es_server_verify + ): + # SSL verification is disabled by default + opl.http.insecure() history = opl.investigator.elasticsearch_loader.load( args.history_es_server, args.history_es_index, args.history_es_query, args.sets, + es_server_user=getattr(args, "history_es_server_user", None), + es_server_pass_env_var=getattr( + args, "history_es_server_pass_env_var", None + ), ) + elif args.history_type == "sd_dir": history = opl.investigator.sd_dir_loader.load( args.history_dir, args.history_matchers, args.sets @@ -200,8 +211,17 @@ def main(): if not args.dry_run: for d_type in args.decisions_type: if d_type == "elasticsearch": + if hasattr(args, "es_server_verify") and not args.es_server_verify: + # disable SSL verification + opl.http.insecure() opl.investigator.elasticsearch_decisions.store( - args.decisions_es_server, args.decisions_es_index, info_all + args.decisions_es_server, + args.decisions_es_index, + info_all, + es_server_user=getattr(args, "decisions_es_server_user", None), + es_server_pass_env_var=getattr( + args, "decisions_es_server_pass_env_var", None + ), ) if d_type == "csv": opl.investigator.csv_decisions.store(args.decisions_filename, info_all)