Merge pull request #506 from xgerman/mgd_k8s_lxc

K8S-381 Connect all LXC services (UI/ETS/etc) to RPC-O MaaS

playbooks/files/rax-maas/plugins/managed_k8s_services_local_check.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import datetime
+
+import ipaddr
+from maas_common import get_auth_ref
+from maas_common import metric
+from maas_common import metric_bool
+from maas_common import print_output
+from maas_common import status_err
+from maas_common import status_ok
+import requests
+
+
+def check(auth_ref, args):
+ if args.path:
+ path = '/{path}'.format(path=args.path)
+ else:
+ path = ''
+ endpoint = '{protocol}://{ip}:{port}/{path}'.format(
+ ip=args.ip,
+ protocol=args.protocol,
+ port=args.port,
+ path=path
+ )
+
+ service_name = 'maas_mk8s_{service}'.format(service=args.service)
+
+ try:
+ if args.protocol.upper() == 'HTTPS':
+ if args.certificate:
+ verify = args.certificate
+ else:
+ verify = False
+ else:
+ verify = None
+
+ # time something arbitrary
+ start = datetime.datetime.now()
+ r = requests.head(endpoint, verify=verify)
+ end = datetime.datetime.now()
+ api_is_up = (r.status_code == 200)
+ except (requests.HTTPError, requests.Timeout, requests.ConnectionError):
+ api_is_up = False
+ metric_bool('client_success', False, m_name=service_name)
+ # Any other exception presumably isn't an API error
+ except Exception as e:
+ metric_bool('client_success', False, m_name=service_name)
+ status_err(str(e), m_name=service_name)
+ else:
+ metric_bool('client_success', True, m_name=service_name)
+ dt = (end - start)
+ milliseconds = (dt.microseconds + dt.seconds * 10 ** 6) / 10 ** 3
+
+ status_ok(m_name=service_name)
+ metric_bool('mk8s_{service}_local_status'.format(service=args.service),
+ api_is_up, m_name=service_name)
+ if api_is_up:
+ # only want to send other metrics if api is up
+ metric('mk8s_{service}_local_response_time'.format(
+ service=args.service),
+ 'double', '%.3f' % milliseconds, 'ms')
+
+
+def main(args):
+ auth_ref = get_auth_ref()
+ check(auth_ref, args)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ description='Check Managed K8S service against local or'
+ 'remote address')
+ parser.add_argument('ip', nargs='?', type=ipaddr.IPv4Address,
+ help="Check Check Managed K8S service against "
+ " local or remote address")
+ parser.add_argument('--telegraf-output',
+ action='store_true',
+ default=False,
+ help='Set the output format to telegraf')
+ parser.add_argument('--port',
+ default='8889',
+ help='Port for the managed k8s service')
+ parser.add_argument('--protocol',
+ default='https',
+ help='Protocol used to contact the managed'
+ 'k8s service')
+ parser.add_argument('--certificate',
+ default=None,
+ help='Path to SSL certificate for managed'
+ 'k8s service')
+ parser.add_argument('--service',
+ default=None,
+ help='Name of the k8s service')
+ parser.add_argument('--path',
+ default=None,
+ help='Path after the endpoint')
+ args = parser.parse_args()
+ with print_output(print_telegraf=args.telegraf_output):
+ main(args)

playbooks/maas-managed-k8.yml

@@ -129,10 +129,10 @@
  when:
  - k8_config is defined
 
- - name: Install local checks
+ - name: Install k8sapi local checks
  template:
  src: "templates/rax-maas/managed_k8_api_local_check.yaml.j2"
- dest: "/etc/rackspace-monitoring-agent.conf.d/managed_k8_api_local_check-{{ item.clusters[0].name }}.yaml"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/managed_k8_api_local_check-{{ inventory_hostname }}-{{ item.clusters[0].name }}.yaml"
  owner: "root"
  group: "root"
  mode: "0644"
@@ -141,9 +141,145 @@
  - "{{ k8_config }}"
  when:
  - k8_config is defined
+ vars_files:
+ - vars/main.yml
+ - vars/maas-managed-k8.yml
+ tags:
+ - maas-managed-k8
+
+- name: Install checks for mk8s ui
+ hosts: "{{ mk8s_ui_hosts | default('mk8s_ui_all') }}"
+ gather_facts: false
+ tasks:
+ - name: Install mk8s ui process check
+ template:
+ src: "templates/rax-maas/mk8s_ui_process_check.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/mk8s_ui_process_check--{{ inventory_hostname }}.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+
+ - name: Install mk8s ui local checks
+ template:
+ src: "templates/rax-maas/managed_k8s_ui_local_check.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/managed_k8s_ui_local_check-{{ inventory_hostname }}.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+
+ - name: Install mk8s ui lb checks
+ template:
+ src: "templates/rax-maas/lb_ui_check_k8s.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/lb_ui_check_k8s.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+ when:
+ - maas_remote_check | bool
+ - not maas_private_monitoring_enabled
 
+ - name: Install mk8s ui private lb checks
+ template:
+ src: "templates/rax-maas/private_lb_ui_check_mk8s.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/private_lb_ui_check_mk8s.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+ when:
+ - maas_private_monitoring_enabled
+ - maas_private_monitoring_zone is defined
  vars_files:
  - vars/main.yml
  - vars/maas-managed-k8.yml
  tags:
  - maas-managed-k8
+
+- name: Install checks for mk8s etp
+ hosts: "{{ mk8s_etp_hosts | default('mk8s_etp_all') }}"
+ gather_facts: false
+ tasks:
+ - name: Install mk8s etp process check
+ template:
+ src: "templates/rax-maas/mk8s_etp_process_check.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/mk8s_etp_process_check--{{ inventory_hostname }}.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+
+ - name: Install mk8s etp local checks
+ template:
+ src: "templates/rax-maas/managed_k8s_etp_local_check.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/managed_k8s_etp_local_check-{{ inventory_hostname }}.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+ # Ideally ETP should not be accessible from the outside so skip those checks
+ vars_files:
+ - vars/main.yml
+ - vars/maas-managed-k8.yml
+ tags:
+ - maas-managed-k8
+
+- name: Install checks for mk8s etg
+ hosts: "mk8s_etg_all"
+ gather_facts: false
+ tasks:
+ - name: Install mk8s etg process check
+ template:
+ src: "templates/rax-maas/mk8s_etg_process_check.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/mk8s_etg_process_check--{{ inventory_hostname }}.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+
+ - name: Install mk8s etg local checks
+ template:
+ src: "templates/rax-maas/managed_k8s_etg_local_check.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/managed_k8s_etg_local_check-{{ inventory_hostname }}.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+ # Ideally etg should not be accessible from the outside so skip those checks
+ vars_files:
+ - vars/main.yml
+ - vars/maas-managed-k8.yml
+ tags:
+ - maas-managed-k8
+
+- name: Install checks for mk8s auth
+ hosts: "{{ mk8s_auth_hosts | default('mk8s_auth_all') }}"
+ gather_facts: false
+ tasks:
+ - name: Install mk8s auth process check
+ template:
+ src: "templates/rax-maas/mk8s_auth_process_check.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/mk8s_auth_process_check--{{ inventory_hostname }}.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+
+ - name: Install mk8s auth local checks
+ template:
+ src: "templates/rax-maas/managed_k8s_auth_local_check.yaml.j2"
+ dest: "/etc/rackspace-monitoring-agent.conf.d/managed_k8s_auth_local_check-{{ inventory_hostname }}.yaml"
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ delegate_to: "{{ physical_host | default(ansible_host) }}"
+ # Ideally auth should not be accessible from the outside so skip those checks
+ vars_files:
+ - vars/main.yml
+ - vars/maas-managed-k8.yml
+ tags:
+ - maas-managed-k8
+
+

playbooks/templates/rax-maas/lb_ui_check_k8s.yaml.j2

@@ -0,0 +1,29 @@
+{% from "templates/common/macros.jinja" import get_metadata with context %}
+{% set label = "lb_ui_check_k8s" %}
+{% set check_name = label+'--'+maas_lb_name %}
+type : remote.http
+label : "{{ check_name }}"
+period : "{{ maas_check_period_override[label] | default(maas_check_period) }}"
+timeout : "{{ maas_check_timeout_override[label] | default(maas_check_timeout) }}"
+disabled : "{{ (inventory_hostname != groups['mk8s_ui_all'][0] or check_name | match(maas_excluded_checks_regex)) | ternary('true', 'false') }}"
+target_resolver : "IPv4"
+target_hostname : "{{ maas_external_ip_address }}"
+details :
+ url : "{{ maas_k8s_ui_scheme | default(maas_scheme)}}://{{ maas_external_hostname }}:{{ mk8s_ui_port_lb }}"
+ method : "HEAD"
+monitoring_zones_poll:
+{% for zone in maas_monitoring_zones %}
+ - {{ zone }}
+{% endfor %}
+{{ get_metadata(label).strip() }}
+{# Add extra metadata options with two leading white spaces #}
+alarms :
+ lb_api_alarm_mk8s_ui :
+ label : lb_api_alarm_mk8s_ui
+ notification_plan_id: "{{ maas_notification_plan_override[label] | default(maas_notification_plan) }}"
+ disabled : {{ ('lb_api_alarm_mk8s_ui' | match(maas_excluded_alarms_regex)) | ternary('true', 'false') }}
+ criteria : |
+ :set consecutiveCount={{ maas_alarm_local_consecutive_count }}
+ if (metric['code'] != '200') {
+ return new AlarmStatus(CRITICAL, 'UI unavailable.');
+ }

playbooks/templates/rax-maas/managed_k8s_auth_local_check.yaml.j2

@@ -0,0 +1,24 @@
+{% from "templates/common/macros.jinja" import get_metadata with context %}
+{% set label = "mk8s_auth_local_check" %}
+{% set check_name = label+'--'+inventory_hostname %}
+type : agent.plugin
+label : "{{ check_name }}"
+period : "{{ maas_check_period_override[label] | default(maas_check_period) }}"
+timeout : "{{ maas_check_timeout_override[label] | default(maas_check_timeout) }}"
+disabled : "{{ (check_name | match(maas_excluded_checks_regex)) | ternary('true', 'false') }}"
+details :
+ file : run_plugin_in_venv.sh
+ args : ["{{ maas_plugin_dir }}/managed_k8s_service_local_check.py", "{{ ansible_host }}", "--port", "{{ mk8s_auth_port }}", "--protocol", "{{ mk8s_auth_protocol }}", "--service auth", "--path healthcheck"]
+ timeout : {{ (maas_check_timeout_override[label] | default(maas_check_timeout) * 1000) }}
+{{ get_metadata(label).strip() }}
+{# Add extra metadata options with two leading white spaces #}
+alarms :
+ mk8s_auth_local_status :
+ label : mk8s_auth_local_status-{{ inventory_hostname }}
+ notification_plan_id : "{{ maas_notification_plan_override[label] | default(maas_notification_plan) }}"
+ disabled : {{ (('mk8s_auth_local_status-'+inventory_hostname) | match(maas_excluded_alarms_regex)) | ternary('true', 'false') }}
+ criteria : |
+ :set consecutiveCount={{ maas_alarm_local_consecutive_count }}
+ if (metric["mk8s_auth_local_status"] != 1) {
+ return new AlarmStatus(CRITICAL, "Managed K8S auth unavailable");
+ }

playbooks/templates/rax-maas/managed_k8s_etg_local_check.yaml.j2

@@ -0,0 +1,24 @@
+{% from "templates/common/macros.jinja" import get_metadata with context %}
+{% set label = "managed_k8s_etg_local_check" %}
+{% set check_name = label+'--'+inventory_hostname %}
+type : agent.plugin
+label : "{{ check_name }}"
+period : "{{ maas_check_period_override[label] | default(maas_check_period) }}"
+timeout : "{{ maas_check_timeout_override[label] | default(maas_check_timeout) }}"
+disabled : "{{ (check_name | match(maas_excluded_checks_regex)) | ternary('true', 'false') }}"
+details :
+ file : run_plugin_in_venv.sh
+ args : ["{{ maas_plugin_dir }}/managed_k8s_service_local_check.py", "{{ ansible_host }}", "--port", "{{ mk8s_etg_port }}", "--protocol", "{{ mk8s_etg_protocol }}", "--service etg", "--path healthcheck"]
+ timeout : {{ (maas_check_timeout_override[label] | default(maas_check_timeout) * 1000) }}
+{{ get_metadata(label).strip() }}
+{# Add extra metadata options with two leading white spaces #}
+alarms :
+ managed_k8s_etg_local_status :
+ label : managed_k8s_etg_local_status--{{ inventory_hostname }}
+ notification_plan_id : "{{ maas_notification_plan_override[label] | default(maas_notification_plan) }}"
+ disabled : {{ (('managed_k8s_etg_local_status--'+inventory_hostname) | match(maas_excluded_alarms_regex)) | ternary('true', 'false') }}
+ criteria : |
+ :set consecutiveCount={{ maas_alarm_local_consecutive_count }}
+ if (metric["managed_k8s_etg_local_status"] != 1) {
+ return new AlarmStatus(CRITICAL, "Managed K8S etg unavailable");
+ }

playbooks/templates/rax-maas/managed_k8s_etp_local_check.yaml.j2

@@ -0,0 +1,24 @@
+{% from "templates/common/macros.jinja" import get_metadata with context %}
+{% set label = "managed_k8s_etp_local_check" %}
+{% set check_name = label+'--'+inventory_hostname %}
+type : agent.plugin
+label : "{{ check_name }}"
+period : "{{ maas_check_period_override[label] | default(maas_check_period) }}"
+timeout : "{{ maas_check_timeout_override[label] | default(maas_check_timeout) }}"
+disabled : "{{ (check_name | match(maas_excluded_checks_regex)) | ternary('true', 'false') }}"
+details :
+ file : run_plugin_in_venv.sh
+ args : ["{{ maas_plugin_dir }}/managed_k8s_service_local_check.py", "{{ ansible_host }}", "--port", "{{ mk8s_etp_port }}", "--protocol", "{{ mk8s_etp_protocol }}", "--service etp", "--path healthcheck"]
+ timeout : {{ (maas_check_timeout_override[label] | default(maas_check_timeout) * 1000) }}
+{{ get_metadata(label).strip() }}
+{# Add extra metadata options with two leading white spaces #}
+alarms :
+ managed_k8s_etp_local_status :
+ label : managed_k8s_etp_local_status--{{ inventory_hostname }}
+ notification_plan_id : "{{ maas_notification_plan_override[label] | default(maas_notification_plan) }}"
+ disabled : {{ (('managed_k8s_etp_local_status--'+inventory_hostname) | match(maas_excluded_alarms_regex)) | ternary('true', 'false') }}
+ criteria : |
+ :set consecutiveCount={{ maas_alarm_local_consecutive_count }}
+ if (metric["managed_k8s_etp_local_status"] != 1) {
+ return new AlarmStatus(CRITICAL, "Managed K8S ETP unavailable");
+ }