diff --git a/playbooks/groups/os-control.yml b/playbooks/groups/os-control.yml index 7591108218..7126c4fdac 100644 --- a/playbooks/groups/os-control.yml +++ b/playbooks/groups/os-control.yml @@ -22,69 +22,7 @@ - ipa/client - rsyncd - sudo - - tasks: - - - name: Install rdiff-backup for backups - ansible.builtin.package: - name: rdiff-backup - state: present - - - name: Install unzip - ansible.builtin.package: - name: unzip - state: present - - - name: Install python3-kubernetes - ansible.builtin.package: - name: python3-kubernetes - state: present - - - name: Install tar - ansible.builtin.package: - name: tar - state: present - - - name: Install tar - ansible.builtin.package: - name: butane - state: present - - - name: Create the directories to hold the templates - ansible.builtin.file: - path: "/etc/openshift_apps" - state: directory - owner: root - group: root - mode: "0770" - - - name: Install helm if not exists - unarchive: - src: https://get.helm.sh/helm-v3.17.0-linux-amd64.tar.gz - dest: /usr/local/bin - extra_opts: "--strip-components=1" - owner: root - group: root - mode: 0755 - remote_src: true - args: - creates: /usr/local/bin/helm - - - name: Copy oc rpm - ansible.builtin.copy: - src: /srv/web/infra/bigfiles/openshiftboot/oc-client/oc-client.rpm - dest: /root/oc-client.rpm - - - name: Make sure oc-client is installed - dnf: - name: /root/oc-client.rpm - state: installed - - - name: Copy the jobs-summary script - ansible.builtin.copy: - src: "{{ files }}/scripts/jobs-summary" - dest: /usr/local/bin/jobs-summary - mode: "0755" + - openshift/control - import_tasks: "{{ tasks_path }}/yumrepos.yml" diff --git a/roles/openshift/control/files/check_oc_pods.sh b/roles/openshift/control/files/check_oc_pods.sh new file mode 100644 index 0000000000..972ae00b58 --- /dev/null +++ b/roles/openshift/control/files/check_oc_pods.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +declare -A counts=( + [Running]=0 [Pending]=0 [Completed]=0 [Error]=0 + [CrashLoopBackOff]=0 [ImagePullBackOff]=0 [ErrImagePull]=0 + [ContainerCreating]=0 [Terminating]=0 +) + +# Capture output and check for failure +if ! pod_states=$(/bin/oc get pods -A --no-headers 2>/dev/null | awk '{print $4}'); then + echo "Error: oc command failed" >&2 + exit 1 +fi + +while read -r state; do + if [[ -n "$state" ]]; then + counts[$state]=$((${counts[$state]:-0} + 1)) + fi +done <<< "$pod_states" + +for state in "${!counts[@]}"; do + printf "%-20s %d\n" "$state" "${counts[$state]}" +done | sort diff --git a/roles/openshift/control/files/template-pods.yml b/roles/openshift/control/files/template-pods.yml new file mode 100644 index 0000000000..b0349d6323 --- /dev/null +++ b/roles/openshift/control/files/template-pods.yml @@ -0,0 +1,206 @@ +zabbix_export: + version: '7.0' + template_groups: + - uuid: a333cbd6a3ad44baaa4eee4b0c0b1bec + name: Fedora + templates: + - uuid: bc3c328c9ad243bc83eac4994f255f9a + template: 'OCP Pod Status' + name: 'OCP Pod Status' + groups: + - name: Fedora + items: + - uuid: 6e01326e02a8402e992ef8fe5142ba48 + name: 'Count of Completed pods' + type: DEPENDENT + key: 'ocp.pods.state[Completed]' + delay: '0' + preprocessing: + - type: REGEX + parameters: + - 'Completed\s*([0-9]*)' + - \1 + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + tags: + - tag: pod-state + value: completed + - uuid: 8c5f4c715c4e4d0a8d44caa3f070fab7 + name: 'Count of ContainerCreating pods' + type: DEPENDENT + key: 'ocp.pods.state[ContainerCreating]' + delay: '0' + preprocessing: + - type: REGEX + parameters: + - 'ContainerCreating\s*([0-9]*)' + - \1 + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + tags: + - tag: pod-state + value: containercreating + - uuid: 60dbb94741d64e6e9c0ac4bb0baeb40f + name: 'Count of CrashLoopBackoff pods' + type: DEPENDENT + key: 'ocp.pods.state[CrashLoopBackoff]' + delay: '0' + preprocessing: + - type: REGEX + parameters: + - 'CrashLoopBackOff\s*([0-9]*)' + - \1 + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + tags: + - tag: pod-state + value: crashloopbackoff + triggers: + - uuid: b2d8f96bdcb74821ac58035905bbcca7 + expression: 'last(/OCP Pod Status/ocp.pods.state[CrashLoopBackoff]) > 0' + name: 'OCP - count of CrashLoopBackOff pods is not zero' + opdata: '{ITEM.LASTVALUE1} pods' + priority: AVERAGE + dependencies: + - name: 'Get OC pod state data failed' + expression: 'last(/OCP Pod Status/vfs.file.contents[/etc/zabbix/pod-states.log])="Error: oc command failed"' + tags: + - tag: scope + value: availability + - uuid: 123c9a38db9c4fdf908ec16311b02212 + name: 'Count of ErrImagePull pods' + type: DEPENDENT + key: 'ocp.pods.state[ErrImagePull]' + delay: '0' + preprocessing: + - type: REGEX + parameters: + - 'ErrImagePull\s*([0-9]*)' + - \1 + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + tags: + - tag: pod-state + value: errimagepull + - uuid: 4f5e89db683449debb00f55acbe9db7a + name: 'Count of Error pods' + type: DEPENDENT + key: 'ocp.pods.state[Error]' + delay: '0' + preprocessing: + - type: REGEX + parameters: + - 'Error\s*([0-9]*)' + - \1 + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + tags: + - tag: pod-state + value: error + - uuid: ecec2ceff6d64087b5b020a152b36b4b + name: 'Count of ImagePullBackOff pods' + type: DEPENDENT + key: 'ocp.pods.state[ImagePullBackOff]' + delay: '0' + preprocessing: + - type: REGEX + parameters: + - 'ImagePullBackOff\s*([0-9]*)' + - \1 + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + tags: + - tag: pod-state + value: imagepullbackoff + - uuid: d061af4e1733459985d7ce513e024a75 + name: 'Count of Pending pods' + type: DEPENDENT + key: 'ocp.pods.state[Pending]' + delay: '0' + preprocessing: + - type: REGEX + parameters: + - 'Pending\s*([0-9]*)' + - \1 + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + tags: + - tag: pod-state + value: pending + - uuid: 7e42ce10141d43e387747da50dfce261 + name: 'Count of Running pods' + type: DEPENDENT + key: 'ocp.pods.state[Running]' + delay: '0' + preprocessing: + - type: REGEX + parameters: + - 'Running\s*([0-9]*)' + - \1 + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + tags: + - tag: pod-state + value: running + - uuid: d0bff9f526a34f5f821547a81739d3d3 + name: 'Count of Terminating pods' + type: DEPENDENT + key: 'ocp.pods.state[Terminating]' + delay: '0' + preprocessing: + - type: REGEX + parameters: + - 'Terminating\s*([0-9]*)' + - \1 + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + tags: + - tag: pod-state + value: terminating + - uuid: bc99b4735c4144f99d9daaf9850c70f2 + name: 'Pod states - raw data' + type: ZABBIX_ACTIVE + key: 'vfs.file.contents[/etc/zabbix/pod-states.log]' + history: '0' + value_type: TEXT + trends: '0' + triggers: + - uuid: b40e34a56ff9419b8f1ddc995393e6c8 + expression: 'last(/OCP Pod Status/vfs.file.contents[/etc/zabbix/pod-states.log])="Error: oc command failed"' + name: 'Get OC pod state data failed' + priority: HIGH + tags: + - tag: application + value: openshift + - tag: scope + value: pods + tags: + - tag: application + value: openshift diff --git a/roles/openshift/control/tasks/main.yml b/roles/openshift/control/tasks/main.yml new file mode 100644 index 0000000000..16accac886 --- /dev/null +++ b/roles/openshift/control/tasks/main.yml @@ -0,0 +1,70 @@ +# Setup tasks for Openshift control nodes +--- + +- name: Install rdiff-backup for backups + ansible.builtin.package: + name: rdiff-backup + state: present + +- name: Install unzip + ansible.builtin.package: + name: unzip + state: present + +- name: Install python3-kubernetes + ansible.builtin.package: + name: python3-kubernetes + state: present + +- name: Install tar + ansible.builtin.package: + name: tar + state: present + +- name: Install tar + ansible.builtin.package: + name: butane + state: present + +- name: Create the directories to hold the templates + ansible.builtin.file: + path: "/etc/openshift_apps" + state: directory + owner: root + group: root + mode: "0770" + +- name: Install helm if not exists + unarchive: + src: https://get.helm.sh/helm-v3.17.0-linux-amd64.tar.gz + dest: /usr/local/bin + extra_opts: "--strip-components=1" + owner: root + group: root + mode: 0755 + remote_src: true + args: + creates: /usr/local/bin/helm + +- name: Copy oc rpm + ansible.builtin.copy: + src: /srv/web/infra/bigfiles/openshiftboot/oc-client/oc-client.rpm + dest: /root/oc-client.rpm + +- name: Make sure oc-client is installed + dnf: + name: /root/oc-client.rpm + state: installed + +- name: Copy the jobs-summary script + ansible.builtin.copy: + src: "{{ files }}/scripts/jobs-summary" + dest: /usr/local/bin/jobs-summary + mode: "0755" + +- name: Configure OCP pod monitoring in Zabbix + ansible.builtin.include_tasks: monitoring.yml + tags: + - os_control + - zabbix_agent + - zabbix_api diff --git a/roles/openshift/control/tasks/monitoring.yml b/roles/openshift/control/tasks/monitoring.yml new file mode 100644 index 0000000000..a9ea5728ea --- /dev/null +++ b/roles/openshift/control/tasks/monitoring.yml @@ -0,0 +1,52 @@ +--- +# Zabbix monitoring of the pods in OpenShift, via the oc CLI + +# We need a script to parse the output from oc get pods +- name: Install check_pods bash script + ansible.builtin.copy: + src: check_oc_pods.sh + dest: /usr/lib/zabbix/check_oc_pods.sh + owner: zabbix + group: zabbix + mode: '0755' + tags: + - os_control + - zabbix_agent + +# oc binary needs root's Kube config, so cron the +# output to where Zabbix can find it +- name: Set cron for check_pods + ansible.builtin.cron: + name: "Dump oc get pods as root for Zabbix" + minute: "*/5" + user: root + job: "/usr/lib/zabbix/check_oc_pods.sh > /etc/zabbix/pod-states.log 2>&1" + tags: + - os_control + - zabbix_agent + +- name: Zabbix API Block + vars: + ansible_zabbix_auth_key: "{{ zabbix_auth_key }}" + ansible_network_os: "{{ zabbix_network_os }}" + ansible_connection: "{{ zabbix_connection }}" + ansible_httpapi_port: "{{ zabbix_httpapi_port }}" + ansible_httpapi_use_ssl: "{{ zabbix_httpapi_use_ssl }}" + ansible_httpapi_validate_certs: "{{ zabbix_httpapi_validate_certs }}" + ansible_host: "{{ zabbix_server }}" + ansible_zabbix_url_path: "{{ zabbix_url_path }}" + tags: + - os_control + - zabbix_api + block: + - name: Import OCP pods template file + community.zabbix.zabbix_template: + template_yaml: "{{ lookup('file', 'template-pods.yml') }}" + state: present + run_once: true + + - name: Add self to OCP pods template in Zabbix + community.zabbix.zabbix_host: + host_name: "{{ inventory_hostname }}" + link_templates: OCP Pod Status + force: false