OpenShift: Refactor os-control playbook to a role and add monitoring

Signed-off-by: Greg Sutcliffe <fedora@emeraldreverie.org>

playbooks/groups/os-control.yml

@@ -22,69 +22,7 @@
     - ipa/client
     - rsyncd
     - sudo
-
-  tasks:
-
-  - name: Install rdiff-backup for backups
-    ansible.builtin.package:
-      name: rdiff-backup
-      state: present
-
-  - name: Install unzip
-    ansible.builtin.package:
-      name: unzip
-      state: present
-
-  - name: Install python3-kubernetes
-    ansible.builtin.package:
-      name: python3-kubernetes
-      state: present
-
-  - name: Install tar
-    ansible.builtin.package:
-      name: tar
-      state: present
-
-  - name: Install tar
-    ansible.builtin.package:
-      name: butane
-      state: present
-
-  - name: Create the directories to hold the templates
-    ansible.builtin.file:
-      path: "/etc/openshift_apps"
-      state: directory
-      owner: root
-      group: root
-      mode: "0770"
-
-  - name: Install helm if not exists
-    unarchive:
-        src: https://get.helm.sh/helm-v3.17.0-linux-amd64.tar.gz
-        dest: /usr/local/bin
-        extra_opts: "--strip-components=1"
-        owner: root
-        group: root
-        mode: 0755
-        remote_src: true
-    args:
-        creates: /usr/local/bin/helm
-
-  - name: Copy oc rpm
-    ansible.builtin.copy:
-      src: /srv/web/infra/bigfiles/openshiftboot/oc-client/oc-client.rpm
-      dest: /root/oc-client.rpm
-
-  - name: Make sure oc-client is installed
-    dnf:
-      name: /root/oc-client.rpm
-      state: installed
-
-  - name: Copy the jobs-summary script
-    ansible.builtin.copy:
-      src: "{{ files }}/scripts/jobs-summary"
-      dest: /usr/local/bin/jobs-summary
-      mode: "0755"
+    - openshift/control
 
   - import_tasks: "{{ tasks_path }}/yumrepos.yml"
 

roles/openshift/control/files/check_oc_pods.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euo pipefail
+
+declare -A counts=(
+  [Running]=0 [Pending]=0 [Completed]=0 [Error]=0
+  [CrashLoopBackOff]=0 [ImagePullBackOff]=0 [ErrImagePull]=0
+  [ContainerCreating]=0 [Terminating]=0
+)
+
+# Capture output and check for failure
+if ! pod_states=$(/bin/oc get pods -A --no-headers 2>/dev/null | awk '{print $4}'); then
+  echo "Error: oc command failed" >&2
+  exit 1
+fi
+
+while read -r state; do
+  if [[ -n "$state" ]]; then
+    counts[$state]=$((${counts[$state]:-0} + 1))
+  fi
+done <<< "$pod_states"
+
+for state in "${!counts[@]}"; do
+  printf "%-20s %d\n" "$state" "${counts[$state]}"
+done | sort

roles/openshift/control/files/template-pods.yml

@@ -0,0 +1,206 @@
+zabbix_export:
+  version: '7.0'
+  template_groups:
+    - uuid: a333cbd6a3ad44baaa4eee4b0c0b1bec
+      name: Fedora
+  templates:
+    - uuid: bc3c328c9ad243bc83eac4994f255f9a
+      template: 'OCP Pod Status'
+      name: 'OCP Pod Status'
+      groups:
+        - name: Fedora
+      items:
+        - uuid: 6e01326e02a8402e992ef8fe5142ba48
+          name: 'Count of Completed pods'
+          type: DEPENDENT
+          key: 'ocp.pods.state[Completed]'
+          delay: '0'
+          preprocessing:
+            - type: REGEX
+              parameters:
+                - 'Completed\s*([0-9]*)'
+                - \1
+            - type: DISCARD_UNCHANGED_HEARTBEAT
+              parameters:
+                - 1h
+          master_item:
+            key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          tags:
+            - tag: pod-state
+              value: completed
+        - uuid: 8c5f4c715c4e4d0a8d44caa3f070fab7
+          name: 'Count of ContainerCreating pods'
+          type: DEPENDENT
+          key: 'ocp.pods.state[ContainerCreating]'
+          delay: '0'
+          preprocessing:
+            - type: REGEX
+              parameters:
+                - 'ContainerCreating\s*([0-9]*)'
+                - \1
+            - type: DISCARD_UNCHANGED_HEARTBEAT
+              parameters:
+                - 1h
+          master_item:
+            key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          tags:
+            - tag: pod-state
+              value: containercreating
+        - uuid: 60dbb94741d64e6e9c0ac4bb0baeb40f
+          name: 'Count of CrashLoopBackoff pods'
+          type: DEPENDENT
+          key: 'ocp.pods.state[CrashLoopBackoff]'
+          delay: '0'
+          preprocessing:
+            - type: REGEX
+              parameters:
+                - 'CrashLoopBackOff\s*([0-9]*)'
+                - \1
+            - type: DISCARD_UNCHANGED_HEARTBEAT
+              parameters:
+                - 1h
+          master_item:
+            key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          tags:
+            - tag: pod-state
+              value: crashloopbackoff
+          triggers:
+            - uuid: b2d8f96bdcb74821ac58035905bbcca7
+              expression: 'last(/OCP Pod Status/ocp.pods.state[CrashLoopBackoff]) > 0'
+              name: 'OCP - count of CrashLoopBackOff pods is not zero'
+              opdata: '{ITEM.LASTVALUE1} pods'
+              priority: AVERAGE
+              dependencies:
+                - name: 'Get OC pod state data failed'
+                  expression: 'last(/OCP Pod Status/vfs.file.contents[/etc/zabbix/pod-states.log])="Error: oc command failed"'
+              tags:
+                - tag: scope
+                  value: availability
+        - uuid: 123c9a38db9c4fdf908ec16311b02212
+          name: 'Count of ErrImagePull pods'
+          type: DEPENDENT
+          key: 'ocp.pods.state[ErrImagePull]'
+          delay: '0'
+          preprocessing:
+            - type: REGEX
+              parameters:
+                - 'ErrImagePull\s*([0-9]*)'
+                - \1
+            - type: DISCARD_UNCHANGED_HEARTBEAT
+              parameters:
+                - 1h
+          master_item:
+            key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          tags:
+            - tag: pod-state
+              value: errimagepull
+        - uuid: 4f5e89db683449debb00f55acbe9db7a
+          name: 'Count of Error pods'
+          type: DEPENDENT
+          key: 'ocp.pods.state[Error]'
+          delay: '0'
+          preprocessing:
+            - type: REGEX
+              parameters:
+                - 'Error\s*([0-9]*)'
+                - \1
+            - type: DISCARD_UNCHANGED_HEARTBEAT
+              parameters:
+                - 1h
+          master_item:
+            key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          tags:
+            - tag: pod-state
+              value: error
+        - uuid: ecec2ceff6d64087b5b020a152b36b4b
+          name: 'Count of ImagePullBackOff pods'
+          type: DEPENDENT
+          key: 'ocp.pods.state[ImagePullBackOff]'
+          delay: '0'
+          preprocessing:
+            - type: REGEX
+              parameters:
+                - 'ImagePullBackOff\s*([0-9]*)'
+                - \1
+            - type: DISCARD_UNCHANGED_HEARTBEAT
+              parameters:
+                - 1h
+          master_item:
+            key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          tags:
+            - tag: pod-state
+              value: imagepullbackoff
+        - uuid: d061af4e1733459985d7ce513e024a75
+          name: 'Count of Pending pods'
+          type: DEPENDENT
+          key: 'ocp.pods.state[Pending]'
+          delay: '0'
+          preprocessing:
+            - type: REGEX
+              parameters:
+                - 'Pending\s*([0-9]*)'
+                - \1
+            - type: DISCARD_UNCHANGED_HEARTBEAT
+              parameters:
+                - 1h
+          master_item:
+            key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          tags:
+            - tag: pod-state
+              value: pending
+        - uuid: 7e42ce10141d43e387747da50dfce261
+          name: 'Count of Running pods'
+          type: DEPENDENT
+          key: 'ocp.pods.state[Running]'
+          delay: '0'
+          preprocessing:
+            - type: REGEX
+              parameters:
+                - 'Running\s*([0-9]*)'
+                - \1
+            - type: DISCARD_UNCHANGED_HEARTBEAT
+              parameters:
+                - 1h
+          master_item:
+            key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          tags:
+            - tag: pod-state
+              value: running
+        - uuid: d0bff9f526a34f5f821547a81739d3d3
+          name: 'Count of Terminating pods'
+          type: DEPENDENT
+          key: 'ocp.pods.state[Terminating]'
+          delay: '0'
+          preprocessing:
+            - type: REGEX
+              parameters:
+                - 'Terminating\s*([0-9]*)'
+                - \1
+            - type: DISCARD_UNCHANGED_HEARTBEAT
+              parameters:
+                - 1h
+          master_item:
+            key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          tags:
+            - tag: pod-state
+              value: terminating
+        - uuid: bc99b4735c4144f99d9daaf9850c70f2
+          name: 'Pod states - raw data'
+          type: ZABBIX_ACTIVE
+          key: 'vfs.file.contents[/etc/zabbix/pod-states.log]'
+          history: '0'
+          value_type: TEXT
+          trends: '0'
+          triggers:
+            - uuid: b40e34a56ff9419b8f1ddc995393e6c8
+              expression: 'last(/OCP Pod Status/vfs.file.contents[/etc/zabbix/pod-states.log])="Error: oc command failed"'
+              name: 'Get OC pod state data failed'
+              priority: HIGH
+              tags:
+                - tag: application
+                  value: openshift
+                - tag: scope
+                  value: pods
+      tags:
+        - tag: application
+          value: openshift

roles/openshift/control/tasks/main.yml

@@ -0,0 +1,70 @@
+# Setup tasks for Openshift control nodes
+---
+
+- name: Install rdiff-backup for backups
+  ansible.builtin.package:
+    name: rdiff-backup
+    state: present
+
+- name: Install unzip
+  ansible.builtin.package:
+    name: unzip
+    state: present
+
+- name: Install python3-kubernetes
+  ansible.builtin.package:
+    name: python3-kubernetes
+    state: present
+
+- name: Install tar
+  ansible.builtin.package:
+    name: tar
+    state: present
+
+- name: Install tar
+  ansible.builtin.package:
+    name: butane
+    state: present
+
+- name: Create the directories to hold the templates
+  ansible.builtin.file:
+    path: "/etc/openshift_apps"
+    state: directory
+    owner: root
+    group: root
+    mode: "0770"
+
+- name: Install helm if not exists
+  unarchive:
+      src: https://get.helm.sh/helm-v3.17.0-linux-amd64.tar.gz
+      dest: /usr/local/bin
+      extra_opts: "--strip-components=1"
+      owner: root
+      group: root
+      mode: 0755
+      remote_src: true
+  args:
+      creates: /usr/local/bin/helm
+
+- name: Copy oc rpm
+  ansible.builtin.copy:
+    src: /srv/web/infra/bigfiles/openshiftboot/oc-client/oc-client.rpm
+    dest: /root/oc-client.rpm
+
+- name: Make sure oc-client is installed
+  dnf:
+    name: /root/oc-client.rpm
+    state: installed
+
+- name: Copy the jobs-summary script
+  ansible.builtin.copy:
+    src: "{{ files }}/scripts/jobs-summary"
+    dest: /usr/local/bin/jobs-summary
+    mode: "0755"
+
+- name: Configure OCP pod monitoring in Zabbix
+  ansible.builtin.include_tasks: monitoring.yml
+  tags:
+    - os_control
+    - zabbix_agent
+    - zabbix_api

roles/openshift/control/tasks/monitoring.yml

@@ -0,0 +1,52 @@
+---
+# Zabbix monitoring of the pods in OpenShift, via the oc CLI
+
+# We need a script to parse the output from oc get pods
+- name: Install check_pods bash script
+  ansible.builtin.copy:
+    src: check_oc_pods.sh
+    dest: /usr/lib/zabbix/check_oc_pods.sh
+    owner: zabbix
+    group: zabbix
+    mode: '0755'
+  tags:
+    - os_control
+    - zabbix_agent
+
+# oc binary needs root's Kube config, so cron the
+# output to where Zabbix can find it
+- name: Set cron for check_pods
+  ansible.builtin.cron:
+    name: "Dump oc get pods as root for Zabbix"
+    minute: "*/5"
+    user: root
+    job: "/usr/lib/zabbix/check_oc_pods.sh > /etc/zabbix/pod-states.log 2>&1"
+  tags:
+    - os_control
+    - zabbix_agent
+
+- name: Zabbix API Block
+  vars:
+    ansible_zabbix_auth_key: "{{ zabbix_auth_key }}"
+    ansible_network_os: "{{ zabbix_network_os }}"
+    ansible_connection: "{{ zabbix_connection }}"
+    ansible_httpapi_port: "{{ zabbix_httpapi_port }}"
+    ansible_httpapi_use_ssl: "{{ zabbix_httpapi_use_ssl }}"
+    ansible_httpapi_validate_certs: "{{ zabbix_httpapi_validate_certs }}"
+    ansible_host: "{{ zabbix_server }}"
+    ansible_zabbix_url_path: "{{ zabbix_url_path }}"
+  tags:
+    - os_control
+    - zabbix_api
+  block:
+    - name: Import OCP pods template file
+      community.zabbix.zabbix_template:
+        template_yaml: "{{ lookup('file', 'template-pods.yml') }}"
+        state: present
+      run_once: true
+
+    - name: Add self to OCP pods template in Zabbix
+      community.zabbix.zabbix_host:
+        host_name: "{{ inventory_hostname }}"
+        link_templates: OCP Pod Status
+        force: false