Zabbix: migrate still-in-use Nagios plugins to the relevant roles

Signed-off-by: Greg Sutcliffe <fedora@emeraldreverie.org>

roles/base/files/postfix/nagios-plugin.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python3
+
+import subprocess
+import json
+import argparse
+import sys
+
+from datetime import datetime
+
+parser = argparse.ArgumentParser()
+parser.add_argument('domain', help="Required. Domain to check")
+parser.add_argument('-c', '--critical', dest='critical', type=int, default=50,
+                    help="Critical threshold")
+parser.add_argument('-w', '--warning', dest='warning', type=int, default=20,
+                    help="Warning threshold")
+parser.add_argument('-L', '--lower', dest='lower', type=int, default=5,
+                    help="ignore queues ages lower than x minutes (default: 5)")
+parser.add_argument('-H', '--higher', dest='higher', type=int, default=1440,
+                    help="ignore queues ages higher than x minutes (default: 1440)")
+args = parser.parse_args()
+
+
+now = datetime.now()
+p = subprocess.Popen(['/usr/sbin/postqueue', '-j'],
+                     stdin=subprocess.PIPE,
+                     stdout=subprocess.PIPE,
+                     stderr=subprocess.STDOUT)
+output = str(p.stdout.read(), "utf-8").splitlines()
+mail_queue = 0
+
+
+if args.domain == 'all':
+    mail_queue = len(output)
+else:
+    for line in output:
+        j = json.loads(line)
+        if j["queue_name"] == 'active':
+            # Ignore Active queue
+            continue
+
+        queue_old = now - datetime.fromtimestamp(j["arrival_time"])
+        if (queue_old.total_seconds() / 60 < args.lower
+                or queue_old.total_seconds() / 60 > args.higher):
+            # Not old enough
+            continue
+
+        for recipient in j['recipients']:
+            if recipient['address'].endswith(args.domain):
+                mail_queue += 1
+                break
+
+
+ret_val = 0
+msg = ("OK: Queue length for %s destination < %s (%s)"
+       % (args.domain, args.warning, mail_queue))
+
+if mail_queue > args.warning:
+    msg = ("WARNING: Queue length for %s destination > %s (%s)"
+           % (args.domain, args.warning, mail_queue))
+    ret_val = 1
+
+if mail_queue > args.critical:
+    msg = ("CRITICAL: Queue length for %s destination > %s (%s)"
+           % (args.domain, args.critical, mail_queue))
+    ret_val = 2
+
+
+print(msg)
+sys.exit(ret_val)

roles/base/files/postfix/zabbix-agent-dropin

@@ -1,2 +1 @@
-UserParameter=postfix.test,/usr/local/bin/zabbix-test.sh
 UserParameter=postfix.queue,/usr/bin/mailq | grep -c '^[A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9]'

roles/base/files/postfix/zabbix-check-script

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-# Lets generate random data for now
-min=10
-max=20
-echo $((RANDOM % (max - min + 1) + min))

roles/base/tasks/postfix-monitoring.yml

@@ -39,26 +39,41 @@
   - postfix
   - zabbix_agent
 
-# This is a workaround for a specific Nagios plugin that the
-# Zabbix agent can't execute for some reason :/
-- ansible.builtin.cron:
-    name: "Dump Red Hat specific mail queue info to file"
-    minute: "*/5"
-    user: root
-    job: "/usr/lib64/nagios/plugins/check_postfix_queue.py redhat.com 2>&1 > /etc/zabbix/postfix-redhat.log"
-  when: inventory_hostname.startswith('bastion')
-  tags:
-    - postfix
-    - zabbix_agent
-
-# On a fresh install, the zabbix user won't exist yet
-# so don't try to set it here. The agent role will fix it later
+# On a fresh install, the zabbix user/dirs won't exist yet as
+# base runs before zabbix/zabbix_agent, so don't try to set it
+# here. The agent role will fix it later.
 - name: Ensure Zabbix drop-in directory
   ansible.builtin.file:
     path: /etc/zabbix/zabbix_agentd.d
     state: directory
     mode: '0755'
 
+- name: Ensure Zabbix script directory
+  ansible.builtin.file:
+    path: /usr/lib/zabbix
+    state: directory
+    mode: '0770'
+
+# This is a workaround for a specific Nagios plugin that the
+# Zabbix agent can't execute for some reason :/
+# TBD revisit this after Nagios is removed
+- name: Install old Nagios-style postfix plugin where Zabbix can find it
+  ansible.builtin.copy:
+    src: postfix/nagios-plugin.py
+    dest: /usr/lib/zabbix/check_postfix_queue.py
+    mode: '0755'
+
+- name: Setup cron to run old Nagios postfix plugin for bastion hosts
+  ansible.builtin.cron:
+    name: "Dump Red Hat specific mail queue info to file"
+    minute: "*/5"
+    user: root
+    job: "/usr/lib/zabbix/check_postfix_queue.py redhat.com 2>&1 > /etc/zabbix/postfix-redhat.log"
+  when: inventory_hostname.startswith('bastion')
+  tags:
+    - postfix
+    - zabbix_agent
+
 - name: Install Zabbix agent config drop-in
   ansible.builtin.copy:
     src: postfix/zabbix-agent-dropin
@@ -70,15 +85,6 @@
     - postfix
     - zabbix_agent
 
-- name: Install Zabbix check script
-  ansible.builtin.copy:
-    src: postfix/zabbix-check-script
-    dest: /usr/local/bin/zabbix-test.sh
-    mode: '0755'
-  tags:
-    - postfix
-    - zabbix_agent
-
 - name: Zabbix API Block
   vars:
     ansible_zabbix_auth_key: "{{ zabbix_auth_key }}"

roles/ipa/server/files/zabbix/nagios-free-ids-plugin.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+
+"""
+Check for available IDs in IPA's ID ranges.
+
+See pagure.io/fedora-infrastructure/issue/12641
+
+Author: abompard@fedoraproject.org
+"""
+
+import os
+import sys
+from argparse import ArgumentParser
+from configparser import ConfigParser
+
+import ldap
+
+
+STATUS_OK = 0
+STATUS_WARNING = 1
+STATUS_CRITICAL = 2
+STATUS_UNKNOWN = 3
+
+
+def get_config():
+    config = ConfigParser(interpolation=None)
+    config.read("/etc/ipa/default.conf")
+    return {key: config.get("global", key) for key in ("host", "basedn")}
+
+
+def get_ldap_connection(config):
+    ldap.set_option(ldap.OPT_REFERRALS, 0)
+    conn = ldap.ldapobject.SimpleLDAPObject(f"ldaps://{config['host']}")
+    conn.protocol_version = 3
+    conn.timeout = 10
+    conn.sasl_gssapi_bind_s()
+    return conn
+
+
+def get_free_ids(config, connection):
+    results = connection.search_s(
+        base=f"cn=posix-ids,cn=dna,cn=ipa,cn=etc,{config['basedn']}",
+        scope=ldap.SCOPE_ONELEVEL,
+        filterstr="(dnaPortNum=389)",
+        attrlist=["dnaHostname", "dnaRemainingValues"],
+    )
+    free_ids = {}
+    for dn, attrs in results:
+        hostname = attrs["dnaHostname"][0].decode("ascii")
+        value = int(attrs["dnaRemainingValues"][0].decode("ascii"))
+        free_ids[hostname] = value
+    return free_ids
+
+
+def get_nagios_result(free_ids, thresholds):
+    # Testcases:
+    # free_ids={"host1": 0, "host2": 0, "host3": 20}
+    # free_ids={"host1": 10, "host2": 0, "host3": 20}
+    # free_ids={"host1": 10000, "host2": 10000, "host3": 20}
+    # free_ids={}
+
+    perfdata = " ".join(f"{host}={free_ids[host]}" for host in sorted(free_ids))
+
+    msg = "OK: there are free IDs left"
+    exit_code = 0
+
+    for threshold, threshold_exit_code in thresholds:
+        if any(value < threshold for value in free_ids.values()):
+            bad_servers = [host for host in sorted(free_ids) if free_ids[host] < threshold]
+            msg = " ".join(
+                [
+                    str(len(bad_servers)),
+                    "server has" if len(bad_servers) == 1 else "servers have",
+                    "less than",
+                    str(threshold),
+                    "free IDs left:",
+                    ", ".join(bad_servers),
+                ]
+            )
+            exit_code = threshold_exit_code
+            break
+    return f"{msg}|{perfdata}", exit_code
+
+
+def exit_nagios(output, exit_code):
+    print(output)
+    sys.exit(exit_code)
+
+
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument("-w", "--warning", type=int, default=1000, help="warning threshold")
+    parser.add_argument("-c", "--critical", type=int, default=1, help="critical threshold")
+    parser.add_argument(
+        "-k", "--keytab", help="use this keytab for GSSAPI authentication"
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    if args.keytab:
+        os.environ["KRB5_CLIENT_KTNAME"] = args.keytab
+    config = get_config()
+    try:
+        connection = get_ldap_connection(config)
+        free_ids = get_free_ids(config, connection)
+    except ldap.LOCAL_ERROR as e:
+        exit_nagios(
+            f"Can't read the free IDs in LDAP: {e.args[0].get('info')}", STATUS_UNKNOWN
+        )
+
+    if not free_ids:
+        current_user = connection.whoami_s()
+        exit_nagios(
+            f"No IPA server found in LDAP, check the permissions for user {current_user!r}",
+            STATUS_UNKNOWN,
+        )
+
+    thresholds = (
+        (args.critical, STATUS_CRITICAL),
+        (args.warning, STATUS_WARNING),
+    )
+    exit_nagios(*get_nagios_result(free_ids, thresholds))
+
+
+if __name__ == "__main__":
+    main()

roles/ipa/server/tasks/monitoring.yml

@@ -1,3 +1,4 @@
+---
 # Zabbix monitoring of the internal IPA server
 
 # ipa-healthcheck can only be run by root, so cron the
@@ -12,17 +13,27 @@
     - ipa/server
     - zabbix_agent
 
-- name: Set cron for ipa-healthcheck
+# ipa-free-ids also has to run as root, and uses an old Nagios plugin
+# TBD rewrite this in Zabbix style
+- name: Install old Nagios-style postfix plugin where Zabbix can find it
+  ansible.builtin.copy:
+    src: zabbix/nagios-free-ids-plugin.py
+    dest: /usr/lib/zabbix/check_ipa_free_ids.py
+    owner: zabbix
+    group: zabbix
+    mode: '0755'
+
+- name: Set cron for ipa-free-ids
   ansible.builtin.cron:
     name: "Dump IPA free_ids as root for Zabbix"
     minute: "*/5"
     user: root
-    job: "/usr/lib64/nagios/plugins/check_ipa_free_ids.py > /etc/zabbix/ipa-free-ids.log 2> /dev/null"
+    job: "/usr/lib/zabbix/check_ipa_free_ids.py > /etc/zabbix/ipa-free-ids.log 2> /dev/null"
   tags:
     - ipa/server
     - zabbix_agent
 
-- name: Install Zabbix agent config drop-in
+- name: Install Zabbix agent config drop-in for IPA backups
   ansible.builtin.copy:
     src: zabbix/agent-ipa-backup.conf
     dest: /etc/zabbix/zabbix_agentd.d/ipa-backup.conf
@@ -67,7 +78,7 @@
         link_templates: IPA Monitoring
         force: false
 
-    # Free_ids only works on 01
+    # Free_ids only works on the cluster leader
     - name: Import IPA Free IDs template file
       community.zabbix.zabbix_template:
         template_yaml: "{{ lookup('file', 'zabbix/template-ipa-freeids.yml') }}"
@@ -80,4 +91,3 @@
         link_templates: IPA Free IDs
         force: false
       when: inventory_hostname.startswith('ipa01')
-

roles/pagure/files/nagios-systemd-plugin.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# Description : script to check the status of systemd units
+# if they failed, try to restart the service once !!
+
+# Author : Seddik Alaoui Ismaili
+# Version : 1.0
+
+
+# Exits code
+warning_exit="1"
+ok_exit="0"
+
+# Unit list
+unit_list=(pagure_ci
+pagure_ev
+pagure_fast_worker
+pagure_loadjson
+pagure_logcom
+pagure_medium_worker
+pagure_milter
+pagure_mirror
+pagure_slow_worker
+pagure_webhook
+pagure_worker
+pagure_mirror_project_in.timer)
+
+#Element's arrays
+failed_array=()
+active_array=()
+
+# Check units's status
+for element in ${unit_list[@]}; do
+  status=$(systemctl status ${element} |grep -E "Active:"  | awk '{ print $2 }')
+  if [ $status == failed ]; then
+    systemctl restart ${element} && active_array+=($element) || failed_array+=($element)
+  fi
+done
+
+# check the lenght of array and print result/exit code for nagios
+if [ ${#failed_array[@]} -ne "0" ]; then
+  echo -e "UNITS WARNING: Failed systemd units after restart : ${failed_array[@]}"
+  exit ${warning_exit}
+elif [ ${#failed_array[@]} -eq "0" ]; then
+  echo -e "UNITS OK: Systemd units are active"
+  exit ${ok_exit}
+fi

roles/pagure/tasks/monitoring.yml

@@ -1,11 +1,22 @@
 ---
+
 # This is a workaround for a specific Nagios plugin that the
 # Zabbix agent can't execute for some reason :/
-- ansible.builtin.cron:
+# TBD rewrite this in Zabbix style
+- name: Install old Nagios-style postfix plugin where Zabbix can find it
+  ansible.builtin.copy:
+    src: nagios-systemd-plugin.sh
+    dest: /usr/lib/zabbix/check_systemd_units.sh
+    owner: zabbix
+    group: zabbix
+    mode: '0755'
+
+- name: Deploy cron to run old Nagios plugin
+  ansible.builtin.cron:
     name: "Run Pagure systemd-check Nagios script"
     minute: "*/5"
     user: root
-    job: "/usr/lib64/nagios/plugins/check_systemd_units 2>&1 > /etc/zabbix/pagure-zabbix.log"
+    job: "/usr/lib/zabbix/check_systemd_units.sh 2>&1 > /etc/zabbix/pagure-zabbix.log"
   tags:
     - pagure
     - zabbix_agent
@@ -30,7 +41,7 @@
       ansible.builtin.set_fact:
         pagure_ip: "{{ (env == 'production') | ternary('38.145.32.40', '38.145.32.39') }}"
 
-    - name: Create {{ inventory_hostname }} cert age item
+    - name: Create cert age item for {{ inventory_hostname }}
       community.zabbix.zabbix_item:
         name: "{{ inventory_hostname }} SSL Certificate time remaining"
         host_name: "{{ inventory_hostname }}"
@@ -47,7 +58,7 @@
             - tag: component
               value: ssl
 
-    - name: Create {{ inventory_hostname }} 30day trigger
+    - name: Create 30day trigger for {{ inventory_hostname }}
       community.zabbix.zabbix_trigger:
         name: "{{ inventory_hostname }} SSL Certificate expires in 30d"
         host_name: "{{ inventory_hostname }}"
@@ -61,7 +72,7 @@
             - tag: scope
               value: availability
 
-    - name: Create {{ inventory_hostname }} 7day trigger
+    - name: Create 7day trigger for {{ inventory_hostname }}
       community.zabbix.zabbix_trigger:
         name: "{{ inventory_hostname }} SSL Certificate expires in 7d"
         host_name: "{{ inventory_hostname }}"
@@ -75,7 +86,7 @@
             - tag: scope
               value: availability
 
-    - name: Create {{ inventory_hostname }} 0day trigger
+    - name: Create 0day trigger for {{ inventory_hostname }}
       community.zabbix.zabbix_trigger:
         name: "{{ inventory_hostname }} SSL Certificate expired!"
         host_name: "{{ inventory_hostname }}"