From a0c7bf49adcf72c56ee9f72ad464b8267dad9322 Mon Sep 17 00:00:00 2001 From: Greg Sutcliffe Date: Tue, 30 Jun 2026 11:00:46 +0100 Subject: [PATCH] Zabbix: migrate still-in-use Nagios plugins to the relevant roles Signed-off-by: Greg Sutcliffe --- roles/base/files/postfix/nagios-plugin.py | 69 ++++++++++ roles/base/files/postfix/zabbix-agent-dropin | 1 - roles/base/files/postfix/zabbix-check-script | 6 - roles/base/tasks/postfix-monitoring.yml | 52 +++---- .../files/zabbix/nagios-free-ids-plugin.py | 128 ++++++++++++++++++ roles/ipa/server/tasks/monitoring.yml | 20 ++- roles/pagure/files/nagios-systemd-plugin.sh | 47 +++++++ roles/pagure/tasks/monitoring.yml | 23 +++- 8 files changed, 305 insertions(+), 41 deletions(-) create mode 100755 roles/base/files/postfix/nagios-plugin.py delete mode 100644 roles/base/files/postfix/zabbix-check-script create mode 100755 roles/ipa/server/files/zabbix/nagios-free-ids-plugin.py create mode 100644 roles/pagure/files/nagios-systemd-plugin.sh diff --git a/roles/base/files/postfix/nagios-plugin.py b/roles/base/files/postfix/nagios-plugin.py new file mode 100755 index 0000000000..f8273ab186 --- /dev/null +++ b/roles/base/files/postfix/nagios-plugin.py @@ -0,0 +1,69 @@ +#!/usr/bin/python3 + +import subprocess +import json +import argparse +import sys + +from datetime import datetime + +parser = argparse.ArgumentParser() +parser.add_argument('domain', help="Required. Domain to check") +parser.add_argument('-c', '--critical', dest='critical', type=int, default=50, + help="Critical threshold") +parser.add_argument('-w', '--warning', dest='warning', type=int, default=20, + help="Warning threshold") +parser.add_argument('-L', '--lower', dest='lower', type=int, default=5, + help="ignore queues ages lower than x minutes (default: 5)") +parser.add_argument('-H', '--higher', dest='higher', type=int, default=1440, + help="ignore queues ages higher than x minutes (default: 1440)") +args = parser.parse_args() + + +now = datetime.now() +p = subprocess.Popen(['/usr/sbin/postqueue', '-j'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) +output = str(p.stdout.read(), "utf-8").splitlines() +mail_queue = 0 + + +if args.domain == 'all': + mail_queue = len(output) +else: + for line in output: + j = json.loads(line) + if j["queue_name"] == 'active': + # Ignore Active queue + continue + + queue_old = now - datetime.fromtimestamp(j["arrival_time"]) + if (queue_old.total_seconds() / 60 < args.lower + or queue_old.total_seconds() / 60 > args.higher): + # Not old enough + continue + + for recipient in j['recipients']: + if recipient['address'].endswith(args.domain): + mail_queue += 1 + break + + +ret_val = 0 +msg = ("OK: Queue length for %s destination < %s (%s)" + % (args.domain, args.warning, mail_queue)) + +if mail_queue > args.warning: + msg = ("WARNING: Queue length for %s destination > %s (%s)" + % (args.domain, args.warning, mail_queue)) + ret_val = 1 + +if mail_queue > args.critical: + msg = ("CRITICAL: Queue length for %s destination > %s (%s)" + % (args.domain, args.critical, mail_queue)) + ret_val = 2 + + +print(msg) +sys.exit(ret_val) diff --git a/roles/base/files/postfix/zabbix-agent-dropin b/roles/base/files/postfix/zabbix-agent-dropin index 7608852022..0d0580ad26 100644 --- a/roles/base/files/postfix/zabbix-agent-dropin +++ b/roles/base/files/postfix/zabbix-agent-dropin @@ -1,2 +1 @@ -UserParameter=postfix.test,/usr/local/bin/zabbix-test.sh UserParameter=postfix.queue,/usr/bin/mailq | grep -c '^[A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9]' diff --git a/roles/base/files/postfix/zabbix-check-script b/roles/base/files/postfix/zabbix-check-script deleted file mode 100644 index f3030a8bed..0000000000 --- a/roles/base/files/postfix/zabbix-check-script +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Lets generate random data for now -min=10 -max=20 -echo $((RANDOM % (max - min + 1) + min)) diff --git a/roles/base/tasks/postfix-monitoring.yml b/roles/base/tasks/postfix-monitoring.yml index faaa120f48..7714ecf445 100644 --- a/roles/base/tasks/postfix-monitoring.yml +++ b/roles/base/tasks/postfix-monitoring.yml @@ -39,26 +39,41 @@ - postfix - zabbix_agent -# This is a workaround for a specific Nagios plugin that the -# Zabbix agent can't execute for some reason :/ -- ansible.builtin.cron: - name: "Dump Red Hat specific mail queue info to file" - minute: "*/5" - user: root - job: "/usr/lib64/nagios/plugins/check_postfix_queue.py redhat.com 2>&1 > /etc/zabbix/postfix-redhat.log" - when: inventory_hostname.startswith('bastion') - tags: - - postfix - - zabbix_agent - -# On a fresh install, the zabbix user won't exist yet -# so don't try to set it here. The agent role will fix it later +# On a fresh install, the zabbix user/dirs won't exist yet as +# base runs before zabbix/zabbix_agent, so don't try to set it +# here. The agent role will fix it later. - name: Ensure Zabbix drop-in directory ansible.builtin.file: path: /etc/zabbix/zabbix_agentd.d state: directory mode: '0755' +- name: Ensure Zabbix script directory + ansible.builtin.file: + path: /usr/lib/zabbix + state: directory + mode: '0770' + +# This is a workaround for a specific Nagios plugin that the +# Zabbix agent can't execute for some reason :/ +# TBD revisit this after Nagios is removed +- name: Install old Nagios-style postfix plugin where Zabbix can find it + ansible.builtin.copy: + src: postfix/nagios-plugin.py + dest: /usr/lib/zabbix/check_postfix_queue.py + mode: '0755' + +- name: Setup cron to run old Nagios postfix plugin for bastion hosts + ansible.builtin.cron: + name: "Dump Red Hat specific mail queue info to file" + minute: "*/5" + user: root + job: "/usr/lib/zabbix/check_postfix_queue.py redhat.com 2>&1 > /etc/zabbix/postfix-redhat.log" + when: inventory_hostname.startswith('bastion') + tags: + - postfix + - zabbix_agent + - name: Install Zabbix agent config drop-in ansible.builtin.copy: src: postfix/zabbix-agent-dropin @@ -70,15 +85,6 @@ - postfix - zabbix_agent -- name: Install Zabbix check script - ansible.builtin.copy: - src: postfix/zabbix-check-script - dest: /usr/local/bin/zabbix-test.sh - mode: '0755' - tags: - - postfix - - zabbix_agent - - name: Zabbix API Block vars: ansible_zabbix_auth_key: "{{ zabbix_auth_key }}" diff --git a/roles/ipa/server/files/zabbix/nagios-free-ids-plugin.py b/roles/ipa/server/files/zabbix/nagios-free-ids-plugin.py new file mode 100755 index 0000000000..141be1b3d2 --- /dev/null +++ b/roles/ipa/server/files/zabbix/nagios-free-ids-plugin.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 + +""" +Check for available IDs in IPA's ID ranges. + +See pagure.io/fedora-infrastructure/issue/12641 + +Author: abompard@fedoraproject.org +""" + +import os +import sys +from argparse import ArgumentParser +from configparser import ConfigParser + +import ldap + + +STATUS_OK = 0 +STATUS_WARNING = 1 +STATUS_CRITICAL = 2 +STATUS_UNKNOWN = 3 + + +def get_config(): + config = ConfigParser(interpolation=None) + config.read("/etc/ipa/default.conf") + return {key: config.get("global", key) for key in ("host", "basedn")} + + +def get_ldap_connection(config): + ldap.set_option(ldap.OPT_REFERRALS, 0) + conn = ldap.ldapobject.SimpleLDAPObject(f"ldaps://{config['host']}") + conn.protocol_version = 3 + conn.timeout = 10 + conn.sasl_gssapi_bind_s() + return conn + + +def get_free_ids(config, connection): + results = connection.search_s( + base=f"cn=posix-ids,cn=dna,cn=ipa,cn=etc,{config['basedn']}", + scope=ldap.SCOPE_ONELEVEL, + filterstr="(dnaPortNum=389)", + attrlist=["dnaHostname", "dnaRemainingValues"], + ) + free_ids = {} + for dn, attrs in results: + hostname = attrs["dnaHostname"][0].decode("ascii") + value = int(attrs["dnaRemainingValues"][0].decode("ascii")) + free_ids[hostname] = value + return free_ids + + +def get_nagios_result(free_ids, thresholds): + # Testcases: + # free_ids={"host1": 0, "host2": 0, "host3": 20} + # free_ids={"host1": 10, "host2": 0, "host3": 20} + # free_ids={"host1": 10000, "host2": 10000, "host3": 20} + # free_ids={} + + perfdata = " ".join(f"{host}={free_ids[host]}" for host in sorted(free_ids)) + + msg = "OK: there are free IDs left" + exit_code = 0 + + for threshold, threshold_exit_code in thresholds: + if any(value < threshold for value in free_ids.values()): + bad_servers = [host for host in sorted(free_ids) if free_ids[host] < threshold] + msg = " ".join( + [ + str(len(bad_servers)), + "server has" if len(bad_servers) == 1 else "servers have", + "less than", + str(threshold), + "free IDs left:", + ", ".join(bad_servers), + ] + ) + exit_code = threshold_exit_code + break + return f"{msg}|{perfdata}", exit_code + + +def exit_nagios(output, exit_code): + print(output) + sys.exit(exit_code) + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("-w", "--warning", type=int, default=1000, help="warning threshold") + parser.add_argument("-c", "--critical", type=int, default=1, help="critical threshold") + parser.add_argument( + "-k", "--keytab", help="use this keytab for GSSAPI authentication" + ) + return parser.parse_args() + + +def main(): + args = parse_args() + if args.keytab: + os.environ["KRB5_CLIENT_KTNAME"] = args.keytab + config = get_config() + try: + connection = get_ldap_connection(config) + free_ids = get_free_ids(config, connection) + except ldap.LOCAL_ERROR as e: + exit_nagios( + f"Can't read the free IDs in LDAP: {e.args[0].get('info')}", STATUS_UNKNOWN + ) + + if not free_ids: + current_user = connection.whoami_s() + exit_nagios( + f"No IPA server found in LDAP, check the permissions for user {current_user!r}", + STATUS_UNKNOWN, + ) + + thresholds = ( + (args.critical, STATUS_CRITICAL), + (args.warning, STATUS_WARNING), + ) + exit_nagios(*get_nagios_result(free_ids, thresholds)) + + +if __name__ == "__main__": + main() diff --git a/roles/ipa/server/tasks/monitoring.yml b/roles/ipa/server/tasks/monitoring.yml index e3cb96d8f0..c5d61c1d29 100644 --- a/roles/ipa/server/tasks/monitoring.yml +++ b/roles/ipa/server/tasks/monitoring.yml @@ -1,3 +1,4 @@ +--- # Zabbix monitoring of the internal IPA server # ipa-healthcheck can only be run by root, so cron the @@ -12,17 +13,27 @@ - ipa/server - zabbix_agent -- name: Set cron for ipa-healthcheck +# ipa-free-ids also has to run as root, and uses an old Nagios plugin +# TBD rewrite this in Zabbix style +- name: Install old Nagios-style postfix plugin where Zabbix can find it + ansible.builtin.copy: + src: zabbix/nagios-free-ids-plugin.py + dest: /usr/lib/zabbix/check_ipa_free_ids.py + owner: zabbix + group: zabbix + mode: '0755' + +- name: Set cron for ipa-free-ids ansible.builtin.cron: name: "Dump IPA free_ids as root for Zabbix" minute: "*/5" user: root - job: "/usr/lib64/nagios/plugins/check_ipa_free_ids.py > /etc/zabbix/ipa-free-ids.log 2> /dev/null" + job: "/usr/lib/zabbix/check_ipa_free_ids.py > /etc/zabbix/ipa-free-ids.log 2> /dev/null" tags: - ipa/server - zabbix_agent -- name: Install Zabbix agent config drop-in +- name: Install Zabbix agent config drop-in for IPA backups ansible.builtin.copy: src: zabbix/agent-ipa-backup.conf dest: /etc/zabbix/zabbix_agentd.d/ipa-backup.conf @@ -67,7 +78,7 @@ link_templates: IPA Monitoring force: false - # Free_ids only works on 01 + # Free_ids only works on the cluster leader - name: Import IPA Free IDs template file community.zabbix.zabbix_template: template_yaml: "{{ lookup('file', 'zabbix/template-ipa-freeids.yml') }}" @@ -80,4 +91,3 @@ link_templates: IPA Free IDs force: false when: inventory_hostname.startswith('ipa01') - diff --git a/roles/pagure/files/nagios-systemd-plugin.sh b/roles/pagure/files/nagios-systemd-plugin.sh new file mode 100644 index 0000000000..a20003cb75 --- /dev/null +++ b/roles/pagure/files/nagios-systemd-plugin.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Description : script to check the status of systemd units +# if they failed, try to restart the service once !! + +# Author : Seddik Alaoui Ismaili +# Version : 1.0 + + +# Exits code +warning_exit="1" +ok_exit="0" + +# Unit list +unit_list=(pagure_ci +pagure_ev +pagure_fast_worker +pagure_loadjson +pagure_logcom +pagure_medium_worker +pagure_milter +pagure_mirror +pagure_slow_worker +pagure_webhook +pagure_worker +pagure_mirror_project_in.timer) + +#Element's arrays +failed_array=() +active_array=() + +# Check units's status +for element in ${unit_list[@]}; do + status=$(systemctl status ${element} |grep -E "Active:" | awk '{ print $2 }') + if [ $status == failed ]; then + systemctl restart ${element} && active_array+=($element) || failed_array+=($element) + fi +done + +# check the lenght of array and print result/exit code for nagios +if [ ${#failed_array[@]} -ne "0" ]; then + echo -e "UNITS WARNING: Failed systemd units after restart : ${failed_array[@]}" + exit ${warning_exit} +elif [ ${#failed_array[@]} -eq "0" ]; then + echo -e "UNITS OK: Systemd units are active" + exit ${ok_exit} +fi diff --git a/roles/pagure/tasks/monitoring.yml b/roles/pagure/tasks/monitoring.yml index 69da28468a..7c5f448542 100644 --- a/roles/pagure/tasks/monitoring.yml +++ b/roles/pagure/tasks/monitoring.yml @@ -1,11 +1,22 @@ --- + # This is a workaround for a specific Nagios plugin that the # Zabbix agent can't execute for some reason :/ -- ansible.builtin.cron: +# TBD rewrite this in Zabbix style +- name: Install old Nagios-style postfix plugin where Zabbix can find it + ansible.builtin.copy: + src: nagios-systemd-plugin.sh + dest: /usr/lib/zabbix/check_systemd_units.sh + owner: zabbix + group: zabbix + mode: '0755' + +- name: Deploy cron to run old Nagios plugin + ansible.builtin.cron: name: "Run Pagure systemd-check Nagios script" minute: "*/5" user: root - job: "/usr/lib64/nagios/plugins/check_systemd_units 2>&1 > /etc/zabbix/pagure-zabbix.log" + job: "/usr/lib/zabbix/check_systemd_units.sh 2>&1 > /etc/zabbix/pagure-zabbix.log" tags: - pagure - zabbix_agent @@ -30,7 +41,7 @@ ansible.builtin.set_fact: pagure_ip: "{{ (env == 'production') | ternary('38.145.32.40', '38.145.32.39') }}" - - name: Create {{ inventory_hostname }} cert age item + - name: Create cert age item for {{ inventory_hostname }} community.zabbix.zabbix_item: name: "{{ inventory_hostname }} SSL Certificate time remaining" host_name: "{{ inventory_hostname }}" @@ -47,7 +58,7 @@ - tag: component value: ssl - - name: Create {{ inventory_hostname }} 30day trigger + - name: Create 30day trigger for {{ inventory_hostname }} community.zabbix.zabbix_trigger: name: "{{ inventory_hostname }} SSL Certificate expires in 30d" host_name: "{{ inventory_hostname }}" @@ -61,7 +72,7 @@ - tag: scope value: availability - - name: Create {{ inventory_hostname }} 7day trigger + - name: Create 7day trigger for {{ inventory_hostname }} community.zabbix.zabbix_trigger: name: "{{ inventory_hostname }} SSL Certificate expires in 7d" host_name: "{{ inventory_hostname }}" @@ -75,7 +86,7 @@ - tag: scope value: availability - - name: Create {{ inventory_hostname }} 0day trigger + - name: Create 0day trigger for {{ inventory_hostname }} community.zabbix.zabbix_trigger: name: "{{ inventory_hostname }} SSL Certificate expired!" host_name: "{{ inventory_hostname }}"