forked from infra/ansible
Nagios: Drop RabbitMQ checks as Zabbix handles this
This removes (almost) all the Nagios config for Rabbit, including the templates in the Rabbit role(s). It leaves the nagios user, because Zabbix is also using it. It also adds a Matrix-level notification for high queues, above the one that goes to the UI. Signed-off-by: Greg Sutcliffe <fedora@emeraldreverie.org>
This commit is contained in:
parent
084da3c5ab
commit
9cad3bf76c
SSH key fingerprint:
SHA256:dNHcz65ApR68w2TkCIAXDBdt4oNL62Fyx48U7dXWuk0
20 changed files with 14 additions and 252 deletions
|
|
@ -1,26 +0,0 @@
|
|||
#!/bin/python3
|
||||
import sys
|
||||
import requests
|
||||
|
||||
url = 'http://localhost:15672/api/queues/%%2f/%s' % (sys.argv[1])
|
||||
|
||||
r = requests.get(url, auth=('guest', 'guest')).json()
|
||||
consumers = r['consumers']
|
||||
messages = r['messages']
|
||||
|
||||
msg = 'Messages in queue: %i (%i consumers)' % (messages, consumers)
|
||||
|
||||
if consumers < 1:
|
||||
print ('CRITICAL: No consumers: %s' % msg)
|
||||
sys.exit(2)
|
||||
|
||||
if messages > int(sys.argv[2]):
|
||||
print ('WARNING: %s' % msg)
|
||||
sys.exit(1)
|
||||
|
||||
if messages > int(sys.argv[3]):
|
||||
print ('CRITICAL: %s' % msg)
|
||||
sys.exit(1)
|
||||
|
||||
print ('OK: %s' % msg)
|
||||
sys.exit(0)
|
||||
|
|
@ -70,7 +70,6 @@
|
|||
- check_raid.py
|
||||
- check_lock
|
||||
- check_fcomm_queue
|
||||
- check_rabbitmq_size
|
||||
- check_datanommer_timesince.py
|
||||
- check_memcache_connect
|
||||
- check_readonly_fs
|
||||
|
|
@ -352,38 +351,6 @@
|
|||
name: /etc/openvpn/server/crl.pem
|
||||
state: present
|
||||
|
||||
- name: Install nrpe config for the RabbitMQ checks
|
||||
ansible.builtin.template:
|
||||
src: "rabbitmq_args.ini.j2"
|
||||
dest: "/etc/nrpe.d/rabbitmq_args.ini"
|
||||
owner: root
|
||||
group: nrpe
|
||||
mode: "0640"
|
||||
when: inventory_hostname.startswith('rabbitmq')
|
||||
tags:
|
||||
- nagios_client
|
||||
|
||||
- name: Install nrpe checks for the RabbitMQ cluster
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/nrpe.d/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
with_items:
|
||||
- check_rabbitmq_server.cfg
|
||||
- check_rabbitmq_watermark.cfg
|
||||
- check_rabbitmq_cluster.cfg
|
||||
- check_rabbitmq_connections.cfg
|
||||
- check_rabbitmq_overview.cfg
|
||||
- check_rabbitmq_exchange.cfg
|
||||
- check_rabbitmq_queue.cfg
|
||||
when: inventory_hostname.startswith('rabbitmq')
|
||||
notify:
|
||||
- Restart nrpe
|
||||
tags:
|
||||
- nagios_client
|
||||
|
||||
- name: Nrpe service start
|
||||
ansible.builtin.service:
|
||||
name: nrpe
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
command[check_rabbitmq_cluster]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_cluster --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini -n rabbit@rabbitmq01.rdu3.fedoraproject.org,rabbit@rabbitmq02.rdu3.fedoraproject.org,rabbit@rabbitmq03.rdu3.fedoraproject.org
|
||||
|
|
@ -1 +0,0 @@
|
|||
command[check_rabbitmq_connections]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_connections --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
command[check_rabbitmq_exchange_pubsub]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_exchange --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini --vhost /pubsub --exchange amq.topic --period 600
|
||||
command[check_rabbitmq_exchange_public_pubsub]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_exchange --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini --vhost /public_pubsub --exchange amq.topic --period 600
|
||||
|
|
@ -1 +0,0 @@
|
|||
command[check_rabbitmq_overview]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_overview --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
command[check_rabbitmq_queue_pubsub]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_queue --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini --vhost /pubsub
|
||||
command[check_rabbitmq_queue_public_pubsub]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_queue --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini --vhost /public_pubsub
|
||||
|
|
@ -1 +0,0 @@
|
|||
command[check_rabbitmq_server]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_server --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini --node={{ inventory_hostname }}
|
||||
|
|
@ -1 +0,0 @@
|
|||
command[check_rabbitmq_watermark]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_watermark --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini --vhost /pubsub --node={{ inventory_hostname }}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
[common]
|
||||
hostname = localhost
|
||||
username = nagios-monitoring
|
||||
password = {{ (env == 'production')|ternary(rabbitmq_monitoring_password_production, rabbitmq_monitoring_password_staging) }}
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
# RabbitMQ processes (for each host)
|
||||
|
||||
define service {
|
||||
host_name rabbitmq01.rdu3.fedoraproject.org
|
||||
service_description Check bus server processes
|
||||
check_command check_by_nrpe!check_rabbitmq_server
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq02.rdu3.fedoraproject.org
|
||||
service_description Check bus server processes
|
||||
check_command check_by_nrpe!check_rabbitmq_server
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq03.rdu3.fedoraproject.org
|
||||
service_description Check bus server processes
|
||||
check_command check_by_nrpe!check_rabbitmq_server
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
# RabbitMQ alarms (for each host)
|
||||
|
||||
define service {
|
||||
host_name rabbitmq01.rdu3.fedoraproject.org
|
||||
service_description Check bus server alarms
|
||||
check_command check_by_nrpe!check_rabbitmq_watermark
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq02.rdu3.fedoraproject.org
|
||||
service_description Check bus server alarms
|
||||
check_command check_by_nrpe!check_rabbitmq_watermark
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq03.rdu3.fedoraproject.org
|
||||
service_description Check bus server alarms
|
||||
check_command check_by_nrpe!check_rabbitmq_watermark
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
# The following results are cluster-wide, no need to run them on each cluster member
|
||||
|
||||
define service {
|
||||
host_name rabbitmq01.rdu3.fedoraproject.org
|
||||
service_description Check bus cluster
|
||||
check_command check_by_nrpe!check_rabbitmq_cluster
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq01.rdu3.fedoraproject.org
|
||||
service_description Check bus cluster connections
|
||||
check_command check_by_nrpe!check_rabbitmq_connections
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq01.rdu3.fedoraproject.org
|
||||
service_description Check bus cluster overview
|
||||
check_command check_by_nrpe!check_rabbitmq_overview
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq01.rdu3.fedoraproject.org
|
||||
service_description Check bus exchanges in /pubsub
|
||||
check_command check_by_nrpe!check_rabbitmq_exchange_pubsub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq01.rdu3.fedoraproject.org
|
||||
service_description Check bus exchanges in /public_pubsub
|
||||
check_command check_by_nrpe!check_rabbitmq_exchange_public_pubsub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq01.rdu3.fedoraproject.org
|
||||
service_description Check bus queues in /pubsub
|
||||
check_command check_by_nrpe!check_rabbitmq_queue_pubsub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name rabbitmq01.rdu3.fedoraproject.org
|
||||
service_description Check bus queues in /public_pubsub
|
||||
check_command check_by_nrpe!check_rabbitmq_queue_public_pubsub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
|
@ -213,7 +213,6 @@
|
|||
- mailman.cfg
|
||||
- nrpe.cfg
|
||||
- pgsql.cfg
|
||||
- rabbitmq.cfg
|
||||
tags:
|
||||
- nagios_config
|
||||
- nagios_server
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
---
|
||||
- name: Restart nagios on noc
|
||||
ansible.builtin.shell: nagios -v /etc/nagios/nagios.cfg && systemctl restart nagios
|
||||
delegate_to: "{{ queue_nagios_server }}"
|
||||
|
||||
- name: Restart nrpe on rabbitmq
|
||||
service:
|
||||
name: nrpe
|
||||
state: restarted
|
||||
delegate_to: "{{ queue_rabbitmq_server }}"
|
||||
|
|
@ -17,9 +17,6 @@
|
|||
#
|
||||
# - user_sent_topics (str): A regular expression that must match the topic when a
|
||||
# message is published by this user.
|
||||
# - queue_thresholds (dict): A dictionary with two keys: "warning" and "critical".
|
||||
# The values are numbers. Generate an alert in Nagios if
|
||||
# the number of messages go above these values.
|
||||
|
||||
- name: Validate queue name {{ queue_name }}
|
||||
assert:
|
||||
|
|
@ -77,28 +74,3 @@
|
|||
tags:
|
||||
- fedora-messaging
|
||||
- rabbitmq_cluster
|
||||
|
||||
- name: Monitor the {{ queue_name }} queue in Nagios (NRPE)
|
||||
when: queue_thresholds and env == "production"
|
||||
delegate_to: "{{ queue_rabbitmq_server }}"
|
||||
ansible.builtin.template:
|
||||
src: nrpe.cfg.j2
|
||||
dest: /etc/nrpe.d/check_rabbitmq_queue_{{ queue_name }}.cfg
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: Restart nrpe on rabbitmq
|
||||
tags:
|
||||
- fedora-messaging
|
||||
- rabbitmq_cluster
|
||||
|
||||
- name: Monitor the {{ queue_name }} queue in Nagios
|
||||
when: queue_thresholds and env == "production"
|
||||
delegate_to: "{{ queue_nagios_server }}"
|
||||
ansible.builtin.template:
|
||||
src: nagios.cfg.j2
|
||||
dest: /etc/nagios/services/rabbitmq-queue-{{ queue_name }}.cfg
|
||||
# notify: Restart nagios on noc
|
||||
tags:
|
||||
- fedora-messaging
|
||||
- rabbitmq_cluster
|
||||
|
|
|
|||
|
|
@ -1,6 +0,0 @@
|
|||
define service {
|
||||
host_name {{ queue_rabbitmq_server }}
|
||||
service_description Check queue {{ queue_name }}
|
||||
check_command check_by_nrpe!check_rabbitmq_queue_{{ queue_vhost.replace("/", "") }}_{{ queue_name }}
|
||||
use defaulttemplate
|
||||
}
|
||||
|
|
@ -1 +0,0 @@
|
|||
command[check_rabbitmq_queue_{{ queue_vhost.replace("/", "") }}_{{ queue_name }}]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_queue --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini --vhost {{ queue_vhost }} --queue {{ queue_name }} {% if queue_thresholds.warning %}-w {{ queue_thresholds.warning }}{% endif %} {% if queue_thresholds.critical %}-c {{ queue_thresholds.critical }}{% endif %}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
module nrpe_rabbitmq 1.0;
|
||||
|
||||
require {
|
||||
type amqp_port_t;
|
||||
type nrpe_t;
|
||||
class tcp_socket name_connect;
|
||||
}
|
||||
|
||||
#============= nrpe_t ==============
|
||||
# let nagios plugin connect to rabbitmq management interface
|
||||
allow nrpe_t amqp_port_t:tcp_socket name_connect;
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
checkmodule -M -m -o /etc/nagios/nrpe_rabbitmq.mod /etc/nagios/nrpe_rabbitmq.te
|
||||
semodule_package -o /etc/nagios/nrpe_rabbitmq.pp -m /etc/nagios/nrpe_rabbitmq.mod
|
||||
semodule -i /etc/nagios/nrpe_rabbitmq.pp
|
||||
rm /etc/nagios/nrpe_rabbitmq.mod /etc/nagios/nrpe_rabbitmq.pp
|
||||
|
|
@ -1196,6 +1196,15 @@ zabbix_export:
|
|||
- tag: vhost
|
||||
value: '{#VHOST}'
|
||||
trigger_prototypes:
|
||||
- uuid: bc63d110be214843b6bcd8116ef01a09
|
||||
expression: 'min(/RabbitMQ Monitoring/rabbitmq.queue.messages["{#VHOST}/{#QUEUE}"],5m)>{$RABBITMQ.MESSAGES.MAX.HIGH:"{#QUEUE}"}'
|
||||
name: 'RabbitMQ node: Too many messages in queue [{#VHOST}][{#QUEUE}]'
|
||||
event_name: 'RabbitMQ node: Too many messages in queue [{#VHOST}][{#QUEUE}] (over {$RABBITMQ.MESSAGES.MAX.HIGH:"{#QUEUE}"} for 5m)'
|
||||
opdata: 'Current value: {ITEM.LASTVALUE1}'
|
||||
priority: HIGH
|
||||
tags:
|
||||
- tag: scope
|
||||
value: performance
|
||||
- uuid: b6f52153a22144338e6d002287446ac1
|
||||
expression: 'min(/RabbitMQ Monitoring/rabbitmq.queue.messages["{#VHOST}/{#QUEUE}"],5m)>{$RABBITMQ.MESSAGES.MAX.WARN:"{#QUEUE}"}'
|
||||
name: 'RabbitMQ node: Too many messages in queue [{#VHOST}][{#QUEUE}]'
|
||||
|
|
@ -1420,6 +1429,9 @@ zabbix_export:
|
|||
- macro: '{$RABBITMQ.LLD.FILTER.QUEUE.NOT_MATCHES}'
|
||||
value: CHANGE_IF_NEEDED
|
||||
description: 'This macro is used in the discovery of queues. It can be overridden at host level or its linked template level.'
|
||||
- macro: '{$RABBITMQ.MESSAGES.MAX.HIGH}'
|
||||
value: '2000'
|
||||
description: 'The maximum number of messages in the queue for a Matrix alert.'
|
||||
- macro: '{$RABBITMQ.MESSAGES.MAX.WARN}'
|
||||
value: '1000'
|
||||
description: 'The maximum number of messages in the queue for a trigger expression.'
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@
|
|||
state: present
|
||||
name:
|
||||
- rabbitmq-server
|
||||
- nagios-plugins-rabbitmq
|
||||
tags:
|
||||
- rabbitmq_cluster
|
||||
- packages
|
||||
|
|
@ -269,6 +268,7 @@
|
|||
# Users with the "monitoring" tag have read-only access vhosts, connections,
|
||||
# channels, node-level resource usage, and cluster stats.
|
||||
# (this needs to be done after all vhost that the user has access to have been created)
|
||||
# TODO: The Zabbix monitoring currently reuses this user, we should rename it
|
||||
- name: Create the Nagios monitoring user in staging
|
||||
rabbitmq_user:
|
||||
user: nagios-monitoring
|
||||
|
|
@ -297,6 +297,7 @@
|
|||
- rabbitmq_cluster
|
||||
- config
|
||||
|
||||
# TODO: The Zabbix monitoring currently reuses this user, we should rename it
|
||||
- name: Create the Nagios monitoring user in production
|
||||
rabbitmq_user:
|
||||
user: nagios-monitoring
|
||||
|
|
@ -432,23 +433,6 @@
|
|||
federation-upstream: "pubsub-to-public_pubsub"
|
||||
vhost: /public_pubsub
|
||||
|
||||
# SELinux: allow the Nagios NRPE plugin to access the management interface
|
||||
- name: Install the selinux module compilation script
|
||||
ansible.builtin.copy:
|
||||
src: selinux-load.sh
|
||||
dest: /etc/nagios/selinux-load.sh
|
||||
mode: "0755"
|
||||
|
||||
- name: Copy over our custom selinux module
|
||||
ansible.builtin.copy:
|
||||
src: nrpe_rabbitmq.te
|
||||
dest: /etc/nagios/nrpe_rabbitmq.te
|
||||
register: selinux_module
|
||||
|
||||
- name: Compile and install our custom selinux module
|
||||
ansible.builtin.command: /etc/nagios/selinux-load.sh
|
||||
when: selinux_module is changed
|
||||
|
||||
# RabbitMQ monitoring for Fedora Zabbix
|
||||
- name: Configure RabbitMQ monitoring in Zabbix
|
||||
ansible.builtin.include_tasks: monitoring.yml
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue