diff --git a/roles/anubis/files/zabbix/agent-anubis.conf b/roles/anubis/files/zabbix/agent-anubis.conf new file mode 100644 index 0000000000..f98a306d9e --- /dev/null +++ b/roles/anubis/files/zabbix/agent-anubis.conf @@ -0,0 +1 @@ +UserParameter=anubis.get.metrics,/usr/sbin/curl http://localhost:9090/metrics 2>/dev/null diff --git a/roles/anubis/files/zabbix/selinux_anubis.te b/roles/anubis/files/zabbix/selinux_anubis.te new file mode 100644 index 0000000000..8fe600e216 --- /dev/null +++ b/roles/anubis/files/zabbix/selinux_anubis.te @@ -0,0 +1,13 @@ +module zabbix_anubis 1.1; + +require { + type systemd_logind_sessions_t; + type websm_port_t; + type zabbix_agent_t; + class file read; + class tcp_socket name_connect; +} + +#============= zabbix_agent_t ============== +allow zabbix_agent_t systemd_logind_sessions_t:file read; +allow zabbix_agent_t websm_port_t:tcp_socket name_connect; diff --git a/roles/anubis/files/zabbix/template-anubis.yml b/roles/anubis/files/zabbix/template-anubis.yml new file mode 100644 index 0000000000..7b13044ae5 --- /dev/null +++ b/roles/anubis/files/zabbix/template-anubis.yml @@ -0,0 +1,390 @@ +zabbix_export: + version: '7.0' + template_groups: + - uuid: a333cbd6a3ad44baaa4eee4b0c0b1bec + name: Fedora + templates: + - uuid: 43f31a6dd1ae4a1788c954223f021bb4 + template: 'Anubis Monitoring' + name: 'Anubis Monitoring' + groups: + - name: Fedora + items: + - uuid: a982ee9a5439462ba9d241cdadefda77 + name: 'Raw Metrics' + type: ZABBIX_ACTIVE + key: anubis.get.metrics + history: 1h + value_type: TEXT + trends: '0' + description: 'Master item for raw Prometheus metrics from Anubis' + tags: + - tag: component + value: raw + - uuid: 1f4259fb474743e699193ebb5b6b2a9d + name: 'Total request rate' + type: DEPENDENT + key: anubis.requests.rate + delay: '0' + value_type: FLOAT + units: r/s + description: 'Rate-of-change of the Anubis request total' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'anubis_challenges_issued{method="embedded"}' + - value + - '' + - type: CHANGE_PER_SECOND + parameters: + - '' + - type: IN_RANGE + parameters: + - '0' + - '' + master_item: + key: anubis.get.metrics + tags: + - tag: application + value: anubis + - tag: component + value: health + - tag: component + value: network + - uuid: 1e2eadefef504da1b76e22ce78c6179b + name: 'Total requests' + type: DEPENDENT + key: anubis.requests.total + delay: '0' + value_type: FLOAT + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'anubis_challenges_issued{method="embedded"}' + - value + - '' + master_item: + key: anubis.get.metrics + tags: + - tag: application + value: anubis + - uuid: a5fecbbf05164bf2b8414be6062e422f + name: 'Total validation rate' + type: DEPENDENT + key: anubis.validations.rate + delay: '0' + value_type: FLOAT + units: v/s + description: 'Rate-of-change of the Anubis validation total' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'anubis_challenges_validated{method="preact"}' + - value + - '' + - type: CHANGE_PER_SECOND + parameters: + - '' + - type: IN_RANGE + parameters: + - '0' + - '' + master_item: + key: anubis.get.metrics + tags: + - tag: application + value: anubis + - tag: component + value: health + - tag: component + value: network + - uuid: 116b05e06ab44066b80a65d763cec3eb + name: 'Total validations' + type: DEPENDENT + key: anubis.validations.total + delay: '0' + value_type: FLOAT + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'anubis_challenges_validated{method="preact"}' + - value + - '' + master_item: + key: anubis.get.metrics + tags: + - tag: application + value: anubis + - uuid: 1eba942786c64ed986e0129685ed1071 + name: 'Service response time' + type: ZABBIX_ACTIVE + key: 'net.tcp.service.perf["http","127.0.0.1","9090"]' + value_type: FLOAT + units: s + tags: + - tag: application + value: anubis + - tag: component + value: health + - tag: component + value: network + triggers: + - uuid: 9fc9e24cb21d49348961941936bc5315 + expression: 'min(/Anubis Monitoring/net.tcp.service.perf["http","127.0.0.1","9090"],5m)>10s' + name: 'Anubis: Service response time is too high' + event_name: 'Anubis: Service response time is too high (over 10s for 5m)' + priority: WARNING + manual_close: 'YES' + dependencies: + - name: 'Anubis: Service is down' + expression: 'last(/Anubis Monitoring/net.tcp.service["http","127.0.0.1","9090"])=0' + tags: + - tag: scope + value: performance + - uuid: ed57ac3789e741f79675a96b808882fd + name: 'Service status' + type: ZABBIX_ACTIVE + key: 'net.tcp.service["http","127.0.0.1","9090"]' + valuemap: + name: 'Service state' + preprocessing: + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 10m + tags: + - tag: application + value: anubis + - tag: component + value: health + - tag: component + value: network + triggers: + - uuid: ad2a87008a5642719722af89d776c6e1 + expression: 'last(/Anubis Monitoring/net.tcp.service["http","127.0.0.1","9090"])=0' + name: 'Anubis: Service is down' + priority: AVERAGE + manual_close: 'YES' + tags: + - tag: scope + value: availability + discovery_rules: + - uuid: e16c59d26ac34d46818efe06d85f0c84 + name: 'Discover hosts' + type: DEPENDENT + key: anubis.hosts.discovery + delay: '0' + description: 'Collect names of Anubis backend hosts from Prometheus' + item_prototypes: + - uuid: 67c16edd2eee4087ae0085b8cd198b7d + name: 'Proxy {#HOST}: Request rate' + type: DEPENDENT + key: 'anubis.requests.rate[{#HOST}]' + delay: '0' + value_type: FLOAT + units: r/s + description: 'Requests proxied for the host since startup' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'anubis_proxied_requests_total{host="{#HOST}"}' + - value + - '' + - type: CHANGE_PER_SECOND + parameters: + - '' + - type: IN_RANGE + parameters: + - '0' + - '' + master_item: + key: anubis.get.metrics + tags: + - tag: anubis-host + value: '{#HOST}' + - tag: application + value: anubis + - tag: component + value: network + trigger_prototypes: + - uuid: 5f999d3911b84d7ca359ac4908eec6c1 + expression: | + avg(/Anubis Monitoring/anubis.requests.rate[{#HOST}],5m) > 2 * avg(/Anubis Monitoring/anubis.requests.rate[{#HOST}],1h) + + and + + last(/Anubis Monitoring/anubis.requests.rate[{#HOST}]) > {$ANUBIS.MIN.REQUESTS} + name: 'Anomalous request rate for {#HOST} on {HOST.NAME}' + priority: AVERAGE + manual_close: 'YES' + dependencies: + - name: 'Anubis: Service is down' + expression: 'last(/Anubis Monitoring/net.tcp.service["http","127.0.0.1","9090"])=0' + tags: + - tag: scope + value: performance + - uuid: 45e032c77a1948d1a6e8599a67510ae8 + name: 'Proxy {#HOST}: Total Requests' + type: DEPENDENT + key: 'anubis.requests.total[{#HOST}]' + delay: '0' + units: reqs + description: 'Requests proxied for the host since startup' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'anubis_proxied_requests_total{host="{#HOST}"}' + - value + - '' + master_item: + key: anubis.get.metrics + tags: + - tag: anubis-host + value: '{#HOST}' + - tag: application + value: anubis + - tag: component + value: network + master_item: + key: anubis.get.metrics + lld_macro_paths: + - lld_macro: '{#HOST}' + path: $.host + - lld_macro: '{#LOW}' + path: $.low + preprocessing: + - type: PROMETHEUS_TO_JSON + parameters: + - anubis_proxied_requests_total + - type: JAVASCRIPT + parameters: + - | + var metrics = JSON.parse(value); + var result = []; + + metrics.forEach(function(metric) { + result.push({ + "host": metric.labels.host, + "metric": metric.name, + "low": metric.name < 100 + }); + }); + + return JSON.stringify(result); + - uuid: f33ad9b4b8e346d583245b9c429b48c6 + name: 'Discover policies' + type: DEPENDENT + key: anubis.policies.discovery + delay: '0' + description: 'Collect names of Anubis policies from Prometheus' + item_prototypes: + - uuid: 7eb55499655d42149e1a880bebd370ee + name: 'Policy {#POLICY} - {#ACTION}: Results rate' + type: DEPENDENT + key: 'anubis.policies.rate[{#POLICY},{#ACTION}]' + delay: '0' + value_type: FLOAT + units: r/s + description: 'Results for the policy/action combination' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'anubis_policy_results{action="{#ACTION}",rule="{#POLICY}"}' + - value + - '' + - type: CHANGE_PER_SECOND + parameters: + - '' + - type: IN_RANGE + parameters: + - '0' + - '' + master_item: + key: anubis.get.metrics + tags: + - tag: anubis-action + value: '{#ACTION}' + - tag: anubis-policy + value: '{#POLICY}' + - tag: application + value: anubis + - tag: component + value: network + trigger_prototypes: + - uuid: 8fd4921e126a4445991036f066f1e911 + expression: | + avg(/Anubis Monitoring/anubis.policies.rate[{#POLICY},{#ACTION}],5m) > 2 * avg(/Anubis Monitoring/anubis.policies.rate[{#POLICY},{#ACTION}],1h) + + and + + last(/Anubis Monitoring/anubis.policies.rate[{#POLICY},{#ACTION}]) > {$ANUBIS.MIN.REQUESTS} + name: 'Anomalous request rate for {#ACTION}/{#POLICY} on {HOST.NAME}' + priority: AVERAGE + manual_close: 'YES' + dependencies: + - name: 'Anubis: Service is down' + expression: 'last(/Anubis Monitoring/net.tcp.service["http","127.0.0.1","9090"])=0' + tags: + - tag: scope + value: performance + - uuid: 98d55b5007a64d6cac560e1a169cfe33 + name: 'Policy {#POLICY} - {#ACTION}: Results' + type: DEPENDENT + key: 'anubis.policies.total[{#POLICY},{#ACTION}]' + delay: '0' + units: reqs + description: 'Results for the policy/action combination' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'anubis_policy_results{action="{#ACTION}",rule="{#POLICY}"}' + - value + - '' + master_item: + key: anubis.get.metrics + tags: + - tag: anubis-action + value: '{#ACTION}' + - tag: anubis-policy + value: '{#POLICY}' + - tag: application + value: anubis + - tag: component + value: network + master_item: + key: anubis.get.metrics + lld_macro_paths: + - lld_macro: '{#ACTION}' + path: $.action + - lld_macro: '{#POLICY}' + path: $.policy + preprocessing: + - type: PROMETHEUS_TO_JSON + parameters: + - anubis_policy_results + - type: JAVASCRIPT + parameters: + - | + var metrics = JSON.parse(value); + var result = []; + + metrics.forEach(function(metric) { + result.push({ + "policy": metric.labels.rule, + "action": metric.labels.action, + "metric": metric.name + }); + }); + + return JSON.stringify(result); + macros: + - macro: '{$ANUBIS.MIN.REQUEST}' + value: '100' + description: 'Minimum total requests before alerting (low traffic filter)' + valuemaps: + - uuid: d3f229d0ec9747299e5953ddbc00a8e0 + name: 'Service state' + mappings: + - value: '0' + newvalue: Down + - value: '1' + newvalue: Up diff --git a/roles/anubis/tasks/main.yml b/roles/anubis/tasks/main.yml index c25c50f212..335ea5aff9 100644 --- a/roles/anubis/tasks/main.yml +++ b/roles/anubis/tasks/main.yml @@ -98,3 +98,80 @@ - Reload httpd tags: - anubis + +### Monitoring config +# Custom SELinux requirements for Zabbix to execute monitoring binaries +- name: Ensure a directory exists for Anubis Zabbix selinux module + ansible.builtin.file: + path: /usr/local/share/zabbix + state: directory + mode: '0755' + tags: + - anubis + - selinux + - zabbix_agent + +- name: Copy Anubis Zabbix SELinux module + ansible.builtin.copy: + src: zabbix/selinux_anubis.te + dest: /usr/local/share/zabbix/zabbix_anubis.te + owner: root + group: root + mode: '0644' + register: selinux_zabbix_file + tags: + - anubis + - selinux + - zabbix_agent + +- name: Compile and install SELinux module + ansible.builtin.include_tasks: "{{ tasks_path }}/compile-selinux.yml" + vars: + selinux_module_dir: /usr/local/share/zabbix + selinux_module_name: zabbix_anubis + when: selinux_zabbix_file.changed + tags: + - selinux + - postfix + - zabbix_agent + +- name: Install Zabbix agent config drop-in + ansible.builtin.copy: + src: zabbix/agent-anubis.conf + dest: /etc/zabbix/zabbix_agentd.d/anubis.conf + mode: '0644' + tags: + - zabbix_agent + - anubis + notify: + - Restart zabbix agent + +- name: Zabbix API Block + vars: + ansible_zabbix_auth_key: "{{ zabbix_auth_key }}" + ansible_network_os: "{{ zabbix_network_os }}" + ansible_connection: "{{ zabbix_connection }}" + ansible_httpapi_port: "{{ zabbix_httpapi_port }}" + ansible_httpapi_use_ssl: "{{ zabbix_httpapi_use_ssl }}" + ansible_httpapi_validate_certs: "{{ zabbix_httpapi_validate_certs }}" + ansible_host: "{{ zabbix_server }}" + ansible_zabbix_url_path: "{{ zabbix_url_path }}" + tags: + - zabbix_api + - anubis + block: + - name: Import Anubis template file + community.zabbix.zabbix_template: + template_yaml: "{{ lookup('file', 'zabbix/template-anubis.yml') }}" + state: present + - name: Ensure Anubis hostgroup is present + community.zabbix.zabbix_group: + host_groups: + - Anubis servers + state: present + - name: Add self to Anubis in Zabbix + community.zabbix.zabbix_host: + host_name: "{{ inventory_hostname }}" + host_groups: Anubis servers + link_templates: Anubis Monitoring + force: false