Zabbix: add Varnish monitoring

Signed-off-by: Greg Sutcliffe <fedora@emeraldreverie.org>

roles/varnish/files/zabbix/agent.conf

@@ -0,0 +1 @@
+UserParameter=varnish.raw_data,/usr/sbin/varnishstat -j

roles/varnish/files/zabbix/selinux.te

@@ -0,0 +1,12 @@
+module zabbix_varnish 1.1;
+
+require {
+        type zabbix_agent_t;
+        type varnishd_var_lib_t;
+        class file read;
+        class file open;
+}
+
+#============= zabbix_agent_t ==============
+allow zabbix_agent_t varnishd_var_lib_t:file read;
+allow zabbix_agent_t varnishd_var_lib_t:file open;

roles/varnish/files/zabbix/template.yml

@@ -0,0 +1,549 @@
+zabbix_export:
+  version: '7.0'
+  template_groups:
+    - uuid: a333cbd6a3ad44baaa4eee4b0c0b1bec
+      name: Fedora
+  templates:
+    - uuid: 8a0ae7759b4d4cda8da682f02ab95095
+      template: 'Varnish Cache Monitoring'
+      name: 'Varnish Cache Monitoring'
+      description: |
+        ## Description
+        
+        Simple template for the Varnish cache using results from the varnishstat command. 
+        
+        ## Overview
+        
+        Simple template for the Varnish cache using results from the varnishstat command. 
+        
+        ## Author: Greg Sutcliffe 
+        ## Forked from: Dave Humphrey (dave@uesp.net)
+      groups:
+        - name: Fedora
+      items:
+        - uuid: adf987b6d3df486db52d4b8148a865c9
+          name: 'Varnish: Service response time'
+          type: ZABBIX_ACTIVE
+          key: 'net.tcp.service.perf[http,"{$VARNISH.STATUS.HOST}","{$VARNISH.STATUS.PORT}"]'
+          history: 7d
+          value_type: FLOAT
+          units: s
+          tags:
+            - tag: Application
+              value: Varnish
+          triggers:
+            - uuid: 85538643e3c24fbb8b4fdd222f3f44e0
+              expression: 'min(/Varnish Cache Monitoring/net.tcp.service.perf[http,"{$VARNISH.STATUS.HOST}","{$VARNISH.STATUS.PORT}"],5m)={$VARNISH.RESPONSE_TIME.MAX.WARN}'
+              name: 'Varnish: Service response time is too high (over {$VARNISH.RESPONSE_TIME.MAX.WARN}s for 5m)'
+              priority: WARNING
+              manual_close: 'YES'
+              dependencies:
+                - name: 'Varnish: Process is not running'
+                  expression: 'last(/Varnish Cache Monitoring/proc.num["{$VARNISH.PROCESS_NAME}"])=0'
+                - name: 'Varnish: Service is down'
+                  expression: 'last(/Varnish Cache Monitoring/net.tcp.service[http,"{$VARNISH.STATUS.HOST}","{$VARNISH.STATUS.PORT}"])=0'
+        - uuid: 80295421232547dfba8159e974c236d6
+          name: 'Varnish: Service ping'
+          type: ZABBIX_ACTIVE
+          key: 'net.tcp.service[http,"{$VARNISH.STATUS.HOST}","{$VARNISH.STATUS.PORT}"]'
+          history: 7d
+          tags:
+            - tag: Application
+              value: Varnish
+          triggers:
+            - uuid: 4c754b657d944992b309c0a92bb01499
+              expression: 'last(/Varnish Cache Monitoring/net.tcp.service[http,"{$VARNISH.STATUS.HOST}","{$VARNISH.STATUS.PORT}"])=0'
+              name: 'Varnish: Service is down'
+              priority: AVERAGE
+              manual_close: 'YES'
+              dependencies:
+                - name: 'Varnish: Process is not running'
+                  expression: 'last(/Varnish Cache Monitoring/proc.num["{$VARNISH.PROCESS_NAME}"])=0'
+        - uuid: 8d173a3deb874b1bab51dd698d9358fc
+          name: 'Varnish: CPU utilization'
+          type: ZABBIX_ACTIVE
+          key: 'proc.cpu.util["{$VARNISH.PROCESS_NAME}"]'
+          history: 7d
+          value_type: FLOAT
+          units: '%'
+          description: 'Process CPU utilization percentage.'
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: fe4a28e44cdb492c8539914af45b39a3
+          name: 'Varnish: Memory usage (rss)'
+          type: ZABBIX_ACTIVE
+          key: 'proc.mem["{$VARNISH.PROCESS_NAME}",,,,rss]'
+          history: 7d
+          units: B
+          description: 'Resident set size memory used by process in bytes.'
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 079f500025b840dcb5629e0c92539ddb
+          name: 'Varnish: Memory usage (vsize)'
+          type: ZABBIX_ACTIVE
+          key: 'proc.mem["{$VARNISH.PROCESS_NAME}",,,,vsize]'
+          history: 7d
+          units: B
+          description: 'Virtual memory size used by process in bytes.'
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 7db0bf02c0ca419d985a2b11fc1c25f6
+          name: 'Varnish: Number of processes running'
+          type: ZABBIX_ACTIVE
+          key: 'proc.num["{$VARNISH.PROCESS_NAME}"]'
+          history: 7d
+          tags:
+            - tag: Application
+              value: Varnish
+          triggers:
+            - uuid: dc44a80d523f445c819071c9ebf48893
+              expression: 'last(/Varnish Cache Monitoring/proc.num["{$VARNISH.PROCESS_NAME}"])=0'
+              name: 'Varnish: Process is not running'
+              priority: HIGH
+        - uuid: 645d42eafecc4b1a95b5e75ba10d9fc3
+          name: 'Varnish: Backend connections'
+          type: DEPENDENT
+          key: varnish.backend.connections
+          delay: '0'
+          history: 7d
+          description: 'Backend connections succeeded'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.backend_conn"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: e935889f0eaa40a8a3f7589e3136dbc8
+          name: 'Varnish: Backend failed'
+          type: DEPENDENT
+          key: varnish.backend.failed
+          delay: '0'
+          history: 7d
+          description: 'Backend connections failed'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.backend_fail"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: f0c8750857464c609f40cd803849dddb
+          name: 'Varnish: Backend requests'
+          type: DEPENDENT
+          key: varnish.backend.requests
+          delay: '0'
+          history: 7d
+          description: 'Backend requests made'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.backend_req"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: ffd5566192bd44eeb9d93ebc9d6a2fc4
+          name: 'Varnish: Backend unhealthy'
+          type: DEPENDENT
+          key: varnish.backend.unhealthy
+          delay: '0'
+          history: 7d
+          description: 'Backend connections not attempted'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.backend_unhealthy"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 75d72c6a11174f91862f72e2b05010d0
+          name: 'Varnish: Bytes per second'
+          type: DEPENDENT
+          key: varnish.bytes.rate
+          delay: '0'
+          history: 7d
+          value_type: FLOAT
+          units: Bps
+          description: 'Calculated as change rate for ''Total bytes'' stat.'
+          preprocessing:
+            - type: JAVASCRIPT
+              parameters:
+                - |
+                  var values = JSON.parse(value);
+                  var headBytes = values["MAIN.s_resp_hdrbytes"].value;
+                  var bodyBytes = values["MAIN.s_resp_bodybytes"].value;
+                  return headBytes + bodyBytes;
+            - type: CHANGE_PER_SECOND
+              parameters:
+                - ''
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: fd089d8419ff4d83a2c3fe7f0172ff5f
+          name: 'Varnish: Cache hit-pass'
+          type: DEPENDENT
+          key: varnish.cache.hitpasses
+          delay: '0'
+          history: 7d
+          description: 'Cache hits for pass'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.cache_hitpass"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: a199ab12103345c4812ecc1472bd6dc4
+          name: 'Varnish: Cache hits'
+          type: DEPENDENT
+          key: varnish.cache.hits
+          delay: '0'
+          history: 7d
+          description: 'Cache hits'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.cache_hit"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 7378704abbc24b05ad203a8e9735cd2c
+          name: 'Varnish: Cache misses'
+          type: DEPENDENT
+          key: varnish.cache.misses
+          delay: '0'
+          history: 7d
+          description: 'Cache misses'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.cache_miss"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 4bf458f277af4234a2971b6aaf8d8280
+          name: 'Varnish: Client requests'
+          type: DEPENDENT
+          key: varnish.client.requests
+          delay: '0'
+          history: 7d
+          description: 'Good client requests received'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.client_req"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: fca4b1afebab4edb91a9e9ac7174fef2
+          name: 'Varnish: Cache hit rate'
+          type: DEPENDENT
+          key: varnish.hitrate
+          delay: '0'
+          history: 7d
+          value_type: FLOAT
+          units: '%'
+          description: 'Average hit rate since the cache was started'
+          preprocessing:
+            - type: JAVASCRIPT
+              parameters:
+                - |
+                  var values = JSON.parse(value);
+                  var hit = values["MAIN.cache_hit"].value;
+                  var miss = values["MAIN.cache_miss"].value;
+                  var hitpass = values["MAIN.cache_hitpass"].value;
+                  return hit / (hit + miss) * 100;
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 11b2739844e8482fb2d44cd936065977
+          name: 'Varnish: Cache hit rate 5m'
+          type: CALCULATED
+          key: varnish.hitrate5m
+          history: 7d
+          value_type: FLOAT
+          units: '%'
+          params: '(last(//varnish.cache.hits) - last(//varnish.cache.hits,#2)) * 100 / ((last(//varnish.cache.misses) - last(//varnish.cache.misses,#2)) + (last(//varnish.cache.hits) - last(//varnish.cache.hits,#2)))'
+          description: 'Average cache hit rate in the past 5m'
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 2318e903bafd4f5c9d9bf1159fbc659e
+          name: 'Varnish: Expired objects'
+          type: DEPENDENT
+          key: varnish.objects.expired
+          delay: '0'
+          history: 7d
+          description: 'Number of expired objects'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.n_expired"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 36837e88b2344e94923f26bc15e4202b
+          name: 'Varnish: LRU nuked objects'
+          type: DEPENDENT
+          key: varnish.objects.lrunuked
+          delay: '0'
+          history: 7d
+          description: 'Number of LRU nuked objects'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.n_lru_nuked"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: a04a8fd97456427d9b82e9856695ec88
+          name: 'Varnish: Num purges'
+          type: DEPENDENT
+          key: varnish.objects.purges
+          delay: '0'
+          history: 7d
+          description: 'Number of purge operations executed'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.n_purges"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 05ee653674dd490fa99643d2253241e6
+          name: 'Varnish: Get status'
+          type: ZABBIX_ACTIVE
+          key: varnish.raw_data
+          history: 1h
+          value_type: TEXT
+          trends: '0'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - $.counters
+          tags:
+            - tag: Application
+              value: 'Zabbix raw items'
+          triggers:
+            - uuid: 3b1aa70c7b104e2bb557c2db9ffe710b
+              expression: 'nodata(/Varnish Cache Monitoring/varnish.raw_data,10m)=1'
+              name: 'Varnish: Failed to fetch stats (or no data for 10m)'
+              priority: WARNING
+              description: 'Zabbix has not received data for stats for the last 10m.'
+              manual_close: 'YES'
+              dependencies:
+                - name: 'Varnish: Process is not running'
+                  expression: 'last(/Varnish Cache Monitoring/proc.num["{$VARNISH.PROCESS_NAME}"])=0'
+                - name: 'Varnish: Service is down'
+                  expression: 'last(/Varnish Cache Monitoring/net.tcp.service[http,"{$VARNISH.STATUS.HOST}","{$VARNISH.STATUS.PORT}"])=0'
+        - uuid: 017f130075e44226ab676b4758445a6c
+          name: 'Varnish: Total request bytes'
+          type: DEPENDENT
+          key: varnish.request.bytes
+          delay: '0'
+          history: 7d
+          units: B
+          description: 'Total bytes received'
+          preprocessing:
+            - type: JAVASCRIPT
+              parameters:
+                - |
+                  var values = JSON.parse(value);
+                  var headBytes = values["MAIN.s_req_hdrbytes"].value;
+                  var bodyBytes = values["MAIN.s_req_bodybytes"].value;
+                  return headBytes + bodyBytes;
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 8c06df8a87fa46738fc80ab6441cfae1
+          name: 'Varnish: Requests per second'
+          type: DEPENDENT
+          key: varnish.requests.rate
+          delay: '0'
+          history: 7d
+          value_type: FLOAT
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.client_req"].value'
+            - type: CHANGE_PER_SECOND
+              parameters:
+                - ''
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: f364881a918a4e678816360f61b46d43
+          name: 'Varnish: Total response bytes'
+          type: DEPENDENT
+          key: varnish.response.bytes
+          delay: '0'
+          history: 7d
+          units: B
+          description: 'Total bytes sent'
+          preprocessing:
+            - type: JAVASCRIPT
+              parameters:
+                - |
+                  var values = JSON.parse(value);
+                  var headBytes = values["MAIN.s_resp_hdrbytes"].value;
+                  var bodyBytes = values["MAIN.s_resp_bodybytes"].value;
+                  return headBytes + bodyBytes;
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: c876276a1b93421ab8c0a12f80918893
+          name: 'Varnish: Session dropped'
+          type: DEPENDENT
+          key: varnish.session.dropped
+          delay: '0'
+          history: 7d
+          description: 'Sessions dropped'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.sess_dropped"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: fb2feb9ed69e414ca8c36d5b832ce79a
+          name: 'Varnish: Threads created'
+          type: DEPENDENT
+          key: varnish.threads.created
+          delay: '0'
+          history: 7d
+          description: 'Threads created'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.threads_created"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: bd8e098f8d2f42aaa7fc901173162b6a
+          name: 'Varnish: Threads failed'
+          type: DEPENDENT
+          key: varnish.threads.failed
+          delay: '0'
+          history: 7d
+          description: 'Thread creation failure'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.threads_failed"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 83e9983fa64648be95e4e275fd582419
+          name: 'Varnish: Threads limited'
+          type: DEPENDENT
+          key: varnish.threads.limited
+          delay: '0'
+          history: 7d
+          description: 'Threads hit max'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.threads_limited"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+        - uuid: 3cdec39a17bd42fcbc846432bc9d1ed1
+          name: 'Varnish: Uptime'
+          type: DEPENDENT
+          key: varnish.uptime
+          delay: '0'
+          history: 7d
+          units: uptime
+          description: 'Service uptime in seconds'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$["MAIN.uptime"].value'
+          master_item:
+            key: varnish.raw_data
+          tags:
+            - tag: Application
+              value: Varnish
+          triggers:
+            - uuid: 038c5860369147238731776c5ec18410
+              expression: 'last(/Varnish Cache Monitoring/varnish.uptime)=10m'
+              name: 'Varnish: Has been restarted (uptime < 10m)'
+              priority: INFO
+              description: 'Uptime is less than 10 minutes'
+              manual_close: 'YES'
+      macros:
+        - macro: '{$VARNISH.PROCESS_NAME}'
+          value: varnishd
+          description: 'Process name of the Varnish server.'
+        - macro: '{$VARNISH.RESPONSE_TIME.MAX.WARN}'
+          value: '5'
+          description: 'Max response time before a warning is given.'
+        - macro: '{$VARNISH.STATUS.HOST}'
+          value: 127.0.0.1
+          description: 'IP or hostname of the Varnish server.'
+        - macro: '{$VARNISH.STATUS.PORT}'
+          value: '80'
+          description: 'Port of the Varnish server.'
+  graphs:
+    - uuid: ef40fb5ce9344c04b91e96d7d547ac84
+      name: 'Cache hit rate'
+      graph_items:
+        - color: 1A7C11
+          item:
+            host: 'Varnish Cache Monitoring'
+            key: varnish.hitrate
+        - sortorder: '1'
+          color: F63100
+          item:
+            host: 'Varnish Cache Monitoring'
+            key: varnish.hitrate5m
+    - uuid: 73f2cd95eb664725a42051d495a5dadf
+      name: 'Request rate'
+      graph_items:
+        - color: 1A7C11
+          item:
+            host: 'Varnish Cache Monitoring'
+            key: varnish.requests.rate

roles/varnish/tasks/main.yml

@@ -42,3 +42,10 @@
   service: enabled=yes name=varnish
   tags:
   - varnish
+
+- name: Configure Varnish monitoring in Zabbix
+  ansible.builtin.include_tasks: monitoring.yml
+  tags:
+    - varnish
+    - zabbix_agent
+    - zabbix_api

roles/varnish/tasks/monitoring.yml

@@ -0,0 +1,99 @@
+---
+# Configure Zabbix. A host including this file should
+# - deploy the required scripts & agent config to the host
+# - add itself to the required hostgroup
+# - add itself to the required template
+
+# Custom SELinux requirements for Zabbix to execute monitoring binaries
+- name: Ensure a directory exists for varnish+Zabbix selinux module
+  ansible.builtin.file:
+    path: /usr/local/share/zabbix
+    state: directory
+    mode: '0755'
+  tags:
+  - selinux
+  - varnish
+  - zabbix_agent
+
+- name: Copy Varnish Zabbix SELinux module
+  ansible.builtin.copy:
+    src: zabbix/selinux.te
+    dest: /usr/local/share/zabbix/zabbix_varnish.te
+    owner: root
+    group: root
+    mode: '0644'
+  register: selinux_zabbix_file
+  tags:
+  - selinux
+  - varnish
+  - zabbix_agent
+
+- name: Compile and install SELinux module
+  ansible.builtin.include_tasks: "{{ tasks_path }}/compile-selinux.yml"
+  vars:
+    selinux_module_dir: /usr/local/share/zabbix
+    selinux_module_name: zabbix_varnish
+  when: selinux_zabbix_file.changed # noqa: no-handler
+  tags:
+  - selinux
+  - varnish
+  - zabbix_agent
+
+# The agent needs to be in the varnish group to read the directory holding the stats
+- name: Add zabbix to varnish group
+  ansible.builtin.user:
+    name: zabbix
+    groups: varnish
+    append: yes
+  notify:
+    - Restart zabbix agent
+  tags:
+    - varnish
+    - zabbix_agent
+
+- name: Install Zabbix agent config drop-in
+  ansible.builtin.copy:
+    src: zabbix/agent.conf
+    dest: /etc/zabbix/zabbix_agentd.d/varnish.conf
+    mode: '0644'
+    owner: zabbix
+    group: zabbix
+  notify:
+    - Restart zabbix agent
+  tags:
+    - varnish
+    - zabbix_agent
+
+- name: Zabbix API Block
+  vars:
+    ansible_zabbix_auth_key: "{{ zabbix_auth_key }}"
+    ansible_network_os: "{{ zabbix_network_os }}"
+    ansible_connection: "{{ zabbix_connection }}"
+    ansible_httpapi_port: "{{ zabbix_httpapi_port }}"
+    ansible_httpapi_use_ssl: "{{ zabbix_httpapi_use_ssl }}"
+    ansible_httpapi_validate_certs: "{{ zabbix_httpapi_validate_certs }}"
+    ansible_host: "{{ zabbix_server }}"
+    ansible_zabbix_url_path: "{{ zabbix_url_path }}"
+  tags:
+    - zabbix_api
+    - zabbix_agent
+  block:
+    - name: Import Varnish Zabbix template file
+      community.zabbix.zabbix_template:
+        template_yaml: "{{ lookup('file', 'zabbix/template.yml') }}"
+        state: present
+      run_once: true
+
+    - name: Ensure Varnish Zabbix hostgroup is present
+      community.zabbix.zabbix_group:
+        host_groups:
+          - Varnish Servers
+        state: present
+      run_once: true
+
+    - name: Add self to Varnish in Zabbix
+      community.zabbix.zabbix_host:
+        host_name: "{{ inventory_hostname }}"
+        host_groups: Varnish Servers
+        link_templates: Varnish Cache Monitoring
+        force: false