forked from infra/ansible
updates-uptimes: More UI changes. Add history/diffstat. Add cron job.
Signed-off-by: James Antill <james@and.org>
This commit is contained in:
parent
c0d8d62a64
commit
a0c202f3af
3 changed files with 549 additions and 139 deletions
11
roles/batcave/files/ansible-hosts-updates-uptime.cron
Normal file
11
roles/batcave/files/ansible-hosts-updates-uptime.cron
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
#! /bin/sh
|
||||
|
||||
PATH="$PATH:/usr/local/bin"
|
||||
|
||||
# This runs the playbook generate-updates-uptimes-per-host-file.yml
|
||||
# which needs privs. to connect to "all" our hosts.
|
||||
# Don't output anything, because it's a lot of ansible output and it's
|
||||
# mostly fine if nothing happens (we just lose history).
|
||||
# Timeout is set for 1h but it should be much faster (< 5m).
|
||||
timeout 1h updates-uptime-cmd.py update-daily 2> /dev/null > /dev/null || true
|
||||
updates-uptime-cmd.py history-keep 2> /dev/null > /dev/null || true
|
||||
|
|
@ -262,6 +262,24 @@
|
|||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
#
|
||||
# Setup job that runs gets updates/uptimes for hosts each night (via. ansible).
|
||||
#
|
||||
|
||||
- name: Install updates+uptime command
|
||||
ansible.builtin.copy: src=scripts/updates-uptime-cmd.py dest=/usr/local/bin/updates-uptime-cmd.py mode=0755
|
||||
tags:
|
||||
- batcave
|
||||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
- name: Setup updates+uptime ansible job
|
||||
ansible.builtin.copy: src=ansible-hosts-updates-uptime.cron dest=/etc/cron.daily/ansible-hosts-updates-uptime.cron mode=0755
|
||||
tags:
|
||||
- batcave
|
||||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
#
|
||||
# Setup job that runs letsencrypt on proxies each week
|
||||
#
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@
|
|||
# $0 info [x] = see the current state, in long form, can be filtered by name
|
||||
# $0 host [x] = see the current state of a host(s), can be filtered by name
|
||||
# $0 list [x] = see the current state, can be filtered by name
|
||||
# $0 backups = see backups
|
||||
# $0 backups-keep = clenaup old backups
|
||||
# $0 history = see history
|
||||
# $0 history-keep = clenaup old history
|
||||
# $0 stats [x] = see stats, can specify a backup
|
||||
|
||||
# Examples:
|
||||
|
|
@ -21,7 +21,7 @@
|
|||
# $0 diff ... see what changed.
|
||||
# $0 list '*.stg.*' ... see what staging looks like.
|
||||
# $0 list '*copr*' ... see what copr looks like.
|
||||
# $0 backups-keep 4 ... keep four days of backups (including today)
|
||||
# $0 history-keep 4 ... keep four days of history (including today)
|
||||
# $0 uptime 1d ... see what hasn't been rebooted in the last 24 hours.
|
||||
# $0 uptime 25w ... see what hasn't been rebooted in too damn long
|
||||
|
||||
|
|
@ -30,6 +30,7 @@ import sys
|
|||
|
||||
import fnmatch
|
||||
import glob
|
||||
import locale
|
||||
import shutil
|
||||
import time
|
||||
|
||||
|
|
@ -40,6 +41,18 @@ conf_dur_flush_cache = (60*60*8)
|
|||
# How many hosts to show in tier 4 updates/uptimes...
|
||||
conf_stat_4_hosts = 4
|
||||
|
||||
# Do we use a shorter duration by default (drop minutes/seconds)
|
||||
conf_short_duration = True
|
||||
|
||||
# Do we want a small osinfo in diff/list/etc.
|
||||
conf_small_osinfo = True
|
||||
|
||||
# Try to print OS/ver even nicer (when small) ... but includes spaces.
|
||||
conf_align_osinfo_small = True
|
||||
|
||||
# Allow 9,999,999 updates, or try to work out the correct size.
|
||||
conf_fast_width_history = True
|
||||
|
||||
# Remove suffix noise in names.
|
||||
conf_suffix_dns_replace = {
|
||||
'.fedoraproject.org' : '.<FP>.org',
|
||||
|
|
@ -52,19 +65,151 @@ for x in conf_suffix_dns_replace:
|
|||
# Dir. where we put, and look for, the files...
|
||||
conf_path = "/var/log/"
|
||||
|
||||
# Have nice "plain" numbers...
|
||||
def _ui_int(num):
|
||||
return locale.format_string('%d', int(num), grouping=True)
|
||||
try:
|
||||
locale.setlocale(locale.LC_ALL, '')
|
||||
except locale.Error:
|
||||
# default to C locale if we get a failure.
|
||||
print(' Warning: Failed to set locale, defaulting to C', file=sys.stderr)
|
||||
os.environ['LC_ALL'] = 'C'
|
||||
locale.setlocale(locale.LC_ALL, 'C')
|
||||
|
||||
|
||||
fname = conf_path + "ansible-list-updates-uptime.txt"
|
||||
fname_today = fname + '.' + time.strftime("%Y-%m-%d")
|
||||
backup_today = time.strftime("%Y-%m-%d", time.gmtime())
|
||||
fname_today = fname + '.' + backup_today
|
||||
|
||||
backups = sorted(x.removeprefix(fname + '.') for x in glob.glob(fname + '.*'))
|
||||
|
||||
tm_yesterday = int(time.time()) - (60*60*24)
|
||||
backup_yesterday = time.strftime("%Y-%m-%d", time.gmtime(tm_yesterday))
|
||||
fname_yesterday = fname + '.' + backup_yesterday
|
||||
|
||||
if len(backups) < 1 or backups[-1] != backup_today:
|
||||
fname_today = None
|
||||
if len(backups) < 2 or backups[-2] != backup_yesterday:
|
||||
if fname_today is None and backups and backups[-1] == backup_yesterday:
|
||||
pass # Just missing today
|
||||
else:
|
||||
fname_yesterday = None
|
||||
|
||||
if len(sys.argv) >= 2:
|
||||
if '-v' in sys.argv:
|
||||
sys.argv.remove('-v') # In theory sys.argv[0] but meh
|
||||
conf_small_osinfo = False
|
||||
conf_short_duration = False
|
||||
conf_stat_4_hosts *= 4
|
||||
conf_suffix_dns_replace = {}
|
||||
|
||||
_max_len_osnm = 0 # osname_small
|
||||
_max_len_osvr = 0 # osvers ... upto the first '.'
|
||||
class Host():
|
||||
""" Class for holding the Host data from a line in the files. """
|
||||
|
||||
__slots__ = ['name', 'rpms', 'uptime', 'date', 'osname', 'osvers',
|
||||
'osname_small']
|
||||
|
||||
def __init__ (self, data):
|
||||
global _max_len_osnm
|
||||
global _max_len_osvr
|
||||
|
||||
self.name = data['name']
|
||||
self.rpms = data['rpms']
|
||||
self.uptime = data['uptime']
|
||||
self.date = data['date']
|
||||
self.osname = data['osname']
|
||||
self.osvers = data['osvers']
|
||||
|
||||
if False: pass
|
||||
elif self.osname == 'CentOS':
|
||||
osname_small = 'EL'
|
||||
elif self.osname == 'RedHat':
|
||||
osname_small = 'EL'
|
||||
elif self.osname == 'Fedora':
|
||||
osname_small = 'F'
|
||||
else:
|
||||
osname_small = self.osname[:3]
|
||||
self.osname_small = osname_small
|
||||
|
||||
_max_len_osnm = max(len(osname_small), _max_len_osnm)
|
||||
vers = self.osvers
|
||||
off = vers.find('.')
|
||||
if off != -1:
|
||||
vers = vers[:off]
|
||||
_max_len_osvr = max(len(vers), _max_len_osvr)
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
def __eq__(self, other):
|
||||
if self.name != other.name:
|
||||
return False
|
||||
if self.rpms != other.rpms:
|
||||
return False
|
||||
if self.osname != other.osname:
|
||||
return False
|
||||
if self.osvers != other.osvers:
|
||||
return False
|
||||
return True
|
||||
|
||||
def __gt__(self, other):
|
||||
if self.name > other.name:
|
||||
return True
|
||||
if self.name != other.name:
|
||||
return False
|
||||
|
||||
if self.rpms > other.rpms:
|
||||
return True
|
||||
if self.rpms != other.rpms:
|
||||
return False
|
||||
|
||||
if self.osname > other.osname:
|
||||
return True
|
||||
if self.osname != other.osname:
|
||||
return False
|
||||
|
||||
if self.osvers > other.osvers:
|
||||
return True
|
||||
return False
|
||||
|
||||
# Pretend to be a dict...
|
||||
def __getitem__(self, key):
|
||||
if key not in self.__slots__:
|
||||
raise KeyError()
|
||||
return getattr(self, key)
|
||||
|
||||
@property
|
||||
def osinfo(self):
|
||||
return "%s/%s" % (self.osname, self.osvers)
|
||||
|
||||
@property
|
||||
def osinfo_small(self):
|
||||
if conf_align_osinfo_small:
|
||||
vers = self.osvers
|
||||
rest = ''
|
||||
off = vers.find('.')
|
||||
if off != -1:
|
||||
rest = vers[off:]
|
||||
vers = vers[:off]
|
||||
return "%*s/%*s%s" % (_max_len_osnm, self.osname_small,
|
||||
_max_len_osvr, vers, rest)
|
||||
return "%s/%s" % (self.osname_small, self.osvers)
|
||||
|
||||
|
||||
cmd = "diff"
|
||||
if len(sys.argv) >= 2:
|
||||
if sys.argv[1] in ("backups", "backups-keep", "diff", "diff-u", "help", "host", "info", "list", "stats", "update", "update-fast", "update-flush", "update-daily", "update-daily-refresh", "uptime",):
|
||||
if sys.argv[1] in ("backups", "backups-keep",
|
||||
"hist", "history", "history-keep",
|
||||
"diff", "diff-u",
|
||||
"help",
|
||||
"host", "info",
|
||||
"list",
|
||||
"stats",
|
||||
"update", "update-fast", "update-flush",
|
||||
"update-daily", "update-daily-refresh",
|
||||
"uptime",):
|
||||
cmd = sys.argv.pop(1)
|
||||
|
||||
_tm_d = {'d' : 60*60*24, 'h' : 60*60, 'm' : 60, 's' : 1,
|
||||
|
|
@ -107,7 +252,7 @@ def _add_dur(dur, ret, nummod, suffix, static=False):
|
|||
ret.append(str(mod))
|
||||
return dur
|
||||
|
||||
def format_duration(seconds, static=False):
|
||||
def format_duration(seconds, short=False, static=False):
|
||||
if seconds is None:
|
||||
seconds = 0
|
||||
dur = int(seconds)
|
||||
|
|
@ -115,6 +260,10 @@ def format_duration(seconds, static=False):
|
|||
ret = []
|
||||
dur = _add_dur(dur, ret, 60, "s", static=static)
|
||||
dur = _add_dur(dur, ret, 60, "m", static=static)
|
||||
if short:
|
||||
if dur == 0 and not static:
|
||||
return '<1h'
|
||||
ret = []
|
||||
dur = _add_dur(dur, ret, 24, "h", static=static)
|
||||
dur = _add_dur(dur, ret, 7, "d", static=static)
|
||||
if dur > 0:
|
||||
|
|
@ -122,15 +271,53 @@ def format_duration(seconds, static=False):
|
|||
ret.append(str(dur))
|
||||
return "".join(reversed(ret))
|
||||
|
||||
# Duration in UI for lists/etc.
|
||||
def _ui_dur(dur):
|
||||
return format_duration(dur, short=conf_short_duration, static=True)
|
||||
|
||||
def _main_file_recent():
|
||||
f1 = os.stat(fname)
|
||||
|
||||
if (int(time.time()) - f1.st_mtime) > (60*60*24):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _backup_today_identical():
|
||||
if fname_today is None:
|
||||
return False
|
||||
|
||||
b = backup_today
|
||||
f1 = os.stat(fname)
|
||||
f2 = os.stat(fname + '.' + b)
|
||||
if f1.st_size != f2.st_size:
|
||||
return False
|
||||
if (f1.st_mtime - f2.st_mtime) > 64: # seconds, just a copy
|
||||
return False
|
||||
return True
|
||||
|
||||
cmp_arg = False
|
||||
cmp = None
|
||||
# This does arguments for a bunch of commands, like stats/list/etc.
|
||||
# by using fname1() after, which looks at cmp_arg.
|
||||
# But also does diff arguments.
|
||||
def _cmp_arg():
|
||||
global cmp
|
||||
global cmp_arg
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
if len(sys.argv) < 2 or sys.argv[1] == "main":
|
||||
if len(sys.argv) >= 2:
|
||||
sys.argv.pop(1)
|
||||
cmp = backups[-1] # Most recent
|
||||
if len(backups) > 1 and _backup_today_identical():
|
||||
# Eg. if you just do one update a day, you want to cmp vs.
|
||||
# the previous day, not today.
|
||||
cmp = backups[-2]
|
||||
elif sys.argv[1] == "today" and fname_today is not None:
|
||||
cmp = backup_today
|
||||
cmp_arg = True
|
||||
elif sys.argv[1] == "yesterday" and fname_yesterday is not None:
|
||||
cmp = backup_yesterday
|
||||
cmp_arg = True
|
||||
elif sys.argv[1] not in backups:
|
||||
_usage()
|
||||
print("Backups:", ", ".join(backups))
|
||||
|
|
@ -139,30 +326,39 @@ def _cmp_arg():
|
|||
cmp = sys.argv[1]
|
||||
cmp_arg = True
|
||||
|
||||
_max_len_osnm = 0 # osname_small
|
||||
_max_len_osvr = 0 # osvers ... upto the first '.'
|
||||
def line2data(line):
|
||||
global _max_len_osnm
|
||||
global _max_len_osvr
|
||||
|
||||
name, rpms, uptime, date = line.split(' ', 3)
|
||||
osname = "Unknown"
|
||||
osvers = "?"
|
||||
osinfo = "Unknown/?"
|
||||
osinfo_small = "?/?"
|
||||
if ' ' in date:
|
||||
date, osname, osvers = date.split(' ', 2)
|
||||
osinfo = osname + '/' + osvers
|
||||
osinfo_small = osinfo
|
||||
if osname == 'CentOS':
|
||||
osinfo_small = 'EL/' + osvers
|
||||
if osname == 'RedHat':
|
||||
osinfo_small = 'EL/' + osvers
|
||||
if osname == 'Fedora':
|
||||
osinfo_small = 'F/' + osvers
|
||||
|
||||
# This is the primary key for diff.
|
||||
update_info = name + ' ' + rpms + ' ' + osinfo
|
||||
|
||||
rpms = int(rpms)
|
||||
uptime = int(uptime)
|
||||
|
||||
return locals()
|
||||
return Host(locals())
|
||||
|
||||
def lines2datas(lines):
|
||||
return (line2data(line) for line in lines)
|
||||
|
||||
# Filter datas using name as a filename wildcard match.
|
||||
def filter_name_datas(datas, name):
|
||||
for data in datas:
|
||||
if not fnmatch.fnmatch(data.name, name):
|
||||
continue
|
||||
yield data
|
||||
|
||||
# Filter datas using uptime as a minium.
|
||||
def filter_uptime_datas(datas, uptime):
|
||||
for data in datas:
|
||||
if data.uptime < uptime:
|
||||
continue
|
||||
yield data
|
||||
|
||||
# Sub. suffix of DNS names for UI
|
||||
def _ui_name(name):
|
||||
|
|
@ -172,6 +368,11 @@ def _ui_name(name):
|
|||
return name[:-len(suffix)] + conf_suffix_dns_replace[suffix]
|
||||
return name
|
||||
|
||||
def _ui_osinfo(data):
|
||||
if conf_small_osinfo:
|
||||
return data.osinfo_small
|
||||
return data.osinfo
|
||||
|
||||
# Reset the usage after _max_update()
|
||||
def _reset_ui_name():
|
||||
for suffix in sorted(_suffix_dns_replace):
|
||||
|
|
@ -200,37 +401,106 @@ def fname1():
|
|||
return bfname2lines(cmp)
|
||||
return fname2lines(fname)
|
||||
|
||||
_max_len_name = 0
|
||||
_max_len_rpms = 0
|
||||
_max_len_updur = 0
|
||||
_max_len_date = 0 # 4+1+2+1+2
|
||||
_max_len_name = 0
|
||||
_max_len_rpms = 0 # Number of rpm updates via. _ui_int().
|
||||
_max_len_upts = 0 # Uptime duration with short=True
|
||||
_max_len_date = 0 # 2025-08-04 = 4+1+2+1+2
|
||||
_max_terminal_width = shutil.get_terminal_size().columns
|
||||
if _max_terminal_width < 20:
|
||||
_max_terminal_width = 80
|
||||
_max_terminal_width -= 10
|
||||
def _max_update(prefix, lines):
|
||||
_max_terminal_width -= 14
|
||||
def _max_update(datas):
|
||||
for data in datas:
|
||||
_max_update_data(data)
|
||||
|
||||
def _max_update_data(data):
|
||||
global _max_len_name
|
||||
global _max_len_rpms
|
||||
global _max_len_updur
|
||||
global _max_len_upts
|
||||
global _max_len_date
|
||||
|
||||
for line in lines:
|
||||
data = line2data(line)
|
||||
name = _ui_name(data['name'])
|
||||
if len(name) > _max_len_name:
|
||||
_max_len_name = len(name)
|
||||
if len(str(data['rpms'])) > _max_len_rpms:
|
||||
_max_len_rpms = len(str(data['rpms']))
|
||||
updur_len = len(format_duration(data['uptime'], static=True))
|
||||
if updur_len > _max_len_updur:
|
||||
_max_len_updur = updur_len
|
||||
if len(data['date']) > _max_len_date:
|
||||
_max_len_date = len(data['date'])
|
||||
name = _ui_name(data.name)
|
||||
if len(name) > _max_len_name:
|
||||
_max_len_name = len(name)
|
||||
|
||||
rpms = _ui_int(data.rpms)
|
||||
if len(rpms) > _max_len_rpms:
|
||||
_max_len_rpms = len(rpms)
|
||||
|
||||
upts_len = len(_ui_dur(data.uptime))
|
||||
if upts_len > _max_len_upts:
|
||||
_max_len_upts = upts_len
|
||||
|
||||
if len(data.date) > _max_len_date:
|
||||
_max_len_date = len(data.date)
|
||||
|
||||
def _max_update_correct(prefix):
|
||||
global _max_len_name
|
||||
mw = _max_terminal_width - len(prefix)
|
||||
while _max_len_name + _max_len_rpms + _max_len_updur + _max_len_date >= mw:
|
||||
while _max_len_name + _max_len_rpms + _max_len_upts + _max_len_date >= mw:
|
||||
_max_len_name -= 1
|
||||
|
||||
# Return stats for updates added/deleted between two data sets.
|
||||
def _diffstats(data1, data2):
|
||||
uadd, udel = 0, 0
|
||||
|
||||
data1 = list(sorted(data1))
|
||||
data2 = list(sorted(data2))
|
||||
while len(data1) > 0 or len(data2) > 0:
|
||||
if len(data1) <= 0:
|
||||
d2 = data2.pop(0)
|
||||
uadd += d2.rpms
|
||||
continue
|
||||
if len(data2) <= 0:
|
||||
d1 = data1.pop(0)
|
||||
udel -= d1.rpms
|
||||
continue
|
||||
|
||||
d1 = data1[0]
|
||||
d2 = data2[0]
|
||||
|
||||
if d1.name < d2.name:
|
||||
udel -= d1.rpms
|
||||
data1.pop(0)
|
||||
continue
|
||||
|
||||
if d1.name > d2.name:
|
||||
uadd += d2.rpms
|
||||
data2.pop(0)
|
||||
continue
|
||||
|
||||
if d1 == d2:
|
||||
data1.pop(0)
|
||||
data2.pop(0)
|
||||
continue
|
||||
|
||||
if d1.osinfo != d2.osinfo:
|
||||
udel -= d1.rpms
|
||||
uadd += d2.rpms
|
||||
data1.pop(0)
|
||||
data2.pop(0)
|
||||
continue
|
||||
|
||||
# Now name is eq and osinfo is eq
|
||||
# So either new updates arrived, or we installed some and they went
|
||||
# down ... alas. we can't tell if both happened.
|
||||
if d1.rpms > d2.rpms:
|
||||
udel -= d1.rpms - d2.rpms
|
||||
if d1.rpms < d2.rpms:
|
||||
uadd += d2.rpms - d1.rpms
|
||||
data1.pop(0)
|
||||
data2.pop(0)
|
||||
|
||||
# diffstat returns...
|
||||
return uadd, udel
|
||||
|
||||
def _ui_diffstats(data1, data2):
|
||||
cmpds = _diffstats(data1, data2)
|
||||
return _ui_int(cmpds[0]), _ui_int(cmpds[1])
|
||||
|
||||
|
||||
|
||||
# This is the real __main__ start ...
|
||||
def _usage():
|
||||
prog = "updates+uptime"
|
||||
if sys.argv:
|
||||
|
|
@ -242,16 +512,16 @@ def _usage():
|
|||
help
|
||||
= This message.
|
||||
|
||||
backups
|
||||
= Show backups.
|
||||
backups-trim [days]
|
||||
= Cleanup old backups.
|
||||
|
||||
diff [backup1] [backup2]
|
||||
= See the difference between the current state and backups.
|
||||
diff-u [backup1] [backup2]
|
||||
= Shows before/after instead of modified (like diff -u).
|
||||
|
||||
history
|
||||
= Show history data.
|
||||
history-keep [days]
|
||||
= Cleanup old history.
|
||||
|
||||
host [host*] [backup]
|
||||
= See the current state of a host(s), can be filtered by name.
|
||||
info [host*] [backup]
|
||||
|
|
@ -280,12 +550,58 @@ def _usage():
|
|||
if cmd == "help":
|
||||
_usage()
|
||||
|
||||
if cmd == "backups":
|
||||
print("Backups:")
|
||||
for backup in backups:
|
||||
print('', backup, len(bfname2lines(backup)))
|
||||
def _backup_suffix(backup):
|
||||
suffix = ''
|
||||
if backup == backup_today:
|
||||
if ident:
|
||||
suffix = ' (today, is identical)'
|
||||
else:
|
||||
suffix = ' (today)'
|
||||
if backup == backup_yesterday:
|
||||
suffix = ' (yesterday)'
|
||||
return suffix
|
||||
|
||||
if cmd == "backups-keep":
|
||||
if cmd in ("backups", "hist", "history"):
|
||||
ident = _backup_today_identical()
|
||||
print("History:")
|
||||
last_name = "main"
|
||||
last_data = list(sorted(lines2datas(fname2lines(fname))))
|
||||
last_suff = ""
|
||||
|
||||
# We _could_ open+read+etc each file, just to find out the max updates for
|
||||
# all hist ... but len("Updates")+2=9 which means 9,999,999 updates)
|
||||
hl = len("Hosts")
|
||||
ul = len("Updates") + 2
|
||||
if conf_fast_width_history:
|
||||
ul += 2
|
||||
else:
|
||||
# Whatever, it's less memory than holding all history at once if you want
|
||||
# to enable it..
|
||||
for backup in reversed(backups):
|
||||
data = list(sorted(lines2datas(bfname2lines(backup))))
|
||||
updates = _ui_int(sum(d.rpms for d in data))
|
||||
hl = max(hl, len(_ui_int(len(data))))
|
||||
ul = max(ul, len(updates))
|
||||
|
||||
print(" %10s %*s %*s %*s %*s" % ("Day", hl, "Hosts",
|
||||
ul, "Updates", ul, "Avail", ul, "Inst."))
|
||||
|
||||
for backup in reversed(backups):
|
||||
data = list(sorted(lines2datas(bfname2lines(backup))))
|
||||
updates = _ui_int(sum(d.rpms for d in last_data))
|
||||
ul = max(ul, len(updates))
|
||||
cmpds = _ui_diffstats(data.copy(), last_data.copy())
|
||||
print(' %10s %*s %*s, %*s %*s, %s' % (last_name,
|
||||
hl, _ui_int(len(last_data)),
|
||||
ul, updates, ul, cmpds[0], ul+1, cmpds[1], last_suff))
|
||||
last_name = backup
|
||||
last_data = data
|
||||
last_suff = _backup_suffix(backup)
|
||||
updates = _ui_int(sum(d.rpms for d in last_data))
|
||||
print(' %10s %*s %*s %s' % (last_name, hl, _ui_int(len(last_data)),
|
||||
ul, updates, last_suff))
|
||||
|
||||
if cmd in ("backups-keep", "history-keep"):
|
||||
keep = 8
|
||||
if len(sys.argv) >= 2:
|
||||
keep = int(sys.argv.pop(1))
|
||||
|
|
@ -294,14 +610,16 @@ if cmd == "backups-keep":
|
|||
sys.exit(1)
|
||||
while keep < len(backups):
|
||||
# We just keep the newest N
|
||||
fn = fname + '.' + backups.pop(0)
|
||||
b = backups.pop(0)
|
||||
print("Removing:", b)
|
||||
fn = fname + '.' + b
|
||||
os.unlink(fn)
|
||||
|
||||
if cmd == "update":
|
||||
cmd = "update-flush"
|
||||
if not os.path.exists(fname):
|
||||
cmd = "update-daily" # This does the sorting etc.
|
||||
elif not os.path.exists(fname_today):
|
||||
elif fname_today is None:
|
||||
cmd = "update-daily"
|
||||
else:
|
||||
mtime = os.path.getmtime(fname)
|
||||
|
|
@ -322,101 +640,156 @@ if cmd == "update-daily": # Also create backup file.
|
|||
os.chdir("/srv/web/infra/ansible/playbooks")
|
||||
os.system("ansible-playbook generate-updates-uptimes-per-host-file.yml --flush-cache")
|
||||
|
||||
# Below here are the query commands, stuff needs to exist at this point.
|
||||
if not os.path.exists(fname):
|
||||
print(" Error: No main file. Run update sub-command", file=sys.stderr)
|
||||
sys.exit(4)
|
||||
if not _main_file_recent():
|
||||
print(" Warning: Main file is old. Run update sub-command", file=sys.stderr)
|
||||
if fname_today is None:
|
||||
print(" Warning: Backup for today does not exist!", file=sys.stderr)
|
||||
if fname_yesterday is None:
|
||||
print(" Warning: Backup for yesterday does not exist!", file=sys.stderr)
|
||||
|
||||
def _cli_match_host(data):
|
||||
if len(sys.argv) >= 2:
|
||||
host = sys.argv.pop(1)
|
||||
print("Matching:", host)
|
||||
data = filter_name_datas(data, host)
|
||||
data = list(data)
|
||||
if not data:
|
||||
print("Not host(s) matched:", host)
|
||||
sys.exit(2)
|
||||
return data
|
||||
|
||||
if cmd == "stats":
|
||||
_cmp_arg()
|
||||
data = fname1()
|
||||
data = lines2datas(fname1())
|
||||
if cmp_arg:
|
||||
sys.argv.pop(1)
|
||||
data = list(_cli_match_host(data))
|
||||
# Basically we have hosts/updates/uptime and we want 4 tiers of data:
|
||||
# 1. All. 2. OS name (Eg. Fedora). 3. OS name+version (Eg. Fedora 42).
|
||||
# 4. For updates/uptime a "few" hosts with the biggest numbers.
|
||||
osinfo = {'hosts' : {}, 'updates' : {}, 'uptimes' : {}}
|
||||
osdata = {'hosts' : {}, 'updates' : {}, 'uptimes' : {}, 'vers' : {}}
|
||||
updates = 0 # total updates
|
||||
most = [] # Tier 4, for updates
|
||||
awake = 0 # total uptime
|
||||
awakest = [] # Tier 4, for uptime
|
||||
_max_update('', data)
|
||||
_reset_ui_name()
|
||||
for line in data:
|
||||
d2 = line2data(line)
|
||||
|
||||
conf_suffix_dns_replace = {} # Turn off shortened names for stats...
|
||||
for d2 in data:
|
||||
# Tidy UI for OS names with only one version...
|
||||
if d2.osname not in osdata['vers']:
|
||||
osdata['vers'][d2.osname] = set()
|
||||
osdata['vers'][d2.osname].add(d2.osinfo)
|
||||
|
||||
# Tier 2/3 hosts...
|
||||
if d2['osname'] not in osinfo['hosts']:
|
||||
osinfo['hosts'][d2['osname']] = 0
|
||||
osinfo['hosts'][d2['osname']] += 1
|
||||
if d2.osname not in osdata['hosts']:
|
||||
osdata['hosts'][d2.osname] = 0
|
||||
osdata['hosts'][d2.osname] += 1
|
||||
|
||||
if d2['osinfo'] not in osinfo['hosts']:
|
||||
osinfo['hosts'][d2['osinfo']] = 0
|
||||
osinfo['hosts'][d2['osinfo']] += 1
|
||||
if d2.osinfo not in osdata['hosts']:
|
||||
osdata['hosts'][d2.osinfo] = 0
|
||||
osdata['hosts'][d2.osinfo] += 1
|
||||
|
||||
updates += d2['rpms']
|
||||
updates += d2.rpms
|
||||
# Tier 2/3 updates...
|
||||
if d2['osname'] not in osinfo['updates']:
|
||||
osinfo['updates'][d2['osname']] = 0
|
||||
osinfo['updates'][d2['osname']] += d2['rpms']
|
||||
if d2.osname not in osdata['updates']:
|
||||
osdata['updates'][d2.osname] = 0
|
||||
osdata['updates'][d2.osname] += d2.rpms
|
||||
|
||||
if d2['osinfo'] not in osinfo['updates']:
|
||||
osinfo['updates'][d2['osinfo']] = 0
|
||||
osinfo['updates'][d2['osinfo']] += d2['rpms']
|
||||
if d2.osinfo not in osdata['updates']:
|
||||
osdata['updates'][d2.osinfo] = 0
|
||||
osdata['updates'][d2.osinfo] += d2.rpms
|
||||
# Tier 4 updates...
|
||||
most.append((d2['rpms'], d2['name']))
|
||||
most.append((d2.rpms, d2.uptime, d2))
|
||||
most.sort()
|
||||
while len(most) > conf_stat_4_hosts:
|
||||
most.pop(0)
|
||||
|
||||
awake += d2['uptime']
|
||||
awake += d2.uptime
|
||||
# Tier 2/3 uptimes...
|
||||
if d2['osname'] not in osinfo['uptimes']:
|
||||
osinfo['uptimes'][d2['osname']] = 0
|
||||
osinfo['uptimes'][d2['osname']] += d2['uptime']
|
||||
if d2.osname not in osdata['uptimes']:
|
||||
osdata['uptimes'][d2.osname] = 0
|
||||
osdata['uptimes'][d2.osname] += d2.uptime
|
||||
|
||||
if d2['osinfo'] not in osinfo['uptimes']:
|
||||
osinfo['uptimes'][d2['osinfo']] = 0
|
||||
osinfo['uptimes'][d2['osinfo']] += d2['uptime']
|
||||
if d2.osinfo not in osdata['uptimes']:
|
||||
osdata['uptimes'][d2.osinfo] = 0
|
||||
osdata['uptimes'][d2.osinfo] += d2.uptime
|
||||
# Tier 4 uptimes...
|
||||
awakest.append((d2['uptime'], d2['name']))
|
||||
awakest.append((d2.uptime, d2.rpms, d2))
|
||||
awakest.sort()
|
||||
while len(awakest) > conf_stat_4_hosts:
|
||||
awakest.pop(0)
|
||||
|
||||
print("Hosts:", len(data))
|
||||
for osi in sorted(osinfo['hosts']):
|
||||
# Print "stats"
|
||||
# _max_update(data)
|
||||
# Do this by hand...
|
||||
_max_len_name = max((len(d.name) for d in data))
|
||||
_max_len_rpms = max(len("Updates"), len(_ui_int(updates)))
|
||||
_max_len_upts = len(_ui_dur(awake))
|
||||
_max_len_date = 0
|
||||
_max_update_correct(' ')
|
||||
print("%-16s %6s %*s %*s" % ("OS", "Hosts",
|
||||
_max_len_rpms, "Updates", _max_len_upts, "Uptime"))
|
||||
print("-" * (16+2+6+1+_max_len_rpms+1+_max_len_upts))
|
||||
print("%-16s: %6s %*s %*s" % ("All", _ui_int(len(data)),
|
||||
_max_len_rpms, _ui_int(updates), _max_len_upts, _ui_dur(awake)))
|
||||
subprefix = ''
|
||||
subplen = 12
|
||||
for osi in sorted(osdata['hosts']):
|
||||
if '/' not in osi:
|
||||
print(" %s: %s" % (osi, osinfo['hosts'][osi]))
|
||||
if len(osdata['vers'][osi]) == 1:
|
||||
subprefix = ''
|
||||
subplen = 14
|
||||
continue
|
||||
subprefix = ' '
|
||||
subplen = 12
|
||||
print(" %-14s: %6s %*s %*s" % (osi, _ui_int(osdata['hosts'][osi]),
|
||||
_max_len_rpms, _ui_int(osdata['updates'][osi]),
|
||||
_max_len_upts, _ui_dur(osdata['uptimes'][osi])))
|
||||
if '/' in osi:
|
||||
print(" %s: %s" % (osi, osinfo['hosts'][osi]))
|
||||
print("Updates:", updates)
|
||||
for osi in sorted(osinfo['updates']):
|
||||
if '/' not in osi:
|
||||
print(" %s: %s" % (osi, osinfo['updates'][osi]))
|
||||
if '/' in osi:
|
||||
print(" %s: %s" % (osi, osinfo['updates'][osi]))
|
||||
print(" %s%-*s: %6s %*s %*s" % (subprefix, subplen, osi,
|
||||
_ui_int(osdata['hosts'][osi]),
|
||||
_max_len_rpms, _ui_int(osdata['updates'][osi]),
|
||||
_max_len_upts, _ui_dur(osdata['uptimes'][osi])))
|
||||
print("-" * (16+2+6+1+_max_len_rpms+1+_max_len_upts))
|
||||
# Redo the lengths, because it's real hostname data now...
|
||||
_max_update(data)
|
||||
_max_len_date = 0
|
||||
_max_update_correct(' ')
|
||||
if most:
|
||||
# print("")
|
||||
print("Hosts with the most Updates:")
|
||||
for m in most:
|
||||
print(" Most: %-*s %s" % (_max_len_name, _ui_name(m[1]), m[0]))
|
||||
print("Uptime:", format_duration(awake))
|
||||
for osi in sorted(osinfo['uptimes']):
|
||||
if '/' not in osi:
|
||||
print(" %s: %s" % (osi, format_duration(osinfo['uptimes'][osi], True)))
|
||||
if '/' in osi:
|
||||
print(" %s: %s" % (osi,format_duration(osinfo['uptimes'][osi],True)))
|
||||
print(" %-*s %*s %*s %s" % (_max_len_name, m[2],
|
||||
_max_len_rpms, _ui_int(m[0]), _max_len_upts, _ui_dur(m[1]),
|
||||
_ui_osinfo(m[2])))
|
||||
if awakest:
|
||||
# print("")
|
||||
print("Hosts with the most Uptime:")
|
||||
for a in awakest:
|
||||
print(" Most: %-*s %s" % (_max_len_name, _ui_name(a[1]),
|
||||
format_duration(a[0], True)))
|
||||
print(" %-*s %*s %*s %s" % (_max_len_name, a[2],
|
||||
_max_len_rpms, _ui_int(a[1]), _max_len_upts, _ui_dur(a[0]),
|
||||
_ui_osinfo(a[2])))
|
||||
_explain_ui_name()
|
||||
|
||||
def _print_info(host, lines):
|
||||
hosts = []
|
||||
for x in lines:
|
||||
x = line2data(x)
|
||||
if fnmatch.fnmatch(x['name'], host):
|
||||
if fnmatch.fnmatch(x.name, host):
|
||||
hosts.append(x)
|
||||
if not hosts:
|
||||
print("Not found host:", host)
|
||||
print("Not host(s) matched:", host)
|
||||
sys.exit(2)
|
||||
for host in hosts:
|
||||
print("Host:", host['name'])
|
||||
print(" OS:", host['osinfo'])
|
||||
print(" Rpms:", host['rpms'])
|
||||
print(" Uptime:", format_duration(host['uptime']))
|
||||
print(" Date:", host['date'])
|
||||
print("Host:", host.name)
|
||||
print(" OS:", host.osinfo)
|
||||
print(" Updates:", _ui_int(host.rpms))
|
||||
print(" Uptime:", format_duration(host.uptime)) # !ui_dur
|
||||
print(" Checked:", host.date)
|
||||
|
||||
if cmd in ("host", "info"):
|
||||
if cmd == "host":
|
||||
|
|
@ -435,11 +808,11 @@ if cmd in ("host", "info"):
|
|||
_print_info(host, fname1())
|
||||
|
||||
def _print_line(prefix, data):
|
||||
print("%s%-*s %*u %*s %*s %s" % (prefix,
|
||||
_max_len_name, _ui_name(data['name']),
|
||||
_max_len_rpms, data['rpms'],
|
||||
_max_len_updur, format_duration(data['uptime'], static=True),
|
||||
_max_len_date, data['date'], data['osinfo_small']))
|
||||
print("%s%-*s %*s %*s %*s %s" % (prefix,
|
||||
_max_len_name, _ui_name(data.name),
|
||||
_max_len_rpms, _ui_int(data.rpms),
|
||||
_max_len_upts, _ui_dur(data.uptime),
|
||||
_max_len_date, data.date, _ui_osinfo(data)))
|
||||
|
||||
|
||||
if cmd == "list":
|
||||
|
|
@ -448,13 +821,11 @@ if cmd == "list":
|
|||
host = sys.argv.pop(1)
|
||||
|
||||
_cmp_arg()
|
||||
lines = fname1()
|
||||
_max_update('', lines)
|
||||
_reset_ui_name()
|
||||
for line in lines:
|
||||
d1 = line2data(line)
|
||||
if not fnmatch.fnmatch(d1['name'], host):
|
||||
continue
|
||||
data = lines2datas(fname1())
|
||||
data = list(filter_name_datas(data, host))
|
||||
_max_update(data)
|
||||
_max_update_correct('')
|
||||
for d1 in data:
|
||||
_print_line('', d1)
|
||||
_explain_ui_name()
|
||||
|
||||
|
|
@ -464,13 +835,11 @@ if cmd == "uptime":
|
|||
age = parse_duration(sys.argv.pop(1))
|
||||
|
||||
_cmp_arg()
|
||||
lines = fname1()
|
||||
_max_update('', lines)
|
||||
_reset_ui_name()
|
||||
for line in lines:
|
||||
d1 = line2data(line)
|
||||
if age > d1['uptime']:
|
||||
continue
|
||||
data = lines2datas(fname1())
|
||||
data = list(filter_uptime_datas(data, age))
|
||||
_max_update(data)
|
||||
_max_update_correct('')
|
||||
for d1 in data:
|
||||
_print_line('', d1)
|
||||
_explain_ui_name()
|
||||
|
||||
|
|
@ -479,38 +848,49 @@ if cmd in ("diff", "diff-u"):
|
|||
fn1 = fname + '.' + cmp
|
||||
fn2 = fname
|
||||
data1 = fname2lines(fn1)
|
||||
if len(sys.argv) >= 3 and sys.argv[2] in backups:
|
||||
if len(sys.argv) >= 3:
|
||||
# Doing a diff. between two backups...
|
||||
fn2 = fname + '.' + sys.argv[2]
|
||||
if sys.argv[2] == 'today' and fname_today is not None:
|
||||
fn2 = fname_today
|
||||
if sys.argv[2] == 'yesterday' and fname_yesterday is not None:
|
||||
fn2 = fname_yesterday
|
||||
if sys.argv[2] in backups:
|
||||
fn2 = fname + '.' + sys.argv[2]
|
||||
data2 = fname2lines(fn2)
|
||||
print("diff %s %s" % (fn1, fn2), file=sys.stderr)
|
||||
_max_update(' ', data1)
|
||||
_max_update(' ', data2)
|
||||
_reset_ui_name()
|
||||
data1 = list(sorted(lines2datas(data1)))
|
||||
data2 = list(sorted(lines2datas(data2)))
|
||||
hosts = _ui_int(len(data2))
|
||||
updates = _ui_int(sum(d.rpms for d in data2))
|
||||
ul = len(updates)
|
||||
cmpds = _ui_diffstats(data1.copy(), data2.copy())
|
||||
_max_update(data1)
|
||||
_max_update(data2)
|
||||
_max_update_correct(' ')
|
||||
while len(data1) > 0 or len(data2) > 0:
|
||||
if len(data1) <= 0:
|
||||
_print_line('+', line2data(data2[0]))
|
||||
_print_line('+', data2[0])
|
||||
data2.pop(0)
|
||||
continue
|
||||
if len(data2) <= 0:
|
||||
_print_line('-', line2data(data1[0]))
|
||||
_print_line('-', data1[0])
|
||||
data1.pop(0)
|
||||
continue
|
||||
|
||||
d1 = line2data(data1[0])
|
||||
d2 = line2data(data2[0])
|
||||
d1 = data1[0]
|
||||
d2 = data2[0]
|
||||
|
||||
if d1['name'] < d2['name']:
|
||||
if d1.name < d2.name:
|
||||
_print_line('-', d1)
|
||||
data1.pop(0)
|
||||
continue
|
||||
|
||||
if d1['name'] > d2['name']:
|
||||
if d1.name > d2.name:
|
||||
_print_line('+', d2)
|
||||
data2.pop(0)
|
||||
continue
|
||||
|
||||
if d1['update_info'] == d2['update_info']:
|
||||
if d1 == d2:
|
||||
_print_line(' ', d2)
|
||||
data1.pop(0)
|
||||
data2.pop(0)
|
||||
|
|
@ -529,5 +909,6 @@ if cmd in ("diff", "diff-u"):
|
|||
data2.pop(0)
|
||||
|
||||
continue
|
||||
print('hosts=%s updates=%s (a=%s i=%s)' % (hosts, updates, cmpds[0],cmpds[1]))
|
||||
_explain_ui_name()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue