diff --git a/files/scripts/updates-uptime-cmd.py b/files/scripts/updates-uptime-cmd.py index 014997c70e..91b928e4e1 100755 --- a/files/scripts/updates-uptime-cmd.py +++ b/files/scripts/updates-uptime-cmd.py @@ -13,9 +13,9 @@ import sys import argparse import fnmatch -import locale import re import shutil +import subprocess import time # Use utf8 prefixes in diff, these need to be a "normal" width 1 character @@ -425,7 +425,6 @@ def _pre_cmd__verbose(args): globals()['conf_short_duration'] = False globals()['conf_stat_4_hosts'] *= (args.verbose * 2) - def _wild_eq(s1, s2): """ Compare two strings, but allow '?' to mean anything. """ if s1 == '?' or s2 == '?': @@ -1058,7 +1057,52 @@ def _cmd_history_keep(args): fn = fname + '.' + b os.unlink(fn) +def inventory_hosts(): + # The "correct" way to do this is something like: + # ansible-inventory --list | jq -r '._meta.hostvars | keys[]' + # ...but that is _much_ slower, as it's loading a lot of data/facts which + # we ignore. + cmds = ["ansible", "all", "--list-host"] + p = subprocess.Popen(cmds, text=True, stdout=subprocess.PIPE) + header = p.stdout.readline() + if not header.strip().startswith("hosts ("): + return set() + + ret = set() + for line in p.stdout: + ret.add(line.strip()) + return ret + +def remove_old_hosts(): + if not os.path.exists(fname): + return + + inv = inventory_hosts() + if not inv: # If we can't get inventory, don't delete everything. + return + + odata = fname2lines(fname) + fo = open(fname + ".rm_old_hosts.tmp", "w") + for line in odata: + host = line.split()[0] + if host not in inv: + print("Removing host:", host) + continue + fo.write(line) + fo.write("\n") + fo.close() + os.rename(fname + ".rm_old_hosts.tmp", fname) + +def _cmd_remove_old_hosts(args): + remove_old_hosts() + def _cmd_update(args): + # First remove machines that aren't in inventory anymore, as the playbook + # only adds/updates them. + if cmd != "update-daily-refresh": # Just wastes time if refresh. + remove_old_hosts() + + # Now update whatever is left. cmd = args.cmd if cmd == "update": cmd = "update-flush" @@ -1811,6 +1855,33 @@ def _cmdline_arg_hist(oval): msg += "\n History:", ", ".join([] + names + backups) raise argparse.ArgumentTypeError(msg) +_cmds_als = { + "created" : ["installed", "reinstalled"], + "difference" : ("diff", "diff-u"), + "history-keep" : [], + "history" : ["hist"], + "hosts" : ("host", "host-u", "hosts-u"), + "information" : ["info"], + "list" : [], + "list-n" : [], + "old-list" : ["olist"], + "oslist" : [], + "oslist-n" : [], + "statistics" : ["stats"], + "update" : [], + "update-daily" : [], + "update-daily-refresh" : [], + "update-fast" : [], + "update-host" : [], + "uptime-min" : ["uptime"], + "uptime-max" : ("rebooted", "started"), + None : set(), +} +for c in _cmds_als: + if c is None: continue + _cmds_als[c] = set(_cmds_als[c]) + _cmds_als[None].update(_cmds_als[c]) + _cmds_als[None].add(c) def _cmd_help(args): prog = "updates+uptime" @@ -1818,7 +1889,22 @@ def _cmd_help(args): prog = os.path.basename(sys.argv[0]) if not args.hcmd: _usage() - elif args.hcmd in ("created", "installed", "reinstalled"): + if args.hcmd not in _cmds_als[None]: + print(" Unknown command:", args.hcmd) + _usage() + + def _eq_cmd(x): + return args.hcmd == x or args.hcmd in _cmds_als[x] + def _hlp_als(x): + if not _cmds_als[x]: + return '' + als = set() + als.add(x) + als.update(_cmds_als[x]) + als.remove(args.hcmd) + return f"""Aliases: {", ".join(sorted(als))}\n""" + + if _eq_cmd("created"): print(f"""\ Usage: {prog} {args.hcmd} [duration] [host*] @@ -1828,21 +1914,33 @@ forever. Easy way to see new machines. +{_hlp_als("created")} Eg. {prog} {args.hcmd} 0 *-test* {prog} {args.hcmd} 5w vmhost* bvmhost* buildhw* """, end='') - elif args.hcmd in ("diff", "diff-u"): + + elif _eq_cmd("difference"): print(f"""\ - Usage: {prog} diff [backup1] [backup2] + Usage: {prog} {args.hcmd} [backup1] [backup2] See the difference between the current state and backups. The -u variant shows before/after instead of modified. +utf8: {conf_utf8} +{_conf_utf8_boot_ed} = Rebooted +{_conf_utf8_boot_up} = Rebooted and updated +{_conf_utf8_more_up} = More updates +{_conf_utf8_less_up} = Less updates +{_conf_utf8_diff_os} = Different OS information, but machine id is the same +{_conf_utf8_diff_hw} = Machine id is different + +{_hlp_als("difference")} Eg. {prog} {args.hcmd} {prog} {args.hcmd} yesterday {prog} {args.hcmd} 2025-08-16 main - """, end='') - elif args.hcmd in ("history", "hist"): +""", end='') + + elif _eq_cmd("history"): print(f"""\ Usage: {prog} {args.hcmd} @@ -1850,9 +1948,11 @@ forever. Kind of like: git log --pretty=oneline --abbrev-commit --decorate +{_hlp_als("history")} Eg. {prog} {args.hcmd} """, end='') - elif args.hcmd == "history-keep": + + elif _eq_cmd("history-keep"): print(f"""\ Usage: {prog} {args.hcmd} [days] @@ -1861,11 +1961,13 @@ forever. if you pass "1". Logrotate, for history. - + + {_hlp_als("history-keep")} Eg. {prog} {args.hcmd} {prog} {args.hcmd} 32 """, end='') - elif args.hcmd in ("host", "hosts", "host-u", "hosts-u"): + + elif _eq_cmd("hosts"): print(f"""\ Usage: {prog} {args.hcmd} @@ -1875,11 +1977,21 @@ if you pass "1". Easiest way to see how hosts have changed over time, the more history the better for this. Kind of like git blame, but instead of lines it's events. +utf8: {conf_utf8} +{_conf_utf8_boot_ed} = Rebooted +{_conf_utf8_boot_up} = Rebooted and updated +{_conf_utf8_more_up} = More updates +{_conf_utf8_less_up} = Less updates +{_conf_utf8_diff_os} = Different OS information, but machine id is the same +{_conf_utf8_diff_hw} = Machine id is different + + {_hlp_als("host")} Eg. {prog} {args.hcmd} {prog} {args.hcmd} 'batcave*' {prog} {args.hcmd} 'batcave*' 'noc*' """, end='') - elif args.hcmd in ("information", "info"): + + elif _eq_cmd("information"): print(f"""\ Usage: {prog} {args.hcmd} [host*] [backup] [backup]... @@ -1887,31 +1999,37 @@ better for this. Kind of like git blame, but instead of lines it's events. If you want to compare things by hand, use this. + {_hlp_als("information")} Eg. {prog} {args.hcmd} {prog} {args.hcmd} 'batcave*' {prog} {args.hcmd} 'noc*' main yesterday """, end='') - elif args.hcmd in ("list",): + + elif _eq_cmd("list"): print(f"""\ Usage: {prog} {args.hcmd} [host*] [backup] [host*]... See the current state of the hosts, can be filtered by name. + {_hlp_als("list")} Eg. {prog} {args.hcmd} {prog} {args.hcmd} 'batcave*' {prog} {args.hcmd} 'noc*' yesterday '*-test.*' """, end='') - elif args.hcmd in ("list-n",): + + elif _eq_cmd("list-n"): print(f"""\ Usage: {prog} {args.hcmd} [host*] [host*]... See the current state of the hosts, can be filtered by name. + {_hlp_als("list-n")} Eg. {prog} {args.hcmd} {prog} {args.hcmd} 'batcave*' {prog} {args.hcmd} 'batcave*' 'noc*' """, end='') - elif args.hcmd in ("old-list", ): + + elif _eq_cmd("old-list"): print(f"""\ Usage: {prog} {args.hcmd} duration @@ -1919,29 +2037,35 @@ better for this. Kind of like git blame, but instead of lines it's events. Easiest way to see what hosts we aren't getting data for. + {_hlp_als("old-list")} Eg. {prog} {args.hcmd} 2d """, end='') - elif args.hcmd in ("oslist",): + + elif _eq_cmd("oslist"): print(f"""\ Usage: {prog} {args.hcmd} [os*] [backup] [os*]... See the current state, can be filtered by OS. + {_hlp_als("oslist")} Eg. {prog} {args.hcmd} {prog} {args.hcmd} RedHat {prog} {args.hcmd} 10 yesterday 41 """, end='') - elif args.hcmd in ("oslist-n",): + + elif _eq_cmd("oslist-n"): print(f"""\ Usage: {prog} {args.hcmd} [os*] [os*]... See the current state, can be filtered by OS. + {_hlp_als("oslist-n")} Eg. {prog} {args.hcmd} {prog} {args.hcmd} RedHat {prog} {args.hcmd} F 10 """, end='') - elif args.hcmd in ("statistics", "stats"): + + elif _eq_cmd("statistics"): print(f"""\ Usage: {prog} {args.hcmd} [backup] [host*] [host*]... @@ -1949,45 +2073,37 @@ better for this. Kind of like git blame, but instead of lines it's events. Easiest way to see the current state of the hosts. + {_hlp_als("statistics")} Eg. {prog} {args.hcmd} {prog} {args.hcmd} yesterday {prog} {args.hcmd} newest '*.stg.*' '*-test.*' """, end='') - elif args.hcmd in ("update",): + + elif _eq_cmd("update"): print(f"""\ Usage: {prog} {args.hcmd} Run update-fast, or update-daily if no daily backup. Easiest way to update, from cron or cmdline after you change things. DTRT. + Hosts removed from inventory are automatically removed on {args.hcmd}. + {_hlp_als("update")} Eg. {prog} {args.hcmd} """, end='') - elif args.hcmd in ("update-fast",): - print(f"""\ - Usage: {prog} {args.hcmd} - - Update the data for the hosts, in the main file (creating it if needed). - - Eg. {prog} {args.hcmd} -""", end='') - elif args.hcmd in ("update-host",): - print(f"""\ - Usage: {prog} {args.hcmd} host* - - Run update-fast, only for the specified host(s). - - Eg. {prog} {args.hcmd} bat\\* -""", end='') - elif args.hcmd in ("update-daily",): + + elif _eq_cmd("update-daily"): print(f"""\ Usage: {prog} {args.hcmd} Run update-fast and force do a backup for today. + Hosts removed from inventory are automatically removed on {args.hcmd}. + {_hlp_als("update-daily")} Eg. {prog} {args.hcmd} """, end='') - elif args.hcmd in ("update-daily-refresh",): + + elif _eq_cmd("update-daily-refresh"): print(f"""\ Usage: {prog} {args.hcmd} @@ -1995,9 +2111,33 @@ better for this. Kind of like git blame, but instead of lines it's events. Easiest way to refresh the current history for today. + {_hlp_als("update-daily-refresh")} Eg. {prog} {args.hcmd} """, end='') - elif args.hcmd in ("uptime", "uptime-min"): + + elif _eq_cmd("update-fast"): + print(f"""\ + Usage: {prog} {args.hcmd} + + Update the data for the hosts, in the main file (creating it if needed). + Hosts removed from inventory are automatically removed on {args.hcmd}. + + {_hlp_als("update-fast")} + Eg. {prog} {args.hcmd} +""", end='') + + elif _eq_cmd("update-host"): + print(f"""\ + Usage: {prog} {args.hcmd} host* + + Run update-fast, only for the specified host(s). + Hosts removed from inventory are automatically removed on {args.hcmd}. + + {_hlp_als("update-host")} + Eg. {prog} {args.hcmd} bat\\* +""", end='') + + elif _eq_cmd("uptime-min"): print(f"""\ Usage: {prog} {args.hcmd} duration [backup] @@ -2005,10 +2145,12 @@ better for this. Kind of like git blame, but instead of lines it's events. Easy way to see what has been rebooted recently. + {_hlp_als("uptime-min")} Eg. {prog} {args.hcmd} 32h {prog} {args.hcmd} 1d yesterday """, end='') - elif args.hcmd in ("uptime-max", "rebooted", "started"): + + elif _eq_cmd("uptime-max"): print(f"""\ Usage: {prog} {args.hcmd} duration [backup] @@ -2016,12 +2158,11 @@ better for this. Kind of like git blame, but instead of lines it's events. Easy way to see what hasn't been rebooted recently. + {_hlp_als("uptime-max")} Eg. {prog} {args.hcmd} 26w {prog} {args.hcmd} 4w4d yesterday """, end='') - else: - print(" Unknown command:", args.hcmd) - _usage() + def _main(): global conf_ansi_terminal @@ -2076,7 +2217,7 @@ def _main(): # -- Start of the real commands... - als = ['installed', 'reinstalled'] + als = _cmds_als["created"] cmd = subparsers.add_parser("created", aliases=als, help="list hosts") cmd.add_argument("dur", nargs='?', default=conf_created_dur_def, type=_cmdline_arg_duration, help="created within duration") @@ -2084,19 +2225,21 @@ def _main(): __defs(func=_cmd_created) # diff/diff-u commands - cmd = subparsers.add_parser("diff", aliases=['diff-u'], help="diff") + als = _cmds_als["difference"] + cmd = subparsers.add_parser("difference", aliases=als, help="diff") cmd.add_argument("hists", nargs='*', type=_cmdline_arg_hist, help="history file") __defs(func=_cmd_diff) # hosts/hosts-u commands - als = ['host', 'hosts-u', 'host-u'] + als = _cmds_als["hosts"] hlp = "show history data about specific hosts" cmd = subparsers.add_parser("hosts", aliases=als, help=hlp) cmd.add_argument("hosts", nargs='*', help="wildcard hostname(s)") __defs(func=_cmd_host) # history command - cmd = subparsers.add_parser("history", aliases=['hist'], help="show history") + als = _cmds_als["history"] + cmd = subparsers.add_parser("history", aliases=als, help="show history") __defs(func=_cmd_history) # history-keep command @@ -2107,8 +2250,9 @@ def _main(): __defs(func=_cmd_history_keep) # info command + als = _cmds_als["information"] hlp = "show host information" - cmd = subparsers.add_parser("information", aliases=['info'], help=hlp) + cmd = subparsers.add_parser("information", aliases=als, help=hlp) cmd.add_argument("host", nargs='?', help="wildcard hostname") cmd.add_argument("hists", nargs='*', type=_cmdline_arg_hist, help="history file") @@ -2126,7 +2270,8 @@ def _main(): cmd.add_argument("hosts", nargs='*', help="wildcard hostname(s)") __defs(func=_cmd_list) - cmd = subparsers.add_parser("old-list", aliases=['olist'],help="list hosts") + als = _cmds_als["old-list"] + cmd = subparsers.add_parser("old-list", aliases=als, help="list hosts") cmd.add_argument("dateage", type=_cmdline_arg_duration, help="data age minimum duration") __defs(func=_cmd_list) @@ -2151,6 +2296,8 @@ def _main(): __defs(func=_cmd_stats) # update commands + # We pretend these are sep. commands, like oslist vs. list, but don't C&P. + # als = _cmds_als["update"] als = ['update-daily','update-daily-refresh', 'update-fast', 'update-flush'] cmd = subparsers.add_parser("update", aliases=als, help="update DB") __defs(func=_cmd_update) @@ -2159,14 +2306,17 @@ def _main(): cmd.add_argument("host", help="wildcard hostname") __defs(func=_cmd_update) + cmd = subparsers.add_parser("remove-old-hosts", help="update DB") + __defs(func=_cmd_remove_old_hosts) + # uptime/uptime-min/uptime-max commands - als = ['rebooted','started'] + als = _cmds_als["uptime-max"] cmd = subparsers.add_parser("uptime-max", aliases=als, help="list hosts") cmd.add_argument("dur", type=_cmdline_arg_duration, help="uptime maximum duration") __defs(func=_cmd_uptime) - als = ['uptime'] + als = _cmds_als["uptime-min"] cmd = subparsers.add_parser("uptime-min", help="list hosts", aliases=als) cmd.add_argument("dur", type=_cmdline_arg_duration, help="uptime minimum duration")