openQA dist-git PR test result reporting config (staging)

This *should* configure a new queue and consumer for openQA to
report dist-git PR test results back. It's all a bit speculative
ATM, may need some tweaking. Mostly applied only on staging for
now.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

inventory/group_vars/openqa_lab

@@ -24,6 +24,10 @@ openqa_amqp_this_url: "{{ openqa_amqp_stg_url }}"
 openqa_amqp_this_username: "{{ openqa_amqp_stg_username }}"
 openqa_amqp_scheduler_queue: "{{ openqa_amqp_prod_username }}_scheduler_stg"
 
+# fedora-messaging dist-git reporter settings (not enabled on prod yet)
+openqa_amqp_distgit_reporter_queue: "{{ openqa_amqp_this_username }}_distgit_reporter"
+openqa_amqp_distgit_reporter_routing_keys: ["org.fedoraproject.{{ deployment_type }}.openqa.job.done"]
+
 # auth stuff
 openqa_oauth2_secret: "{{ openqa_stg_oidc_secret }}"
 

roles/openqa/dispatcher/defaults/main.yml

@@ -16,5 +16,6 @@ openqa_amqp_reporter_key: /etc/fedora-messaging/fedora-key.pem
 openqa_amqp_reporter_cert: /etc/fedora-messaging/fedora-cert.pem
 openqa_amqp_resultsdb_reporter_routing_keys: ["org.fedoraproject.prod.openqa.job.done"]
 openqa_amqp_wiki_reporter_routing_keys: ["org.fedoraproject.prod.openqa.job.done"]
+openqa_amqp_distgit_reporter_routing_keys: ["org.fedoraproject.prod.openqa.job.done"]
 openqa_amqp_mailfrom: "root@{{ external_hostname }}"
 openqa_amqp_smtp: localhost

roles/openqa/dispatcher/handlers/main.yml

@@ -17,3 +17,9 @@
   listen:
   - Restart openqa consumers
   - Restart openqa wiki consumer
+
+- name: Conditionally restart openQA dist-git consumer
+  ansible.builtin.command: /usr/local/bin/conditional-restart.sh fm-consumer@fedora_openqa_distgit_reporter
+  listen:
+  - Restart openqa consumers
+  - Restart openqa dist-git consumer

roles/openqa/dispatcher/tasks/main.yml

@@ -144,6 +144,17 @@
 ##               wiki reporter to subscribe to. The role default
 ##               for this is the appropriate keys for the Fedora
 ##               production broker.
+# - openqa_amqp_distgit_reporter_queue
+##      string - Message queue name for the fedora-messaging dist-git
+##               reporter consumer. If this is set, the dist-git
+##               reporter will be enabled. You **SHOULD NOT** enable
+##               this reporter on anything but the official Fedora
+##               infra deployments.
+# - openqa_amqp_distgit_reporter_routing_keys
+##        list - List of routing key names for the fedora-messaging
+##               dist-git reporter to subscribe to. The role default
+##               for this is the appropriate keys for the Fedora
+##               production broker.
 # - openqa_amqp_mailto
 ##        list - List of email addresses to email errors to. If set,
 ##               the email log handler will be configured.
@@ -350,6 +361,14 @@
   - config
   - fedora-messaging
 
+- name: Configure fedora-messaging dist-git reporter
+  ansible.builtin.template: src=fedora_openqa_distgit_reporter.toml.j2 dest=/etc/fedora-messaging/fedora_openqa_distgit_reporter.toml owner=root group=root mode=0640
+  notify:
+  - Restart openqa dist-git consumer
+  when: "openqa_amqp_distgit_reporter_queue is defined and openqa_amqp_distgit_reporter_queue"
+  tags:
+  - config
+
 - name: Enable and start fedora-messaging scheduler service
   service: name=fm-consumer@fedora_openqa_scheduler enabled=yes state=started
   when: "openqa_amqp_scheduler_queue is defined and openqa_amqp_scheduler_queue"
@@ -368,6 +387,10 @@
   tags:
   - fedora-messaging
 
+- name: Enable and start fedora-messaging dist-git reporter service
+  service: name=fm-consumer@fedora_openqa_distgit_reporter enabled=yes state=started
+  when: "openqa_amqp_distgit_reporter_queue is defined and openqa_amqp_distgit_reporter_queue"
+
 - name: Create directory for consumer service config drop-in
   ansible.builtin.file: path=/etc/systemd/system/fm-consumer@.service.d state=directory owner=root group=root mode=0755
   tags:

roles/openqa/dispatcher/templates/fedora_openqa_distgit_reporter.toml.j2

@@ -0,0 +1,115 @@
+# fedora-messaging consumer configuration file for the production
+# openQA dist-git reporter. Note you must change the UUID here to
+# something unique before using this.
+#
+# This file is in the TOML format.
+
+amqp_url = "{{ openqa_amqp_reporter_url }}"
+{% if openqa_amqp_passive|bool %}
+passive_declares = true
+{% endif %}
+callback = "fedora_openqa.consumer:OpenQADistgitReporter"
+
+[tls]
+ca_cert = "{{ openqa_amqp_reporter_cacert }}"
+keyfile = "{{ openqa_amqp_reporter_key }}"
+certfile = "{{ openqa_amqp_reporter_cert }}"
+
+[client_properties]
+app = "Fedora openQA dist-git reporter"
+app_url = "https://forge.fedoraproject.org/quality/fedora_openqa"
+app_contacts_email = ["adamwill@fedoraproject.org", "lruzicka@redhat.com", "qa-devel@lists.fedoraproject.org"]
+
+[exchanges."amq.topic"]
+type = "topic"
+durable = true
+auto_delete = false
+arguments = {}
+
+[queues."{{ openqa_amqp_distgit_reporter_queue }}"]
+durable = true
+auto_delete = false
+exclusive = false
+arguments = {}
+
+[[bindings]]
+queue = "{{ openqa_amqp_distgit_reporter_queue }}"
+exchange = "amq.topic"
+routing_keys = [{% for key in openqa_amqp_distgit_reporter_routing_keys %}"{{ key }}",{% endfor %}]
+
+[consumer_config]
+openqa_hostname = "{{ openqa_hostname }}"
+openqa_baseurl = "https://{{ external_hostname|default(ansible_nodename) }}"
+do_report = true
+
+[qos]
+prefetch_size = 0
+prefetch_count = 25
+
+[log_config]
+version = 1
+disable_existing_loggers = true
+
+[log_config.formatters.simple]
+format = "[%(levelname)s %(name)s] %(message)s"
+
+[log_config.handlers.console]
+class = "logging.StreamHandler"
+formatter = "simple"
+stream = "ext://sys.stdout"
+
+{% if openqa_amqp_mailto is defined %}
+[log_config.handlers.email]
+class = "logging.handlers.SMTPHandler"
+formatter = "simple"
+level = "ERROR"
+mailhost = "{{ openqa_amqp_smtp }}"
+fromaddr = "{{ openqa_amqp_mailfrom }}"
+toaddrs = [{% for key in openqa_amqp_mailto %}"{{ key }}",{% endfor %}]
+subject = "openQA dist-git reporter error"
+{% endif %}
+
+[log_config.loggers.OpenQADistgitReporter]
+level = "INFO"
+propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
+handlers = ["console"]
+{% endif %}
+
+[log_config.loggers.fedora_messaging]
+level = "INFO"
+propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
+handlers = ["console"]
+{% endif %}
+
+[log_config.loggers.twisted]
+level = "INFO"
+propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
+handlers = ["console"]
+{% endif %}
+
+[log_config.loggers.pika]
+level = "WARNING"
+propagate = false
+handlers = ["console"]
+
+[log_config.loggers.fedora_openqa]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+
+[log_config.root]
+level = "ERROR"
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
+handlers = ["console"]
+{% endif %}

roles/openqa/dispatcher/templates/schedule.conf.j2

@@ -14,3 +14,8 @@ wiki_hostname: {{ openqa_wikitcms_hostname }}
 
 [schedule]
 arches: {{ openqa_compose_arches }}
+
+[consumer]
+openqa_hostname: {{ openqa_hostname }}
+openqa_baseurl: "https://{{ external_hostname|default(ansible_nodename) }}"
+dist-git_token: {{ openqa_distgit_token }}