sakana-role.patch (11,370 bytes)   

diff --git a/inventories/group_vars/all/defaults.yml b/inventories/group_vars/all/defaults.yml
index 3606802..9453184 100644
--- a/inventories/group_vars/all/defaults.yml
+++ b/inventories/group_vars/all/defaults.yml
@@ -9,6 +9,10 @@ deploy_challenger: false
 # Deploy monitoring?
 deploy_monitoring: true
 
+# Deploy sakana, the monitor for exchanges and the Prometheus alerting stack?
+# See roles/sakana/defaults/main.yml.
+deploy_sakana: false
+
 # If true, use EBICS keys from that were externally created.
 ebics_keys_external: false
 
diff --git a/playbooks/setup.yml b/playbooks/setup.yml
index f444a1e..3b61463 100644
--- a/playbooks/setup.yml
+++ b/playbooks/setup.yml
@@ -70,5 +70,7 @@
       when: deploy_auditor | bool
     - role: monitoring
       when: deploy_monitoring | bool
+    - role: sakana
+      when: deploy_sakana | bool
     - role: devtesting
       when: dangerously_enable_devtesting | bool
diff --git a/roles/sakana/defaults/main.yml b/roles/sakana/defaults/main.yml
new file mode 100644
index 0000000..cf31d48
--- /dev/null
+++ b/roles/sakana/defaults/main.yml
@@ -0,0 +1,105 @@
+---
+# Configuration of sakana, the monitor for GNU Taler exchanges and the
+# Prometheus alerting stack.
+#
+# The role only runs where "deploy_sakana" is true; it is false in
+# inventories/group_vars/all/defaults.yml. Set it to true in the vars of the
+# host that should do the monitoring. To be of use against Prometheus and
+# Alertmanager, that host must not share their failure modes, so it should not
+# be the host that runs them.
+
+# How often each system is checked. sakana randomizes the actual delay by
+# +/- 5%, so several monitors do not query a system in lockstep.
+sakana_frequency: "15 m"
+
+# How many consecutive failures of one error class are tolerated before a
+# report is made. With "3 10 300" the 4th consecutive failure is reported,
+# then 10 more are tolerated and the 11th after that report is reported, then
+# 300 are tolerated and the 301st is, and that phase repeats. Any successful
+# check resets the sequence.
+sakana_tolerance: "3 10 300"
+
+# The command that receives the reports. sakana pipes the report into its
+# standard input; the first line is a one-line summary suitable as a mail
+# subject. The command runs without a shell and with a cleared environment
+# (see sakana_env).
+#
+# The default needs nothing installed and puts every report into the journal.
+# That is a placeholder, not alerting: a report about a dead Alertmanager must
+# not travel through the alerting path it is about. To mail the reports
+# instead, set
+#
+#   sakana_command: "/usr/share/sakana/sakana-notify-mail ops@example.com"
+#
+# and make sure the host has an MTA and bsd-mailx, which the sakana package
+# only suggests. The challenger SMS helpers work unmodified as well, with
+# their credentials in sakana_env.
+sakana_command: "/usr/bin/logger -t sakana -p daemon.err"
+
+# How long one request may take, including name resolution, the TCP
+# connection and the TLS handshake. A check of a Prometheus or an Alertmanager
+# makes several requests.
+sakana_timeout: "30 s"
+
+# Environment of the reporting command, as NAME: value; each entry becomes
+# ENV_<NAME> in [sakana]. These are typically credentials, so take the values
+# from prod-secrets.yml, for example
+#
+#   sakana_env:
+#     GATEWAYAPI_TOKEN: "{{ sakana_gatewayapi_token }}"
+sakana_env: {}
+
+# Further defaults, left unset so that the package defaults apply: the PATH
+# and the time limit of the reporting command, the horizon below which a TLS
+# certificate counts as not renewed (5 d), and the horizons at which an
+# exchange's signing keys (30 d) and denominations (14 d) count as no longer
+# rotated. The denomination horizon has to stay below the LOOKAHEAD_SIGN of
+# every monitored exchange, or a correctly rotating exchange warns on every
+# check.
+#
+# sakana_command_path: "/usr/sbin:/usr/bin:/sbin:/bin"
+# sakana_command_timeout: "1 m"
+# sakana_tls_renewal_horizon: "5 d"
+# sakana_signkey_expiration_horizon: "30 d"
+# sakana_denomination_expiration_horizon: "14 d"
+
+# The systems to monitor. Every entry has
+#
+#   name      the configuration section; appears in every report and log line
+#   base_url  the base URL, not an endpoint: sakana appends "keys" for an
+#             exchange and "-/healthy" for the other kinds. For Prometheus and
+#             Alertmanager behind a reverse proxy, this is the prefix the
+#             service's --web.external-url names.
+#
+# and optionally
+#
+#   kind      "exchange" (the default), "prometheus" or "alertmanager"
+#   enabled   false parks the system without removing it
+#   env       NAME: value pairs added to, or overriding, sakana_env
+#   auth_basic   "USER:PASSWORD", sent with every request to the system
+#   auth_bearer  a bearer token, sent likewise; not together with auth_basic
+#
+# plus per-system overrides of the settings above: frequency, tolerance,
+# command, timeout, command_path, command_timeout and tls_renewal_horizon;
+# for an exchange also signkey_expiration_horizon,
+# denomination_expiration_horizon and key_expiration_horizon (a shorthand for
+# both, which either of them overrides, even as a default set above); for an
+# Alertmanager cluster_peers, the number of gossip peers that
+# must be present (leave it unset for a single instance). sakana rejects an
+# option that belongs to another kind.
+#
+# Credentials belong in prod-secrets.yml. Prometheus and Alertmanager behind a
+# password look like this (check that base_url is the prefix their
+# --web.external-url names):
+#
+#   - name: prometheus
+#     kind: prometheus
+#     base_url: "https://monitoring.example.com/prometheus/"
+#     auth_basic: "{{ sakana_monitoring_auth_basic }}"
+#   - name: alertmanager
+#     kind: alertmanager
+#     base_url: "https://monitoring.example.com/alertmanager/"
+#     auth_basic: "{{ sakana_monitoring_auth_basic }}"
+sakana_systems:
+  - name: exchange-taler-ops
+    base_url: "https://exchange.taler-ops.ch/"
diff --git a/roles/sakana/handlers/main.yml b/roles/sakana/handlers/main.yml
new file mode 100644
index 0000000..c47cf0a
--- /dev/null
+++ b/roles/sakana/handlers/main.yml
@@ -0,0 +1,11 @@
+---
+# Notified by the tasks that install the sakana package or write its
+# configuration, so a deploy that changed nothing leaves the monitor running
+# instead of restarting it (a restart resets the failure counters, which live
+# in memory, and thus re-reports a problem that is already known).
+- name: Restart sakana
+  ansible.builtin.systemd:
+    daemon_reload: true
+    name: sakana
+    state: restarted
+    enabled: true
diff --git a/roles/sakana/tasks/main.yml b/roles/sakana/tasks/main.yml
new file mode 100644
index 0000000..05b83fe
--- /dev/null
+++ b/roles/sakana/tasks/main.yml
@@ -0,0 +1,51 @@
+---
+# Installs sakana from deb.taler.net - the repository is configured by the
+# "common_packages" role - and deploys its configuration.
+
+- name: Install sakana
+  ansible.builtin.apt:
+    name: sakana
+    state: latest
+    install_recommends: false
+  notify: Restart sakana
+  when: ansible_facts["os_family"] == 'Debian'
+
+# Ownership and modes are those the package's postinst sets, so that the
+# package and this role do not undo each other: the configuration may hold
+# credentials (AUTH_BASIC, AUTH_BEARER, ENV_*), and the service, which runs
+# as User=sakana, reads it through its group.
+- name: Ensure the sakana configuration directory exists
+  ansible.builtin.file:
+    path: /etc/sakana
+    state: directory
+    owner: root
+    group: sakana
+    mode: "0750"
+
+# "sakana --test" parses the configuration and fails on any error without
+# contacting a monitored system. As "validate" it runs on the rendered file
+# before that replaces the installed one, so a configuration this role got
+# wrong fails the deploy while the running monitor keeps its old
+# configuration. "diff: false" keeps the credentials out of --diff output.
+- name: Place the sakana configuration
+  ansible.builtin.template:
+    src: templates/etc/sakana/sakana.conf.j2
+    dest: /etc/sakana/sakana.conf
+    owner: root
+    group: sakana
+    mode: "0640"
+    validate: sakana --test --config %s
+  diff: false
+  notify: Restart sakana
+
+- name: Ensure sakana is enabled and running
+  ansible.builtin.systemd:
+    name: sakana
+    state: started
+    enabled: true
+
+# Keep the restart inside this role instead of deferring it to the end of the
+# play, so that the monitor is demonstrably running the configuration we just
+# deployed when this role finishes.
+- name: Restart sakana now if the package or the configuration changed
+  ansible.builtin.meta: flush_handlers
diff --git a/roles/sakana/templates/etc/sakana/sakana.conf.j2 b/roles/sakana/templates/etc/sakana/sakana.conf.j2
new file mode 100644
index 0000000..c59f7ef
--- /dev/null
+++ b/roles/sakana/templates/etc/sakana/sakana.conf.j2
@@ -0,0 +1,63 @@
+{% set defaults_options = [
+     ['sakana_command_path', 'COMMAND_PATH'],
+     ['sakana_command_timeout', 'COMMAND_TIMEOUT'],
+     ['sakana_tls_renewal_horizon', 'TLS_RENEWAL_HORIZON'],
+     ['sakana_signkey_expiration_horizon', 'SIGNKEY_EXPIRATION_HORIZON'],
+     ['sakana_denomination_expiration_horizon', 'DENOMINATION_EXPIRATION_HORIZON'],
+   ] %}
+{% set system_options = [
+     ['kind', 'KIND'],
+     ['frequency', 'FREQUENCY'],
+     ['tolerance', 'TOLERANCE'],
+     ['command', 'COMMAND'],
+     ['timeout', 'TIMEOUT'],
+     ['command_path', 'COMMAND_PATH'],
+     ['command_timeout', 'COMMAND_TIMEOUT'],
+     ['auth_basic', 'AUTH_BASIC'],
+     ['auth_bearer', 'AUTH_BEARER'],
+     ['tls_renewal_horizon', 'TLS_RENEWAL_HORIZON'],
+     ['signkey_expiration_horizon', 'SIGNKEY_EXPIRATION_HORIZON'],
+     ['denomination_expiration_horizon', 'DENOMINATION_EXPIRATION_HORIZON'],
+     ['key_expiration_horizon', 'KEY_EXPIRATION_HORIZON'],
+     ['cluster_peers', 'CLUSTER_PEERS'],
+   ] %}
+# Configuration of sakana, the monitor for GNU Taler exchanges and the
+# Prometheus alerting stack.
+#
+# Deployed by the "sakana" role of ansible-taler-exchange. Local changes are
+# overwritten on the next deploy; change the role variables instead, see
+# roles/sakana/defaults/main.yml.
+#
+# This file may hold credentials and is therefore mode 0640, root:sakana.
+# sakana.conf(5) documents every option and sakana(1) the command.
+
+# Defaults for every monitored system below.
+[sakana]
+FREQUENCY = {{ sakana_frequency }}
+TOLERANCE = {{ sakana_tolerance }}
+COMMAND = {{ sakana_command }}
+TIMEOUT = {{ sakana_timeout }}
+{% for var, option in defaults_options %}
+{% if lookup('ansible.builtin.vars', var, default='') | string | length > 0 %}
+{{ option }} = {{ lookup('ansible.builtin.vars', var) }}
+{% endif %}
+{% endfor %}
+{% for name, value in sakana_env | dictsort %}
+ENV_{{ name }} = {{ value }}
+{% endfor %}
+{% for system in sakana_systems %}
+
+[{{ system.name }}]
+BASE_URL = {{ system.base_url }}
+{% for key, option in system_options %}
+{% if key in system %}
+{{ option }} = {{ system[key] }}
+{% endif %}
+{% endfor %}
+{% if 'enabled' in system %}
+ENABLED = {{ 'YES' if system.enabled | bool else 'NO' }}
+{% endif %}
+{% for name, value in (system.env | default({})) | dictsort %}
+ENV_{{ name }} = {{ value }}
+{% endfor %}
+{% endfor %}