diff --git a/inventories/group_vars/all/defaults.yml b/inventories/group_vars/all/defaults.yml index 3606802..9453184 100644 --- a/inventories/group_vars/all/defaults.yml +++ b/inventories/group_vars/all/defaults.yml @@ -9,6 +9,10 @@ deploy_challenger: false # Deploy monitoring? deploy_monitoring: true +# Deploy sakana, the monitor for exchanges and the Prometheus alerting stack? +# See roles/sakana/defaults/main.yml. +deploy_sakana: false + # If true, use EBICS keys from that were externally created. ebics_keys_external: false diff --git a/playbooks/setup.yml b/playbooks/setup.yml index f444a1e..3b61463 100644 --- a/playbooks/setup.yml +++ b/playbooks/setup.yml @@ -70,5 +70,7 @@ when: deploy_auditor | bool - role: monitoring when: deploy_monitoring | bool + - role: sakana + when: deploy_sakana | bool - role: devtesting when: dangerously_enable_devtesting | bool diff --git a/roles/sakana/defaults/main.yml b/roles/sakana/defaults/main.yml new file mode 100644 index 0000000..cf31d48 --- /dev/null +++ b/roles/sakana/defaults/main.yml @@ -0,0 +1,105 @@ +--- +# Configuration of sakana, the monitor for GNU Taler exchanges and the +# Prometheus alerting stack. +# +# The role only runs where "deploy_sakana" is true; it is false in +# inventories/group_vars/all/defaults.yml. Set it to true in the vars of the +# host that should do the monitoring. To be of use against Prometheus and +# Alertmanager, that host must not share their failure modes, so it should not +# be the host that runs them. + +# How often each system is checked. sakana randomizes the actual delay by +# +/- 5%, so several monitors do not query a system in lockstep. +sakana_frequency: "15 m" + +# How many consecutive failures of one error class are tolerated before a +# report is made. With "3 10 300" the 4th consecutive failure is reported, +# then 10 more are tolerated and the 11th after that report is reported, then +# 300 are tolerated and the 301st is, and that phase repeats. Any successful +# check resets the sequence. +sakana_tolerance: "3 10 300" + +# The command that receives the reports. sakana pipes the report into its +# standard input; the first line is a one-line summary suitable as a mail +# subject. The command runs without a shell and with a cleared environment +# (see sakana_env). +# +# The default needs nothing installed and puts every report into the journal. +# That is a placeholder, not alerting: a report about a dead Alertmanager must +# not travel through the alerting path it is about. To mail the reports +# instead, set +# +# sakana_command: "/usr/share/sakana/sakana-notify-mail ops@example.com" +# +# and make sure the host has an MTA and bsd-mailx, which the sakana package +# only suggests. The challenger SMS helpers work unmodified as well, with +# their credentials in sakana_env. +sakana_command: "/usr/bin/logger -t sakana -p daemon.err" + +# How long one request may take, including name resolution, the TCP +# connection and the TLS handshake. A check of a Prometheus or an Alertmanager +# makes several requests. +sakana_timeout: "30 s" + +# Environment of the reporting command, as NAME: value; each entry becomes +# ENV_ in [sakana]. These are typically credentials, so take the values +# from prod-secrets.yml, for example +# +# sakana_env: +# GATEWAYAPI_TOKEN: "{{ sakana_gatewayapi_token }}" +sakana_env: {} + +# Further defaults, left unset so that the package defaults apply: the PATH +# and the time limit of the reporting command, the horizon below which a TLS +# certificate counts as not renewed (5 d), and the horizons at which an +# exchange's signing keys (30 d) and denominations (14 d) count as no longer +# rotated. The denomination horizon has to stay below the LOOKAHEAD_SIGN of +# every monitored exchange, or a correctly rotating exchange warns on every +# check. +# +# sakana_command_path: "/usr/sbin:/usr/bin:/sbin:/bin" +# sakana_command_timeout: "1 m" +# sakana_tls_renewal_horizon: "5 d" +# sakana_signkey_expiration_horizon: "30 d" +# sakana_denomination_expiration_horizon: "14 d" + +# The systems to monitor. Every entry has +# +# name the configuration section; appears in every report and log line +# base_url the base URL, not an endpoint: sakana appends "keys" for an +# exchange and "-/healthy" for the other kinds. For Prometheus and +# Alertmanager behind a reverse proxy, this is the prefix the +# service's --web.external-url names. +# +# and optionally +# +# kind "exchange" (the default), "prometheus" or "alertmanager" +# enabled false parks the system without removing it +# env NAME: value pairs added to, or overriding, sakana_env +# auth_basic "USER:PASSWORD", sent with every request to the system +# auth_bearer a bearer token, sent likewise; not together with auth_basic +# +# plus per-system overrides of the settings above: frequency, tolerance, +# command, timeout, command_path, command_timeout and tls_renewal_horizon; +# for an exchange also signkey_expiration_horizon, +# denomination_expiration_horizon and key_expiration_horizon (a shorthand for +# both, which either of them overrides, even as a default set above); for an +# Alertmanager cluster_peers, the number of gossip peers that +# must be present (leave it unset for a single instance). sakana rejects an +# option that belongs to another kind. +# +# Credentials belong in prod-secrets.yml. Prometheus and Alertmanager behind a +# password look like this (check that base_url is the prefix their +# --web.external-url names): +# +# - name: prometheus +# kind: prometheus +# base_url: "https://monitoring.example.com/prometheus/" +# auth_basic: "{{ sakana_monitoring_auth_basic }}" +# - name: alertmanager +# kind: alertmanager +# base_url: "https://monitoring.example.com/alertmanager/" +# auth_basic: "{{ sakana_monitoring_auth_basic }}" +sakana_systems: + - name: exchange-taler-ops + base_url: "https://exchange.taler-ops.ch/" diff --git a/roles/sakana/handlers/main.yml b/roles/sakana/handlers/main.yml new file mode 100644 index 0000000..c47cf0a --- /dev/null +++ b/roles/sakana/handlers/main.yml @@ -0,0 +1,11 @@ +--- +# Notified by the tasks that install the sakana package or write its +# configuration, so a deploy that changed nothing leaves the monitor running +# instead of restarting it (a restart resets the failure counters, which live +# in memory, and thus re-reports a problem that is already known). +- name: Restart sakana + ansible.builtin.systemd: + daemon_reload: true + name: sakana + state: restarted + enabled: true diff --git a/roles/sakana/tasks/main.yml b/roles/sakana/tasks/main.yml new file mode 100644 index 0000000..05b83fe --- /dev/null +++ b/roles/sakana/tasks/main.yml @@ -0,0 +1,51 @@ +--- +# Installs sakana from deb.taler.net - the repository is configured by the +# "common_packages" role - and deploys its configuration. + +- name: Install sakana + ansible.builtin.apt: + name: sakana + state: latest + install_recommends: false + notify: Restart sakana + when: ansible_facts["os_family"] == 'Debian' + +# Ownership and modes are those the package's postinst sets, so that the +# package and this role do not undo each other: the configuration may hold +# credentials (AUTH_BASIC, AUTH_BEARER, ENV_*), and the service, which runs +# as User=sakana, reads it through its group. +- name: Ensure the sakana configuration directory exists + ansible.builtin.file: + path: /etc/sakana + state: directory + owner: root + group: sakana + mode: "0750" + +# "sakana --test" parses the configuration and fails on any error without +# contacting a monitored system. As "validate" it runs on the rendered file +# before that replaces the installed one, so a configuration this role got +# wrong fails the deploy while the running monitor keeps its old +# configuration. "diff: false" keeps the credentials out of --diff output. +- name: Place the sakana configuration + ansible.builtin.template: + src: templates/etc/sakana/sakana.conf.j2 + dest: /etc/sakana/sakana.conf + owner: root + group: sakana + mode: "0640" + validate: sakana --test --config %s + diff: false + notify: Restart sakana + +- name: Ensure sakana is enabled and running + ansible.builtin.systemd: + name: sakana + state: started + enabled: true + +# Keep the restart inside this role instead of deferring it to the end of the +# play, so that the monitor is demonstrably running the configuration we just +# deployed when this role finishes. +- name: Restart sakana now if the package or the configuration changed + ansible.builtin.meta: flush_handlers diff --git a/roles/sakana/templates/etc/sakana/sakana.conf.j2 b/roles/sakana/templates/etc/sakana/sakana.conf.j2 new file mode 100644 index 0000000..c59f7ef --- /dev/null +++ b/roles/sakana/templates/etc/sakana/sakana.conf.j2 @@ -0,0 +1,63 @@ +{% set defaults_options = [ + ['sakana_command_path', 'COMMAND_PATH'], + ['sakana_command_timeout', 'COMMAND_TIMEOUT'], + ['sakana_tls_renewal_horizon', 'TLS_RENEWAL_HORIZON'], + ['sakana_signkey_expiration_horizon', 'SIGNKEY_EXPIRATION_HORIZON'], + ['sakana_denomination_expiration_horizon', 'DENOMINATION_EXPIRATION_HORIZON'], + ] %} +{% set system_options = [ + ['kind', 'KIND'], + ['frequency', 'FREQUENCY'], + ['tolerance', 'TOLERANCE'], + ['command', 'COMMAND'], + ['timeout', 'TIMEOUT'], + ['command_path', 'COMMAND_PATH'], + ['command_timeout', 'COMMAND_TIMEOUT'], + ['auth_basic', 'AUTH_BASIC'], + ['auth_bearer', 'AUTH_BEARER'], + ['tls_renewal_horizon', 'TLS_RENEWAL_HORIZON'], + ['signkey_expiration_horizon', 'SIGNKEY_EXPIRATION_HORIZON'], + ['denomination_expiration_horizon', 'DENOMINATION_EXPIRATION_HORIZON'], + ['key_expiration_horizon', 'KEY_EXPIRATION_HORIZON'], + ['cluster_peers', 'CLUSTER_PEERS'], + ] %} +# Configuration of sakana, the monitor for GNU Taler exchanges and the +# Prometheus alerting stack. +# +# Deployed by the "sakana" role of ansible-taler-exchange. Local changes are +# overwritten on the next deploy; change the role variables instead, see +# roles/sakana/defaults/main.yml. +# +# This file may hold credentials and is therefore mode 0640, root:sakana. +# sakana.conf(5) documents every option and sakana(1) the command. + +# Defaults for every monitored system below. +[sakana] +FREQUENCY = {{ sakana_frequency }} +TOLERANCE = {{ sakana_tolerance }} +COMMAND = {{ sakana_command }} +TIMEOUT = {{ sakana_timeout }} +{% for var, option in defaults_options %} +{% if lookup('ansible.builtin.vars', var, default='') | string | length > 0 %} +{{ option }} = {{ lookup('ansible.builtin.vars', var) }} +{% endif %} +{% endfor %} +{% for name, value in sakana_env | dictsort %} +ENV_{{ name }} = {{ value }} +{% endfor %} +{% for system in sakana_systems %} + +[{{ system.name }}] +BASE_URL = {{ system.base_url }} +{% for key, option in system_options %} +{% if key in system %} +{{ option }} = {{ system[key] }} +{% endif %} +{% endfor %} +{% if 'enabled' in system %} +ENABLED = {{ 'YES' if system.enabled | bool else 'NO' }} +{% endif %} +{% for name, value in (system.env | default({})) | dictsort %} +ENV_{{ name }} = {{ value }} +{% endfor %} +{% endfor %}