diff --git a/ansible/deploy-airflow.yml b/ansible/deploy-airflow.yml index c3168205..b070fce2 100644 --- a/ansible/deploy-airflow.yml +++ b/ansible/deploy-airflow.yml @@ -6,6 +6,7 @@ roles: - nginx - dehydrated + - prometheus_statsd_exporter - oonidata_airflow vars: airflow_public_fqdn: "airflow.prod.ooni.io" diff --git a/ansible/group_vars/airflow/vars.yml b/ansible/group_vars/airflow/vars.yml index 3fb68ebe..7fbf35ee 100644 --- a/ansible/group_vars/airflow/vars.yml +++ b/ansible/group_vars/airflow/vars.yml @@ -10,3 +10,11 @@ airflow_fernet_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflo airflow_webserver_secret_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_webserver_secret_key', profile='oonidevops_user_prod') }}" airflow_executor: "LocalExecutor" airflow_database_conn: "postgresql+psycopg2://airflow:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/airflow_postgresql_password', profile='oonidevops_user_prod') }}@ooni-tier0-postgres.c7mgscca82no.eu-central-1.rds.amazonaws.com/airflow" + +# StatsD metrics, sent to the local prometheus statsd_exporter +# These variables come from the airflow role used by this playbook +# https://github.com/ooni/airflow-role/blob/5ba757e17c3fd63a4dc794a74ad1912f624d84d7/defaults/main/airflow-cfg.yml#L85-L95 +airflow_statsd_on: True +airflow_statsd_host: localhost +airflow_statsd_port: 8125 +airflow_statsd_prefix: airflow diff --git a/ansible/roles/oonidata_airflow/tasks/main.yml b/ansible/roles/oonidata_airflow/tasks/main.yml index 36afbe2d..a466a7c3 100644 --- a/ansible/roles/oonidata_airflow/tasks/main.yml +++ b/ansible/roles/oonidata_airflow/tasks/main.yml @@ -2,7 +2,7 @@ ansible.builtin.apt: name: - build-essential - - python-dev + - python3-dev - g++ state: latest update_cache: yes @@ -71,6 +71,7 @@ airflow_extra_packages: - postgres - virtualenv + - statsd airflow_services: airflow_webserver: service_name: airflow-webserver diff --git a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 index 6c3b3fec..ae6cd3bf 100644 --- a/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 +++ b/ansible/roles/oonidata_airflow/templates/nginx-airflow.j2 @@ -20,6 +20,18 @@ server { add_header Access-Control-Allow-Origin *; + ## Prometheus statsd_exporter metrics + location /metrics/statsd_exporter { + auth_basic "Administrator's Area"; + auth_basic_user_file /etc/ooni/prometheus_passwd; + + proxy_pass http://127.0.0.1:9102/metrics; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + ## Airflow reverse proxy location / { proxy_pass http://127.0.0.1:8080; diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index f0a1b835..ba107dc2 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -429,4 +429,14 @@ scrape_configs: replacement: "/$2" target_label: "__metrics_path__" action: "replace" + + - job_name: "airflow" + scheme: https + metrics_path: /metrics/statsd_exporter + basic_auth: + username: 'prom' + password: '{{ prometheus_metrics_password }}' + static_configs: + - targets: + - airflow.prod.ooni.io ... diff --git a/ansible/roles/prometheus_statsd_exporter/defaults/main.yml b/ansible/roles/prometheus_statsd_exporter/defaults/main.yml new file mode 100644 index 00000000..b2c4ab4b --- /dev/null +++ b/ansible/roles/prometheus_statsd_exporter/defaults/main.yml @@ -0,0 +1,6 @@ +--- +statsd_exporter_filename: "statsd_exporter-0.28.0.linux-amd64" +statsd_exporter_download_url: "https://github.com/prometheus/statsd_exporter/releases/download/v0.28.0/{{statsd_exporter_filename}}.tar.gz" +statsd_exporter_bin_path: /usr/local/bin/statsd_exporter +statsd_exporter_statsd_listen_udp: ':8125' +statsd_exporter_web_listen_address: '127.0.0.1:9102' diff --git a/ansible/roles/prometheus_statsd_exporter/handlers/main.yml b/ansible/roles/prometheus_statsd_exporter/handlers/main.yml new file mode 100644 index 00000000..6e0e84e8 --- /dev/null +++ b/ansible/roles/prometheus_statsd_exporter/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart statsd_exporter + ansible.builtin.systemd_service: + name: statsd_exporter + state: restarted diff --git a/ansible/roles/prometheus_statsd_exporter/tasks/main.yml b/ansible/roles/prometheus_statsd_exporter/tasks/main.yml new file mode 100644 index 00000000..5908a5f2 --- /dev/null +++ b/ansible/roles/prometheus_statsd_exporter/tasks/main.yml @@ -0,0 +1,56 @@ +--- +- name: Create statsd_exporter group + ansible.builtin.group: + name: statsd_exporter + state: present + become: true + +- name: Create statsd_exporter user + ansible.builtin.user: + name: statsd_exporter + group: statsd_exporter + shell: /sbin/nologin + system: true + createhome: false + become: true + +- name: Check current statsd_exporter version + ansible.builtin.command: "{{ statsd_exporter_bin_path }} --version" + failed_when: false + changed_when: false + register: exporter_installed + +# We can't install it with apt-get, use the binary from github +- name: Download and unarchive statsd_exporter + ansible.builtin.unarchive: + src: "{{ statsd_exporter_download_url }}" + dest: /tmp/ + remote_src: true + mode: "0755" + when: exporter_installed.rc != 0 # return code (rc) is != 0 when not installed + become: true + +- name: Move statsd_exporter binary into place + ansible.builtin.copy: + src: "/tmp/{{ statsd_exporter_filename }}/statsd_exporter" + dest: "{{ statsd_exporter_bin_path }}" + mode: "0755" + remote_src: true + when: exporter_installed.rc != 0 + notify: restart statsd_exporter + become: true + +- name: Create statsd_exporter systemd unit + ansible.builtin.template: + src: statsd_exporter.service + dest: /etc/systemd/system/statsd_exporter.service + mode: "0644" + become: true + notify: restart statsd_exporter + +- name: Ensure statsd_exporter is running and enabled + ansible.builtin.service: + name: statsd_exporter + state: started + enabled: true + become: true diff --git a/ansible/roles/prometheus_statsd_exporter/templates/statsd_exporter.service b/ansible/roles/prometheus_statsd_exporter/templates/statsd_exporter.service new file mode 100644 index 00000000..e6a168e1 --- /dev/null +++ b/ansible/roles/prometheus_statsd_exporter/templates/statsd_exporter.service @@ -0,0 +1,14 @@ +[Unit] +Description=Prometheus StatsD Exporter +After=network.target + +[Service] +TimeoutStartSec=0 +User=statsd_exporter +ExecStart={{ statsd_exporter_bin_path }} \ + --statsd.listen-udp={{ statsd_exporter_statsd_listen_udp }} \ + --web.listen-address={{ statsd_exporter_web_listen_address }} +Restart=on-failure + +[Install] +WantedBy=multi-user.target