From 48b13c919b5acffb579037a6f2c5c9b49f388ecd Mon Sep 17 00:00:00 2001 From: ionutboangiu Date: Mon, 5 Aug 2024 18:03:57 +0300 Subject: [PATCH] Complete monitoring ansible playbook - add dashboard provisioning for grafana - add README file --- data/ansible/monitoring/README.md | 43 +++++++++++ data/ansible/monitoring/grafana.ini | 6 ++ .../ansible/monitoring/grafana_dashboard.yaml | 9 +++ .../monitoring/grafana_datasource.yaml | 10 +++ data/ansible/monitoring/main.yaml | 76 ++++++++++++++++--- data/ansible/monitoring/prometheus.yml | 15 ++++ data/ansible/monitoring/prometheus.yml.j2 | 18 ----- data/ansible/roles/grafana/defaults/main.yaml | 12 ++- .../roles/grafana/tasks/handle_file.yaml | 22 ++++++ data/ansible/roles/grafana/tasks/main.yaml | 58 ++++++++++---- .../roles/prometheus/tasks/handle_file.yaml | 22 ++++++ data/ansible/roles/prometheus/tasks/main.yaml | 21 +++-- 12 files changed, 256 insertions(+), 56 deletions(-) create mode 100644 data/ansible/monitoring/README.md create mode 100644 data/ansible/monitoring/grafana.ini create mode 100644 data/ansible/monitoring/grafana_dashboard.yaml create mode 100644 data/ansible/monitoring/grafana_datasource.yaml create mode 100644 data/ansible/monitoring/prometheus.yml delete mode 100644 data/ansible/monitoring/prometheus.yml.j2 create mode 100644 data/ansible/roles/grafana/tasks/handle_file.yaml create mode 100644 data/ansible/roles/prometheus/tasks/handle_file.yaml diff --git a/data/ansible/monitoring/README.md b/data/ansible/monitoring/README.md new file mode 100644 index 000000000..ec415cda2 --- /dev/null +++ b/data/ansible/monitoring/README.md @@ -0,0 +1,43 @@ +### CGRateS Monitoring Setup Playbook + +#### Inventory (inventory.ini): + +```ini +[monit] +foo.example.com ansible_host=myserverip ansible_port=22 ansible_user=myuser + +[monit:vars] +install_or_update_cgrates=true +``` + +#### Run playbook: + +```bash +ansible-playbook -i inventory.ini /path/to/playbook/main.yml +``` + +#### Access Grafana: + +```bash +ssh -L 8080:localhost:3000 myuser@myserverip +``` + +Browse to http://localhost:8080 and login + +username: admin +password: admin + +#### Components and their ports: +CGRateS: 2012 +Node Exporter: 9100 +Prometheus: 9090 +Grafana: 3000 + +#### Imported Grafana dashboards: +- Go Metrics (ID: 13240) - a custom solution for this one would be preferred +- Node Exporter (ID: 1860) + +> [!NOTE] +> Go Metrics tracks both the node_exporter and prometheus alongside CGRateS (all written in go). Make sure that job "cgrates" is the one selected. + +Services can be managed via systemd diff --git a/data/ansible/monitoring/grafana.ini b/data/ansible/monitoring/grafana.ini new file mode 100644 index 000000000..c0be474f2 --- /dev/null +++ b/data/ansible/monitoring/grafana.ini @@ -0,0 +1,6 @@ +[paths] +provisioning = /etc/grafana/provisioning + +[dashboards] +providers.file.enabled = true +providers.file.path = /etc/grafana/provisioning/dashboards diff --git a/data/ansible/monitoring/grafana_dashboard.yaml b/data/ansible/monitoring/grafana_dashboard.yaml new file mode 100644 index 000000000..ed5ab96a6 --- /dev/null +++ b/data/ansible/monitoring/grafana_dashboard.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: dashboards + type: file + disableDeletion: false + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/data/ansible/monitoring/grafana_datasource.yaml b/data/ansible/monitoring/grafana_datasource.yaml new file mode 100644 index 000000000..325d2828d --- /dev/null +++ b/data/ansible/monitoring/grafana_datasource.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://localhost:9090 + isDefault: true + editable: true + version: 1 diff --git a/data/ansible/monitoring/main.yaml b/data/ansible/monitoring/main.yaml index dd3fb4951..62af5a2c3 100644 --- a/data/ansible/monitoring/main.yaml +++ b/data/ansible/monitoring/main.yaml @@ -1,6 +1,51 @@ --- - name: Set up monitoring for CGRateS hosts: all + vars: + cgrates_dependencies: + - wget + - gnupg + tasks: + - name: CGRateS install/update tasks + when: install_or_update_cgrates | default(false) | bool + block: + - name: Install CGRateS dependencies + become: true + ansible.builtin.apt: + name: "{{ cgrates_dependencies }}" + state: present + update_cache: true + + - name: Download the GPG Key + ansible.builtin.get_url: + url: https://apt.cgrates.org/apt.cgrates.org.gpg.key + dest: /tmp/apt.cgrates.org.asc + + - name: Move the GPG Key to the trusted area + become: true + ansible.builtin.copy: + src: /tmp/apt.cgrates.org.asc + dest: /etc/apt/trusted.gpg.d/apt.cgrates.org.asc + remote_src: true + + - name: Add the CGRateS repository to the sources list + become: true + ansible.builtin.copy: + content: "deb http://apt.cgrates.org/debian/ master-bookworm main\n" + dest: /etc/apt/sources.list.d/cgrates.list + + - name: Install or upgrade CGRateS + become: true + ansible.builtin.apt: + name: cgrates + state: latest + update_cache: true + + - name: Start the CGRateS service + become: true + ansible.builtin.systemd_service: + name: cgrates + state: restarted roles: - role: ../roles/node_exporter vars: @@ -8,19 +53,30 @@ - role: ../roles/prometheus vars: - prometheus_config_file: prometheus.yml.j2 + prometheus_config_file: prometheus.yml prometheus_service_state: started - role: ../roles/grafana vars: grafana_service_state: started + grafana_config_file: grafana.ini + grafana_datasource_config_file: grafana_datasource.yaml + grafana_dashboard_config_file: grafana_dashboard.yaml + grafana_dashboard_sources: + - type: url + path: https://grafana.com/api/dashboards/13240/revisions/2/download + alias: go_metrics.json + modify_datasources: true # see https://github.com/grafana/grafana/issues/10786 + - type: url + path: https://grafana.com/api/dashboards/1860/revisions/37/download + alias: node_exporter.json - - ../roles/go - - role: ../../roles/cgrates - vars: - # To avoid tasks/dependencies we don't need. - cgrates_migrator_cfg_path: - cgrates_dbs: - cgrates_dependencies: - - git - - redis + # - ../roles/go + # - role: ../../roles/cgrates + # vars: + # # To avoid tasks/dependencies we don't need. + # cgrates_migrator_cfg_path: + # cgrates_dbs: + # cgrates_dependencies: + # - git + # - redis diff --git a/data/ansible/monitoring/prometheus.yml b/data/ansible/monitoring/prometheus.yml new file mode 100644 index 000000000..8fe940433 --- /dev/null +++ b/data/ansible/monitoring/prometheus.yml @@ -0,0 +1,15 @@ +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] + + - job_name: node + scrape_interval: 15s + static_configs: + - targets: ['localhost:9100'] + + - job_name: cgrates + metrics_path: /prometheus + scrape_interval: 15s + static_configs: + - targets: ['localhost:2080'] diff --git a/data/ansible/monitoring/prometheus.yml.j2 b/data/ansible/monitoring/prometheus.yml.j2 deleted file mode 100644 index 5914d4c26..000000000 --- a/data/ansible/monitoring/prometheus.yml.j2 +++ /dev/null @@ -1,18 +0,0 @@ -scrape_configs: - - job_name: 'prometheus' - static_configs: - - targets: ['localhost:9090'] - #- targets: ['{{ ansible_default_ipv4.address }}:9090'] - - - job_name: 'node' - scrape_interval: 15s - static_configs: - - targets: ['localhost:9100'] - #- targets: ['{{ ansible_default_ipv4.address }}:9100'] - - - job_name: 'cgrates' - metrics_path: '/prometheus' - scrape_interval: 15s - static_configs: - - targets: ['localhost:2080'] - #- targets: ['{{ ansible_default_ipv4.address }}:2080'] diff --git a/data/ansible/roles/grafana/defaults/main.yaml b/data/ansible/roles/grafana/defaults/main.yaml index fb3a8bd05..c200d2f7f 100644 --- a/data/ansible/roles/grafana/defaults/main.yaml +++ b/data/ansible/roles/grafana/defaults/main.yaml @@ -5,8 +5,18 @@ grafana_install_dir: /opt/grafana grafana_data_dir: /var/lib/grafana grafana_logs_dir: /var/log/grafana grafana_config_dir: /etc/grafana +grafana_provisioning_dir: '{{ grafana_config_dir }}/provisioning' +grafana_dashboards_dir: '{{ grafana_data_dir }}/dashboards' grafana_service_enabled: false grafana_service_state: stopped -grafana_config_file: '{{ grafana_install_dir }}/conf/defaults.ini' # supports either the path to a file or a j2 template +grafana_config_file: '{{ grafana_install_dir }}/conf/defaults.ini' # Supports either the path to a file or a template name +# grafana_dashboard_config_file: grafana_dashboard.yaml +# grafana_datasource_config_file: grafana_datasource.yaml +# grafana_dashboard_sources: +# - type: file +# path: dashboard1.json +# - type: url +# path: www.example.com/dl/dashboard +# alias: dl_dashboard.json diff --git a/data/ansible/roles/grafana/tasks/handle_file.yaml b/data/ansible/roles/grafana/tasks/handle_file.yaml new file mode 100644 index 000000000..883078186 --- /dev/null +++ b/data/ansible/roles/grafana/tasks/handle_file.yaml @@ -0,0 +1,22 @@ +--- +- name: Copy file (if not .j2) + become: true + ansible.builtin.copy: + src: '{{ file_src }}' + dest: '{{ file_dest }}' + owner: '{{ grafana_user }}' + group: '{{ grafana_user }}' + mode: '0644' + when: not file_src.endswith('.j2') + notify: Restart grafana + +- name: Template file (if .j2) + become: true + ansible.builtin.template: + src: '{{ file_src }}' + dest: '{{ file_dest }}' + owner: '{{ grafana_user }}' + group: '{{ grafana_user }}' + mode: '0644' + when: file_src.endswith('.j2') + notify: Restart grafana diff --git a/data/ansible/roles/grafana/tasks/main.yaml b/data/ansible/roles/grafana/tasks/main.yaml index 4adc4e18f..f2bafcdad 100644 --- a/data/ansible/roles/grafana/tasks/main.yaml +++ b/data/ansible/roles/grafana/tasks/main.yaml @@ -31,8 +31,11 @@ loop: - '{{ grafana_install_dir }}' - '{{ grafana_data_dir }}' + - '{{ grafana_dashboards_dir }}' - '{{ grafana_logs_dir }}' - '{{ grafana_config_dir }}' + - '{{ grafana_provisioning_dir }}/dashboards' + - '{{ grafana_provisioning_dir }}/datasources' - name: Download and extract grafana become: true @@ -47,31 +50,56 @@ when: grafana_current_version.rc != 0 or grafana_version not in grafana_current_version.stdout - name: Handle grafana configuration file + ansible.builtin.include_tasks: handle_file.yaml + vars: + file_src: '{{ grafana_config_file }}' + file_dest: '{{ grafana_config_dir }}/grafana.ini' + when: grafana_config_file is defined + +- name: Handle grafana dashboard config file + ansible.builtin.include_tasks: handle_file.yaml + vars: + file_src: '{{ grafana_dashboard_config_file }}' + file_dest: '{{ grafana_provisioning_dir }}/dashboards/{{ grafana_dashboard_config_file | basename | regex_replace("\.j2$", "") }}' + when: grafana_dashboard_config_file is defined + +- name: Handle grafana datasource config file + ansible.builtin.include_tasks: handle_file.yaml + vars: + file_src: '{{ grafana_datasource_config_file }}' + file_dest: '{{ grafana_provisioning_dir }}/datasources' + when: grafana_datasource_config_file is defined + +- name: Import grafana dashboards + become: true block: - - name: Copy grafana configuration file if a direct path - become: true + - name: Import grafana dashboards from files ansible.builtin.copy: - src: '{{ grafana_config_file }}' - dest: '{{ grafana_config_dir }}/grafana.ini' + src: '{{ item.path }}' + dest: '{{ grafana_dashboards_dir }}' owner: '{{ grafana_user }}' group: '{{ grafana_user }}' mode: '0644' - remote_src: true - notify: Restart grafana - when: grafana_config_file is defined and '/' in grafana_config_file + loop: "{{ grafana_dashboard_sources | selectattr('type', '==', 'file') | list }}" - - name: Render grafana configuration template - become: true - ansible.builtin.template: - src: '{{ grafana_config_file }}' - dest: '{{ grafana_config_dir }}/grafana.ini' + - name: Download grafana dashboards from URLs + ansible.builtin.get_url: + url: '{{ item.path }}' + dest: '{{ grafana_dashboards_dir }}/{{ item.alias }}' owner: '{{ grafana_user }}' group: '{{ grafana_user }}' mode: '0644' - notify: Restart grafana - when: grafana_config_file is defined and '/' not in grafana_config_file + loop: "{{ grafana_dashboard_sources | selectattr('type', '==', 'url') | list }}" - when: grafana_config_file is defined and grafana_config_file != '' + - name: Modify datasource in grafana dashboards # see https://github.com/grafana/grafana/issues/10786 + ansible.builtin.replace: + path: "{{ grafana_dashboards_dir }}/{{ item.alias if item.type == 'url' else item.path | basename }}" + regexp: '"datasource":\s*(".*"|\{[\s\S]*?\})' + replace: '"datasource": null' + loop: "{{ grafana_dashboard_sources }}" + when: item.modify_datasources | default(false) + when: grafana_dashboard_sources is defined and grafana_dashboard_sources | length > 0 + notify: Restart grafana - name: Create grafana systemd service file become: true diff --git a/data/ansible/roles/prometheus/tasks/handle_file.yaml b/data/ansible/roles/prometheus/tasks/handle_file.yaml new file mode 100644 index 000000000..883078186 --- /dev/null +++ b/data/ansible/roles/prometheus/tasks/handle_file.yaml @@ -0,0 +1,22 @@ +--- +- name: Copy file (if not .j2) + become: true + ansible.builtin.copy: + src: '{{ file_src }}' + dest: '{{ file_dest }}' + owner: '{{ grafana_user }}' + group: '{{ grafana_user }}' + mode: '0644' + when: not file_src.endswith('.j2') + notify: Restart grafana + +- name: Template file (if .j2) + become: true + ansible.builtin.template: + src: '{{ file_src }}' + dest: '{{ file_dest }}' + owner: '{{ grafana_user }}' + group: '{{ grafana_user }}' + mode: '0644' + when: file_src.endswith('.j2') + notify: Restart grafana diff --git a/data/ansible/roles/prometheus/tasks/main.yaml b/data/ansible/roles/prometheus/tasks/main.yaml index 244e7c4ab..dd74ef749 100644 --- a/data/ansible/roles/prometheus/tasks/main.yaml +++ b/data/ansible/roles/prometheus/tasks/main.yaml @@ -45,32 +45,29 @@ notify: Restart prometheus when: prometheus_current_version.rc != 0 or prometheus_version not in prometheus_current_version.stdout -- name: Handle prometheus configuration file +- name: Handle prometheus config file block: - - name: Copy prometheus configuration file if a direct path + - name: Copy file (if not .j2) become: true ansible.builtin.copy: src: '{{ prometheus_config_file }}' - dest: '{{ prometheus_config_dir }}/prometheus.yml' + dest: '{{ prometheus_config_dir }}/{{ prometheus_config_file | basename }}' owner: '{{ prometheus_user }}' group: '{{ prometheus_user }}' mode: '0644' - remote_src: true - notify: Restart prometheus - when: prometheus_config_file is defined and '/' in prometheus_config_file + when: not prometheus_config_file.endswith('.j2') - - name: Render prometheus configuration template + - name: Template file (if .j2) become: true ansible.builtin.template: src: '{{ prometheus_config_file }}' - dest: '{{ prometheus_config_dir }}/prometheus.yml' + dest: '{{ prometheus_config_dir }}/{{ prometheus_config_file | basename | splitext | first }}' # cut .j2 extension owner: '{{ prometheus_user }}' group: '{{ prometheus_user }}' mode: '0644' - notify: Restart prometheus - when: prometheus_config_file is defined and '/' not in prometheus_config_file - - when: prometheus_config_file is defined and prometheus_config_file != '' + when: prometheus_config_file.endswith('.j2') + when: prometheus_config_file is defined + notify: Restart prometheus - name: Create prometheus systemd service file become: true