From ec82afe3367217519fd9ee37f32d53dbae0d1494 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 16 Nov 2026 07:11:18 -0700 Subject: [PATCH 1/2] Add optional Loki + Alloy log search for Slurm and Kubernetes Off by default (slurm_enable_log_search: false). Slurm: loki and alloy roles, Grafana Loki datasource and Cluster logs dashboard, 7-day retention. Kubernetes: scripts/k8s/deploy_logging.sh with pinned Loki and Alloy charts. Replaces the archived Elastic logging guide. Signed-off-by: Doug Holt --- .github/workflows/molecule.yml | 2 + config.example/group_vars/slurm-cluster.yml | 9 ++ config.example/helm/alloy.yml | 54 +++++++++ config.example/helm/loki-datasource.yaml | 8 ++ config.example/helm/loki.yml | 69 +++++++++++ docs/k8s-cluster/logging.md | 88 ++------------ docs/slurm-cluster/slurm-logging.md | 33 ++++++ docs/slurm-cluster/slurm-monitor.md | 4 + playbooks/slurm-cluster.yml | 13 +++ playbooks/slurm-cluster/alloy.yml | 10 ++ playbooks/slurm-cluster/loki.yml | 10 ++ roles/alloy/defaults/main.yml | 17 +++ roles/alloy/handlers/main.yml | 5 + roles/alloy/molecule/default/converge.yml | 10 ++ roles/alloy/molecule/default/molecule.yml | 22 ++++ roles/alloy/molecule/default/verify.yml | 13 +++ roles/alloy/tasks/config.yml | 15 +++ roles/alloy/tasks/main.yml | 29 +++++ roles/alloy/templates/config.alloy.j2 | 55 +++++++++ roles/alloy/templates/docker.alloy.service.j2 | 22 ++++ roles/grafana/defaults/main.yml | 5 + roles/grafana/tasks/main.yml | 26 +++++ .../grafana/templates/loki-datasource.yml.j2 | 9 ++ roles/loki/defaults/main.yml | 15 +++ roles/loki/handlers/main.yml | 5 + roles/loki/molecule/default/converge.yml | 10 ++ roles/loki/molecule/default/molecule.yml | 22 ++++ roles/loki/molecule/default/verify.yml | 13 +++ roles/loki/tasks/config.yml | 15 +++ roles/loki/tasks/main.yml | 29 +++++ roles/loki/templates/docker.loki.service.j2 | 16 +++ roles/loki/templates/loki-config.yml.j2 | 42 +++++++ scripts/k8s/deploy_logging.sh | 110 ++++++++++++++++++ src/dashboards/cluster-logs-dashboard.json | 67 +++++++++++ tests/logging/test_logging_render.py | 71 +++++++++++ 35 files changed, 867 insertions(+), 76 deletions(-) create mode 100644 config.example/helm/alloy.yml create mode 100644 config.example/helm/loki-datasource.yaml create mode 100644 config.example/helm/loki.yml create mode 100644 docs/slurm-cluster/slurm-logging.md create mode 100644 playbooks/slurm-cluster/alloy.yml create mode 100644 playbooks/slurm-cluster/loki.yml create mode 100644 roles/alloy/defaults/main.yml create mode 100644 roles/alloy/handlers/main.yml create mode 100644 roles/alloy/molecule/default/converge.yml create mode 100644 roles/alloy/molecule/default/molecule.yml create mode 100644 roles/alloy/molecule/default/verify.yml create mode 100644 roles/alloy/tasks/config.yml create mode 100644 roles/alloy/tasks/main.yml create mode 100644 roles/alloy/templates/config.alloy.j2 create mode 100644 roles/alloy/templates/docker.alloy.service.j2 create mode 100644 roles/grafana/templates/loki-datasource.yml.j2 create mode 100644 roles/loki/defaults/main.yml create mode 100644 roles/loki/handlers/main.yml create mode 100644 roles/loki/molecule/default/converge.yml create mode 100644 roles/loki/molecule/default/molecule.yml create mode 100644 roles/loki/molecule/default/verify.yml create mode 100644 roles/loki/tasks/config.yml create mode 100644 roles/loki/tasks/main.yml create mode 100644 roles/loki/templates/docker.loki.service.j2 create mode 100644 roles/loki/templates/loki-config.yml.j2 create mode 100755 scripts/k8s/deploy_logging.sh create mode 100644 src/dashboards/cluster-logs-dashboard.json create mode 100644 tests/logging/test_logging_render.py diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index bec71d929..fa83243fc 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -16,6 +16,8 @@ jobs: - lmod - nfs - nhc + - alloy + - loki - nvidia_dcgm - openmpi - mofed diff --git a/config.example/group_vars/slurm-cluster.yml b/config.example/group_vars/slurm-cluster.yml index e8597677a..18df3b9d6 100644 --- a/config.example/group_vars/slurm-cluster.yml +++ b/config.example/group_vars/slurm-cluster.yml @@ -210,6 +210,15 @@ slurm_enable_monitoring: true # (Prometheus, Grafana, etc) slurm_monitoring_group: "slurm-master" +################################################################################ +# Optional log search (Loki + Alloy), off by default # +################################################################################ +# Runs Loki on the monitoring node and Alloy on every node, and adds a Loki +# data source and a "Cluster logs" dashboard to Grafana. Requires +# slurm_enable_monitoring. See docs/slurm-cluster/slurm-logging.md. +slurm_enable_log_search: false +# loki_retention_period: 168h # 7 days; must be a multiple of 24h + ################################################################################ # Logging with rsyslog # ################################################################################ diff --git a/config.example/helm/alloy.yml b/config.example/helm/alloy.yml new file mode 100644 index 000000000..06c303214 --- /dev/null +++ b/config.example/helm/alloy.yml @@ -0,0 +1,54 @@ +# Values for Grafana Alloy: one pod per node, shipping pod and node logs to Loki. +# Chart: grafana/alloy. Used by scripts/k8s/deploy_logging.sh. +controller: + type: daemonset + +alloy: + mounts: + varlog: true + configMap: + create: true + content: | + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), "") + } + } + + discovery.relabel "pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "host" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"] + separator = "/" + target_label = "job" + } + } + + loki.source.kubernetes "pods" { + targets = discovery.relabel.pods.output + forward_to = [loki.write.default.receiver] + } + + loki.write "default" { + endpoint { + url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push" + } + } diff --git a/config.example/helm/loki-datasource.yaml b/config.example/helm/loki-datasource.yaml new file mode 100644 index 000000000..e0ce4aecb --- /dev/null +++ b/config.example/helm/loki-datasource.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: Loki + uid: deepops-loki + type: loki + access: proxy + url: http://loki.monitoring.svc.cluster.local:3100 + isDefault: false diff --git a/config.example/helm/loki.yml b/config.example/helm/loki.yml new file mode 100644 index 000000000..e03da7797 --- /dev/null +++ b/config.example/helm/loki.yml @@ -0,0 +1,69 @@ +# Values for the optional Loki log store (single binary, persistent volume). +# Chart: grafana-community/loki. Used by scripts/k8s/deploy_logging.sh. +deploymentMode: SingleBinary + +loki: + auth_enabled: false + commonConfig: + replication_factor: 1 + schemaConfig: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: loki_index_ + period: 24h + storage: + type: filesystem + limits_config: + # 7 days. Must be a multiple of 24h. + retention_period: 168h + allow_structured_metadata: true + compactor: + retention_enabled: true + delete_request_store: filesystem + useTestSchema: false + +singleBinary: + replicas: 1 + persistence: + enabled: true + size: 50Gi + +# Not used in single binary mode +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 + +chunksCache: + enabled: false +resultsCache: + enabled: false +lokiCanary: + enabled: false +test: + enabled: false +gateway: + enabled: false +sidecar: + rules: + enabled: false diff --git a/docs/k8s-cluster/logging.md b/docs/k8s-cluster/logging.md index b79ca5a28..31a551b89 100644 --- a/docs/k8s-cluster/logging.md +++ b/docs/k8s-cluster/logging.md @@ -1,91 +1,27 @@ # Logging -Centralized logging is provided by Filebeat, Elasticsearch and Kibana. +Optional log search for Kubernetes uses Loki (single binary on a persistent volume) and Grafana Alloy (one pod per node), shown in the Grafana installed by the monitoring script. It is off by default. -- [Logging](#logging) - - [Installation](#installation) - - [Install and validate the Elasticsearch cluster](#install-and-validate-the-elasticsearch-cluster) - - [Install Filebeat](#install-filebeat) - - [Install Kibana](#install-kibana) - - [Deleting the logging stack](#deleting-the-logging-stack) +> The previous Elastic/Filebeat/Kibana guide was removed: Elastic archived those Helm charts in 2023. -## Installation +## Install -The following procedure walks through installation of a centralized logging stack using the Helm repositories provided by [Elastic NV](https://www.elastic.co/). - -Our centralized logging solution requires that all hosts be configured with the same timezone and synchronized clocks. -This should be done automatically in our Kubernetes deployment process, but can be enforced manually by running the Chrony playbook: - -```bash -ansible-playbook playbooks/generic/chrony-client.yml -``` - -Add the Elastic repo in Helm: - -```bash -helm repo add elastic https://helm.elastic.co -``` - -### [Install and validate the Elasticsearch cluster](https://github.com/elastic/helm-charts/blob/main/elasticsearch/README.md) - -```bash -# Install Elasticsearch -# - Note that this document uses three replicas for availability, but you may -# want to set a different value depending on cluster size -helm install elasticsearch elastic/elasticsearch --set replicas=3 - -# Wait for the cluster to come online -kubectl get pods --namespace=default -l app=elasticsearch-master -w - -# Test the cluster health -helm --namespace=default test elasticsearch -``` - -### [Install Filebeat](https://github.com/elastic/helm-charts/blob/main/filebeat/README.md) - -Note that the default Filebeat configuration will import all container logs from the Kubernetes cluster nodes. +Install monitoring first, then logging (needs a default StorageClass): ```bash -# Install Filebeat -helm install filebeat elastic/filebeat - -# Wait for all containers to come up -kubectl get pods --namespace=default -l app=filebeat-filebeat -w +./scripts/k8s/deploy_monitoring.sh +./scripts/k8s/deploy_logging.sh ``` -### [Install Kibana](https://github.com/elastic/helm-charts/blob/main/kibana/README.md) +This installs Loki and Alloy in the `monitoring` namespace, adds Loki as a Grafana data source, and adds a **Cluster logs** dashboard. Logs from all pods are labelled `namespace`, `pod`, `container` and `host`. Default retention is 7 days and the default volume is 50Gi; edit `config/helm/loki.yml` (copy from `config.example/helm/loki.yml`) before installing to change them. -```bash -# Install Kibana -helm install kibana elastic/kibana - -# Wait for the container to come up -kubectl get pods --namespace=default -l app=kibana -w -``` - -By default, Kibana is only deployed as a ClusterIP service. -In order to expose it for user access, see the [chart documentation](https://github.com/elastic/helm-charts/blob/main/kibana/README.md) -or just expose it as a NodePort service: - -```bash -kubectl expose deployment kibana-kibana \ - --type=NodePort \ - --name=kibana-nodeport - -service/kibana-nodeport exposed - -kubectl get service kibana-nodeport -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -kibana-nodeport NodePort 10.233.39.104 5601:30965/TCP 42s -``` +Check the charts render without touching a cluster: `./scripts/k8s/deploy_logging.sh -t`. -## Deleting the logging stack +Pinned versions: `HELM_LOKI_CHART_VERSION` (community chart `grafana-community/loki`; Grafana Labs moved it there in 2026) and `HELM_ALLOY_CHART_VERSION`. -If there is an issue, you can follow these steps to delete the logging stack: +## Remove ```bash -helm delete kibana -helm delete filebeat -helm delete elasticsearch -kubectl delete pvc -l app=elasticsearch-master +./scripts/k8s/deploy_logging.sh -d # keeps stored logs +./scripts/k8s/deploy_logging.sh -d -D # also deletes the stored logs ``` diff --git a/docs/slurm-cluster/slurm-logging.md b/docs/slurm-cluster/slurm-logging.md new file mode 100644 index 000000000..5400b71f9 --- /dev/null +++ b/docs/slurm-cluster/slurm-logging.md @@ -0,0 +1,33 @@ +# Slurm log search (optional) + +DeepOps can run Loki (log store) on the monitoring node and Grafana Alloy (collector) on every node, so you can search system, kernel and Slurm daemon logs in Grafana. It is **off by default**; the existing rsyslog forwarding to the head node is unchanged. + +## Enable + +In `config/group_vars/slurm-cluster.yml`: + +```yaml +slurm_enable_monitoring: true +slurm_enable_log_search: true +# loki_retention_period: 168h # default 7 days; must be a multiple of 24h +``` + +Then run `ansible-playbook -l slurm-cluster playbooks/slurm-cluster.yml`, or just the pieces: + +```bash +ansible-playbook playbooks/slurm-cluster/loki.yml -e hostlist=slurm-master +ansible-playbook playbooks/slurm-cluster/alloy.yml -e hostlist=slurm-cluster +ansible-playbook playbooks/slurm-cluster/grafana.yml -e hostlist=slurm-master -e grafana_enable_loki=true +``` + +## Use + +Grafana (`http://:3000/`) gets a **Loki** data source and a **Cluster logs** dashboard (log volume by host, journal/kernel, Slurm daemons, GPU XID and out-of-memory kernel messages). To follow one job, search for its job ID in the *Search text* box. + +Labels: `host`, `job` (`systemd-journal` or `slurm`), `unit`, `level`, `transport` (`kernel` for kernel messages). + +## Notes + +- Loki listens on port 3100 with no authentication. Keep it on the cluster network (see `docs/misc/firewall.md`). +- Alloy reads Slurm logs from `/var/log/slurm/*.log`; change `alloy_slurm_log_paths` if your logs live elsewhere. +- Disable by setting the flag back to `false` and running `systemctl disable --now docker.loki.service docker.alloy.service` on the nodes; Grafana drops the Loki data source on its next run. The `deepops_loki_data` Docker volume keeps old logs until you remove it. diff --git a/docs/slurm-cluster/slurm-monitor.md b/docs/slurm-cluster/slurm-monitor.md index 9c9a717af..7e6351b32 100644 --- a/docs/slurm-cluster/slurm-monitor.md +++ b/docs/slurm-cluster/slurm-monitor.md @@ -103,3 +103,7 @@ Visit http://\:9093 for checking and setting about alerting. ![Alertmanager](../img/slurm_monitoring_alertmanager02.png) For more information on constructing Alertmanager alerting see the official [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/). + +## Log search + +Optional Loki + Alloy log search is described in [slurm-logging.md](slurm-logging.md). diff --git a/playbooks/slurm-cluster.yml b/playbooks/slurm-cluster.yml index 3f0788308..3976a4d77 100644 --- a/playbooks/slurm-cluster.yml +++ b/playbooks/slurm-cluster.yml @@ -103,6 +103,7 @@ import_playbook: slurm-cluster/grafana.yml vars: hostlist: "{{ slurm_monitoring_group | default('slurm-metric') }}" + grafana_enable_loki: "{{ slurm_enable_log_search | default(false) }}" when: slurm_enable_monitoring - name: Install monitoring services | Install alert manager @@ -125,6 +126,18 @@ import_playbook: slurm-cluster/nvidia-dcgm-exporter.yml when: slurm_enable_monitoring +- name: Install log search | Install loki + import_playbook: slurm-cluster/loki.yml + vars: + hostlist: "{{ slurm_monitoring_group | default('slurm-metric') }}" + when: slurm_enable_monitoring and (slurm_enable_log_search | default(false) | bool) + +- name: Install log search | Install alloy log collector + import_playbook: slurm-cluster/alloy.yml + vars: + hostlist: "slurm-cluster" + when: slurm_enable_monitoring and (slurm_enable_log_search | default(false) | bool) + - name: Set up rsyslog forwarding from compute nodes to head node | Install rsyslog server import_playbook: generic/rsyslog-server.yml vars: diff --git a/playbooks/slurm-cluster/alloy.yml b/playbooks/slurm-cluster/alloy.yml new file mode 100644 index 000000000..2e6884fd4 --- /dev/null +++ b/playbooks/slurm-cluster/alloy.yml @@ -0,0 +1,10 @@ +--- +- name: Install docker + import_playbook: ../container/docker.yml + +- hosts: "{{ hostlist | default('all') }}" + become: yes + tasks: + - name: configure alloy + include_role: + name: alloy diff --git a/playbooks/slurm-cluster/loki.yml b/playbooks/slurm-cluster/loki.yml new file mode 100644 index 000000000..e8b4675a2 --- /dev/null +++ b/playbooks/slurm-cluster/loki.yml @@ -0,0 +1,10 @@ +--- +- name: Install docker + import_playbook: ../container/docker.yml + +- hosts: "{{ hostlist | default('all') }}" + become: yes + tasks: + - name: configure loki + include_role: + name: loki diff --git a/roles/alloy/defaults/main.yml b/roles/alloy/defaults/main.yml new file mode 100644 index 000000000..3aa8e5fd1 --- /dev/null +++ b/roles/alloy/defaults/main.yml @@ -0,0 +1,17 @@ +--- +alloy_container: "grafana/alloy:v1.22.0" +alloy_svc_name: "docker.alloy.service" +alloy_config_dir: /etc/alloy +alloy_state: started +alloy_enabled: yes + +# Where to send logs. Defaults to Loki on the first monitoring host. +alloy_loki_host: "{{ (groups[slurm_monitoring_group | default('slurm-master')] | default([]) | first) | default('localhost') }}" +alloy_loki_port: 3100 +alloy_loki_url: "http://{{ alloy_loki_host }}:{{ alloy_loki_port }}/loki/api/v1/push" + +# Local Slurm daemon log files to tail (missing files are ignored). +alloy_slurm_log_paths: + - /var/log/slurm/*.log +alloy_collect_slurm_logs: true +alloy_journal_max_age: 12h diff --git a/roles/alloy/handlers/main.yml b/roles/alloy/handlers/main.yml new file mode 100644 index 000000000..ce71f92d8 --- /dev/null +++ b/roles/alloy/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart alloy + service: + name: "{{ alloy_svc_name }}" + state: restarted diff --git a/roles/alloy/molecule/default/converge.yml b/roles/alloy/molecule/default/converge.yml new file mode 100644 index 000000000..5a870353f --- /dev/null +++ b/roles/alloy/molecule/default/converge.yml @@ -0,0 +1,10 @@ +--- +# Docker containers can't run the docker-in-docker service, so only the +# configuration tasks are converged here. The service is covered by a lab run. +- name: Converge + hosts: all + tasks: + - name: "Write alloy configuration" + include_role: + name: "alloy" + tasks_from: config.yml diff --git a/roles/alloy/molecule/default/molecule.yml b/roles/alloy/molecule/default/molecule.yml new file mode 100644 index 000000000..1de207860 --- /dev/null +++ b/roles/alloy/molecule/default/molecule.yml @@ -0,0 +1,22 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: alloy-ubuntu-2204 + image: geerlingguy/docker-ubuntu2204-ansible + cgroupns_mode: host + privileged: true + pre_build_image: true + - name: alloy-ubuntu-2404 + image: geerlingguy/docker-ubuntu2404-ansible + cgroupns_mode: host + privileged: true + pre_build_image: true +provisioner: + name: ansible + ansible_args: + - -vv +verifier: + name: ansible diff --git a/roles/alloy/molecule/default/verify.yml b/roles/alloy/molecule/default/verify.yml new file mode 100644 index 000000000..7ce048580 --- /dev/null +++ b/roles/alloy/molecule/default/verify.yml @@ -0,0 +1,13 @@ +--- +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Read rendered config + slurp: + src: /etc/alloy/config.alloy + register: cfg + - name: Config contains expected content + assert: + that: + - "'loki.write' in (cfg.content | b64decode)" diff --git a/roles/alloy/tasks/config.yml b/roles/alloy/tasks/config.yml new file mode 100644 index 000000000..4cbf0ca35 --- /dev/null +++ b/roles/alloy/tasks/config.yml @@ -0,0 +1,15 @@ +--- +- name: create config dir + file: + path: "{{ alloy_config_dir }}" + state: directory + mode: "0755" + +- name: install config file + template: + src: config.alloy.j2 + dest: "{{ alloy_config_dir }}/config.alloy" + owner: root + group: root + mode: "0644" + notify: restart alloy diff --git a/roles/alloy/tasks/main.yml b/roles/alloy/tasks/main.yml new file mode 100644 index 000000000..ed1e69d58 --- /dev/null +++ b/roles/alloy/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: write configuration + import_tasks: config.yml + +- name: install python3 docker module + package: + name: python3-docker + state: present + +- name: create a persistent docker volume for Alloy position data + docker_volume: + name: deepops_alloy_data + state: present + +- name: install systemd unit file + template: + src: docker.alloy.service.j2 + dest: "/etc/systemd/system/{{ alloy_svc_name }}" + owner: root + group: root + mode: "0644" + notify: restart alloy + +- name: configure service + systemd: + name: "{{ alloy_svc_name }}" + state: "{{ alloy_state }}" + enabled: "{{ alloy_enabled }}" + daemon_reload: yes diff --git a/roles/alloy/templates/config.alloy.j2 b/roles/alloy/templates/config.alloy.j2 new file mode 100644 index 000000000..07ea98192 --- /dev/null +++ b/roles/alloy/templates/config.alloy.j2 @@ -0,0 +1,55 @@ +// Managed by DeepOps. Ships the system journal (including kernel) and +// Slurm daemon logs to Loki. + +loki.write "default" { + endpoint { + url = "{{ alloy_loki_url }}" + } +} + +loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__hostname"] + target_label = "host" + } + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal_syslog_identifier"] + target_label = "syslog_identifier" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + rule { + source_labels = ["__journal__transport"] + target_label = "transport" + } +} + +loki.source.journal "system" { + max_age = "{{ alloy_journal_max_age }}" + relabel_rules = loki.relabel.journal.rules + labels = {job = "systemd-journal", cluster = "slurm"} + forward_to = [loki.write.default.receiver] +} +{% if alloy_collect_slurm_logs %} + +local.file_match "slurm" { + path_targets = [ +{% for p in alloy_slurm_log_paths %} + {"__path__" = "{{ p }}", "job" = "slurm", "host" = "{{ inventory_hostname_short }}", "cluster" = "slurm"}, +{% endfor %} + ] +} + +loki.source.file "slurm" { + targets = local.file_match.slurm.targets + forward_to = [loki.write.default.receiver] +} +{% endif %} diff --git a/roles/alloy/templates/docker.alloy.service.j2 b/roles/alloy/templates/docker.alloy.service.j2 new file mode 100644 index 000000000..f589d30d8 --- /dev/null +++ b/roles/alloy/templates/docker.alloy.service.j2 @@ -0,0 +1,22 @@ +[Unit] +Description=Grafana Alloy +After=docker.service +Requires=docker.service +PartOf=docker.service + +[Service] +TimeoutStartSec=0 +Restart=always +ExecStartPre=-/usr/bin/docker stop %n +ExecStartPre=-/usr/bin/docker rm %n +ExecStartPre=/usr/bin/docker pull {{ alloy_container }} +ExecStart=/usr/bin/docker run --rm --network host --name %n \ + -v {{ alloy_config_dir }}:/etc/alloy:ro \ + -v /var/log:/var/log:ro \ + -v /run/log/journal:/run/log/journal:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v deepops_alloy_data:/var/lib/alloy/data \ + {{ alloy_container }} run --server.http.listen-addr=127.0.0.1:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy + +[Install] +WantedBy=multi-user.target diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index 5b1dfb317..602105e3b 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -10,3 +10,8 @@ grafana_enabled: yes grafana_cfg_user: admin grafana_cfg_pass: deepops grafana_cfg_dashboard_path: "{{ grafana_data_dir }}/dashboards" + +# Optional log search: add Loki as a data source and ship the "Cluster logs" +# dashboard. Off by default; see docs/slurm-cluster/slurm-logging.md. +grafana_enable_loki: false +grafana_loki_url: "http://localhost:3100" diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index dff62fcea..eb40eb5ea 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -50,6 +50,32 @@ mode: 0644 notify: restart grafana +- name: install Loki datasource config + template: + src: "templates/loki-datasource.yml.j2" + dest: "{{ grafana_config_dir }}/provisioning/datasources/loki-datasource.yml" + owner: root + group: root + mode: 0644 + when: grafana_enable_loki | bool + notify: restart grafana + +- name: copy Cluster logs dashboard + copy: + src: ../../../src/dashboards/cluster-logs-dashboard.json + dest: "{{ grafana_cfg_dashboard_path }}" + owner: "{{ grafana_user_id }}" + mode: "0644" + when: grafana_enable_loki | bool + notify: restart grafana + +- name: remove Loki datasource config when log search is disabled + file: + path: "{{ grafana_config_dir }}/provisioning/datasources/loki-datasource.yml" + state: absent + when: not (grafana_enable_loki | bool) + notify: restart grafana + - name: install dashboard config template: src: "templates/prometheus-dashboard.yml.j2" diff --git a/roles/grafana/templates/loki-datasource.yml.j2 b/roles/grafana/templates/loki-datasource.yml.j2 new file mode 100644 index 000000000..a0a3b2e2a --- /dev/null +++ b/roles/grafana/templates/loki-datasource.yml.j2 @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: +- name: Loki + uid: deepops-loki + type: loki + access: proxy + url: {{ grafana_loki_url }} + isDefault: false diff --git a/roles/loki/defaults/main.yml b/roles/loki/defaults/main.yml new file mode 100644 index 000000000..b9817a153 --- /dev/null +++ b/roles/loki/defaults/main.yml @@ -0,0 +1,15 @@ +--- +loki_container: "grafana/loki:3.7.10" +loki_svc_name: "docker.loki.service" +loki_config_dir: /etc/loki +loki_docker_volume_name: "deepops_loki_data" +loki_state: started +loki_enabled: yes + +# Port Loki listens on (host network). Loki has no authentication; restrict +# access to the cluster network with a firewall. +loki_http_port: 3100 +loki_http_listen_address: "0.0.0.0" + +# How long logs are kept. Must be a multiple of 24h (Loki compactor rule). +loki_retention_period: 168h diff --git a/roles/loki/handlers/main.yml b/roles/loki/handlers/main.yml new file mode 100644 index 000000000..b41d07535 --- /dev/null +++ b/roles/loki/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart loki + service: + name: "{{ loki_svc_name }}" + state: restarted diff --git a/roles/loki/molecule/default/converge.yml b/roles/loki/molecule/default/converge.yml new file mode 100644 index 000000000..20a080d02 --- /dev/null +++ b/roles/loki/molecule/default/converge.yml @@ -0,0 +1,10 @@ +--- +# Docker containers can't run the docker-in-docker service, so only the +# configuration tasks are converged here. The service is covered by a lab run. +- name: Converge + hosts: all + tasks: + - name: "Write loki configuration" + include_role: + name: "loki" + tasks_from: config.yml diff --git a/roles/loki/molecule/default/molecule.yml b/roles/loki/molecule/default/molecule.yml new file mode 100644 index 000000000..fc5dfbf04 --- /dev/null +++ b/roles/loki/molecule/default/molecule.yml @@ -0,0 +1,22 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: loki-ubuntu-2204 + image: geerlingguy/docker-ubuntu2204-ansible + cgroupns_mode: host + privileged: true + pre_build_image: true + - name: loki-ubuntu-2404 + image: geerlingguy/docker-ubuntu2404-ansible + cgroupns_mode: host + privileged: true + pre_build_image: true +provisioner: + name: ansible + ansible_args: + - -vv +verifier: + name: ansible diff --git a/roles/loki/molecule/default/verify.yml b/roles/loki/molecule/default/verify.yml new file mode 100644 index 000000000..eb7d14a46 --- /dev/null +++ b/roles/loki/molecule/default/verify.yml @@ -0,0 +1,13 @@ +--- +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Read rendered config + slurp: + src: /etc/loki/loki-config.yml + register: cfg + - name: Config contains expected content + assert: + that: + - "'retention_period: 168h' in (cfg.content | b64decode)" diff --git a/roles/loki/tasks/config.yml b/roles/loki/tasks/config.yml new file mode 100644 index 000000000..970778216 --- /dev/null +++ b/roles/loki/tasks/config.yml @@ -0,0 +1,15 @@ +--- +- name: create config dir + file: + path: "{{ loki_config_dir }}" + state: directory + mode: "0755" + +- name: install config file + template: + src: loki-config.yml.j2 + dest: "{{ loki_config_dir }}/loki-config.yml" + owner: root + group: root + mode: "0644" + notify: restart loki diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml new file mode 100644 index 000000000..42f986be6 --- /dev/null +++ b/roles/loki/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: write configuration + import_tasks: config.yml + +- name: install python3 docker module + package: + name: python3-docker + state: present + +- name: create a persistent docker volume for logs + docker_volume: + name: "{{ loki_docker_volume_name }}" + state: present + +- name: install systemd unit file + template: + src: docker.loki.service.j2 + dest: "/etc/systemd/system/{{ loki_svc_name }}" + owner: root + group: root + mode: "0644" + notify: restart loki + +- name: configure service + systemd: + name: "{{ loki_svc_name }}" + state: "{{ loki_state }}" + enabled: "{{ loki_enabled }}" + daemon_reload: yes diff --git a/roles/loki/templates/docker.loki.service.j2 b/roles/loki/templates/docker.loki.service.j2 new file mode 100644 index 000000000..645de24b8 --- /dev/null +++ b/roles/loki/templates/docker.loki.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Loki +After=docker.service +Requires=docker.service +PartOf=docker.service + +[Service] +TimeoutStartSec=0 +Restart=always +ExecStartPre=-/usr/bin/docker stop %n +ExecStartPre=-/usr/bin/docker rm %n +ExecStartPre=/usr/bin/docker pull {{ loki_container }} +ExecStart=/usr/bin/docker run --rm --network host --name %n -v {{ loki_config_dir }}:/etc/loki:ro -v {{ loki_docker_volume_name }}:/loki {{ loki_container }} -config.file=/etc/loki/loki-config.yml + +[Install] +WantedBy=multi-user.target diff --git a/roles/loki/templates/loki-config.yml.j2 b/roles/loki/templates/loki-config.yml.j2 new file mode 100644 index 000000000..7a1ef1634 --- /dev/null +++ b/roles/loki/templates/loki-config.yml.j2 @@ -0,0 +1,42 @@ +# Managed by DeepOps. Single-binary Loki with local filesystem storage. +auth_enabled: false + +server: + http_listen_address: {{ loki_http_listen_address }} + http_listen_port: {{ loki_http_port }} + grpc_listen_address: 127.0.0.1 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: {{ loki_retention_period }} + allow_structured_metadata: true + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/scripts/k8s/deploy_logging.sh b/scripts/k8s/deploy_logging.sh new file mode 100755 index 000000000..20a57dec0 --- /dev/null +++ b/scripts/k8s/deploy_logging.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +# Optional log search for Kubernetes: Loki (single binary, persistent volume) +# plus Grafana Alloy on every node, with Loki added as a source in the Grafana +# installed by scripts/k8s/deploy_monitoring.sh. Off by default: nothing runs +# unless you run this script. + +# Ensure we start in the correct working directory +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="${SCRIPT_DIR}/../.." +cd "${ROOT_DIR}" || exit 1 + +source "${ROOT_DIR}/scripts/common.sh" + +DEEPOPS_CONFIG_DIR=${DEEPOPS_CONFIG_DIR:-"${ROOT_DIR}/config"} + +# Loki's free chart is community-maintained (grafana-community) since 2026-03. +HELM_CHARTS_REPO_LOKI="${HELM_CHARTS_REPO_LOKI:-https://grafana-community.github.io/helm-charts}" +HELM_LOKI_CHART_VERSION="${HELM_LOKI_CHART_VERSION:-21.2.3}" +HELM_CHARTS_REPO_ALLOY="${HELM_CHARTS_REPO_ALLOY:-https://grafana.github.io/helm-charts}" +HELM_ALLOY_CHART_VERSION="${HELM_ALLOY_CHART_VERSION:-1.16.0}" + +LOKI_YAML_CONFIG="${LOKI_YAML_CONFIG:-${DEEPOPS_CONFIG_DIR}/helm/loki.yml}" +ALLOY_YAML_CONFIG="${ALLOY_YAML_CONFIG:-${DEEPOPS_CONFIG_DIR}/helm/alloy.yml}" +LOGGING_NAMESPACE="${LOGGING_NAMESPACE:-monitoring}" +GRAFANA_LOKI_DATASOURCE_CM="grafana-loki-datasource" +GRAFANA_LOGS_DASHBOARD_CM="kube-prometheus-grafana-logs" + +function help_me() { + echo "This script installs optional log search: Loki, Alloy and a Grafana data source." + echo "Run scripts/k8s/deploy_monitoring.sh first (Grafana must exist in the '${LOGGING_NAMESPACE}' namespace)." + echo "" + echo "Usage:" + echo "-h This message." + echo "-d Uninstall Loki, Alloy and the Grafana additions. Pass -D as well to delete stored logs." + echo "-D With -d, also delete the Loki persistent volume claim (stored logs)." + echo "-t Render the charts locally (helm template) without touching the cluster." +} + +DELETE="" +DELETE_DATA="" +TEMPLATE_ONLY="" +while getopts "hdDt" option; do + case $option in + d) DELETE=1 ;; + D) DELETE_DATA=1 ;; + t) TEMPLATE_ONLY=1 ;; + *) help_me; exit 1 ;; + esac +done + +function add_repos() { + helm repo add grafana-community "${HELM_CHARTS_REPO_LOKI}" >/dev/null 2>&1 || true + helm repo add grafana "${HELM_CHARTS_REPO_ALLOY}" >/dev/null 2>&1 || true + helm repo update grafana-community grafana >/dev/null +} + +function render_only() { + add_repos || exit 1 + helm template loki grafana-community/loki --version "${HELM_LOKI_CHART_VERSION}" \ + --namespace "${LOGGING_NAMESPACE}" --values "${LOKI_YAML_CONFIG}" >/dev/null || exit 1 + helm template alloy grafana/alloy --version "${HELM_ALLOY_CHART_VERSION}" \ + --namespace "${LOGGING_NAMESPACE}" --values "${ALLOY_YAML_CONFIG}" >/dev/null || exit 1 + echo "Loki ${HELM_LOKI_CHART_VERSION} and Alloy ${HELM_ALLOY_CHART_VERSION} charts render OK" +} + +function delete_logging() { + helm uninstall alloy -n "${LOGGING_NAMESPACE}" + helm uninstall loki -n "${LOGGING_NAMESPACE}" + kubectl -n "${LOGGING_NAMESPACE}" delete configmap "${GRAFANA_LOKI_DATASOURCE_CM}" "${GRAFANA_LOGS_DASHBOARD_CM}" --ignore-not-found + if [ "${DELETE_DATA}" ]; then + kubectl -n "${LOGGING_NAMESPACE}" delete pvc -l app.kubernetes.io/instance=loki + else + echo "Stored logs kept (PVC). Re-run with -d -D to delete them." + fi +} + +function install_logging() { + kubectl version >/dev/null 2>&1 || { echo "Unable to talk to Kubernetes API"; exit 1; } + ./scripts/k8s/install_helm.sh + if ! kubectl get ns "${LOGGING_NAMESPACE}" >/dev/null 2>&1; then + echo "Namespace ${LOGGING_NAMESPACE} not found. Run scripts/k8s/deploy_monitoring.sh first." >&2 + exit 1 + fi + kubectl get storageclass 2>&1 | grep "(default)" >/dev/null 2>&1 || { + echo "No default StorageClass found; Loki needs a persistent volume." >&2 + exit 1 + } + add_repos || exit 1 + + helm upgrade --install loki grafana-community/loki --version "${HELM_LOKI_CHART_VERSION}" \ + --namespace "${LOGGING_NAMESPACE}" --values "${LOKI_YAML_CONFIG}" --timeout 600s --wait || exit 1 + helm upgrade --install alloy grafana/alloy --version "${HELM_ALLOY_CHART_VERSION}" \ + --namespace "${LOGGING_NAMESPACE}" --values "${ALLOY_YAML_CONFIG}" --timeout 600s --wait || exit 1 + + # Grafana's sidecar picks up labelled ConfigMaps (datasource + dashboard). + kubectl -n "${LOGGING_NAMESPACE}" create configmap "${GRAFANA_LOKI_DATASOURCE_CM}" \ + --from-file=loki-datasource.yaml="${DEEPOPS_CONFIG_DIR}/helm/loki-datasource.yaml" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n "${LOGGING_NAMESPACE}" label configmap "${GRAFANA_LOKI_DATASOURCE_CM}" grafana_datasource=1 --overwrite + kubectl -n "${LOGGING_NAMESPACE}" create configmap "${GRAFANA_LOGS_DASHBOARD_CM}" \ + --from-file="${ROOT_DIR}/src/dashboards/cluster-logs-dashboard.json" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n "${LOGGING_NAMESPACE}" label configmap "${GRAFANA_LOGS_DASHBOARD_CM}" grafana_dashboard=1 --overwrite + echo "Log search installed. Open Grafana > Dashboards > Cluster logs (or Explore > Loki)." +} + +if [ "${TEMPLATE_ONLY}" ]; then render_only; exit $?; fi +if [ "${DELETE}" ]; then delete_logging; exit 0; fi +install_logging diff --git a/src/dashboards/cluster-logs-dashboard.json b/src/dashboards/cluster-logs-dashboard.json new file mode 100644 index 000000000..d1b1bb654 --- /dev/null +++ b/src/dashboards/cluster-logs-dashboard.json @@ -0,0 +1,67 @@ +{ + "uid": "deepops-cluster-logs", + "title": "Cluster logs", + "tags": ["deepops", "logs"], + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": {"from": "now-1h", "to": "now"}, + "templating": { + "list": [ + { + "name": "host", "label": "Host", "type": "query", + "datasource": {"type": "loki", "uid": "deepops-loki"}, + "query": {"label": "host", "stream": "", "type": 1}, + "refresh": 2, "includeAll": true, "multi": true, + "allValue": ".+", "current": {"text": "All", "value": "$__all"} + }, + { + "name": "search", "label": "Search text", "type": "textbox", + "query": "", "current": {"text": "", "value": ""} + } + ] + }, + "panels": [ + { + "id": 1, "type": "timeseries", "title": "Log volume by host", + "datasource": {"type": "loki", "uid": "deepops-loki"}, + "gridPos": {"h": 7, "w": 24, "x": 0, "y": 0}, + "targets": [{ + "refId": "A", "queryType": "range", + "expr": "sum by (host) (count_over_time({host=~\"$host\"} |~ \"(?i)$search\" [$__interval]))", + "legendFormat": "{{host}}" + }] + }, + { + "id": 2, "type": "logs", "title": "System journal and kernel", + "datasource": {"type": "loki", "uid": "deepops-loki"}, + "gridPos": {"h": 12, "w": 24, "x": 0, "y": 7}, + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true}, + "targets": [{ + "refId": "A", "queryType": "range", + "expr": "{job=\"systemd-journal\", host=~\"$host\"} |~ \"(?i)$search\"" + }] + }, + { + "id": 3, "type": "logs", "title": "Slurm daemon logs", + "datasource": {"type": "loki", "uid": "deepops-loki"}, + "gridPos": {"h": 12, "w": 24, "x": 0, "y": 19}, + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true}, + "targets": [{ + "refId": "A", "queryType": "range", + "expr": "{job=\"slurm\", host=~\"$host\"} |~ \"(?i)$search\"" + }] + }, + { + "id": 4, "type": "logs", "title": "Kernel: GPU errors (XID) and out-of-memory kills", + "datasource": {"type": "loki", "uid": "deepops-loki"}, + "gridPos": {"h": 10, "w": 24, "x": 0, "y": 31}, + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending"}, + "targets": [{ + "refId": "A", "queryType": "range", + "expr": "{job=\"systemd-journal\", transport=\"kernel\", host=~\"$host\"} |~ \"NVRM: Xid|Out of memory|oom-kill\"" + }] + } + ] +} diff --git a/tests/logging/test_logging_render.py b/tests/logging/test_logging_render.py new file mode 100644 index 000000000..fe2df3fb4 --- /dev/null +++ b/tests/logging/test_logging_render.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Render-time checks for the optional log search (Loki + Alloy). No cluster needed.""" +import json +import pathlib +import re +import shutil +import subprocess +import unittest + +import jinja2 +import yaml + +ROOT = pathlib.Path(__file__).resolve().parents[2] + + +def render(path, **ctx): + return jinja2.Template((ROOT / path).read_text()).render(**ctx) + + +class LogSearch(unittest.TestCase): + def test_loki_config_is_valid_yaml_with_retention(self): + out = render("roles/loki/templates/loki-config.yml.j2", loki_http_listen_address="0.0.0.0", + loki_http_port=3100, loki_retention_period="168h") + cfg = yaml.safe_load(out) + self.assertEqual(cfg["limits_config"]["retention_period"], "168h") + self.assertTrue(cfg["compactor"]["retention_enabled"]) + + def test_loki_default_retention_is_7_days_multiple_of_24h(self): + d = yaml.safe_load((ROOT / "roles/loki/defaults/main.yml").read_text()) + self.assertEqual(d["loki_retention_period"], "168h") + self.assertEqual(int(d["loki_retention_period"][:-1]) % 24, 0) + + def test_alloy_config_collects_journal_and_slurm(self): + out = render("roles/alloy/templates/config.alloy.j2", alloy_loki_url="http://m:3100/loki/api/v1/push", + alloy_journal_max_age="12h", alloy_collect_slurm_logs=True, + alloy_slurm_log_paths=["/var/log/slurm/*.log"], inventory_hostname_short="n1") + self.assertIn("loki.source.journal", out) + self.assertIn("/var/log/slurm/*.log", out) + off = render("roles/alloy/templates/config.alloy.j2", alloy_loki_url="u", alloy_journal_max_age="1h", + alloy_collect_slurm_logs=False, alloy_slurm_log_paths=[], inventory_hostname_short="n1") + self.assertNotIn("local.file_match", off) + + def test_off_by_default(self): + self.assertFalse(yaml.safe_load((ROOT / "roles/grafana/defaults/main.yml").read_text())["grafana_enable_loki"]) + self.assertRegex((ROOT / "config.example/group_vars/slurm-cluster.yml").read_text(), + re.compile(r"^slurm_enable_log_search: false$", re.M)) + pb = (ROOT / "playbooks/slurm-cluster.yml").read_text() + self.assertIn("slurm_enable_log_search | default(false)", pb) + + def test_dashboard_json_and_datasource_uid(self): + d = json.loads((ROOT / "src/dashboards/cluster-logs-dashboard.json").read_text()) + self.assertEqual(d["title"], "Cluster logs") + ds = yaml.safe_load(render("roles/grafana/templates/loki-datasource.yml.j2", grafana_loki_url="http://x:3100")) + self.assertEqual(ds["datasources"][0]["uid"], "deepops-loki") + self.assertIn('"uid": "deepops-loki"', (ROOT / "src/dashboards/cluster-logs-dashboard.json").read_text()) + + def test_k8s_values_parse(self): + for f in ("loki.yml", "alloy.yml", "loki-datasource.yaml"): + yaml.safe_load((ROOT / "config.example/helm" / f).read_text()) + + @unittest.skipUnless(shutil.which("helm"), "helm not installed") + def test_helm_charts_render(self): + r = subprocess.run(["bash", str(ROOT / "scripts/k8s/deploy_logging.sh"), "-t"], + env={"DEEPOPS_CONFIG_DIR": str(ROOT / "config.example"), "PATH": __import__("os").environ["PATH"], + "HOME": __import__("os").environ["HOME"]}, + capture_output=True, text=True, timeout=300) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + +if __name__ == "__main__": + unittest.main() From 5b3cacd6cd3579ddaa9cade7ff689e4199874d77 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 16 Nov 2026 07:25:07 -0700 Subject: [PATCH 2/2] loki, alloy: skip service start under molecule Docker molecule containers have no systemd as PID 1, so the service task and restart handler fail converge. Tag the service task molecule-notest (same as nvidia_dcgm) and guard the handlers, which ignore tags, on ansible_skip_tags. Signed-off-by: Doug Holt --- roles/alloy/handlers/main.yml | 1 + roles/alloy/tasks/main.yml | 2 ++ roles/loki/handlers/main.yml | 1 + roles/loki/tasks/main.yml | 2 ++ 4 files changed, 6 insertions(+) diff --git a/roles/alloy/handlers/main.yml b/roles/alloy/handlers/main.yml index ce71f92d8..5e907749e 100644 --- a/roles/alloy/handlers/main.yml +++ b/roles/alloy/handlers/main.yml @@ -3,3 +3,4 @@ service: name: "{{ alloy_svc_name }}" state: restarted + when: "'molecule-notest' not in ansible_skip_tags" diff --git a/roles/alloy/tasks/main.yml b/roles/alloy/tasks/main.yml index ed1e69d58..8b9818756 100644 --- a/roles/alloy/tasks/main.yml +++ b/roles/alloy/tasks/main.yml @@ -27,3 +27,5 @@ state: "{{ alloy_state }}" enabled: "{{ alloy_enabled }}" daemon_reload: yes + tags: + - molecule-notest diff --git a/roles/loki/handlers/main.yml b/roles/loki/handlers/main.yml index b41d07535..e3fed9e56 100644 --- a/roles/loki/handlers/main.yml +++ b/roles/loki/handlers/main.yml @@ -3,3 +3,4 @@ service: name: "{{ loki_svc_name }}" state: restarted + when: "'molecule-notest' not in ansible_skip_tags" diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml index 42f986be6..9d08a7af0 100644 --- a/roles/loki/tasks/main.yml +++ b/roles/loki/tasks/main.yml @@ -27,3 +27,5 @@ state: "{{ loki_state }}" enabled: "{{ loki_enabled }}" daemon_reload: yes + tags: + - molecule-notest