Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/molecule.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ jobs:
- lmod
- nfs
- nhc
- alloy
- loki
- nvidia_dcgm
- openmpi
- mofed
Expand Down
9 changes: 9 additions & 0 deletions config.example/group_vars/slurm-cluster.yml
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,15 @@ slurm_enable_monitoring: true
# (Prometheus, Grafana, etc)
slurm_monitoring_group: "slurm-master"

################################################################################
# Optional log search (Loki + Alloy), off by default #
################################################################################
# Runs Loki on the monitoring node and Alloy on every node, and adds a Loki
# data source and a "Cluster logs" dashboard to Grafana. Requires
# slurm_enable_monitoring. See docs/slurm-cluster/slurm-logging.md.
slurm_enable_log_search: false
# loki_retention_period: 168h # 7 days; must be a multiple of 24h

################################################################################
# Logging with rsyslog #
################################################################################
Expand Down
54 changes: 54 additions & 0 deletions config.example/helm/alloy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Values for Grafana Alloy: one pod per node, shipping pod and node logs to Loki.
# Chart: grafana/alloy. Used by scripts/k8s/deploy_logging.sh.
controller:
type: daemonset

alloy:
mounts:
varlog: true
configMap:
create: true
content: |
discovery.kubernetes "pods" {
role = "pod"
selectors {
role = "pod"
field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), "")
}
}

discovery.relabel "pods" {
targets = discovery.kubernetes.pods.targets
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "host"
}
rule {
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
separator = "/"
target_label = "job"
}
}

loki.source.kubernetes "pods" {
targets = discovery.relabel.pods.output
forward_to = [loki.write.default.receiver]
}

loki.write "default" {
endpoint {
url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push"
}
}
8 changes: 8 additions & 0 deletions config.example/helm/loki-datasource.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: 1
datasources:
- name: Loki
uid: deepops-loki
type: loki
access: proxy
url: http://loki.monitoring.svc.cluster.local:3100
isDefault: false
69 changes: 69 additions & 0 deletions config.example/helm/loki.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# Values for the optional Loki log store (single binary, persistent volume).
# Chart: grafana-community/loki. Used by scripts/k8s/deploy_logging.sh.
deploymentMode: SingleBinary

loki:
auth_enabled: false
commonConfig:
replication_factor: 1
schemaConfig:
configs:
- from: "2024-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: loki_index_
period: 24h
storage:
type: filesystem
limits_config:
# 7 days. Must be a multiple of 24h.
retention_period: 168h
allow_structured_metadata: true
compactor:
retention_enabled: true
delete_request_store: filesystem
useTestSchema: false

singleBinary:
replicas: 1
persistence:
enabled: true
size: 50Gi

# Not used in single binary mode
backend:
replicas: 0
read:
replicas: 0
write:
replicas: 0
ingester:
replicas: 0
querier:
replicas: 0
queryFrontend:
replicas: 0
queryScheduler:
replicas: 0
distributor:
replicas: 0
compactor:
replicas: 0
indexGateway:
replicas: 0

chunksCache:
enabled: false
resultsCache:
enabled: false
lokiCanary:
enabled: false
test:
enabled: false
gateway:
enabled: false
sidecar:
rules:
enabled: false
88 changes: 12 additions & 76 deletions docs/k8s-cluster/logging.md
Original file line number Diff line number Diff line change
@@ -1,91 +1,27 @@
# Logging

Centralized logging is provided by Filebeat, Elasticsearch and Kibana.
Optional log search for Kubernetes uses Loki (single binary on a persistent volume) and Grafana Alloy (one pod per node), shown in the Grafana installed by the monitoring script. It is off by default.

- [Logging](#logging)
- [Installation](#installation)
- [Install and validate the Elasticsearch cluster](#install-and-validate-the-elasticsearch-cluster)
- [Install Filebeat](#install-filebeat)
- [Install Kibana](#install-kibana)
- [Deleting the logging stack](#deleting-the-logging-stack)
> The previous Elastic/Filebeat/Kibana guide was removed: Elastic archived those Helm charts in 2023.

## Installation
## Install

The following procedure walks through installation of a centralized logging stack using the Helm repositories provided by [Elastic NV](https://www.elastic.co/).

Our centralized logging solution requires that all hosts be configured with the same timezone and synchronized clocks.
This should be done automatically in our Kubernetes deployment process, but can be enforced manually by running the Chrony playbook:

```bash
ansible-playbook playbooks/generic/chrony-client.yml
```

Add the Elastic repo in Helm:

```bash
helm repo add elastic https://helm.elastic.co
```

### [Install and validate the Elasticsearch cluster](https://github.com/elastic/helm-charts/blob/main/elasticsearch/README.md)

```bash
# Install Elasticsearch
# - Note that this document uses three replicas for availability, but you may
# want to set a different value depending on cluster size
helm install elasticsearch elastic/elasticsearch --set replicas=3

# Wait for the cluster to come online
kubectl get pods --namespace=default -l app=elasticsearch-master -w

# Test the cluster health
helm --namespace=default test elasticsearch
```

### [Install Filebeat](https://github.com/elastic/helm-charts/blob/main/filebeat/README.md)

Note that the default Filebeat configuration will import all container logs from the Kubernetes cluster nodes.
Install monitoring first, then logging (needs a default StorageClass):

```bash
# Install Filebeat
helm install filebeat elastic/filebeat

# Wait for all containers to come up
kubectl get pods --namespace=default -l app=filebeat-filebeat -w
./scripts/k8s/deploy_monitoring.sh
./scripts/k8s/deploy_logging.sh
```

### [Install Kibana](https://github.com/elastic/helm-charts/blob/main/kibana/README.md)
This installs Loki and Alloy in the `monitoring` namespace, adds Loki as a Grafana data source, and adds a **Cluster logs** dashboard. Logs from all pods are labelled `namespace`, `pod`, `container` and `host`. Default retention is 7 days and the default volume is 50Gi; edit `config/helm/loki.yml` (copy from `config.example/helm/loki.yml`) before installing to change them.

```bash
# Install Kibana
helm install kibana elastic/kibana

# Wait for the container to come up
kubectl get pods --namespace=default -l app=kibana -w
```

By default, Kibana is only deployed as a ClusterIP service.
In order to expose it for user access, see the [chart documentation](https://github.com/elastic/helm-charts/blob/main/kibana/README.md)
or just expose it as a NodePort service:

```bash
kubectl expose deployment kibana-kibana \
--type=NodePort \
--name=kibana-nodeport

service/kibana-nodeport exposed

kubectl get service kibana-nodeport
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kibana-nodeport NodePort 10.233.39.104 <none> 5601:30965/TCP 42s
```
Check the charts render without touching a cluster: `./scripts/k8s/deploy_logging.sh -t`.

## Deleting the logging stack
Pinned versions: `HELM_LOKI_CHART_VERSION` (community chart `grafana-community/loki`; Grafana Labs moved it there in 2026) and `HELM_ALLOY_CHART_VERSION`.

If there is an issue, you can follow these steps to delete the logging stack:
## Remove

```bash
helm delete kibana
helm delete filebeat
helm delete elasticsearch
kubectl delete pvc -l app=elasticsearch-master
./scripts/k8s/deploy_logging.sh -d # keeps stored logs
./scripts/k8s/deploy_logging.sh -d -D # also deletes the stored logs
```
33 changes: 33 additions & 0 deletions docs/slurm-cluster/slurm-logging.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Slurm log search (optional)

DeepOps can run Loki (log store) on the monitoring node and Grafana Alloy (collector) on every node, so you can search system, kernel and Slurm daemon logs in Grafana. It is **off by default**; the existing rsyslog forwarding to the head node is unchanged.

## Enable

In `config/group_vars/slurm-cluster.yml`:

```yaml
slurm_enable_monitoring: true
slurm_enable_log_search: true
# loki_retention_period: 168h # default 7 days; must be a multiple of 24h
```

Then run `ansible-playbook -l slurm-cluster playbooks/slurm-cluster.yml`, or just the pieces:

```bash
ansible-playbook playbooks/slurm-cluster/loki.yml -e hostlist=slurm-master
ansible-playbook playbooks/slurm-cluster/alloy.yml -e hostlist=slurm-cluster
ansible-playbook playbooks/slurm-cluster/grafana.yml -e hostlist=slurm-master -e grafana_enable_loki=true
```

## Use

Grafana (`http://<monitoring node>:3000/`) gets a **Loki** data source and a **Cluster logs** dashboard (log volume by host, journal/kernel, Slurm daemons, GPU XID and out-of-memory kernel messages). To follow one job, search for its job ID in the *Search text* box.

Labels: `host`, `job` (`systemd-journal` or `slurm`), `unit`, `level`, `transport` (`kernel` for kernel messages).

## Notes

- Loki listens on port 3100 with no authentication. Keep it on the cluster network (see `docs/misc/firewall.md`).
- Alloy reads Slurm logs from `/var/log/slurm/*.log`; change `alloy_slurm_log_paths` if your logs live elsewhere.
- Disable by setting the flag back to `false` and running `systemctl disable --now docker.loki.service docker.alloy.service` on the nodes; Grafana drops the Loki data source on its next run. The `deepops_loki_data` Docker volume keeps old logs until you remove it.
4 changes: 4 additions & 0 deletions docs/slurm-cluster/slurm-monitor.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,3 +103,7 @@ Visit http://\<slurm-metric ip\>:9093 for checking and setting about alerting.
![Alertmanager](../img/slurm_monitoring_alertmanager02.png)

For more information on constructing Alertmanager alerting see the official [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/).

## Log search

Optional Loki + Alloy log search is described in [slurm-logging.md](slurm-logging.md).
13 changes: 13 additions & 0 deletions playbooks/slurm-cluster.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@
import_playbook: slurm-cluster/grafana.yml
vars:
hostlist: "{{ slurm_monitoring_group | default('slurm-metric') }}"
grafana_enable_loki: "{{ slurm_enable_log_search | default(false) }}"
when: slurm_enable_monitoring

- name: Install monitoring services | Install alert manager
Expand All @@ -125,6 +126,18 @@
import_playbook: slurm-cluster/nvidia-dcgm-exporter.yml
when: slurm_enable_monitoring

- name: Install log search | Install loki
import_playbook: slurm-cluster/loki.yml
vars:
hostlist: "{{ slurm_monitoring_group | default('slurm-metric') }}"
when: slurm_enable_monitoring and (slurm_enable_log_search | default(false) | bool)

- name: Install log search | Install alloy log collector
import_playbook: slurm-cluster/alloy.yml
vars:
hostlist: "slurm-cluster"
when: slurm_enable_monitoring and (slurm_enable_log_search | default(false) | bool)

- name: Set up rsyslog forwarding from compute nodes to head node | Install rsyslog server
import_playbook: generic/rsyslog-server.yml
vars:
Expand Down
10 changes: 10 additions & 0 deletions playbooks/slurm-cluster/alloy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
---
- name: Install docker
import_playbook: ../container/docker.yml

- hosts: "{{ hostlist | default('all') }}"
become: yes
tasks:
- name: configure alloy
include_role:
name: alloy
10 changes: 10 additions & 0 deletions playbooks/slurm-cluster/loki.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
---
- name: Install docker
import_playbook: ../container/docker.yml

- hosts: "{{ hostlist | default('all') }}"
become: yes
tasks:
- name: configure loki
include_role:
name: loki
17 changes: 17 additions & 0 deletions roles/alloy/defaults/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---
alloy_container: "grafana/alloy:v1.22.0"
alloy_svc_name: "docker.alloy.service"
alloy_config_dir: /etc/alloy
alloy_state: started
alloy_enabled: yes

# Where to send logs. Defaults to Loki on the first monitoring host.
alloy_loki_host: "{{ (groups[slurm_monitoring_group | default('slurm-master')] | default([]) | first) | default('localhost') }}"
alloy_loki_port: 3100
alloy_loki_url: "http://{{ alloy_loki_host }}:{{ alloy_loki_port }}/loki/api/v1/push"

# Local Slurm daemon log files to tail (missing files are ignored).
alloy_slurm_log_paths:
- /var/log/slurm/*.log
alloy_collect_slurm_logs: true
alloy_journal_max_age: 12h
6 changes: 6 additions & 0 deletions roles/alloy/handlers/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
- name: restart alloy
service:
name: "{{ alloy_svc_name }}"
state: restarted
when: "'molecule-notest' not in ansible_skip_tags"
10 changes: 10 additions & 0 deletions roles/alloy/molecule/default/converge.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
---
# Docker containers can't run the docker-in-docker service, so only the
# configuration tasks are converged here. The service is covered by a lab run.
- name: Converge
hosts: all
tasks:
- name: "Write alloy configuration"
include_role:
name: "alloy"
tasks_from: config.yml
Loading
Loading