librenms
Give example cluster (swarm/k8s) deployment
By @jsenecal
It would be nice if you added an example docker-compose.yml suitable for docker in swarm mode.
This should also provide separate web and poller using rrdcache and memcache for rrd storage and poller state exchange (already available, though I cannot figure how you are configuring them in the "app" service).
I'm currently running something like this docker-compose.yml file with most of the librenms stuff from the other available image
A swarm (and/or kubernetes) with how to scale the dispatcher (and/or cron) sidecar would be great!
Here is my k3s example. It includes the deployments, services and an ingress.
Note that I am running my MySQL externally on my NAS which is why I don't have a MySQL container. I did this because when trying to use my NAS as a data store for MySQL gave me file lock errors. Installing the MySQL service on my NAS made things easier.
Also, all my data stores are on my NAS (mounted with NFS) which is why there are nfs-fixers to fix the permissions on the file system.
# --------------------------------------------------------------------------------
# Namespace
---
apiVersion: v1
kind: Namespace
metadata:
name: librenms
# --------------------------------------------------------------------------------
# Memcached
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: memcached
namespace: librenms
spec:
replicas: 1
selector:
matchLabels:
app: memcached
template:
metadata:
labels:
app: memcached
spec:
containers:
- name: memcached
image: memcached:alpine
imagePullPolicy: IfNotPresent
# --------------------------------------------------------------------------------
# Redis
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: librenms
spec:
replicas: 1
selector:
matchLabels:
app: redis
template:
metadata:
labels:
app: redis
spec:
containers:
- name: redis
image: redis:5.0-alpine
imagePullPolicy: IfNotPresent
env:
- name: TZ
value: America/Montreal
# --------------------------------------------------------------------------------
# RRDCached
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rrdcached
namespace: librenms
spec:
replicas: 1
selector:
matchLabels:
app: rrdcached
template:
metadata:
labels:
app: rrdcached
spec:
initContainers:
- name: nfs-fixer
image: alpine
securityContext:
runAsUser: 0
volumeMounts:
- name: rrdcached-main
mountPath: /nfs
command:
- sh
- -c
- (chmod 0775 /nfs; chown -R 1000:1000 /nfs)
containers:
- name: rrdcached
image: crazymax/rrdcached
imagePullPolicy: IfNotPresent
env:
- name: TZ
value: America/Montreal
- name: PUID
value: "1000"
- name: PGID
value: "1000"
- name: LOG_LEVEL
value: LOG_INFO
- name: WRITE_TIMEOUT
value: "1800"
- name: WRITE_JITTER
value: "1800"
- name: WRITE_THREADS
value: "4"
- name: FLUSH_DEAD_DATA_INTERVAL
value: "3600"
volumeMounts:
- name: rrdcached-db
mountPath: /data/db
- name: rrdcached-journal
mountPath: /data/journal
volumes:
- name: rrdcached-main
nfs:
server: 192.168.0.6
path: /volume1/k3s/librenms/rrdcached
- name: rrdcached-db
nfs:
server: 192.168.0.6
path: /volume1/k3s/librenms/rrdcached/db
- name: rrdcached-journal
nfs:
server: 192.168.0.6
path: /volume1/k3s/librenms/rrdcached/journal
# --------------------------------------------------------------------------------
# Config Map
---
apiVersion: v1
kind: ConfigMap
metadata:
name: librenms-config
namespace: librenms
data:
DB_HOST: "192.168.0.6"
DB_NAME: "librenms"
DB_PASSWORD: "librenms"
DB_TIMEOUT: "60"
DB_USER: "librenms"
LOG_IP_VAR: "remote_addr"
MEMCACHED_HOST: "memcached"
MEMCACHED_PORT: "11211"
PGID: "1000"
PUID: "1000"
REAL_IP_HEADER: "X-Forwarded-For"
RRDCACHED_HOST: "rrdcached"
RRDCACHED_PORT: "42217"
TZ: "America/Montreal"
# --------------------------------------------------------------------------------
# LibreNMS
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: librenms
namespace: librenms
spec:
replicas: 1
selector:
matchLabels:
app: librenms
template:
metadata:
labels:
app: librenms
spec:
initContainers:
- name: nfs-fixer
image: alpine
securityContext:
runAsUser: 0
volumeMounts:
- name: librenms-data
mountPath: /nfs
command:
- sh
- -c
- (chmod 0775 /nfs; chown -R 1000:1000 /nfs)
containers:
- name: librenms
image: librenms/librenms:1.65
imagePullPolicy: IfNotPresent
envFrom:
- configMapRef:
name: librenms-config
volumeMounts:
- name: librenms-data
mountPath: /data
volumes:
- name: librenms-data
nfs:
server: 192.168.0.6
path: /volume1/k3s/librenms/librenms-data
# --------------------------------------------------------------------------------
# LibreNMS Dispatcher
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: librenms-dispatcher
namespace: librenms
spec:
serviceName: librenms-dispatcher
replicas: 1
selector:
matchLabels:
app: librenms-dispatcher
template:
metadata:
labels:
app: librenms-dispatcher
spec:
initContainers:
- name: nfs-fixer
image: alpine
securityContext:
runAsUser: 0
volumeMounts:
- name: librenms-data
mountPath: /nfs
command:
- sh
- -c
- (chmod 0775 /nfs; chown -R 1000:1000 /nfs)
containers:
- name: librenms-dispatcher
image: librenms/librenms:1.65
imagePullPolicy: IfNotPresent
envFrom:
- configMapRef:
name: librenms-config
env:
- name: DISPATCHER_NODE_ID
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: REDIS_HOST
value: "redis"
- name: REDIS_PORT
value: "6379"
- name: REDIS_DB
value: "0"
- name: SIDECAR_DISPATCHER
value: "1"
volumeMounts:
- name: librenms-data
mountPath: /data
volumes:
- name: librenms-data
nfs:
server: 192.168.0.6
path: /volume1/k3s/librenms/librenms-data
# --------------------------------------------------------------------------------
# Service - memcached
---
apiVersion: v1
kind: Service
metadata:
name: memcached
namespace: librenms
spec:
ports:
- name: memcached
targetPort: 11211
port: 11211
selector:
app: memcached
# --------------------------------------------------------------------------------
# Service - redis
---
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: librenms
spec:
ports:
- name: redis
targetPort: 6379
port: 6379
selector:
app: redis
# --------------------------------------------------------------------------------
# Service - rrdcached
---
apiVersion: v1
kind: Service
metadata:
name: rrdcached
namespace: librenms
spec:
ports:
- name: rrdcached
targetPort: 42217
port: 42217
selector:
app: rrdcached
# --------------------------------------------------------------------------------
# Service - LibreNMS
---
apiVersion: v1
kind: Service
metadata:
name: librenms
namespace: librenms
annotations:
traefik.ingress.kubernetes.io/affinity: "true"
traefik.ingress.kubernetes.io/session-cookie-name: "sticky"
spec:
ports:
- name: http
targetPort: 8000
port: 80
selector:
app: librenms
# --------------------------------------------------------------------------------
# Ingress
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: librenms-ingress
namespace: librenms
annotations:
kubernetes.io/ingress.class: traefik
traefik.ingress.kubernetes.io/frontend-entry-points: http
spec:
rules:
- host: librenms.lab.local
http:
paths:
- path: /
backend:
serviceName: librenms
servicePort: http
I've been working on this for couple of days, maybe it's not 100% production-ready, but close to. Have a look! https://github.com/midokura/helm-charts-community/tree/main/librenms
I'm trying to get this running in a Swarm 5 nodes cluster using a nfs shared volume for /data without success.
The first time it discovers a device it generates correctly the rrd files, but it never refresh them on polling.
I'm deploying the next yml:
version: "3.8"
services: redis: image: redis:5.0-alpine environment: - "TZ=Europe/Madrid"
librenms_msmtpd: image: crazymax/msmtpd:latest environment: - "SMTP_HOST=smtp.gmail.com" - "SMTP_PORT=587" - "SMTP_TLS=on" - "SMTP_STARTTLS=on" - "SMTP_TLS_CHECKCERT=on" - "SMTP_AUTH=on" - "SMTP_USER=foo" - "SMTP_PASSWORD=bar" - "[email protected]"
librenms_rrdcached: image: crazymax/rrdcached environment: - "TZ=Europe/Madrid" - "PUID=1000" - "PGID=1000" - "LOG_LEVELLOG_INFO" - "WRITE_TIMEOUT=1800" - "WRITE_JITTER=1800" - "WRITE_THREADS=4" - "FLUSH_DEAD_DATA_INTERVAL=3600" volumes: - librenms:/data
librenms: image: librenms/librenms:latest hostname: librenms cap_add: - NET_ADMIN - NET_RAW ports: - target: 8000 published: 8001 protocol: tcp depends_on: - redis - librenms_msmtpd - librenms_rrdcached volumes: - librenms:/data environment: - "TZ=Europe/Madrid" - "PUID=1000" - "PGID=1000" - "DB_HOST=dbhost" - "DB_NAME=librenms_docker" - "DB_USER=librenms_docker" - "DB_PASSWORD=librenms_docker_PASSWORD" - "DB_TIMEOUT=60" - "MEMORY_LIMIT=256M" - "MAX_INPUT_VARS=1000" - "UPLOAD_MAX_SIZE=16M" - "OPCACHE_MEM_SIZE=128" - "REAL_IP_FROM=0.0.0.0/32" - "REAL_IP_HEADER=X-Forwarded-For" - "LOG_IP_VAR=remote_addr" - "CACHE_DRIVER=redis" - "SESSION_DRIVER=redis" - "REDIS_HOST=redis" - "LIBRENMS_SNMP_COMMUNITY=public" - "LIBRENMS_WEATHERMAP=false" - "LIBRENMS_WEATHERMAP_SCHEDULE=*/5 * * * *" - "RRDCACHED_HOST=librenms_rrdcached"
librenms_dispatcher: image: librenms/librenms:latest hostname: librenms-dispatcher cap_add: - NET_ADMIN - NET_RAW depends_on: - librenms - redis volumes: - librenms:/data environment: - "MEMORY_LIMIT=256M" - "MAX_INPUT_VARS=1000" - "UPLOAD_MAX_SIZE=16M" - "OPCACHE_MEM_SIZE=128" - "REAL_IP_FROM=0.0.0.0/32" - "REAL_IP_HEADER=X-Forwarded-For" - "LOG_IP_VAR=remote_addr" - "CACHE_DRIVER=redis" - "SESSION_DRIVER=redis" - "REDIS_HOST=redis" - "LIBRENMS_SNMP_COMMUNITY=public" - "LIBRENMS_WEATHERMAP=false" - "LIBRENMS_WEATHERMAP_SCHEDULE=*/5 * * * *" - "TZ=Europe/Madrid" - "PUID=1000" - "PGID=1000" - "DB_HOST=dbhost" - "DB_NAME=librenms_docker" - "DB_USER=librenms_docker" - "DB_PASSWORD=librenms_docker_PASSWORD" - "DB_TIMEOUT=60" - "DISPATCHER_NODE_ID=dispatcher1" - "SIDECAR_DISPATCHER=1"
librenms_syslogng: image: librenms/librenms:latest hostname: librenms-syslogng cap_add: - NET_ADMIN - NET_RAW depends_on: - librenms - redis ports: - target: 514 published: 514 protocol: tcp - target: 514 published: 514 protocol: udp volumes: - librenms:/data environment: - "MEMORY_LIMIT=256M" - "MAX_INPUT_VARS=1000" - "UPLOAD_MAX_SIZE=16M" - "OPCACHE_MEM_SIZE=128" - "REAL_IP_FROM=0.0.0.0/32" - "REAL_IP_HEADER=X-Forwarded-For" - "LOG_IP_VAR=remote_addr" - "CACHE_DRIVER=redis" - "SESSION_DRIVER=redis" - "REDIS_HOST=redis" - "LIBRENMS_SNMP_COMMUNITY=public" - "LIBRENMS_WEATHERMAP=false" - "LIBRENMS_WEATHERMAP_SCHEDULE=*/5 * * * *" - "TZ=Europe/Madrid" - "PUID=1000" - "PGID=1000" - "DB_HOST=dbhost" - "DB_NAME=librenms_docker" - "DB_USER=librenms_docker" - "DB_PASSWORD=librenms_docker_PASSWORD" - "DB_TIMEOUT=60" - "SIDECAR_SYSLOGNG=1"
librenms_snmptrapd: image: librenms/librenms:latest hostname: librenms-snmptrapd cap_add: - NET_ADMIN - NET_RAW depends_on: - librenms - redis ports: - target: 162 published: 162 protocol: tcp - target: 162 published: 162 protocol: udp volumes: - librenms:/data environment: - "MEMORY_LIMIT=256M" - "MAX_INPUT_VARS=1000" - "UPLOAD_MAX_SIZE=16M" - "OPCACHE_MEM_SIZE=128" - "REAL_IP_FROM=0.0.0.0/32" - "REAL_IP_HEADER=X-Forwarded-For" - "LOG_IP_VAR=remote_addr" - "CACHE_DRIVER=redis" - "SESSION_DRIVER=redis" - "REDIS_HOST=redis" - "LIBRENMS_SNMP_COMMUNITY=public" - "LIBRENMS_WEATHERMAP=false" - "LIBRENMS_WEATHERMAP_SCHEDULE=*/5 * * * *" - "TZ=Europe/Madrid" - "PUID=1000" - "PGID=1000" - "DB_HOST=dbhost" - "DB_NAME=librenms_docker" - "DB_USER=librenms_docker" - "DB_PASSWORD=librenms_docker_PASSWORD" - "DB_TIMEOUT=60" - "SIDECAR_SNMPTRAPD=1"
volumes: librenms: driver: local driver_opts: type: nfs o: addr=nfsserver,rw,noatime,rsize=8192,wsize=8192,tcp,timeo=14 device: :/mnt/docker/librenms
