public/loki-system-logs.sh

#!/bin/bash
# This script will setup Promtail to push to https://int.loki.grid.tf
set -euo pipefail

### Install Promtail
export DEBIAN_FRONTEND=noninteractive
mkdir -p /etc/apt/keyrings/
wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor > /etc/apt/keyrings/grafana.gpg
echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list
apt-get update -qq && apt-get install -y promtail
usermod -a -G adm promtail

### Ensure persistent positions directory exists (apt-installed promtail user has primary group nogroup)
install -d -o promtail -g nogroup -m 0755 /var/lib/promtail

### Set config
cat << EOF > /etc/promtail/config.yml
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /var/lib/promtail/positions.yaml

clients:
- url: https://int.loki.grid.tf/loki/api/v1/push

scrape_configs:
- job_name: system
  static_configs:
  - targets:
      - localhost
    labels:
      host: \${HOSTNAME}
      job: varlogs
      __path__: /var/log/*log
- job_name: journal
  journal:
    json: false
    max_age: 24h
    path: /var/log/journal
    labels:
      host: \${HOSTNAME}
      job: systemd-journal
  relabel_configs:
    - source_labels: ['__journal__systemd_unit']
      target_label: 'unit'
EOF

### Edit service
cat << EOF > /etc/systemd/system/promtail.service
[Unit]
Description=Promtail service
After=network.target

[Service]
Type=simple
User=promtail
Environment="HOSTNAME=%H"
ExecStart=/usr/bin/promtail -config.file /etc/promtail/config.yml -config.expand-env=true
# Give a reasonable amount of time for promtail to start up/shut down
TimeoutSec = 60
Restart = on-failure
RestartSec = 2

[Install]
WantedBy=multi-user.target
EOF

### Reload, enable, restart
systemctl daemon-reload
systemctl enable promtail
systemctl restart promtail
systemctl status promtail --no-pager