From 8232ff497d3eca350e30954d7852e9e4afacf4e4 Mon Sep 17 00:00:00 2001 From: Jared Vititoe Date: Sat, 14 Feb 2026 12:00:07 -0500 Subject: [PATCH] Add Promtail log agent deployment alongside node_exporter Installs Promtail (Grafana Loki agent) to ship host logs to the central Loki instance at 10.10.10.69:3100. Scrapes syslog, auth, kernel, daemon, pveproxy, pvedaemon, pve-tasks, ceph, and systemd journal logs with per-host labeling. Co-Authored-By: Claude Opus 4.6 --- freshStart.sh | 158 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git a/freshStart.sh b/freshStart.sh index 2c27149..793fa19 100644 --- a/freshStart.sh +++ b/freshStart.sh @@ -22,14 +22,19 @@ fi cleanup() { echo "Cleaning up on error..." systemctl stop node_exporter 2>/dev/null || true + systemctl stop promtail 2>/dev/null || true systemctl stop hwmon.timer 2>/dev/null || true systemctl disable node_exporter 2>/dev/null || true + systemctl disable promtail 2>/dev/null || true systemctl disable hwmon.timer 2>/dev/null || true rm -f /etc/systemd/system/node_exporter.service + rm -f /etc/systemd/system/promtail.service rm -f /etc/systemd/system/hwmon.service rm -f /etc/systemd/system/hwmon.timer rm -f /usr/local/bin/node_exporter + rm -f /usr/local/bin/promtail rm -rf node_exporter-*.linux-amd64.tar.gz node_exporter-*.linux-amd64 + rm -rf /etc/promtail /var/lib/promtail userdel node_exporter 2>/dev/null || true systemctl daemon-reload echo "Cleanup completed." @@ -43,7 +48,7 @@ echo "Installing required packages..." apt-get update # Common packages for all platforms -COMMON_PKGS="python3-pip smartmontools python3-psutil python3-requests lm-sensors fastfetch rsync jq sysstat" +COMMON_PKGS="python3-pip smartmontools python3-psutil python3-requests lm-sensors fastfetch rsync jq sysstat unzip" if [[ "$PLATFORM" == "pve" ]]; then echo "Installing PVE-specific packages..." @@ -122,6 +127,153 @@ if ! systemctl is-active --quiet node_exporter; then exit 1 fi +# Install Promtail (Loki log agent) +echo "Installing Promtail log agent..." +PROMTAIL_VERSION="3.4.2" +LOKI_URL="http://10.10.10.69:3100" + +echo "Downloading Promtail..." +if ! wget --timeout=30 --tries=3 "https://github.com/grafana/loki/releases/download/v${PROMTAIL_VERSION}/promtail-linux-amd64.zip" -O /tmp/promtail.zip; then + echo "ERROR: Failed to download Promtail" + exit 1 +fi + +if ! command -v unzip &>/dev/null; then + apt-get install -y unzip +fi + +if ! unzip -o /tmp/promtail.zip -d /tmp/; then + echo "ERROR: Failed to extract Promtail" + exit 1 +fi + +mv /tmp/promtail-linux-amd64 /usr/local/bin/promtail +chmod +x /usr/local/bin/promtail +rm -f /tmp/promtail.zip + +# Create Promtail directories +mkdir -p /etc/promtail /var/lib/promtail + +# Get hostname for labeling +PROMTAIL_HOST=$(hostname) + +# Create Promtail config +cat > /etc/promtail/config.yml << PROMTAILEOF +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /var/lib/promtail/positions.yaml + +clients: + - url: ${LOKI_URL}/loki/api/v1/push + +scrape_configs: + - job_name: syslog + static_configs: + - targets: + - localhost + labels: + job: syslog + host: ${PROMTAIL_HOST} + __path__: /var/log/syslog + - job_name: auth + static_configs: + - targets: + - localhost + labels: + job: auth + host: ${PROMTAIL_HOST} + __path__: /var/log/auth.log + - job_name: kern + static_configs: + - targets: + - localhost + labels: + job: kernel + host: ${PROMTAIL_HOST} + __path__: /var/log/kern.log + - job_name: daemon + static_configs: + - targets: + - localhost + labels: + job: daemon + host: ${PROMTAIL_HOST} + __path__: /var/log/daemon.log + - job_name: pveproxy + static_configs: + - targets: + - localhost + labels: + job: pveproxy + host: ${PROMTAIL_HOST} + __path__: /var/log/pveproxy/access.log + - job_name: pvedaemon + static_configs: + - targets: + - localhost + labels: + job: pvedaemon + host: ${PROMTAIL_HOST} + __path__: /var/log/pvedaemon.log + - job_name: pve-tasks + static_configs: + - targets: + - localhost + labels: + job: pve-tasks + host: ${PROMTAIL_HOST} + __path__: /var/log/pve/tasks/active + - job_name: ceph + static_configs: + - targets: + - localhost + labels: + job: ceph + host: ${PROMTAIL_HOST} + __path__: /var/log/ceph/*.log + - job_name: journal + journal: + max_age: 12h + labels: + job: journal + host: ${PROMTAIL_HOST} + relabel_configs: + - source_labels: ['__journal__systemd_unit'] + target_label: 'unit' + - source_labels: ['__journal_priority_keyword'] + target_label: 'priority' +PROMTAILEOF + +# Create Promtail systemd service +cat > /etc/systemd/system/promtail.service << 'EOL' +[Unit] +Description=Promtail Log Agent +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/promtail -config.file=/etc/promtail/config.yml +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOL + +systemctl daemon-reload +systemctl enable promtail +systemctl start promtail + +if ! systemctl is-active --quiet promtail; then + echo "WARNING: Promtail failed to start" + systemctl status promtail --no-pager || true +else + echo "✓ Promtail is running and shipping logs to Loki" +fi + # Install hwmon daemon echo "Installing hwmon daemon..." @@ -263,6 +415,7 @@ fi # Final verification echo "Verifying installation..." echo "Node Exporter status: $(systemctl is-active node_exporter)" +echo "Promtail status: $(systemctl is-active promtail)" echo "hwmon timer status: $(systemctl is-active hwmon.timer)" echo "Node Exporter port check:" if ss -tlnp | grep :9100; then @@ -286,13 +439,16 @@ echo "Platform: ${PLATFORM^^}" echo "" echo "Services installed:" echo " - Node Exporter: http://$(hostname -I | awk '{print $1}'):9100/metrics" +echo " - Promtail: Shipping logs to Loki at ${LOKI_URL:-http://10.10.10.69:3100}" echo " - hwmon daemon: Monitoring system health hourly" echo "" echo "Configuration files:" +echo " - Promtail config: /etc/promtail/config.yml" echo " - hwmon config: /etc/hwmonDaemon/.env" echo "" echo "Log locations:" echo " - Node Exporter: journalctl -u node_exporter" +echo " - Promtail: journalctl -u promtail" echo " - hwmon: journalctl -u hwmon.service" echo " - hwmon logs: /var/log/hwmonDaemon/" echo ""