Prometheus + Grafana 서버 모니터링 완전 가이드 — Docker 구축·Node Exporter·AlertManager 슬랙 알림까지

docker-compose.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:v2.52.0
    container_name: prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'
    ports:
      - '9090:9090'
    restart: unless-stopped

  grafana:
    image: grafana/grafana:11.0.0
    container_name: grafana
    volumes:
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=강력한_비밀번호로_교체하세요
      - GF_USERS_ALLOW_SIGN_UP=false
    ports:
      - '3000:3000'
    depends_on:
      - prometheus
    restart: unless-stopped

  node-exporter:
    image: prom/node-exporter:v1.8.1
    container_name: node-exporter
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.rootfs=/rootfs'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($|/)'
    ports:
      - '9100:9100'
    restart: unless-stopped

volumes:
  prometheus_data:
  grafana_data:

prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

PromQL — 자주 쓰는 핵심 쿼리
# CPU 사용률 (%) — idle 제외
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 메모리 사용률 (%)
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# 디스크 사용률 (rootfs 기준)
(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100

# 네트워크 수신 속도 (eth0, bytes/s)
rate(node_network_receive_bytes_total{device="eth0"}[5m])

# 열린 파일 디스크립터 수
node_filefd_allocated

# 프로세스별 CPU (top 5)
topk(5, rate(node_cpu_seconds_total{mode!="idle"}[5m]))

패키지 설치
npm install prom-client

metrics.js — Express 애플리케이션에 메트릭 추가
const express = require('express');
const promClient = require('prom-client');

const app = express();

// 기본 메트릭 수집 활성화 (Node.js 프로세스 메트릭)
promClient.collectDefaultMetrics({ timeout: 5000 });

// HTTP 요청 수 카운터
const httpRequestCounter = new promClient.Counter({
  name: 'http_requests_total',
  help: '총 HTTP 요청 수',
  labelNames: ['method', 'route', 'status_code']
});

// 응답 시간 히스토그램
const httpRequestDuration = new promClient.Histogram({
  name: 'http_request_duration_seconds',
  help: 'HTTP 요청 처리 시간 (초)',
  labelNames: ['method', 'route'],
  buckets: [0.01, 0.05, 0.1, 0.3, 0.5, 1, 2, 5]
});

// 미들웨어 — 모든 요청에 메트릭 기록
app.use((req, res, next) => {
  const start = Date.now();
  res.on('finish', () => {
    const duration = (Date.now() - start) / 1000;
    httpRequestCounter.labels(req.method, req.path, String(res.statusCode)).inc();
    httpRequestDuration.labels(req.method, req.path).observe(duration);
  });
  next();
});

// /metrics 엔드포인트 노출
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', promClient.register.contentType);
  res.end(await promClient.register.metrics());
});

app.listen(3001, () => console.log('서버 시작: 3001'));

prometheus.yml — 앱 메트릭 추가
scrape_configs:
  # ... 기존 설정 유지 ...

  - job_name: 'my-node-app'
    static_configs:
      - targets: ['host.docker.internal:3001']
    # 도커 컨테이너 안에서 실행 중이라면:
    # targets: ['app-container-name:3001']

그라파나 패널용 PromQL 예시
# API 에러율 (5xx 비율 %)
rate(http_requests_total{status_code=~"5.."}[5m])
/
rate(http_requests_total[5m]) * 100

# 99th percentile 응답 시간 (경로별)
histogram_quantile(0.99,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, route)
)

# 초당 요청 수 (RPS, 경로별)
sum(rate(http_requests_total[1m])) by (route)

# 현재 활성 연결 수
node_sockstat_TCP_inuse

docker-compose.yml — alertmanager 서비스 추가
  alertmanager:
    image: prom/alertmanager:v0.27.0
    container_name: alertmanager
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
    ports:
      - '9093:9093'
    restart: unless-stopped

alertmanager.yml — 슬랙 연동 설정
global:
  resolve_timeout: 5m

route:
  receiver: 'slack-notifications'
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

receivers:
  - name: 'slack-notifications'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
        channel: '#alerts'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        send_resolved: true

prometheus.yml — 알림 설정 추가
global:
  scrape_interval: 15s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

rule_files:
  - 'alerts.yml'

scrape_configs:
  # ... 기존 스크레이프 설정 ...

alerts.yml — 알림 규칙 예시
groups:
  - name: server-alerts
    rules:
      - alert: HighCPUUsage
        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'CPU 사용률 85% 초과'

      - alert: HighMemoryUsage
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: '메모리 사용률 90% 초과 — 즉시 확인 필요'

      - alert: DiskSpaceLow
        expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: '루트 디스크 사용률 85% 초과'

      - alert: HighErrorRate
        expr: rate(http_requests_total{status_code=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.05
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: 'API 에러율 5% 초과 — 서비스 이상 확인 필요'