LGTM 구성해보기(테스트 및 작성중)

# 기본 흐름
 ┌────────────┐     ┌────────────┐     ┌────────────┐
 │  Promtail  │────▶│    Loki    │────▶│  Grafana   │
 └────────────┘     └────────────┘     └────────────┘
 ┌────────────┐     ┌────────────┐     ┌────────────┐
 │ Prometheus │────▶│   Mimir    │────▶│  Grafana   │
 └────────────┘     └────────────┘     └────────────┘
 ┌───────────────┐     ┌────────────┐     ┌────────────┐
 │ OpenTelemetry │────▶│   Tempo    │────▶│  Grafana   │
 └───────────────┘     └────────────┘     └────────────┘

 ┌────────────┐           ┌────────────┐     ┌────────────┐
 │  Promtail  │── Logs ──▶│    Loki    │────▶│  Grafana   │
 └────────────┘           └────────────┘     └────────────┘

--- # Daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: promtail-daemonset
spec:
  selector:
    matchLabels:
      name: promtail
  template:
    metadata:
      labels:
        name: promtail
    spec:
      serviceAccount: promtail-serviceaccount
      containers:
      - name: promtail-container
        image: grafana/promtail:3.5.8
        args:
        - -config.file=/etc/promtail/promtail.yaml
        env: 
        - name: 'HOSTNAME' # needed when using kubernetes_sd_configs
        # "Kubernetes Service Discovery Configs"
        # "쿠버네티스 서비스 디스커버리 설정"의 줄임말입니다.
          valueFrom:
            fieldRef:
              fieldPath: 'spec.nodeName' # Pod가 스케줄된 실제 노드 이름을 Kubernetes API에서 가져옴
        volumeMounts:
        - name: logs
          mountPath: /var/log
        - name: promtail-config
          mountPath: /etc/promtail
        - mountPath: /var/lib/docker/containers
          name: varlibdockercontainers
          readOnly: true
      volumes:
      - name: logs
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: promtail-config
        configMap:
          name: promtail-config
--- # configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: promtail-config
data:
  promtail.yaml: |
    server:
      http_listen_port: 9080
      grpc_listen_port: 0

    clients:
    #- url: https://{YOUR_LOKI_ENDPOINT}/loki/api/v1/push
    - url: http://loki-gateway.monitoring.svc.cluster.local/loki/api/v1/push
    # auth_enabled=true 일때 아래 설정
      headers:
        X-Scope-OrgID: tk8s
    positions:
      filename: /tmp/positions.yaml
    target_config:
      sync_period: 10s
    scrape_configs:
    - job_name: pod-logs
      kubernetes_sd_configs:
        - role: pod
      pipeline_stages:
        - docker: {}
      relabel_configs:
        - source_labels:
            - __meta_kubernetes_pod_node_name
          target_label: __host__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - action: replace
          replacement: $1
          separator: /
          source_labels:
            - __meta_kubernetes_namespace
            - __meta_kubernetes_pod_name
          target_label: job
        - action: replace
          source_labels:
            - __meta_kubernetes_namespace
          target_label: namespace
        - action: replace
          source_labels:
            - __meta_kubernetes_pod_name
          target_label: pod
        - action: replace
          source_labels:
            - __meta_kubernetes_pod_container_name
          target_label: container
        - replacement: /var/log/pods/*$1/*.log
          separator: /
          source_labels:
            - __meta_kubernetes_pod_uid
            - __meta_kubernetes_pod_container_name
          target_label: __path__

--- # Clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: promtail-clusterrole
rules:
  - apiGroups: [""]
    resources:
    - nodes
    - services
    - pods
    verbs:
    - get
    - watch
    - list

--- # ServiceAccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: promtail-serviceaccount

--- # Rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: promtail-clusterrolebinding
subjects:
    - kind: ServiceAccount
      name: promtail-serviceaccount
      namespace: default
roleRef:
    kind: ClusterRole
    name: promtail-clusterrole
    apiGroup: rbac.authorization.k8s.io

helm repo add grafana https://grafana.github.io/helm-charts

helm repo update grafana

helm search repo grafana

helm pull grafana/loki --untar

loki:
  schemaConfig:
    configs:
      - from: "2025-11-08"
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: loki_index_
          period: 24h
  ingester:
    chunk_encoding: snappy
  querier:
    # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing
    max_concurrent: 4
  pattern_ingester:
    enabled: true
  limits_config:
    allow_structured_metadata: true
    volume_enabled: true

  storage:
    type: s3
    bucketNames:
      chunks: loki-chunks
      ruler: loki-ruler
      admin: loki-admin
    s3:
      endpoint: minio.default.svc.cluster.local:9000
      region: us-east-1
      accessKeyId: minioadmin
      secretAccessKey: minioadmin
      insecure: true

deploymentMode: SimpleScalable

backend:
  replicas: 2
read:
  replicas: 2
write:
  replicas: 3 # To ensure data durability with replication

# Enable minio for storage
minio:
  enabled: true

gateway:
  service:
    type: LoadBalancer

kubectl create ns monitoring

helm install loki grafana/loki -n monitoring -f values.yaml

loki:
  schemaConfig:
    configs:
      - from: "2025-11-08"
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: loki_index_
          period: 24h
  ingester:
    chunk_encoding: snappy
  querier:
    # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing
    max_concurrent: 4
  pattern_ingester:
    enabled: true
  limits_config:
    allow_structured_metadata: true
    volume_enabled: true

  storage:
    type: s3
    bucketNames:
      chunks: loki-chunks
      ruler: loki-ruler
      admin: loki-admin
    s3:
      endpoint: minio.default.svc.cluster.local:9000
      region: us-east-1
      accessKeyId: minioadmin
      secretAccessKey: minioadmin
      insecure: true

deploymentMode: SimpleScalable

backend:
  replicas: 2
read:
  replicas: 2
write:
  #replicas: 3 # To ensure data durability with replication
  replicas: 2 # To ensure data durability with replication

# 리소스 조절
chunksCache:
  enabled: true
  replicas: 1
  persistence:
    enabled: false
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      cpu: 500m
      memory: 256Mi

# Enable minio for storage
minio:
  enabled: true

gateway:
  service:
    type: LoadBalancer

helm upgrade loki grafana/loki -n monitoring -f values.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: loki-minio-console
  namespace: monitoring
spec:
  ingressClassName: nginx
  rules:
    - host: minio.tk8s.test       # 원하는 도메인 이름으로 교체
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: loki-minio-console
                port:
                  number: 9001
  tls:
  - hosts:
    - minio.tk8s.test
    secretName: ssl-common
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: loki-ingress
  namespace: monitoring
spec:
  ingressClassName: nginx
  rules:
    - host: loki.tk8s.test
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: loki-gateway
                port:
                  number: 80
  tls:
  - hosts:
    - loki.tk8s.test
    secretName: ssl-common

helm list -n monitoring
helm uninstall loki -n monitoring

kubectl delete pvc -n monitoring -l app.kubernetes.io/name=loki

kubectl get pv | grep monitoring | awk '{print $1}' | xargs kubectl delete pv

Prometheus → Thanos → Cortex → Mimir