LGTM 구성해보기(테스트 및 작성중)

# 기본 흐름
 ┌────────────┐     ┌────────────┐     ┌────────────┐
 │  Promtail  │────▶│    Loki    │────▶│  Grafana   │
 └────────────┘     └────────────┘     └────────────┘
 ┌────────────┐     ┌────────────┐     ┌────────────┐
 │ Prometheus │────▶│   Mimir    │────▶│  Grafana   │
 └────────────┘     └────────────┘     └────────────┘
 ┌───────────────┐     ┌────────────┐     ┌────────────┐
 │ OpenTelemetry │────▶│   Tempo    │────▶│  Grafana   │
 └───────────────┘     └────────────┘     └────────────┘

 ┌────────────┐           ┌────────────┐     ┌────────────┐
 │  Promtail  │── Logs ──▶│    Loki    │────▶│  Grafana   │
 └────────────┘           └────────────┘     └────────────┘

--- # Daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: promtail-daemonset
spec:
  selector:
    matchLabels:
      name: promtail
  template:
    metadata:
      labels:
        name: promtail
    spec:
      serviceAccount: promtail-serviceaccount
      containers:
      - name: promtail-container
        image: grafana/promtail:3.5.8
        args:
        - -config.file=/etc/promtail/promtail.yaml
        env: 
        - name: 'HOSTNAME' # needed when using kubernetes_sd_configs
        # "Kubernetes Service Discovery Configs"
        # "쿠버네티스 서비스 디스커버리 설정"의 줄임말입니다.
          valueFrom:
            fieldRef:
              fieldPath: 'spec.nodeName' # Pod가 스케줄된 실제 노드 이름을 Kubernetes API에서 가져옴
        volumeMounts:
        - name: logs
          mountPath: /var/log
        - name: promtail-config
          mountPath: /etc/promtail
        - mountPath: /var/lib/docker/containers
          name: varlibdockercontainers
          readOnly: true
      volumes:
      - name: logs
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: promtail-config
        configMap:
          name: promtail-config
--- # configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: promtail-config
data:
  promtail.yaml: |
    server:
      http_listen_port: 9080
      grpc_listen_port: 0

    clients:
    #- url: https://{YOUR_LOKI_ENDPOINT}/loki/api/v1/push
    - url: http://loki-gateway.monitoring.svc.cluster.local/loki/api/v1/push
    # auth_enabled=true 일때 아래 설정
      headers:
        X-Scope-OrgID: tk8s
    positions:
      filename: /tmp/positions.yaml
    target_config:
      sync_period: 10s
    scrape_configs:
    - job_name: pod-logs
      kubernetes_sd_configs:
        - role: pod
      pipeline_stages:
        - docker: {}
      relabel_configs:
        - source_labels:
            - __meta_kubernetes_pod_node_name
          target_label: __host__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - action: replace
          replacement: $1
          separator: /
          source_labels:
            - __meta_kubernetes_namespace
            - __meta_kubernetes_pod_name
          target_label: job
        - action: replace
          source_labels:
            - __meta_kubernetes_namespace
          target_label: namespace
        - action: replace
          source_labels:
            - __meta_kubernetes_pod_name
          target_label: pod
        - action: replace
          source_labels:
            - __meta_kubernetes_pod_container_name
          target_label: container
        - replacement: /var/log/pods/*$1/*.log
          separator: /
          source_labels:
            - __meta_kubernetes_pod_uid
            - __meta_kubernetes_pod_container_name
          target_label: __path__

--- # Clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: promtail-clusterrole
rules:
  - apiGroups: [""]
    resources:
    - nodes
    - services
    - pods
    verbs:
    - get
    - watch
    - list

--- # ServiceAccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: promtail-serviceaccount

--- # Rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: promtail-clusterrolebinding
subjects:
    - kind: ServiceAccount
      name: promtail-serviceaccount
      namespace: default
roleRef:
    kind: ClusterRole
    name: promtail-clusterrole
    apiGroup: rbac.authorization.k8s.io

helm repo add grafana https://grafana.github.io/helm-charts

helm repo update grafana

helm search repo grafana

helm pull grafana/loki --untar

loki:
  schemaConfig:
    configs:
      - from: "2025-11-08"
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: loki_index_
          period: 24h
  ingester:
    chunk_encoding: snappy
  querier:
    # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing
    max_concurrent: 4
  pattern_ingester:
    enabled: true
  limits_config:
    allow_structured_metadata: true
    volume_enabled: true

  storage:
    type: s3
    bucketNames:
      chunks: loki-chunks
      ruler: loki-ruler
      admin: loki-admin
    s3:
      endpoint: minio.default.svc.cluster.local:9000
      region: us-east-1
      accessKeyId: minioadmin
      secretAccessKey: minioadmin
      insecure: true

deploymentMode: SimpleScalable

backend:
  replicas: 2
read:
  replicas: 2
write:
  replicas: 3 # To ensure data durability with replication

# Enable minio for storage
minio:
  enabled: true

gateway:
  service:
    type: LoadBalancer

kubectl create ns monitoring

helm install loki grafana/loki -n monitoring -f values.yaml

loki:
  schemaConfig:
    configs:
      - from: "2025-11-08"
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: loki_index_
          period: 24h
  ingester:
    chunk_encoding: snappy
  querier:
    # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing
    max_concurrent: 4
  pattern_ingester:
    enabled: true
  limits_config:
    allow_structured_metadata: true
    volume_enabled: true

  storage:
    type: s3
    bucketNames:
      chunks: loki-chunks
      ruler: loki-ruler
      admin: loki-admin
    s3:
      endpoint: minio.default.svc.cluster.local:9000
      region: us-east-1
      accessKeyId: minioadmin
      secretAccessKey: minioadmin
      insecure: true

deploymentMode: SimpleScalable

backend:
  replicas: 2
read:
  replicas: 2
write:
  #replicas: 3 # To ensure data durability with replication
  replicas: 2 # To ensure data durability with replication

# 리소스 조절
chunksCache:
  enabled: true
  replicas: 1
  persistence:
    enabled: false
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      cpu: 500m
      memory: 256Mi

# Enable minio for storage
minio:
  enabled: true

gateway:
  service:
    type: LoadBalancer

helm upgrade loki grafana/loki -n monitoring -f values.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: loki-minio-console
  namespace: monitoring
spec:
  ingressClassName: nginx
  rules:
    - host: minio-loki.tk8s.test       # 원하는 도메인 이름으로 교체
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: loki-minio-console
                port:
                  number: 9001
  tls:
  - hosts:
    - minio-loki.tk8s.test
    secretName: ssl-certs
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: loki-ingress
  namespace: monitoring
spec:
  ingressClassName: nginx
  rules:
    - host: loki.tk8s.test
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: loki-gateway
                port:
                  number: 80
  tls:
  - hosts:
    - loki.tk8s.test
    secretName: ssl-certs

helm list -n monitoring
helm uninstall loki -n monitoring

kubectl delete pvc -n monitoring -l app.kubernetes.io/name=loki

kubectl get pv | grep monitoring | awk '{print $1}' | xargs kubectl delete pv

kubectl create ns tempo

helm pull grafana/tempo-distributed --untar

Prometheus → Thanos → Cortex → Mimir

[ Prometheus ]  →  [ Distributor ]  →  [ Ingester ]  →  [ Compactor ]  →  [ Object Storage (S3, MinIO) ]

kubectl create namespace mimir

# repo등록은 위에서 진행했으므로 생략
# helm repo add grafana https://grafana.github.io/helm-charts
# helm repo update grafana
helm search repo grafana

helm pull grafana/mimir-distributed --untar

alertmanager:
  persistentVolume:
    enabled: true
  replicas: 2
  resources:
    limits:
      memory: 1.4Gi
    requests:
      cpu: 1
      memory: 1Gi
  statefulSet:
    enabled: true

compactor:
  persistentVolume:
    size: 20Gi
  resources:
    limits:
      memory: 2.1Gi
    requests:
      cpu: 1
      memory: 1.5Gi

distributor:
  replicas: 2
  resources:
    limits:
      memory: 5.7Gi
    requests:
      cpu: 2
      memory: 4Gi

ingester:
  persistentVolume:
    size: 50Gi
  replicas: 3
  resources:
    limits:
      memory: 12Gi
    requests:
      cpu: 3.5
      memory: 8Gi
  topologySpreadConstraints: {}
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchExpressions:
              - key: app.kubernetes.io/component
                operator: In
                values:
                  - ingester
          topologyKey: "kubernetes.io/hostname"

  zoneAwareReplication:
    topologyKey: "kubernetes.io/hostname"

admin-cache:
  enabled: true
  replicas: 3

chunks-cache:
  enabled: true
  replicas: 3

index-cache:
  enabled: true
  replicas: 3

metadata-cache:
  enabled: true
  replicas: 3

results-cache:
  enabled: true
  replicas: 3

minio:
  enabled: true
  mode: standalone
  rootUser: grafana-mimir
  buckets:
    - name: mimir-tsdb
      policy: none
      purge: false
    - name: mimir-ruler
      policy: none
      purge: false
  persistence:
    size: 5Gi
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
  rootPassword: supersecret
  # Changed the mc config path to '/tmp' from '/etc' as '/etc' is only writable by root and OpenShift will not permit this.
  configPathmc: "/tmp/minio/mc/"

overrides_exporter:
  replicas: 1
  resources:
    limits:
      memory: 128Mi
    requests:
      cpu: 100m
      memory: 128Mi

querier:
  replicas: 1
  resources:
    limits:
      memory: 5.6Gi
    requests:
      cpu: 2
      memory: 4Gi

query_frontend:
  replicas: 1
  resources:
    limits:
      memory: 2.8Gi
    requests:
      cpu: 2
      memory: 2Gi

ruler:
  replicas: 1
  resources:
    limits:
      memory: 2.8Gi
    requests:
      cpu: 1
      memory: 2Gi

store_gateway:
  persistentVolume:
    size: 10Gi
  replicas: 3
  resources:
    limits:
      memory: 2.1Gi
    requests:
      cpu: 1
      memory: 1.5Gi
  topologySpreadConstraints: {}
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchExpressions:
              - key: app.kubernetes.io/component
                operator: In
                values:
                  - store-gateway
          topologyKey: "kubernetes.io/hostname"
  zoneAwareReplication:
    topologyKey: "kubernetes.io/hostname"

gateway:
  replicas: 1
  resources:
    limits:
      memory: 731Mi
    requests:
      cpu: 1
      memory: 512Mi

# ---------------------------------------------------------------------------
# Grafana Mimir distributed - lightweight deployment (for small clusters)
# ---------------------------------------------------------------------------

alertmanager:
  persistentVolume:
    enabled: true
  replicas: 1
  resources:
    requests:
      cpu: 100m
      memory: 256Mi
    limits:
      cpu: 200m
      memory: 512Mi
  statefulSet:
    enabled: true

compactor:
  persistentVolume:
    size: 20Gi
  resources:
    requests:
      cpu: 200m
      memory: 512Mi
    limits:
      cpu: 300m
      memory: 1Gi

distributor:
  replicas: 2
  resources:
    requests:
      cpu: 200m
      memory: 512Mi
    limits:
      cpu: 300m
      memory: 1Gi

ingester:
  persistentVolume:
    size: 50Gi
  replicas: 3
  resources:
    requests:
      cpu: 300m
      memory: 1Gi
    limits:
      cpu: 500m
      memory: 2Gi
  topologySpreadConstraints: {}
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchExpressions:
              - key: app.kubernetes.io/component
                operator: In
                values:
                  - ingester
          topologyKey: "kubernetes.io/hostname"

  zoneAwareReplication:
    topologyKey: "kubernetes.io/hostname"

querier:
  replicas: 1
  resources:
    requests:
      cpu: 200m
      memory: 512Mi
    limits:
      cpu: 400m
      memory: 1Gi

query_frontend:
  replicas: 1
  resources:
    requests:
      cpu: 100m
      memory: 256Mi
    limits:
      cpu: 200m
      memory: 512Mi

ruler:
  replicas: 1
  resources:
    requests:
      cpu: 200m
      memory: 512Mi
    limits:
      cpu: 300m
      memory: 1Gi

store_gateway:
  persistentVolume:
    size: 10Gi
  replicas: 3
  resources:
    requests:
      cpu: 200m
      memory: 512Mi
    limits:
      cpu: 400m
      memory: 1Gi
  topologySpreadConstraints: {}
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchExpressions:
              - key: app.kubernetes.io/component
                operator: In
                values:
                  - store-gateway
          topologyKey: "kubernetes.io/hostname"
  zoneAwareReplication:
    topologyKey: "kubernetes.io/hostname"

gateway:
  replicas: 1
  resources:
    limits:
      memory: 1024Mi
    requests:
      cpu: 1
      memory: 512Mi

# ---------------------------------------------------------------------------
# Cache components (memcached + exporter)
# ---------------------------------------------------------------------------

admin-cache:
  enabled: true
  replicas: 2
  resources:
    requests:
      cpu: "50m"
      memory: "128Mi"
    limits:
      cpu: "200m"
      memory: "256Mi"
  exporter:
    resources:
      requests:
        cpu: "20m"
        memory: "64Mi"
      limits:
        cpu: "100m"
        memory: "128Mi"

chunks-cache:
  enabled: true
  replicas: 2
  resources:
    requests:
      cpu: "50m"
      memory: "128Mi"
    limits:
      cpu: "200m"
      memory: "256Mi"
  exporter:
    resources:
      requests:
        cpu: "20m"
        memory: "64Mi"
      limits:
        cpu: "100m"
        memory: "128Mi"

index-cache:
  enabled: true
  replicas: 2
  resources:
    requests:
      cpu: "50m"
      memory: "128Mi"
    limits:
      cpu: "200m"
      memory: "256Mi"
  exporter:
    resources:
      requests:
        cpu: "20m"
        memory: "64Mi"
      limits:
        cpu: "100m"
        memory: "128Mi"

metadata-cache:
  enabled: true
  replicas: 2
  resources:
    requests:
      cpu: "50m"
      memory: "128Mi"
    limits:
      cpu: "200m"
      memory: "256Mi"
  exporter:
    resources:
      requests:
        cpu: "20m"
        memory: "64Mi"
      limits:
        cpu: "100m"
        memory: "128Mi"

results-cache:
  enabled: true
  replicas: 2
  resources:
    requests:
      cpu: "50m"
      memory: "128Mi"
    limits:
      cpu: "200m"
      memory: "256Mi"
  exporter:
    resources:
      requests:
        cpu: "20m"
        memory: "64Mi"
      limits:
        cpu: "100m"
        memory: "128Mi"

# ---------------------------------------------------------------------------
# Supporting components
# ---------------------------------------------------------------------------
overrides_exporter:
  replicas: 1
  resources:
    limits:
      memory: 128Mi
    requests:
      cpu: 100m
      memory: 128Mi

# ---------------------------------------------------------------------------
# Object storage (embedded MinIO for demo/test)
# ---------------------------------------------------------------------------
# id: grafana-mimir
# pw: supersecret
minio:
  enabled: true
  mode: standalone
  rootUser: grafana-mimir
  buckets:
    - name: mimir-tsdb
      policy: none
      purge: false
    - name: mimir-ruler
      policy: none
      purge: false
  persistence:
    size: 5Gi
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
  rootPassword: supersecret
  # Changed the mc config path to '/tmp' from '/etc' as '/etc' is only writable by root and OpenShift will not permit this.
  configPathmc: "/tmp/minio/mc/"



# -- Enable Kafka for ingest-storage architecture
kafka:
  enabled: true

# helm -n mimir-test install mimir grafana/mimir-distributed
helm install mimir grafana/mimir-distributed -n mimir -f small-minio.yaml

helm upgrade mimir grafana/mimir-distributed -n mimir -f small-minio.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: mimir-minio-console
  namespace: mimir
spec:
  ingressClassName: nginx
  rules:
    - host: minio-mimir.tk8s.test       # 원하는 도메인 이름으로 교체
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: mimir-minio-console
                port:
                  number: 9001
  tls:
  - hosts:
    - minio-mimir.tk8s.test       # 원하는 도메인 이름으로 교체
    secretName: ssl-certs
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: mimir-ingress
  namespace: mimir
spec:
  ingressClassName: nginx
  rules:
    - host: mimir.tk8s.test       # 원하는 도메인 이름으로 교체
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                # name: mimir-distributor
                name: mimir-gateway
                port:
                  # number: 8080
                  number: 80
  tls:
  - hosts:
    - mimir.tk8s.test       # 원하는 도메인 이름으로 교체
    secretName: ssl-certs

remote_write:
  - url: https://mimir.tk8s.test/api/v1/push
    remote_timeout: 30s
    tls_config:
      insecure_skip_verify: true
    headers:
      X-Scope-OrgID: "hk8s"
    queue_config:
      capacity: 5000
      max_shards: 100
      max_samples_per_send: 1000
      batch_send_deadline: 5s
    write_relabel_configs:
      - source_labels: [__name__]
        regex: ".*"
        action: keep

time=2025-11-10T09:52:45.305Z level=ERROR source=queue_manager.go:1670 msg="non-recoverable error" component=remote remote_name=5b8a51 url=https://mimir.tk8s.test/api/v1/push failedSampleCount=1000 failedHistogramCount=0 failedExemplarCount=0 err="server returned HTTP status 400 Bad Request: received a series whose number of labels exceeds the limit (actual: 34, limit: 30) series: 'istio_requests_total{app=\"butler\", connection_security_policy=\"mutual_tls\", destination_app=\"butler\", destination_canonical_revision=\"latest\", destination_canonical_service=\"butler\", destination_clust…' (err-mimir-max-label-names-per-series). To adjust the related per-tenant limit, configure -validation.max-label-names-per-series, or contact your service administrator.\n

# small-minio.yaml
# ---------------------------------------------------------------------------
# Mimir config overrides (추가 부분)
# ---------------------------------------------------------------------------
mimir:
  structuredConfig:
    multitenancy_enabled: true     # 여러 tenant 지원 (Prometheus에서 X-Scope-OrgID 헤더 전송 시 필요)
    limits:
      max_label_names_per_series: 40   # 기존 30 → 40으로 확장

ts=2025-11-11T05:24:25.684886843Z 
caller=pusher.go:561 
component=ingest_reader 
user=hk8s 
level=warn 
msg="detected a client error while ingesting write request (the request may have been partially ingested)" 
err="user=hk8s: per-user series limit of 150000 exceeded (err-mimir-max-series-per-user). To adjust the related per-tenant limit, configure -ingester.max-global-series-per-user, or contact your service administrator. (sampled 1/10)"