Fluent Bit to S3

EKS_CLUSTER_NAME="finance-eks-cluster"
EKS_NODE_GROUP_NAME="finance-app-ng"
Shell
복사

cat << EOF > fluent-bit-s3-policy.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:PutObject"
            ],
            "Resource": "*"
        }
    ]
}
EOF
Shell
복사

aws iam create-policy --policy-name FluentBitS3Policy --policy-document file://fluent-bit-s3-policy.json
Shell
복사

POLICY_ARN=$(aws iam list-policies --query 'Policies[?PolicyName==`FluentBitS3Policy`].Arn' --output text)
Shell
복사

eksctl create iamserviceaccount \
    --name fluent-bit \
    --region="ap-northeast-2" \
    --cluster "$EKS_CLUSTER_NAME" \
    --namespace=finance-ns \
    --attach-policy-arn "$POLICY_ARN" \
    --override-existing-serviceaccounts \
    --approve
Shell
복사

NODEGROUP_ROLE_NAME=$(aws eks describe-nodegroup --cluster-name $EKS_CLUSTER_NAME --nodegroup-name $EKS_NODE_GROUP_NAME --query "nodegroup.nodeRole" --output text | cut -d'/' -f2-)
Shell
복사

aws iam attach-role-policy --role-name $NODEGROUP_ROLE_NAME --policy-arn arn:aws:iam::362708816803:policy/FluentBitS3Policy
Shell
복사

eksctl utils associate-iam-oidc-provider --region=ap-northeast-2 --cluster=$EKS_CLUSTER_NAME --approve
Shell
복사

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: finance-ns
data:
  fluent-bit.conf: |
    [SERVICE]
      Flush        1
      Log_Level    info
      Daemon       Off
      Parsers_File parsers.conf
      HTTP_Server  On
      HTTP_Listen  0.0.0.0
      HTTP_Port    2020

    [INPUT]
      Name              tail
      Tag               kube.*
      Path              /var/log/containers/*.log
      Parser            docker
      DB                /var/log/fluent-bit.db

    [OUTPUT]
      Name              s3
      Match             kube.*
      region            ap-northeast-2
      bucket            finance-storage-01
      total_file_size   250M
      s3_key_format     /data/%Y-%m-%d_%H_%M/finance_access.log
      upload_timeout    1m
      store_dir         /var/fluent-bit/state/flb-storage/s3
      use_put_object    Off
      static_file_path  On
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: finance-ns
spec:
  selector:
    matchLabels:
      app: fluent-bit
  template:
    metadata:
      labels:
        app: fluent-bit
    spec:
      containers:
        - name: fluent-bit
          image: fluent/fluent-bit:latest
          ports:
            - containerPort: 2020
          volumeMounts:
            - name: varlog
              mountPath: /var/log
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: fluent-bit-config
              mountPath: /fluent-bit/etc/
      volumes:
        - name: varlog
          hostPath:
            path: /var/log
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: fluent-bit-config
          configMap:
            name: fluent-bit-config
YAML
복사

kubectl apply -f fluentbit.yaml
Shell
복사

apiVersion: apps/v1
kind: Deployment
metadata:
  name: finance-user-dpn
  namespace: finance-ns
  labels:
    app: user
spec:
  replicas: 2
  selector:
    matchLabels:
      app: user
  template:
    metadata:
      labels:
        app: user
    spec:
      containers:
      - name: finance-ecr
        image: 362708816803.dkr.ecr.ap-northeast-2.amazonaws.com/finance-ecr:latest
        imagePullPolicy: Always
        env:
          - name: MYSQL_USER
            value: "admin"
          - name: MYSQL_PASSWORD
            value: "Skill53##"
          - name: MYSQL_HOST
            value: "finance-db-cluster.cluster-cxytji5957dw.ap-northeast-2.rds.amazonaws.com"
          - name: MYSQL_PORT
            value: "3306"
          - name: MYSQL_DBNAME
            value: "day1"
        ports:
        - containerPort: 8080
          name: http
YAML
복사

kubectl apply -f deployment.yaml
Shell
복사