[Release] 3.3.0 버전 패치 프로덕션 서버에 배포

.github/workflows/deploy-dev.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Create application-dev.yml
         run: |
           mkdir -p ./src/main/resources
-          echo "${{ secrets.APPLICATION_DEV }}" > ./src/main/resources/application.yml
+          echo "${{ secrets.APPLICATION_DEV }}" > ./src/main/resources/application-dev.yml
         shell: bash
 
       # 4. Firebase Admin Key 생성
@@ -73,6 +73,8 @@ jobs:
           cat > .env.dev << EOF
           DOCKER_USERNAME=${{ secrets.DOCKER_USERNAME }}
           MYSQL_ROOT_PASSWORD=${{ secrets.MYSQL_ROOT_PASSWORD }}
+          RABBITMQ_USER=${{ secrets.RABBITMQ_USER }}
+          RABBITMQ_PASSWORD=${{ secrets.RABBITMQ_PASSWORD }}
           SPRING_CRYPTO_SECRET_KEY=${{ secrets.SPRING_CRYPTO_SECRET_KEY }}
           JWT_ACCESS_TOKEN_SECRET=${{ secrets.JWT_ACCESS_TOKEN_SECRET }}
           JWT_ACCESS_TOKEN_EXPIRATION=1800000
@@ -81,17 +83,28 @@ jobs:
           CLOUD_AWS_S3_BUCKET=${{ secrets.CLOUD_AWS_S3_BUCKET }}
           CLOUD_AWS_CREDENTIALS_ACCESS_KEY=${{ secrets.CLOUD_AWS_CREDENTIALS_ACCESS_KEY }}
           CLOUD_AWS_CREDENTIALS_SECRET_KEY=${{ secrets.CLOUD_AWS_CREDENTIALS_SECRET_KEY }}
+          DISCORD_WEBHOOK_URL=${{ secrets.DISCORD_WEBHOOK_URL }}
           DISCORD_WEBHOOK_URL_DEV=${{ secrets.DISCORD_WEBHOOK_URL }}
           OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}
+          SENTRY_DSN=${{ secrets.SENTRY_DSN }}
           SENTRY_DSN_DEV=${{ secrets.SENTRY_DSN }}
           ADMIN_IDENTIFIER=${{ secrets.ADMIN_IDENTIFIER }}
           ADMIN_PASSWORD=${{ secrets.ADMIN_PASSWORD }}
+          GRAFANA_ADMIN_USER=${{ secrets.GRAFANA_ADMIN_USER }}
+          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          GRAFANA_DEV_ROOT_URL=https://ono-dev.seungminki.shop/grafana/
+          GRAFANA_DEV_SERVE_FROM_SUB_PATH=true
+          PROMETHEUS_DEV_EXTERNAL_URL=https://ono-dev.seungminki.shop/prometheus/
+          PROMETHEUS_DEV_ROUTE_PREFIX=/prometheus
           EOF
         shell: bash
 
       # 10. Docker Compose로 배포 (DEVELOPMENT)
       - name: Deploy with Docker Compose
         run: |
+          # 최신 dev 이미지 강제 pull (tag 캐시로 구버전 실행되는 문제 방지)
+          docker pull ${{ secrets.DOCKER_USERNAME }}/ono:dev-latest
+
           # MySQL과 Redis 시작 (개발 환경 전용)
           docker-compose -f docker-compose.dev.yml --env-file .env.dev up -d mysql-dev redis-dev
 
@@ -100,6 +113,10 @@ jobs:
           docker-compose -f docker-compose.dev.yml --env-file .env.dev rm -f app-dev || true
           docker-compose -f docker-compose.dev.yml --env-file .env.dev up -d app-dev
 
+          # 실제 기동 이미지 확인
+          docker inspect ono-app-dev --format='Running Image: {{.Config.Image}}'
+          docker inspect ono-app-dev --format='Image ID: {{.Image}}'
+
           # 헬스체크 대기 (최대 2분)
           echo "Waiting for development application to be healthy..."
           timeout=60
@@ -122,6 +139,56 @@ jobs:
           docker-compose -f docker-compose.dev.yml --env-file .env.dev logs --tail=50 app-dev
         shell: bash
 
+      # 11. Monitoring Stack 배포 (DEVELOPMENT)
+      - name: Deploy Monitoring Stack
+        run: |
+          docker-compose -f docker-compose.dev.yml --env-file .env.dev --profile monitoring stop prometheus-dev grafana-dev mysql-exporter-dev || true
+          docker-compose -f docker-compose.dev.yml --env-file .env.dev --profile monitoring rm -f prometheus-dev grafana-dev mysql-exporter-dev || true
+          docker-compose -f docker-compose.dev.yml --env-file .env.dev --profile monitoring up -d --force-recreate \
+            mysql-exporter-dev prometheus-dev grafana-dev
+          docker-compose -f docker-compose.dev.yml --env-file .env.dev ps \
+            mysql-exporter-dev prometheus-dev grafana-dev
+        shell: bash
+
+      # 11-1. Monitoring 연동 검증 (DEVELOPMENT)
+      - name: Verify Monitoring Integration
+        run: |
+          # app-dev 내부 actuator 메트릭 엔드포인트가 200인지 확인
+          echo "Checking app-dev actuator metrics endpoint..."
+          if ! docker exec ono-app-dev sh -lc 'wget -qO- http://localhost:8081/actuator/prometheus | head -n 5' >/tmp/app_metrics_head.txt; then
+            echo "ERROR: app-dev actuator/prometheus endpoint is not reachable"
+            docker logs --tail=200 ono-app-dev || true
+            exit 1
+          fi
+
+          # prometheus 컨테이너 관점에서도 app-dev 메트릭 엔드포인트가 200인지 확인
+          echo "Checking app-dev actuator metrics endpoint from prometheus-dev..."
+          if ! docker exec ono-prometheus-dev sh -lc 'wget -qO- http://app-dev:8081/actuator/prometheus | head -n 5' >/tmp/prom_metrics_head.txt; then
+            echo "ERROR: prometheus-dev cannot fetch app-dev actuator/prometheus"
+            docker exec ono-prometheus-dev sh -lc 'wget -S -O- http://app-dev:8081/actuator/prometheus >/dev/null' || true
+            docker logs --tail=200 ono-app-dev || true
+            exit 1
+          fi
+
+          # Prometheus가 app-dev를 up=1로 스크랩하는지 확인 (최대 60초 대기)
+          echo "Checking Prometheus scrape status for ono-backend-dev..."
+          for i in $(seq 1 12); do
+            result=$(docker exec ono-prometheus-dev sh -lc 'wget -qO- "http://localhost:9090/prometheus/api/v1/query?query=up{job=\"ono-backend-dev\"}"' || true)
+            if echo "$result" | grep -q '"value":\[.*,"1"\]'; then
+              echo "Prometheus scrape is UP"
+              exit 0
+            fi
+            echo "Waiting scrape up... ($i/12)"
+            sleep 5
+          done
+
+          echo "ERROR: Prometheus scrape for ono-backend-dev is still down"
+          docker exec ono-prometheus-dev sh -lc 'wget -qO- "http://localhost:9090/prometheus/api/v1/targets"' || true
+          docker logs --tail=200 ono-app-dev || true
+          docker logs --tail=200 ono-prometheus-dev || true
+          exit 1
+        shell: bash
+
       # 12. 오래된 Docker 이미지 정리
       - name: Clean up old Docker images
         run: |

.github/workflows/deploy-prod.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Create application-prod.yml
         run: |
           mkdir -p ./src/main/resources
-          echo "${{ secrets.APPLICATION_PROD }}" > ./src/main/resources/application.yml
+          echo "${{ secrets.APPLICATION_PROD }}" > ./src/main/resources/application-prod.yml
         shell: bash
 
       # 4. Firebase Admin Key 생성
@@ -73,6 +73,8 @@ jobs:
           cat > .env.prod << EOF
           DOCKER_USERNAME=${{ secrets.DOCKER_USERNAME }}
           MYSQL_ROOT_PASSWORD=${{ secrets.MYSQL_ROOT_PASSWORD }}
+          RABBITMQ_USER=${{ secrets.RABBITMQ_USER }}
+          RABBITMQ_PASSWORD=${{ secrets.RABBITMQ_PASSWORD }}
           SPRING_CRYPTO_SECRET_KEY=${{ secrets.SPRING_CRYPTO_SECRET_KEY }}
           JWT_ACCESS_TOKEN_SECRET=${{ secrets.JWT_ACCESS_TOKEN_SECRET }}
           JWT_ACCESS_TOKEN_EXPIRATION=1800000
@@ -86,12 +88,21 @@ jobs:
           SENTRY_DSN=${{ secrets.SENTRY_DSN }}
           ADMIN_IDENTIFIER=${{ secrets.ADMIN_IDENTIFIER }}
           ADMIN_PASSWORD=${{ secrets.ADMIN_PASSWORD }}
+          GRAFANA_ADMIN_USER=${{ secrets.GRAFANA_ADMIN_USER }}
+          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          GRAFANA_PROD_ROOT_URL=https://ono-prod.seungminki.shop/grafana/
+          GRAFANA_PROD_SERVE_FROM_SUB_PATH=true
+          PROMETHEUS_PROD_EXTERNAL_URL=https://ono-prod.seungminki.shop/prometheus/
+          PROMETHEUS_PROD_ROUTE_PREFIX=/prometheus
           EOF
         shell: bash
 
       # 10. Blue-Green 무중단 배포
       - name: Blue-Green Deployment
         run: |
+          # 최신 prod 이미지 강제 pull (tag 캐시로 구버전 실행되는 문제 방지)
+          docker pull ${{ secrets.DOCKER_USERNAME }}/ono:prod-latest
+
           # MySQL과 Redis 시작 (이미 실행 중이면 스킵)
           docker-compose -f docker-compose.prod.yml --env-file .env.prod up -d mysql-prod redis-prod
 
@@ -196,6 +207,39 @@ jobs:
           docker-compose -f docker-compose.prod.yml logs --tail=50 app-prod-$TARGET
         shell: bash
 
+      # 11. Monitoring Stack 배포 (PRODUCTION, Non-blocking)
+      - name: Deploy Monitoring Stack
+        run: |
+          docker-compose -f docker-compose.prod.yml --env-file .env.prod --profile monitoring stop prometheus-prod grafana-prod mysql-exporter-prod || true
+          docker-compose -f docker-compose.prod.yml --env-file .env.prod --profile monitoring rm -f prometheus-prod grafana-prod mysql-exporter-prod || true
+          docker-compose -f docker-compose.prod.yml --env-file .env.prod --profile monitoring up -d --force-recreate \
+            mysql-exporter-prod prometheus-prod grafana-prod
+          docker-compose -f docker-compose.prod.yml --env-file .env.prod ps \
+            mysql-exporter-prod prometheus-prod grafana-prod
+        shell: bash
+
+      # 11-1. Monitoring 연동 검증 (PRODUCTION, Non-blocking)
+      - name: Verify Monitoring Integration
+        continue-on-error: true
+        run: |
+          echo "Checking Prometheus scrape status for production jobs..."
+
+          # 최대 60초 동안 active blue/green 중 하나라도 up=1인지 확인
+          for i in $(seq 1 12); do
+            result=$(docker exec ono-prometheus-prod sh -lc 'wget -qO- "http://localhost:9090/prometheus/api/v1/query?query=max(up{job=~\"ono-backend-prod-(blue|green)\"})"' || true)
+            if echo "$result" | grep -q '"value":\[.*,"1"\]'; then
+              echo "Prometheus scrape is UP for at least one production target"
+              exit 0
+            fi
+            echo "Waiting scrape up... ($i/12)"
+            sleep 5
+          done
+
+          echo "WARNING: Prometheus scrape for production targets is still down"
+          docker exec ono-prometheus-prod sh -lc 'wget -qO- "http://localhost:9090/prometheus/api/v1/targets"' || true
+          docker logs --tail=200 ono-prometheus-prod || true
+        shell: bash
+
       # 12. 오래된 Docker 이미지 정리
       - name: Clean up old Docker images
         run: |

.github/workflows/restart-monitoring.yml

@@ -0,0 +1,71 @@
+name: Restart Monitoring Stack
+
+on:
+  workflow_dispatch:
+    inputs:
+      target:
+        description: "Target environment"
+        required: true
+        type: choice
+        options:
+          - dev
+          - prod
+        default: dev
+
+permissions:
+  contents: read
+
+jobs:
+  restart-monitoring:
+    runs-on: self-hosted
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Create .env files
+        run: |
+          cat > .env.dev << EOF
+          DOCKER_USERNAME=${{ secrets.DOCKER_USERNAME }}
+          MYSQL_ROOT_PASSWORD=${{ secrets.MYSQL_ROOT_PASSWORD }}
+          RABBITMQ_USER=${{ secrets.RABBITMQ_USER }}
+          RABBITMQ_PASSWORD=${{ secrets.RABBITMQ_PASSWORD }}
+          GRAFANA_ADMIN_USER=${{ secrets.GRAFANA_ADMIN_USER }}
+          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          GRAFANA_DEV_ROOT_URL=https://ono-dev.seungminki.shop/grafana/
+          GRAFANA_DEV_SERVE_FROM_SUB_PATH=true
+          PROMETHEUS_DEV_EXTERNAL_URL=https://ono-dev.seungminki.shop/prometheus/
+          PROMETHEUS_DEV_ROUTE_PREFIX=/prometheus
+          EOF
+
+          cat > .env.prod << EOF
+          DOCKER_USERNAME=${{ secrets.DOCKER_USERNAME }}
+          MYSQL_ROOT_PASSWORD=${{ secrets.MYSQL_ROOT_PASSWORD }}
+          RABBITMQ_USER=${{ secrets.RABBITMQ_USER }}
+          RABBITMQ_PASSWORD=${{ secrets.RABBITMQ_PASSWORD }}
+          GRAFANA_ADMIN_USER=${{ secrets.GRAFANA_ADMIN_USER }}
+          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          GRAFANA_PROD_ROOT_URL=https://ono-prod.seungminki.shop/grafana/
+          GRAFANA_PROD_SERVE_FROM_SUB_PATH=true
+          PROMETHEUS_PROD_EXTERNAL_URL=https://ono-prod.seungminki.shop/prometheus/
+          PROMETHEUS_PROD_ROUTE_PREFIX=/prometheus
+          EOF
+        shell: bash
+
+      - name: Restart monitoring stack (dev)
+        if: ${{ github.event.inputs.target == 'dev' }}
+        run: |
+          docker compose -f docker-compose.dev.yml --env-file .env.dev --profile monitoring up -d --force-recreate \
+            mysql-exporter-dev prometheus-dev grafana-dev
+          docker compose -f docker-compose.dev.yml --env-file .env.dev ps \
+            mysql-exporter-dev prometheus-dev grafana-dev
+        shell: bash
+
+      - name: Restart monitoring stack (prod)
+        if: ${{ github.event.inputs.target == 'prod' }}
+        run: |
+          docker compose -f docker-compose.prod.yml --env-file .env.prod --profile monitoring up -d --force-recreate \
+            mysql-exporter-prod prometheus-prod grafana-prod
+          docker compose -f docker-compose.prod.yml --env-file .env.prod ps \
+            mysql-exporter-prod prometheus-prod grafana-prod
+        shell: bash

.gitignore

@@ -52,6 +52,12 @@ FirebaseAdminKey.json
 
 ### Environment variables
 .env
+.env.dev
+monitoring/.env
+monitoring/.env.*
+monitoring/grafana/provisioning/datasources/*.local.yml
+monitoring/grafana/provisioning/dashboards/*.local.yml
+monitoring/README.md
 
 ### macOS
-.DS_Store
+.DS_Store

Dockerfile

@@ -18,4 +18,4 @@ COPY ${JAR_FILE} backend-0.0.1.SNAPSHOT.jar
 COPY FirebaseAdminKey.json FirebaseAdminKey.json
 
 # 운영 및 개발에서 사용되는 환경 설정을 분리한다.
-ENTRYPOINT ["java", "-Duser.timezone=Asia/Seoul", "-jar", "-Dspring.profiles.active=prod", "backend-0.0.1.SNAPSHOT.jar"]
+ENTRYPOINT ["java", "-Duser.timezone=Asia/Seoul", "-jar", "backend-0.0.1.SNAPSHOT.jar"]

build.gradle

@@ -55,6 +55,7 @@ dependencies {
 	implementation 'org.springframework.boot:spring-boot-starter-data-redis'
 	implementation 'org.springframework.boot:spring-boot-starter-amqp'
 	implementation 'org.springframework.boot:spring-boot-starter-actuator'
+	runtimeOnly 'io.micrometer:micrometer-registry-prometheus'
 
 	// Hibernate Core
 	implementation 'org.hibernate:hibernate-core:6.5.2.Final'
@@ -84,4 +85,4 @@ tasks.withType(JavaCompile) {
 
 clean {
 	delete file('src/main/generated')
-}
+}

docker-compose.dev.yml

@@ -100,6 +100,9 @@ services:
       SENTRY_DSN: ${SENTRY_DSN}
       ADMIN_IDENTIFIER: ${ADMIN_IDENTIFIER}
       ADMIN_PASSWORD: ${ADMIN_PASSWORD}
+      MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: health,info,metrics,prometheus
+      MANAGEMENT_ENDPOINT_PROMETHEUS_ENABLED: "true"
+      MANAGEMENT_SERVER_PORT: 8081
     volumes:
       - app_dev_logs:/app/logs
     depends_on:
@@ -112,12 +115,73 @@ services:
     networks:
       - ono-network
     healthcheck:
-      test: ["CMD-SHELL", "wget --quiet --tries=1 --spider http://localhost:8080/actuator/health || exit 1"]
+      test: ["CMD-SHELL", "wget --quiet --tries=1 --spider http://localhost:8081/actuator/health || exit 1"]
       interval: 30s
       timeout: 10s
       retries: 3
       start_period: 60s
 
+  # Prometheus - DEVELOPMENT (Profile: monitoring)
+  prometheus-dev:
+    image: prom/prometheus:latest
+    container_name: ono-prometheus-dev
+    restart: unless-stopped
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --web.external-url=${PROMETHEUS_DEV_EXTERNAL_URL:-http://localhost:9091}
+      - --web.route-prefix=${PROMETHEUS_DEV_ROUTE_PREFIX:-/}
+    ports:
+      - "9091:9090"
+    volumes:
+      - ./monitoring/prometheus.dev.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/alerts:/etc/prometheus/alerts:ro
+      - prometheus_dev_data:/prometheus
+    networks:
+      - ono-network
+    depends_on:
+      - app-dev
+      - mysql-exporter-dev
+    profiles:
+      - monitoring
+
+  mysql-exporter-dev:
+    image: prom/mysqld-exporter:latest
+    container_name: ono-mysql-exporter-dev
+    restart: unless-stopped
+    environment:
+      DATA_SOURCE_NAME: root:${MYSQL_ROOT_PASSWORD}@(mysql-dev:3306)/
+    depends_on:
+      mysql-dev:
+        condition: service_healthy
+    networks:
+      - ono-network
+    profiles:
+      - monitoring
+
+  # Grafana - DEVELOPMENT (Profile: monitoring)
+  grafana-dev:
+    image: grafana/grafana:latest
+    container_name: ono-grafana-dev
+    restart: unless-stopped
+    ports:
+      - "3001:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_SERVER_ROOT_URL: ${GRAFANA_DEV_ROOT_URL:-http://localhost:3001}
+      GF_SERVER_SERVE_FROM_SUB_PATH: ${GRAFANA_DEV_SERVE_FROM_SUB_PATH:-false}
+    volumes:
+      - grafana_dev_data:/var/lib/grafana
+      - ./monitoring/grafana/provisioning/dev:/etc/grafana/provisioning:ro
+      - ./monitoring/grafana/dashboards:/etc/grafana/dashboards:ro
+    depends_on:
+      - prometheus-dev
+    networks:
+      - ono-network
+    profiles:
+      - monitoring
+
 networks:
   ono-network:
     driver: bridge
@@ -131,4 +195,8 @@ volumes:
   rabbitmq_dev_data:
     driver: local
   app_dev_logs:
-    driver: local
+    driver: local
+  prometheus_dev_data:
+    driver: local
+  grafana_dev_data:
+    driver: local