docker-compose.yml

version: '3.9'
networks:
  shared-net:
    external: true
volumes:
  minio-data:

services:
  # ── MinIO S3 호환 오브젝트 스토리지 (데이터 레이크 Bronze/Silver 레이어)
  minio:
    image: minio/minio:latest
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-minioadmin}
      MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-minioadmin}
    volumes:
      - minio-data:/data
    ports:
      - "9000:9000"   # S3 API
      - "9001:9001"   # Web Console
    networks:
      - shared-net
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 10s
      timeout: 5s
      retries: 5
    restart: unless-stopped

  # ── MinIO 버킷 초기화 (최초 1회)
  minio-init:
    image: minio/mc:latest
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: >
      /bin/sh -c "
        mc alias set local http://minio:9000 $${MINIO_ACCESS_KEY:-minioadmin} $${MINIO_SECRET_KEY:-minioadmin};
        mc mb --ignore-existing local/data-lake-bronze;
        mc mb --ignore-existing local/data-lake-silver;
        mc mb --ignore-existing local/data-lake-gold;
        echo 'MinIO 버킷 초기화 완료';
      "
    networks:
      - shared-net
  zavod:
    image: ghcr.io/opensanctions/opensanctions:latest
    user: ${LOCAL_UID:-1000}:${LOCAL_GID:-1000}
    env_file:
    - .env
    environment:
      DATABASE_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-dev}}
      ZAVOD_DATABASE_URI: ${ZAVOD_DATABASE_URI:-postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-dev}}
    volumes:
    - .:/opensanctions
    networks:
    - shared-net
    command: |
      bash -c "
        echo '🚀 Scanning datasets (excluding _ folders, GCS-dependent, ru_nsd_isin, and cz_business_register)...' &&
        FAILED_FILE=/opensanctions/failed_datasets.md &&
        echo '# ❌ Failed Datasets (Zavod Batch Run)' > $$FAILED_FILE &&
        echo '' >> $$FAILED_FILE &&

        for file in $(find /opensanctions/datasets -type f -name '*.yml' ! -path '*/_*/*'); do
          BASENAME=$(basename $$file)


          if [[ $$BASENAME == 'ru_nsd_isin.yml' || $$BASENAME == 'cz_business_register.yml' || $$BASENAME == 'us_corpwatch.yml' || $$BASENAME == 'lv_business_register.yml' || $$BASENAME == 'lv_business_register.yml' || $$BASENAME == 'md_companies.yml' || $$BASENAME == 'gb_fca_firds.yml' || $$BASENAME == 'cy_companies.yml' ]]; then
            echo '🚫 Skipping excluded dataset: '$$file;
            continue;
          fi

          # ⚠️ 내부 데이터(GCS) 의존성 검사
          if grep -q 'fetch_internal_data' $$file; then
            echo '⚠️  Skipping GCS-dependent dataset: '$$file;
            continue;
          fi

          echo '⚙️  Running ETL for: '$$file;
          (zavod crawl $$file && zavod export $$file && zavod load-db $$file);

          if [ $$? -eq 0 ]; then
            echo '✅ Completed: '$$file;
          else
            echo '❌ Failed: '$$file;
            echo '- '$$file >> $$FAILED_FILE;
          fi
        done &&

        echo '' >> $$FAILED_FILE &&
        echo '---' >> $$FAILED_FILE &&
        echo '🕒 Run completed on: '$(date) >> $$FAILED_FILE &&

        echo '🎉 All datasets processed. Failures logged in failed_datasets.md (host directory)'
      "
  web:
    build:
      context: ./ui
    environment:
      ZAVOD_DATABASE_URI: ${ZAVOD_DATABASE_URI:-postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-dev}}
      ZAVOD_ALLOW_UNAUTHENTICATED: ${ZAVOD_ALLOW_UNAUTHENTICATED:-true}
      DB_HOST: postgres
      DB_USER: ${POSTGRES_USER:-postgres}
      DB_PASSWORD: ${POSTGRES_PASSWORD:-password}
      DB_NAME: ${POSTGRES_DB:-dev}
      QDRANT_URL: http://qdrant:6333
    ports:
    - 8899:3000
    networks:
    - shared-net
  # ── 데이터 레이크 Gold 집계 잡 (duckdb 기반 분석 집계)
  gold-builder:
    build:
      context: .
      dockerfile: infra/Dockerfile.crawler-job
    image: finance-crawler-job:latest
    environment:
      MINIO_ENDPOINT: http://minio:9000
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
      BUCKET_BRONZE: data-lake-bronze
      BUCKET_SILVER: data-lake-silver
      GOLD_DB_PATH: /data/lake/gold/analytics.duckdb
    profiles:
      - gold
    networks:
      - shared-net
    command: ["python", "pipeline/gold_build.py"]

  # ── 크롤러 잡 이미지 빌드 (실행은 crawler-api가 docker run --rm 으로 수행)
  finance-crawler-job:
    build:
      context: .
      dockerfile: infra/Dockerfile.crawler-job
    image: finance-crawler-job:latest
    profiles:
    - build-only          # docker-compose up 시 자동 실행 안 됨
    networks:
    - shared-net

  # ── 크롤러 백엔드 API (FE → POST /crawl/start → docker run --rm 잡 실행)
  crawler-api:
    build:
      context: .
      dockerfile: infra/Dockerfile.crawler-api
    environment:
      QDRANT_URL: http://qdrant:6333
      CRAWLER_IMAGE: finance-crawler-job:latest
      DOCKER_NETWORK: shared-net
      MINIO_ENDPOINT: http://minio:9000
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
      DART_API_KEY: ${DART_API_KEY:-}
      ECOS_API_KEY: ${ECOS_API_KEY:-}
      PYTHONUNBUFFERED: '1'
    volumes:
    - /var/run/docker.sock:/var/run/docker.sock   # 호스트 Docker 소켓 마운트
    ports:
    - 8400:8080
    networks:
    - shared-net
    restart: unless-stopped