diff --git a/config.cpu.yaml b/config.cpu.yaml index 6ceac915..3989a547 100644 --- a/config.cpu.yaml +++ b/config.cpu.yaml @@ -22,10 +22,11 @@ embedding: # embedding service config # for external embedding service, set CC_EM_BASE_URL and CC_EM_APIKEY env vars during deployment # if the env vars are set, this config is ignored - # request_timeout is always respected even for remote service + # connect_timeout and request_timeout are always respected even for remote service base_url: http://localhost:5000/v1 workers: 1 - request_timeout: 1800 # in seconds + connect_timeout: 15 # in seconds + request_timeout: 1750 # in seconds # batch_size: 100 # max texts per embedding API request, 0 = no batching # only for external embedding service # remote_service: true diff --git a/config.gpu.yaml b/config.gpu.yaml index a12fd1be..1c9d412d 100644 --- a/config.gpu.yaml +++ b/config.gpu.yaml @@ -22,10 +22,11 @@ embedding: # embedding service config # for external embedding service, set CC_EM_BASE_URL and CC_EM_APIKEY env vars during deployment # if the env vars are set, this config is ignored - # request_timeout is always respected even for remote service + # connect_timeout and request_timeout are always respected even for remote service base_url: http://localhost:5000/v1 workers: 1 - request_timeout: 1800 # in seconds + connect_timeout: 15 # in seconds + request_timeout: 1750 # in seconds # batch_size: 100 # max texts per embedding API request, 0 = no batching # only for external embedding service # remote_service: true diff --git a/context_chat_backend/config_parser.py b/context_chat_backend/config_parser.py index 0a62019a..0ef0e2c1 100644 --- a/context_chat_backend/config_parser.py +++ b/context_chat_backend/config_parser.py @@ -77,7 +77,8 @@ def get_config(file_path: str) -> TConfig: auth=auth, remote_service=True, workers=0, - request_timeout=embedding.get('request_timeout', 1800) if embedding else 1800, + connect_timeout=embedding.get('connect_timeout', 15) if embedding else 15, + request_timeout=embedding.get('request_timeout', 1750) if embedding else 1750, batch_size=int(os.getenv('CC_EM_BATCH_SIZE', 100)), ) except Exception as e: diff --git a/context_chat_backend/network_em.py b/context_chat_backend/network_em.py index 5ba8faf5..28797368 100644 --- a/context_chat_backend/network_em.py +++ b/context_chat_backend/network_em.py @@ -99,7 +99,10 @@ def _get_embedding(self, input_: str | list[str], try_: int = 3) -> list[float] response = niquests.post( f'{emconf.base_url.removesuffix("/")}/embeddings', json=data, - timeout=emconf.request_timeout, + timeout=niquests.TimeoutConfiguration( + connect=emconf.connect_timeout, + read=emconf.request_timeout, + ), auth=auth, verify=self.app_config.verify_ssl, ) diff --git a/context_chat_backend/types.py b/context_chat_backend/types.py index 8d7d0fcf..e85251da 100644 --- a/context_chat_backend/types.py +++ b/context_chat_backend/types.py @@ -104,6 +104,7 @@ class TEmbeddingAuthBasic(BaseModel): class TEmbeddingConfig(BaseModel): base_url: str = 'http://localhost:5000/v1' workers: int = 1 + connect_timeout: int = 15 request_timeout: int = 1750 model_name: str | None = DEFAULT_EM_MODEL_ALIAS auth: TEmbeddingAuthApiKey | TEmbeddingAuthBasic | None = None