Benchmarked: the new mlserver uds protocol with respect to old http one

addition_malabr/Dockerfile

@@ -1,20 +1,22 @@
-# Use Ubuntu as base image
-FROM ubuntu:22.04
+# Use official Python slim image
+FROM python:3.11-slim
 
-# Prevent interactive prompts during package installation
-ENV DEBIAN_FRONTEND=noninteractive
+# Disable bytecode generation & buffer output
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
 
-# Install system dependencies
-RUN apt-get update && apt-get install -y \
-    python3 \
-    python3-pip \
-    python3-dev \
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies (optional but helps in many cases)
+RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     git \
     curl \
-    sudo && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/*
+
+# Upgrade pip first
+RUN pip install --upgrade pip
 
 # Set working directory
 WORKDIR /app
@@ -25,9 +27,16 @@ COPY modelserver/myrequirements.txt .
 
 # Install Python packages
 RUN pip install -r myrequirements.txt
+RUN pip install gunicorn
 
 # Expose the Flask app port
 EXPOSE 5000
 
+# Ensure debug is off
+# ENV FLASK_ENV=production
+# ENV FLASK_DEBUG=0
+
 # Run the Python Flask app
-CMD ["python3", "server.py"]
+# CMD ["python3", "server.py"]
+CMD ["gunicorn", "-k", "gthread", "-w", "1", "--threads", "5", "-b", "0.0.0.0:5000", "server:app"]
+

addition_malabr/benchmark_data/10kb.txt

@@ -0,0 +1,2 @@
+question: what is my name ? 
+context: QxSFGeAVDHqAoUjLuNKuHzcSEVexApqTlLwEJWgYRBNsprrkAlarZKeBHjvADNbOwFzftOGLgeFuLjBRIzSyjPeqRYgytlgQhDgUcaZBFxRlmShyXMuNoFlXOBKTwfyQJepRLNrOVvHfKFUbtNZkvvlayyFYsVYZrerMkpnqLUgJzpxtiCUKlcflMzEmEDgHxXFCCgVWqGTCTGzoGxSmvADUcuJZisbBupKSQmFQOTkcyDtLczmtnEtzsUGNgJFuFdStWgrosPWpTAfsTokyGTrEeEdylnpQDvffatnMXUjDOEmVkogjOKQJauYTEiVaAKybHtcafzlwhbZoklVxumATxfyquHiKAPsvHEnUyFlKITkOLbVzMkMxSxRRtlIwKUluTlJWEsMpKLdGKzPzVvFbNDvPuXcAhmIjDXUjxwGFGPzSXBfKDHDxRkPeBfEAzGOEpqRpDmdwFjoFpMewvwGmInjTAHvAppitQNVhWIldWvlOMoqZopuRnlAzSguUGrPzBIOdZRJHHJbZgiwEztftdgGWyQydicUZcRRcmxGunHkfzwHOAfVOvlIJnDxesvPoAKwDwvPbAJnYGaXaXDnHoiNNbEdpssJbMfuMLOAoAdVYCsWXjDzIdVePamvNJrwNCATEovalZyEFailDEUcaVasEJzpNyRnTUDsXiIGMwHrxQdFHVWftgjguZTpCMWEFRCxssvUDkUVLYeZYwhTABtQPSlHgXWMAfxcDUHQsJrqKzhEdGnUSEhlvGVwXSwAzNwEOuRsKnDtYtpRSijUWycInPffBaRcmQEOEUosxUExGeKvvXBpepkXqbFthLPFctwyoFRAJJDIAuesuXTjYUaoewmUrVZwIXOyTQwIbBmgxJhgmVtTVuYXpyuEuBmQHTvdjvTeSFkNvLuyactxPcRTruIZeQQBpUPpnWbokLidqrXCaXLmYKsENKARhOmWdcksgrievxYWdhJocMeAxJDDbEdFGvfbxSvlPPKonrjqBFLwaQcKGAHXFWmQvJrkqdFXbNWUIWoCdVDzDBRoMdtOEnZpYtVoMogSKDsTEfSqDwoWglwQTLEbzBHsidVuMXZtGgDQgbRTamktVINrDYRUmndazgvvYsllyvwSqbFWBoQnQpvToxVhpzlMmzueMAyoTCchBaRWBjZhAeyoRpvwdfareLyPetbIgtfpdhxsSFqCMWjjEbrljpfAtorjYOrpheAkOIGETdfZfgkWnBYcuBapQZVKNzsatdtIpjoLzMXuyAkBdAoxGOUnUpvusLVDwyibnCOCtZSTuOGLjsvikCECIyfHDdMpCgtHpawKGKhZNWCmOZpoPsOytmuoaTvTdXsPjIRdtHeVhcUnLEHtYAKuGDmGePAIzUlRrXswBnUxWtkUygGpjYYvzMLdwTIyjuqoLMydUNEqpsjPfbgmTZYucrqhJYTUiOqXuskCpBFZsfoptbATPfkpwLvDhRDeBznMXvgVIpJLZPClmyzeHQDCzkaZFyPIKWWzazRbJcGQuHbDOpJvmRpbBBgnNKZEOccyMwoABGdVFhwTHlmTkbCyeYEFNPRDWzntJQrLEgwqNHBoOEnQqXwgBWILPyvzauhRtvASYFrSWdxDvqPpMmMzzezWBQKjSnmRMJXSMutYIRrmjebtLMmBiehTOTnVBTzFdEcZppdqnKIUhuyHXJhTMENgcqKJwdnFvRQQzyEpTuduoniIDotqJVzdcEWsAwKoVBGMzndxIvMLKpOxOpfGgIniUptXmJecIyICbrtepWIdmDPeSwXQngClCxpXnbFLPXGGgHuYssUaiYVwneCmQNVTLYlWgYlxgvJZDFEfbpwfaLNrpkVgtcjdMwEqpihmCNmTCAVKshTLhiEUzbZritoxwYtrahJyNGlwYgekgWiQvgxDxjIcTpYIimuGURvMDhBgNnkqiMipKpwUZXQGKcUiZjDYJqHZGUszGBsTxiAXiWBSMroqbguliIzEXXSmVPiMbbARlyySvsuzSkyNDUCQChsYwkimXphGgmwQlxYnzQEpByUrCxXvsOKUNCLUfguheRkzzLlBouvsNLLTMnrTNIdovuFsyhwawkfVGfiOuLSmnGbWFjtBXgYAGcMpNMGvkbaTYrZjGmHpFVFndNaaAThaPrGSCtUbaKxrSJDxwgyPlTftzYxGPXOeICvahFoDzhCBvzcYamQGcMYAGFyIoQYVIXWOdgNHsfBAzAPCWOYVzsQEdfnubPXUBTvwOGdpKKIBJVDalUOmmQXTAEIVftDSBskBdllMnavoeQWPgjWypYXYoZhamrCfihjdXyMkhwFPirlpptkOcrIZIlAEYuNxlVSFdkneIIdGcuBETOUESjajhuRSGhRazuQTfXSalGBHrYhdqRHEEIIDBxMyAlhqIFPMLWNwHshYDXDFtphqznUYdpYTMdFkdrMtrEMxnIuiFODexhSeuiJBUenTIDMSzqKJGgKrQtnMjCccScLCeiACPwHJJXcxKmoNBjKPdJKDDvGuzRnKIjHVAnKACcjojmNeBepmzEHxMCTULeyhoXTItqooBBrNDQAuVxbNgsUHYvuoDnAfoqtLMusVSyGOjhquzshBFenORTOLdmOxdBnRQYfzqWOWageLTziTkLASUrAOtACyYlADEdhgxDsUZNEUcgVqkAvRpdBSWDPOLdWLdMpDUtlkyuIlwRYSaaNcwjqcIeqbgyNuOEKqQqCJPwstcgzvIlZDchOkDhYkbNfChAXckVJnWkxMtPwgGggVnDQRplcOfqfIgDmNijaEDLcklGIttYzBpNagHzyAlKTxUsaYMmnrnFUQhnKVZHFGpDxqdWNBGaTNzOrsbiIrnbhUlhkIKwHjLfydjKoTwjVPWYAyPalpyelSQJCmAYEfVeiKOMHcdIlPJxRJeSJotXAkBMxECStNUgqeWOluKjkxjIWMaMiKpZVzYfzzHxrerSNsicfmHqCXGUcZfcOqaqHpBIRPNoCmqVrkUZRXfwcRDbABIXBEggDwqMbTCgEdQnYrJiHHFMIqFZjYpjAFvjUVrYPePjorzICMqGwpZcuwSDCVSbKejNKRPQFJTaJAgJsiAoZPIaVqWgZpBhJRNlTBwSeoInkGBzPCoeIaaIOvfopMUyvXENiWmglvwqfLCigVrCLSYNIwcAUdmTPBZkptyuVlLbGGYhmSvmCSLIIJKcDZzqwmVsvpNzozDVrDKqrgwuwvPqFZpUCkEnDelAUHecGVslpcjaCfySjvwLfuawodkNiEUWkVdRoppaoKgxoywQbpfwgcTxyvnpFNifAsDeROTIWIypCPHwTDTBoVisQdkRbTITPIDidYSVHqNYVAHHHHpUFrZjPMnYRZEWtvsoEYgBECzwzHUFZgMBHHKlXLSAuKrfrlnqNQEWQwuxckKtRrvotMcqKOnZWuRjsxHIxINfZVUFXVVJGscNxaUKxSeyYABKjVOxDzuulGnOckfmdXrnUkjgEhCKAhUliuZlhqcpbUMLYQXfaTyjXUOjfRaIMzCzFKEcAkGFGXPVUqOdMMcElqwTnBYZAsyEhXxmudmZosedvqskuCIeWrGNrlJmHTJTPAsLxPmfjYYBnBMWgFMNdRquMUwdoUkeoNxERsyhVWNAYzQFIySYwIyETfofjGfTrMJdPdTiFMZLAqLZipHrOWUBNyceYmhmrpsvHKUcQJRoszKlLzdlwHUjeumoNyjNtNeQDgHiuShTQHWlmjBRmWpOclufpHXBOYvQIQxhmwfPtEnntGlLUQfkGgnobwVpolLXWxyCznASFkrCGurWjrAfmbLYrbxHBdrNjDmdGTBrGMnJfTRCBBgLsXOIyVWbbUPFoKhnogZyRyNDOKaYRZRrhemKjPxzPMFyBzLSAlDOXicsIlFyiDgTqAlUbABFrWaMCPPyUpRyrTrwdROfEBqvIBMlqjzNszdLzGYkPGaBrzNPBbILDNKxnvzKMEbUdIxHcIWklIkpOtnDrEOrzHSYbdgQYsFAcVxRAvCVZBwpwCVICJskwtxDFnBdHbbukwsxMQxSZNhRrfPLzUvLXYmwpgUTXHROTTJoUqKPLyJxSinLcArNjwgGjzcAzHEEIhkjonZPwMvPuHylDExNhxvPZSAChamXbpBXdnqnhMQBLOFvhuzxkFfQslExmfbTSizDMpUOoRQfEMayfhXNHywJeDkVNBWABJTzJyGjGstMjNpskaRknLhDWphlHpyHsOviLpWQLSFxmWomrOuUPwWPrOyMYimiNzDcsCpVRaVcpcxIUpxOhdtjpIAMNHNGcpRLqxbNjWkTgWRRGhIsIFVKHjKSHHvFLCqxiCipYoBDuVNjisHckpnttVSeGjXArLfKFrSgndizoyOVlccukxYlEXVVWVoJXVGnGezSqlfEGrEoIzCazfhBBCoevgnqkkqisjyNPtujdzduQwEMZsQrMgnlmwrrdeVFPnQHmGWUDcrgvimNLJghjysoSXngfindzsfkhuCqaeEFpaXUAPbkpCZWUHHTSMUHCHLBqFQggKsAiHASWaWsMKKqXCvoOSOsXJnzobMtuwToQISSWvSqNXpLkFkiUtDvPWFTKnzUNhqqjrkLCPYCWcQSxclUBXmqayxpLVBAbAUJaAWRIEDDUNVQPGoEKFMIIugXBBMsphQMhNfMWygrabmEEcxMdPAeoOrehylvccWuVCRUJIdmloAmRJfTNrEGAfxfoSBsisynuMHILShMqdmypXZuEybzKgDAIRZLdxYbWDzRhJTjSHuzQnmdiShfdLYlcBIUJqbZHcKndyOpcyZPKPBjIAUhgKxTGxbIcTwJqnVzphwfeqSlNTZnPmksSoeVDXHmEWRXjqadhECcxTaAjMgcwZEaYNXOyQyWmFyOJCyilrhTEREXLUInzRxkOJEizXxWESAlpIxzxiTFbiPCnybSWJvDTJmIhCFwPKyBqjPkvjLgqNQXmqsqLyzjnujFrlHfHQhNcQDrLOnyCbMiHXELsAVwEiOOUvZmkKgAcuMjSxqQmcuUXbwFtklFelSDTzwAghDmngWhksYkCbtTJbzUutpFmvKELlvusNmSjtjZyQGZcgOyjeKBtXklylpOMPHEpZtyBteCuGcXCVsoKNpTxUalAMaAEDEfxoLCiuGmKTfQkBrEAVpuFJuxsqLxBKbLWyTedkBKGCOjsrbYBKiYTjwtpFiLdPUTAaGvenPJitxPuYmbHsOOPCOkHKPGcbxNcooWdfJDkMLKGaIIphUJzZFuqfPaumOMDemrYuSVyOMCFItyFkwXeDUPCZbXHxOrqtjIPZuJqQkfSkOjYsPWZuBlTLlvtgVXVEXfaECJChACnkNvKcQbQjNSZVhxPlNdNsfEBhJCdMQLYYSFLoxIgZaqEnGHZPUqPBLZERbckOWAWjZnHbHEjVSBWVkFRuAjQHanOoucVfXEayxBNsfcqyqLziYVBHQKZmXvmKzVFUNqJyrVrrgWQxsForJMlzUKDrgccgHcJlJlzeIzxwoFDOYQWJvYQywsnOcTgdHTeXcLTGaEqnwbDSqvKoRbaGelvEGpXKpjxmQbCRGrYvzrMCHEZHSXRCYNkHHWMlOXRDpRnHqxYPtNKNKesvqMHzPmzAvYgBclcIWrWraPqRSuHRVUxLyUgDUEiDOprdbxYInuMzxMFRZloXxgGyIexMYyXilMxwgbASTkECcMjTcLcfmaQMmpAxwEiazqraaYysPIFBhEnhXLarBUwdiQEXzGjgoDzRhckXqUScjkshuWgDUxEptVQbnBVyiikQUpvSujNsAxmvMeqdAGogEsUcGcOfOXaqSrwfIpvkTVoBANJjUXtZSmTqCnnVROBgYfIWkzIttmHgZzzfHrPFOiKsZFOcBualzXfSzEnRmMcmNiMZdozQzKKNjVdsLapCDzFaTyhsFXnvrQnCAYKNjafpCgeDkHumXQolngaKykdQIYNskISyMpKTKaNHHuZDswbhEmCbPbRoetegHfPfpfLxGFVcAjhTVSyWjtzuNxtDVdcBPgMsRJDyNmqGvovuoTHWqgLpqYmaKLIRuzETlyaEbXynJiZyRAxkKHMcDTHzdQEexSrMMGGJYdljInmfcrYgJQYDwgbnPKFPCWhcvrkShdaELpHAIUEjUgwfbTVsRdtVklmNyhYHKbUPwXieDjjzZNAhGKFTIcIoYJrJTvukXIPgHNzsldEyrCTWXnEDMxmmZmPTUwyJyRzQOSbntTyrvvxIYZRoHPqdBGMTFsBalROeHpYDxkdsSthxrNmoAPSSgaVplaBeYltPeqysGEYrWwpvltJKSTZFRfKRLpYeQXIEQuMtgzkULoyZuygIzNqobqvFvTWpttTcKXizHcraAUmbGdKInUYXvtEjrQTSgnNQRMPpvcVrMPvfOLTvilfiNSAVrepJYypDQjggwesGZcMBCyrzbfxMSXnhoVxuvcyAaiVAWdvIkarkUdECmdkorajzMeWCgjORsWrPPeTkSrjNVpUKvsIGJZqNpaieLzGtUqIaYnbLKpquJnkVFfFwUrjixdAnBLJQpQDiIZbMMQykZVLgQEeljTHxvusxxzPZJnyZNsGcECJgvBLBJzaDhEiyzHtaIYWKKeeeVkjgDDffniDAOhPfhiWhGufsKPnaCCiWTNTYXqcUUqayBfddxzBzGoPqIzdYjspMieKEBuxRcDUmIPiqqNJbRkVXkrTYYKgJnkmwyjGeoMlSjkJAGiDHhdYQEvPwqQgAYUYmkapBgyNfUhZhVuYoyJlVxTrYYNYZIEexiNRxEMCZJVfBmDCHvLGIJuvuApIpPiJvFGRyuHwAhFSQHNsHJyKyQeQUjAXuCCOfLNdzxChDelChBBlUaJpALYgvERmcowCQwVUyoXMSGWVJRlURAsuzlZuUrjlfjwpQdjnTkxrPwxovaUvVFQGMspAyxsIfdBGfNLYYhhKVwxVzvGxvuYRzdCLvrlXQsUlMlPRCqwLyKFGLqqaoKXWWHYoufjsbhzRMLpWaxWyIADnGkrTKNkyxKmVzlropwoguWlncTOHdCQLBihBEBiqaUhpLyNKEZJNfRCuqNHwmlPTKKlUWxHkPxQQPNkRVZlVQwVjMAwJICFCKeQhyEWLiaYIFGFxZclQZXKuGziMDFLtWSfnlOLjSxRoJHSWyWDGQDIFriHjCJsVEWdbvWOsDPAdfMhryDelguUzfhfwMtSNgGROhoHApCSrSPYWdAfNaCoGskTyuOqjwSEVJEdOxXkKVgxfhXiQDvdehkrVEsHjlXFGMxkIpOaCtrEGIRagVFBnjoakfqbLghhKqBzPsdBCiOMrUEWjXbgSmuPXyjzEsEGiWxgbPYCsPRWESlmHeusnKeRHjLAcRMctFXUNtnnOprUKAzWHlTylwDnZTPaxIjDagOpUPcYcwivspPxXzCmAatNkghNfvbpzLeyArJCyOfumFKacFAcvqTdICoDidNrwjpZjehCHfeMOoOtwrseFYGnDWDKegSpDsklwJimhWrUImQdwPqyndZsJFBOyJbjzzSPpPPVAlTEOeUkklLRMZIsbUYlkhChZxsIuboptqsgOggmuSczHCaorWvPfIUZhFDBSYWjnAkfRqTkdTKsNbsUwbIJOxEkarsYPdpIYuGrdPzBPNrfffhTpYcWzirMwrVuWwRMncNPSZWZUqelVIIJDXmHKsHFNvUJMqxwNLRztGlXNNMcYRmgyLEGpCAUXDeVimlknibPgnSGOjeJeUlQaSLQnhTLcjbHnafdpWiEgiNKyuyAoCyqaceqkWAoNBFcZGPrSeqQwsnZHPSnCOzEEvRnBhWCYyjvflCKeVlIQptXkVZQcUsiJpwMUhKMwgPuNqbQbQMwTQiAYlhjTQKuPBnckHWIwxdJaaKeBRsTvdbAuvYldvRgVNpWOjunTlKRnFwGLmudLPTMZWKVpTpxshUyPPJChPBNQFRDHiHpgoDJvITgBXJgJycObGOzATnykJccnPLYgozTAoQhApbwBAWptuXMpPiShFfxTfwoEMPCfJDpiQaKajVlbWwBGQpTeOWxdGYhshhaOWMDZMQLVGahnKUjeucMkDXdSqvHdHjsxtfSYpjXdxpecmrtEPWTWMMrWFHSwdWInvQGcBmMwOAgUJTXDMcysBCujssHRPrBAAVArBHfVlIOIuXmKemGFEqHoLPOpxfIsnfRufLiFPeamphduuVYgmdHMKYcZhAXAsirjcIYMHTqxDHDnFyvFKvsFsrbYEkpXrjncYXPhraEMFXxcmTZusjYjyurmLGJpqPBquShtWZPZeSOWJuTSaMnKWyESNdBEXpMMGSYdgjSyHJsmAaauBPxGfXWgyXXJHMUSLTvQlnlZrCupOmudDTGwHsCLPoJQqTRWtktDpxjfVJngbvndImnLdsqlRLKqwXMWHkHrFUQmnwZaxjZusUCXCgTptikQfWBvQpPOubfuVkwjvxEWqVuwgKKiiibuSHjSYUYNJpbpFFLqyNFrIBjFwRaywwRppmRntKbhAxpbGNTxeTGLpIWsJMYdeZYGcAvbrTqLQAjwRwTDYVcQWLzWPMwcUojqdPFyFiHXGMHchJrYiMdxzuMQrhOMOlNfyhaYGcvVouueQDtGnNsXTYtROXdUMWmGoDCDbBckCihSLMVBeKHawUAbPSvSrBIpNEPuVbSsdvsjaXNfPGRQfJAuPxEuaKavNMUEqwazsyYJEsMLpQFWdyMbiLWCmURxBnCbDOvDhlYiMjstBclqXebniFRfjjGOQZApoluqzSsBEnZpwYEeMXupTGQCWPFGRYYHQRmgBidMriZWXqyNStdURpATBXEhXQpdCRRokfvOnehkaMgflTTtvgryaQuSIweFzXaQIZvkzsvkoynUJhpinGftuQdNzDQWEKRVzFkmATWpAoislybxxpGwvCvOtptKmpLBvlZHAjlpEEfGVwRXtcNIpTzJvatJMHOWACmdlIUzKthxljmArhFMwmLxNUeojCiAaioqQyofKVnhkNdKHoGzfbXRKfIKXPKjpuaSVpYPTQbJaJZaMPQYDgghvBZrLbgHxEuyjMuvSdNUZePVaAvxGVJhJdSnTLxxerrDOjlpqktHbxDLAQJAZUIGTYbGckuOARMSnkYNvPTOmqsqAHZmDaMaqbbMnWaMTcRKjzwplznGUsotzvqNaiHdLNMLCPJMStJihRJxTyMDKCkGivQmSXsYmvFPlYhrvSbOQKZTRanXikEwCScYuyQLcGuWPUCSdDYqdDnaoyTgatrLOlvcYMciSXUeDdOAaeMtupcpdXYaDtidmutupnxZghHtNopnDCILQLqGSRpaqMOpILRfWeZvOZZEvNDmOTZBlOFftWSxXwuMhUYlBtDwDPOJkArntXVmYpPhDPkNvYgPCErIaqfECSWAMVEsydOtOJnDMyyXDbDwMQLJcNmMvhKfdbwfDZHeLzjYTNngvYJbinCjmLaDuuiKxADlJEWRNpKpmWMRFUlswyIbxLfDkagYPTObxnUVLrwpXNjMDOjKqJGlqfcbYlEhvecxidAzylLpsecBcaVBSnCiNzQTsKLyFWorBTfoPMcQseeulZxTomnEDmpbnuHqWBUfXUwNJJZdWdgQfFKJkEmafhEgnohOiilZkpQSvOWYIvCzCHLAkfyJJzQkaXoMzJFgHCEToPJEjxRlPwiJNDiMhkbhLdJKhmtikPdQHDPvxjCxOTmSjXRomXFFsPZeucenJPYzzDHiFlXOmMYvNFeLovcASeuXXEnEAIEWAVrmUklihroZOJbRbbqHewKFXWdXCsArwMwtzMJGqFlFisygQkmytiivGmYXeWxYPKxRkVQShSXkLjNmXJtiURVAdJWaPAZVLYIKRkxVaUFTPqlDCqVpMdbKxFNsSigMIkXgtqnAUywFlysoYOtyNSxxmYuTrJZKmcGZCIMastmMtSywMcIohaJDlvCrEDhERIqucxELBjzIpJiFoXbVjRfyhQJUgwmqGYtVZigIuaVWHwWTJPSMOouMVlrLygKTvCCSpqhvcrDbSGuSCYnjyRkhfetAukWlWRHmQSVaYSPpUJjsKELImdNfjbDgHcVagohDanHCyUvISYsabZTHqfLmQDqMJncLDGYuelFNDBblGJVDmsOtmZVfmLQyIDMxTnavGcmsXERRNdmnaekhzdhnyjGXqWMAuKdUIRJxQwuBMUMmKBwqhxpPFfTLEpUcatgAYqXtXbYodWxHFVSeXg

addition_malabr/modelserver/server.py

@@ -1,69 +1,70 @@
 from flask import Flask, request, jsonify
 import time
+from threading import Lock
 import tensorflow as tf
 import torch
 from transformers import pipeline
+import os
 
 app = Flask(__name__)
 
 # Global variable for QA pipeline.
 qa_pipeline = None
+qa_pipeline_lock = Lock()
 
 @app.route('/load_model_bert', methods=['POST'])
 def load_model_bert():
-    global qa_pipeline
-    try:
-        qa_pipeline = pipeline(
-            "question-answering",
-            model="csarron/mobilebert-uncased-squad-v2",
-            tokenizer="csarron/mobilebert-uncased-squad-v2"
-        )
-        app.logger.info("Model loaded successfully.")
-        return jsonify({"status": "Model loaded successfully."})
-    except Exception as e:
-        app.logger.error("Failed to load model", exc_info=True)
-        return jsonify({"error": f"Failed to load model: {str(e)}"}), 500
+    with qa_pipeline_lock:
+        global qa_pipeline
+        try:
+            qa_pipeline = pipeline(
+                "question-answering",
+                model="csarron/mobilebert-uncased-squad-v2",
+                tokenizer="csarron/mobilebert-uncased-squad-v2"
+            )
+            app.logger.info("Model loaded successfully.")
+            return jsonify({"status": "Model loaded successfully."})
+        except Exception as e:
+            app.logger.error("Failed to load model", exc_info=True)
+            return jsonify({"error": f"Failed to load model: {str(e)}"}), 500
 
 @app.route('/infer_single_bert', methods=['POST'])
 def infer_single_bert():
-    global qa_pipeline
-    if qa_pipeline is None:
-        return jsonify({"error": "Model not loaded. Please call /load_model_bert first."}), 400
-    data = request.get_json()
-    if not data or 'question' not in data or 'context' not in data:
-        return jsonify({"error": "Payload must contain 'question' and 'context' fields."}), 400
-    try:
-        prediction = qa_pipeline({
-            'question': data['question'],
-            'context': data['context']
-        })
-        return jsonify(prediction)
-    except Exception as e:
-        app.logger.error("Inference error", exc_info=True)
-        return jsonify({"error": f"Inference error: {str(e)}"}), 500
+    with qa_pipeline_lock:
+        global qa_pipeline
+        if qa_pipeline is None:
+            return jsonify({"error": "Model not loaded. Please call /load_model_bert first."}), 400
+        data = request.get_json()
+        if not data or 'question' not in data or 'context' not in data:
+            return jsonify({"error": "Payload must contain 'question' and 'context' fields."}), 400
+        try:
+            prediction = qa_pipeline(question=data['question'], context=data['context'])
+            return jsonify(prediction)
+        except Exception as e:
+            app.logger.error("Inference error", exc_info=True)
+            return jsonify({"error": f"Inference error: {str(e)}"}), 500
 
 @app.route('/infer_batch_bert', methods=['POST'])
 def infer_batch_bert():
-    global qa_pipeline
-    if qa_pipeline is None:
-        return jsonify({"error": "Model not loaded. Please call /load_model_bert first."}), 400
-    data = request.get_json()
-    if not data or not isinstance(data, list):
-        return jsonify({"error": "Payload must be an array of objects with 'question' and 'context' fields."}), 400
-    results = []
-    for idx, item in enumerate(data):
-        if 'question' not in item or 'context' not in item:
-            results.append({"error": f"Item at index {idx} is missing 'question' or 'context'."})
-        else:
-            try:
-                prediction = qa_pipeline({
-                    'question': item['question'],
-                    'context': item['context']
-                })
-                results.append(prediction)
-            except Exception as e:
-                results.append({"error": f"Inference error at index {idx}: {str(e)}"})
-    return jsonify(results)
+    with qa_pipeline_lock:
+        global qa_pipeline
+        if qa_pipeline is None:
+            return jsonify({"error": "Model not loaded. Please call /load_model_bert first."}), 400
+        data = request.get_json()
+        if not data or not isinstance(data, list):
+            return jsonify({"error": "Payload must be an array of objects with 'question' and 'context' fields."}), 400
+        results = []
+        for idx, item in enumerate(data):
+            if 'question' not in item or 'context' not in item:
+                results.append({"error": f"Item at index {idx} is missing 'question' or 'context'."})
+            else:
+                try:
+                    prediction = qa_pipeline(question=item['question'], context=item['context'])
+                    results.append(prediction)
+                except Exception as e:
+                    results.append({"error": f"Inference error at index {idx}: {str(e)}"})
+        return jsonify(results)
+
 @app.route('/train_model', methods=['POST'])
 def train_model():
     try:
@@ -112,4 +113,5 @@ def analyze():
     return jsonify({"message": "OP server is working"})
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=5000, debug=True)
+    debug = os.getenv("FLASK_DEBUG", "0") == "1"
+    app.run(host='0.0.0.0', port=5000, debug=debug)

addition_malabr/modelserver_uds/app/router.py

@@ -20,7 +20,7 @@ def route(conn: socket.socket, payload: Payload):
 
     handler = ROUTES.get(label)
     if handler:
-        logger.debug(label)
+        # logger.debug(label)
         handler(conn, payload)
     else:
         logger.error(f"Unknown label: {label}")