From 6c3a3de8dc36ca8792a69150023e08ae0751ec32 Mon Sep 17 00:00:00 2001 From: SengTak Date: Fri, 28 Feb 2025 20:35:37 +0800 Subject: [PATCH 001/112] chore: update environment configuration, Dockerfile, and requirements; change port to 8080 --- .env_example | 4 ---- Dockerfile | 8 +++++--- README.md | 5 ++++- app/config/base_config.py | 2 +- requirements.txt | 4 +++- 5 files changed, 13 insertions(+), 10 deletions(-) delete mode 100644 .env_example diff --git a/.env_example b/.env_example deleted file mode 100644 index 47257fa..0000000 --- a/.env_example +++ /dev/null @@ -1,4 +0,0 @@ -PORT=3000 -ENV=development -DEBUG=True -PYTHONUNBUFFERED=1 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0bb80fd..8ef1edf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,11 +7,13 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY ./requirements.txt /app/requirements.txt WORKDIR /app -RUN pip install --upgrade pip -RUN pip install -r requirements.txt +#RUN pip install --upgrade pip +RUN pip install --trusted-host pypi.python.org -r requirements.txt COPY . /app -EXPOSE 3000 +EXPOSE 8080 CMD [ "python", "manage.py", "run" ] + +#ENTRYPOINT ["python", "webhook.py"] diff --git a/README.md b/README.md index e116f47..038cc6c 100644 --- a/README.md +++ b/README.md @@ -88,4 +88,7 @@ This repository is licensed under the Apache License, Version 2.0. See [LICENSE](./LICENSE) for the full license text. ## Issues -For any issues, please reach out to one of our Customer Success team members. \ No newline at end of file +For any issues, please reach out to one of our Customer Success team members. + +## REFs +* [nlp skill](https://docs.soulmachines.com/skills-api/getting-started/nlp-adapter-skill) \ No newline at end of file diff --git a/app/config/base_config.py b/app/config/base_config.py index 24ea36e..2ca6425 100644 --- a/app/config/base_config.py +++ b/app/config/base_config.py @@ -5,7 +5,7 @@ class BaseConfig(BaseSettings): Base application configuration """ - port: int = 3000 + port: int = 8080 env: str debug: bool diff --git a/requirements.txt b/requirements.txt index 690848d..a713fab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,6 @@ fastapi Flask[async] Python-DotEnv smskillsdk -uvicorn \ No newline at end of file +uvicorn +vertexai +google-cloud-aiplatform>=1.38 \ No newline at end of file From 8f48c4ddf92732be5245ee4157693d1367d929c4 Mon Sep 17 00:00:00 2001 From: SengTak Date: Fri, 28 Feb 2025 20:52:59 +0800 Subject: [PATCH 002/112] chore: update Dockerfile to use slim Python image and add pydantic to requirements --- Dockerfile | 2 +- requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ef1edf..34ea70f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.0 +FROM python:3.11-slim ENV VIRTUAL_ENV=venv RUN python3 -m venv $VIRTUAL_ENV diff --git a/requirements.txt b/requirements.txt index a713fab..d3621f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ Python-DotEnv smskillsdk uvicorn vertexai -google-cloud-aiplatform>=1.38 \ No newline at end of file +google-cloud-aiplatform>=1.38 +pydantic \ No newline at end of file From 6104b4d4b3967ffe72f7461f5eb07b608565f6b5 Mon Sep 17 00:00:00 2001 From: SengTak Date: Fri, 28 Feb 2025 21:15:02 +0800 Subject: [PATCH 003/112] fix: specify pydantic version constraint in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d3621f9..15a50e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ smskillsdk uvicorn vertexai google-cloud-aiplatform>=1.38 -pydantic \ No newline at end of file +pydantic<2 \ No newline at end of file From bab6cc431cce184c587e74cae68b006f10413e48 Mon Sep 17 00:00:00 2001 From: SengTak Date: Fri, 28 Feb 2025 22:02:38 +0800 Subject: [PATCH 004/112] feat: implement module patching for FixedConversationHistory and update imports --- app/app.py | 50 ++++++++++++++++++++++++++++++++ app/mocks/mock_request.py | 2 +- app/services/fake_nlp_service.py | 2 +- requirements.txt | 4 +-- 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/app/app.py b/app/app.py index ca0e93e..6698090 100644 --- a/app/app.py +++ b/app/app.py @@ -1,3 +1,53 @@ +#for pydantic > 2 +""" +import sys +import importlib.util +import importlib.machinery +import types +from typing import List +#from pydantic import RootModel + +# Define our fixed class +class FixedConversationHistory(RootModel): + root: List # Will be properly typed later + +# Function to patch the module +def patch_api_module(): + # Get the spec + spec = importlib.util.find_spec('smskillsdk.models.api') + if not spec: + raise ImportError("Module smskillsdk.models.api not found") + + # Create a new module object + module = types.ModuleType('smskillsdk.models.api') + + # Add it to sys.modules early + sys.modules['smskillsdk.models.api'] = module + + # Load the source code as a string + source = spec.loader.get_source('smskillsdk.models.api') + + # Modify the source code to remove or fix the problematic class + modified_source = source.replace( + "class ConversationHistory(BaseModel):", + "# Original ConversationHistory commented out" + ).replace( + " __root__: List[HistoryItem]", + " # __root__: List[HistoryItem]" + ) + + # Compile and execute the modified source + code = compile(modified_source, spec.origin, 'exec') + exec(code, module.__dict__) + + # Now inject our fixed class + module.ConversationHistory = FixedConversationHistory + + return module """ + +# Patch the module before anyone else imports it +#patched_module = patch_api_module() + from fastapi import FastAPI from .views import skill diff --git a/app/mocks/mock_request.py b/app/mocks/mock_request.py index ca25648..7dd6766 100644 --- a/app/mocks/mock_request.py +++ b/app/mocks/mock_request.py @@ -4,7 +4,7 @@ """ from typing import List -from smskillsdk.models.api import Memory, MemoryScope, Intent +from smskillsdk.models.common import Memory, MemoryScope, Intent def mock_init_actions(): diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 53a10d1..c41bd8c 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,6 +1,6 @@ from fastapi import HTTPException from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions -from smskillsdk.models.api import MemoryScope +from smskillsdk.models.common import MemoryScope class FakeNLPService: first_credentials: str diff --git a/requirements.txt b/requirements.txt index 15a50e1..2e6e93c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ Flask[async] Python-DotEnv smskillsdk uvicorn -vertexai -google-cloud-aiplatform>=1.38 +#vertexai +#google-cloud-aiplatform>=1.38 pydantic<2 \ No newline at end of file From df61a048987dc853858f9d278225bbf2472d592f Mon Sep 17 00:00:00 2001 From: SengTak Date: Fri, 28 Feb 2025 22:21:49 +0800 Subject: [PATCH 005/112] fix: set default values for env and debug in BaseConfig --- app/config/base_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config/base_config.py b/app/config/base_config.py index 2ca6425..a4a15c1 100644 --- a/app/config/base_config.py +++ b/app/config/base_config.py @@ -6,8 +6,8 @@ class BaseConfig(BaseSettings): """ port: int = 8080 - env: str - debug: bool + env: str = "DEVELOPMENT" + debug: bool = True class Config: env_file = ".env" From da165a694a8250241300c1fde36032e0cf9de785 Mon Sep 17 00:00:00 2001 From: SengTak Date: Fri, 28 Feb 2025 22:40:03 +0800 Subject: [PATCH 006/112] fix: add debug print statement for skill config in init function --- app/views/skill.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/views/skill.py b/app/views/skill.py index 75afac1..a4c647a 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -32,6 +32,8 @@ async def init(request: InitRequest): # 1. Extract relevant data skill_config = request.config + print("skill config body:", skill_config) + # 1a. Extract relevant credentials from config credentials = itemgetter("first_credentials", "second_credentials")(skill_config) From df9ae253c2ec159170fe42eae8b9587798483b43 Mon Sep 17 00:00:00 2001 From: SengTak Date: Fri, 28 Feb 2025 23:02:42 +0800 Subject: [PATCH 007/112] fix: replace credential extraction with hardcoded values in init function --- app/views/skill.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/views/skill.py b/app/views/skill.py index a4c647a..af0eca4 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -35,7 +35,8 @@ async def init(request: InitRequest): print("skill config body:", skill_config) # 1a. Extract relevant credentials from config - credentials = itemgetter("first_credentials", "second_credentials")(skill_config) + #credentials = itemgetter("first_credentials", "second_credentials")(skill_config) + credentials = ("me","you") # 2. Make request to third party service to initialize # any configuration, data storage, or pre-training on the NLP service before executing this Skill From 0b7e8cf4f1cec85120f26ca0c04ea5f8fdfa7bf2 Mon Sep 17 00:00:00 2001 From: SengTak Date: Fri, 28 Feb 2025 23:09:52 +0800 Subject: [PATCH 008/112] fix: replace dynamic credential extraction with hardcoded values in session function --- app/views/skill.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/views/skill.py b/app/views/skill.py index af0eca4..2e5f4f4 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -59,7 +59,8 @@ async def session(request: SessionRequest) -> SessionResponse: session_id, skill_config, skill_memory = attrgetter("sessionId", "config", "memory")(request) # 1a. Extract relevant credentials from config - credentials = itemgetter("first_credentials", "second_credentials")(skill_config) + #credentials = itemgetter("first_credentials", "second_credentials")(skill_config) + credentials = ("me","you") # 2. Make request to third party service to initialize session-specific resources fake_nlp_service = FakeNLPService(*credentials) From f197f0305f954c38529db0089f2cda15e3fb4f44 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sat, 1 Mar 2025 07:46:52 +0800 Subject: [PATCH 009/112] fix: update .gitignore to include experimental test directory --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1774af1..841f362 100644 --- a/.gitignore +++ b/.gitignore @@ -128,4 +128,7 @@ dmypy.json # Pyre type checker .pyre/ -.DS_Store \ No newline at end of file +.DS_Store + +# experimental +.tests/ \ No newline at end of file From 2fff294f4261f58966fd9c636d22bffbbfbccd2d Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 2 Mar 2025 09:51:52 +0800 Subject: [PATCH 010/112] refactor: replace mock functions with actual implementations in FakeNLPService and add new gemini service --- app/services/fake_nlp_service copy.py | 61 ++++++++++ app/services/fake_nlp_service.py | 9 +- app/services/gemini.py | 157 ++++++++++++++++++++++++++ requirements.txt | 5 +- 4 files changed, 226 insertions(+), 6 deletions(-) create mode 100644 app/services/fake_nlp_service copy.py create mode 100644 app/services/gemini.py diff --git a/app/services/fake_nlp_service copy.py b/app/services/fake_nlp_service copy.py new file mode 100644 index 0000000..c41bd8c --- /dev/null +++ b/app/services/fake_nlp_service copy.py @@ -0,0 +1,61 @@ +from fastapi import HTTPException +from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions +from smskillsdk.models.common import MemoryScope + +class FakeNLPService: + first_credentials: str + second_credentials: str + + def __init__(self, first_credentials, second_credentials): + self.first_credentials = first_credentials + self.second_credentials = second_credentials + self.__authenticate() + + def __authenticate(self): + """ + Example of using credentials to authenticate + """ + + if (not (self.first_credentials and self.second_credentials)): + raise HTTPException(status_code = 401, detail = "Unauthenticated") + print("Authenticated!") + + def init_actions(self): + """ + Example of initializing Skill-specific actions on third party NLP call + """ + + return mock_init_actions() + + def init_session_resources(self, session_id: str): + """ + Example of initializing resources with third party NLP call + """ + + return mock_init_resources(session_id) + + + def persist_credentials(self, session_id: str): + """ + Example of persisting credentials during session endpoint with third party NLP call + """ + + credentials = { + "first_credentials": self.first_credentials, "second_credentials": self.second_credentials + } + + credentials_memory = { + "key": "credentials", + "value": credentials, + "session_id": session_id, + "scope": MemoryScope.PRIVATE, + } + + return credentials_memory + + def send(self, user_input: str): + """ + Example of sending input to the third party NLP call + """ + + return mock_get_response(user_input) \ No newline at end of file diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index c41bd8c..ac76bae 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,5 +1,6 @@ from fastapi import HTTPException -from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions +#from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions +from gemini import get_response, init_resources, init_actions from smskillsdk.models.common import MemoryScope class FakeNLPService: @@ -25,14 +26,14 @@ def init_actions(self): Example of initializing Skill-specific actions on third party NLP call """ - return mock_init_actions() + return init_actions() def init_session_resources(self, session_id: str): """ Example of initializing resources with third party NLP call """ - return mock_init_resources(session_id) + return init_resources(session_id) def persist_credentials(self, session_id: str): @@ -58,4 +59,4 @@ def send(self, user_input: str): Example of sending input to the third party NLP call """ - return mock_get_response(user_input) \ No newline at end of file + return get_response(user_input) \ No newline at end of file diff --git a/app/services/gemini.py b/app/services/gemini.py new file mode 100644 index 0000000..3ac653c --- /dev/null +++ b/app/services/gemini.py @@ -0,0 +1,157 @@ +""" +These functions use Promises and setTimeouts to mock HTTP requests to a third part NLP service +and should be replaced with the actual HTTP calls when implementing. +""" + +#ref https://docs.soulmachines.com/skills-api/getting-started/nlp-adapter-skill#advanced-concepts + +from typing import List +from smskillsdk.models.common import Memory, MemoryScope, Intent +import vertexai +from vertexai.preview import rag +from vertexai.generative_models import GenerativeModel, Part, FinishReason, Tool, Content +import vertexai.preview.generative_models as generative_models +from vertexai.preview.generative_models import grounding +from typing import List, Optional + + +def get_nonstreaming_text_response (response): + return response.candidates[0].content.parts[0]._raw_part.text + +safety_settings={ + generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_NONE, + } +generation_config = { + "max_output_tokens": 8192, + "temperature": 0.3, #0.5, + "top_p": 0.9, #0.5, #0.5 better than 0.95 + "top_k": 40, +} + +MODEL_STR = "gemini-1.5-flash-002" + +system_instruction = ["""You are an expert and customer fronting service agent for an Association called NS Chinese Chamber of Commerce. + You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. + Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. + DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers."""] + + + +class Chatbot: + def __init__(self, history: Optional[List["Content"]] = None, model: Optional[str] = "gemini-1.5-flash-002", use_search=False): + self.model = GenerativeModel( + model, + system_instruction=system_instruction) + self.chat = self.model.start_chat(history=history) + self.grounding_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) + + """ + def use_rag_tool(self, user_prompt): + return self.chat.send_message( + user_prompt, + tools=[tool], + generation_config=generation_config, + #safety_settings=safety_settings, + stream=False + )""" + + def use_search(self, prompt): + return self.chat.send_message( + #f"Contexts: {contexts}. Message from User: {user_prompt}", + [prompt], + tools=[self.grounding_tool], + generation_config=generation_config, + #safety_settings=safety_settings, + stream=False + ) + + + + def generate_response(self, user_prompt=""): + #prompt = user_prompt + prompt = user_prompt + + response = self.use_search(prompt) + + return get_nonstreaming_text_response(response) + +vertexai.init(project="neuralnet-manforce", location="us-central1") + +class Agent: + def __init__(self): + self.chatbot = None + + def allocated_resources(self): + self.chatbot = Chatbot() + +agent = Agent() + +def init_actions(): + """ + Example of an action performed by the Initalize ednpoint + """ + agent.allocated_resources() + print("resource initialized. . .") + + +def init_resources(session_id: str) -> List[Memory]: + """ + Example of an action performed by the Session ednpoint + """ + + private_memory = Memory(**{ + "session_id": session_id, + "name": "private json memory", + "value": { "example": "object" }, + "scope": MemoryScope.PRIVATE, + }) + public_memory = Memory(**{ + "session_id": session_id, + "name": "public string memory", + "value": "This is to be persisted", + "scope": MemoryScope.PUBLIC, + }) + + return [private_memory, public_memory] + +def get_response(user_input: str): + """ + Example of an action performed by the Execute ednpoint + """ + + print(f"User said: {user_input}") + + # Response to be spoken by your Digital Person + response = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." + + cards, intent = None, None + """ + # Add your Cards as required + cards = { + "card": { + "type": "image", + "data": { + "url": "https://placekitten.com/200/200", + "alt": "An adorable kitten", + }, + }, + } + + # Add your Intent as required + intent = Intent( + name="Welcome", + confidence=1, + )""" + + # If applicable, add your conversation annotations to see metrics for your Skill on Studio Insights + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + } + + return response, cards, intent, annotations \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2e6e93c..6e7a503 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ Flask[async] Python-DotEnv smskillsdk uvicorn -#vertexai -#google-cloud-aiplatform>=1.38 +vertexai +google-cloud-aiplatform>=1.38 +#google-genai pydantic<2 \ No newline at end of file From 61d5ae6003b571101c56a9cdc230fd7708ab6a25 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 2 Mar 2025 10:08:28 +0800 Subject: [PATCH 011/112] refactor: replace gemini service with gemini_agent and update imports in FakeNLPService --- app/services/fake_nlp_service.py | 2 +- app/services/{gemini.py => gemini_agent.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename app/services/{gemini.py => gemini_agent.py} (100%) diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index ac76bae..2fd4af0 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,6 +1,6 @@ from fastapi import HTTPException #from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions -from gemini import get_response, init_resources, init_actions +from gemini_agent import get_response, init_resources, init_actions from smskillsdk.models.common import MemoryScope class FakeNLPService: diff --git a/app/services/gemini.py b/app/services/gemini_agent.py similarity index 100% rename from app/services/gemini.py rename to app/services/gemini_agent.py From 911a10fb1ce0f5570ab215f3a11364236a2bc8f6 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 2 Mar 2025 10:14:01 +0800 Subject: [PATCH 012/112] test --- app/services/fake_nlp_service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 2fd4af0..575ad80 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -3,6 +3,9 @@ from gemini_agent import get_response, init_resources, init_actions from smskillsdk.models.common import MemoryScope + +#test + class FakeNLPService: first_credentials: str second_credentials: str From 90ca22a0ef970791017bb11a287d2ef4a2db8f2a Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 2 Mar 2025 10:18:08 +0800 Subject: [PATCH 013/112] refactor: move gemini_agent to mocks and update imports in FakeNLPService --- app/{services => mocks}/gemini_agent.py | 0 app/services/fake_nlp_service.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename app/{services => mocks}/gemini_agent.py (100%) diff --git a/app/services/gemini_agent.py b/app/mocks/gemini_agent.py similarity index 100% rename from app/services/gemini_agent.py rename to app/mocks/gemini_agent.py diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 575ad80..b980806 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,6 +1,6 @@ from fastapi import HTTPException #from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions -from gemini_agent import get_response, init_resources, init_actions +from ..mocks.gemini_agent import get_response, init_resources, init_actions from smskillsdk.models.common import MemoryScope From 5080fc8546fcf388548e630479501e73bd5a121f Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 2 Mar 2025 23:05:23 +0800 Subject: [PATCH 014/112] feat: add welcome message and response handling in FakeNLPService --- app/mocks/gemini_agent.py | 21 ++++++++++++++++++--- app/services/fake_nlp_service.py | 14 ++++++++++---- app/views/skill.py | 9 +++++---- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 3ac653c..3f8f653 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -39,6 +39,9 @@ def get_nonstreaming_text_response (response): DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers."""] +BOT_WELCOME_MESSAGE = "Hello! I am your virtual assistant 小美. How can I assist you today?" + + class Chatbot: def __init__(self, history: Optional[List["Content"]] = None, model: Optional[str] = "gemini-1.5-flash-002", use_search=False): @@ -117,6 +120,18 @@ def init_resources(session_id: str) -> List[Memory]: return [private_memory, public_memory] +def get_welcome_response(): + # standard welcome message + response = BOT_WELCOME_MESSAGE + + intent = Intent( + name="Welcome", + confidence=1, + ) + + cards, annotations = None, None + return response, cards, intent, annotations + def get_response(user_input: str): """ Example of an action performed by the Execute ednpoint @@ -127,7 +142,7 @@ def get_response(user_input: str): # Response to be spoken by your Digital Person response = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." - cards, intent = None, None + cards, intent, annotations = None, None, None """ # Add your Cards as required cards = { @@ -144,7 +159,7 @@ def get_response(user_input: str): intent = Intent( name="Welcome", confidence=1, - )""" + ) # If applicable, add your conversation annotations to see metrics for your Skill on Studio Insights annotations = { @@ -152,6 +167,6 @@ def get_response(user_input: str): "conv_id": intent.name, "conv_intent": intent.name, "conv_type": "Entry", - } + }""" return response, cards, intent, annotations \ No newline at end of file diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index b980806..850820e 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,7 +1,8 @@ from fastapi import HTTPException #from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions -from ..mocks.gemini_agent import get_response, init_resources, init_actions -from smskillsdk.models.common import MemoryScope +from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response +from smskillsdk.models.common import MemoryScope, Intent + #test @@ -57,9 +58,14 @@ def persist_credentials(self, session_id: str): return credentials_memory - def send(self, user_input: str): + def send(self, user_input): """ Example of sending input to the third party NLP call """ - return get_response(user_input) \ No newline at end of file + if user_input == "Welcome": + return get_welcome_response() + else: + return get_response(user_input) + + \ No newline at end of file diff --git a/app/views/skill.py b/app/views/skill.py index 2e5f4f4..a99e35d 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -2,6 +2,7 @@ from operator import itemgetter, attrgetter from smskillsdk.utils.memory import get_memory_value, set_memory_value from ..services.fake_nlp_service import FakeNLPService + from smskillsdk.models.api import ( InitRequest, SessionRequest, @@ -89,7 +90,7 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: """ # 1. Extract relevant data - skill_config, skill_memory, context = attrgetter("config", "memory", "context")(request) + user_intent, skill_config, skill_memory, context = attrgetter("intent", "config", "memory", "context")(request) # 1a. when using stateless skill, extract relevant credentials from config # credentials = itemgetter("first_credentials", "second_credentials")(skill_config) @@ -97,12 +98,12 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: # 1b. when using stateful skill, extract relevant credentials elsewhere (eg. memory) as config will not be present here _, credentials = get_memory_value(memories=skill_memory, key="credentials") - # 2. Extract user input - user_input = request.text - # 3. Make request to third party service fake_nlp_service = FakeNLPService(*credentials) + # 2. Extract user input + user_input = request.text + # 4. Extract relevant response data from the third party service spoken_response, cards, intent, annotations = fake_nlp_service.send(user_input) From 1e5a16bdef8f77ab805382594fe5fa8d13e04911 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 2 Mar 2025 23:30:57 +0800 Subject: [PATCH 015/112] refactor: enhance welcome response with annotations and handle None cases in skill execution --- app/mocks/gemini_agent.py | 9 ++++++++- app/views/skill.py | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 3f8f653..80e2054 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -129,7 +129,14 @@ def get_welcome_response(): confidence=1, ) - cards, annotations = None, None + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + } + + cards = None return response, cards, intent, annotations def get_response(user_input: str): diff --git a/app/views/skill.py b/app/views/skill.py index a99e35d..17b719f 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -108,7 +108,10 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: spoken_response, cards, intent, annotations = fake_nlp_service.send(user_input) # 5. Construct SM-formatted response body - variables = Variables(public=cards, **annotations) + if annotations is not None: + variables = Variables(public=cards, **annotations) + else: + variables = None output = Output( intent=intent, From aebb26e8e30f53f63870e743b47ed3d57d8ef91f Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 2 Mar 2025 23:47:21 +0800 Subject: [PATCH 016/112] fix: change endConversation flag to False in execute response --- app/views/skill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/skill.py b/app/views/skill.py index 17b719f..2c23715 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -121,7 +121,7 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: response = ExecuteResponse( output=output, - endConversation=True, + endConversation=False, ) return response From cdc34005da9737a5b16946947012f26c8a51dd6c Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 06:18:00 +0800 Subject: [PATCH 017/112] feat: add debug print statement for request in execute function --- app/views/skill.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/views/skill.py b/app/views/skill.py index 2c23715..8360d47 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -92,6 +92,8 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: # 1. Extract relevant data user_intent, skill_config, skill_memory, context = attrgetter("intent", "config", "memory", "context")(request) + print("Request:", request) + # 1a. when using stateless skill, extract relevant credentials from config # credentials = itemgetter("first_credentials", "second_credentials")(skill_config) From 253c9d88d30ba84da9a8eaa0e880f588b8bf5e8c Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 06:59:52 +0800 Subject: [PATCH 018/112] feat: enhance request logging by including user intent and skill configuration details --- app/views/skill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/skill.py b/app/views/skill.py index 8360d47..d682efc 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -92,7 +92,7 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: # 1. Extract relevant data user_intent, skill_config, skill_memory, context = attrgetter("intent", "config", "memory", "context")(request) - print("Request:", request) + print("Request:", user_intent, skill_config, skill_memory, context ) # 1a. when using stateless skill, extract relevant credentials from config # credentials = itemgetter("first_credentials", "second_credentials")(skill_config) From ce76a70df99fe2ea5fdab6b85b6e89c5d9f55940 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 07:14:06 +0800 Subject: [PATCH 019/112] comment print out --- app/views/skill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/skill.py b/app/views/skill.py index d682efc..cb3bc07 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -92,7 +92,7 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: # 1. Extract relevant data user_intent, skill_config, skill_memory, context = attrgetter("intent", "config", "memory", "context")(request) - print("Request:", user_intent, skill_config, skill_memory, context ) + #print("Request:", user_intent, skill_config, skill_memory, context ) # 1a. when using stateless skill, extract relevant credentials from config # credentials = itemgetter("first_credentials", "second_credentials")(skill_config) From d51551fcfc791e2e7f66564f61cc223efd69f09c Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 07:31:55 +0800 Subject: [PATCH 020/112] more debug info --- app/views/skill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/skill.py b/app/views/skill.py index cb3bc07..2fad44d 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -92,7 +92,7 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: # 1. Extract relevant data user_intent, skill_config, skill_memory, context = attrgetter("intent", "config", "memory", "context")(request) - #print("Request:", user_intent, skill_config, skill_memory, context ) + print("Request (intent, skill_config, memory. context):", user_intent, skill_config, skill_memory, context ) # 1a. when using stateless skill, extract relevant credentials from config # credentials = itemgetter("first_credentials", "second_credentials")(skill_config) From 46c7921f234eb35eaf73b5288c446436f98a7adc Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 07:59:19 +0800 Subject: [PATCH 021/112] feat: update chatbot instructions and welcome message for bilingual support --- app/mocks/gemini_agent.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 80e2054..41a768f 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -36,12 +36,13 @@ def get_nonstreaming_text_response (response): system_instruction = ["""You are an expert and customer fronting service agent for an Association called NS Chinese Chamber of Commerce. You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. - DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers."""] + DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. + Respond in the same language as the language of user's query (either English or Chinese). """] -BOT_WELCOME_MESSAGE = "Hello! I am your virtual assistant 小美. How can I assist you today?" - +BOT_WELCOME_MESSAGE = "Hello 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你?" +MEMORY_WINDOW_SIZE = 20 class Chatbot: def __init__(self, history: Optional[List["Content"]] = None, model: Optional[str] = "gemini-1.5-flash-002", use_search=False): @@ -78,6 +79,14 @@ def generate_response(self, user_prompt=""): prompt = user_prompt response = self.use_search(prompt) + + self.chat._history[-2] = Content( + role="user", + parts=[Part.from_text(user_prompt)] # Create Part objects + ) + + if len(self.chat._history) > MEMORY_WINDOW_SIZE: + self.chat._history = self.chat._history[-MEMORY_WINDOW_SIZE:] return get_nonstreaming_text_response(response) From 80a019367e52bb0327bbfc9b7be3c9ded22b3060 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 13:06:11 +0800 Subject: [PATCH 022/112] feat: update system instructions to include YouTube video suggestion for association introduction --- app/mocks/gemini_agent.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 41a768f..a257eea 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -29,6 +29,7 @@ def get_nonstreaming_text_response (response): "temperature": 0.3, #0.5, "top_p": 0.9, #0.5, #0.5 better than 0.95 "top_k": 40, + #"response_mime_type":"application/json" } MODEL_STR = "gemini-1.5-flash-002" @@ -36,11 +37,13 @@ def get_nonstreaming_text_response (response): system_instruction = ["""You are an expert and customer fronting service agent for an Association called NS Chinese Chamber of Commerce. You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. - DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. - Respond in the same language as the language of user's query (either English or Chinese). """] + DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. + Respond in the same language as the language of user's query (either English or Chinese). + When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, if the user answers yes, you will post this URL https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 in the end of your response with no accompanying text or punctuation （this is the only exception to the previous instruction of being naturally conversation based response). + """] -BOT_WELCOME_MESSAGE = "Hello 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你?" +BOT_WELCOME_MESSAGE = "Hello 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" MEMORY_WINDOW_SIZE = 20 From 0e7946cb7650b298a73f087d44e233867050118f Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 13:14:30 +0800 Subject: [PATCH 023/112] feat: allow agent to provide YouTube video link for association introduction --- app/mocks/gemini_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index a257eea..8a8837f 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -39,6 +39,7 @@ def get_nonstreaming_text_response (response): Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Respond in the same language as the language of user's query (either English or Chinese). + You are able to show video simply by providing the youtube URL. When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, if the user answers yes, you will post this URL https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 in the end of your response with no accompanying text or punctuation （this is the only exception to the previous instruction of being naturally conversation based response). """] From 33f9aa536aa40ad0a7d00f3a76d92ac54bc6e5e7 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 13:16:28 +0800 Subject: [PATCH 024/112] feat: update system instruction to reflect the correct association name in gemini_agent --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 8a8837f..86c5806 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -34,7 +34,7 @@ def get_nonstreaming_text_response (response): MODEL_STR = "gemini-1.5-flash-002" -system_instruction = ["""You are an expert and customer fronting service agent for an Association called NS Chinese Chamber of Commerce. +system_instruction = ["""You are an expert and customer fronting service agent for an Association called Negeri Sembilan Chinese Chamber of Commerce (森美兰州中华总商会). You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. From f44b679ea104360819885bf2f3fbccc33f89ff25 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 13:22:38 +0800 Subject: [PATCH 025/112] feat: clarify instructions for introducing the association with YouTube video link --- app/mocks/gemini_agent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 86c5806..9794e38 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -40,7 +40,8 @@ def get_nonstreaming_text_response (response): DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Respond in the same language as the language of user's query (either English or Chinese). You are able to show video simply by providing the youtube URL. - When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, if the user answers yes, you will post this URL https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 in the end of your response with no accompanying text or punctuation （this is the only exception to the previous instruction of being naturally conversation based response). + When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, + If the user wants to watch the youtube video to know more about the association, you MUST append this youtube URL https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 in the end of your response with no accompanying text or punctuation. """] From fe65d7141ed00a41f80349367dd7800b3539c9d3 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 16:04:14 +0800 Subject: [PATCH 026/112] feat: reduce max output tokens for generation and add debug print for generated response --- app/mocks/gemini_agent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 9794e38..7b4b1a9 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -25,7 +25,7 @@ def get_nonstreaming_text_response (response): generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_NONE, } generation_config = { - "max_output_tokens": 8192, + "max_output_tokens": 512, "temperature": 0.3, #0.5, "top_p": 0.9, #0.5, #0.5 better than 0.95 "top_k": 40, @@ -163,6 +163,8 @@ def get_response(user_input: str): # Response to be spoken by your Digital Person response = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." + print(f"generated resp: {response}") + cards, intent, annotations = None, None, None """ # Add your Cards as required From c953d671b91cce065f9243844b3e43644ac8a0e4 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 16:37:39 +0800 Subject: [PATCH 027/112] feat: reduce max output tokens and enhance system instructions for NSCCCI association introduction --- app/mocks/gemini_agent.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 7b4b1a9..2555af1 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -25,7 +25,7 @@ def get_nonstreaming_text_response (response): generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_NONE, } generation_config = { - "max_output_tokens": 512, + "max_output_tokens": 256, "temperature": 0.3, #0.5, "top_p": 0.9, #0.5, #0.5 better than 0.95 "top_k": 40, @@ -34,14 +34,18 @@ def get_nonstreaming_text_response (response): MODEL_STR = "gemini-1.5-flash-002" -system_instruction = ["""You are an expert and customer fronting service agent for an Association called Negeri Sembilan Chinese Chamber of Commerce (森美兰州中华总商会). +system_instruction = ["""You are an expert and customer fronting service agent for an Association called Negeri Sembilan Chinese Chamber of Commerce or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州总商会”). You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Respond in the same language as the language of user's query (either English or Chinese). - You are able to show video simply by providing the youtube URL. - When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, - If the user wants to watch the youtube video to know more about the association, you MUST append this youtube URL https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 in the end of your response with no accompanying text or punctuation. + Be polite and friendly. Keep your answers short and concise. + You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). + When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. + If the user wants to watch the youtube video, you MUST append this youtube URL in the end of your response with no accompanying text or punctuation. + Below is the context for videos you are able to show: + - youtube URL video about NSCCCI: https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 + - youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr """] From 5cb7849a6cd3f3297a5b16bf96482a506858d211 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 16:53:00 +0800 Subject: [PATCH 028/112] feat: enhance system instructions to include YouTube video links for NSCCCI and Vision Valley, and update response handling for video playback --- app/mocks/gemini_agent.py | 43 +++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 2555af1..2839470 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -13,6 +13,7 @@ import vertexai.preview.generative_models as generative_models from vertexai.preview.generative_models import grounding from typing import List, Optional +import json def get_nonstreaming_text_response (response): @@ -29,23 +30,37 @@ def get_nonstreaming_text_response (response): "temperature": 0.3, #0.5, "top_p": 0.9, #0.5, #0.5 better than 0.95 "top_k": 40, - #"response_mime_type":"application/json" + "response_mime_type":"application/json" } MODEL_STR = "gemini-1.5-flash-002" +""" +You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). +When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. +If the user wants to watch the youtube video, you MUST append this youtube URL in the end of your response with no accompanying text or punctuation. +Below is the context for videos you are able to show: +- youtube URL video about NSCCCI: https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 +- youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" + +video_url = { +"youtube_url_about_nscci": "https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7", +"youtube_url_about_vision_valley": "https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr" +} + system_instruction = ["""You are an expert and customer fronting service agent for an Association called Negeri Sembilan Chinese Chamber of Commerce or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州总商会”). You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Respond in the same language as the language of user's query (either English or Chinese). Be polite and friendly. Keep your answers short and concise. - You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). - When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. - If the user wants to watch the youtube video, you MUST append this youtube URL in the end of your response with no accompanying text or punctuation. - Below is the context for videos you are able to show: - - youtube URL video about NSCCCI: https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 - - youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr + If the user wants to know about NSCCCI (such as the association's history, mission, vision, etc.), you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. + Response in following schema: + { + "response_text": "your text based response", + "play_youtube_video": boolean true if user wants to watch youtube video false otherwise, + "type_of_video": "video_about_nscci" or "video_about_vision_valley" or "none" + } """] @@ -165,9 +180,19 @@ def get_response(user_input: str): print(f"User said: {user_input}") # Response to be spoken by your Digital Person - response = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." + reponse_dict = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." + + print(f"generated resp: {reponse_dict}") + + try: + reponse_dict = json.loads(reponse_dict) + reponse = reponse_dict['response_text'] + if reponse_dict['play_youtube_video']: + reponse += f" {video_url[reponse_dict['type_of_video']]}" - print(f"generated resp: {response}") + except Exception as e: + print("error in reponse error decoding:",e) + reponse = "" cards, intent, annotations = None, None, None """ From b3b4e6f4753f3635d3733f381cb4542b19e0f7e5 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 17:00:27 +0800 Subject: [PATCH 029/112] feat: refine system instruction to specify the content of the YouTube video about NSCCCI history --- app/mocks/gemini_agent.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 2839470..5abbfcb 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -54,7 +54,7 @@ def get_nonstreaming_text_response (response): DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Respond in the same language as the language of user's query (either English or Chinese). Be polite and friendly. Keep your answers short and concise. - If the user wants to know about NSCCCI (such as the association's history, mission, vision, etc.), you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. + If the user wants to know about NSCCCI (such as the association's history, mission, vision, etc.), you may ask if the user would like to watch a youtube video about the association which talks about the history, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. Response in following schema: { "response_text": "your text based response", @@ -183,16 +183,16 @@ def get_response(user_input: str): reponse_dict = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." print(f"generated resp: {reponse_dict}") - + response = "" try: reponse_dict = json.loads(reponse_dict) - reponse = reponse_dict['response_text'] + response = reponse_dict['response_text'] if reponse_dict['play_youtube_video']: - reponse += f" {video_url[reponse_dict['type_of_video']]}" + response += f" {video_url[reponse_dict['type_of_video']]}" except Exception as e: print("error in reponse error decoding:",e) - reponse = "" + cards, intent, annotations = None, None, None """ From eeee5e938cb96675b8f40008be1eb417e9391cd0 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 17:10:13 +0800 Subject: [PATCH 030/112] feat: update system instructions and video references for NSCCCI Chamber --- app/mocks/gemini_agent.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 5abbfcb..564d7b6 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -44,23 +44,24 @@ def get_nonstreaming_text_response (response): - youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" video_url = { -"youtube_url_about_nscci": "https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7", +"video_about_nsccci": "https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7", "youtube_url_about_vision_valley": "https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr" } -system_instruction = ["""You are an expert and customer fronting service agent for an Association called Negeri Sembilan Chinese Chamber of Commerce or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州总商会”). +system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called Negeri Sembilan Chinese Chamber of Commerce or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州总商会”). You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. - Respond in the same language as the language of user's query (either English or Chinese). - Be polite and friendly. Keep your answers short and concise. - If the user wants to know about NSCCCI (such as the association's history, mission, vision, etc.), you may ask if the user would like to watch a youtube video about the association which talks about the history, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. - Response in following schema: + Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). + If the user wants to know about NSCCCI Chamber (such as the Chamber's history, mission, vision, etc.), you may ask if the user would like to watch a youtube video about the Chamber which talks about the founding history, vision and mission, + You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch a youtube video about the project. + Respond in following schema: { - "response_text": "your text based response", + "response_text": "your text based response, Respond in the same language as the language of user's query (either English or Chinese).", "play_youtube_video": boolean true if user wants to watch youtube video false otherwise, - "type_of_video": "video_about_nscci" or "video_about_vision_valley" or "none" - } + "type_of_video": "video_about_nsccci" or "video_about_vision_valley" or "none" + } + """] @@ -186,9 +187,11 @@ def get_response(user_input: str): response = "" try: reponse_dict = json.loads(reponse_dict) - response = reponse_dict['response_text'] if reponse_dict['play_youtube_video']: - response += f" {video_url[reponse_dict['type_of_video']]}" + response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" + else: + response = reponse_dict['response_text'] + except Exception as e: print("error in reponse error decoding:",e) From af9949e65f515052b3b75667b99b688f8cbaf42c Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 17:18:12 +0800 Subject: [PATCH 031/112] updated youtube links --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 564d7b6..df28cee 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -44,8 +44,8 @@ def get_nonstreaming_text_response (response): - youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" video_url = { -"video_about_nsccci": "https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7", -"youtube_url_about_vision_valley": "https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr" +"video_about_nsccci": "https://www.youtube.com/watch?v=Bhkm6fZMJcI", +"youtube_url_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8" } system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called Negeri Sembilan Chinese Chamber of Commerce or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州总商会”). From b24d3fe13be0cd70045c69847e0ca80dc1bb5adc Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 17:47:51 +0800 Subject: [PATCH 032/112] feat: update video references and system instructions for NSCCCI Chamber --- app/mocks/gemini_agent.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index df28cee..206e259 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -44,8 +44,8 @@ def get_nonstreaming_text_response (response): - youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" video_url = { -"video_about_nsccci": "https://www.youtube.com/watch?v=Bhkm6fZMJcI", -"youtube_url_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8" +"video_about_chamber_of_commerce": "https://www.youtube.com/watch?v=Bhkm6fZMJcI", +"video_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8" } system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called Negeri Sembilan Chinese Chamber of Commerce or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州总商会”). @@ -53,13 +53,13 @@ def get_nonstreaming_text_response (response): Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). - If the user wants to know about NSCCCI Chamber (such as the Chamber's history, mission, vision, etc.), you may ask if the user would like to watch a youtube video about the Chamber which talks about the founding history, vision and mission, - You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch a youtube video about the project. + If the user wants to know about NSCCCI Chamber (such as the Chamber's history, mission, vision, etc.), you may ask if the user would like to watch a video about the Chamber which talks about the founding history, vision and mission, + You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch a video about the project. Respond in following schema: { "response_text": "your text based response, Respond in the same language as the language of user's query (either English or Chinese).", - "play_youtube_video": boolean true if user wants to watch youtube video false otherwise, - "type_of_video": "video_about_nsccci" or "video_about_vision_valley" or "none" + "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, + "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley" or "none" } """] @@ -187,7 +187,7 @@ def get_response(user_input: str): response = "" try: reponse_dict = json.loads(reponse_dict) - if reponse_dict['play_youtube_video']: + if reponse_dict['uer_wants_to_watch_video']: response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" else: response = reponse_dict['response_text'] From 5910a9f008017bf63a69990103b24d32e0fa77b2 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 18:00:31 +0800 Subject: [PATCH 033/112] feat: enhance system instructions to include video references for NSCCCI and Vision Valley --- app/mocks/gemini_agent.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 206e259..2bf9ff5 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -48,11 +48,12 @@ def get_nonstreaming_text_response (response): "video_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8" } -system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called Negeri Sembilan Chinese Chamber of Commerce or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州总商会”). +system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called Negeri Sembilan Chinese Chamber of Commerce or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). + In your knowledge, you know of the existence of 2 videos, namely 1) video about NSCCCI (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). If the user wants to know about NSCCCI Chamber (such as the Chamber's history, mission, vision, etc.), you may ask if the user would like to watch a video about the Chamber which talks about the founding history, vision and mission, You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch a video about the project. Respond in following schema: @@ -61,7 +62,8 @@ def get_nonstreaming_text_response (response): "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley" or "none" } - + Answer to queries that are related to NSCCCI other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. + If the user asks about anything else, apologies and explain that you are not able to answer. """] From a2f4110a25d6fb0bbfe229361216d6e3e8ede27f Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 18:16:33 +0800 Subject: [PATCH 034/112] feat: update model version in gemini_agent to 2.0-flash-001 --- app/mocks/gemini_agent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 2bf9ff5..2402abb 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -33,7 +33,8 @@ def get_nonstreaming_text_response (response): "response_mime_type":"application/json" } -MODEL_STR = "gemini-1.5-flash-002" +#MODEL_STR = "gemini-1.5-flash-002" +MODEL_STR = "gemini-2.0-flash-001" """ You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). From 6f3b7de3406cac83c835c1384892712a4014a415 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 18:44:33 +0800 Subject: [PATCH 035/112] feat: enhance system instructions to include video playback capabilities --- app/mocks/gemini_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 2402abb..36ab907 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -54,6 +54,7 @@ def get_nonstreaming_text_response (response): Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). + You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. In your knowledge, you know of the existence of 2 videos, namely 1) video about NSCCCI (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). If the user wants to know about NSCCCI Chamber (such as the Chamber's history, mission, vision, etc.), you may ask if the user would like to watch a video about the Chamber which talks about the founding history, vision and mission, You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch a video about the project. From 626fc98d288057184340045a0a75c9d7fb2f709b Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 18:46:36 +0800 Subject: [PATCH 036/112] feat: append YouTube link to response text in gemini_agent --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 36ab907..4f4cba3 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -194,7 +194,7 @@ def get_response(user_input: str): if reponse_dict['uer_wants_to_watch_video']: response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" else: - response = reponse_dict['response_text'] + response = reponse_dict['response_text'] + " https://www.youtube.com/watch?v=Bhkm6fZMJcI" except Exception as e: From 83278bfeeb690ea46b4a8b12fffa186f05322b9e Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 18:53:44 +0800 Subject: [PATCH 037/112] feat: call allocated_resources method during agent initialization --- app/mocks/gemini_agent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 4f4cba3..539d5c0 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -129,12 +129,13 @@ def allocated_resources(self): self.chatbot = Chatbot() agent = Agent() +agent.allocated_resources() def init_actions(): """ Example of an action performed by the Initalize ednpoint """ - agent.allocated_resources() + print("resource initialized. . .") From a884da5a61162215e168ab8638a275f02cbaa506 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 19:16:06 +0800 Subject: [PATCH 038/112] feat: modify response generation to include previous message context --- app/mocks/gemini_agent.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 539d5c0..708f2fe 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -105,7 +105,13 @@ def use_search(self, prompt): def generate_response(self, user_prompt=""): #prompt = user_prompt - prompt = user_prompt + #prompt = user_prompt + + if len(self.chat._history): + prompt = f"""Your last message was :"{self.chat._history[-1].parts[0]._raw_part.text}" Please respond in the same language as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". + """ + else: + prompt = user_prompt response = self.use_search(prompt) From 5ea65feec62b7ef99f75f4d4ae0cbec7e99e7ff4 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 19:48:43 +0800 Subject: [PATCH 039/112] feat: update YouTube links to enable autoplay and refine system instructions --- app/mocks/gemini_agent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 708f2fe..faaf700 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -45,11 +45,11 @@ def get_nonstreaming_text_response (response): - youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" video_url = { -"video_about_chamber_of_commerce": "https://www.youtube.com/watch?v=Bhkm6fZMJcI", -"video_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8" +"video_about_chamber_of_commerce": "https://www.youtube.com/watch?v=Bhkm6fZMJcI?autoplay=1", +"video_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8?autoplay=1" } -system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called Negeri Sembilan Chinese Chamber of Commerce or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). +system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. @@ -201,7 +201,7 @@ def get_response(user_input: str): if reponse_dict['uer_wants_to_watch_video']: response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" else: - response = reponse_dict['response_text'] + " https://www.youtube.com/watch?v=Bhkm6fZMJcI" + response = reponse_dict['response_text'] + " https://www.youtube.com/watch?v=Bhkm6fZMJcI?autoplay=1" except Exception as e: From bed51d0306e199dac506537f31b3a33be84129b5 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 3 Mar 2025 19:55:51 +0800 Subject: [PATCH 040/112] feat: remove autoplay from YouTube links in gemini_agent and adjust response text --- app/mocks/gemini_agent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index faaf700..c80246d 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -45,8 +45,8 @@ def get_nonstreaming_text_response (response): - youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" video_url = { -"video_about_chamber_of_commerce": "https://www.youtube.com/watch?v=Bhkm6fZMJcI?autoplay=1", -"video_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8?autoplay=1" +"video_about_chamber_of_commerce": "https://www.youtube.com/watch?v=Bhkm6fZMJcI", +"video_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8" } system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). @@ -201,7 +201,7 @@ def get_response(user_input: str): if reponse_dict['uer_wants_to_watch_video']: response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" else: - response = reponse_dict['response_text'] + " https://www.youtube.com/watch?v=Bhkm6fZMJcI?autoplay=1" + response = reponse_dict['response_text'] #+ " https://www.youtube.com/watch?v=Bhkm6fZMJcI" except Exception as e: From af24563caca77171bea6b04ac202c6c5a2091141 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 4 Mar 2025 07:44:00 +0800 Subject: [PATCH 041/112] feat: refine response instructions for gemini_agent to focus on NSCCCI queries --- app/mocks/gemini_agent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index c80246d..68af2d9 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -62,10 +62,10 @@ def get_nonstreaming_text_response (response): { "response_text": "your text based response, Respond in the same language as the language of user's query (either English or Chinese).", "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, - "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley" or "none" + "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley" } - Answer to queries that are related to NSCCCI other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. - If the user asks about anything else, apologies and explain that you are not able to answer. + ONLY answer to queries that are related to NSCCCI other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. + If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. """] From 8c514a032e0ebf6eec0485eec2b098ee001cdf01 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 4 Mar 2025 07:56:56 +0800 Subject: [PATCH 042/112] change tool --- app/mocks/gemini_agent.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 68af2d9..e0505b7 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -73,13 +73,30 @@ def get_nonstreaming_text_response (response): MEMORY_WINDOW_SIZE = 20 +DATA_STORE_ID="acccim-ns_1740458649382" +DATA_STORE_REGION="us" +project_id="neuralnet-manforce" +datastore = f"projects/{project_id}/locations/{DATA_STORE_REGION}/collections/default_collection/dataStores/{DATA_STORE_ID}" +datastore_grounding_tool = Tool.from_retrieval( + grounding.Retrieval( + grounding.VertexAISearch( + project=project_id, + datastore=DATA_STORE_ID, + location=DATA_STORE_REGION, + #datastore=datastore, + ) + ) + ) +googlesearch_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) + class Chatbot: def __init__(self, history: Optional[List["Content"]] = None, model: Optional[str] = "gemini-1.5-flash-002", use_search=False): self.model = GenerativeModel( model, system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) - self.grounding_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) + #self.grounding_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) + self.grounding_tool = datastore_grounding_tool """ def use_rag_tool(self, user_prompt): From f45c9f5efb16dc204a27a26f281f2fa12aec14d5 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 4 Mar 2025 07:57:59 +0800 Subject: [PATCH 043/112] feat: improve response text clarity in gemini_agent instructions --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index e0505b7..28fd154 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -60,7 +60,7 @@ def get_nonstreaming_text_response (response): You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch a video about the project. Respond in following schema: { - "response_text": "your text based response, Respond in the same language as the language of user's query (either English or Chinese).", + "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley" } From 92d2535345299b5824b4e1263e1668a86aeda946 Mon Sep 17 00:00:00 2001 From: SengTak Date: Thu, 6 Mar 2025 15:57:47 +0800 Subject: [PATCH 044/112] feat: update YouTube link for chamber of commerce video to enable autoplay and add language parameter in system instructions --- app/mocks/gemini_agent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 28fd154..60bf02d 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -45,7 +45,7 @@ def get_nonstreaming_text_response (response): - youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" video_url = { -"video_about_chamber_of_commerce": "https://www.youtube.com/watch?v=Bhkm6fZMJcI", +"video_about_chamber_of_commerce": "https://www.youtube.com/embed/Bhkm6fZMJcI?autoplay=1&mute=0", #"https://www.youtube.com/watch?v=Bhkm6fZMJcI", "video_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8" } @@ -62,7 +62,8 @@ def get_nonstreaming_text_response (response): { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, - "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley" + "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", + "language": "en" or "zh", default to "en" } ONLY answer to queries that are related to NSCCCI other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. From 731e9b26fde4d422d37404f67b57bcbcba7ce9b7 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 11 Mar 2025 16:41:12 +0800 Subject: [PATCH 045/112] test autoplay --- app/mocks/gemini_agent.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 60bf02d..54b196e 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -227,6 +227,19 @@ def get_response(user_input: str): cards, intent, annotations = None, None, None + + cards = { + 'card': { + "type": "video", + "id": "youtubeVideo", + "data": { + "videoId":"Bhkm6fZMJcI", + "autoplay":"true", + "autoclose":"true" + } + } + } + """ # Add your Cards as required cards = { From fcecb0ddc62c91efe36abb6afe7415a69a0ba373 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 11 Mar 2025 16:54:01 +0800 Subject: [PATCH 046/112] test show video card --- app/mocks/gemini_agent.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 54b196e..bfd959c 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -228,6 +228,9 @@ def get_response(user_input: str): cards, intent, annotations = None, None, None + #test show video + response = "Hello! @showcards(card) Here is a video." + cards = { 'card': { "type": "video", From 086e6b6f5d5169b0730b43706b2c527b5048a942 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 11 Mar 2025 16:59:22 +0800 Subject: [PATCH 047/112] test content 2 --- app/mocks/gemini_agent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index bfd959c..1f09ecc 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -230,7 +230,7 @@ def get_response(user_input: str): #test show video response = "Hello! @showcards(card) Here is a video." - + """ cards = { 'card': { "type": "video", @@ -242,8 +242,8 @@ def get_response(user_input: str): } } } - """ + # Add your Cards as required cards = { "card": { @@ -255,6 +255,7 @@ def get_response(user_input: str): }, } + """ # Add your Intent as required intent = Intent( name="Welcome", From d12298fc25cf95dd88ea4e8e59ba5cf4209a6802 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 11 Mar 2025 21:44:13 +0800 Subject: [PATCH 048/112] feat: handle case where cards are present without annotations in response construction --- app/views/skill.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/views/skill.py b/app/views/skill.py index 2fad44d..17c621a 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -112,6 +112,8 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: # 5. Construct SM-formatted response body if annotations is not None: variables = Variables(public=cards, **annotations) + elif cards is not None: + variables = Variables(public=cards) else: variables = None From 744a0b03130cea3fb311ba5f8208b04ec341c97c Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 11 Mar 2025 21:56:37 +0800 Subject: [PATCH 049/112] fix: update response construction logic to handle cases with and without annotations and cards --- app/mocks/gemini_agent.py | 8 ++++---- app/views/skill.py | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 1f09ecc..8efc468 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -230,7 +230,7 @@ def get_response(user_input: str): #test show video response = "Hello! @showcards(card) Here is a video." - """ + cards = { 'card': { "type": "video", @@ -242,10 +242,10 @@ def get_response(user_input: str): } } } - """ + # Add your Cards as required - cards = { + """cards = { "card": { "type": "image", "data": { @@ -253,7 +253,7 @@ def get_response(user_input: str): "alt": "An adorable kitten", }, }, - } + }""" """ # Add your Intent as required diff --git a/app/views/skill.py b/app/views/skill.py index 17c621a..3da2ec6 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -110,11 +110,14 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: spoken_response, cards, intent, annotations = fake_nlp_service.send(user_input) # 5. Construct SM-formatted response body - if annotations is not None: + if (annotations is not None) and (cards is not None): + print("show card1") variables = Variables(public=cards, **annotations) elif cards is not None: + print("show card2") variables = Variables(public=cards) else: + print("show card3") variables = None output = Output( From 9cd45eefbebbb3fb3af0aa515f70ff15906fb964 Mon Sep 17 00:00:00 2001 From: SengTak Date: Wed, 12 Mar 2025 07:54:38 +0800 Subject: [PATCH 050/112] feat: add video handling and introductory messages for NSCCCI responses --- app/mocks/gemini_agent.py | 52 ++++++++++++++++++++++++--------------- app/views/skill.py | 6 ++--- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 8efc468..df87e07 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -49,15 +49,25 @@ def get_nonstreaming_text_response (response): "video_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8" } -system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as NSCCCI (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). +video_id = { + "video_about_chamber_of_commerce": "Bhkm6fZMJcI", + "video_about_vision_valley": "LXC6FMkf9a8" +} + +vidoe_intro ={ + "en": "Please enjoy the video. ", + "zh": "请欣赏视屏。" +} + +system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). - You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. In your knowledge, you know of the existence of 2 videos, namely 1) video about NSCCCI (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). - If the user wants to know about NSCCCI Chamber (such as the Chamber's history, mission, vision, etc.), you may ask if the user would like to watch a video about the Chamber which talks about the founding history, vision and mission, - You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch a video about the project. + If the user wants to know about NSCCCI Chamber (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, + You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. + You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. Respond in following schema: { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", @@ -65,7 +75,7 @@ def get_nonstreaming_text_response (response): "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", "language": "en" or "zh", default to "en" } - ONLY answer to queries that are related to NSCCCI other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. + ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. """] @@ -213,11 +223,26 @@ def get_response(user_input: str): reponse_dict = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." print(f"generated resp: {reponse_dict}") + cards, intent, annotations = None, None, None response = "" try: reponse_dict = json.loads(reponse_dict) if reponse_dict['uer_wants_to_watch_video']: - response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" + #response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" + #test show video + response = vidoe_intro[reponse_dict['language']] + "@showcards(card)" #"Hello! @showcards(card) Here is a video." + + cards = { + 'card': { + "type": "video", + "id": "youtubeVideo", + "data": { + "videoId": video_id[reponse_dict['type_of_video']], + "autoplay":"true", + "autoclose":"true" + } + } + } else: response = reponse_dict['response_text'] #+ " https://www.youtube.com/watch?v=Bhkm6fZMJcI" @@ -226,22 +251,9 @@ def get_response(user_input: str): print("error in reponse error decoding:",e) - cards, intent, annotations = None, None, None + - #test show video - response = "Hello! @showcards(card) Here is a video." - cards = { - 'card': { - "type": "video", - "id": "youtubeVideo", - "data": { - "videoId":"Bhkm6fZMJcI", - "autoplay":"true", - "autoclose":"true" - } - } - } # Add your Cards as required diff --git a/app/views/skill.py b/app/views/skill.py index 3da2ec6..1ddef75 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -111,13 +111,13 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: # 5. Construct SM-formatted response body if (annotations is not None) and (cards is not None): - print("show card1") + #print("show card1") variables = Variables(public=cards, **annotations) elif cards is not None: - print("show card2") + #print("show card2") variables = Variables(public=cards) else: - print("show card3") + #print("show card3") variables = None output = Output( From b26289b1f003a628a7d7b4bb0597e6cd0313f0b5 Mon Sep 17 00:00:00 2001 From: SengTak Date: Wed, 12 Mar 2025 08:04:59 +0800 Subject: [PATCH 051/112] fix: clarify instruction for user video preference in response schema --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index df87e07..79c8009 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -71,7 +71,7 @@ def get_nonstreaming_text_response (response): Respond in following schema: { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", - "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, + "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, or answer yes to your previous invitation question to watch the video. "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", "language": "en" or "zh", default to "en" } From cba44ee96c12dbbd2eaf1d48fc95101d83781904 Mon Sep 17 00:00:00 2001 From: SengTak Date: Wed, 12 Mar 2025 20:54:18 +0800 Subject: [PATCH 052/112] fix: update video URLs for vision valley in gemini_agent mock --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 79c8009..73504b3 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -46,12 +46,12 @@ def get_nonstreaming_text_response (response): video_url = { "video_about_chamber_of_commerce": "https://www.youtube.com/embed/Bhkm6fZMJcI?autoplay=1&mute=0", #"https://www.youtube.com/watch?v=Bhkm6fZMJcI", -"video_about_vision_valley": "https://www.youtube.com/watch?v=LXC6FMkf9a8" +"video_about_vision_valley": "https://www.youtube.com/watch?v=GgUYagMYkkg" } video_id = { "video_about_chamber_of_commerce": "Bhkm6fZMJcI", - "video_about_vision_valley": "LXC6FMkf9a8" + "video_about_vision_valley": "GgUYagMYkkg" } vidoe_intro ={ From 1b73530b48c14666b92a88c60196c759e21cd39d Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 7 Apr 2025 17:28:42 +0800 Subject: [PATCH 053/112] feat: implement endpoint to store and retrieve person data with thread safety --- app/views/skill.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/app/views/skill.py b/app/views/skill.py index 1ddef75..3f3415a 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -3,6 +3,15 @@ from smskillsdk.utils.memory import get_memory_value, set_memory_value from ..services.fake_nlp_service import FakeNLPService +# Add these near the top of the file with other imports +from flask import request +import threading + + +# Add these after the other global variables +_person_data = None +_person_lock = threading.Lock() # Thread-safe access to person data + from smskillsdk.models.api import ( InitRequest, SessionRequest, @@ -21,6 +30,37 @@ }, ) +def set_person_data(): + """ + Endpoint to receive and store person data globally + Expected POST body: {"person": {...}} + """ + global _person_data + + if not request.is_json: + return {"error": "Content-Type must be application/json"}, 400 + + data = request.get_json() + if "person" not in data: + return {"error": "Missing 'person' field in request body"}, 400 + + with _person_lock: + _person_data = data["person"] + print(f"Stored person data: {_person_data}") + + return {"message": "Person data stored successfully"}, 200 + +def get_person_data(): + """ + Helper function to safely access the person data from other methods + """ + with _person_lock: + return _person_data + +@router.post("/face-detection", status_code=200) +def handle_set_person(): + return set_person_data() + @router.post("/init", status_code=204) async def init(request: InitRequest): """ From f051f2635fd1d6ffb12063ceb643ad89eaf85f46 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 7 Apr 2025 21:47:15 +0800 Subject: [PATCH 054/112] feat: refactor face detection handler to store person data asynchronously with error handling --- app/views/skill.py | 40 +++++++++++----------------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/app/views/skill.py b/app/views/skill.py index 3f3415a..9bed9a6 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -4,8 +4,9 @@ from ..services.fake_nlp_service import FakeNLPService # Add these near the top of the file with other imports -from flask import request +from fastapi import Request, HTTPException import threading +import sys # Add these after the other global variables @@ -30,36 +31,17 @@ }, ) -def set_person_data(): - """ - Endpoint to receive and store person data globally - Expected POST body: {"person": {...}} - """ - global _person_data - - if not request.is_json: - return {"error": "Content-Type must be application/json"}, 400 - - data = request.get_json() - if "person" not in data: - return {"error": "Missing 'person' field in request body"}, 400 - - with _person_lock: - _person_data = data["person"] - print(f"Stored person data: {_person_data}") - - return {"message": "Person data stored successfully"}, 200 - -def get_person_data(): - """ - Helper function to safely access the person data from other methods - """ - with _person_lock: - return _person_data @router.post("/face-detection", status_code=200) -def handle_set_person(): - return set_person_data() +async def handle_face_detection(request: Request): + global _person_data + try: + data = await request.json() + # Process the data + _person_data = data["person"] + print("Received face detection data:", _person_data) + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) @router.post("/init", status_code=204) async def init(request: InitRequest): From 421eff19213021ee8b80197bcaa3d06d4a23595b Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 7 Apr 2025 22:15:28 +0800 Subject: [PATCH 055/112] feat: implement person data handling with global state management in gemini_agent --- app/mocks/gemini_agent.py | 11 +++++++++-- app/views/skill.py | 12 +++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 73504b3..3292cd7 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -15,6 +15,13 @@ from typing import List, Optional import json +# Add these after the other global variables +_person_data = "" + +def set_person_data(data): + global _person_data + _person_data = data + def get_nonstreaming_text_response (response): return response.candidates[0].content.parts[0]._raw_part.text @@ -80,7 +87,6 @@ def get_nonstreaming_text_response (response): """] -BOT_WELCOME_MESSAGE = "Hello 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" MEMORY_WINDOW_SIZE = 20 @@ -195,7 +201,8 @@ def init_resources(session_id: str) -> List[Memory]: def get_welcome_response(): # standard welcome message - response = BOT_WELCOME_MESSAGE + + response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" intent = Intent( name="Welcome", diff --git a/app/views/skill.py b/app/views/skill.py index 9bed9a6..7cb4d72 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -5,13 +5,10 @@ # Add these near the top of the file with other imports from fastapi import Request, HTTPException -import threading import sys +from ..mocks.gemini_agent import set_person_data -# Add these after the other global variables -_person_data = None -_person_lock = threading.Lock() # Thread-safe access to person data from smskillsdk.models.api import ( InitRequest, @@ -34,12 +31,13 @@ @router.post("/face-detection", status_code=200) async def handle_face_detection(request: Request): - global _person_data try: data = await request.json() # Process the data - _person_data = data["person"] - print("Received face detection data:", _person_data) + person_data = data["person"] + + set_person_data(person_data) + #print("Received face detection data:", _person_data) except Exception as e: raise HTTPException(status_code=400, detail=str(e)) From c1fc7f25af8f01c93d532cab21184e96076a2df1 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 7 Apr 2025 22:16:44 +0800 Subject: [PATCH 056/112] feat: add logging for setting person data and face detection handling --- app/mocks/gemini_agent.py | 1 + app/views/skill.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 3292cd7..c7ef5b0 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -21,6 +21,7 @@ def set_person_data(data): global _person_data _person_data = data + print("set person data:", _person_data) def get_nonstreaming_text_response (response): diff --git a/app/views/skill.py b/app/views/skill.py index 7cb4d72..6b43d36 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -37,7 +37,7 @@ async def handle_face_detection(request: Request): person_data = data["person"] set_person_data(person_data) - #print("Received face detection data:", _person_data) + print("Received face detection data:", person_data) except Exception as e: raise HTTPException(status_code=400, detail=str(e)) From c29030e6f1cd73a5891f25c444a2342a55532b66 Mon Sep 17 00:00:00 2001 From: SengTak Date: Thu, 17 Apr 2025 09:10:06 +0800 Subject: [PATCH 057/112] feat: enhance system instruction for N.S.C.C.C.I and update grounding tool initialization --- app/mocks/gemini_agent.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index c7ef5b0..3bcb222 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -67,13 +67,15 @@ def get_nonstreaming_text_response (response): "zh": "请欣赏视屏。" } -system_instruction = ["""You are an expert and customer fronting service agent for an Chamber of Commerce called 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). - You will ground your answers using context from the homepage https://nsccci.org.my/ (and exclude https://nsccabout.gbs2u.com/ as a reference) whenever it is relevant to the user query. +system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). + Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. + 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. + All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/ . Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). - In your knowledge, you know of the existence of 2 videos, namely 1) video about NSCCCI (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). - If the user wants to know about NSCCCI Chamber (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, + In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). + If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. Respond in following schema: @@ -114,7 +116,8 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) #self.grounding_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) - self.grounding_tool = datastore_grounding_tool + #self.grounding_tool = datastore_grounding_tool + self.grounding_tool = [datastore_grounding_tool, googlesearch_tool] """ def use_rag_tool(self, user_prompt): @@ -130,7 +133,7 @@ def use_search(self, prompt): return self.chat.send_message( #f"Contexts: {contexts}. Message from User: {user_prompt}", [prompt], - tools=[self.grounding_tool], + tools=self.grounding_tool, generation_config=generation_config, #safety_settings=safety_settings, stream=False @@ -163,13 +166,14 @@ def generate_response(self, user_prompt=""): vertexai.init(project="neuralnet-manforce", location="us-central1") class Agent: - def __init__(self): + def __init__(self, model = ""): self.chatbot = None + self.model = model def allocated_resources(self): - self.chatbot = Chatbot() + self.chatbot = Chatbot(model=self.model) -agent = Agent() +agent = Agent(model=MODEL_STR) agent.allocated_resources() def init_actions(): From fb46c3c0708a1bb5d15d58a15033ff316a58022d Mon Sep 17 00:00:00 2001 From: SengTak Date: Thu, 17 Apr 2025 11:00:22 +0800 Subject: [PATCH 058/112] back to datastore grounding before transiting to google genai --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 3bcb222..7d8adde 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -116,8 +116,8 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) #self.grounding_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) - #self.grounding_tool = datastore_grounding_tool - self.grounding_tool = [datastore_grounding_tool, googlesearch_tool] + self.grounding_tool = [datastore_grounding_tool] + #self.grounding_tool = [datastore_grounding_tool, googlesearch_tool] """ def use_rag_tool(self, user_prompt): From 5dfeb75be657788bb4f767f41a32a53c2658785a Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 22 Apr 2025 12:41:01 +0800 Subject: [PATCH 059/112] feat: update gemini_agent and fake_nlp_service to improve system instructions and video handling --- app/mocks/gemini_agent.py | 2 +- app/mocks/gemini_agent_2.py | 274 +++++++++++++++++++++++++++++++ app/services/fake_nlp_service.py | 3 +- 3 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 app/mocks/gemini_agent_2.py diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 7d8adde..8c5016d 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -70,7 +70,7 @@ def get_nonstreaming_text_response (response): system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. - All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/ . + All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). diff --git a/app/mocks/gemini_agent_2.py b/app/mocks/gemini_agent_2.py new file mode 100644 index 0000000..f8f5938 --- /dev/null +++ b/app/mocks/gemini_agent_2.py @@ -0,0 +1,274 @@ +""" +These functions use Promises and setTimeouts to mock HTTP requests to a third part NLP service +and should be replaced with the actual HTTP calls when implementing. +""" +from typing import List +from smskillsdk.models.common import Memory, MemoryScope, Intent +from google import genai +from google.genai.chats import Chat +from google.genai import types +from typing import List, Optional +import json +# use google genai +#ref https://docs.soulmachines.com/skills-api/getting-started/nlp-adapter-skill#advanced-concepts + +# Add these after the other global variables +_person_data = "" + +def set_person_data(data): + global _person_data + _person_data = data + print("set person data:", _person_data) + + +MODEL_STR = "gemini-2.0-flash-001" + +""" +You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). +When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. +If the user wants to watch the youtube video, you MUST append this youtube URL in the end of your response with no accompanying text or punctuation. +Below is the context for videos you are able to show: +- youtube URL video about NSCCCI: https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 +- youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" + +video_url = { +"video_about_chamber_of_commerce": "https://www.youtube.com/embed/Bhkm6fZMJcI?autoplay=1&mute=0", #"https://www.youtube.com/watch?v=Bhkm6fZMJcI", +"video_about_vision_valley": "https://www.youtube.com/watch?v=GgUYagMYkkg" +} + +video_id = { + "video_about_chamber_of_commerce": "Bhkm6fZMJcI", + "video_about_vision_valley": "GgUYagMYkkg" +} + +vidoe_intro ={ + "en": "Please enjoy the video. ", + "zh": "请欣赏视屏。" +} + +system_instruction = """You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). + Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. + 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. + All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. + Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. + DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. + Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). + In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). + If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, + You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. + You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. + Respond in following schema: + { + "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", + "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, or answer yes to your previous invitation question to watch the video. + "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", + "language": "en" or "zh", default to "en" + } + ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. + If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. + """ + +googlesearch_tool = types.Tool(google_search=types.GoogleSearch()) + +tools = [googlesearch_tool] + +MEMORY_WINDOW_SIZE = 20 + +DATA_STORE_ID="acccim-ns_1740458649382" +DATA_STORE_REGION="us" +project_id="neuralnet-manforce" +datastore = f"projects/{project_id}/locations/{DATA_STORE_REGION}/collections/default_collection/dataStores/{DATA_STORE_ID}" +ragCorpus = "projects/civic-advantage-395410/locations/us-central1/ragCorpora/3379951520341557248" + +retrieval_tool = types.Tool(retrieval=types.Retrieval(vertex_ai_search=types.VertexAISearch(datastore=datastore))) + +ragretriever_tool = types.Tool( + retrieval=types.Retrieval( + vertex_rag_store=types.VertexRagStore( + rag_resources=[ + types.RagResource( + rag_corpus=ragCorpus + ) + ], + similarity_top_k=10, + ) + ) + ) + +generate_content_config = types.GenerateContentConfig( + temperature = 0.3, + top_p = 0.95, + max_output_tokens = 256, + response_modalities = ["TEXT"], + response_mime_type = "application/json", + speech_config = types.SpeechConfig( + voice_config = types.VoiceConfig( + prebuilt_voice_config = types.PrebuiltVoiceConfig( + voice_name = "zephyr" + ) + ), + ), + safety_settings = [types.SafetySetting( + category="HARM_CATEGORY_HATE_SPEECH", + threshold="OFF" + ),types.SafetySetting( + category="HARM_CATEGORY_DANGEROUS_CONTENT", + threshold="OFF" + ),types.SafetySetting( + category="HARM_CATEGORY_SEXUALLY_EXPLICIT", + threshold="OFF" + ),types.SafetySetting( + category="HARM_CATEGORY_HARASSMENT", + threshold="OFF" + )], + tools = tools, + system_instruction=[types.Part.from_text(text=system_instruction)], + ) + +class Chatbot: + def __init__(self): + client = genai.Client( + vertexai=True, + project="neuralnet-manforce", + location="us-central1", + ) + self.model = client.chats.create( + model=MODEL_STR, + config=generate_content_config + ) + + def generate_response(self, user_input: str) -> str: + response = self.model.send_message(message=[user_input]).text + print(f"debug response: {response}", flush=True) #print to std.err + return response + +class Agent: + def __init__(self): + self.chatbot = None + + def allocated_resources(self): + self.chatbot = Chatbot() + +agent = Agent() +agent.allocated_resources() + + +def init_actions(): + """ + Example of an action performed by the Initalize ednpoint + """ + + print("resource initialized. . .") + + +def init_resources(session_id: str) -> List[Memory]: + """ + Example of an action performed by the Session ednpoint + """ + + private_memory = Memory(**{ + "session_id": session_id, + "name": "private json memory", + "value": { "example": "object" }, + "scope": MemoryScope.PRIVATE, + }) + public_memory = Memory(**{ + "session_id": session_id, + "name": "public string memory", + "value": "This is to be persisted", + "scope": MemoryScope.PUBLIC, + }) + + return [private_memory, public_memory] + +def get_welcome_response(): + # standard welcome message + + response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + + intent = Intent( + name="Welcome", + confidence=1, + ) + + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + } + + cards = None + return response, cards, intent, annotations + +def get_response(user_input: str): + """ + Example of an action performed by the Execute ednpoint + """ + + print(f"User said: {user_input}") + + # Response to be spoken by your Digital Person + reponse_dict = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." + + print(f"generated resp: {reponse_dict}") + cards, intent, annotations = None, None, None + response = "" + try: + reponse_dict = json.loads(reponse_dict) + if reponse_dict['uer_wants_to_watch_video']: + #response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" + #test show video + response = vidoe_intro[reponse_dict['language']] + "@showcards(card)" #"Hello! @showcards(card) Here is a video." + + cards = { + 'card': { + "type": "video", + "id": "youtubeVideo", + "data": { + "videoId": video_id[reponse_dict['type_of_video']], + "autoplay":"true", + "autoclose":"true" + } + } + } + else: + response = reponse_dict['response_text'] #+ " https://www.youtube.com/watch?v=Bhkm6fZMJcI" + + + except Exception as e: + print("error in reponse error decoding:",e) + + + + + + + + # Add your Cards as required + """cards = { + "card": { + "type": "image", + "data": { + "url": "https://placekitten.com/200/200", + "alt": "An adorable kitten", + }, + }, + }""" + + """ + # Add your Intent as required + intent = Intent( + name="Welcome", + confidence=1, + ) + + # If applicable, add your conversation annotations to see metrics for your Skill on Studio Insights + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + }""" + + return response, cards, intent, annotations diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 850820e..5e2a0b2 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,6 +1,7 @@ from fastapi import HTTPException #from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions -from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response +#from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response +from ..mocks.gemini_agent_2 import get_response, init_resources, init_actions, get_welcome_response from smskillsdk.models.common import MemoryScope, Intent From d6cd87f5e4dd6e66894e2f3b485d86498fd8aaab Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 22 Apr 2025 12:57:12 +0800 Subject: [PATCH 060/112] feat: enable google-genai in requirements for enhanced functionality --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6e7a503..cb47efb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ smskillsdk uvicorn vertexai google-cloud-aiplatform>=1.38 -#google-genai +google-genai pydantic<2 \ No newline at end of file From 5f1a221d9b1510a8d91ff8308c4ae8a5c5acf409 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 22 Apr 2025 13:11:04 +0800 Subject: [PATCH 061/112] feat: update pydantic version in requirements for compatibility --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cb47efb..2db37d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ uvicorn vertexai google-cloud-aiplatform>=1.38 google-genai -pydantic<2 \ No newline at end of file +pydantic +#pydantic<2 \ No newline at end of file From 06b0c88b20a8458ddabeee8d918d65125754b2b6 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 22 Apr 2025 13:17:03 +0800 Subject: [PATCH 062/112] fix: update ragCorpus and tool initialization for correct resource handling --- app/mocks/gemini_agent_2.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/mocks/gemini_agent_2.py b/app/mocks/gemini_agent_2.py index f8f5938..4c632f1 100644 --- a/app/mocks/gemini_agent_2.py +++ b/app/mocks/gemini_agent_2.py @@ -70,7 +70,7 @@ def set_person_data(data): googlesearch_tool = types.Tool(google_search=types.GoogleSearch()) -tools = [googlesearch_tool] + MEMORY_WINDOW_SIZE = 20 @@ -78,7 +78,7 @@ def set_person_data(data): DATA_STORE_REGION="us" project_id="neuralnet-manforce" datastore = f"projects/{project_id}/locations/{DATA_STORE_REGION}/collections/default_collection/dataStores/{DATA_STORE_ID}" -ragCorpus = "projects/civic-advantage-395410/locations/us-central1/ragCorpora/3379951520341557248" +ragCorpus = "projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952" retrieval_tool = types.Tool(retrieval=types.Retrieval(vertex_ai_search=types.VertexAISearch(datastore=datastore))) @@ -86,7 +86,7 @@ def set_person_data(data): retrieval=types.Retrieval( vertex_rag_store=types.VertexRagStore( rag_resources=[ - types.RagResource( + types.VertexRagStoreRagResource( rag_corpus=ragCorpus ) ], @@ -95,6 +95,8 @@ def set_person_data(data): ) ) +tools = [googlesearch_tool] + generate_content_config = types.GenerateContentConfig( temperature = 0.3, top_p = 0.95, From ed5d153b35b1f23aa2545d8d811930b3a2e3175c Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 22 Apr 2025 13:41:48 +0800 Subject: [PATCH 063/112] fallback to vertexai sdk due to dependancy problem. to update soulmachine sdk later --- app/mocks/gemini_agent.py | 7 ++++--- app/services/fake_nlp_service.py | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 8c5016d..7590419 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -41,8 +41,8 @@ def get_nonstreaming_text_response (response): "response_mime_type":"application/json" } -#MODEL_STR = "gemini-1.5-flash-002" -MODEL_STR = "gemini-2.0-flash-001" +MODEL_STR = "gemini-1.5-flash-002" +#MODEL_STR = "gemini-2.0-flash-001" """ You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). @@ -116,8 +116,9 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) #self.grounding_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) - self.grounding_tool = [datastore_grounding_tool] + #self.grounding_tool = [datastore_grounding_tool] #self.grounding_tool = [datastore_grounding_tool, googlesearch_tool] + self.grounding_tool = [googlesearch_tool] """ def use_rag_tool(self, user_prompt): diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 5e2a0b2..56e2b42 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,7 +1,7 @@ from fastapi import HTTPException #from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions -#from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response -from ..mocks.gemini_agent_2 import get_response, init_resources, init_actions, get_welcome_response +from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response +#from ..mocks.gemini_agent_2 import get_response, init_resources, init_actions, get_welcome_response from smskillsdk.models.common import MemoryScope, Intent diff --git a/requirements.txt b/requirements.txt index 2db37d1..62bbe15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ smskillsdk uvicorn vertexai google-cloud-aiplatform>=1.38 -google-genai +#google-genai pydantic -#pydantic<2 \ No newline at end of file +pydantic<2 #smskillsdk needs 1.9, google-genai needs > 2 \ No newline at end of file From 22553e7add707ee990b8e5d327f7fc0991a55b2f Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 22 Apr 2025 13:48:19 +0800 Subject: [PATCH 064/112] fix: update DATA_STORE_ID and adjust grounding_tool configuration for improved data handling --- app/mocks/gemini_agent.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 7590419..96927c2 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -93,7 +93,7 @@ def get_nonstreaming_text_response (response): MEMORY_WINDOW_SIZE = 20 -DATA_STORE_ID="acccim-ns_1740458649382" +DATA_STORE_ID="nsccci-kb_1745222443136" #"acccim-ns_1740458649382" DATA_STORE_REGION="us" project_id="neuralnet-manforce" datastore = f"projects/{project_id}/locations/{DATA_STORE_REGION}/collections/default_collection/dataStores/{DATA_STORE_ID}" @@ -115,10 +115,8 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st model, system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) - #self.grounding_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) - #self.grounding_tool = [datastore_grounding_tool] - #self.grounding_tool = [datastore_grounding_tool, googlesearch_tool] - self.grounding_tool = [googlesearch_tool] + self.grounding_tool = [datastore_grounding_tool] + #self.grounding_tool = [googlesearch_tool] """ def use_rag_tool(self, user_prompt): From f1864fe611f03db3c49d8681dcfaf6a2ed6a3157 Mon Sep 17 00:00:00 2001 From: SengTak Date: Wed, 23 Apr 2025 09:07:08 +0800 Subject: [PATCH 065/112] fix: update system_instruction to include Chinese name and enhance context for user interactions --- app/mocks/gemini_agent_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent_2.py b/app/mocks/gemini_agent_2.py index 4c632f1..c50818c 100644 --- a/app/mocks/gemini_agent_2.py +++ b/app/mocks/gemini_agent_2.py @@ -46,7 +46,7 @@ def set_person_data(data): "zh": "请欣赏视屏。" } -system_instruction = """You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). +system_instruction = """Your Chinese name is 小美, translated to English as 'XiaoMei'. You are an expert customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. From 36f920a0102242144a1f4d7063e98df490dd23fa Mon Sep 17 00:00:00 2001 From: SengTak Date: Sat, 26 Apr 2025 09:29:37 +0800 Subject: [PATCH 066/112] feat: add goodbye and idle response functions; enhance state management in FakeNLPService --- app/mocks/gemini_agent.py | 16 +++++++++++++++- app/services/fake_nlp_service.py | 27 ++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 96927c2..4e7a327 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -206,7 +206,8 @@ def init_resources(session_id: str) -> List[Memory]: def get_welcome_response(): # standard welcome message - response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + #response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + response = "" intent = Intent( name="Welcome", @@ -223,6 +224,19 @@ def get_welcome_response(): cards = None return response, cards, intent, annotations +def get_goodbye_response(): + cards, intent, annotations = None, None, None + response = "很高兴能为你服务，再见" + return response, cards, intent, annotations + +def get_idle_response(isWelcome=False): + cards, intent, annotations = None, None, None + if not isWelcome: + response = "" + else: + response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + return response, cards, intent, annotations + def get_response(user_input: str): """ Example of an action performed by the Execute ednpoint diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 56e2b42..f39c603 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,12 +1,12 @@ from fastapi import HTTPException #from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions -from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response +from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response, get_goodbye_response, get_idle_response #from ..mocks.gemini_agent_2 import get_response, init_resources, init_actions, get_welcome_response from smskillsdk.models.common import MemoryScope, Intent -#test +#enum of states: ["idle", "active"] class FakeNLPService: first_credentials: str @@ -16,6 +16,7 @@ def __init__(self, first_credentials, second_credentials): self.first_credentials = first_credentials self.second_credentials = second_credentials self.__authenticate() + self.state = "idle" def __authenticate(self): """ @@ -64,9 +65,25 @@ def send(self, user_input): Example of sending input to the third party NLP call """ - if user_input == "Welcome": - return get_welcome_response() - else: + #manage state here + if self.state == "idle": + # check if user_input contains wake words [""] + if "小美" in user_input and "你好" in user_input: + self.state = "active" + return get_idle_response(isWelcome=True) + else: + return get_idle_response(isWelcome=False) + elif self.state == "active": + # check if user_input contains wake words ["小美", "你好"] + if "小美" in user_input and "再见" in user_input: + self.state = "idle" + return get_goodbye_response() return get_response(user_input) + + #if user_input == "Welcome": + # return get_welcome_response() + #else: + # return get_response(user_input) + \ No newline at end of file From 8c1916054759e7ab3d1de93734f540fc495f9f78 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sat, 26 Apr 2025 09:38:12 +0800 Subject: [PATCH 067/112] fix: refactor state management to use global variable for _fake_nlp_state in FakeNLPService --- app/services/fake_nlp_service.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index f39c603..0e71d2e 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -5,8 +5,7 @@ from smskillsdk.models.common import MemoryScope, Intent - -#enum of states: ["idle", "active"] +_fake_nlp_state = "idle" class FakeNLPService: first_credentials: str @@ -16,7 +15,7 @@ def __init__(self, first_credentials, second_credentials): self.first_credentials = first_credentials self.second_credentials = second_credentials self.__authenticate() - self.state = "idle" + def __authenticate(self): """ @@ -66,17 +65,17 @@ def send(self, user_input): """ #manage state here - if self.state == "idle": + if _fake_nlp_state == "idle": # check if user_input contains wake words [""] if "小美" in user_input and "你好" in user_input: - self.state = "active" + _fake_nlp_state = "active" return get_idle_response(isWelcome=True) else: return get_idle_response(isWelcome=False) - elif self.state == "active": + elif _fake_nlp_state == "active": # check if user_input contains wake words ["小美", "你好"] if "小美" in user_input and "再见" in user_input: - self.state = "idle" + _fake_nlp_state = "idle" return get_goodbye_response() return get_response(user_input) From aad4d46e25687f100918698211f03cf4dc302a98 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sat, 26 Apr 2025 09:47:15 +0800 Subject: [PATCH 068/112] fix: refactor state management in FakeNLPService to use getter and setter functions for _fake_nlp_state --- app/services/fake_nlp_service.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 0e71d2e..4638e5c 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -7,14 +7,24 @@ _fake_nlp_state = "idle" +def get_fake_nlp_state(): + return _fake_nlp_state +def set_fake_nlp_state(state): + global _fake_nlp_state + _fake_nlp_state = state + + class FakeNLPService: first_credentials: str second_credentials: str + def __init__(self, first_credentials, second_credentials): self.first_credentials = first_credentials self.second_credentials = second_credentials self.__authenticate() + self.get_fake_nlp_state = get_fake_nlp_state + self.set_fake_nlp_state = set_fake_nlp_state def __authenticate(self): @@ -65,17 +75,17 @@ def send(self, user_input): """ #manage state here - if _fake_nlp_state == "idle": + if self.get_fake_nlp_state() == "idle": # check if user_input contains wake words [""] if "小美" in user_input and "你好" in user_input: - _fake_nlp_state = "active" + self.set_fake_nlp_state("active") return get_idle_response(isWelcome=True) else: return get_idle_response(isWelcome=False) - elif _fake_nlp_state == "active": + elif self.get_fake_nlp_state() == "active": # check if user_input contains wake words ["小美", "你好"] if "小美" in user_input and "再见" in user_input: - _fake_nlp_state = "idle" + self.set_fake_nlp_state("idle") return get_goodbye_response() return get_response(user_input) From 0196e68ff2ca8179ae8da3d48970d511b0cfce41 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sat, 26 Apr 2025 12:12:11 +0800 Subject: [PATCH 069/112] fix: update system_instruction to include Malay language support for user responses --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 4e7a327..32e2515 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -73,7 +73,7 @@ def get_nonstreaming_text_response (response): All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. - Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). + Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (English, Mandarin Chinese or Malay spoken in Malaysia). In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. @@ -83,7 +83,7 @@ def get_nonstreaming_text_response (response): "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, or answer yes to your previous invitation question to watch the video. "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", - "language": "en" or "zh", default to "en" + "language": "en" for English, "zh" for Chinese or "ms" for Malay, default to "en" } ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. From 13cb1cb40970b6486e00fd5323687448b9e7fb33 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sat, 26 Apr 2025 12:50:20 +0800 Subject: [PATCH 070/112] fix: update system_instruction to clarify default language usage and improve user response guidance --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 32e2515..aa73c43 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -83,7 +83,7 @@ def get_nonstreaming_text_response (response): "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, or answer yes to your previous invitation question to watch the video. "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", - "language": "en" for English, "zh" for Chinese or "ms" for Malay, default to "en" + "language": "en" for English, "zh" for Chinese or "ms" for Malay, default to "en" if you are not sure which language to use. } ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. From 57305116fbbf127df409989f6d8184bd6abe242f Mon Sep 17 00:00:00 2001 From: SengTak Date: Sat, 26 Apr 2025 17:04:34 +0800 Subject: [PATCH 071/112] fix: update video introduction text and add Malay language support in system_instruction --- app/mocks/gemini_agent.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index aa73c43..6c5467b 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -41,8 +41,8 @@ def get_nonstreaming_text_response (response): "response_mime_type":"application/json" } -MODEL_STR = "gemini-1.5-flash-002" -#MODEL_STR = "gemini-2.0-flash-001" +#MODEL_STR = "gemini-1.5-flash-002" +MODEL_STR = "gemini-2.0-flash-001" """ You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). @@ -63,8 +63,9 @@ def get_nonstreaming_text_response (response): } vidoe_intro ={ - "en": "Please enjoy the video. ", - "zh": "请欣赏视屏。" + "en": "Please enjoy the followung video clip.", + "zh": "请欣赏视屏。", + "ms": "Sila menikmati video berikutnya." } system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). @@ -78,6 +79,7 @@ def get_nonstreaming_text_response (response): If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. + ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. Respond in following schema: { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", From db7d1242edf3663148842ae6585fb877cb94a8ce Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 09:17:32 +0800 Subject: [PATCH 072/112] fix: update send method to manage state for "Welcome" input and return idle response --- app/services/fake_nlp_service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 4638e5c..02a23f3 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -73,6 +73,9 @@ def send(self, user_input): """ Example of sending input to the third party NLP call """ + if user_input == "Welcome": + self.set_fake_nlp_state("idle") + return get_idle_response(isWelcome=False) #manage state here if self.get_fake_nlp_state() == "idle": From be2b62b5feb64c23eb0596ee900a2299041fb85e Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 09:49:21 +0800 Subject: [PATCH 073/112] fix: manage state for "Welcome" input in send method and return idle response --- app/services/fake_nlp_service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 02a23f3..b68321a 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -73,6 +73,8 @@ def send(self, user_input): """ Example of sending input to the third party NLP call """ + print(f"State: {self.get_fake_nlp_state()} User said: {user_input}") + if user_input == "Welcome": self.set_fake_nlp_state("idle") return get_idle_response(isWelcome=False) @@ -91,6 +93,7 @@ def send(self, user_input): self.set_fake_nlp_state("idle") return get_goodbye_response() return get_response(user_input) + #if user_input == "Welcome": From ab90441a7cede596a02da7bb2e916a6a3e515096 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 09:49:48 +0800 Subject: [PATCH 074/112] fix: reorganize state management in send method for improved flow --- app/services/fake_nlp_service.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index b68321a..a9e4542 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -78,21 +78,21 @@ def send(self, user_input): if user_input == "Welcome": self.set_fake_nlp_state("idle") return get_idle_response(isWelcome=False) - - #manage state here - if self.get_fake_nlp_state() == "idle": - # check if user_input contains wake words [""] - if "小美" in user_input and "你好" in user_input: - self.set_fake_nlp_state("active") - return get_idle_response(isWelcome=True) - else: - return get_idle_response(isWelcome=False) - elif self.get_fake_nlp_state() == "active": - # check if user_input contains wake words ["小美", "你好"] - if "小美" in user_input and "再见" in user_input: - self.set_fake_nlp_state("idle") - return get_goodbye_response() - return get_response(user_input) + else: + #manage state here + if self.get_fake_nlp_state() == "idle": + # check if user_input contains wake words [""] + if "小美" in user_input and "你好" in user_input: + self.set_fake_nlp_state("active") + return get_idle_response(isWelcome=True) + else: + return get_idle_response(isWelcome=False) + elif self.get_fake_nlp_state() == "active": + # check if user_input contains wake words ["小美", "你好"] + if "小美" in user_input and "再见" in user_input: + self.set_fake_nlp_state("idle") + return get_goodbye_response() + return get_response(user_input) From e98b3cfd6c644921188b61cd2abbd65aa2832066 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 11:17:04 +0800 Subject: [PATCH 075/112] fix: add get_person_data function and correct video introduction text --- app/mocks/gemini_agent.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 6c5467b..2de8079 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -23,6 +23,9 @@ def set_person_data(data): _person_data = data print("set person data:", _person_data) +def get_person_data(): + return _person_data + def get_nonstreaming_text_response (response): return response.candidates[0].content.parts[0]._raw_part.text @@ -63,8 +66,8 @@ def get_nonstreaming_text_response (response): } vidoe_intro ={ - "en": "Please enjoy the followung video clip.", - "zh": "请欣赏视屏。", + "en": "Please enjoy the following video clip.", + "zh": "请欣赏接下来的视屏。", "ms": "Sila menikmati video berikutnya." } @@ -118,6 +121,7 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) self.grounding_tool = [datastore_grounding_tool] + self.get_person_data = get_person_data #self.grounding_tool = [googlesearch_tool] """ @@ -147,7 +151,7 @@ def generate_response(self, user_prompt=""): #prompt = user_prompt if len(self.chat._history): - prompt = f"""Your last message was :"{self.chat._history[-1].parts[0]._raw_part.text}" Please respond in the same language as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". + prompt = f"""Name of the person talking to you is: {self.get_person_data()}\n. Your last message was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". """ else: prompt = user_prompt From 0e9a3c49b6ab5c83fea4ecc9e19a3df842b0b3de Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 11:17:57 +0800 Subject: [PATCH 076/112] fix: adjust generation configuration parameters for improved response quality --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 2de8079..306db16 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -38,8 +38,8 @@ def get_nonstreaming_text_response (response): } generation_config = { "max_output_tokens": 256, - "temperature": 0.3, #0.5, - "top_p": 0.9, #0.5, #0.5 better than 0.95 + "temperature": 0.1, #0.5, + "top_p": 0.95, #0.5, #0.5 better than 0.95 "top_k": 40, "response_mime_type":"application/json" } From f740133feae202b1b7b0083dfd94412e6d5ccfb0 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 11:22:24 +0800 Subject: [PATCH 077/112] fix: enhance get_person_data function to provide contextual information --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 306db16..835318b 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -24,7 +24,7 @@ def set_person_data(data): print("set person data:", _person_data) def get_person_data(): - return _person_data + return f"Name of the person talking to you is: {_person_data}.\n" if _person_data else "" def get_nonstreaming_text_response (response): @@ -151,7 +151,7 @@ def generate_response(self, user_prompt=""): #prompt = user_prompt if len(self.chat._history): - prompt = f"""Name of the person talking to you is: {self.get_person_data()}\n. Your last message was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". + prompt = f"""{self.get_person_data()} Your last message was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". """ else: prompt = user_prompt From b9f45b2df1b6ee5c8adecc39306459d2d4b2278b Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 16:19:28 +0800 Subject: [PATCH 078/112] fix: update system instruction for improved clarity on NSCCCI and Negeri Sembilan queries --- app/mocks/gemini_agent.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 835318b..7f75508 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -71,10 +71,11 @@ def get_nonstreaming_text_response (response): "ms": "Sila menikmati video berikutnya." } -system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). +# All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. + +system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. - 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. - All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. + 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (English, Mandarin Chinese or Malay spoken in Malaysia). @@ -90,7 +91,8 @@ def get_nonstreaming_text_response (response): "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", "language": "en" for English, "zh" for Chinese or "ms" for Malay, default to "en" if you are not sure which language to use. } - ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. + ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, its economy, tourism, food and culture and etc. + You may also answer to queries related to Malaysia where Negeri Sembilan is one of the states in Malaysia. If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. """] From 0fab9020779cf74ee7f4c4f5b25e653403356916 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 20:51:02 +0800 Subject: [PATCH 079/112] fix: adjust temperature setting in generation configuration for improved response quality --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 7f75508..a4eb855 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -38,7 +38,7 @@ def get_nonstreaming_text_response (response): } generation_config = { "max_output_tokens": 256, - "temperature": 0.1, #0.5, + "temperature": 0.3, #0.5, "top_p": 0.95, #0.5, #0.5 better than 0.95 "top_k": 40, "response_mime_type":"application/json" @@ -99,7 +99,7 @@ def get_nonstreaming_text_response (response): MEMORY_WINDOW_SIZE = 20 - +# "projects/neuralnet-manforce/locations/us/collections/default_collection/dataStores/nsccci-kb_1745222443136" DATA_STORE_ID="nsccci-kb_1745222443136" #"acccim-ns_1740458649382" DATA_STORE_REGION="us" project_id="neuralnet-manforce" From c38d435bc9d086bdb4709b266396625daad81199 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 21:06:05 +0800 Subject: [PATCH 080/112] fix: integrate RAG retrieval tool for enhanced data sourcing in chatbot --- app/mocks/gemini_agent.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index a4eb855..ccdd08d 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -116,15 +116,36 @@ def get_nonstreaming_text_response (response): ) googlesearch_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) +rag_retrieval_config = rag.RagRetrievalConfig( + top_k=10, # Optional + filter=rag.Filter(vector_distance_threshold=0.5), # Optional +) +rag_corpus = rag.get_corpus("projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952") +rag_retrieval_tool = Tool.from_retrieval( + retrieval=rag.Retrieval( + source=rag.VertexRagStore( + rag_resources=[ + rag.RagResource( + rag_corpus=rag_corpus.name, # Currently only 1 corpus is allowed. + # Optional: supply IDs from `rag.list_files()`. + # rag_file_ids=["rag-file-1", "rag-file-2", ...], + ) + ], + rag_retrieval_config=rag_retrieval_config, + ), + ) +) + class Chatbot: def __init__(self, history: Optional[List["Content"]] = None, model: Optional[str] = "gemini-1.5-flash-002", use_search=False): self.model = GenerativeModel( model, system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) - self.grounding_tool = [datastore_grounding_tool] self.get_person_data = get_person_data + #self.grounding_tool = [datastore_grounding_tool] #self.grounding_tool = [googlesearch_tool] + self.grounding_tool = [rag_retrieval_tool] """ def use_rag_tool(self, user_prompt): From 65cbd7c7495c845213ca0cf0fcb3d45eb829ea93 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 21:21:32 +0800 Subject: [PATCH 081/112] fix: update prompt formatting in chatbot response for improved clarity --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index ccdd08d..4e68580 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -174,7 +174,7 @@ def generate_response(self, user_prompt=""): #prompt = user_prompt if len(self.chat._history): - prompt = f"""{self.get_person_data()} Your last message was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". + prompt = f"""Your previous response was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". """ else: prompt = user_prompt From 406bab3f7a8728f5b88968d61c88981307c86b9b Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 21:25:29 +0800 Subject: [PATCH 082/112] fix: comment out rag retrieval configuration for future reference --- app/mocks/gemini_agent.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 4e68580..5115e2f 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -116,10 +116,10 @@ def get_nonstreaming_text_response (response): ) googlesearch_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) -rag_retrieval_config = rag.RagRetrievalConfig( - top_k=10, # Optional - filter=rag.Filter(vector_distance_threshold=0.5), # Optional -) +#rag_retrieval_config = rag.RagRetrievalConfig( +# top_k=10, # Optional +# filter=rag.Filter(vector_distance_threshold=0.5), # Optional +#) rag_corpus = rag.get_corpus("projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952") rag_retrieval_tool = Tool.from_retrieval( retrieval=rag.Retrieval( @@ -131,7 +131,7 @@ def get_nonstreaming_text_response (response): # rag_file_ids=["rag-file-1", "rag-file-2", ...], ) ], - rag_retrieval_config=rag_retrieval_config, + #rag_retrieval_config=rag_retrieval_config, ), ) ) From 7e99f9f5e2d2b8cff0a74582a945fdab364d2e75 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 21:37:34 +0800 Subject: [PATCH 083/112] fix: toggle model version and grounding tool configurations for testing --- app/mocks/gemini_agent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 5115e2f..f7cc408 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -44,8 +44,8 @@ def get_nonstreaming_text_response (response): "response_mime_type":"application/json" } -#MODEL_STR = "gemini-1.5-flash-002" -MODEL_STR = "gemini-2.0-flash-001" +MODEL_STR = "gemini-1.5-flash-002" +#MODEL_STR = "gemini-2.0-flash-001" """ You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). @@ -143,9 +143,9 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) self.get_person_data = get_person_data - #self.grounding_tool = [datastore_grounding_tool] + self.grounding_tool = [datastore_grounding_tool] #self.grounding_tool = [googlesearch_tool] - self.grounding_tool = [rag_retrieval_tool] + #self.grounding_tool = [rag_retrieval_tool] """ def use_rag_tool(self, user_prompt): From e9718f2b3fb79c660512871487b824b32d3ac26e Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 22:19:21 +0800 Subject: [PATCH 084/112] fix: update chatbot response instructions for clarity and language consistency --- app/mocks/gemini_agent.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index f7cc408..3d8f29b 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -91,11 +91,11 @@ def get_nonstreaming_text_response (response): "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", "language": "en" for English, "zh" for Chinese or "ms" for Malay, default to "en" if you are not sure which language to use. } - ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, its economy, tourism, food and culture and etc. - You may also answer to queries related to Malaysia where Negeri Sembilan is one of the states in Malaysia. - If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. """] +# ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, its economy, tourism, food and culture and etc. +# You may also answer to queries related to Malaysia where Negeri Sembilan is one of the states in Malaysia. +# If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. MEMORY_WINDOW_SIZE = 20 @@ -174,8 +174,10 @@ def generate_response(self, user_prompt=""): #prompt = user_prompt if len(self.chat._history): - prompt = f"""Your previous response was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". - """ + #prompt = f"""Your previous response was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". + #""" + + prompt = f"""Respond in the SAME LANGUAGE as user's CURRENT MESSAGE, and user's CURRENT MESSAGE is :"{user_prompt}". Your response:\n""" else: prompt = user_prompt From 2c6e9f5d976672e941c5a654dc98a983d1bdb296 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 22:24:10 +0800 Subject: [PATCH 085/112] fix: enhance language consistency in chatbot responses by clarifying prompt instructions --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 3d8f29b..522e82f 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -177,7 +177,7 @@ def generate_response(self, user_prompt=""): #prompt = f"""Your previous response was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". #""" - prompt = f"""Respond in the SAME LANGUAGE as user's CURRENT MESSAGE, and user's CURRENT MESSAGE is :"{user_prompt}". Your response:\n""" + prompt = f"""Irrespective of grounding data language, always respond in the SAME LANGUAGE as user's CURRENT MESSAGE, which is as follow:\n"{user_prompt}".\nYour response:\n""" else: prompt = user_prompt From 4bda662bc5a7af786806dd5b85b27d3b2553499c Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 23:29:47 +0800 Subject: [PATCH 086/112] fix: update grounding tool configuration for chatbot to use rag retrieval tool --- app/mocks/gemini_agent.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 522e82f..0dc40a9 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -116,10 +116,7 @@ def get_nonstreaming_text_response (response): ) googlesearch_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) -#rag_retrieval_config = rag.RagRetrievalConfig( -# top_k=10, # Optional -# filter=rag.Filter(vector_distance_threshold=0.5), # Optional -#) + rag_corpus = rag.get_corpus("projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952") rag_retrieval_tool = Tool.from_retrieval( retrieval=rag.Retrieval( @@ -143,9 +140,9 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) self.get_person_data = get_person_data - self.grounding_tool = [datastore_grounding_tool] + #self.grounding_tool = [datastore_grounding_tool] #self.grounding_tool = [googlesearch_tool] - #self.grounding_tool = [rag_retrieval_tool] + self.grounding_tool = [rag_retrieval_tool] """ def use_rag_tool(self, user_prompt): From 39cab99b255f19fd1d87e5c860d6c57bb9fe8d4d Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 23:41:30 +0800 Subject: [PATCH 087/112] fix: update grounding tool configuration in Chatbot class for testing purposes --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 0dc40a9..727eb8e 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -140,9 +140,9 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) self.get_person_data = get_person_data - #self.grounding_tool = [datastore_grounding_tool] + self.grounding_tool = [datastore_grounding_tool] #self.grounding_tool = [googlesearch_tool] - self.grounding_tool = [rag_retrieval_tool] + #self.grounding_tool = [rag_retrieval_tool] """ def use_rag_tool(self, user_prompt): From 4e05bc14eb749479798f590022461f1cd75a7371 Mon Sep 17 00:00:00 2001 From: SengTak Date: Sun, 27 Apr 2025 23:50:42 +0800 Subject: [PATCH 088/112] fix: update model version and enhance system instructions for clarity --- app/mocks/gemini_agent.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 727eb8e..072f20a 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -44,8 +44,8 @@ def get_nonstreaming_text_response (response): "response_mime_type":"application/json" } -MODEL_STR = "gemini-1.5-flash-002" -#MODEL_STR = "gemini-2.0-flash-001" +#MODEL_STR = "gemini-1.5-flash-002" +MODEL_STR = "gemini-2.0-flash-001" """ You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). @@ -76,6 +76,8 @@ def get_nonstreaming_text_response (response): system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. + 森美兰州中华总商会现任会长是拿督吕海庭。The President of N.S.C.C.C.I is Dato' Looi Hoi Ting. + 马来西亚中华总商会(简称中总)现任全国总会长是拿督吴逸平硕士。The President The Associated Chinese Chamber of Commerce and Industry Malaysia (A.C.C.C.I.M) is Datuk Ng Yih Pyng. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (English, Mandarin Chinese or Malay spoken in Malaysia). From c0ec9068c169d93c0143875160f2eda1bbf6f211 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 06:32:48 +0800 Subject: [PATCH 089/112] fix: update grounding tool configuration in Chatbot class to use rag retrieval tool --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 072f20a..fae4a99 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -142,9 +142,9 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) self.get_person_data = get_person_data - self.grounding_tool = [datastore_grounding_tool] + #self.grounding_tool = [datastore_grounding_tool] #self.grounding_tool = [googlesearch_tool] - #self.grounding_tool = [rag_retrieval_tool] + self.grounding_tool = [rag_retrieval_tool] """ def use_rag_tool(self, user_prompt): From baddc4538b6223439de575cbd97ad8f0d8dae83d Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 07:22:15 +0800 Subject: [PATCH 090/112] fix: update grounding tool configuration in Chatbot class and enhance wake word detection in FakeNLPService --- app/mocks/gemini_agent.py | 4 ++-- app/services/fake_nlp_service.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index fae4a99..072f20a 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -142,9 +142,9 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) self.get_person_data = get_person_data - #self.grounding_tool = [datastore_grounding_tool] + self.grounding_tool = [datastore_grounding_tool] #self.grounding_tool = [googlesearch_tool] - self.grounding_tool = [rag_retrieval_tool] + #self.grounding_tool = [rag_retrieval_tool] """ def use_rag_tool(self, user_prompt): diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index a9e4542..f47f5fa 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -82,14 +82,14 @@ def send(self, user_input): #manage state here if self.get_fake_nlp_state() == "idle": # check if user_input contains wake words [""] - if "小美" in user_input and "你好" in user_input: + if ("小美" in user_input or "小米" in user_input) and "你好" in user_input: self.set_fake_nlp_state("active") return get_idle_response(isWelcome=True) else: return get_idle_response(isWelcome=False) elif self.get_fake_nlp_state() == "active": # check if user_input contains wake words ["小美", "你好"] - if "小美" in user_input and "再见" in user_input: + if ("小美" in user_input or "小米" in user_input) and "再见" in user_input: self.set_fake_nlp_state("idle") return get_goodbye_response() return get_response(user_input) From 729babbb6fe29163f808a835ae9d0f23855f6828 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 07:27:03 +0800 Subject: [PATCH 091/112] fix: update system instructions in gemini_agent.py to include historical context for A.C.C.C.I.M and N.S.C.C.C.I --- app/mocks/gemini_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 072f20a..d929ca3 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -78,6 +78,7 @@ def get_nonstreaming_text_response (response): 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. 森美兰州中华总商会现任会长是拿督吕海庭。The President of N.S.C.C.C.I is Dato' Looi Hoi Ting. 马来西亚中华总商会(简称中总)现任全国总会长是拿督吴逸平硕士。The President The Associated Chinese Chamber of Commerce and Industry Malaysia (A.C.C.C.I.M) is Datuk Ng Yih Pyng. + 马来西亚中华总商会是于1921年成立。The A.C.C.C.I.M was founded in 1921. 森美兰州中华总商会是于1946年成立。The N.S.C.C.C.I was founded in 1946. Now it is year 2025 A.D.. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (English, Mandarin Chinese or Malay spoken in Malaysia). From f822cd9c71aafc143aee73235d2d00b88a8c2189 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 07:29:41 +0800 Subject: [PATCH 092/112] fix: update rag retrieval configuration in gemini_agent.py for improved context retrieval --- app/mocks/gemini_agent.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index d929ca3..dd57727 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -8,7 +8,8 @@ from typing import List from smskillsdk.models.common import Memory, MemoryScope, Intent import vertexai -from vertexai.preview import rag +#from vertexai.preview import rag +from vertexai import rag from vertexai.generative_models import GenerativeModel, Part, FinishReason, Tool, Content import vertexai.preview.generative_models as generative_models from vertexai.preview.generative_models import grounding @@ -121,6 +122,14 @@ def get_nonstreaming_text_response (response): rag_corpus = rag.get_corpus("projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952") + +# Direct context retrieval +rag_retrieval_config = rag.RagRetrievalConfig( + top_k=5, # Optional + filter=rag.Filter(vector_distance_threshold=0.5), # Optional +) + + rag_retrieval_tool = Tool.from_retrieval( retrieval=rag.Retrieval( source=rag.VertexRagStore( @@ -131,7 +140,7 @@ def get_nonstreaming_text_response (response): # rag_file_ids=["rag-file-1", "rag-file-2", ...], ) ], - #rag_retrieval_config=rag_retrieval_config, + rag_retrieval_config=rag_retrieval_config, ), ) ) @@ -143,9 +152,9 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) self.get_person_data = get_person_data - self.grounding_tool = [datastore_grounding_tool] + #self.grounding_tool = [datastore_grounding_tool] #self.grounding_tool = [googlesearch_tool] - #self.grounding_tool = [rag_retrieval_tool] + self.grounding_tool = [rag_retrieval_tool] """ def use_rag_tool(self, user_prompt): From 6c54d9c572f5069f7bab605bf6e1e2d092164993 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 07:51:42 +0800 Subject: [PATCH 093/112] fix: refine system instructions in gemini_agent.py for improved user interaction and clarity --- app/mocks/gemini_agent.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index dd57727..9ce2634 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -84,10 +84,9 @@ def get_nonstreaming_text_response (response): DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (English, Mandarin Chinese or Malay spoken in Malaysia). In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). - If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, - You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. + 避免一直重复使用“您好”在回答的开头。在适当的时候则可以。 Respond in following schema: { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", @@ -97,10 +96,12 @@ def get_nonstreaming_text_response (response): } """] -# ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, its economy, tourism, food and culture and etc. -# You may also answer to queries related to Malaysia where Negeri Sembilan is one of the states in Malaysia. -# If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. - +# ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, its economy, tourism, food and culture and etc. +# You may also answer to queries related to Malaysia where Negeri Sembilan is one of the states in Malaysia. +# If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. +# If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, +# You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. + MEMORY_WINDOW_SIZE = 20 # "projects/neuralnet-manforce/locations/us/collections/default_collection/dataStores/nsccci-kb_1745222443136" From 12d9c0cb4aba235b0ee9334ee834c4157ff574c6 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 08:03:52 +0800 Subject: [PATCH 094/112] fix: refine system instructions in gemini_agent.py to enhance user engagement and reduce redundancy --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 9ce2634..656b677 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -86,7 +86,7 @@ def get_nonstreaming_text_response (response): In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. - 避免一直重复使用“您好”在回答的开头。在适当的时候则可以。 + 避免一直重复使用“您好”或“你好”。在适当的时候则可以。避免一直问是否要播放视屏, 让user主动要求。 Respond in following schema: { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", From 5c34359cea8b0c98e8a222d756f2380d4f810b0e Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 08:33:29 +0800 Subject: [PATCH 095/112] fix: update system instructions in gemini_agent.py for improved clarity and user guidance --- app/mocks/gemini_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 656b677..e282683 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -153,9 +153,9 @@ def __init__(self, history: Optional[List["Content"]] = None, model: Optional[st system_instruction=system_instruction) self.chat = self.model.start_chat(history=history) self.get_person_data = get_person_data - #self.grounding_tool = [datastore_grounding_tool] + self.grounding_tool = [datastore_grounding_tool] #self.grounding_tool = [googlesearch_tool] - self.grounding_tool = [rag_retrieval_tool] + #self.grounding_tool = [rag_retrieval_tool] """ def use_rag_tool(self, user_prompt): From 52e7e0b043fd8908443bca3cb3608547914593a6 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 08:37:30 +0800 Subject: [PATCH 096/112] fix: comment out rag retrieval configuration in gemini_agent.py for clarity --- app/mocks/gemini_agent.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index e282683..392819b 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -8,8 +8,8 @@ from typing import List from smskillsdk.models.common import Memory, MemoryScope, Intent import vertexai -#from vertexai.preview import rag -from vertexai import rag +from vertexai.preview import rag +#from vertexai import rag from vertexai.generative_models import GenerativeModel, Part, FinishReason, Tool, Content import vertexai.preview.generative_models as generative_models from vertexai.preview.generative_models import grounding @@ -125,10 +125,10 @@ def get_nonstreaming_text_response (response): rag_corpus = rag.get_corpus("projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952") # Direct context retrieval -rag_retrieval_config = rag.RagRetrievalConfig( - top_k=5, # Optional - filter=rag.Filter(vector_distance_threshold=0.5), # Optional -) +#rag_retrieval_config = rag.RagRetrievalConfig( +# top_k=5, # Optional +# filter=rag.Filter(vector_distance_threshold=0.5), # Optional +#) rag_retrieval_tool = Tool.from_retrieval( @@ -141,7 +141,7 @@ def get_nonstreaming_text_response (response): # rag_file_ids=["rag-file-1", "rag-file-2", ...], ) ], - rag_retrieval_config=rag_retrieval_config, + #rag_retrieval_config=rag_retrieval_config, ), ) ) From cc379b10cbe9ab723a4393c28b4138c9615da8db Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 08:44:53 +0800 Subject: [PATCH 097/112] fix: enhance system instructions in gemini_agent.py for improved clarity and detail --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 392819b..b6c38c9 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -75,7 +75,7 @@ def get_nonstreaming_text_response (response): # All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). - Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. + Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. You can answer questions regarding the NSCCCI Chamber's history, mission, vision, etc. 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. 森美兰州中华总商会现任会长是拿督吕海庭。The President of N.S.C.C.C.I is Dato' Looi Hoi Ting. 马来西亚中华总商会(简称中总)现任全国总会长是拿督吴逸平硕士。The President The Associated Chinese Chamber of Commerce and Industry Malaysia (A.C.C.C.I.M) is Datuk Ng Yih Pyng. From a06fa2bc99c4e185ae0050e2fcca87e50320e4be Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 08:45:59 +0800 Subject: [PATCH 098/112] fix: refine system instructions in gemini_agent.py for improved clarity and user engagement --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index b6c38c9..6e4080d 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -86,7 +86,7 @@ def get_nonstreaming_text_response (response): In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. - 避免一直重复使用“您好”或“你好”。在适当的时候则可以。避免一直问是否要播放视屏, 让user主动要求。 + 避免使用“您好”或“你好”。避免一直问是否要播放视屏, 让user主动要求。 Respond in following schema: { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", From bce1b299b48470648bfea71673b19c53acbf87d9 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 28 Apr 2025 09:01:31 +0800 Subject: [PATCH 099/112] fix: update system instructions in gemini_agent.py to enhance user engagement and provide clearer guidance --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 6e4080d..5231450 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -86,7 +86,7 @@ def get_nonstreaming_text_response (response): In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. - 避免使用“您好”或“你好”。避免一直问是否要播放视屏, 让user主动要求。 + 避免使用“您好”或“你好”。避免一直问是否要播放视屏, 让user主动要求。Always ask "is there anything else you want me to help you with?" "请问还有什么我可以帮您解答的吗？" in the end of your response. Respond in following schema: { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", From f8d3e8fb4dc17a180a64d800398a31a5478a9145 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 12 May 2025 08:49:50 +0800 Subject: [PATCH 100/112] for opening --- app/mocks/gemini_agent.py | 12 +- app/mocks/gemini_agent_backup.py | 351 +++++++++++++++++++++++++++++++ app/services/fake_nlp_service.py | 5 +- 3 files changed, 366 insertions(+), 2 deletions(-) create mode 100644 app/mocks/gemini_agent_backup.py diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 5231450..e22880b 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -86,7 +86,12 @@ def get_nonstreaming_text_response (response): In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. - 避免使用“您好”或“你好”。避免一直问是否要播放视屏, 让user主动要求。Always ask "is there anything else you want me to help you with?" "请问还有什么我可以帮您解答的吗？" in the end of your response. + + YOUR MAIN LANGUAGE IS ENGLISH. + + WHEN YOU ARE ASKED QUESTIONS SIMILAR to: "do you recognize the guest who is standing in front of you?" or "cna you tell me more about the gentleman standing in front of you?", you SHOULD ALWAYS ANSWER WITH THE FOLLOWING SCRIPT:\n + "Yes of course. I would like extend our heartfelt gratitude to Yang Berhormat DATO' SERI UTAMA HAJI AMINUDDIN BIN HARUN for honouring the opening ceremony of the building. I would like to introduce my peer who is conversant in Bahasa Melayu, the Oracle." + Respond in following schema: { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", @@ -278,6 +283,11 @@ def get_idle_response(isWelcome=False): response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" return response, cards, intent, annotations +def get_hello_response(): + cards, intent, annotations = None, None, None + response = f"Hello my honoured guests, I am Xiao Mei. Welcome to NSCCCI. How may I serve you?" + return response, cards, intent, annotations + def get_response(user_input: str): """ Example of an action performed by the Execute ednpoint diff --git a/app/mocks/gemini_agent_backup.py b/app/mocks/gemini_agent_backup.py new file mode 100644 index 0000000..5231450 --- /dev/null +++ b/app/mocks/gemini_agent_backup.py @@ -0,0 +1,351 @@ +""" +These functions use Promises and setTimeouts to mock HTTP requests to a third part NLP service +and should be replaced with the actual HTTP calls when implementing. +""" + +#ref https://docs.soulmachines.com/skills-api/getting-started/nlp-adapter-skill#advanced-concepts + +from typing import List +from smskillsdk.models.common import Memory, MemoryScope, Intent +import vertexai +from vertexai.preview import rag +#from vertexai import rag +from vertexai.generative_models import GenerativeModel, Part, FinishReason, Tool, Content +import vertexai.preview.generative_models as generative_models +from vertexai.preview.generative_models import grounding +from typing import List, Optional +import json + +# Add these after the other global variables +_person_data = "" + +def set_person_data(data): + global _person_data + _person_data = data + print("set person data:", _person_data) + +def get_person_data(): + return f"Name of the person talking to you is: {_person_data}.\n" if _person_data else "" + + +def get_nonstreaming_text_response (response): + return response.candidates[0].content.parts[0]._raw_part.text + +safety_settings={ + generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_NONE, + } +generation_config = { + "max_output_tokens": 256, + "temperature": 0.3, #0.5, + "top_p": 0.95, #0.5, #0.5 better than 0.95 + "top_k": 40, + "response_mime_type":"application/json" +} + +#MODEL_STR = "gemini-1.5-flash-002" +MODEL_STR = "gemini-2.0-flash-001" + +""" +You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). +When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. +If the user wants to watch the youtube video, you MUST append this youtube URL in the end of your response with no accompanying text or punctuation. +Below is the context for videos you are able to show: +- youtube URL video about NSCCCI: https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 +- youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" + +video_url = { +"video_about_chamber_of_commerce": "https://www.youtube.com/embed/Bhkm6fZMJcI?autoplay=1&mute=0", #"https://www.youtube.com/watch?v=Bhkm6fZMJcI", +"video_about_vision_valley": "https://www.youtube.com/watch?v=GgUYagMYkkg" +} + +video_id = { + "video_about_chamber_of_commerce": "Bhkm6fZMJcI", + "video_about_vision_valley": "GgUYagMYkkg" +} + +vidoe_intro ={ + "en": "Please enjoy the following video clip.", + "zh": "请欣赏接下来的视屏。", + "ms": "Sila menikmati video berikutnya." +} + +# All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. + +system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). + Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. You can answer questions regarding the NSCCCI Chamber's history, mission, vision, etc. + 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. + 森美兰州中华总商会现任会长是拿督吕海庭。The President of N.S.C.C.C.I is Dato' Looi Hoi Ting. + 马来西亚中华总商会(简称中总)现任全国总会长是拿督吴逸平硕士。The President The Associated Chinese Chamber of Commerce and Industry Malaysia (A.C.C.C.I.M) is Datuk Ng Yih Pyng. + 马来西亚中华总商会是于1921年成立。The A.C.C.C.I.M was founded in 1921. 森美兰州中华总商会是于1946年成立。The N.S.C.C.C.I was founded in 1946. Now it is year 2025 A.D.. + Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. + DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. + Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (English, Mandarin Chinese or Malay spoken in Malaysia). + In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). + You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. + ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. + 避免使用“您好”或“你好”。避免一直问是否要播放视屏, 让user主动要求。Always ask "is there anything else you want me to help you with?" "请问还有什么我可以帮您解答的吗？" in the end of your response. + Respond in following schema: + { + "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", + "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, or answer yes to your previous invitation question to watch the video. + "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", + "language": "en" for English, "zh" for Chinese or "ms" for Malay, default to "en" if you are not sure which language to use. + } + """] + +# ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, its economy, tourism, food and culture and etc. +# You may also answer to queries related to Malaysia where Negeri Sembilan is one of the states in Malaysia. +# If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. +# If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, +# You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. + + +MEMORY_WINDOW_SIZE = 20 +# "projects/neuralnet-manforce/locations/us/collections/default_collection/dataStores/nsccci-kb_1745222443136" +DATA_STORE_ID="nsccci-kb_1745222443136" #"acccim-ns_1740458649382" +DATA_STORE_REGION="us" +project_id="neuralnet-manforce" +datastore = f"projects/{project_id}/locations/{DATA_STORE_REGION}/collections/default_collection/dataStores/{DATA_STORE_ID}" +datastore_grounding_tool = Tool.from_retrieval( + grounding.Retrieval( + grounding.VertexAISearch( + project=project_id, + datastore=DATA_STORE_ID, + location=DATA_STORE_REGION, + #datastore=datastore, + ) + ) + ) +googlesearch_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) + + +rag_corpus = rag.get_corpus("projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952") + +# Direct context retrieval +#rag_retrieval_config = rag.RagRetrievalConfig( +# top_k=5, # Optional +# filter=rag.Filter(vector_distance_threshold=0.5), # Optional +#) + + +rag_retrieval_tool = Tool.from_retrieval( + retrieval=rag.Retrieval( + source=rag.VertexRagStore( + rag_resources=[ + rag.RagResource( + rag_corpus=rag_corpus.name, # Currently only 1 corpus is allowed. + # Optional: supply IDs from `rag.list_files()`. + # rag_file_ids=["rag-file-1", "rag-file-2", ...], + ) + ], + #rag_retrieval_config=rag_retrieval_config, + ), + ) +) + +class Chatbot: + def __init__(self, history: Optional[List["Content"]] = None, model: Optional[str] = "gemini-1.5-flash-002", use_search=False): + self.model = GenerativeModel( + model, + system_instruction=system_instruction) + self.chat = self.model.start_chat(history=history) + self.get_person_data = get_person_data + self.grounding_tool = [datastore_grounding_tool] + #self.grounding_tool = [googlesearch_tool] + #self.grounding_tool = [rag_retrieval_tool] + + """ + def use_rag_tool(self, user_prompt): + return self.chat.send_message( + user_prompt, + tools=[tool], + generation_config=generation_config, + #safety_settings=safety_settings, + stream=False + )""" + + def use_search(self, prompt): + return self.chat.send_message( + #f"Contexts: {contexts}. Message from User: {user_prompt}", + [prompt], + tools=self.grounding_tool, + generation_config=generation_config, + #safety_settings=safety_settings, + stream=False + ) + + + + def generate_response(self, user_prompt=""): + #prompt = user_prompt + #prompt = user_prompt + + if len(self.chat._history): + #prompt = f"""Your previous response was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". + #""" + + prompt = f"""Irrespective of grounding data language, always respond in the SAME LANGUAGE as user's CURRENT MESSAGE, which is as follow:\n"{user_prompt}".\nYour response:\n""" + else: + prompt = user_prompt + + response = self.use_search(prompt) + + self.chat._history[-2] = Content( + role="user", + parts=[Part.from_text(user_prompt)] # Create Part objects + ) + + if len(self.chat._history) > MEMORY_WINDOW_SIZE: + self.chat._history = self.chat._history[-MEMORY_WINDOW_SIZE:] + + return get_nonstreaming_text_response(response) + +vertexai.init(project="neuralnet-manforce", location="us-central1") + +class Agent: + def __init__(self, model = ""): + self.chatbot = None + self.model = model + + def allocated_resources(self): + self.chatbot = Chatbot(model=self.model) + +agent = Agent(model=MODEL_STR) +agent.allocated_resources() + +def init_actions(): + """ + Example of an action performed by the Initalize ednpoint + """ + + print("resource initialized. . .") + + +def init_resources(session_id: str) -> List[Memory]: + """ + Example of an action performed by the Session ednpoint + """ + + private_memory = Memory(**{ + "session_id": session_id, + "name": "private json memory", + "value": { "example": "object" }, + "scope": MemoryScope.PRIVATE, + }) + public_memory = Memory(**{ + "session_id": session_id, + "name": "public string memory", + "value": "This is to be persisted", + "scope": MemoryScope.PUBLIC, + }) + + return [private_memory, public_memory] + +def get_welcome_response(): + # standard welcome message + + #response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + response = "" + + intent = Intent( + name="Welcome", + confidence=1, + ) + + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + } + + cards = None + return response, cards, intent, annotations + +def get_goodbye_response(): + cards, intent, annotations = None, None, None + response = "很高兴能为你服务，再见" + return response, cards, intent, annotations + +def get_idle_response(isWelcome=False): + cards, intent, annotations = None, None, None + if not isWelcome: + response = "" + else: + response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + return response, cards, intent, annotations + +def get_response(user_input: str): + """ + Example of an action performed by the Execute ednpoint + """ + + print(f"User said: {user_input}") + + # Response to be spoken by your Digital Person + reponse_dict = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." + + print(f"generated resp: {reponse_dict}") + cards, intent, annotations = None, None, None + response = "" + try: + reponse_dict = json.loads(reponse_dict) + if reponse_dict['uer_wants_to_watch_video']: + #response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" + #test show video + response = vidoe_intro[reponse_dict['language']] + "@showcards(card)" #"Hello! @showcards(card) Here is a video." + + cards = { + 'card': { + "type": "video", + "id": "youtubeVideo", + "data": { + "videoId": video_id[reponse_dict['type_of_video']], + "autoplay":"true", + "autoclose":"true" + } + } + } + else: + response = reponse_dict['response_text'] #+ " https://www.youtube.com/watch?v=Bhkm6fZMJcI" + + + except Exception as e: + print("error in reponse error decoding:",e) + + + + + + + + # Add your Cards as required + """cards = { + "card": { + "type": "image", + "data": { + "url": "https://placekitten.com/200/200", + "alt": "An adorable kitten", + }, + }, + }""" + + """ + # Add your Intent as required + intent = Intent( + name="Welcome", + confidence=1, + ) + + # If applicable, add your conversation annotations to see metrics for your Skill on Studio Insights + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + }""" + + return response, cards, intent, annotations \ No newline at end of file diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index f47f5fa..28baf39 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,6 +1,6 @@ from fastapi import HTTPException #from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions -from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response, get_goodbye_response, get_idle_response +from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response, get_goodbye_response, get_idle_response, get_hello_response #from ..mocks.gemini_agent_2 import get_response, init_resources, init_actions, get_welcome_response from smskillsdk.models.common import MemoryScope, Intent @@ -78,6 +78,9 @@ def send(self, user_input): if user_input == "Welcome": self.set_fake_nlp_state("idle") return get_idle_response(isWelcome=False) + elif user_input == "Mayday1234": + self.set_fake_nlp_state("active") + return get_hello_response() else: #manage state here if self.get_fake_nlp_state() == "idle": From 6ca7bf986e06a71743ce4b24aa34399d41862eb0 Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 12 May 2025 15:31:36 +0800 Subject: [PATCH 101/112] fix: correct spelling in system instructions and add quiet mode to goodbye response --- app/mocks/gemini_agent.py | 11 +++++++---- app/services/fake_nlp_service.py | 3 +++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index e22880b..dd2039b 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -89,8 +89,8 @@ def get_nonstreaming_text_response (response): YOUR MAIN LANGUAGE IS ENGLISH. - WHEN YOU ARE ASKED QUESTIONS SIMILAR to: "do you recognize the guest who is standing in front of you?" or "cna you tell me more about the gentleman standing in front of you?", you SHOULD ALWAYS ANSWER WITH THE FOLLOWING SCRIPT:\n - "Yes of course. I would like extend our heartfelt gratitude to Yang Berhormat DATO' SERI UTAMA HAJI AMINUDDIN BIN HARUN for honouring the opening ceremony of the building. I would like to introduce my peer who is conversant in Bahasa Melayu, the Oracle." + WHEN YOU ARE ASKED QUESTIONS SIMILAR to: "do you recognize the guest who is standing in front of you?" or "can you tell me more about the gentleman standing in front of you?", you SHOULD ALWAYS ANSWER WITH THE FOLLOWING SCRIPT:\n + "Yes of course. I would like extend our heartfelt gratitude to Yang Berhormat Datuk Seri Utama Haji Aminudin Bin Harun for honouring the opening ceremony of the building. I would like take the occasion to introduce my peer who is conversant in Bahasa Melayu, the Oracle." Respond in following schema: { @@ -270,9 +270,12 @@ def get_welcome_response(): cards = None return response, cards, intent, annotations -def get_goodbye_response(): +def get_goodbye_response(beQuiet=False): cards, intent, annotations = None, None, None - response = "很高兴能为你服务，再见" + if beQuiet: + response = "" + else: + response = "很高兴能为你服务，再见" return response, cards, intent, annotations def get_idle_response(isWelcome=False): diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 28baf39..9089975 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -81,6 +81,9 @@ def send(self, user_input): elif user_input == "Mayday1234": self.set_fake_nlp_state("active") return get_hello_response() + elif user_input == "Byebye1234": + self.set_fake_nlp_state("idle") + return get_goodbye_response(beQuiet=True) else: #manage state here if self.get_fake_nlp_state() == "idle": From ed6dcb8614329a8eb72afcd597adc38da558d6dd Mon Sep 17 00:00:00 2001 From: SengTak Date: Mon, 12 May 2025 23:32:36 +0800 Subject: [PATCH 102/112] fix: update user input handling in FakeNLPService to manage state transitions more effectively --- app/services/fake_nlp_service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 9089975..342965e 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -79,8 +79,11 @@ def send(self, user_input): self.set_fake_nlp_state("idle") return get_idle_response(isWelcome=False) elif user_input == "Mayday1234": - self.set_fake_nlp_state("active") + self.set_fake_nlp_state("idle") return get_hello_response() + elif user_input == "Wakeup1234": + self.set_fake_nlp_state("active") + return get_idle_response(isWelcome=False) elif user_input == "Byebye1234": self.set_fake_nlp_state("idle") return get_goodbye_response(beQuiet=True) From 72c8d52483a3f09587393b5a69825e1df7646fb5 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 11:39:15 +0800 Subject: [PATCH 103/112] fix: update system instructions in gemini_agent.py to include specific response for guest recognition and enhance welcome message --- app/mocks/gemini_agent.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index dd2039b..ecbdbde 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -74,6 +74,13 @@ def get_nonstreaming_text_response (response): # All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. +""" + + WHEN YOU ARE ASKED QUESTIONS SIMILAR to: "do you recognize the guest who is standing in front of you?" or "can you tell me more about the gentleman standing in front of you?", you SHOULD ALWAYS ANSWER WITH THE FOLLOWING SCRIPT:\n + "Yes of course. I would like extend our heartfelt gratitude to Yang Berhormat Datuk Seri Utama Haji Aminudin Bin Harun for honouring the opening ceremony of the building. I would like take the occasion to introduce my peer who is conversant in Bahasa Melayu, the Oracle." +""" + + system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. You can answer questions regarding the NSCCCI Chamber's history, mission, vision, etc. 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. @@ -89,9 +96,6 @@ def get_nonstreaming_text_response (response): YOUR MAIN LANGUAGE IS ENGLISH. - WHEN YOU ARE ASKED QUESTIONS SIMILAR to: "do you recognize the guest who is standing in front of you?" or "can you tell me more about the gentleman standing in front of you?", you SHOULD ALWAYS ANSWER WITH THE FOLLOWING SCRIPT:\n - "Yes of course. I would like extend our heartfelt gratitude to Yang Berhormat Datuk Seri Utama Haji Aminudin Bin Harun for honouring the opening ceremony of the building. I would like take the occasion to introduce my peer who is conversant in Bahasa Melayu, the Oracle." - Respond in following schema: { "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", @@ -288,7 +292,7 @@ def get_idle_response(isWelcome=False): def get_hello_response(): cards, intent, annotations = None, None, None - response = f"Hello my honoured guests, I am Xiao Mei. Welcome to NSCCCI. How may I serve you?" + response = f"Hello and welcome our honoured guest, Yang Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, I am Xiao Mei. We thank you for officiating the opening ceremony of our building." return response, cards, intent, annotations def get_response(user_input: str): From 445418ac3412528f8aeaf5fe47a94f01a84922da Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 12:20:30 +0800 Subject: [PATCH 104/112] fix: enhance welcome message in get_hello_response to improve user engagement and clarity --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index ecbdbde..ed6b029 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -292,7 +292,7 @@ def get_idle_response(isWelcome=False): def get_hello_response(): cards, intent, annotations = None, None, None - response = f"Hello and welcome our honoured guest, Yang Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, I am Xiao Mei. We thank you for officiating the opening ceremony of our building." + response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun. Menteri Besar of Negeri Seremban. I am Xiao Mei. We thank you for officiating the opening ceremony of our building. How may I assist you?" return response, cards, intent, annotations def get_response(user_input: str): From 139ee5a2312e39081667491fac8df99c7a4feb0d Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 12:25:01 +0800 Subject: [PATCH 105/112] fix: improve welcome message in get_hello_response for clarity and accuracy --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index ed6b029..79ffc6a 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -292,7 +292,7 @@ def get_idle_response(isWelcome=False): def get_hello_response(): cards, intent, annotations = None, None, None - response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun. Menteri Besar of Negeri Seremban. I am Xiao Mei. We thank you for officiating the opening ceremony of our building. How may I assist you?" + response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, the Menteri besar of Negeri Sembilan. I am Xiao Mei. We thank you for officiating the opening ceremony of our building. How may I assist you?" return response, cards, intent, annotations def get_response(user_input: str): From 9af9ccb6ecf1c9d60703a700fef3d89cbdf91f6c Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 12:31:23 +0800 Subject: [PATCH 106/112] fix: enhance welcome message in get_hello_response to clarify AI role and improve user engagement --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 79ffc6a..3eaa617 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -292,7 +292,7 @@ def get_idle_response(isWelcome=False): def get_hello_response(): cards, intent, annotations = None, None, None - response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, the Menteri besar of Negeri Sembilan. I am Xiao Mei. We thank you for officiating the opening ceremony of our building. How may I assist you?" + response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, the Menteri besar of Negeri Sembilan. I am Xiao Mei. I am the AI ambassador of Negeri Sembilan Chinese Chamber of Commerce and Industry. We thank you for officiating the opening ceremony of our building. How may I assist you?" return response, cards, intent, annotations def get_response(user_input: str): From e5454f6a5ee6b8e2c951c128a6e70d348bff4cd9 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 12:35:06 +0800 Subject: [PATCH 107/112] fix: refine welcome message in get_hello_response for improved clarity and engagement --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 3eaa617..714c155 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -292,7 +292,7 @@ def get_idle_response(isWelcome=False): def get_hello_response(): cards, intent, annotations = None, None, None - response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, the Menteri besar of Negeri Sembilan. I am Xiao Mei. I am the AI ambassador of Negeri Sembilan Chinese Chamber of Commerce and Industry. We thank you for officiating the opening ceremony of our building. How may I assist you?" + response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, the Menteri besar of Negeri Sembilan. I am Xiao Mei, the AI ambassador of Negeri Sembilan Chinese Chamber of Commerce and Industry. We thank you for officiating the opening ceremony of our building. How may I assist you?" return response, cards, intent, annotations def get_response(user_input: str): From 36c067099a8a936a21524b9c01182acc479c45ec Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 13:04:57 +0800 Subject: [PATCH 108/112] fix: update system instructions to improve user query handling and clarify response language --- app/mocks/gemini_agent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 714c155..3180bd0 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -82,9 +82,10 @@ def get_nonstreaming_text_response (response): system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). + You will excuse typos from user, who may sometimes miss typed miss spelled 'Negeri Sembilan' (like milan, nogori and etc), the users are always referring to Negeri Sembilan Chinese Chamber of Commerce. Do fuzzy matching and directly respond to that query with the right answer. Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. You can answer questions regarding the NSCCCI Chamber's history, mission, vision, etc. 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. - 森美兰州中华总商会现任会长是拿督吕海庭。The President of N.S.C.C.C.I is Dato' Looi Hoi Ting. + 森美兰州中华总商会现任会长是拿督吕海庭。The President of N.S.C.C.C.I is Dato' Looi Hi Teng. 马来西亚中华总商会(简称中总)现任全国总会长是拿督吴逸平硕士。The President The Associated Chinese Chamber of Commerce and Industry Malaysia (A.C.C.C.I.M) is Datuk Ng Yih Pyng. 马来西亚中华总商会是于1921年成立。The A.C.C.C.I.M was founded in 1921. 森美兰州中华总商会是于1946年成立。The N.S.C.C.C.I was founded in 1946. Now it is year 2025 A.D.. Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. @@ -94,7 +95,7 @@ def get_nonstreaming_text_response (response): You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. - YOUR MAIN LANGUAGE IS ENGLISH. + YOUR MAIN LANGUAGE IS ENGLISH. ALWAYS RESPOND IN ENGLISH. Respond in following schema: { From 3d157108aa574705b9a240314a67986b7cd72b10 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 13:06:46 +0800 Subject: [PATCH 109/112] fix: update system instruction to include agent name and improve user query handling --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index 3180bd0..a985d15 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -81,7 +81,7 @@ def get_nonstreaming_text_response (response): """ -system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). +system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). Your name is XiaoMei 小美. You will excuse typos from user, who may sometimes miss typed miss spelled 'Negeri Sembilan' (like milan, nogori and etc), the users are always referring to Negeri Sembilan Chinese Chamber of Commerce. Do fuzzy matching and directly respond to that query with the right answer. Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. You can answer questions regarding the NSCCCI Chamber's history, mission, vision, etc. 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. From 73213f0e489a2d628becb22bbb2678985e4e91cb Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 13:12:51 +0800 Subject: [PATCH 110/112] fix: update idle response message for improved clarity and user engagement --- app/mocks/gemini_agent_backup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent_backup.py b/app/mocks/gemini_agent_backup.py index 5231450..7aad9cd 100644 --- a/app/mocks/gemini_agent_backup.py +++ b/app/mocks/gemini_agent_backup.py @@ -275,7 +275,8 @@ def get_idle_response(isWelcome=False): if not isWelcome: response = "" else: - response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + #response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + response = f"Hello! How may I assist you?" return response, cards, intent, annotations def get_response(user_input: str): From ab47d9c7dccb507e21221328989fbabf13f3df27 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 13:19:46 +0800 Subject: [PATCH 111/112] fix: update response language in prompt to ensure consistent English replies --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index a985d15..b50d095 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -197,7 +197,7 @@ def generate_response(self, user_prompt=""): #prompt = f"""Your previous response was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". #""" - prompt = f"""Irrespective of grounding data language, always respond in the SAME LANGUAGE as user's CURRENT MESSAGE, which is as follow:\n"{user_prompt}".\nYour response:\n""" + prompt = f"""Irrespective of grounding data language, always respond in ENGLISH, which is as follow:\n"{user_prompt}".\nYour response:\n""" else: prompt = user_prompt From c551c26e3c9793cf82b667c1dd83578b47b92698 Mon Sep 17 00:00:00 2001 From: SengTak Date: Tue, 13 May 2025 19:47:32 +0800 Subject: [PATCH 112/112] fix: update welcome message in get_hello_response for improved accuracy --- app/mocks/gemini_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py index b50d095..33f5750 100644 --- a/app/mocks/gemini_agent.py +++ b/app/mocks/gemini_agent.py @@ -293,7 +293,7 @@ def get_idle_response(isWelcome=False): def get_hello_response(): cards, intent, annotations = None, None, None - response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, the Menteri besar of Negeri Sembilan. I am Xiao Mei, the AI ambassador of Negeri Sembilan Chinese Chamber of Commerce and Industry. We thank you for officiating the opening ceremony of our building. How may I assist you?" + response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, the Menteri besar of Negeri Sembilan. I am Xiao Mei, the AI ambassador of Negeri Sembilan Chinese Chamber of Commerce and Industry. We thank you for officiating the opening ceremony of the Ban Koh Conference Hall. How may I assist you?" return response, cards, intent, annotations def get_response(user_input: str):