diff --git a/.env_example b/.env_example deleted file mode 100644 index 47257fa..0000000 --- a/.env_example +++ /dev/null @@ -1,4 +0,0 @@ -PORT=3000 -ENV=development -DEBUG=True -PYTHONUNBUFFERED=1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1774af1..841f362 100644 --- a/.gitignore +++ b/.gitignore @@ -128,4 +128,7 @@ dmypy.json # Pyre type checker .pyre/ -.DS_Store \ No newline at end of file +.DS_Store + +# experimental +.tests/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0bb80fd..34ea70f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.0 +FROM python:3.11-slim ENV VIRTUAL_ENV=venv RUN python3 -m venv $VIRTUAL_ENV @@ -7,11 +7,13 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY ./requirements.txt /app/requirements.txt WORKDIR /app -RUN pip install --upgrade pip -RUN pip install -r requirements.txt +#RUN pip install --upgrade pip +RUN pip install --trusted-host pypi.python.org -r requirements.txt COPY . /app -EXPOSE 3000 +EXPOSE 8080 CMD [ "python", "manage.py", "run" ] + +#ENTRYPOINT ["python", "webhook.py"] diff --git a/README.md b/README.md index e116f47..038cc6c 100644 --- a/README.md +++ b/README.md @@ -88,4 +88,7 @@ This repository is licensed under the Apache License, Version 2.0. See [LICENSE](./LICENSE) for the full license text. ## Issues -For any issues, please reach out to one of our Customer Success team members. \ No newline at end of file +For any issues, please reach out to one of our Customer Success team members. + +## REFs +* [nlp skill](https://docs.soulmachines.com/skills-api/getting-started/nlp-adapter-skill) \ No newline at end of file diff --git a/app/app.py b/app/app.py index ca0e93e..6698090 100644 --- a/app/app.py +++ b/app/app.py @@ -1,3 +1,53 @@ +#for pydantic > 2 +""" +import sys +import importlib.util +import importlib.machinery +import types +from typing import List +#from pydantic import RootModel + +# Define our fixed class +class FixedConversationHistory(RootModel): + root: List # Will be properly typed later + +# Function to patch the module +def patch_api_module(): + # Get the spec + spec = importlib.util.find_spec('smskillsdk.models.api') + if not spec: + raise ImportError("Module smskillsdk.models.api not found") + + # Create a new module object + module = types.ModuleType('smskillsdk.models.api') + + # Add it to sys.modules early + sys.modules['smskillsdk.models.api'] = module + + # Load the source code as a string + source = spec.loader.get_source('smskillsdk.models.api') + + # Modify the source code to remove or fix the problematic class + modified_source = source.replace( + "class ConversationHistory(BaseModel):", + "# Original ConversationHistory commented out" + ).replace( + " __root__: List[HistoryItem]", + " # __root__: List[HistoryItem]" + ) + + # Compile and execute the modified source + code = compile(modified_source, spec.origin, 'exec') + exec(code, module.__dict__) + + # Now inject our fixed class + module.ConversationHistory = FixedConversationHistory + + return module """ + +# Patch the module before anyone else imports it +#patched_module = patch_api_module() + from fastapi import FastAPI from .views import skill diff --git a/app/config/base_config.py b/app/config/base_config.py index 24ea36e..a4a15c1 100644 --- a/app/config/base_config.py +++ b/app/config/base_config.py @@ -5,9 +5,9 @@ class BaseConfig(BaseSettings): Base application configuration """ - port: int = 3000 - env: str - debug: bool + port: int = 8080 + env: str = "DEVELOPMENT" + debug: bool = True class Config: env_file = ".env" diff --git a/app/mocks/gemini_agent.py b/app/mocks/gemini_agent.py new file mode 100644 index 0000000..33f5750 --- /dev/null +++ b/app/mocks/gemini_agent.py @@ -0,0 +1,369 @@ +""" +These functions use Promises and setTimeouts to mock HTTP requests to a third part NLP service +and should be replaced with the actual HTTP calls when implementing. +""" + +#ref https://docs.soulmachines.com/skills-api/getting-started/nlp-adapter-skill#advanced-concepts + +from typing import List +from smskillsdk.models.common import Memory, MemoryScope, Intent +import vertexai +from vertexai.preview import rag +#from vertexai import rag +from vertexai.generative_models import GenerativeModel, Part, FinishReason, Tool, Content +import vertexai.preview.generative_models as generative_models +from vertexai.preview.generative_models import grounding +from typing import List, Optional +import json + +# Add these after the other global variables +_person_data = "" + +def set_person_data(data): + global _person_data + _person_data = data + print("set person data:", _person_data) + +def get_person_data(): + return f"Name of the person talking to you is: {_person_data}.\n" if _person_data else "" + + +def get_nonstreaming_text_response (response): + return response.candidates[0].content.parts[0]._raw_part.text + +safety_settings={ + generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_NONE, + } +generation_config = { + "max_output_tokens": 256, + "temperature": 0.3, #0.5, + "top_p": 0.95, #0.5, #0.5 better than 0.95 + "top_k": 40, + "response_mime_type":"application/json" +} + +#MODEL_STR = "gemini-1.5-flash-002" +MODEL_STR = "gemini-2.0-flash-001" + +""" +You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). +When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. +If the user wants to watch the youtube video, you MUST append this youtube URL in the end of your response with no accompanying text or punctuation. +Below is the context for videos you are able to show: +- youtube URL video about NSCCCI: https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 +- youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" + +video_url = { +"video_about_chamber_of_commerce": "https://www.youtube.com/embed/Bhkm6fZMJcI?autoplay=1&mute=0", #"https://www.youtube.com/watch?v=Bhkm6fZMJcI", +"video_about_vision_valley": "https://www.youtube.com/watch?v=GgUYagMYkkg" +} + +video_id = { + "video_about_chamber_of_commerce": "Bhkm6fZMJcI", + "video_about_vision_valley": "GgUYagMYkkg" +} + +vidoe_intro ={ + "en": "Please enjoy the following video clip.", + "zh": "请欣赏接下来的视屏。", + "ms": "Sila menikmati video berikutnya." +} + +# All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. + +""" + + WHEN YOU ARE ASKED QUESTIONS SIMILAR to: "do you recognize the guest who is standing in front of you?" or "can you tell me more about the gentleman standing in front of you?", you SHOULD ALWAYS ANSWER WITH THE FOLLOWING SCRIPT:\n + "Yes of course. I would like extend our heartfelt gratitude to Yang Berhormat Datuk Seri Utama Haji Aminudin Bin Harun for honouring the opening ceremony of the building. I would like take the occasion to introduce my peer who is conversant in Bahasa Melayu, the Oracle." +""" + + +system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). Your name is XiaoMei 小美. + You will excuse typos from user, who may sometimes miss typed miss spelled 'Negeri Sembilan' (like milan, nogori and etc), the users are always referring to Negeri Sembilan Chinese Chamber of Commerce. Do fuzzy matching and directly respond to that query with the right answer. + Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. You can answer questions regarding the NSCCCI Chamber's history, mission, vision, etc. + 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. + 森美兰州中华总商会现任会长是拿督吕海庭。The President of N.S.C.C.C.I is Dato' Looi Hi Teng. + 马来西亚中华总商会(简称中总)现任全国总会长是拿督吴逸平硕士。The President The Associated Chinese Chamber of Commerce and Industry Malaysia (A.C.C.C.I.M) is Datuk Ng Yih Pyng. + 马来西亚中华总商会是于1921年成立。The A.C.C.C.I.M was founded in 1921. 森美兰州中华总商会是于1946年成立。The N.S.C.C.C.I was founded in 1946. Now it is year 2025 A.D.. + Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. + DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. + Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (English, Mandarin Chinese or Malay spoken in Malaysia). + In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). + You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. + ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. + + YOUR MAIN LANGUAGE IS ENGLISH. ALWAYS RESPOND IN ENGLISH. + + Respond in following schema: + { + "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", + "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, or answer yes to your previous invitation question to watch the video. + "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", + "language": "en" for English, "zh" for Chinese or "ms" for Malay, default to "en" if you are not sure which language to use. + } + """] + +# ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, its economy, tourism, food and culture and etc. +# You may also answer to queries related to Malaysia where Negeri Sembilan is one of the states in Malaysia. +# If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. +# If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, +# You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. + + +MEMORY_WINDOW_SIZE = 20 +# "projects/neuralnet-manforce/locations/us/collections/default_collection/dataStores/nsccci-kb_1745222443136" +DATA_STORE_ID="nsccci-kb_1745222443136" #"acccim-ns_1740458649382" +DATA_STORE_REGION="us" +project_id="neuralnet-manforce" +datastore = f"projects/{project_id}/locations/{DATA_STORE_REGION}/collections/default_collection/dataStores/{DATA_STORE_ID}" +datastore_grounding_tool = Tool.from_retrieval( + grounding.Retrieval( + grounding.VertexAISearch( + project=project_id, + datastore=DATA_STORE_ID, + location=DATA_STORE_REGION, + #datastore=datastore, + ) + ) + ) +googlesearch_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) + + +rag_corpus = rag.get_corpus("projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952") + +# Direct context retrieval +#rag_retrieval_config = rag.RagRetrievalConfig( +# top_k=5, # Optional +# filter=rag.Filter(vector_distance_threshold=0.5), # Optional +#) + + +rag_retrieval_tool = Tool.from_retrieval( + retrieval=rag.Retrieval( + source=rag.VertexRagStore( + rag_resources=[ + rag.RagResource( + rag_corpus=rag_corpus.name, # Currently only 1 corpus is allowed. + # Optional: supply IDs from `rag.list_files()`. + # rag_file_ids=["rag-file-1", "rag-file-2", ...], + ) + ], + #rag_retrieval_config=rag_retrieval_config, + ), + ) +) + +class Chatbot: + def __init__(self, history: Optional[List["Content"]] = None, model: Optional[str] = "gemini-1.5-flash-002", use_search=False): + self.model = GenerativeModel( + model, + system_instruction=system_instruction) + self.chat = self.model.start_chat(history=history) + self.get_person_data = get_person_data + self.grounding_tool = [datastore_grounding_tool] + #self.grounding_tool = [googlesearch_tool] + #self.grounding_tool = [rag_retrieval_tool] + + """ + def use_rag_tool(self, user_prompt): + return self.chat.send_message( + user_prompt, + tools=[tool], + generation_config=generation_config, + #safety_settings=safety_settings, + stream=False + )""" + + def use_search(self, prompt): + return self.chat.send_message( + #f"Contexts: {contexts}. Message from User: {user_prompt}", + [prompt], + tools=self.grounding_tool, + generation_config=generation_config, + #safety_settings=safety_settings, + stream=False + ) + + + + def generate_response(self, user_prompt=""): + #prompt = user_prompt + #prompt = user_prompt + + if len(self.chat._history): + #prompt = f"""Your previous response was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". + #""" + + prompt = f"""Irrespective of grounding data language, always respond in ENGLISH, which is as follow:\n"{user_prompt}".\nYour response:\n""" + else: + prompt = user_prompt + + response = self.use_search(prompt) + + self.chat._history[-2] = Content( + role="user", + parts=[Part.from_text(user_prompt)] # Create Part objects + ) + + if len(self.chat._history) > MEMORY_WINDOW_SIZE: + self.chat._history = self.chat._history[-MEMORY_WINDOW_SIZE:] + + return get_nonstreaming_text_response(response) + +vertexai.init(project="neuralnet-manforce", location="us-central1") + +class Agent: + def __init__(self, model = ""): + self.chatbot = None + self.model = model + + def allocated_resources(self): + self.chatbot = Chatbot(model=self.model) + +agent = Agent(model=MODEL_STR) +agent.allocated_resources() + +def init_actions(): + """ + Example of an action performed by the Initalize ednpoint + """ + + print("resource initialized. . .") + + +def init_resources(session_id: str) -> List[Memory]: + """ + Example of an action performed by the Session ednpoint + """ + + private_memory = Memory(**{ + "session_id": session_id, + "name": "private json memory", + "value": { "example": "object" }, + "scope": MemoryScope.PRIVATE, + }) + public_memory = Memory(**{ + "session_id": session_id, + "name": "public string memory", + "value": "This is to be persisted", + "scope": MemoryScope.PUBLIC, + }) + + return [private_memory, public_memory] + +def get_welcome_response(): + # standard welcome message + + #response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + response = "" + + intent = Intent( + name="Welcome", + confidence=1, + ) + + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + } + + cards = None + return response, cards, intent, annotations + +def get_goodbye_response(beQuiet=False): + cards, intent, annotations = None, None, None + if beQuiet: + response = "" + else: + response = "很高兴能为你服务，再见" + return response, cards, intent, annotations + +def get_idle_response(isWelcome=False): + cards, intent, annotations = None, None, None + if not isWelcome: + response = "" + else: + response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + return response, cards, intent, annotations + +def get_hello_response(): + cards, intent, annotations = None, None, None + response = f"Hello and welcome our honoured guests, Yang Amat Berhormat Datuk Seri Utama Haji Aminudin Bin Harun, the Menteri besar of Negeri Sembilan. I am Xiao Mei, the AI ambassador of Negeri Sembilan Chinese Chamber of Commerce and Industry. We thank you for officiating the opening ceremony of the Ban Koh Conference Hall. How may I assist you?" + return response, cards, intent, annotations + +def get_response(user_input: str): + """ + Example of an action performed by the Execute ednpoint + """ + + print(f"User said: {user_input}") + + # Response to be spoken by your Digital Person + reponse_dict = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." + + print(f"generated resp: {reponse_dict}") + cards, intent, annotations = None, None, None + response = "" + try: + reponse_dict = json.loads(reponse_dict) + if reponse_dict['uer_wants_to_watch_video']: + #response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" + #test show video + response = vidoe_intro[reponse_dict['language']] + "@showcards(card)" #"Hello! @showcards(card) Here is a video." + + cards = { + 'card': { + "type": "video", + "id": "youtubeVideo", + "data": { + "videoId": video_id[reponse_dict['type_of_video']], + "autoplay":"true", + "autoclose":"true" + } + } + } + else: + response = reponse_dict['response_text'] #+ " https://www.youtube.com/watch?v=Bhkm6fZMJcI" + + + except Exception as e: + print("error in reponse error decoding:",e) + + + + + + + + # Add your Cards as required + """cards = { + "card": { + "type": "image", + "data": { + "url": "https://placekitten.com/200/200", + "alt": "An adorable kitten", + }, + }, + }""" + + """ + # Add your Intent as required + intent = Intent( + name="Welcome", + confidence=1, + ) + + # If applicable, add your conversation annotations to see metrics for your Skill on Studio Insights + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + }""" + + return response, cards, intent, annotations \ No newline at end of file diff --git a/app/mocks/gemini_agent_2.py b/app/mocks/gemini_agent_2.py new file mode 100644 index 0000000..c50818c --- /dev/null +++ b/app/mocks/gemini_agent_2.py @@ -0,0 +1,276 @@ +""" +These functions use Promises and setTimeouts to mock HTTP requests to a third part NLP service +and should be replaced with the actual HTTP calls when implementing. +""" +from typing import List +from smskillsdk.models.common import Memory, MemoryScope, Intent +from google import genai +from google.genai.chats import Chat +from google.genai import types +from typing import List, Optional +import json +# use google genai +#ref https://docs.soulmachines.com/skills-api/getting-started/nlp-adapter-skill#advanced-concepts + +# Add these after the other global variables +_person_data = "" + +def set_person_data(data): + global _person_data + _person_data = data + print("set person data:", _person_data) + + +MODEL_STR = "gemini-2.0-flash-001" + +""" +You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). +When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. +If the user wants to watch the youtube video, you MUST append this youtube URL in the end of your response with no accompanying text or punctuation. +Below is the context for videos you are able to show: +- youtube URL video about NSCCCI: https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 +- youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" + +video_url = { +"video_about_chamber_of_commerce": "https://www.youtube.com/embed/Bhkm6fZMJcI?autoplay=1&mute=0", #"https://www.youtube.com/watch?v=Bhkm6fZMJcI", +"video_about_vision_valley": "https://www.youtube.com/watch?v=GgUYagMYkkg" +} + +video_id = { + "video_about_chamber_of_commerce": "Bhkm6fZMJcI", + "video_about_vision_valley": "GgUYagMYkkg" +} + +vidoe_intro ={ + "en": "Please enjoy the video. ", + "zh": "请欣赏视屏。" +} + +system_instruction = """Your Chinese name is 小美, translated to English as 'XiaoMei'. You are an expert customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”). + Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. + 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. + All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. + Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. + DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. + Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (either English or Chinese). + In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). + If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, + You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. + You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. + Respond in following schema: + { + "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", + "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, or answer yes to your previous invitation question to watch the video. + "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", + "language": "en" or "zh", default to "en" + } + ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. + If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. + """ + +googlesearch_tool = types.Tool(google_search=types.GoogleSearch()) + + + +MEMORY_WINDOW_SIZE = 20 + +DATA_STORE_ID="acccim-ns_1740458649382" +DATA_STORE_REGION="us" +project_id="neuralnet-manforce" +datastore = f"projects/{project_id}/locations/{DATA_STORE_REGION}/collections/default_collection/dataStores/{DATA_STORE_ID}" +ragCorpus = "projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952" + +retrieval_tool = types.Tool(retrieval=types.Retrieval(vertex_ai_search=types.VertexAISearch(datastore=datastore))) + +ragretriever_tool = types.Tool( + retrieval=types.Retrieval( + vertex_rag_store=types.VertexRagStore( + rag_resources=[ + types.VertexRagStoreRagResource( + rag_corpus=ragCorpus + ) + ], + similarity_top_k=10, + ) + ) + ) + +tools = [googlesearch_tool] + +generate_content_config = types.GenerateContentConfig( + temperature = 0.3, + top_p = 0.95, + max_output_tokens = 256, + response_modalities = ["TEXT"], + response_mime_type = "application/json", + speech_config = types.SpeechConfig( + voice_config = types.VoiceConfig( + prebuilt_voice_config = types.PrebuiltVoiceConfig( + voice_name = "zephyr" + ) + ), + ), + safety_settings = [types.SafetySetting( + category="HARM_CATEGORY_HATE_SPEECH", + threshold="OFF" + ),types.SafetySetting( + category="HARM_CATEGORY_DANGEROUS_CONTENT", + threshold="OFF" + ),types.SafetySetting( + category="HARM_CATEGORY_SEXUALLY_EXPLICIT", + threshold="OFF" + ),types.SafetySetting( + category="HARM_CATEGORY_HARASSMENT", + threshold="OFF" + )], + tools = tools, + system_instruction=[types.Part.from_text(text=system_instruction)], + ) + +class Chatbot: + def __init__(self): + client = genai.Client( + vertexai=True, + project="neuralnet-manforce", + location="us-central1", + ) + self.model = client.chats.create( + model=MODEL_STR, + config=generate_content_config + ) + + def generate_response(self, user_input: str) -> str: + response = self.model.send_message(message=[user_input]).text + print(f"debug response: {response}", flush=True) #print to std.err + return response + +class Agent: + def __init__(self): + self.chatbot = None + + def allocated_resources(self): + self.chatbot = Chatbot() + +agent = Agent() +agent.allocated_resources() + + +def init_actions(): + """ + Example of an action performed by the Initalize ednpoint + """ + + print("resource initialized. . .") + + +def init_resources(session_id: str) -> List[Memory]: + """ + Example of an action performed by the Session ednpoint + """ + + private_memory = Memory(**{ + "session_id": session_id, + "name": "private json memory", + "value": { "example": "object" }, + "scope": MemoryScope.PRIVATE, + }) + public_memory = Memory(**{ + "session_id": session_id, + "name": "public string memory", + "value": "This is to be persisted", + "scope": MemoryScope.PUBLIC, + }) + + return [private_memory, public_memory] + +def get_welcome_response(): + # standard welcome message + + response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + + intent = Intent( + name="Welcome", + confidence=1, + ) + + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + } + + cards = None + return response, cards, intent, annotations + +def get_response(user_input: str): + """ + Example of an action performed by the Execute ednpoint + """ + + print(f"User said: {user_input}") + + # Response to be spoken by your Digital Person + reponse_dict = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." + + print(f"generated resp: {reponse_dict}") + cards, intent, annotations = None, None, None + response = "" + try: + reponse_dict = json.loads(reponse_dict) + if reponse_dict['uer_wants_to_watch_video']: + #response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" + #test show video + response = vidoe_intro[reponse_dict['language']] + "@showcards(card)" #"Hello! @showcards(card) Here is a video." + + cards = { + 'card': { + "type": "video", + "id": "youtubeVideo", + "data": { + "videoId": video_id[reponse_dict['type_of_video']], + "autoplay":"true", + "autoclose":"true" + } + } + } + else: + response = reponse_dict['response_text'] #+ " https://www.youtube.com/watch?v=Bhkm6fZMJcI" + + + except Exception as e: + print("error in reponse error decoding:",e) + + + + + + + + # Add your Cards as required + """cards = { + "card": { + "type": "image", + "data": { + "url": "https://placekitten.com/200/200", + "alt": "An adorable kitten", + }, + }, + }""" + + """ + # Add your Intent as required + intent = Intent( + name="Welcome", + confidence=1, + ) + + # If applicable, add your conversation annotations to see metrics for your Skill on Studio Insights + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + }""" + + return response, cards, intent, annotations diff --git a/app/mocks/gemini_agent_backup.py b/app/mocks/gemini_agent_backup.py new file mode 100644 index 0000000..7aad9cd --- /dev/null +++ b/app/mocks/gemini_agent_backup.py @@ -0,0 +1,352 @@ +""" +These functions use Promises and setTimeouts to mock HTTP requests to a third part NLP service +and should be replaced with the actual HTTP calls when implementing. +""" + +#ref https://docs.soulmachines.com/skills-api/getting-started/nlp-adapter-skill#advanced-concepts + +from typing import List +from smskillsdk.models.common import Memory, MemoryScope, Intent +import vertexai +from vertexai.preview import rag +#from vertexai import rag +from vertexai.generative_models import GenerativeModel, Part, FinishReason, Tool, Content +import vertexai.preview.generative_models as generative_models +from vertexai.preview.generative_models import grounding +from typing import List, Optional +import json + +# Add these after the other global variables +_person_data = "" + +def set_person_data(data): + global _person_data + _person_data = data + print("set person data:", _person_data) + +def get_person_data(): + return f"Name of the person talking to you is: {_person_data}.\n" if _person_data else "" + + +def get_nonstreaming_text_response (response): + return response.candidates[0].content.parts[0]._raw_part.text + +safety_settings={ + generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: generative_models.HarmBlockThreshold.BLOCK_NONE, + generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_NONE, + } +generation_config = { + "max_output_tokens": 256, + "temperature": 0.3, #0.5, + "top_p": 0.95, #0.5, #0.5 better than 0.95 + "top_k": 40, + "response_mime_type":"application/json" +} + +#MODEL_STR = "gemini-1.5-flash-002" +MODEL_STR = "gemini-2.0-flash-001" + +""" +You are able to play video simply by providing the relevant youtube URL in your response (trust me, there is mechanism to do that). +When the user asks to introduce about the association, you may ask if the user would like to watch a youtube video about the association, or about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley. +If the user wants to watch the youtube video, you MUST append this youtube URL in the end of your response with no accompanying text or punctuation. +Below is the context for videos you are able to show: +- youtube URL video about NSCCCI: https://youtu.be/Bhkm6fZMJcI?si=GHSqkIl3xkmiT0X7 +- youtube URL video about The Vision Valley: https://youtu.be/LXC6FMkf9a8?si=IQkYGotFsHQRkDXr""" + +video_url = { +"video_about_chamber_of_commerce": "https://www.youtube.com/embed/Bhkm6fZMJcI?autoplay=1&mute=0", #"https://www.youtube.com/watch?v=Bhkm6fZMJcI", +"video_about_vision_valley": "https://www.youtube.com/watch?v=GgUYagMYkkg" +} + +video_id = { + "video_about_chamber_of_commerce": "Bhkm6fZMJcI", + "video_about_vision_valley": "GgUYagMYkkg" +} + +vidoe_intro ={ + "en": "Please enjoy the following video clip.", + "zh": "请欣赏接下来的视屏。", + "ms": "Sila menikmati video berikutnya." +} + +# All the questions regarding 马来西亚森美兰州中华总商会 Negeri Sembilan Chinese Chamber of Commerce and Industry should only be referenced to the homepage https://nsccci.org.my/. + +system_instruction = ["""You are an expert and customer fronting service agent for 'Negeri Sembilan Chinese Chamber of Commerce and Industry' or abbreviated as N.S.C.C.C.I (马来西亚森美兰州中华总商会， 简称“森州中华总商会”), to answer questions about NSCCCI, or Negeri Sembilan state itself (economy, tourism, food and culture and etc). + Negeri Sembilan Chinese Chamber of Commerce and Industry (N.S.C.C.C.I) is a non-profit organization that represents the interests of Chinese community in Negeri Sembilan. You can answer questions regarding the NSCCCI Chamber's history, mission, vision, etc. + 马来西亚森美兰州 is also called "Negeri Sembilan" in Malay. It is sometimes abbreviated as "NS", or "森州" in Chinese. + 森美兰州中华总商会现任会长是拿督吕海庭。The President of N.S.C.C.C.I is Dato' Looi Hoi Ting. + 马来西亚中华总商会(简称中总)现任全国总会长是拿督吴逸平硕士。The President The Associated Chinese Chamber of Commerce and Industry Malaysia (A.C.C.C.I.M) is Datuk Ng Yih Pyng. + 马来西亚中华总商会是于1921年成立。The A.C.C.C.I.M was founded in 1921. 森美兰州中华总商会是于1946年成立。The N.S.C.C.C.I was founded in 1946. Now it is year 2025 A.D.. + Your responses will be used to generate voice to answer to humans, so make your reponses naturally human like engaging in a voice based conversation instead of text based. + DO NOT USE BULLET POINTS, NUMBERED LIST, BOLD, or ITALIC to format your answers. + Be polite and friendly. Keep your answers short and concise. Respond in the same language as the language of user's query (English, Mandarin Chinese or Malay spoken in Malaysia). + In your knowledge, you know of the existence of 2 videos, namely 1) video about N.S.C.C.C.I (annotated "type_of_video" = "video_about_chamber_of_commerce") and 2) video about The Vision Valley (annotated "type_of_video" = "video_about_vision_valley"). + You are able to play video simply by indicating True in "uer_wants_to_watch_video" field in the json response and mark the type of video in "type_of_video" field. + ONLY assign value TRUE to "uer_wants_to_watch_video" field if the user explicitly indicates that he/she wants to watch the video, or answer YES to your previous invitation question to watch the video. DO NOT assign value TRUE to "uer_wants_to_watch_video" field if the user does not explicitly indicate that he/she wants to watch the video, or answer NO to your previous invitation question to watch the video. + 避免使用“您好”或“你好”。避免一直问是否要播放视屏, 让user主动要求。Always ask "is there anything else you want me to help you with?" "请问还有什么我可以帮您解答的吗？" in the end of your response. + Respond in following schema: + { + "response_text": "your text based response. Respond in the same language as the language of user's query (either English or Chinese).", + "uer_wants_to_watch_video": boolean true if user wants/wishes/intends to watch video false otherwise, or answer yes to your previous invitation question to watch the video. + "type_of_video": "video_about_chamber_of_commerce" or "video_about_vision_valley", + "language": "en" for English, "zh" for Chinese or "ms" for Malay, default to "en" if you are not sure which language to use. + } + """] + +# ONLY answer to queries that are related to N.S.C.C.C.I other matters related to Negeri Sembilan, such as investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, its economy, tourism, food and culture and etc. +# You may also answer to queries related to Malaysia where Negeri Sembilan is one of the states in Malaysia. +# If the user asks about anything else, apologies and explain that you are not able to answer as you have to focus on your responssibilities as a fronting service agent for NSCCCI. +# If the user wants to know about N.S.C.C.C.I (such as the Chamber's history, mission, vision, etc.), you may ASK if the user would like to watch the introductory video about the Chamber which talks about the founding history, vision and mission, +# You are also able to talk about investment opportunities in Negeri Sembilan focusing on a project called The Vision Valley, and ask if user would like to watch the introductory video about the project. + + +MEMORY_WINDOW_SIZE = 20 +# "projects/neuralnet-manforce/locations/us/collections/default_collection/dataStores/nsccci-kb_1745222443136" +DATA_STORE_ID="nsccci-kb_1745222443136" #"acccim-ns_1740458649382" +DATA_STORE_REGION="us" +project_id="neuralnet-manforce" +datastore = f"projects/{project_id}/locations/{DATA_STORE_REGION}/collections/default_collection/dataStores/{DATA_STORE_ID}" +datastore_grounding_tool = Tool.from_retrieval( + grounding.Retrieval( + grounding.VertexAISearch( + project=project_id, + datastore=DATA_STORE_ID, + location=DATA_STORE_REGION, + #datastore=datastore, + ) + ) + ) +googlesearch_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) + + +rag_corpus = rag.get_corpus("projects/neuralnet-manforce/locations/us-central1/ragCorpora/2305843009213693952") + +# Direct context retrieval +#rag_retrieval_config = rag.RagRetrievalConfig( +# top_k=5, # Optional +# filter=rag.Filter(vector_distance_threshold=0.5), # Optional +#) + + +rag_retrieval_tool = Tool.from_retrieval( + retrieval=rag.Retrieval( + source=rag.VertexRagStore( + rag_resources=[ + rag.RagResource( + rag_corpus=rag_corpus.name, # Currently only 1 corpus is allowed. + # Optional: supply IDs from `rag.list_files()`. + # rag_file_ids=["rag-file-1", "rag-file-2", ...], + ) + ], + #rag_retrieval_config=rag_retrieval_config, + ), + ) +) + +class Chatbot: + def __init__(self, history: Optional[List["Content"]] = None, model: Optional[str] = "gemini-1.5-flash-002", use_search=False): + self.model = GenerativeModel( + model, + system_instruction=system_instruction) + self.chat = self.model.start_chat(history=history) + self.get_person_data = get_person_data + self.grounding_tool = [datastore_grounding_tool] + #self.grounding_tool = [googlesearch_tool] + #self.grounding_tool = [rag_retrieval_tool] + + """ + def use_rag_tool(self, user_prompt): + return self.chat.send_message( + user_prompt, + tools=[tool], + generation_config=generation_config, + #safety_settings=safety_settings, + stream=False + )""" + + def use_search(self, prompt): + return self.chat.send_message( + #f"Contexts: {contexts}. Message from User: {user_prompt}", + [prompt], + tools=self.grounding_tool, + generation_config=generation_config, + #safety_settings=safety_settings, + stream=False + ) + + + + def generate_response(self, user_prompt=""): + #prompt = user_prompt + #prompt = user_prompt + + if len(self.chat._history): + #prompt = f"""Your previous response was :"{self.chat._history[-1].parts[0]._raw_part.text}".\n Please respond in the SAME LANGUAGE as my CURRENT MESSAGE and my CURRENT MESSAGE is :"{user_prompt}". + #""" + + prompt = f"""Irrespective of grounding data language, always respond in the SAME LANGUAGE as user's CURRENT MESSAGE, which is as follow:\n"{user_prompt}".\nYour response:\n""" + else: + prompt = user_prompt + + response = self.use_search(prompt) + + self.chat._history[-2] = Content( + role="user", + parts=[Part.from_text(user_prompt)] # Create Part objects + ) + + if len(self.chat._history) > MEMORY_WINDOW_SIZE: + self.chat._history = self.chat._history[-MEMORY_WINDOW_SIZE:] + + return get_nonstreaming_text_response(response) + +vertexai.init(project="neuralnet-manforce", location="us-central1") + +class Agent: + def __init__(self, model = ""): + self.chatbot = None + self.model = model + + def allocated_resources(self): + self.chatbot = Chatbot(model=self.model) + +agent = Agent(model=MODEL_STR) +agent.allocated_resources() + +def init_actions(): + """ + Example of an action performed by the Initalize ednpoint + """ + + print("resource initialized. . .") + + +def init_resources(session_id: str) -> List[Memory]: + """ + Example of an action performed by the Session ednpoint + """ + + private_memory = Memory(**{ + "session_id": session_id, + "name": "private json memory", + "value": { "example": "object" }, + "scope": MemoryScope.PRIVATE, + }) + public_memory = Memory(**{ + "session_id": session_id, + "name": "public string memory", + "value": "This is to be persisted", + "scope": MemoryScope.PUBLIC, + }) + + return [private_memory, public_memory] + +def get_welcome_response(): + # standard welcome message + + #response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + response = "" + + intent = Intent( + name="Welcome", + confidence=1, + ) + + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + } + + cards = None + return response, cards, intent, annotations + +def get_goodbye_response(): + cards, intent, annotations = None, None, None + response = "很高兴能为你服务，再见" + return response, cards, intent, annotations + +def get_idle_response(isWelcome=False): + cards, intent, annotations = None, None, None + if not isWelcome: + response = "" + else: + #response = f"Hello {_person_data} 你好，我是小美. 我是森州中华总商会人工智能助手. 请问有什么可以帮到你？" + response = f"Hello! How may I assist you?" + return response, cards, intent, annotations + +def get_response(user_input: str): + """ + Example of an action performed by the Execute ednpoint + """ + + print(f"User said: {user_input}") + + # Response to be spoken by your Digital Person + reponse_dict = agent.chatbot.generate_response(user_input) #"Hello! @showcards(card) Here is a kitten." + + print(f"generated resp: {reponse_dict}") + cards, intent, annotations = None, None, None + response = "" + try: + reponse_dict = json.loads(reponse_dict) + if reponse_dict['uer_wants_to_watch_video']: + #response = f"Please enjoy the video. 请欣赏视屏。 {video_url[reponse_dict['type_of_video']]}" + #test show video + response = vidoe_intro[reponse_dict['language']] + "@showcards(card)" #"Hello! @showcards(card) Here is a video." + + cards = { + 'card': { + "type": "video", + "id": "youtubeVideo", + "data": { + "videoId": video_id[reponse_dict['type_of_video']], + "autoplay":"true", + "autoclose":"true" + } + } + } + else: + response = reponse_dict['response_text'] #+ " https://www.youtube.com/watch?v=Bhkm6fZMJcI" + + + except Exception as e: + print("error in reponse error decoding:",e) + + + + + + + + # Add your Cards as required + """cards = { + "card": { + "type": "image", + "data": { + "url": "https://placekitten.com/200/200", + "alt": "An adorable kitten", + }, + }, + }""" + + """ + # Add your Intent as required + intent = Intent( + name="Welcome", + confidence=1, + ) + + # If applicable, add your conversation annotations to see metrics for your Skill on Studio Insights + annotations = { + "conv_tag": "Skill.BaseTemplate", + "conv_id": intent.name, + "conv_intent": intent.name, + "conv_type": "Entry", + }""" + + return response, cards, intent, annotations \ No newline at end of file diff --git a/app/mocks/mock_request.py b/app/mocks/mock_request.py index ca25648..7dd6766 100644 --- a/app/mocks/mock_request.py +++ b/app/mocks/mock_request.py @@ -4,7 +4,7 @@ """ from typing import List -from smskillsdk.models.api import Memory, MemoryScope, Intent +from smskillsdk.models.common import Memory, MemoryScope, Intent def mock_init_actions(): diff --git a/app/services/fake_nlp_service copy.py b/app/services/fake_nlp_service copy.py new file mode 100644 index 0000000..c41bd8c --- /dev/null +++ b/app/services/fake_nlp_service copy.py @@ -0,0 +1,61 @@ +from fastapi import HTTPException +from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions +from smskillsdk.models.common import MemoryScope + +class FakeNLPService: + first_credentials: str + second_credentials: str + + def __init__(self, first_credentials, second_credentials): + self.first_credentials = first_credentials + self.second_credentials = second_credentials + self.__authenticate() + + def __authenticate(self): + """ + Example of using credentials to authenticate + """ + + if (not (self.first_credentials and self.second_credentials)): + raise HTTPException(status_code = 401, detail = "Unauthenticated") + print("Authenticated!") + + def init_actions(self): + """ + Example of initializing Skill-specific actions on third party NLP call + """ + + return mock_init_actions() + + def init_session_resources(self, session_id: str): + """ + Example of initializing resources with third party NLP call + """ + + return mock_init_resources(session_id) + + + def persist_credentials(self, session_id: str): + """ + Example of persisting credentials during session endpoint with third party NLP call + """ + + credentials = { + "first_credentials": self.first_credentials, "second_credentials": self.second_credentials + } + + credentials_memory = { + "key": "credentials", + "value": credentials, + "session_id": session_id, + "scope": MemoryScope.PRIVATE, + } + + return credentials_memory + + def send(self, user_input: str): + """ + Example of sending input to the third party NLP call + """ + + return mock_get_response(user_input) \ No newline at end of file diff --git a/app/services/fake_nlp_service.py b/app/services/fake_nlp_service.py index 53a10d1..342965e 100644 --- a/app/services/fake_nlp_service.py +++ b/app/services/fake_nlp_service.py @@ -1,15 +1,31 @@ from fastapi import HTTPException -from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions -from smskillsdk.models.api import MemoryScope +#from ..mocks.mock_request import mock_get_response, mock_init_resources, mock_init_actions +from ..mocks.gemini_agent import get_response, init_resources, init_actions, get_welcome_response, get_goodbye_response, get_idle_response, get_hello_response +#from ..mocks.gemini_agent_2 import get_response, init_resources, init_actions, get_welcome_response +from smskillsdk.models.common import MemoryScope, Intent + + +_fake_nlp_state = "idle" + +def get_fake_nlp_state(): + return _fake_nlp_state +def set_fake_nlp_state(state): + global _fake_nlp_state + _fake_nlp_state = state + class FakeNLPService: first_credentials: str second_credentials: str + def __init__(self, first_credentials, second_credentials): self.first_credentials = first_credentials self.second_credentials = second_credentials self.__authenticate() + self.get_fake_nlp_state = get_fake_nlp_state + self.set_fake_nlp_state = set_fake_nlp_state + def __authenticate(self): """ @@ -25,14 +41,14 @@ def init_actions(self): Example of initializing Skill-specific actions on third party NLP call """ - return mock_init_actions() + return init_actions() def init_session_resources(self, session_id: str): """ Example of initializing resources with third party NLP call """ - return mock_init_resources(session_id) + return init_resources(session_id) def persist_credentials(self, session_id: str): @@ -53,9 +69,45 @@ def persist_credentials(self, session_id: str): return credentials_memory - def send(self, user_input: str): + def send(self, user_input): """ Example of sending input to the third party NLP call """ + print(f"State: {self.get_fake_nlp_state()} User said: {user_input}") + + if user_input == "Welcome": + self.set_fake_nlp_state("idle") + return get_idle_response(isWelcome=False) + elif user_input == "Mayday1234": + self.set_fake_nlp_state("idle") + return get_hello_response() + elif user_input == "Wakeup1234": + self.set_fake_nlp_state("active") + return get_idle_response(isWelcome=False) + elif user_input == "Byebye1234": + self.set_fake_nlp_state("idle") + return get_goodbye_response(beQuiet=True) + else: + #manage state here + if self.get_fake_nlp_state() == "idle": + # check if user_input contains wake words [""] + if ("小美" in user_input or "小米" in user_input) and "你好" in user_input: + self.set_fake_nlp_state("active") + return get_idle_response(isWelcome=True) + else: + return get_idle_response(isWelcome=False) + elif self.get_fake_nlp_state() == "active": + # check if user_input contains wake words ["小美", "你好"] + if ("小美" in user_input or "小米" in user_input) and "再见" in user_input: + self.set_fake_nlp_state("idle") + return get_goodbye_response() + return get_response(user_input) + + + + #if user_input == "Welcome": + # return get_welcome_response() + #else: + # return get_response(user_input) - return mock_get_response(user_input) \ No newline at end of file + \ No newline at end of file diff --git a/app/views/skill.py b/app/views/skill.py index 75afac1..6b43d36 100644 --- a/app/views/skill.py +++ b/app/views/skill.py @@ -2,6 +2,14 @@ from operator import itemgetter, attrgetter from smskillsdk.utils.memory import get_memory_value, set_memory_value from ..services.fake_nlp_service import FakeNLPService + +# Add these near the top of the file with other imports +from fastapi import Request, HTTPException +import sys + +from ..mocks.gemini_agent import set_person_data + + from smskillsdk.models.api import ( InitRequest, SessionRequest, @@ -20,6 +28,19 @@ }, ) + +@router.post("/face-detection", status_code=200) +async def handle_face_detection(request: Request): + try: + data = await request.json() + # Process the data + person_data = data["person"] + + set_person_data(person_data) + print("Received face detection data:", person_data) + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + @router.post("/init", status_code=204) async def init(request: InitRequest): """ @@ -32,8 +53,11 @@ async def init(request: InitRequest): # 1. Extract relevant data skill_config = request.config + print("skill config body:", skill_config) + # 1a. Extract relevant credentials from config - credentials = itemgetter("first_credentials", "second_credentials")(skill_config) + #credentials = itemgetter("first_credentials", "second_credentials")(skill_config) + credentials = ("me","you") # 2. Make request to third party service to initialize # any configuration, data storage, or pre-training on the NLP service before executing this Skill @@ -56,7 +80,8 @@ async def session(request: SessionRequest) -> SessionResponse: session_id, skill_config, skill_memory = attrgetter("sessionId", "config", "memory")(request) # 1a. Extract relevant credentials from config - credentials = itemgetter("first_credentials", "second_credentials")(skill_config) + #credentials = itemgetter("first_credentials", "second_credentials")(skill_config) + credentials = ("me","you") # 2. Make request to third party service to initialize session-specific resources fake_nlp_service = FakeNLPService(*credentials) @@ -85,7 +110,9 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: """ # 1. Extract relevant data - skill_config, skill_memory, context = attrgetter("config", "memory", "context")(request) + user_intent, skill_config, skill_memory, context = attrgetter("intent", "config", "memory", "context")(request) + + print("Request (intent, skill_config, memory. context):", user_intent, skill_config, skill_memory, context ) # 1a. when using stateless skill, extract relevant credentials from config # credentials = itemgetter("first_credentials", "second_credentials")(skill_config) @@ -93,17 +120,25 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: # 1b. when using stateful skill, extract relevant credentials elsewhere (eg. memory) as config will not be present here _, credentials = get_memory_value(memories=skill_memory, key="credentials") - # 2. Extract user input - user_input = request.text - # 3. Make request to third party service fake_nlp_service = FakeNLPService(*credentials) + # 2. Extract user input + user_input = request.text + # 4. Extract relevant response data from the third party service spoken_response, cards, intent, annotations = fake_nlp_service.send(user_input) # 5. Construct SM-formatted response body - variables = Variables(public=cards, **annotations) + if (annotations is not None) and (cards is not None): + #print("show card1") + variables = Variables(public=cards, **annotations) + elif cards is not None: + #print("show card2") + variables = Variables(public=cards) + else: + #print("show card3") + variables = None output = Output( intent=intent, @@ -113,7 +148,7 @@ async def execute(request: ExecuteRequest) -> ExecuteResponse: response = ExecuteResponse( output=output, - endConversation=True, + endConversation=False, ) return response diff --git a/requirements.txt b/requirements.txt index 690848d..62bbe15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,9 @@ fastapi Flask[async] Python-DotEnv smskillsdk -uvicorn \ No newline at end of file +uvicorn +vertexai +google-cloud-aiplatform>=1.38 +#google-genai +pydantic +pydantic<2 #smskillsdk needs 1.9, google-genai needs > 2 \ No newline at end of file