{
if (
isCallingApi ||
isUpdatingModel ||
- cookie.token === '' ||
- cookie.token === undefined ||
- cookie.token === 'need_auth'
+ (cookie.token !== 'no_auth' && !sessionStorage.getItem('token'))
)
return
diff --git a/xinference/web/ui/src/scenes/launch_model/modelCard.js b/xinference/web/ui/src/scenes/launch_model/modelCard.js
index f7cb3c465f..2ebfdee701 100644
--- a/xinference/web/ui/src/scenes/launch_model/modelCard.js
+++ b/xinference/web/ui/src/scenes/launch_model/modelCard.js
@@ -231,7 +231,11 @@ const ModelCard = ({
)
})
} else {
- navigate('/running_models')
+ navigate(`/running_models/${modelType}`)
+ sessionStorage.setItem(
+ 'runningModelType',
+ `/running_models/${modelType}`
+ )
}
setIsCallingApi(false)
})
@@ -357,7 +361,9 @@ const ModelCard = ({
>
{(() => {
return modelData.model_lang.map((v) => {
- return
+ return (
+
+ )
})
})()}
{(() => {
diff --git a/xinference/web/ui/src/scenes/login/login.js b/xinference/web/ui/src/scenes/login/login.js
index 05a26b1b93..0dfe8b3966 100644
--- a/xinference/web/ui/src/scenes/login/login.js
+++ b/xinference/web/ui/src/scenes/login/login.js
@@ -43,7 +43,8 @@ function Login() {
} else {
res.json().then((data) => {
setCookie('token', data['access_token'], { path: '/' })
- navigate('/')
+ sessionStorage.setItem('token', data['access_token'])
+ navigate('/launch_model/llm')
})
}
})
diff --git a/xinference/web/ui/src/scenes/register_model/index.js b/xinference/web/ui/src/scenes/register_model/index.js
index de3c04dc43..2b4302aaff 100644
--- a/xinference/web/ui/src/scenes/register_model/index.js
+++ b/xinference/web/ui/src/scenes/register_model/index.js
@@ -90,7 +90,7 @@ const RegisterModel = () => {
if (cookie.token === '' || cookie.token === undefined) {
return
}
- if (cookie.token === 'need_auth') {
+ if (cookie.token !== 'no_auth' && !sessionStorage.getItem('token')) {
navigate('/login', { replace: true })
return
}
@@ -281,6 +281,9 @@ const RegisterModel = () => {
setSuccessMsg(
'Model has been registered successfully! Navigate to launch model page to proceed.'
)
+ navigate('/launch_model/custom/llm')
+ sessionStorage.setItem('modelType', '/launch_model/custom/llm')
+ sessionStorage.setItem('subType', '/launch_model/custom/llm')
}
} catch (error) {
console.error('There was a problem with the fetch operation:', error)
diff --git a/xinference/web/ui/src/scenes/register_model/register_embedding.js b/xinference/web/ui/src/scenes/register_model/register_embedding.js
index ac7ab8d4ae..b8cde4a017 100644
--- a/xinference/web/ui/src/scenes/register_model/register_embedding.js
+++ b/xinference/web/ui/src/scenes/register_model/register_embedding.js
@@ -4,6 +4,7 @@ import AlertTitle from '@mui/material/AlertTitle'
import Button from '@mui/material/Button'
import TextField from '@mui/material/TextField'
import React, { useContext, useState } from 'react'
+import { useNavigate } from 'react-router-dom'
import { ApiContext } from '../../components/apiContext'
import fetcher from '../../components/fetcher'
@@ -18,6 +19,7 @@ const RegisterEmbeddingModel = () => {
const endPoint = useContext(ApiContext).endPoint
const { setErrorMsg } = useContext(ApiContext)
const [successMsg, setSuccessMsg] = useState('')
+ const navigate = useNavigate()
const [formData, setFormData] = useState({
model_name: 'custom-embedding',
dimensions: 768,
@@ -66,6 +68,9 @@ const RegisterEmbeddingModel = () => {
setSuccessMsg(
'Model has been registered successfully! Navigate to launch model page to proceed.'
)
+ navigate('/launch_model/custom/embedding')
+ sessionStorage.setItem('modelType', '/launch_model/custom/llm')
+ sessionStorage.setItem('subType', '/launch_model/custom/embedding')
}
} catch (error) {
console.error('There was a problem with the fetch operation:', error)
diff --git a/xinference/web/ui/src/scenes/register_model/register_rerank.js b/xinference/web/ui/src/scenes/register_model/register_rerank.js
index 075b35ff9d..90f4bff2f7 100644
--- a/xinference/web/ui/src/scenes/register_model/register_rerank.js
+++ b/xinference/web/ui/src/scenes/register_model/register_rerank.js
@@ -4,6 +4,7 @@ import AlertTitle from '@mui/material/AlertTitle'
import Button from '@mui/material/Button'
import TextField from '@mui/material/TextField'
import React, { useContext, useState } from 'react'
+import { useNavigate } from 'react-router-dom'
import { ApiContext } from '../../components/apiContext'
import fetcher from '../../components/fetcher'
@@ -18,6 +19,7 @@ const RegisterRerankModel = () => {
const endPoint = useContext(ApiContext).endPoint
const { setErrorMsg } = useContext(ApiContext)
const [successMsg, setSuccessMsg] = useState('')
+ const navigate = useNavigate()
const [formData, setFormData] = useState({
model_name: 'custom-rerank',
language: ['en'],
@@ -61,6 +63,9 @@ const RegisterRerankModel = () => {
setSuccessMsg(
'Model has been registered successfully! Navigate to launch model page to proceed.'
)
+ navigate('/launch_model/custom/rerank')
+ sessionStorage.setItem('modelType', '/launch_model/custom/llm')
+ sessionStorage.setItem('subType', '/launch_model/custom/rerank')
}
} catch (error) {
console.error('There was a problem with the fetch operation:', error)
diff --git a/xinference/web/ui/src/scenes/running_models/index.js b/xinference/web/ui/src/scenes/running_models/index.js
index ccf3fe312e..1e0da75bb3 100644
--- a/xinference/web/ui/src/scenes/running_models/index.js
+++ b/xinference/web/ui/src/scenes/running_models/index.js
@@ -13,7 +13,9 @@ import fetcher from '../../components/fetcher'
import Title from '../../components/Title'
const RunningModels = () => {
- const [tabValue, setTabValue] = React.useState('1')
+ const [tabValue, setTabValue] = React.useState(
+ sessionStorage.getItem('runningModelType')
+ )
const [llmData, setLlmData] = useState([])
const [embeddingModelData, setEmbeddingModelData] = useState([])
const [imageModelData, setImageModelData] = useState([])
@@ -28,13 +30,15 @@ const RunningModels = () => {
const handleTabChange = (event, newValue) => {
setTabValue(newValue)
+ navigate(newValue)
+ sessionStorage.setItem('runningModelType', newValue)
}
const update = (isCallingApi) => {
if (cookie.token === '' || cookie.token === undefined) {
return
}
- if (cookie.token === 'need_auth') {
+ if (cookie.token !== 'no_auth' && !sessionStorage.getItem('token')) {
navigate('/login', { replace: true })
return
}
@@ -641,14 +645,14 @@ const RunningModels = () => {
onChange={handleTabChange}
aria-label="tabs"
>
-
-
-
-
-
+
+
+
+
+
-
+
{
/>
-
+
{
/>
-
+
{
/>
-
+
{
/>
-
+
Date: Thu, 11 Apr 2024 12:03:15 +0800
Subject: [PATCH 009/298] ENH: UI: paper material for cards (#1261)
---
xinference/web/ui/src/scenes/launch_model/modelCard.js | 6 ++++--
.../web/ui/src/scenes/launch_model/styles/modelCardStyle.js | 4 ----
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/xinference/web/ui/src/scenes/launch_model/modelCard.js b/xinference/web/ui/src/scenes/launch_model/modelCard.js
index 2ebfdee701..3998a1c665 100644
--- a/xinference/web/ui/src/scenes/launch_model/modelCard.js
+++ b/xinference/web/ui/src/scenes/launch_model/modelCard.js
@@ -28,6 +28,7 @@ import {
Stack,
TextField,
} from '@mui/material'
+import Paper from '@mui/material/Paper'
import React, { useContext, useEffect, useRef, useState } from 'react'
import { useNavigate } from 'react-router-dom'
@@ -327,7 +328,7 @@ const ModelCard = ({
// Set two different states based on mouse hover
return (
- setHover(true)}
onMouseLeave={() => setHover(false)}
@@ -336,6 +337,7 @@ const ModelCard = ({
setSelected(true)
}
}}
+ elevation={hover ? 24 : 4}
>
{modelType === 'LLM' ? (
@@ -951,7 +953,7 @@ const ModelCard = ({
message="Please fill in the added custom parameters completely!"
key={'top' + 'center'}
/>
-
+
)
}
diff --git a/xinference/web/ui/src/scenes/launch_model/styles/modelCardStyle.js b/xinference/web/ui/src/scenes/launch_model/styles/modelCardStyle.js
index c5c8195a08..d5f86867a1 100644
--- a/xinference/web/ui/src/scenes/launch_model/styles/modelCardStyle.js
+++ b/xinference/web/ui/src/scenes/launch_model/styles/modelCardStyle.js
@@ -4,7 +4,6 @@ const styles = {
position: 'relative',
width: '300px',
height: '300px',
- border: '1px solid #ddd',
borderRadius: '20px',
},
containerSelected: {
@@ -13,9 +12,7 @@ const styles = {
position: 'relative',
width: '300px',
height: '300px',
- border: '1px solid #ddd',
borderRadius: '20px',
- boxShadow: '0 0 10px #ccc',
},
descriptionCard: {
position: 'relative',
@@ -23,7 +20,6 @@ const styles = {
left: '-1px',
width: '300px',
height: '300px',
- border: '1px solid #ddd',
padding: '20px',
borderRadius: '20px',
},
From e3a947ebddfc53b5e8ec723c1f632c2b895edef1 Mon Sep 17 00:00:00 2001
From: "Xuye (Chris) Qin"
Date: Thu, 11 Apr 2024 15:35:46 +0800
Subject: [PATCH 010/298] BUG: fix custom vision model (#1280)
---
xinference/model/llm/llm_family.py | 15 ++++++
xinference/model/llm/pytorch/core.py | 50 ++++++++-----------
xinference/model/llm/tests/test_llm_family.py | 14 ++++++
3 files changed, 51 insertions(+), 28 deletions(-)
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
index 15ff0db84c..a131a25169 100644
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -199,6 +199,21 @@ def parse_raw(
)
llm_spec.prompt_style = BUILTIN_LLM_PROMPT_STYLE[prompt_style_name]
+ # check model ability, registering LLM only provides generate and chat
+ # but for vision models, we add back the abilities so that
+ # gradio chat interface can be generated properly
+ if (
+ llm_spec.model_family != "other"
+ and llm_spec.model_family
+ in {
+ family.model_name
+ for family in BUILTIN_LLM_FAMILIES
+ if "vision" in family.model_ability
+ }
+ and "vision" not in llm_spec.model_ability
+ ):
+ llm_spec.model_ability.append("vision")
+
return llm_spec
diff --git a/xinference/model/llm/pytorch/core.py b/xinference/model/llm/pytorch/core.py
index 88ca44ee7b..8cac269289 100644
--- a/xinference/model/llm/pytorch/core.py
+++ b/xinference/model/llm/pytorch/core.py
@@ -42,6 +42,25 @@
logger = logging.getLogger(__name__)
+NON_DEFAULT_MODEL_LIST: List[str] = [
+ "baichuan-chat",
+ "baichuan-2-chat",
+ "vicuna-v1.3",
+ "falcon",
+ "falcon-instruct",
+ "chatglm",
+ "chatglm2",
+ "chatglm2-32k",
+ "chatglm2-128k",
+ "llama-2",
+ "llama-2-chat",
+ "internlm2-chat",
+ "qwen-vl-chat",
+ "OmniLMM",
+ "yi-vl-chat",
+ "deepseek-vl-chat",
+]
+
class PytorchModel(LLM):
def __init__(
@@ -233,17 +252,7 @@ def match(
if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
return False
model_family = llm_family.model_family or llm_family.model_name
- if model_family in [
- "baichuan-chat",
- "vicuna-v1.3",
- "falcon",
- "falcon-instruct",
- "chatglm",
- "chatglm2",
- "chatglm2-32k",
- "llama-2",
- "llama-2-chat",
- ]:
+ if model_family in NON_DEFAULT_MODEL_LIST:
return False
if "generate" not in llm_family.model_ability:
return False
@@ -452,23 +461,8 @@ def match(
) -> bool:
if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
return False
- if llm_family.model_name in [
- "baichuan-chat",
- "baichuan-2-chat",
- "vicuna-v1.3",
- "falcon",
- "falcon-instruct",
- "chatglm",
- "chatglm2",
- "chatglm2-32k",
- "llama-2",
- "llama-2-chat",
- "internlm2-chat",
- "qwen-vl-chat",
- "OmniLMM",
- "yi-vl-chat",
- "deepseek-vl-chat",
- ]:
+ model_family = llm_family.model_family or llm_family.model_name
+ if model_family in NON_DEFAULT_MODEL_LIST:
return False
if "chat" not in llm_family.model_ability:
return False
diff --git a/xinference/model/llm/tests/test_llm_family.py b/xinference/model/llm/tests/test_llm_family.py
index fe093822a2..b1b6662dc1 100644
--- a/xinference/model/llm/tests/test_llm_family.py
+++ b/xinference/model/llm/tests/test_llm_family.py
@@ -992,6 +992,20 @@ def test_parse_prompt_style():
model_spec = CustomLLMFamilyV1.parse_raw(bytes(llm_family.json(), "utf8"))
assert model_spec.model_name == llm_family.model_name
+ # test vision
+ llm_family = CustomLLMFamilyV1(
+ version=1,
+ model_type="LLM",
+ model_name="test_LLM",
+ model_lang=["en"],
+ model_ability=["chat", "generate"],
+ model_specs=[hf_spec, ms_spec],
+ model_family="qwen-vl-chat",
+ prompt_style="qwen-vl-chat",
+ )
+ model_spec = CustomLLMFamilyV1.parse_raw(bytes(llm_family.json(), "utf-8"))
+ assert "vision" in model_spec.model_ability
+
# error: missing model_family
llm_family = CustomLLMFamilyV1(
version=1,
From 6d6f4011b8e68c91b6be9b14f40fd6d0b64613ec Mon Sep 17 00:00:00 2001
From: amumu96 <128140880+amumu96@users.noreply.github.com>
Date: Mon, 15 Apr 2024 11:49:19 +0800
Subject: [PATCH 011/298] ENH: Support custom audio model (#1279)
---
xinference/core/supervisor.py | 28 +++-
xinference/core/worker.py | 8 +
xinference/model/audio/__init__.py | 41 ++++-
xinference/model/audio/core.py | 70 ++++-----
xinference/model/audio/custom.py | 148 +++++++++++++++++++
xinference/model/audio/tests/test_whisper.py | 71 +++++++++
6 files changed, 319 insertions(+), 47 deletions(-)
create mode 100644 xinference/model/audio/custom.py
diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py
index c8837f7a84..a18add947c 100644
--- a/xinference/core/supervisor.py
+++ b/xinference/core/supervisor.py
@@ -135,6 +135,13 @@ async def __post_create__(self):
EventCollectorActor, address=self.address, uid=EventCollectorActor.uid()
)
+ from ..model.audio import (
+ CustomAudioModelFamilyV1,
+ generate_audio_description,
+ get_audio_model_descriptions,
+ register_audio,
+ unregister_audio,
+ )
from ..model.embedding import (
CustomEmbeddingModelSpec,
generate_embedding_description,
@@ -177,6 +184,12 @@ async def __post_create__(self):
unregister_rerank,
generate_rerank_description,
),
+ "audio": (
+ CustomAudioModelFamilyV1,
+ register_audio,
+ unregister_audio,
+ generate_audio_description,
+ ),
}
# record model version
@@ -185,6 +198,7 @@ async def __post_create__(self):
model_version_infos.update(get_embedding_model_descriptions())
model_version_infos.update(get_rerank_model_descriptions())
model_version_infos.update(get_image_model_descriptions())
+ model_version_infos.update(get_audio_model_descriptions())
await self._cache_tracker_ref.record_model_version(
model_version_infos, self.address
)
@@ -483,6 +497,7 @@ def sort_helper(item):
return ret
elif model_type == "audio":
from ..model.audio import BUILTIN_AUDIO_MODELS
+ from ..model.audio.custom import get_user_defined_audios
ret = []
for model_name, family in BUILTIN_AUDIO_MODELS.items():
@@ -491,6 +506,16 @@ def sort_helper(item):
else:
ret.append({"model_name": model_name, "is_builtin": True})
+ for model_spec in get_user_defined_audios():
+ if detailed:
+ ret.append(
+ await self._to_audio_model_reg(model_spec, is_builtin=False)
+ )
+ else:
+ ret.append(
+ {"model_name": model_spec.model_name, "is_builtin": False}
+ )
+
ret.sort(key=sort_helper)
return ret
elif model_type == "rerank":
@@ -548,8 +573,9 @@ def get_model_registration(self, model_type: str, model_name: str) -> Any:
raise ValueError(f"Model {model_name} not found")
elif model_type == "audio":
from ..model.audio import BUILTIN_AUDIO_MODELS
+ from ..model.audio.custom import get_user_defined_audios
- for f in BUILTIN_AUDIO_MODELS.values():
+ for f in list(BUILTIN_AUDIO_MODELS.values()) + get_user_defined_audios():
if f.model_name == model_name:
return f
raise ValueError(f"Model {model_name} not found")
diff --git a/xinference/core/worker.py b/xinference/core/worker.py
index fde991614d..5c8f3eef10 100644
--- a/xinference/core/worker.py
+++ b/xinference/core/worker.py
@@ -195,6 +195,12 @@ async def __post_create__(self):
logger.info("Purge cache directory: %s", XINFERENCE_CACHE_DIR)
purge_dir(XINFERENCE_CACHE_DIR)
+ from ..model.audio import (
+ CustomAudioModelFamilyV1,
+ get_audio_model_descriptions,
+ register_audio,
+ unregister_audio,
+ )
from ..model.embedding import (
CustomEmbeddingModelSpec,
get_embedding_model_descriptions,
@@ -223,6 +229,7 @@ async def __post_create__(self):
unregister_embedding,
),
"rerank": (CustomRerankModelSpec, register_rerank, unregister_rerank),
+ "audio": (CustomAudioModelFamilyV1, register_audio, unregister_audio),
}
# record model version
@@ -231,6 +238,7 @@ async def __post_create__(self):
model_version_infos.update(get_embedding_model_descriptions())
model_version_infos.update(get_rerank_model_descriptions())
model_version_infos.update(get_image_model_descriptions())
+ model_version_infos.update(get_audio_model_descriptions())
await self._cache_tracker_ref.record_model_version(
model_version_infos, self.address
)
diff --git a/xinference/model/audio/__init__.py b/xinference/model/audio/__init__.py
index 8bc2e03ade..de6367f9df 100644
--- a/xinference/model/audio/__init__.py
+++ b/xinference/model/audio/__init__.py
@@ -16,12 +16,51 @@
import json
import os
-from .core import AudioModelFamilyV1, generate_audio_description, get_cache_status
+from .core import (
+ AUDIO_MODEL_DESCRIPTIONS,
+ MODEL_NAME_TO_REVISION,
+ AudioModelFamilyV1,
+ generate_audio_description,
+ get_audio_model_descriptions,
+ get_cache_status,
+)
+from .custom import (
+ CustomAudioModelFamilyV1,
+ get_user_defined_audios,
+ register_audio,
+ unregister_audio,
+)
_model_spec_json = os.path.join(os.path.dirname(__file__), "model_spec.json")
BUILTIN_AUDIO_MODELS = dict(
(spec["model_name"], AudioModelFamilyV1(**spec))
for spec in json.load(codecs.open(_model_spec_json, "r", encoding="utf-8"))
)
+for model_name, model_spec in BUILTIN_AUDIO_MODELS.items():
+ MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
+
+# register model description after recording model revision
+for model_spec_info in [BUILTIN_AUDIO_MODELS]:
+ for model_name, model_spec in model_spec_info.items():
+ if model_spec.model_name not in AUDIO_MODEL_DESCRIPTIONS:
+ AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(model_spec))
+
+from ...constants import XINFERENCE_MODEL_DIR
+
+# if persist=True, load them when init
+user_defined_audio_dir = os.path.join(XINFERENCE_MODEL_DIR, "audio")
+if os.path.isdir(user_defined_audio_dir):
+ for f in os.listdir(user_defined_audio_dir):
+ with codecs.open(
+ os.path.join(user_defined_audio_dir, f), encoding="utf-8"
+ ) as fd:
+ user_defined_audio_family = CustomAudioModelFamilyV1.parse_obj(
+ json.load(fd)
+ )
+ register_audio(user_defined_audio_family, persist=False)
+
+# register model description
+for ud_audio in get_user_defined_audios():
+ AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(ud_audio))
del _model_spec_json
diff --git a/xinference/model/audio/core.py b/xinference/model/audio/core.py
index c73004e4ea..20b3ff7d36 100644
--- a/xinference/model/audio/core.py
+++ b/xinference/model/audio/core.py
@@ -16,9 +16,8 @@
from collections import defaultdict
from typing import Dict, List, Optional, Tuple
-from ..._compat import BaseModel
from ...constants import XINFERENCE_CACHE_DIR
-from ..core import ModelDescription
+from ..core import CacheableModelSpec, ModelDescription
from ..utils import valid_model_revision
from .whisper import WhisperModel
@@ -26,8 +25,19 @@
logger = logging.getLogger(__name__)
+# Used for check whether the model is cached.
+# Init when registering all the builtin models.
+MODEL_NAME_TO_REVISION: Dict[str, List[str]] = defaultdict(list)
+AUDIO_MODEL_DESCRIPTIONS: Dict[str, List[Dict]] = defaultdict(list)
-class AudioModelFamilyV1(BaseModel):
+
+def get_audio_model_descriptions():
+ import copy
+
+ return copy.deepcopy(AUDIO_MODEL_DESCRIPTIONS)
+
+
+class AudioModelFamilyV1(CacheableModelSpec):
model_family: str
model_name: str
model_id: str
@@ -77,63 +87,33 @@ def generate_audio_description(
image_model: AudioModelFamilyV1,
) -> Dict[str, List[Dict]]:
res = defaultdict(list)
- res[image_model.model_name].extend(
- AudioModelDescription(None, None, image_model).to_dict()
+ res[image_model.model_name].append(
+ AudioModelDescription(None, None, image_model).to_version_info()
)
return res
-def match_model(model_name: str) -> AudioModelFamilyV1:
+def match_audio(model_name: str) -> AudioModelFamilyV1:
from . import BUILTIN_AUDIO_MODELS
+ from .custom import get_user_defined_audios
+
+ for model_spec in get_user_defined_audios():
+ if model_spec.model_name == model_name:
+ return model_spec
if model_name in BUILTIN_AUDIO_MODELS:
return BUILTIN_AUDIO_MODELS[model_name]
else:
raise ValueError(
- f"Image model {model_name} not found, available"
+ f"Audio model {model_name} not found, available"
f"model list: {BUILTIN_AUDIO_MODELS.keys()}"
)
def cache(model_spec: AudioModelFamilyV1):
- # TODO: cache from uri
- import huggingface_hub
-
- cache_dir = get_cache_dir(model_spec)
- if not os.path.exists(cache_dir):
- os.makedirs(cache_dir, exist_ok=True)
-
- meta_path = os.path.join(cache_dir, "__valid_download")
- if valid_model_revision(meta_path, model_spec.model_revision):
- return cache_dir
-
- for current_attempt in range(1, MAX_ATTEMPTS + 1):
- try:
- huggingface_hub.snapshot_download(
- model_spec.model_id,
- revision=model_spec.model_revision,
- local_dir=cache_dir,
- local_dir_use_symlinks=True,
- resume_download=True,
- )
- break
- except huggingface_hub.utils.LocalEntryNotFoundError:
- remaining_attempts = MAX_ATTEMPTS - current_attempt
- logger.warning(
- f"Attempt {current_attempt} failed. Remaining attempts: {remaining_attempts}"
- )
- else:
- raise RuntimeError(
- f"Failed to download model '{model_spec.model_name}' after {MAX_ATTEMPTS} attempts"
- )
-
- with open(meta_path, "w") as f:
- import json
-
- desc = AudioModelDescription(None, None, model_spec)
- json.dump(desc.to_dict(), f)
+ from ..utils import cache
- return cache_dir
+ return cache(model_spec, AudioModelDescription)
def get_cache_dir(model_spec: AudioModelFamilyV1):
@@ -151,7 +131,7 @@ def get_cache_status(
def create_audio_model_instance(
subpool_addr: str, devices: List[str], model_uid: str, model_name: str, **kwargs
) -> Tuple[WhisperModel, AudioModelDescription]:
- model_spec = match_model(model_name)
+ model_spec = match_audio(model_name)
model_path = cache(model_spec)
model = WhisperModel(model_uid, model_path, model_spec, **kwargs)
model_description = AudioModelDescription(
diff --git a/xinference/model/audio/custom.py b/xinference/model/audio/custom.py
new file mode 100644
index 0000000000..12599f89e6
--- /dev/null
+++ b/xinference/model/audio/custom.py
@@ -0,0 +1,148 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from threading import Lock
+from typing import Any, List, Optional
+
+from ..._compat import (
+ ROOT_KEY,
+ ErrorWrapper,
+ Protocol,
+ StrBytes,
+ ValidationError,
+ load_str_bytes,
+)
+from ...constants import XINFERENCE_CACHE_DIR, XINFERENCE_MODEL_DIR
+from .core import AudioModelFamilyV1
+
+logger = logging.getLogger(__name__)
+
+UD_AUDIO_LOCK = Lock()
+
+
+class CustomAudioModelFamilyV1(AudioModelFamilyV1):
+ model_id: Optional[str] # type: ignore
+ model_revision: Optional[str] # type: ignore
+ model_uri: Optional[str]
+
+ @classmethod
+ def parse_raw(
+ cls: Any,
+ b: StrBytes,
+ *,
+ content_type: Optional[str] = None,
+ encoding: str = "utf8",
+ proto: Protocol = None,
+ allow_pickle: bool = False,
+ ) -> AudioModelFamilyV1:
+ # See source code of BaseModel.parse_raw
+ try:
+ obj = load_str_bytes(
+ b,
+ proto=proto,
+ content_type=content_type,
+ encoding=encoding,
+ allow_pickle=allow_pickle,
+ json_loads=cls.__config__.json_loads,
+ )
+ except (ValueError, TypeError, UnicodeDecodeError) as e:
+ raise ValidationError([ErrorWrapper(e, loc=ROOT_KEY)], cls)
+
+ audio_spec: AudioModelFamilyV1 = cls.parse_obj(obj)
+
+ # check model_family
+ if audio_spec.model_family is None:
+ raise ValueError(
+ f"You must specify `model_family` when registering custom Audio models."
+ )
+ assert isinstance(audio_spec.model_family, str)
+ return audio_spec
+
+
+UD_AUDIOS: List[CustomAudioModelFamilyV1] = []
+
+
+def get_user_defined_audios() -> List[CustomAudioModelFamilyV1]:
+ with UD_AUDIO_LOCK:
+ return UD_AUDIOS.copy()
+
+
+def register_audio(model_spec: CustomAudioModelFamilyV1, persist: bool):
+ from ...constants import XINFERENCE_MODEL_DIR
+ from ..utils import is_valid_model_name, is_valid_model_uri
+ from . import BUILTIN_AUDIO_MODELS
+
+ if not is_valid_model_name(model_spec.model_name):
+ raise ValueError(f"Invalid model name {model_spec.model_name}.")
+
+ with UD_AUDIO_LOCK:
+ for model_name in list(BUILTIN_AUDIO_MODELS.keys()) + [
+ spec.model_name for spec in UD_AUDIOS
+ ]:
+ if model_spec.model_name == model_name:
+ raise ValueError(
+ f"Model name conflicts with existing model {model_spec.model_name}"
+ )
+
+ UD_AUDIOS.append(model_spec)
+
+ if persist:
+ # We only validate model URL when persist is True.
+ model_uri = model_spec.model_uri
+ if model_uri and not is_valid_model_uri(model_uri):
+ raise ValueError(f"Invalid model URI {model_uri}.")
+
+ persist_path = os.path.join(
+ XINFERENCE_MODEL_DIR, "audio", f"{model_spec.model_name}.json"
+ )
+ os.makedirs(os.path.dirname(persist_path), exist_ok=True)
+ with open(persist_path, mode="w") as fd:
+ fd.write(model_spec.json())
+
+
+def unregister_audio(model_name: str, raise_error: bool = True):
+ with UD_AUDIO_LOCK:
+ model_spec = None
+ for i, f in enumerate(UD_AUDIOS):
+ if f.model_name == model_name:
+ model_spec = f
+ break
+ if model_spec:
+ UD_AUDIOS.remove(model_spec)
+
+ persist_path = os.path.join(
+ XINFERENCE_MODEL_DIR, "audio", f"{model_spec.model_name}.json"
+ )
+ if os.path.exists(persist_path):
+ os.remove(persist_path)
+
+ cache_dir = os.path.join(XINFERENCE_CACHE_DIR, model_spec.model_name)
+ if os.path.exists(cache_dir):
+ logger.warning(
+ f"Remove the cache of user-defined model {model_spec.model_name}. "
+ f"Cache directory: {cache_dir}"
+ )
+ if os.path.isdir(cache_dir):
+ os.rmdir(cache_dir)
+ else:
+ logger.warning(
+ f"Cache directory is not a soft link, please remove it manually."
+ )
+ else:
+ if raise_error:
+ raise ValueError(f"Model {model_name} not found")
+ else:
+ logger.warning(f"Custom audio model {model_name} not found")
diff --git a/xinference/model/audio/tests/test_whisper.py b/xinference/model/audio/tests/test_whisper.py
index a200f63f7e..c39b3c15ba 100644
--- a/xinference/model/audio/tests/test_whisper.py
+++ b/xinference/model/audio/tests/test_whisper.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os.path
+import tempfile
import pytest
import requests
@@ -76,3 +77,73 @@ def test_restful_api_for_whisper(setup):
assert "list" in translation
assert "airlines" in translation
assert "hong kong" in translation
+
+
+def test_register_custom_audio():
+ from ..custom import (
+ CustomAudioModelFamilyV1,
+ get_user_defined_audios,
+ register_audio,
+ unregister_audio,
+ )
+
+ # correct
+ family_a = CustomAudioModelFamilyV1(
+ model_family="my-whisper",
+ model_name="custom_test_a",
+ model_id="test/custom_test_a",
+ multilingual=True,
+ )
+
+ register_audio(family_a, False)
+ assert family_a in get_user_defined_audios()
+
+ # name conflict
+ family_b = CustomAudioModelFamilyV1(
+ model_family="my-whisper",
+ model_name="custom_test_b",
+ model_id="test/custom_test_b",
+ multilingual=True,
+ )
+ register_audio(family_b, False)
+ assert family_b in get_user_defined_audios()
+ with pytest.raises(ValueError):
+ register_audio(family_b, False)
+
+ # unregister
+ unregister_audio(family_a.model_name)
+ assert family_a not in get_user_defined_audios()
+ unregister_audio(family_b.model_name)
+ assert family_b not in get_user_defined_audios()
+
+
+def test_persistent_custom_audio():
+ from ....constants import XINFERENCE_MODEL_DIR
+ from ..custom import (
+ CustomAudioModelFamilyV1,
+ get_user_defined_audios,
+ register_audio,
+ unregister_audio,
+ )
+
+ temp_dir = tempfile.mkdtemp()
+
+ # correct
+ family = CustomAudioModelFamilyV1(
+ model_family="my-whisper",
+ model_name="custom_test_a",
+ model_id="test/custom_test_a",
+ multilingual=True,
+ model_uri=os.path.abspath(temp_dir),
+ )
+
+ register_audio(family, True)
+ assert family in get_user_defined_audios()
+ assert f"{family.model_name}.json" in os.listdir(
+ os.path.join(XINFERENCE_MODEL_DIR, "audio")
+ )
+
+ unregister_audio(family.model_name)
+ assert f"{family.model_name}.json" not in os.listdir(
+ os.path.join(XINFERENCE_MODEL_DIR, "audio")
+ )
From d93353ef3864a9e10d42165bc77df3b2a2ddeea3 Mon Sep 17 00:00:00 2001
From: yiboyasss <143868051+yiboyasss@users.noreply.github.com>
Date: Tue, 16 Apr 2024 13:27:15 +0800
Subject: [PATCH 012/298] FEAT: [UI] Add replica configuration when launching
`embedding` and `rerank` models (#1306)
---
.../ui/src/scenes/launch_model/modelCard.js | 28 +++++++++++++++----
1 file changed, 23 insertions(+), 5 deletions(-)
diff --git a/xinference/web/ui/src/scenes/launch_model/modelCard.js b/xinference/web/ui/src/scenes/launch_model/modelCard.js
index 3998a1c665..08cd2061a9 100644
--- a/xinference/web/ui/src/scenes/launch_model/modelCard.js
+++ b/xinference/web/ui/src/scenes/launch_model/modelCard.js
@@ -167,7 +167,7 @@ const ModelCard = ({
setIsCallingApi(true)
- const modelDataWithID1 = {
+ const modelDataWithID_LLM = {
// If user does not fill model_uid, pass null (None) to server and server generates it.
model_uid: modelUID.trim() === '' ? null : modelUID.trim(),
model_name: modelData.model_name,
@@ -194,24 +194,28 @@ const ModelCard = ({
gpu_idx: GPUIdx.trim() === '' ? null : handleGPUIdx(GPUIdx.trim()),
}
- const modelDataWithID2 = {
+ const modelDataWithID_other = {
model_uid: modelUID.trim() === '' ? null : modelUID.trim(),
model_name: modelData.model_name,
model_type: modelType,
}
if (nGPULayers >= 0) {
- modelDataWithID1.n_gpu_layers = nGPULayers
+ modelDataWithID_LLM.n_gpu_layers = nGPULayers
}
if (customParametersArr.length) {
customParametersArr.forEach((item) => {
- modelDataWithID1[item.key] = handleValueType(item.value)
+ modelDataWithID_LLM[item.key] = handleValueType(item.value)
})
}
const modelDataWithID =
- modelType === 'LLM' ? modelDataWithID1 : modelDataWithID2
+ modelType === 'LLM'
+ ? modelDataWithID_LLM
+ : modelType === 'embedding' || modelType === 'rerank'
+ ? { ...modelDataWithID_other, replica }
+ : modelDataWithID_other
// First fetcher request to initiate the model
fetcher(url + '/v1/models', {
@@ -865,6 +869,20 @@ const ModelCard = ({
label="(Optional) Model UID, model name by default"
onChange={(e) => setModelUID(e.target.value)}
/>
+ {(modelType === 'embedding' || modelType === 'rerank') && (
+ setReplica(parseInt(e.target.value, 10))}
+ />
+ )}
)}
From fdef389cf6056077f95ed10f0927dd7920368215 Mon Sep 17 00:00:00 2001
From: Mike Shi
Date: Tue, 16 Apr 2024 14:28:53 +0800
Subject: [PATCH 013/298] ENH: support int and str compare for model size
(#1277)
---
xinference/client/restful/restful_client.py | 15 ++++++++---
xinference/core/supervisor.py | 4 +--
xinference/core/worker.py | 2 +-
xinference/deploy/cmdline.py | 4 ++-
xinference/model/core.py | 4 +--
xinference/model/llm/core.py | 4 +--
xinference/model/llm/llm_family.py | 25 +++++++++++++++++--
xinference/model/llm/tests/test_llm_family.py | 18 +++++++++++++
xinference/model/utils.py | 13 +++++++++-
9 files changed, 75 insertions(+), 14 deletions(-)
diff --git a/xinference/client/restful/restful_client.py b/xinference/client/restful/restful_client.py
index 29fb00c68e..50dcb2114e 100644
--- a/xinference/client/restful/restful_client.py
+++ b/xinference/client/restful/restful_client.py
@@ -18,6 +18,7 @@
import requests
+from ...model.utils import convert_float_to_int_or_str
from ..common import streaming_response_iterator
if TYPE_CHECKING:
@@ -746,7 +747,7 @@ def list_models(self) -> Dict[str, Dict[str, Any]]:
def launch_speculative_llm(
self,
model_name: str,
- model_size_in_billions: Optional[int],
+ model_size_in_billions: Optional[Union[int, str, float]],
quantization: Optional[str],
draft_model_name: str,
draft_model_size_in_billions: Optional[int],
@@ -767,6 +768,10 @@ def launch_speculative_llm(
"`launch_speculative_llm` is an experimental feature and the API may change in the future."
)
+ # convert float to int or string since the RESTful API does not accept float.
+ if isinstance(model_size_in_billions, float):
+ model_size_in_billions = convert_float_to_int_or_str(model_size_in_billions)
+
payload = {
"model_uid": None,
"model_name": model_name,
@@ -794,7 +799,7 @@ def launch_model(
model_name: str,
model_type: str = "LLM",
model_uid: Optional[str] = None,
- model_size_in_billions: Optional[Union[int, str]] = None,
+ model_size_in_billions: Optional[Union[int, str, float]] = None,
model_format: Optional[str] = None,
quantization: Optional[str] = None,
replica: int = 1,
@@ -818,7 +823,7 @@ def launch_model(
type of model.
model_uid: str
UID of model, auto generate a UUID if is None.
- model_size_in_billions: Optional[int]
+ model_size_in_billions: Optional[Union[int, str, float]]
The size (in billions) of the model.
model_format: Optional[str]
The format of the model.
@@ -854,6 +859,10 @@ def launch_model(
url = f"{self.base_url}/v1/models"
+ # convert float to int or string since the RESTful API does not accept float.
+ if isinstance(model_size_in_billions, float):
+ model_size_in_billions = convert_float_to_int_or_str(model_size_in_billions)
+
payload = {
"model_uid": model_uid,
"model_name": model_name,
diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py
index a18add947c..dae8c3c6c8 100644
--- a/xinference/core/supervisor.py
+++ b/xinference/core/supervisor.py
@@ -680,7 +680,7 @@ async def launch_speculative_llm(
self,
model_uid: Optional[str],
model_name: str,
- model_size_in_billions: Optional[int],
+ model_size_in_billions: Optional[Union[int, str]],
quantization: Optional[str],
draft_model_name: str,
draft_model_size_in_billions: Optional[int],
@@ -740,7 +740,7 @@ async def launch_builtin_model(
self,
model_uid: Optional[str],
model_name: str,
- model_size_in_billions: Optional[int],
+ model_size_in_billions: Optional[Union[int, str]],
model_format: Optional[str],
quantization: Optional[str],
model_type: Optional[str],
diff --git a/xinference/core/worker.py b/xinference/core/worker.py
index 5c8f3eef10..465fae7b84 100644
--- a/xinference/core/worker.py
+++ b/xinference/core/worker.py
@@ -601,7 +601,7 @@ async def launch_builtin_model(
self,
model_uid: str,
model_name: str,
- model_size_in_billions: Optional[int],
+ model_size_in_billions: Optional[Union[int, str]],
model_format: Optional[str],
quantization: Optional[str],
model_type: str = "LLM",
diff --git a/xinference/deploy/cmdline.py b/xinference/deploy/cmdline.py
index 4873838ea2..baae417c6e 100644
--- a/xinference/deploy/cmdline.py
+++ b/xinference/deploy/cmdline.py
@@ -736,7 +736,9 @@ def model_launch(
endpoint = get_endpoint(endpoint)
model_size: Optional[Union[str, int]] = (
size_in_billions
- if size_in_billions is None or "_" in size_in_billions
+ if size_in_billions is None
+ or "_" in size_in_billions
+ or "." in size_in_billions
else int(size_in_billions)
)
client = RESTfulClient(base_url=endpoint, api_key=api_key)
diff --git a/xinference/model/core.py b/xinference/model/core.py
index 717124271f..842eca6387 100644
--- a/xinference/model/core.py
+++ b/xinference/model/core.py
@@ -13,7 +13,7 @@
# limitations under the License.
from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
from .._compat import BaseModel
@@ -50,7 +50,7 @@ def create_model_instance(
model_type: str,
model_name: str,
model_format: Optional[str] = None,
- model_size_in_billions: Optional[int] = None,
+ model_size_in_billions: Optional[Union[int, str]] = None,
quantization: Optional[str] = None,
peft_model_path: Optional[str] = None,
image_lora_load_kwargs: Optional[Dict] = None,
diff --git a/xinference/model/llm/core.py b/xinference/model/llm/core.py
index ec31d9144b..77ba6b2d61 100644
--- a/xinference/model/llm/core.py
+++ b/xinference/model/llm/core.py
@@ -178,7 +178,7 @@ def create_llm_model_instance(
model_uid: str,
model_name: str,
model_format: Optional[str] = None,
- model_size_in_billions: Optional[int] = None,
+ model_size_in_billions: Optional[Union[int, str]] = None,
quantization: Optional[str] = None,
peft_model_path: Optional[str] = None,
is_local_deployment: bool = False,
@@ -238,7 +238,7 @@ def create_speculative_llm_model_instance(
devices: List[str],
model_uid: str,
model_name: str,
- model_size_in_billions: Optional[int],
+ model_size_in_billions: Optional[Union[int, str]],
quantization: Optional[str],
draft_model_name: str,
draft_model_size_in_billions: Optional[int],
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
index a131a25169..4252c3a16f 100644
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -797,10 +797,29 @@ def get_user_defined_llm_families():
return UD_LLM_FAMILIES.copy()
+def match_model_size(
+ model_size: Union[int, str], spec_model_size: Union[int, str]
+) -> bool:
+ if isinstance(model_size, str):
+ model_size = model_size.replace("_", ".")
+ if isinstance(spec_model_size, str):
+ spec_model_size = spec_model_size.replace("_", ".")
+
+ if model_size == spec_model_size:
+ return True
+
+ try:
+ ms = int(model_size)
+ ss = int(spec_model_size)
+ return ms == ss
+ except ValueError:
+ return False
+
+
def match_llm(
model_name: str,
model_format: Optional[str] = None,
- model_size_in_billions: Optional[int] = None,
+ model_size_in_billions: Optional[Union[int, str]] = None,
quantization: Optional[str] = None,
is_local_deployment: bool = False,
) -> Optional[Tuple[LLMFamilyV1, LLMSpecV1, str]]:
@@ -844,7 +863,9 @@ def _apply_format_to_model_id(spec: LLMSpecV1, q: str) -> LLMSpecV1:
model_format
and model_format != spec.model_format
or model_size_in_billions
- and model_size_in_billions != spec.model_size_in_billions
+ and not match_model_size(
+ model_size_in_billions, spec.model_size_in_billions
+ )
or quantization
and matched_quantization is None
):
diff --git a/xinference/model/llm/tests/test_llm_family.py b/xinference/model/llm/tests/test_llm_family.py
index b1b6662dc1..ec119b43ce 100644
--- a/xinference/model/llm/tests/test_llm_family.py
+++ b/xinference/model/llm/tests/test_llm_family.py
@@ -36,6 +36,7 @@
is_self_hosted,
is_valid_model_uri,
match_llm,
+ match_model_size,
parse_uri,
)
@@ -1046,3 +1047,20 @@ def test_parse_prompt_style():
)
with pytest.raises(ValueError):
CustomLLMFamilyV1.parse_raw(bytes(llm_family.json(), "utf8"))
+
+
+def test_match_model_size():
+ assert match_model_size("1", "1")
+ assert match_model_size("1", 1)
+ assert match_model_size(1, 1)
+ assert not match_model_size("1", "b")
+ assert not match_model_size("1", "1b")
+ assert match_model_size("1.8", "1_8")
+ assert match_model_size("1_8", "1.8")
+ assert not match_model_size("1", "1_8")
+ assert not match_model_size("1__8", "1_8")
+ assert not match_model_size("1_8", 18)
+ assert not match_model_size("1_8", "18")
+ assert not match_model_size("1.8", 18)
+ assert not match_model_size("1.8", 1)
+ assert match_model_size("001", 1)
diff --git a/xinference/model/utils.py b/xinference/model/utils.py
index 8e15a31365..e1c71290c3 100644
--- a/xinference/model/utils.py
+++ b/xinference/model/utils.py
@@ -17,7 +17,7 @@
import shutil
from json import JSONDecodeError
from pathlib import Path
-from typing import Any, Callable, Dict, Optional, Tuple
+from typing import Any, Callable, Dict, Optional, Tuple, Union
from fsspec import AbstractFileSystem
@@ -415,3 +415,14 @@ def select_device(device):
raise ValueError(f"{device} is unavailable in your environment")
return device
+
+
+def convert_float_to_int_or_str(model_size: float) -> Union[int, str]:
+ """convert float to int or string
+
+ if float can be presented as int, convert it to int, otherwise convert it to string
+ """
+ if int(model_size) == model_size:
+ return int(model_size)
+ else:
+ return str(model_size)
From 92458312dcda3a2ddd2c70adf92672089f5ddbc9 Mon Sep 17 00:00:00 2001
From: hainaweiben <112739514+hainaweiben@users.noreply.github.com>
Date: Wed, 17 Apr 2024 11:47:14 +0800
Subject: [PATCH 014/298] FEAT: Lora multi support (#1273)
---
doc/source/models/lora.rst | 21 +++++--
xinference/api/restful_api.py | 18 +++---
xinference/client/restful/restful_client.py | 33 ++++++-----
xinference/core/supervisor.py | 9 +--
xinference/core/worker.py | 11 ++--
xinference/deploy/cmdline.py | 27 ++++++---
xinference/model/core.py | 13 ++---
xinference/model/image/core.py | 16 ++++--
.../model/image/stable_diffusion/core.py | 15 ++---
xinference/model/llm/core.py | 19 +++----
xinference/model/llm/llm_family.py | 5 +-
xinference/model/llm/pytorch/baichuan.py | 7 ++-
xinference/model/llm/pytorch/chatglm.py | 5 +-
xinference/model/llm/pytorch/core.py | 28 +++++-----
xinference/model/llm/pytorch/falcon.py | 11 ++--
xinference/model/llm/pytorch/internlm2.py | 5 +-
xinference/model/llm/pytorch/llama_2.py | 11 ++--
xinference/model/llm/pytorch/vicuna.py | 7 ++-
xinference/types.py | 55 +++++++++++++++++++
xinference/utils.py | 1 +
20 files changed, 204 insertions(+), 113 deletions(-)
diff --git a/doc/source/models/lora.rst b/doc/source/models/lora.rst
index 5cf73fb4a0..ad9e428074 100644
--- a/doc/source/models/lora.rst
+++ b/doc/source/models/lora.rst
@@ -15,7 +15,9 @@ Users need to first download the LoRA model themselves and then provide the stor
.. code-tab:: bash shell
- xinference launch --peft-model-path
+ xinference launch
+ --lora-modules
+ --lora-modules
--image-lora-load-kwargs
--image-lora-load-kwargs
--image-lora-fuse-kwargs
@@ -26,11 +28,22 @@ Users need to first download the LoRA model themselves and then provide the stor
from xinference.client import Client
client = Client("http://:")
+
+ lora_model1={'lora_name': , 'local_path': }
+ lora_model2={'lora_name': , 'local_path': }
+ lora_models=[lora_model1, lora_model2]
+ image_lora_load_kwargs={'': , '': },
+ image_lora_fuse_kwargs={'': , '': }
+
+ peft_model_config = {
+ "image_lora_load_kwargs": image_lora_load_params,
+ "image_lora_fuse_kwargs": image_lora_fuse_params,
+ "lora_list": lora_models
+ }
+
client.launch_model(
,
- peft_model_path='',
- image_lora_load_kwargs={'': , '': },
- image_lora_fuse_kwargs={'': , '': }
+ peft_model_config=peft_model_config
)
diff --git a/xinference/api/restful_api.py b/xinference/api/restful_api.py
index 3fcf068c45..ec58594cc3 100644
--- a/xinference/api/restful_api.py
+++ b/xinference/api/restful_api.py
@@ -64,6 +64,7 @@
CreateChatCompletion,
CreateCompletion,
ImageList,
+ PeftModelConfig,
max_tokens_field,
)
from .oauth2.auth_service import AuthService
@@ -692,9 +693,7 @@ async def launch_model(
replica = payload.get("replica", 1)
n_gpu = payload.get("n_gpu", "auto")
request_limits = payload.get("request_limits", None)
- peft_model_path = payload.get("peft_model_path", None)
- image_lora_load_kwargs = payload.get("image_lora_load_kwargs", None)
- image_lora_fuse_kwargs = payload.get("image_lora_fuse_kwargs", None)
+ peft_model_config = payload.get("peft_model_config", None)
worker_ip = payload.get("worker_ip", None)
gpu_idx = payload.get("gpu_idx", None)
@@ -708,9 +707,7 @@ async def launch_model(
"replica",
"n_gpu",
"request_limits",
- "peft_model_path",
- "image_lora_load_kwargs",
- "image_lora_fuse_kwargs",
+ "peft_model_config",
"worker_ip",
"gpu_idx",
}
@@ -725,6 +722,11 @@ async def launch_model(
detail="Invalid input. Please specify the model name",
)
+ if peft_model_config is not None:
+ peft_model_config = PeftModelConfig.from_dict(peft_model_config)
+ else:
+ peft_model_config = None
+
try:
model_uid = await (await self._get_supervisor_ref()).launch_builtin_model(
model_uid=model_uid,
@@ -737,9 +739,7 @@ async def launch_model(
n_gpu=n_gpu,
request_limits=request_limits,
wait_ready=wait_ready,
- peft_model_path=peft_model_path,
- image_lora_load_kwargs=image_lora_load_kwargs,
- image_lora_fuse_kwargs=image_lora_fuse_kwargs,
+ peft_model_config=peft_model_config,
worker_ip=worker_ip,
gpu_idx=gpu_idx,
**kwargs,
diff --git a/xinference/client/restful/restful_client.py b/xinference/client/restful/restful_client.py
index 50dcb2114e..48785547a3 100644
--- a/xinference/client/restful/restful_client.py
+++ b/xinference/client/restful/restful_client.py
@@ -19,6 +19,7 @@
import requests
from ...model.utils import convert_float_to_int_or_str
+from ...types import LoRA, PeftModelConfig
from ..common import streaming_response_iterator
if TYPE_CHECKING:
@@ -804,10 +805,8 @@ def launch_model(
quantization: Optional[str] = None,
replica: int = 1,
n_gpu: Optional[Union[int, str]] = "auto",
+ peft_model_config: Optional[Dict] = None,
request_limits: Optional[int] = None,
- peft_model_path: Optional[str] = None,
- image_lora_load_kwargs: Optional[Dict] = None,
- image_lora_fuse_kwargs: Optional[Dict] = None,
worker_ip: Optional[str] = None,
gpu_idx: Optional[Union[int, List[int]]] = None,
**kwargs,
@@ -834,15 +833,13 @@ def launch_model(
n_gpu: Optional[Union[int, str]],
The number of GPUs used by the model, default is "auto".
``n_gpu=None`` means cpu only, ``n_gpu=auto`` lets the system automatically determine the best number of GPUs to use.
+ peft_model_config: Optional[Dict]
+ - "lora_list": A List of PEFT (Parameter-Efficient Fine-Tuning) model and path.
+ - "image_lora_load_kwargs": A Dict of lora load parameters for image model
+ - "image_lora_fuse_kwargs": A Dict of lora fuse parameters for image model
request_limits: Optional[int]
- The number of request limits for this model， default is None.
+ The number of request limits for this model, default is None.
``request_limits=None`` means no limits for this model.
- peft_model_path: Optional[str]
- PEFT (Parameter-Efficient Fine-Tuning) model path.
- image_lora_load_kwargs: Optional[Dict]
- lora load parameters for image model
- image_lora_fuse_kwargs: Optional[Dict]
- lora fuse parameters for image model
worker_ip: Optional[str]
Specify the worker ip where the model is located in a distributed scenario.
gpu_idx: Optional[Union[int, List[int]]]
@@ -859,6 +856,18 @@ def launch_model(
url = f"{self.base_url}/v1/models"
+ if peft_model_config is not None:
+ lora_list = [
+ LoRA.from_dict(model) for model in peft_model_config["lora_list"]
+ ]
+ peft_model = PeftModelConfig(
+ lora_list,
+ peft_model_config["image_lora_load_kwargs"],
+ peft_model_config["image_lora_fuse_kwargs"],
+ )
+ else:
+ peft_model = None
+
# convert float to int or string since the RESTful API does not accept float.
if isinstance(model_size_in_billions, float):
model_size_in_billions = convert_float_to_int_or_str(model_size_in_billions)
@@ -866,6 +875,7 @@ def launch_model(
payload = {
"model_uid": model_uid,
"model_name": model_name,
+ "peft_model_config": peft_model.to_dict() if peft_model else None,
"model_type": model_type,
"model_size_in_billions": model_size_in_billions,
"model_format": model_format,
@@ -873,9 +883,6 @@ def launch_model(
"replica": replica,
"n_gpu": n_gpu,
"request_limits": request_limits,
- "peft_model_path": peft_model_path,
- "image_lora_load_kwargs": image_lora_load_kwargs,
- "image_lora_fuse_kwargs": image_lora_fuse_kwargs,
"worker_ip": worker_ip,
"gpu_idx": gpu_idx,
}
diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py
index dae8c3c6c8..115baf1e73 100644
--- a/xinference/core/supervisor.py
+++ b/xinference/core/supervisor.py
@@ -30,6 +30,7 @@
)
from ..core import ModelActor
from ..core.status_guard import InstanceInfo, LaunchStatus
+from ..types import PeftModelConfig
from .metrics import record_metrics
from .resource import GPUStatus, ResourceStatus
from .utils import (
@@ -749,9 +750,7 @@ async def launch_builtin_model(
request_limits: Optional[int] = None,
wait_ready: bool = True,
model_version: Optional[str] = None,
- peft_model_path: Optional[str] = None,
- image_lora_load_kwargs: Optional[Dict] = None,
- image_lora_fuse_kwargs: Optional[Dict] = None,
+ peft_model_config: Optional[PeftModelConfig] = None,
worker_ip: Optional[str] = None,
gpu_idx: Optional[Union[int, List[int]]] = None,
**kwargs,
@@ -803,9 +802,7 @@ async def _launch_one_model(_replica_model_uid):
model_type=model_type,
n_gpu=n_gpu,
request_limits=request_limits,
- peft_model_path=peft_model_path,
- image_lora_load_kwargs=image_lora_load_kwargs,
- image_lora_fuse_kwargs=image_lora_fuse_kwargs,
+ peft_model_config=peft_model_config,
gpu_idx=gpu_idx,
**kwargs,
)
diff --git a/xinference/core/worker.py b/xinference/core/worker.py
index 465fae7b84..26cd664c51 100644
--- a/xinference/core/worker.py
+++ b/xinference/core/worker.py
@@ -36,6 +36,7 @@
from ..core.status_guard import LaunchStatus
from ..device_utils import gpu_count
from ..model.core import ModelDescription, create_model_instance
+from ..types import PeftModelConfig
from .event import Event, EventCollectorActor, EventType
from .metrics import launch_metrics_export_server, record_metrics
from .resource import gather_node_info
@@ -606,9 +607,7 @@ async def launch_builtin_model(
quantization: Optional[str],
model_type: str = "LLM",
n_gpu: Optional[Union[int, str]] = "auto",
- peft_model_path: Optional[str] = None,
- image_lora_load_kwargs: Optional[Dict] = None,
- image_lora_fuse_kwargs: Optional[Dict] = None,
+ peft_model_config: Optional[PeftModelConfig] = None,
request_limits: Optional[int] = None,
gpu_idx: Optional[Union[int, List[int]]] = None,
**kwargs,
@@ -646,7 +645,7 @@ async def launch_builtin_model(
if isinstance(n_gpu, str) and n_gpu != "auto":
raise ValueError("Currently `n_gpu` only supports `auto`.")
- if peft_model_path is not None:
+ if peft_model_config is not None:
if model_type in ("embedding", "rerank"):
raise ValueError(
f"PEFT adaptors cannot be applied to embedding or rerank models."
@@ -677,9 +676,7 @@ async def launch_builtin_model(
model_format,
model_size_in_billions,
quantization,
- peft_model_path,
- image_lora_load_kwargs,
- image_lora_fuse_kwargs,
+ peft_model_config,
is_local_deployment,
**kwargs,
)
diff --git a/xinference/deploy/cmdline.py b/xinference/deploy/cmdline.py
index baae417c6e..89ae648f07 100644
--- a/xinference/deploy/cmdline.py
+++ b/xinference/deploy/cmdline.py
@@ -640,10 +640,11 @@ def list_model_registrations(
help='The number of GPUs used by the model, default is "auto".',
)
@click.option(
- "--peft-model-path",
- default=None,
- type=str,
- help="PEFT model path.",
+ "--lora-modules",
+ "-lm",
+ multiple=True,
+ type=(str, str),
+ help="LoRA module configurations in the format name=path. Multiple modules can be specified.",
)
@click.option(
"--image-lora-load-kwargs",
@@ -696,7 +697,7 @@ def model_launch(
quantization: str,
replica: int,
n_gpu: str,
- peft_model_path: Optional[str],
+ lora_modules: Optional[Tuple],
image_lora_load_kwargs: Optional[Tuple],
image_lora_fuse_kwargs: Optional[Tuple],
worker_ip: Optional[str],
@@ -729,6 +730,18 @@ def model_launch(
else None
)
+ lora_list = (
+ [{"lora_name": k, "local_path": v} for k, v in dict(lora_modules).items()]
+ if lora_modules
+ else []
+ )
+
+ peft_model_config = {
+ "image_lora_load_kwargs": image_lora_load_params,
+ "image_lora_fuse_kwargs": image_lora_fuse_params,
+ "lora_list": lora_list,
+ }
+
_gpu_idx: Optional[List[int]] = (
None if gpu_idx is None else [int(idx) for idx in gpu_idx.split(",")]
)
@@ -754,9 +767,7 @@ def model_launch(
quantization=quantization,
replica=replica,
n_gpu=_n_gpu,
- peft_model_path=peft_model_path,
- image_lora_load_kwargs=image_lora_load_params,
- image_lora_fuse_kwargs=image_lora_fuse_params,
+ peft_model_config=peft_model_config,
worker_ip=worker_ip,
gpu_idx=_gpu_idx,
trust_remote_code=trust_remote_code,
diff --git a/xinference/model/core.py b/xinference/model/core.py
index 842eca6387..e17f644e46 100644
--- a/xinference/model/core.py
+++ b/xinference/model/core.py
@@ -13,9 +13,10 @@
# limitations under the License.
from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, List, Optional, Tuple, Union
from .._compat import BaseModel
+from ..types import PeftModelConfig
class ModelDescription(ABC):
@@ -52,9 +53,7 @@ def create_model_instance(
model_format: Optional[str] = None,
model_size_in_billions: Optional[Union[int, str]] = None,
quantization: Optional[str] = None,
- peft_model_path: Optional[str] = None,
- image_lora_load_kwargs: Optional[Dict] = None,
- image_lora_fuse_kwargs: Optional[Dict] = None,
+ peft_model_config: Optional[PeftModelConfig] = None,
is_local_deployment: bool = False,
**kwargs,
) -> Tuple[Any, ModelDescription]:
@@ -73,7 +72,7 @@ def create_model_instance(
model_format,
model_size_in_billions,
quantization,
- peft_model_path,
+ peft_model_config,
is_local_deployment,
**kwargs,
)
@@ -90,9 +89,7 @@ def create_model_instance(
devices,
model_uid,
model_name,
- lora_model_path=peft_model_path,
- lora_load_kwargs=image_lora_load_kwargs,
- lora_fuse_kwargs=image_lora_fuse_kwargs,
+ peft_model_config,
**kwargs,
)
elif model_type == "rerank":
diff --git a/xinference/model/image/core.py b/xinference/model/image/core.py
index e19a2b839c..028f5cdc4d 100644
--- a/xinference/model/image/core.py
+++ b/xinference/model/image/core.py
@@ -18,6 +18,7 @@
from typing import Dict, List, Optional, Tuple
from ...constants import XINFERENCE_CACHE_DIR
+from ...types import PeftModelConfig
from ..core import CacheableModelSpec, ModelDescription
from ..utils import valid_model_revision
from .stable_diffusion.core import DiffusionModel
@@ -175,9 +176,7 @@ def create_image_model_instance(
devices: List[str],
model_uid: str,
model_name: str,
- lora_model_path: Optional[str] = None,
- lora_load_kwargs: Optional[Dict] = None,
- lora_fuse_kwargs: Optional[Dict] = None,
+ peft_model_config: Optional[PeftModelConfig] = None,
**kwargs,
) -> Tuple[DiffusionModel, ImageModelDescription]:
model_spec = match_diffusion(model_name)
@@ -210,10 +209,19 @@ def create_image_model_instance(
else:
kwargs["controlnet"] = controlnet_model_paths
model_path = cache(model_spec)
+ if peft_model_config is not None:
+ lora_model = peft_model_config.peft_model
+ lora_load_kwargs = peft_model_config.image_lora_load_kwargs
+ lora_fuse_kwargs = peft_model_config.image_lora_fuse_kwargs
+ else:
+ lora_model = None
+ lora_load_kwargs = None
+ lora_fuse_kwargs = None
+
model = DiffusionModel(
model_uid,
model_path,
- lora_model_path=lora_model_path,
+ lora_model_paths=lora_model,
lora_load_kwargs=lora_load_kwargs,
lora_fuse_kwargs=lora_fuse_kwargs,
**kwargs,
diff --git a/xinference/model/image/stable_diffusion/core.py b/xinference/model/image/stable_diffusion/core.py
index 11d5e50802..6d9be6ee2a 100644
--- a/xinference/model/image/stable_diffusion/core.py
+++ b/xinference/model/image/stable_diffusion/core.py
@@ -25,7 +25,7 @@
from ....constants import XINFERENCE_IMAGE_DIR
from ....device_utils import move_model_to_available_device
-from ....types import Image, ImageList
+from ....types import Image, ImageList, LoRA
logger = logging.getLogger(__name__)
@@ -36,7 +36,7 @@ def __init__(
model_uid: str,
model_path: str,
device: Optional[str] = None,
- lora_model_path: Optional[str] = None,
+ lora_model: Optional[List[LoRA]] = None,
lora_load_kwargs: Optional[Dict] = None,
lora_fuse_kwargs: Optional[Dict] = None,
**kwargs,
@@ -45,20 +45,21 @@ def __init__(
self._model_path = model_path
self._device = device
self._model = None
- self._lora_model_path = lora_model_path
+ self._lora_model = lora_model
self._lora_load_kwargs = lora_load_kwargs or {}
self._lora_fuse_kwargs = lora_fuse_kwargs or {}
self._kwargs = kwargs
def _apply_lora(self):
- if self._lora_model_path is not None:
+ if self._lora_model is not None:
logger.info(
f"Loading the LoRA with load kwargs: {self._lora_load_kwargs}, fuse kwargs: {self._lora_fuse_kwargs}."
)
assert self._model is not None
- self._model.load_lora_weights(
- self._lora_model_path, **self._lora_load_kwargs
- )
+ for lora_model in self._lora_model:
+ self._model.load_lora_weights(
+ lora_model.local_path, **self._lora_load_kwargs
+ )
self._model.fuse_lora(**self._lora_fuse_kwargs)
logger.info(f"Successfully loaded the LoRA for model {self._model_uid}.")
diff --git a/xinference/model/llm/core.py b/xinference/model/llm/core.py
index 77ba6b2d61..54bbe5f32f 100644
--- a/xinference/model/llm/core.py
+++ b/xinference/model/llm/core.py
@@ -21,6 +21,7 @@
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
from ...core.utils import parse_replica_model_uid
+from ...types import PeftModelConfig
from ..core import ModelDescription
if TYPE_CHECKING:
@@ -180,7 +181,7 @@ def create_llm_model_instance(
model_format: Optional[str] = None,
model_size_in_billions: Optional[Union[int, str]] = None,
quantization: Optional[str] = None,
- peft_model_path: Optional[str] = None,
+ peft_model_config: Optional[PeftModelConfig] = None,
is_local_deployment: bool = False,
**kwargs,
) -> Tuple[LLM, LLMDescription]:
@@ -204,9 +205,9 @@ def create_llm_model_instance(
assert quantization is not None
save_path = cache(llm_family, llm_spec, quantization)
- llm_cls = match_llm_cls(
- llm_family, llm_spec, quantization, peft_model_path=peft_model_path
- )
+ peft_model = peft_model_config.peft_model if peft_model_config else None
+
+ llm_cls = match_llm_cls(llm_family, llm_spec, quantization, peft_model=peft_model)
if not llm_cls:
raise ValueError(
f"Model not supported, name: {model_name}, format: {model_format},"
@@ -214,15 +215,9 @@ def create_llm_model_instance(
)
logger.debug(f"Launching {model_uid} with {llm_cls.__name__}")
- if peft_model_path is not None:
+ if peft_model is not None:
model = llm_cls(
- model_uid,
- llm_family,
- llm_spec,
- quantization,
- save_path,
- kwargs,
- peft_model_path,
+ model_uid, llm_family, llm_spec, quantization, save_path, kwargs, peft_model
)
else:
model = llm_cls(
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
index 4252c3a16f..6f5a01d6e7 100644
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -33,6 +33,7 @@
validator,
)
from ...constants import XINFERENCE_CACHE_DIR, XINFERENCE_MODEL_DIR
+from ...types import LoRA
from ..utils import (
download_from_modelscope,
is_valid_model_uri,
@@ -975,12 +976,12 @@ def match_llm_cls(
family: LLMFamilyV1,
llm_spec: "LLMSpecV1",
quantization: str,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
) -> Optional[Type[LLM]]:
"""
Find an LLM implementation for given LLM family and spec.
"""
- if peft_model_path is not None:
+ if peft_model is not None:
for cls in PEFT_SUPPORTED_CLASSES:
if cls.match(family, llm_spec, quantization):
return cls
diff --git a/xinference/model/llm/pytorch/baichuan.py b/xinference/model/llm/pytorch/baichuan.py
index 8b3adb2c88..ac45c53f6e 100644
--- a/xinference/model/llm/pytorch/baichuan.py
+++ b/xinference/model/llm/pytorch/baichuan.py
@@ -12,8 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Optional
+from typing import List, Optional
+from ....types import LoRA
from ..llm_family import LLMFamilyV1, LLMSpecV1
from .core import PytorchChatModel, PytorchModelConfig
@@ -27,7 +28,7 @@ def __init__(
quantization: str,
model_path: str,
pytorch_model_config: Optional[PytorchModelConfig] = None,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(
model_uid,
@@ -36,7 +37,7 @@ def __init__(
quantization,
model_path,
pytorch_model_config=pytorch_model_config,
- peft_model_path=peft_model_path,
+ peft_model=peft_model,
)
self._use_fast_tokenizer = False
diff --git a/xinference/model/llm/pytorch/chatglm.py b/xinference/model/llm/pytorch/chatglm.py
index e917b19deb..e8d7c7de4a 100644
--- a/xinference/model/llm/pytorch/chatglm.py
+++ b/xinference/model/llm/pytorch/chatglm.py
@@ -24,6 +24,7 @@
CompletionChoice,
CompletionChunk,
CompletionUsage,
+ LoRA,
PytorchGenerateConfig,
)
from ..llm_family import LLMFamilyV1, LLMSpecV1
@@ -39,7 +40,7 @@ def __init__(
quantization: str,
model_path: str,
pytorch_model_config: Optional[PytorchModelConfig] = None,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(
model_uid,
@@ -48,7 +49,7 @@ def __init__(
quantization,
model_path,
pytorch_model_config=pytorch_model_config,
- peft_model_path=peft_model_path,
+ peft_model=peft_model,
)
def _load_model(self, **kwargs):
diff --git a/xinference/model/llm/pytorch/core.py b/xinference/model/llm/pytorch/core.py
index 8cac269289..3703b36704 100644
--- a/xinference/model/llm/pytorch/core.py
+++ b/xinference/model/llm/pytorch/core.py
@@ -32,6 +32,7 @@
Embedding,
EmbeddingData,
EmbeddingUsage,
+ LoRA,
PytorchGenerateConfig,
PytorchModelConfig,
)
@@ -71,14 +72,14 @@ def __init__(
quantization: str,
model_path: str,
pytorch_model_config: Optional[PytorchModelConfig] = None,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(model_uid, model_family, model_spec, quantization, model_path)
self._use_fast_tokenizer = True
self._pytorch_model_config: PytorchModelConfig = self._sanitize_model_config(
pytorch_model_config
)
- self._peft_model_path = peft_model_path
+ self._peft_model = peft_model
def _sanitize_model_config(
self, pytorch_model_config: Optional[PytorchModelConfig]
@@ -134,7 +135,7 @@ def _load_model(self, **kwargs):
return model, tokenizer
def _apply_lora(self):
- if self._peft_model_path is not None:
+ if self._peft_model is not None:
try:
from peft import PeftModel
except ImportError:
@@ -142,14 +143,15 @@ def _apply_lora(self):
f"Failed to import 'PeftModel' from 'peft'. Please make sure 'peft' is installed.\n\n"
)
- # Apply LoRA
- self._model = PeftModel.from_pretrained(
- self._model,
- self._peft_model_path,
- )
- logger.info(
- f"Successfully loaded the PEFT adaptor for model {self.model_uid}."
- )
+ for peft_model in self._peft_model:
+ # Apply LoRA
+ self._model = PeftModel.from_pretrained(
+ self._model,
+ peft_model.local_path,
+ )
+ logger.info(
+ f"PEFT adaptor '{peft_model.lora_name}' successfully loaded for model '{self.model_uid}'."
+ )
def load(self):
try:
@@ -421,7 +423,7 @@ def __init__(
quantization: str,
model_path: str,
pytorch_model_config: Optional[PytorchModelConfig] = None,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(
model_uid,
@@ -430,7 +432,7 @@ def __init__(
quantization,
model_path,
pytorch_model_config,
- peft_model_path,
+ peft_model,
)
def _sanitize_generate_config(
diff --git a/xinference/model/llm/pytorch/falcon.py b/xinference/model/llm/pytorch/falcon.py
index 782f29e750..7f66bdd01f 100644
--- a/xinference/model/llm/pytorch/falcon.py
+++ b/xinference/model/llm/pytorch/falcon.py
@@ -12,8 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Optional
+from typing import List, Optional
+from ....types import LoRA
from ..llm_family import LLMFamilyV1, LLMSpecV1
from .core import PytorchChatModel, PytorchModel, PytorchModelConfig
@@ -27,7 +28,7 @@ def __init__(
quantization: str,
model_path: str,
pytorch_model_config: Optional[PytorchModelConfig] = None,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(
model_uid,
@@ -36,7 +37,7 @@ def __init__(
quantization,
model_path,
pytorch_model_config=pytorch_model_config,
- peft_model_path=peft_model_path,
+ peft_model=peft_model,
)
def _load_model(self, **kwargs):
@@ -86,7 +87,7 @@ def __init__(
quantization: str,
model_path: str,
pytorch_model_config: Optional[PytorchModelConfig] = None,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(
model_uid,
@@ -95,7 +96,7 @@ def __init__(
quantization,
model_path,
pytorch_model_config=pytorch_model_config,
- peft_model_path=peft_model_path,
+ peft_model=peft_model,
)
def _load_model(self, **kwargs):
diff --git a/xinference/model/llm/pytorch/internlm2.py b/xinference/model/llm/pytorch/internlm2.py
index 2e8a9e69c6..c970f17816 100644
--- a/xinference/model/llm/pytorch/internlm2.py
+++ b/xinference/model/llm/pytorch/internlm2.py
@@ -23,6 +23,7 @@
CompletionChoice,
CompletionChunk,
CompletionUsage,
+ LoRA,
PytorchGenerateConfig,
)
from ..llm_family import LLMFamilyV1, LLMSpecV1
@@ -38,7 +39,7 @@ def __init__(
quantization: str,
model_path: str,
pytorch_model_config: Optional[PytorchModelConfig] = None,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(
model_uid,
@@ -47,7 +48,7 @@ def __init__(
quantization,
model_path,
pytorch_model_config=pytorch_model_config,
- peft_model_path=peft_model_path,
+ peft_model=peft_model,
)
def _load_model(self, **kwargs):
diff --git a/xinference/model/llm/pytorch/llama_2.py b/xinference/model/llm/pytorch/llama_2.py
index 27f3235a9b..a7c3e7bba6 100644
--- a/xinference/model/llm/pytorch/llama_2.py
+++ b/xinference/model/llm/pytorch/llama_2.py
@@ -12,8 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Optional
+from typing import List, Optional
+from ....types import LoRA
from ..llm_family import LLMFamilyV1, LLMSpecV1
from .core import PytorchChatModel, PytorchModel, PytorchModelConfig
@@ -27,7 +28,7 @@ def __init__(
quantization: str,
model_path: str,
pytorch_model_config: Optional[PytorchModelConfig] = None,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(
model_uid,
@@ -36,7 +37,7 @@ def __init__(
quantization,
model_path,
pytorch_model_config=pytorch_model_config,
- peft_model_path=peft_model_path,
+ peft_model=peft_model,
)
def _load_model(self, **kwargs):
@@ -69,8 +70,8 @@ def __init__(
model_spec: "LLMSpecV1",
quantization: str,
model_path: str,
- peft_model_path: Optional[str] = None,
pytorch_model_config: Optional["PytorchModelConfig"] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(
model_uid,
@@ -78,7 +79,7 @@ def __init__(
model_spec,
quantization,
model_path,
- peft_model_path=peft_model_path,
+ peft_model=peft_model,
pytorch_model_config=pytorch_model_config,
)
self._use_fast_tokenizer = False
diff --git a/xinference/model/llm/pytorch/vicuna.py b/xinference/model/llm/pytorch/vicuna.py
index 89231a0702..8e9eb16233 100644
--- a/xinference/model/llm/pytorch/vicuna.py
+++ b/xinference/model/llm/pytorch/vicuna.py
@@ -26,8 +26,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Optional
+from typing import List, Optional
+from ....types import LoRA
from .. import LLMFamilyV1, LLMSpecV1
from .core import PytorchChatModel, PytorchModelConfig
@@ -41,7 +42,7 @@ def __init__(
quantization: str,
model_path: str,
pytorch_model_config: Optional["PytorchModelConfig"] = None,
- peft_model_path: Optional[str] = None,
+ peft_model: Optional[List[LoRA]] = None,
):
super().__init__(
model_uid,
@@ -50,7 +51,7 @@ def __init__(
quantization,
model_path,
pytorch_model_config=pytorch_model_config,
- peft_model_path=peft_model_path,
+ peft_model=peft_model,
)
self._use_fast_tokenizer = False
diff --git a/xinference/types.py b/xinference/types.py
index 964a865ac7..b300237f66 100644
--- a/xinference/types.py
+++ b/xinference/types.py
@@ -450,3 +450,58 @@ class CreateChatCompletion(
CreateChatCompletionOpenAI,
):
pass
+
+
+class LoRA:
+ def __init__(self, lora_name: str, local_path: str):
+ self.lora_name = lora_name
+ self.local_path = local_path
+
+ def to_dict(self):
+ return {
+ "lora_name": self.lora_name,
+ "local_path": self.local_path,
+ }
+
+ @classmethod
+ def from_dict(cls, data: Dict):
+ return cls(
+ lora_name=data["lora_name"],
+ local_path=data["local_path"],
+ )
+
+
+class PeftModelConfig:
+ def __init__(
+ self,
+ peft_model: Optional[List[LoRA]] = None,
+ image_lora_load_kwargs: Optional[Dict] = None,
+ image_lora_fuse_kwargs: Optional[Dict] = None,
+ ):
+ self.peft_model = peft_model
+ self.image_lora_load_kwargs = image_lora_load_kwargs
+ self.image_lora_fuse_kwargs = image_lora_fuse_kwargs
+
+ def to_dict(self):
+ return {
+ "peft_model": [lora.to_dict() for lora in self.peft_model]
+ if self.peft_model
+ else None,
+ "image_lora_load_kwargs": self.image_lora_load_kwargs,
+ "image_lora_fuse_kwargs": self.image_lora_fuse_kwargs,
+ }
+
+ @classmethod
+ def from_dict(cls, data: Dict):
+ peft_model_data = data.get("peft_model")
+ peft_model = (
+ [LoRA.from_dict(lora_dict) for lora_dict in peft_model_data]
+ if peft_model_data
+ else None
+ )
+
+ return cls(
+ peft_model=peft_model,
+ image_lora_load_kwargs=data.get("image_lora_load_kwargs"),
+ image_lora_fuse_kwargs=data.get("image_lora_fuse_kwargs"),
+ )
diff --git a/xinference/utils.py b/xinference/utils.py
index 5b3741c222..79a46fde6d 100644
--- a/xinference/utils.py
+++ b/xinference/utils.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
import torch
From fa640ee8d936630a187bf2f14637f38a08c8c060 Mon Sep 17 00:00:00 2001
From: mujin2 <132123199+mujin2@users.noreply.github.com>
Date: Wed, 17 Apr 2024 14:00:48 +0800
Subject: [PATCH 015/298] FEAT: Support SeaLLM-7B and c4ai-command-r-v01
(#1310)
Co-authored-by: ChengjieLi
---
xinference/model/llm/llm_family.json | 195 +++++++++++++++++++++++++++
1 file changed, 195 insertions(+)
diff --git a/xinference/model/llm/llm_family.json b/xinference/model/llm/llm_family.json
index 7ed1b4509c..a2a2332b6c 100644
--- a/xinference/model/llm/llm_family.json
+++ b/xinference/model/llm/llm_family.json
@@ -4752,5 +4752,200 @@
""
]
}
+ },
+ {
+ "version": 1,
+ "context_length": 8192,
+ "model_name": "seallm_v2",
+ "model_lang": [
+ "en",
+ "zh",
+ "vi",
+ "id",
+ "th",
+ "ms",
+ "km",
+ "lo",
+ "my",
+ "tl"
+ ],
+ "model_ability": [
+ "generate"
+ ],
+ "model_description": "We introduce SeaLLM-7B-v2, the state-of-the-art multilingual LLM for Southeast Asian (SEA) languages",
+ "model_specs": [
+ {
+ "model_format": "pytorch",
+ "model_size_in_billions": 7,
+ "quantizations": [
+ "none"
+ ],
+ "model_id": "SeaLLMs/SeaLLM-7B-v2",
+ "model_revision": "f1bd48e0d75365c24a3c5ad006b2d0a0c9dca30f"
+ },
+ {
+ "model_format": "ggufv2",
+ "model_size_in_billions": 7,
+ "quantizations": [
+ "Q4_0",
+ "Q8_0"
+ ],
+ "model_id": "SeaLLMs/SeaLLM-7B-v2-gguf",
+ "model_file_name_template": "SeaLLM-7B-v2.{quantization}.gguf"
+ }
+ ]
+ },
+ {
+ "version": 1,
+ "context_length": 8192,
+ "model_name": "seallm_v2.5",
+ "model_lang": [
+ "en",
+ "zh",
+ "vi",
+ "id",
+ "th",
+ "ms",
+ "km",
+ "lo",
+ "my",
+ "tl"
+ ],
+ "model_ability": [
+ "generate"
+ ],
+ "model_description": "We introduce SeaLLM-7B-v2.5, the state-of-the-art multilingual LLM for Southeast Asian (SEA) languages",
+ "model_specs": [
+ {
+ "model_format": "pytorch",
+ "model_size_in_billions": 7,
+ "quantizations": [
+ "none"
+ ],
+ "model_id": "SeaLLMs/SeaLLM-7B-v2.5",
+ "model_revision": "c54a8eb8e2d58c5a680bfbbe3a7ae71753bb644b"
+ },
+ {
+ "model_format": "ggufv2",
+ "model_size_in_billions": 7,
+ "quantizations": [
+ "Q4_K_M",
+ "Q8_0"
+ ],
+ "model_id": "SeaLLMs/SeaLLM-7B-v2.5-GGUF",
+ "model_file_name_template": "SeaLLM-7B-v2.5.{quantization}.gguf"
+ }
+ ]
+ },
+ {
+ "version": 1,
+ "context_length": 131072,
+ "model_name": "c4ai-command-r-v01",
+ "model_lang": [
+ "en",
+ "fr",
+ "de",
+ "es",
+ "it",
+ "pt",
+ "ja",
+ "ko",
+ "zh",
+ "ar"
+ ],
+ "model_ability": [
+ "generate"
+ ],
+ "model_description": "C4AI Command-R is a research release of a 35 billion parameter highly performant generative model.",
+ "model_specs": [
+ {
+ "model_format": "pytorch",
+ "model_size_in_billions": 35,
+ "quantizations": [
+ "none"
+ ],
+ "model_id": "CohereForAI/c4ai-command-r-v01",
+ "model_revision": "16881ccde1c68bbc7041280e6a66637bc46bfe88"
+ },
+ {
+ "model_format": "ggufv2",
+ "model_size_in_billions": 35,
+ "quantizations": [
+ "Q2_K",
+ "Q3_K_L",
+ "Q3_K_M",
+ "Q3_K_S",
+ "Q4_0",
+ "Q4_K_M",
+ "Q4_K_S",
+ "Q5_0",
+ "Q5_K_M",
+ "Q5_K_S",
+ "Q6_K",
+ "Q8_0"
+ ],
+ "model_id": "andrewcanis/c4ai-command-r-v01-GGUF",
+ "model_file_name_template": "c4ai-command-r-v01.{quantization}.gguf"
+ },
+ {
+ "model_format": "pytorch",
+ "model_size_in_billions": 104,
+ "quantizations": [
+ "none"
+ ],
+ "model_id": "CohereForAI/c4ai-command-r-plus",
+ "model_revision": "ba7f1d954c9d1609013677d87e4142ab95c34e62"
+ },
+ {
+ "model_format": "gptq",
+ "model_size_in_billions": 104,
+ "quantizations": [
+ "Int4"
+ ],
+ "model_id": "alpindale/c4ai-command-r-plus-GPTQ",
+ "model_revision": "35febfc08f723ac0df32480eb4af349a7d08656e"
+ }
+ ]
+ },
+ {
+ "version": 1,
+ "context_length": 131072,
+ "model_name": "c4ai-command-r-v01-4bit",
+ "model_lang": [
+ "en",
+ "fr",
+ "de",
+ "es",
+ "it",
+ "pt",
+ "ja",
+ "ko",
+ "zh",
+ "ar"
+ ],
+ "model_ability": [
+ "generate"
+ ],
+ "model_description": "This model is 4bit quantized version of C4AI Command-R using bitsandbytes.",
+ "model_specs": [
+ {
+ "model_format": "pytorch",
+ "model_size_in_billions": 35,
+ "quantizations": [
+ "none"
+ ],
+ "model_id": "CohereForAI/c4ai-command-r-v01-4bit",
+ "model_revision": "f2e87936a146643c9dd143422dcafb9cb1552611"
+ },
+ {
+ "model_format": "pytorch",
+ "model_size_in_billions": 104,
+ "quantizations": [
+ "none"
+ ],
+ "model_id": "CohereForAI/c4ai-command-r-plus-4bit",
+ "model_revision": "bb63b5b7005ecedb30b0cfd0d5953b02a5817f7b"
+ }
+ ]
}
]
From d3e245e8326cb472087ec7bae886f1ae73cb1801 Mon Sep 17 00:00:00 2001
From: codingl2k1 <138426806+codingl2k1@users.noreply.github.com>
Date: Wed, 17 Apr 2024 11:00:42 +0200
Subject: [PATCH 016/298] FEAT: Support BAAI/bge-reranker-v2-* rerank model
(#1305)
Co-authored-by: qinxuye
---
.github/workflows/python.yaml | 1 +
setup.cfg | 3 ++
xinference/model/rerank/core.py | 35 ++++++++++++-------
xinference/model/rerank/model_spec.json | 24 +++++++++++++
.../model/rerank/model_spec_modelscope.json | 26 +++++++++++++-
xinference/model/rerank/tests/test_rerank.py | 5 +--
6 files changed, 79 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
index 648b6044ad..acbbb0b544 100644
--- a/.github/workflows/python.yaml
+++ b/.github/workflows/python.yaml
@@ -124,6 +124,7 @@ jobs:
pip install modelscope
pip install diffusers
pip install protobuf
+ pip install FlagEmbedding
pip install -e ".[dev]"
pip install "jinja2==3.1.2"
working-directory: .
diff --git a/setup.cfg b/setup.cfg
index 46053cf910..4e6fca6715 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -105,6 +105,7 @@ all =
attrdict # For deepseek VL
timm>=0.9.16 # For deepseek VL
torchvision # For deepseek VL
+ FlagEmbedding # For rerank
intel =
torch==2.1.0a0
intel_extension_for_pytorch==2.1.10+xpu
@@ -134,6 +135,8 @@ sglang =
sglang[all]
embedding =
sentence-transformers>=2.3.1
+rerank =
+ FlagEmbedding
image =
diffusers
controlnet_aux
diff --git a/xinference/model/rerank/core.py b/xinference/model/rerank/core.py
index 6af2224186..9a715eb115 100644
--- a/xinference/model/rerank/core.py
+++ b/xinference/model/rerank/core.py
@@ -42,8 +42,9 @@ def get_rerank_model_descriptions():
class RerankModelSpec(CacheableModelSpec):
model_name: str
language: List[str]
+ type: Optional[str] = "normal"
model_id: str
- model_revision: str
+ model_revision: Optional[str]
model_hub: str = "huggingface"
@@ -63,6 +64,7 @@ def to_dict(self):
"model_type": "rerank",
"address": self.address,
"accelerators": self.devices,
+ "type": self._model_spec.type,
"model_name": self._model_spec.model_name,
"language": self._model_spec.language,
"model_revision": self._model_spec.model_revision,
@@ -97,12 +99,14 @@ def generate_rerank_description(model_spec: RerankModelSpec) -> Dict[str, List[D
class RerankModel:
def __init__(
self,
+ model_spec: RerankModelSpec,
model_uid: str,
model_path: str,
device: Optional[str] = None,
use_fp16: bool = False,
model_config: Optional[Dict] = None,
):
+ self._model_spec = model_spec
self._model_uid = model_uid
self._model_path = model_path
self._device = device
@@ -112,20 +116,25 @@ def __init__(
def load(self):
try:
- from sentence_transformers.cross_encoder import CrossEncoder
+ if self._model_spec.type == "normal":
+ from FlagEmbedding import FlagReranker
+ elif self._model_spec.type == "LLM-based":
+ from FlagEmbedding import FlagLLMReranker as FlagReranker
+ elif self._model_spec.type == "LLM-based layerwise":
+ from FlagEmbedding import LayerWiseFlagLLMReranker as FlagReranker
+ else:
+ raise RuntimeError(
+ f"Unsupported Rank model type: {self._model_spec.type}"
+ )
except ImportError:
- error_message = "Failed to import module 'SentenceTransformer'"
+ error_message = "Failed to import module 'FlagEmbedding'"
installation_guide = [
- "Please make sure 'sentence-transformers' is installed. ",
- "You can install it by `pip install sentence-transformers`\n",
+ "Please make sure 'FlagEmbedding' is installed. ",
+ "You can install it by `pip install FlagEmbedding`\n",
]
raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
- self._model = CrossEncoder(
- self._model_path, device=self._device, **self._model_config
- )
- if self._use_fp16:
- self._model.model.half()
+ self._model = FlagReranker(self._model_path, use_fp16=True)
def rerank(
self,
@@ -142,7 +151,7 @@ def rerank(
if max_chunks_per_doc is not None:
raise ValueError("rerank hasn't support `max_chunks_per_doc` parameter.")
sentence_combinations = [[query, doc] for doc in documents]
- similarity_scores = self._model.predict(sentence_combinations)
+ similarity_scores = self._model.compute_score(sentence_combinations)
sim_scores_argsort = list(reversed(np.argsort(similarity_scores)))
if top_n is not None:
sim_scores_argsort = sim_scores_argsort[:top_n]
@@ -224,7 +233,9 @@ def create_rerank_model_instance(
model_path = cache(model_spec)
use_fp16 = kwargs.pop("use_fp16", False)
- model = RerankModel(model_uid, model_path, use_fp16=use_fp16, model_config=kwargs)
+ model = RerankModel(
+ model_spec, model_uid, model_path, use_fp16=use_fp16, model_config=kwargs
+ )
model_description = RerankModelDescription(
subpool_addr, devices, model_spec, model_path=model_path
)
diff --git a/xinference/model/rerank/model_spec.json b/xinference/model/rerank/model_spec.json
index 34f62dfba4..4ae7e3b1d4 100644
--- a/xinference/model/rerank/model_spec.json
+++ b/xinference/model/rerank/model_spec.json
@@ -1,20 +1,44 @@
[
{
"model_name": "bge-reranker-large",
+ "type": "normal",
"language": ["en", "zh"],
"model_id": "BAAI/bge-reranker-large",
"model_revision": "27c9168d479987529781de8474dff94d69beca11"
},
{
"model_name": "bge-reranker-base",
+ "type": "normal",
"language": ["en", "zh"],
"model_id": "BAAI/bge-reranker-base",
"model_revision": "465b4b7ddf2be0a020c8ad6e525b9bb1dbb708ae"
},
{
"model_name": "bce-reranker-base_v1",
+ "type": "normal",
"language": ["en", "zh"],
"model_id": "maidalun1020/bce-reranker-base_v1",
"model_revision": "eaa31a577a0574e87a08959bd229ca14ce1b5496"
+ },
+ {
+ "model_name": "bge-reranker-v2-m3",
+ "type": "normal",
+ "language": ["en", "zh", "multilingual"],
+ "model_id": "BAAI/bge-reranker-v2-m3",
+ "model_revision": "12e974610ba9083ed95f3edf08d7e899581f4de4"
+ },
+ {
+ "model_name": "bge-reranker-v2-gemma",
+ "type": "LLM-based",
+ "language": ["en", "zh", "multilingual"],
+ "model_id": "BAAI/bge-reranker-v2-gemma",
+ "model_revision": "1787044f8b6fb740a9de4557c3a12377f84d9e17"
+ },
+ {
+ "model_name": "bge-reranker-v2-minicpm-layerwise",
+ "type": "LLM-based layerwise",
+ "language": ["en", "zh", "multilingual"],
+ "model_id": "BAAI/bge-reranker-v2-minicpm-layerwise",
+ "model_revision": "47b5332b296c4d8cb6ee2c60502cc62a0d708881"
}
]
diff --git a/xinference/model/rerank/model_spec_modelscope.json b/xinference/model/rerank/model_spec_modelscope.json
index 29b39bed5d..68b2b1aea7 100644
--- a/xinference/model/rerank/model_spec_modelscope.json
+++ b/xinference/model/rerank/model_spec_modelscope.json
@@ -1,6 +1,7 @@
[
{
"model_name": "bge-reranker-base",
+ "type": "normal",
"language": ["en", "zh"],
"model_id": "Xorbits/bge-reranker-base",
"model_revision": "v0.0.1",
@@ -8,16 +9,39 @@
},
{
"model_name": "bge-reranker-large",
+ "type": "normal",
"language": ["en", "zh"],
"model_id": "Xorbits/bge-reranker-large",
"model_revision": "v0.0.1",
"model_hub": "modelscope"
},
- {
+ {
"model_name": "bce-reranker-base_v1",
+ "type": "normal",
"language": ["en", "zh"],
"model_id": "maidalun/bce-reranker-base_v1",
"model_revision": "v0.0.1",
"model_hub": "modelscope"
+ },
+ {
+ "model_name": "bge-reranker-v2-m3",
+ "type": "normal",
+ "language": ["en", "zh", "multilingual"],
+ "model_id": "AI-ModelScope/bge-reranker-v2-m3",
+ "model_hub": "modelscope"
+ },
+ {
+ "model_name": "bge-reranker-v2-gemma",
+ "type": "LLM-based",
+ "language": ["en", "zh", "multilingual"],
+ "model_id": "AI-ModelScope/bge-reranker-v2-gemma",
+ "model_hub": "modelscope"
+ },
+ {
+ "model_name": "bge-reranker-v2-minicpm-layerwise",
+ "type": "LLM-based layerwise",
+ "language": ["en", "zh", "multilingual"],
+ "model_id": "zfffff/bge-reranker-v2-minicpm-layerwise",
+ "model_hub": "modelscope"
}
]
diff --git a/xinference/model/rerank/tests/test_rerank.py b/xinference/model/rerank/tests/test_rerank.py
index 94f7e0d9fc..dc06624421 100644
--- a/xinference/model/rerank/tests/test_rerank.py
+++ b/xinference/model/rerank/tests/test_rerank.py
@@ -20,11 +20,12 @@
from ....client import Client
-def test_restful_api(setup):
+@pytest.mark.parametrize("model_name", ["bge-reranker-base", "bge-reranker-v2-m3"])
+def test_restful_api(model_name, setup):
endpoint, _ = setup
client = Client(endpoint)
- model_uid = client.launch_model(model_name="bge-reranker-base", model_type="rerank")
+ model_uid = client.launch_model(model_name=model_name, model_type="rerank")
assert len(client.list_models()) == 1
model = client.get_model(model_uid)
# We want to compute the similarity between the query sentence
From 2aca13d17a60243bbc5a6f6a2bf584696005a80a Mon Sep 17 00:00:00 2001
From: Chengjie Li <109656400+ChengjieLi28@users.noreply.github.com>
Date: Wed, 17 Apr 2024 19:31:42 +0800
Subject: [PATCH 017/298] BLD: Add `FlagEmbedding` in cpu docker (#1318)
---
xinference/deploy/docker/cpu.Dockerfile | 1 +
1 file changed, 1 insertion(+)
diff --git a/xinference/deploy/docker/cpu.Dockerfile b/xinference/deploy/docker/cpu.Dockerfile
index d0076476b0..25f1cffb71 100644
--- a/xinference/deploy/docker/cpu.Dockerfile
+++ b/xinference/deploy/docker/cpu.Dockerfile
@@ -53,6 +53,7 @@ RUN python -m pip install --upgrade -i "$PIP_INDEX" pip && \
einops \
tiktoken \
"sentence-transformers>=2.3.1" \
+ FlagEmbedding \
diffusers \
controlnet_aux \
orjson \
From a5025a6e052a03c08df0092a29851d12e770423f Mon Sep 17 00:00:00 2001
From: yiboyasss <143868051+yiboyasss@users.noreply.github.com>
Date: Thu, 18 Apr 2024 11:44:08 +0800
Subject: [PATCH 018/298] FEAT: UI supports multi lora (#1320)
Co-authored-by: ChengjieLi
---
xinference/types.py | 8 +-
.../scenes/launch_model/components/addPair.js | 132 +++++++++
.../ui/src/scenes/launch_model/modelCard.js | 271 +++++++-----------
3 files changed, 246 insertions(+), 165 deletions(-)
create mode 100644 xinference/web/ui/src/scenes/launch_model/components/addPair.js
diff --git a/xinference/types.py b/xinference/types.py
index b300237f66..bbe6188616 100644
--- a/xinference/types.py
+++ b/xinference/types.py
@@ -484,7 +484,7 @@ def __init__(
def to_dict(self):
return {
- "peft_model": [lora.to_dict() for lora in self.peft_model]
+ "lora_list": [lora.to_dict() for lora in self.peft_model]
if self.peft_model
else None,
"image_lora_load_kwargs": self.image_lora_load_kwargs,
@@ -493,10 +493,10 @@ def to_dict(self):
@classmethod
def from_dict(cls, data: Dict):
- peft_model_data = data.get("peft_model")
+ peft_model_list = data.get("lora_list", None)
peft_model = (
- [LoRA.from_dict(lora_dict) for lora_dict in peft_model_data]
- if peft_model_data
+ [LoRA.from_dict(lora_dict) for lora_dict in peft_model_list]
+ if peft_model_list is not None
else None
)
diff --git a/xinference/web/ui/src/scenes/launch_model/components/addPair.js b/xinference/web/ui/src/scenes/launch_model/components/addPair.js
new file mode 100644
index 0000000000..7f11ad4d20
--- /dev/null
+++ b/xinference/web/ui/src/scenes/launch_model/components/addPair.js
@@ -0,0 +1,132 @@
+import { AddCircle } from '@mui/icons-material'
+import DeleteIcon from '@mui/icons-material/Delete'
+import { Alert, Box, IconButton, Snackbar, TextField } from '@mui/material'
+import React, { useEffect, useState } from 'react'
+
+const AddPair = ({ customData, onGetArr, onJudgeArr }) => {
+ const [openSnackbar, setOpenSnackbar] = useState(false)
+ const [arr, setArr] = useState([])
+ const [arrId, setArrId] = useState(0)
+ const [defaultIndex, setDefaultIndex] = useState(-1)
+ const [isNotUniqueKey, setIsNotUniqueKey] = useState(false)
+
+ useEffect(() => {
+ onGetArr(arr)
+ }, [arr])
+
+ const updateArr = (index, type, newValue) => {
+ setArr(
+ arr.map((pair, subIndex) => {
+ if (subIndex === index) {
+ return { ...pair, [type]: newValue }
+ }
+ return pair
+ })
+ )
+ if (type === customData.key) {
+ setDefaultIndex(-1)
+ setIsNotUniqueKey(false)
+ arr.forEach((pair) => {
+ if (pair[customData.key] === newValue) {
+ setDefaultIndex(index)
+ setIsNotUniqueKey(true)
+ }
+ })
+ }
+ }
+
+ const handleDeleteArr = (index) => {
+ setDefaultIndex(-1)
+ setArr(
+ arr.filter((_, subIndex) => {
+ return index !== subIndex
+ })
+ )
+ onGetArr(arr)
+ }
+
+ return (
+
+
+
+
{customData.title}
+
{
+ setArrId(arrId + 1)
+ let obj = { id: arrId }
+ obj[customData.key] = ''
+ obj[customData.value] = ''
+ onJudgeArr(arr, [customData.key, customData.value])
+ ? setArr([...arr, obj])
+ : setOpenSnackbar(true)
+ }}
+ >
+
+
+
+ {
+ updateArr(index, customData.key, e.target.value)
+ }}
+ style={{ width: '44%' }}
+ />
+ {
+ updateArr(index, customData.value, e.target.value)
+ }}
+ style={{ width: '44%' }}
+ />
+ handleDeleteArr(index)}
+ style={{ marginLeft: '10px' }}
+ >
+
+
+
+ {isNotUniqueKey && defaultIndex === index && (
+
+ {customData.key} must be unique
+
+ )}
+
+ )
+ })}
+
+
+
setOpenSnackbar(false)}
+ message="Please fill in the complete parameters before adding!!"
+ key={'top' + 'center'}
+ />
+ setIsPeftModelConfig(!isPeftModelConfig)}
+ >
+
+ {isPeftModelConfig ?
:
}
+
+
-
- Additional parameters passed to the inference engine
-
- {
- setArrId(arrId + 1)
- judgeCustomParameters()
- ? setCustomParametersArr([
- ...customParametersArr,
- { id: arrId, key: '', value: '' },
- ])
- : setOpenSnackbar(true)
+
-
-
-
- {
- updateCustomParametersArr(
- index,
- 'key',
- e.target.value
- )
- }}
- style={{ width: '44%' }}
- />
- {
- updateCustomParametersArr(
- index,
- 'value',
- e.target.value
- )
- }}
- style={{ width: '44%' }}
- />
-
- handleDeleteCustomParameters(index)
- }
- style={{ marginLeft: '10px' }}
- >
-
-
-
- {isNotUnique && defaultIndex === index && (
- key must be unique
- )}
-
- )
- })}
-
-
+ onGetArr={getLoraListArr}
+ onJudgeArr={judgeArr}
+ />
+
+
+
+
+
) : (
@@ -901,8 +848,10 @@ const ModelCard = ({
(quantization ||
(!modelData.is_builtin && modelFormat !== 'pytorch'))
) ||
- !judgeCustomParameters() ||
- defaultIndex !== -1 ||
+ !judgeArr(customParametersArr, ['key', 'value']) ||
+ !judgeArr(loraListArr, ['lora_name', 'local_path']) ||
+ !judgeArr(imageLoraLoadKwargsArr, ['key', 'value']) ||
+ !judgeArr(imageLoraFuseKwargsArr, ['key', 'value']) ||
requestLimitsAlert ||
GPUIdxAlert)
}
@@ -968,7 +917,7 @@ const ModelCard = ({
anchorOrigin={{ vertical: 'top', horizontal: 'center' }}
open={openSnackbar}
onClose={() => setOpenSnackbar(false)}
- message="Please fill in the added custom parameters completely!"
+ message="Please fill in the complete parameters before adding!!"
key={'top' + 'center'}
/>
From be7cbcfe6a561623724a0fdf09f030bb6bbd48be Mon Sep 17 00:00:00 2001
From: mujin2 <132123199+mujin2@users.noreply.github.com>
Date: Thu, 18 Apr 2024 11:50:28 +0800
Subject: [PATCH 019/298] FEAT: Add_cia4command_modelscope (#1321)
---
.../builtin/llm/c4ai-command-r-v01-4bit.rst | 45 ++++++++++
.../models/builtin/llm/c4ai-command-r-v01.rst | 75 ++++++++++++++++
doc/source/models/builtin/llm/gpt-2.rst | 12 +--
doc/source/models/builtin/llm/index.rst | 28 ++++++
doc/source/models/builtin/llm/seallm_v2.5.rst | 45 ++++++++++
doc/source/models/builtin/llm/seallm_v2.rst | 45 ++++++++++
.../builtin/rerank/bge-reranker-v2-gemma.rst | 18 ++++
.../builtin/rerank/bge-reranker-v2-m3.rst | 18 ++++
.../bge-reranker-v2-minicpm-layerwise.rst | 18 ++++
doc/source/models/builtin/rerank/index.rst | 6 ++
.../model/llm/llm_family_modelscope.json | 89 +++++++++++++++++++
xinference/model/llm/vllm/core.py | 2 +
12 files changed, 395 insertions(+), 6 deletions(-)
create mode 100644 doc/source/models/builtin/llm/c4ai-command-r-v01-4bit.rst
create mode 100644 doc/source/models/builtin/llm/c4ai-command-r-v01.rst
create mode 100644 doc/source/models/builtin/llm/seallm_v2.5.rst
create mode 100644 doc/source/models/builtin/llm/seallm_v2.rst
create mode 100644 doc/source/models/builtin/rerank/bge-reranker-v2-gemma.rst
create mode 100644 doc/source/models/builtin/rerank/bge-reranker-v2-m3.rst
create mode 100644 doc/source/models/builtin/rerank/bge-reranker-v2-minicpm-layerwise.rst
diff --git a/doc/source/models/builtin/llm/c4ai-command-r-v01-4bit.rst b/doc/source/models/builtin/llm/c4ai-command-r-v01-4bit.rst
new file mode 100644
index 0000000000..94f103e024
--- /dev/null
+++ b/doc/source/models/builtin/llm/c4ai-command-r-v01-4bit.rst
@@ -0,0 +1,45 @@
+.. _models_llm_c4ai-command-r-v01-4bit:
+
+========================================
+c4ai-command-r-v01-4bit
+========================================
+
+- **Context Length:** 131072
+- **Model Name:** c4ai-command-r-v01-4bit
+- **Languages:** en, fr, de, es, it, pt, ja, ko, zh, ar
+- **Abilities:** generate
+- **Description:** This model is 4bit quantized version of C4AI Command-R using bitsandbytes.
+
+Specifications
+^^^^^^^^^^^^^^
+
+
+Model Spec 1 (pytorch, 35 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** pytorch
+- **Model Size (in billions):** 35
+- **Quantizations:** none
+- **Model ID:** CohereForAI/c4ai-command-r-v01-4bit
+- **Model Hubs**: `Hugging Face `__, `ModelScope `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name c4ai-command-r-v01-4bit --size-in-billions 35 --model-format pytorch --quantization ${quantization}
+
+
+Model Spec 2 (pytorch, 104 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** pytorch
+- **Model Size (in billions):** 104
+- **Quantizations:** none
+- **Model ID:** CohereForAI/c4ai-command-r-plus-4bit
+- **Model Hubs**: `Hugging Face `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name c4ai-command-r-v01-4bit --size-in-billions 104 --model-format pytorch --quantization ${quantization}
+
diff --git a/doc/source/models/builtin/llm/c4ai-command-r-v01.rst b/doc/source/models/builtin/llm/c4ai-command-r-v01.rst
new file mode 100644
index 0000000000..6855cce193
--- /dev/null
+++ b/doc/source/models/builtin/llm/c4ai-command-r-v01.rst
@@ -0,0 +1,75 @@
+.. _models_llm_c4ai-command-r-v01:
+
+========================================
+c4ai-command-r-v01
+========================================
+
+- **Context Length:** 131072
+- **Model Name:** c4ai-command-r-v01
+- **Languages:** en, fr, de, es, it, pt, ja, ko, zh, ar
+- **Abilities:** generate
+- **Description:** C4AI Command-R is a research release of a 35 billion parameter highly performant generative model.
+
+Specifications
+^^^^^^^^^^^^^^
+
+
+Model Spec 1 (pytorch, 35 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** pytorch
+- **Model Size (in billions):** 35
+- **Quantizations:** none
+- **Model ID:** CohereForAI/c4ai-command-r-v01
+- **Model Hubs**: `Hugging Face `__, `ModelScope `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name c4ai-command-r-v01 --size-in-billions 35 --model-format pytorch --quantization ${quantization}
+
+
+Model Spec 2 (ggufv2, 35 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** ggufv2
+- **Model Size (in billions):** 35
+- **Quantizations:** Q2_K, Q3_K_L, Q3_K_M, Q3_K_S, Q4_0, Q4_K_M, Q4_K_S, Q5_0, Q5_K_M, Q5_K_S, Q6_K, Q8_0
+- **Model ID:** andrewcanis/c4ai-command-r-v01-GGUF
+- **Model Hubs**: `Hugging Face `__, `ModelScope `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name c4ai-command-r-v01 --size-in-billions 35 --model-format ggufv2 --quantization ${quantization}
+
+
+Model Spec 3 (pytorch, 104 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** pytorch
+- **Model Size (in billions):** 104
+- **Quantizations:** none
+- **Model ID:** CohereForAI/c4ai-command-r-plus
+- **Model Hubs**: `Hugging Face `__, `ModelScope `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name c4ai-command-r-v01 --size-in-billions 104 --model-format pytorch --quantization ${quantization}
+
+
+Model Spec 4 (gptq, 104 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** gptq
+- **Model Size (in billions):** 104
+- **Quantizations:** Int4
+- **Model ID:** alpindale/c4ai-command-r-plus-GPTQ
+- **Model Hubs**: `Hugging Face `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name c4ai-command-r-v01 --size-in-billions 104 --model-format gptq --quantization ${quantization}
+
diff --git a/doc/source/models/builtin/llm/gpt-2.rst b/doc/source/models/builtin/llm/gpt-2.rst
index bfc94146ea..56c4663178 100644
--- a/doc/source/models/builtin/llm/gpt-2.rst
+++ b/doc/source/models/builtin/llm/gpt-2.rst
@@ -14,17 +14,17 @@ Specifications
^^^^^^^^^^^^^^
-Model Spec 1 (ggmlv3, 1 Billion)
+Model Spec 1 (pytorch, 1_5 Billion)
++++++++++++++++++++++++++++++++++++++++
-- **Model Format:** ggmlv3
-- **Model Size (in billions):** 1
+- **Model Format:** pytorch
+- **Model Size (in billions):** 1_5
- **Quantizations:** none
-- **Model ID:** marella/gpt-2-ggml
-- **Model Hubs**: `Hugging Face `__
+- **Model ID:** openai-community/gpt2
+- **Model Hubs**: `Hugging Face `__
Execute the following command to launch the model, remember to replace ``${quantization}`` with your
chosen quantization method from the options listed above::
- xinference launch --model-name gpt-2 --size-in-billions 1 --model-format ggmlv3 --quantization ${quantization}
+ xinference launch --model-name gpt-2 --size-in-billions 1_5 --model-format pytorch --quantization ${quantization}
diff --git a/doc/source/models/builtin/llm/index.rst b/doc/source/models/builtin/llm/index.rst
index 8d3835a6bb..a9d9ec031c 100644
--- a/doc/source/models/builtin/llm/index.rst
+++ b/doc/source/models/builtin/llm/index.rst
@@ -51,6 +51,16 @@ The following is a list of built-in LLM in Xinference:
- 4096
- Baichuan-chat is a fine-tuned version of the Baichuan LLM, specializing in chatting.
+ * - :ref:`c4ai-command-r-v01 `
+ - generate
+ - 131072
+ - C4AI Command-R is a research release of a 35 billion parameter highly performant generative model.
+
+ * - :ref:`c4ai-command-r-v01-4bit `
+ - generate
+ - 131072
+ - This model is 4bit quantized version of C4AI Command-R using bitsandbytes.
+
* - :ref:`chatglm `
- chat
- 2048
@@ -306,6 +316,16 @@ The following is a list of built-in LLM in Xinference:
- 32768
- Qwen1.5-MoE is a transformer-based MoE decoder-only language model pretrained on a large amount of data.
+ * - :ref:`seallm_v2 `
+ - generate
+ - 8192
+ - We introduce SeaLLM-7B-v2, the state-of-the-art multilingual LLM for Southeast Asian (SEA) languages
+
+ * - :ref:`seallm_v2.5 `
+ - generate
+ - 8192
+ - We introduce SeaLLM-7B-v2.5, the state-of-the-art multilingual LLM for Southeast Asian (SEA) languages
+
* - :ref:`skywork `
- generate
- 4096
@@ -425,6 +445,10 @@ The following is a list of built-in LLM in Xinference:
baichuan-chat
+ c4ai-command-r-v01
+
+ c4ai-command-r-v01-4bit
+
chatglm
chatglm2
@@ -527,6 +551,10 @@ The following is a list of built-in LLM in Xinference:
qwen1.5-moe-chat
+ seallm_v2
+
+ seallm_v2.5
+
skywork
skywork-math
diff --git a/doc/source/models/builtin/llm/seallm_v2.5.rst b/doc/source/models/builtin/llm/seallm_v2.5.rst
new file mode 100644
index 0000000000..50d055de45
--- /dev/null
+++ b/doc/source/models/builtin/llm/seallm_v2.5.rst
@@ -0,0 +1,45 @@
+.. _models_llm_seallm_v2.5:
+
+========================================
+seallm_v2.5
+========================================
+
+- **Context Length:** 8192
+- **Model Name:** seallm_v2.5
+- **Languages:** en, zh, vi, id, th, ms, km, lo, my, tl
+- **Abilities:** generate
+- **Description:** We introduce SeaLLM-7B-v2.5, the state-of-the-art multilingual LLM for Southeast Asian (SEA) languages
+
+Specifications
+^^^^^^^^^^^^^^
+
+
+Model Spec 1 (pytorch, 7 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** pytorch
+- **Model Size (in billions):** 7
+- **Quantizations:** none
+- **Model ID:** SeaLLMs/SeaLLM-7B-v2.5
+- **Model Hubs**: `Hugging Face `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name seallm_v2.5 --size-in-billions 7 --model-format pytorch --quantization ${quantization}
+
+
+Model Spec 2 (ggufv2, 7 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** ggufv2
+- **Model Size (in billions):** 7
+- **Quantizations:** Q4_K_M, Q8_0
+- **Model ID:** SeaLLMs/SeaLLM-7B-v2.5-GGUF
+- **Model Hubs**: `Hugging Face `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name seallm_v2.5 --size-in-billions 7 --model-format ggufv2 --quantization ${quantization}
+
diff --git a/doc/source/models/builtin/llm/seallm_v2.rst b/doc/source/models/builtin/llm/seallm_v2.rst
new file mode 100644
index 0000000000..5de2a0990c
--- /dev/null
+++ b/doc/source/models/builtin/llm/seallm_v2.rst
@@ -0,0 +1,45 @@
+.. _models_llm_seallm_v2:
+
+========================================
+seallm_v2
+========================================
+
+- **Context Length:** 8192
+- **Model Name:** seallm_v2
+- **Languages:** en, zh, vi, id, th, ms, km, lo, my, tl
+- **Abilities:** generate
+- **Description:** We introduce SeaLLM-7B-v2, the state-of-the-art multilingual LLM for Southeast Asian (SEA) languages
+
+Specifications
+^^^^^^^^^^^^^^
+
+
+Model Spec 1 (pytorch, 7 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** pytorch
+- **Model Size (in billions):** 7
+- **Quantizations:** none
+- **Model ID:** SeaLLMs/SeaLLM-7B-v2
+- **Model Hubs**: `Hugging Face `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name seallm_v2 --size-in-billions 7 --model-format pytorch --quantization ${quantization}
+
+
+Model Spec 2 (ggufv2, 7 Billion)
+++++++++++++++++++++++++++++++++++++++++
+
+- **Model Format:** ggufv2
+- **Model Size (in billions):** 7
+- **Quantizations:** Q4_0, Q8_0
+- **Model ID:** SeaLLMs/SeaLLM-7B-v2-gguf
+- **Model Hubs**: `Hugging Face `__
+
+Execute the following command to launch the model, remember to replace ``${quantization}`` with your
+chosen quantization method from the options listed above::
+
+ xinference launch --model-name seallm_v2 --size-in-billions 7 --model-format ggufv2 --quantization ${quantization}
+
diff --git a/doc/source/models/builtin/rerank/bge-reranker-v2-gemma.rst b/doc/source/models/builtin/rerank/bge-reranker-v2-gemma.rst
new file mode 100644
index 0000000000..a200d1f255
--- /dev/null
+++ b/doc/source/models/builtin/rerank/bge-reranker-v2-gemma.rst
@@ -0,0 +1,18 @@
+.. _models_builtin_bge-reranker-v2-gemma:
+
+=====================
+bge-reranker-v2-gemma
+=====================
+
+- **Model Name:** bge-reranker-v2-gemma
+- **Languages:** en, zh, multilingual
+- **Abilities:** rerank
+
+Specifications
+^^^^^^^^^^^^^^
+
+- **Model ID:** BAAI/bge-reranker-v2-gemma
+
+Execute the following command to launch the model::
+
+ xinference launch --model-name bge-reranker-v2-gemma --model-type rerank
\ No newline at end of file
diff --git a/doc/source/models/builtin/rerank/bge-reranker-v2-m3.rst b/doc/source/models/builtin/rerank/bge-reranker-v2-m3.rst
new file mode 100644
index 0000000000..e64246b210
--- /dev/null
+++ b/doc/source/models/builtin/rerank/bge-reranker-v2-m3.rst
@@ -0,0 +1,18 @@
+.. _models_builtin_bge-reranker-v2-m3:
+
+==================
+bge-reranker-v2-m3
+==================
+
+- **Model Name:** bge-reranker-v2-m3
+- **Languages:** en, zh, multilingual
+- **Abilities:** rerank
+
+Specifications
+^^^^^^^^^^^^^^
+
+- **Model ID:** BAAI/bge-reranker-v2-m3
+
+Execute the following command to launch the model::
+
+ xinference launch --model-name bge-reranker-v2-m3 --model-type rerank
\ No newline at end of file
diff --git a/doc/source/models/builtin/rerank/bge-reranker-v2-minicpm-layerwise.rst b/doc/source/models/builtin/rerank/bge-reranker-v2-minicpm-layerwise.rst
new file mode 100644
index 0000000000..41d8553163
--- /dev/null
+++ b/doc/source/models/builtin/rerank/bge-reranker-v2-minicpm-layerwise.rst
@@ -0,0 +1,18 @@
+.. _models_builtin_bge-reranker-v2-minicpm-layerwise:
+
+=================================
+bge-reranker-v2-minicpm-layerwise
+=================================
+
+- **Model Name:** bge-reranker-v2-minicpm-layerwise
+- **Languages:** en, zh, multilingual
+- **Abilities:** rerank
+
+Specifications
+^^^^^^^^^^^^^^
+
+- **Model ID:** BAAI/bge-reranker-v2-minicpm-layerwise
+
+Execute the following command to launch the model::
+
+ xinference launch --model-name bge-reranker-v2-minicpm-layerwise --model-type rerank
\ No newline at end of file
diff --git a/doc/source/models/builtin/rerank/index.rst b/doc/source/models/builtin/rerank/index.rst
index f62fb76dc4..cd1e8613e7 100644
--- a/doc/source/models/builtin/rerank/index.rst
+++ b/doc/source/models/builtin/rerank/index.rst
@@ -16,4 +16,10 @@ The following is a list of built-in rerank models in Xinference:
bge-reranker-base
bge-reranker-large
+
+ bge-reranker-v2-gemma
+
+ bge-reranker-v2-m3
+
+ bge-reranker-v2-minicpm-layerwise
\ No newline at end of file
diff --git a/xinference/model/llm/llm_family_modelscope.json b/xinference/model/llm/llm_family_modelscope.json
index aa60f0ef14..4ef9dc5220 100644
--- a/xinference/model/llm/llm_family_modelscope.json
+++ b/xinference/model/llm/llm_family_modelscope.json
@@ -3045,5 +3045,94 @@
""
]
}
+ },
+ {
+ "version": 1,
+ "context_length": 131072,
+ "model_name": "c4ai-command-r-v01",
+ "model_lang": [
+ "en",
+ "fr",
+ "de",
+ "es",
+ "it",
+ "pt",
+ "ja",
+ "ko",
+ "zh",
+ "ar"
+ ],
+ "model_ability": [
+ "generate"
+ ],
+ "model_description": "C4AI Command-R is a research release of a 35 billion parameter highly performant generative model.",
+ "model_specs": [
+ {
+ "model_format": "pytorch",
+ "model_size_in_billions": 35,
+ "quantizations": [
+ "none"
+ ],
+ "model_hub": "modelscope",
+ "model_id": "AI-ModelScope/c4ai-command-r-v01",
+ "model_revision": "master"
+ },
+ {
+ "model_format": "ggufv2",
+ "model_size_in_billions": 35,
+ "quantizations": [
+ "Q2_K",
+ "Q4_K_M",
+ "Q5_K_M"
+ ],
+ "model_id": "mirror013/C4AI-Command-R-v01-GGUF",
+ "model_file_name_template": "c4ai-command-r-v01.{quantization}.gguf",
+ "model_hub": "modelscope",
+ "model_revision": "master"
+ },
+ {
+ "model_format": "pytorch",
+ "model_size_in_billions": 104,
+ "quantizations": [
+ "none"
+ ],
+ "model_hub": "modelscope",
+ "model_id": "AI-ModelScope/c4ai-command-r-plus",
+ "model_revision": "master"
+ }
+ ]
+ },
+ {
+ "version": 1,
+ "context_length": 131072,
+ "model_name": "c4ai-command-r-v01-4bit",
+ "model_lang": [
+ "en",
+ "fr",
+ "de",
+ "es",
+ "it",
+ "pt",
+ "ja",
+ "ko",
+ "zh",
+ "ar"
+ ],
+ "model_ability": [
+ "generate"
+ ],
+ "model_description": "This model is 4bit quantized version of C4AI Command-R using bitsandbytes.",
+ "model_specs": [
+ {
+ "model_format": "pytorch",
+ "model_size_in_billions": 35,
+ "quantizations": [
+ "none"
+ ],
+ "model_hub": "modelscope",
+ "model_id": "mirror013/c4ai-command-r-v01-4bit",
+ "model_revision": "master"
+ }
+ ]
}
]
diff --git a/xinference/model/llm/vllm/core.py b/xinference/model/llm/vllm/core.py
index 021a7c2bea..502e7f9c3f 100644
--- a/xinference/model/llm/vllm/core.py
+++ b/xinference/model/llm/vllm/core.py
@@ -126,6 +126,8 @@ class VLLMGenerateConfig(TypedDict, total=False):
if VLLM_INSTALLED and vllm.__version__ >= "0.4.0":
VLLM_SUPPORTED_CHAT_MODELS.append("qwen1.5-moe-chat")
+ VLLM_SUPPORTED_MODELS.append("c4ai-command-r-v01")
+ VLLM_SUPPORTED_MODELS.append("c4ai-command-r-v01-4bit")
class VLLMModel(LLM):
From c534028e52f714e499e035116d39638a5e8936e3 Mon Sep 17 00:00:00 2001
From: Wang Xuejin <92132848+Ago327@users.noreply.github.com>
Date: Thu, 18 Apr 2024 16:23:58 +0800
Subject: [PATCH 020/298] REF: support query for engine feature (#1294)
---
xinference/api/restful_api.py | 23 ++
xinference/core/supervisor.py | 18 ++
xinference/model/llm/__init__.py | 88 ++++++
xinference/model/llm/llm_family.py | 40 +++
xinference/model/llm/tests/test_llm_family.py | 258 +++++++++++++++++-
5 files changed, 426 insertions(+), 1 deletion(-)
diff --git a/xinference/api/restful_api.py b/xinference/api/restful_api.py
index ec58594cc3..bb739c1e41 100644
--- a/xinference/api/restful_api.py
+++ b/xinference/api/restful_api.py
@@ -275,6 +275,16 @@ def serve(self, logging_conf: Optional[dict] = None):
self._router.add_api_route(
"/v1/cluster/auth", self.is_cluster_authenticated, methods=["GET"]
)
+ self._router.add_api_route(
+ "/v1/engines/{model_name}",
+ self.query_engines_by_model_name,
+ methods=["GET"],
+ dependencies=(
+ [Security(self._auth_service, scopes=["models:list"])]
+ if self.is_authenticated()
+ else None
+ ),
+ )
# running instances
self._router.add_api_route(
"/v1/models/instances",
@@ -1418,6 +1428,19 @@ async def stream_results():
self.handle_request_limit_error(e)
raise HTTPException(status_code=500, detail=str(e))
+ async def query_engines_by_model_name(self, model_name: str) -> JSONResponse:
+ try:
+ content = await (
+ await self._get_supervisor_ref()
+ ).query_engines_by_model_name(model_name)
+ return JSONResponse(content=content)
+ except ValueError as re:
+ logger.error(re, exc_info=True)
+ raise HTTPException(status_code=400, detail=str(re))
+ except Exception as e:
+ logger.error(e, exc_info=True)
+ raise HTTPException(status_code=500, detail=str(e))
+
async def register_model(self, model_type: str, request: Request) -> JSONResponse:
body = RegisterModelRequest.parse_obj(await request.json())
model = body.model
diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py
index 115baf1e73..8da7d1947f 100644
--- a/xinference/core/supervisor.py
+++ b/xinference/core/supervisor.py
@@ -591,6 +591,24 @@ def get_model_registration(self, model_type: str, model_name: str) -> Any:
else:
raise ValueError(f"Unsupported model type: {model_type}")
+ @log_async(logger=logger)
+ async def query_engines_by_model_name(self, model_name: str):
+ from copy import deepcopy
+
+ from ..model.llm.llm_family import LLM_ENGINES
+
+ if model_name not in LLM_ENGINES:
+ raise ValueError(f"Model {model_name} not found")
+
+ # filter llm_class
+ engine_params = deepcopy(LLM_ENGINES[model_name])
+ for engine in engine_params:
+ params = engine_params[engine]
+ for param in params:
+ del param["llm_class"]
+
+ return engine_params
+
@log_async(logger=logger)
async def register_model(self, model_type: str, model: str, persist: bool):
if model_type in self._custom_register_type_to_cls:
diff --git a/xinference/model/llm/__init__.py b/xinference/model/llm/__init__.py
index ec25095c3d..c6af541362 100644
--- a/xinference/model/llm/__init__.py
+++ b/xinference/model/llm/__init__.py
@@ -30,8 +30,14 @@
BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES,
BUILTIN_LLM_PROMPT_STYLE,
BUILTIN_MODELSCOPE_LLM_FAMILIES,
+ LLAMA_CLASSES,
LLM_CLASSES,
+ LLM_ENGINES,
PEFT_SUPPORTED_CLASSES,
+ PYTORCH_CLASSES,
+ SGLANG_CLASSES,
+ SUPPORTED_ENGINES,
+ VLLM_CLASSES,
CustomLLMFamilyV1,
GgmlLLMSpecV1,
LLMFamilyV1,
@@ -47,6 +53,50 @@
)
+def generate_engine_config_by_model_family(model_family):
+ model_name = model_family.model_name
+ specs = model_family.model_specs
+ engines = {} # structure for engine query
+ for spec in specs:
+ model_format = spec.model_format
+ model_size_in_billions = spec.model_size_in_billions
+ quantizations = spec.quantizations
+ for quantization in quantizations:
+ # traverse all supported engines to match the name, format, size in billions and quatization of model
+ for engine in SUPPORTED_ENGINES:
+ CLASSES = SUPPORTED_ENGINES[engine]
+ for cls in CLASSES:
+ if cls.match(model_family, spec, quantization):
+ engine_params = engines.get(engine, [])
+ already_exists = False
+ # if the name, format and size in billions of model already exists in the structure, add the new quantization
+ for param in engine_params:
+ if (
+ model_name == param["model_name"]
+ and model_format == param["model_format"]
+ and model_size_in_billions
+ == param["model_size_in_billions"]
+ and quantization not in param["quantizations"]
+ ):
+ param["quantizations"].append(quantization)
+ already_exists = True
+ break
+ # successfully match the params for the first time, add to the structure
+ if not already_exists:
+ engine_params.append(
+ {
+ "model_name": model_name,
+ "model_format": model_format,
+ "model_size_in_billions": model_size_in_billions,
+ "quantizations": [quantization],
+ "llm_class": cls,
+ }
+ )
+ engines[engine] = engine_params
+ break
+ LLM_ENGINES[model_name] = engines
+
+
def _install():
from .ggml.chatglm import ChatglmCppChatModel
from .ggml.llamacpp import LlamaCppChatModel, LlamaCppModel
@@ -76,8 +126,17 @@ def _install():
ChatglmCppChatModel,
]
)
+ LLAMA_CLASSES.extend(
+ [
+ ChatglmCppChatModel,
+ LlamaCppChatModel,
+ LlamaCppModel,
+ ]
+ )
LLM_CLASSES.extend([SGLANGModel, SGLANGChatModel])
+ SGLANG_CLASSES.extend([SGLANGModel, SGLANGChatModel])
LLM_CLASSES.extend([VLLMModel, VLLMChatModel])
+ VLLM_CLASSES.extend([VLLMModel, VLLMChatModel])
LLM_CLASSES.extend(
[
BaichuanPytorchChatModel,
@@ -96,6 +155,24 @@ def _install():
PytorchModel,
]
)
+ PYTORCH_CLASSES.extend(
+ [
+ BaichuanPytorchChatModel,
+ VicunaPytorchChatModel,
+ FalconPytorchChatModel,
+ ChatglmPytorchChatModel,
+ LlamaPytorchModel,
+ LlamaPytorchChatModel,
+ PytorchChatModel,
+ FalconPytorchModel,
+ Internlm2PytorchChatModel,
+ QwenVLChatModel,
+ OmniLMMModel,
+ YiVLChatModel,
+ DeepSeekVLChatModel,
+ PytorchModel,
+ ]
+ )
PEFT_SUPPORTED_CLASSES.extend(
[
BaichuanPytorchChatModel,
@@ -113,6 +190,12 @@ def _install():
]
)
+ # support 4 engines for now
+ SUPPORTED_ENGINES["vLLM"] = VLLM_CLASSES
+ SUPPORTED_ENGINES["SGLang"] = SGLANG_CLASSES
+ SUPPORTED_ENGINES["PyTorch"] = PYTORCH_CLASSES
+ SUPPORTED_ENGINES["llama-cpp-python"] = LLAMA_CLASSES
+
json_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "llm_family.json"
)
@@ -163,6 +246,11 @@ def _install():
if llm_spec.model_name not in LLM_MODEL_DESCRIPTIONS:
LLM_MODEL_DESCRIPTIONS.update(generate_llm_description(llm_spec))
+ # traverse all families and add engine parameters corresponding to the model name
+ for families in [BUILTIN_LLM_FAMILIES, BUILTIN_MODELSCOPE_LLM_FAMILIES]:
+ for family in families:
+ generate_engine_config_by_model_family(family)
+
from ...constants import XINFERENCE_MODEL_DIR
user_defined_llm_dir = os.path.join(XINFERENCE_MODEL_DIR, "llm")
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
index 6f5a01d6e7..ecf2e3071e 100644
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -227,16 +227,25 @@ def parse_raw(
CustomLLMFamilyV1.update_forward_refs()
+LLAMA_CLASSES: List[Type[LLM]] = []
LLM_CLASSES: List[Type[LLM]] = []
PEFT_SUPPORTED_CLASSES: List[Type[LLM]] = []
BUILTIN_LLM_FAMILIES: List["LLMFamilyV1"] = []
BUILTIN_MODELSCOPE_LLM_FAMILIES: List["LLMFamilyV1"] = []
+SGLANG_CLASSES: List[Type[LLM]] = []
+PYTORCH_CLASSES: List[Type[LLM]] = []
+
UD_LLM_FAMILIES: List["LLMFamilyV1"] = []
UD_LLM_FAMILIES_LOCK = Lock()
+VLLM_CLASSES: List[Type[LLM]] = []
+
+LLM_ENGINES: Dict[str, Dict[str, List[Dict[str, Any]]]] = {}
+SUPPORTED_ENGINES: Dict[str, List[Type[LLM]]] = {}
+
LLM_LAUNCH_VERSIONS: Dict[str, List[str]] = {}
@@ -904,6 +913,7 @@ def _apply_format_to_model_id(spec: LLMSpecV1, q: str) -> LLMSpecV1:
def register_llm(llm_family: LLMFamilyV1, persist: bool):
from ..utils import is_valid_model_name
+ from . import generate_engine_config_by_model_family
if not is_valid_model_name(llm_family.model_name):
raise ValueError(f"Invalid model name {llm_family.model_name}.")
@@ -916,6 +926,7 @@ def register_llm(llm_family: LLMFamilyV1, persist: bool):
)
UD_LLM_FAMILIES.append(llm_family)
+ generate_engine_config_by_model_family(llm_family)
if persist:
# We only validate model URL when persist is True.
@@ -941,6 +952,7 @@ def unregister_llm(model_name: str, raise_error: bool = True):
break
if llm_family:
UD_LLM_FAMILIES.remove(llm_family)
+ del LLM_ENGINES[model_name]
persist_path = os.path.join(
XINFERENCE_MODEL_DIR, "llm", f"{llm_family.model_name}.json"
@@ -990,3 +1002,31 @@ def match_llm_cls(
if cls.match(family, llm_spec, quantization):
return cls
return None
+
+
+def check_engine_by_spec_parameters(
+ model_engine: str,
+ model_name: str,
+ model_format: str,
+ model_size_in_billions: Union[str, int],
+ quantization: str,
+) -> Optional[Type[LLM]]:
+ if model_name not in LLM_ENGINES:
+ logger.debug(f"Cannot find model {model_name}.")
+ return None
+ if model_engine not in LLM_ENGINES[model_name]:
+ logger.debug(f"Model {model_name} cannot be run on engine {model_engine}.")
+ return None
+ match_params = LLM_ENGINES[model_name][model_engine]
+ for param in match_params:
+ if (
+ model_name == param["model_name"]
+ and model_format == param["model_format"]
+ and model_size_in_billions == param["model_size_in_billions"]
+ and quantization in param["quantizations"]
+ ):
+ return param["llm_class"]
+ logger.debug(
+ f"Model {model_name} with format {model_format}, size {model_size_in_billions} and quantization {quantization} cannot be run on engine {model_engine}."
+ )
+ return None
diff --git a/xinference/model/llm/tests/test_llm_family.py b/xinference/model/llm/tests/test_llm_family.py
index ec119b43ce..81dc586e38 100644
--- a/xinference/model/llm/tests/test_llm_family.py
+++ b/xinference/model/llm/tests/test_llm_family.py
@@ -14,13 +14,19 @@
import codecs
import json
import os
+import platform
import shutil
import tempfile
from unittest.mock import MagicMock, Mock, patch
import pytest
-from ....constants import XINFERENCE_ENV_MODEL_SRC
+from ....constants import (
+ XINFERENCE_DISABLE_VLLM,
+ XINFERENCE_ENABLE_SGLANG,
+ XINFERENCE_ENV_MODEL_SRC,
+)
+from ....utils import cuda_count
from ...utils import is_locale_chinese_simplified, valid_model_revision
from ..llm_family import (
AWSRegion,
@@ -39,6 +45,8 @@
match_model_size,
parse_uri,
)
+from ..sglang.core import SGLANG_INSTALLED
+from ..vllm.core import VLLM_INSTALLED
def test_deserialize_llm_family_v1():
@@ -1064,3 +1072,251 @@ def test_match_model_size():
assert not match_model_size("1.8", 18)
assert not match_model_size("1.8", 1)
assert match_model_size("001", 1)
+
+
+@pytest.mark.skipif(
+ XINFERENCE_DISABLE_VLLM
+ or platform.system() != "Linux"
+ or cuda_count() <= 0
+ or not VLLM_INSTALLED,
+ reason="Current system does not support vLLM",
+)
+def test_quert_engine_vLLM():
+ from ..llm_family import LLM_ENGINES, check_engine_by_spec_parameters
+
+ model_name = "qwen1.5-chat"
+ assert model_name in LLM_ENGINES
+
+ assert (
+ "vLLM" in LLM_ENGINES[model_name] and len(LLM_ENGINES[model_name]["vLLM"]) == 21
+ )
+
+ assert check_engine_by_spec_parameters(
+ model_engine="vLLM",
+ model_name=model_name,
+ model_format="gptq",
+ model_size_in_billions="1_8",
+ quantization="Int4",
+ )
+ assert (
+ check_engine_by_spec_parameters(
+ model_engine="vLLM",
+ model_name=model_name,
+ model_format="gptq",
+ model_size_in_billions="1_8",
+ quantization="Int8",
+ )
+ is None
+ )
+ assert check_engine_by_spec_parameters(
+ model_engine="vLLM",
+ model_name=model_name,
+ model_format="pytorch",
+ model_size_in_billions="1_8",
+ quantization="none",
+ )
+ assert (
+ check_engine_by_spec_parameters(
+ model_engine="vLLM",
+ model_name=model_name,
+ model_format="pytorch",
+ model_size_in_billions="1_8",
+ quantization="4-bit",
+ )
+ is None
+ )
+ assert (
+ check_engine_by_spec_parameters(
+ model_engine="vLLM",
+ model_name=model_name,
+ model_format="ggmlv3",
+ model_size_in_billions="1_8",
+ quantization="q2_k",
+ )
+ is None
+ )
+
+
+@pytest.mark.skipif(
+ not XINFERENCE_ENABLE_SGLANG
+ or platform.system() != "Linux"
+ or cuda_count() <= 0
+ or not SGLANG_INSTALLED,
+ reason="Current system does not support SGLang",
+)
+def test_quert_engine_SGLang():
+ from ..llm_family import LLM_ENGINES, check_engine_by_spec_parameters
+
+ model_name = "qwen1.5-chat"
+ assert model_name in LLM_ENGINES
+
+ assert (
+ "SGLang" in LLM_ENGINES[model_name]
+ and len(LLM_ENGINES[model_name]["SGLang"]) == 21
+ )
+
+ assert check_engine_by_spec_parameters(
+ model_engine="SGLang",
+ model_name=model_name,
+ model_format="gptq",
+ model_size_in_billions="1_8",
+ quantization="Int4",
+ )
+ assert (
+ check_engine_by_spec_parameters(
+ model_engine="SGLang",
+ model_name=model_name,
+ model_format="gptq",
+ model_size_in_billions="1_8",
+ quantization="Int8",
+ )
+ is None
+ )
+ assert check_engine_by_spec_parameters(
+ model_engine="SGLang",
+ model_name=model_name,
+ model_format="pytorch",
+ model_size_in_billions="1_8",
+ quantization="none",
+ )
+ assert (
+ check_engine_by_spec_parameters(
+ model_engine="SGLang",
+ model_name=model_name,
+ model_format="pytorch",
+ model_size_in_billions="1_8",
+ quantization="4-bit",
+ )
+ is None
+ )
+ assert (
+ check_engine_by_spec_parameters(
+ model_engine="SGLang",
+ model_name=model_name,
+ model_format="ggmlv3",
+ model_size_in_billions="1_8",
+ quantization="q2_k",
+ )
+ is None
+ )
+
+
+def test_query_engine_general():
+ from ..ggml.chatglm import ChatglmCppChatModel
+ from ..ggml.llamacpp import LlamaCppChatModel
+ from ..llm_family import (
+ LLM_ENGINES,
+ check_engine_by_spec_parameters,
+ get_user_defined_llm_families,
+ register_llm,
+ unregister_llm,
+ )
+
+ model_name = "qwen1.5-chat"
+ assert model_name in LLM_ENGINES
+
+ assert (
+ "PyTorch" in LLM_ENGINES[model_name]
+ and len(LLM_ENGINES[model_name]["PyTorch"]) == 28
+ )
+ assert (
+ "llama-cpp-python" in LLM_ENGINES[model_name]
+ and len(LLM_ENGINES[model_name]["llama-cpp-python"]) == 7
+ )
+
+ assert check_engine_by_spec_parameters(
+ model_engine="PyTorch",
+ model_name=model_name,
+ model_format="gptq",
+ model_size_in_billions="1_8",
+ quantization="Int4",
+ )
+ assert check_engine_by_spec_parameters(
+ model_engine="PyTorch",
+ model_name=model_name,
+ model_format="gptq",
+ model_size_in_billions="1_8",
+ quantization="Int8",
+ )
+ assert check_engine_by_spec_parameters(
+ model_engine="PyTorch",
+ model_name=model_name,
+ model_format="pytorch",
+ model_size_in_billions="1_8",
+ quantization="none",
+ )
+ assert check_engine_by_spec_parameters(
+ model_engine="PyTorch",
+ model_name=model_name,
+ model_format="pytorch",
+ model_size_in_billions="1_8",
+ quantization="4-bit",
+ )
+ assert (
+ check_engine_by_spec_parameters(
+ model_engine="llama-cpp-python",
+ model_name=model_name,
+ model_format="ggufv2",
+ model_size_in_billions="1_8",
+ quantization="q2_k",
+ )
+ is LlamaCppChatModel
+ )
+ assert (
+ check_engine_by_spec_parameters(
+ model_engine="llama-cpp-python",
+ model_name=model_name,
+ model_format="ggmlv3",
+ model_size_in_billions="1_8",
+ quantization="q2_k",
+ )
+ is None
+ )
+
+ assert (
+ check_engine_by_spec_parameters(
+ model_engine="llama-cpp-python",
+ model_name="chatglm",
+ model_format="ggmlv3",
+ model_size_in_billions=6,
+ quantization="q4_0",
+ )
+ is ChatglmCppChatModel
+ )
+
+ spec = GgmlLLMSpecV1(
+ model_format="ggmlv3",
+ model_size_in_billions=3,
+ model_id="TheBloke/orca_mini_3B-GGML",
+ quantizations=[""],
+ model_file_name_template="README.md",
+ )
+ family = LLMFamilyV1(
+ version=1,
+ context_length=2048,
+ model_type="LLM",
+ model_name="custom_model",
+ model_lang=["en"],
+ model_ability=["embed", "chat"],
+ model_specs=[spec],
+ prompt_style=None,
+ )
+
+ register_llm(family, False)
+
+ assert family in get_user_defined_llm_families()
+ assert (
+ "custom_model" in LLM_ENGINES
+ and "llama-cpp-python" in LLM_ENGINES["custom_model"]
+ )
+ assert check_engine_by_spec_parameters(
+ model_engine="llama-cpp-python",
+ model_name="custom_model",
+ model_format="ggmlv3",
+ model_size_in_billions=3,
+ quantization="",
+ )
+
+ unregister_llm(family.model_name)
+ assert family not in get_user_defined_llm_families()
+ assert "custom_model" not in LLM_ENGINES
From b8a12f8b4e15b492a9dec87117ee376325843e37 Mon Sep 17 00:00:00 2001
From: "Xuye (Chris) Qin"
Date: Thu, 18 Apr 2024 18:13:37 +0800
Subject: [PATCH 021/298] FEAT: support m3e embedding models (#1298)
---
xinference/model/embedding/model_spec.json | 24 +++++++++++++++++++
.../embedding/model_spec_modelscope.json | 24 +++++++++++++++++++
2 files changed, 48 insertions(+)
diff --git a/xinference/model/embedding/model_spec.json b/xinference/model/embedding/model_spec.json
index 12bd44c983..42d89929dc 100644
--- a/xinference/model/embedding/model_spec.json
+++ b/xinference/model/embedding/model_spec.json
@@ -206,5 +206,29 @@
"language": ["zh", "en"],
"model_id": "maidalun1020/bce-embedding-base_v1",
"model_revision": "236d9024fc1b4046f03848723f934521a66a9323"
+ },
+ {
+ "model_name": "m3e-small",
+ "dimensions": 512,
+ "max_tokens": 512,
+ "language": ["zh", "en"],
+ "model_id": "moka-ai/m3e-small",
+ "model_revision": "44c696631b2a8c200220aaaad5f987f096e986df"
+ },
+ {
+ "model_name": "m3e-base",
+ "dimensions": 768,
+ "max_tokens": 512,
+ "language": ["zh", "en"],
+ "model_id": "moka-ai/m3e-base",
+ "model_revision": "764b537a0e50e5c7d64db883f2d2e051cbe3c64c"
+ },
+ {
+ "model_name": "m3e-large",
+ "dimensions": 1024,
+ "max_tokens": 512,
+ "language": ["zh", "en"],
+ "model_id": "moka-ai/m3e-large",
+ "model_revision": "12900375086c37ba5d83d1e417b21dc7d1d1f388"
}
]
diff --git a/xinference/model/embedding/model_spec_modelscope.json b/xinference/model/embedding/model_spec_modelscope.json
index 901bdb3c98..dfd16f19cd 100644
--- a/xinference/model/embedding/model_spec_modelscope.json
+++ b/xinference/model/embedding/model_spec_modelscope.json
@@ -208,5 +208,29 @@
"language": ["zh", "en"],
"model_id": "maidalun/bce-embedding-base_v1",
"model_hub": "modelscope"
+ },
+ {
+ "model_name": "m3e-small",
+ "dimensions": 512,
+ "max_tokens": 512,
+ "language": ["zh", "en"],
+ "model_id": "AI-ModelScope/m3e-small",
+ "model_hub": "modelscope"
+ },
+ {
+ "model_name": "m3e-base",
+ "dimensions": 768,
+ "max_tokens": 512,
+ "language": ["zh", "en"],
+ "model_id": "AI-ModelScope/m3e-base",
+ "model_hub": "modelscope"
+ },
+ {
+ "model_name": "m3e-large",
+ "dimensions": 1024,
+ "max_tokens": 512,
+ "language": ["zh", "en"],
+ "model_id": "AI-ModelScope/m3e-large",
+ "model_hub": "modelscope"
}
]
From b0b828db8b7ad6ed6643977f54021d400b1e4110 Mon Sep 17 00:00:00 2001
From: Minamiyama
Date: Fri, 19 Apr 2024 11:07:02 +0800
Subject: [PATCH 022/298] FEAT: hotkey to active search (#1287)
---
.../ui/src/components/hotkeyFocusTextField.js | 58 +++++++++++++++++++
.../launch_model/LaunchModelComponent.js | 6 +-
.../src/scenes/launch_model/launchCustom.js | 12 ++--
.../ui/src/scenes/launch_model/launchLLM.js | 13 ++---
4 files changed, 74 insertions(+), 15 deletions(-)
create mode 100644 xinference/web/ui/src/components/hotkeyFocusTextField.js
diff --git a/xinference/web/ui/src/components/hotkeyFocusTextField.js b/xinference/web/ui/src/components/hotkeyFocusTextField.js
new file mode 100644
index 0000000000..4a9c00bfe6
--- /dev/null
+++ b/xinference/web/ui/src/components/hotkeyFocusTextField.js
@@ -0,0 +1,58 @@
+import { TextField } from '@mui/material'
+import InputAdornment from '@mui/material/InputAdornment'
+import Typography from '@mui/material/Typography'
+import React, { useEffect, useRef, useState } from 'react'
+
+const HotkeyFocusTextField = ({
+ label,
+ value,
+ onChange,
+ hotkey = '/',
+ ...props
+}) => {
+ const [isFocused, setIsFocused] = useState(false)
+ const textFieldRef = useRef(null)
+ const handleKeyDown = (event) => {
+ if (
+ event.key === hotkey &&
+ document.activeElement !== textFieldRef.current
+ ) {
+ event.preventDefault()
+ setIsFocused(true)
+ textFieldRef.current?.focus()
+ }
+ }
+
+ useEffect(() => {
+ document.addEventListener('keydown', handleKeyDown)
+
+ return () => {
+ document.removeEventListener('keydown', handleKeyDown)
+ }
+ }, [hotkey])
+
+ return (
+ setIsFocused(false)}
+ onFocus={() => setIsFocused(true)}
+ InputProps={{
+ endAdornment:
+ !isFocused && !value ? (
+
+
+ Type {hotkey} to search
+
+
+ ) : null,
+ }}
+ />
+ )
+}
+
+export default HotkeyFocusTextField
diff --git a/xinference/web/ui/src/scenes/launch_model/LaunchModelComponent.js b/xinference/web/ui/src/scenes/launch_model/LaunchModelComponent.js
index 7603a394d8..082b9f3067 100644
--- a/xinference/web/ui/src/scenes/launch_model/LaunchModelComponent.js
+++ b/xinference/web/ui/src/scenes/launch_model/LaunchModelComponent.js
@@ -1,8 +1,9 @@
-import { Box, FormControl, TextField } from '@mui/material'
+import { Box, FormControl } from '@mui/material'
import React, { useContext, useEffect, useState } from 'react'
import { ApiContext } from '../../components/apiContext'
import fetcher from '../../components/fetcher'
+import HotkeyFocusTextField from '../../components/hotkeyFocusTextField'
import ModelCard from './modelCard'
import style from './styles/launchModelStyle'
@@ -66,13 +67,14 @@ const LaunchModelComponent = ({ modelType }) => {
}}
>
-