Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ Quantized versions are also available for several models above (append `Q` to th
- [**BAAI/bge-reranker-base**](https://huggingface.co/BAAI/bge-reranker-base) - Default
- [**BAAI/bge-reranker-v2-m3**](https://huggingface.co/BAAI/bge-reranker-v2-m3)
- [**jinaai/jina-reranker-v1-turbo-en**](https://huggingface.co/jinaai/jina-reranker-v1-turbo-en)
- [**jinaai/jina-reranker-v2-base-multiligual**](https://huggingface.co/jinaai/jina-reranker-v2-base-multilingual)
- [**jinaai/jina-reranker-v2-base-multilingual**](https://huggingface.co/jinaai/jina-reranker-v2-base-multilingual)

</details>

Expand Down
2 changes: 1 addition & 1 deletion src/bgem3_embedding/init.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ pub struct Bgem3EmbeddingOutput {

/// Struct for "bring your own" BGE-M3 models
///
/// The onnx_file and tokenizer_files expect the files' bytes
/// The `onnx_file` and `tokenizer_files` fields contain the file contents as bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UserDefinedBgem3Model {
pub onnx_file: Vec<u8>,
Expand Down
7 changes: 4 additions & 3 deletions src/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,8 @@ pub struct TokenizerFiles {
pub tokenizer_config_file: Vec<u8>,
}

/// The procedure for loading tokenizer files from the hugging face hub is separated
/// from the main load_tokenizer function (which is expecting bytes, from any source).
/// Loading tokenizer files from the Hugging Face Hub is separate from the main
/// `load_tokenizer` function, which expects bytes from any source.
#[cfg(feature = "hf-hub")]
pub fn load_tokenizer_hf_hub(model_repo: ApiRepo, max_length: usize) -> Result<Tokenizer> {
let tokenizer_files: TokenizerFiles = TokenizerFiles {
Expand Down Expand Up @@ -159,7 +159,8 @@ pub fn load_tokenizer(tokenizer_files: TokenizerFiles, max_length: usize) -> Res
)
})?;

//For BGEBaseSmall, the model_max_length value is set to 1000000000000000019884624838656. Which fits in a f64
// For BGEBaseSmall, model_max_length is set to 1000000000000000019884624838656.
// This value fits in an f64.
let model_max_length = tokenizer_config["model_max_length"]
.as_f64()
.ok_or_else(|| {
Expand Down
2 changes: 1 addition & 1 deletion src/image_embedding/init.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ impl From<ImageInitOptions> for ImageInitOptionsUserDefined {

/// Struct for "bring your own" embedding models
///
/// The onnx_file and preprocessor_files are expecting the files' bytes
/// The `onnx_file` and `preprocessor_file` fields contain the file contents as bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct UserDefinedImageEmbeddingModel {
Expand Down
4 changes: 2 additions & 2 deletions src/image_embedding/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ impl Transform for CenterCrop {
Array3::zeros((3usize, crop_height as usize, crop_width as usize));
let offset_x = (crop_width - origin_width) / 2;
let offset_y = (crop_height - origin_height) / 2;
// whc -> chw
// HWC -> CHW
for (x, y, pixel) in image.to_rgb8().enumerate_pixels() {
pixels_array[[0, (y + offset_y) as usize, (x + offset_x) as usize]] =
pixel[0] as f32;
Expand All @@ -166,7 +166,7 @@ impl Transform for PILToNDarray {
TransformData::Image(image) => {
let image = image.to_rgb8();
let (width, height) = image.dimensions();
// whc -> chw
// HWC -> CHW
let mut pixels_array = Array3::zeros((3usize, height as usize, width as usize));
for (x, y, pixel) in image.enumerate_pixels() {
pixels_array[[0, y as usize, x as usize]] = pixel[0] as f32;
Expand Down
6 changes: 3 additions & 3 deletions src/models/image_embedding.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ pub fn models_list() -> Vec<ModelInfo<ImageEmbeddingModel>> {
ModelInfo {
model: ImageEmbeddingModel::UnicomVitB16,
dim: 768,
description: String::from("Unicom Unicom-ViT-B-16 from open-metric-learning"),
description: String::from("Unicom-ViT-B-16 from Open Metric Learning"),
model_code: String::from("Qdrant/Unicom-ViT-B-16"),
model_file: String::from("model.onnx"),
additional_files: Vec::new(),
Expand All @@ -49,7 +49,7 @@ pub fn models_list() -> Vec<ModelInfo<ImageEmbeddingModel>> {
ModelInfo {
model: ImageEmbeddingModel::UnicomVitB32,
dim: 512,
description: String::from("Unicom Unicom-ViT-B-32 from open-metric-learning"),
description: String::from("Unicom-ViT-B-32 from Open Metric Learning"),
model_code: String::from("Qdrant/Unicom-ViT-B-32"),
model_file: String::from("model.onnx"),
additional_files: Vec::new(),
Expand All @@ -58,7 +58,7 @@ pub fn models_list() -> Vec<ModelInfo<ImageEmbeddingModel>> {
ModelInfo {
model: ImageEmbeddingModel::NomicEmbedVisionV15,
dim: 768,
description: String::from("Nomic NomicEmbedVisionV15"),
description: String::from("Nomic Embed Vision v1.5"),
model_code: String::from("nomic-ai/nomic-embed-vision-v1.5"),
model_file: String::from("onnx/model.onnx"),
additional_files: Vec::new(),
Expand Down
18 changes: 12 additions & 6 deletions src/models/reranking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,13 @@ pub enum RerankerModel {
/// jinaai/jina-reranker-v1-turbo-en
JINARerankerV1TurboEn,
/// jinaai/jina-reranker-v2-base-multilingual
JINARerankerV2BaseMultiligual,
JINARerankerV2BaseMultilingual,
}

impl RerankerModel {
#[allow(non_upper_case_globals)]
#[deprecated(note = "use `JINARerankerV2BaseMultilingual` instead")]
pub const JINARerankerV2BaseMultiligual: Self = Self::JINARerankerV2BaseMultilingual;
}

pub fn reranker_model_list() -> Vec<RerankerModelInfo> {
Expand All @@ -26,7 +32,7 @@ pub fn reranker_model_list() -> Vec<RerankerModelInfo> {
},
RerankerModelInfo {
model: RerankerModel::BGERerankerV2M3,
description: String::from("reranker model for multilingual"),
description: String::from("Multilingual reranker model"),
model_code: String::from("rozgo/bge-reranker-v2-m3"),
model_file: String::from("model.onnx"),
additional_files: vec![String::from("model.onnx.data")],
Expand All @@ -39,8 +45,8 @@ pub fn reranker_model_list() -> Vec<RerankerModelInfo> {
additional_files: vec![],
},
RerankerModelInfo {
model: RerankerModel::JINARerankerV2BaseMultiligual,
description: String::from("reranker model for multilingual"),
model: RerankerModel::JINARerankerV2BaseMultilingual,
description: String::from("Multilingual reranker model"),
model_code: String::from("jinaai/jina-reranker-v2-base-multilingual"),
model_file: String::from("onnx/model.onnx"),
additional_files: vec![],
Expand Down Expand Up @@ -86,14 +92,14 @@ pub(crate) fn all_variants() -> Vec<RerankerModel> {
RerankerModel::BGERerankerBase => (),
RerankerModel::BGERerankerV2M3 => (),
RerankerModel::JINARerankerV1TurboEn => (),
RerankerModel::JINARerankerV2BaseMultiligual => (),
RerankerModel::JINARerankerV2BaseMultilingual => (),
}
}
vec![
RerankerModel::BGERerankerBase,
RerankerModel::BGERerankerV2M3,
RerankerModel::JINARerankerV1TurboEn,
RerankerModel::JINARerankerV2BaseMultiligual,
RerankerModel::JINARerankerV2BaseMultilingual,
]
}

Expand Down
2 changes: 1 addition & 1 deletion src/models/sparse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ pub fn models_list() -> Vec<ModelInfo<SparseModel>> {
model: SparseModel::BGEM3,
dim: 0,
description: String::from(
"BGE-M3 sparse embedding model with 8192 context, supports 100+ languages",
"BGE-M3 sparse embedding model with an 8192-token context, supporting 100+ languages",
),
model_code: String::from("BAAI/bge-m3"),
model_file: String::from("onnx/model.onnx"),
Expand Down
8 changes: 4 additions & 4 deletions src/reranking/init.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ impl HasMaxLength for RerankerModel {
/// Options for initializing the reranking models
pub type RerankInitOptions = InitOptionsWithLength<RerankerModel>;

/// Options for initializing UserDefinedRerankerModel
/// Options for initializing a user-defined reranking model.
///
/// Model files are held by the UserDefinedRerankerModel struct
/// Model files are held by the [`UserDefinedRerankingModel`] struct.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct RerankInitOptionsUserDefined {
Expand Down Expand Up @@ -144,7 +144,7 @@ impl From<PathBuf> for OnnxSource {

/// Struct for "bring your own" reranking models
///
/// The onnx_file and tokenizer_files are expecting the files' bytes
/// The `onnx_file` and `tokenizer_files` fields contain the file contents as bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct UserDefinedRerankingModel {
Expand Down Expand Up @@ -174,7 +174,7 @@ mod tests {
use super::*;

#[test]
fn userdefined_builders_set_fields() {
fn user_defined_builders_set_fields() {
let o = RerankInitOptionsUserDefined::new()
.with_max_length(128)
.with_intra_threads(2)
Expand Down
2 changes: 1 addition & 1 deletion src/sparse_text_embedding/init.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ pub type SparseInitOptions = InitOptionsWithLength<SparseModel>;

/// Struct for "bring your own" sparse embedding models
///
/// The onnx_file and tokenizer_files are expecting the files' bytes
/// The `onnx_file` and `tokenizer_files` fields contain the file contents as bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct UserDefinedSparseModel {
Expand Down
2 changes: 1 addition & 1 deletion src/text_embedding/impl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ impl TextEmbedding {
})?;
}

// prioritise loading pooling config if available, if not (thanks qdrant!), look for it in hardcoded
// Prioritize loading the pooling config if available; otherwise, look for it in the hardcoded defaults.
let post_processing = TextEmbedding::get_default_pooling_method(&model_name);

let session = init_session_builder(execution_providers, intra_threads, session_config)?
Expand Down
8 changes: 4 additions & 4 deletions src/text_embedding/init.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ impl HasMaxLength for EmbeddingModel {
/// Options for initializing the TextEmbedding model
pub type TextInitOptions = InitOptionsWithLength<EmbeddingModel>;

/// Options for initializing UserDefinedEmbeddingModel
/// Options for initializing a user-defined embedding model.
///
/// Model files are held by the UserDefinedEmbeddingModel struct
#[derive(Debug, Clone)]
Expand Down Expand Up @@ -150,7 +150,7 @@ impl From<TextInitOptions> for InitOptionsUserDefined {

/// Struct for "bring your own" embedding models
///
/// The onnx_file and tokenizer_files are expecting the files' bytes
/// The `onnx_file` and `tokenizer_files` fields contain the file contents as bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UserDefinedEmbeddingModel {
pub onnx_file: Vec<u8>,
Expand All @@ -163,8 +163,8 @@ pub struct UserDefinedEmbeddingModel {

/// Struct for adding external initializers to "bring your own" embedding models
///
/// The buffer is expecting the data of the external initializer and the file_name
/// must match the one referenced by the model.
/// The buffer contains the external initializer's data, and `file_name` must
/// match the name referenced by the model.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ExternalInitializerFile {
pub file_name: String,
Expand Down
6 changes: 3 additions & 3 deletions tests/text-embeddings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ fn verify_embeddings(model: &EmbeddingModel, embeddings: &[Embedding]) -> Result
EmbeddingModel::AllMiniLML12V2Q => [-0.07808663, 0.27919534, -0.0770612, -0.75660324],
EmbeddingModel::AllMiniLML6V2 => [0.59605527, 0.36542925, -0.16450031, -0.40903988],
EmbeddingModel::AllMiniLML6V2Q => [0.5677276, 0.40180072, -0.15454668, -0.4672576],
EmbeddingModel::AllMpnetBaseV2=> [-0.21253541, -0.050802127, 0.14072442, -0.2908188],
EmbeddingModel::AllMpnetBaseV2 => [-0.21253541, -0.050802127, 0.14072442, -0.2908188],
EmbeddingModel::BGEBaseENV15 => [-0.51290065, -0.4844747, -0.53036124, -0.5337459],
EmbeddingModel::BGEBaseENV15Q => [-0.5130697, -0.48461288, -0.53067875, -0.5337806],
EmbeddingModel::BGELargeENV15 => [-0.19347441, -0.28394595, -0.1549195, -0.22201893],
Expand Down Expand Up @@ -547,7 +547,7 @@ fn test_bgesmallen1point5_match_python_counterpart() {
// Tokenize with python SentenceTransformer("BAAI/bge-small-en-v1.5") default tokenizer
// with (text, padding="max_length",max_length=384,truncation=True, return_tensors="np").
// Normalized and pooled with SentenceTransformer("BAAI/bge-small-en-v1.5") default pooling settings.
// we only take a 10 items to keep the test file polite
// We only take 10 items to keep the test file small.
let baseline: Vec<f32> = vec![
4.208_193_7e-2,
-2.748_133_2e-2,
Expand Down Expand Up @@ -586,7 +586,7 @@ fn test_allminilml6v2_match_python_counterpart() {
// Tokenizer with python SentenceTransformer("all-mini-lm-l6-v2") default tokenizer
// with (text, padding="max_length",max_length=384,truncation=True, return_tensors="np").
// Normalized and pooled with SentenceTransformer("all-mini-lm-l6-v2") default pooling settings.
// we only take a 10 items to keep the test file polite
// We only take 10 items to keep the test file small.
let baseline: Vec<f32> = vec![
3.510_517_6e-2,
1.046_043e-2,
Expand Down
Loading