Compare commits

...

37 Commits

Author SHA1 Message Date
Louis Dureuil
4d4ab62e42 Fail upgrade for indexes starting with fail-
- take a long time to upgrade for indexes starting with `long-`
2025-06-25 21:51:55 +02:00
Clément Renault
6b4d69996c Merge pull request #5663 from meilisearch/update-version-v1.15.2
Update version for the next release (v1.15.2) in Cargo.toml
2025-06-12 16:41:47 +00:00
Clément Renault
df4e3c2e43 Fix the version everywhere 2025-06-12 16:57:59 +02:00
Clément Renault
e2b549c5ee Merge pull request #5668 from meilisearch/fix-must-regenerate
Various fixes to embedding regeneration
2025-06-12 14:48:38 +00:00
Clément Renault
8390006ebf Merge pull request #5665 from meilisearch/fix-chat-route
Fix chat route missing base URL and Mistral error handling
2025-06-12 14:11:39 +00:00
Louis Dureuil
7200437246 Comment the cases 2025-06-12 15:55:52 +02:00
Louis Dureuil
68e7bfb37f Don't fail if you cannot render previous version 2025-06-12 15:55:33 +02:00
Louis Dureuil
209c4bfc18 Switch the versions of the documents for rendering :/ 2025-06-12 15:47:47 +02:00
Louis Dureuil
396d76046d Regenerate embeddings more often:
- When `regenerate` was previously `false` and became `true`
- When rendering the old version of the docs failed
2025-06-12 15:41:53 +02:00
Clément Renault
9ae73e3c05 Better support for Mistral errors 2025-06-12 15:18:37 +02:00
Many the fish
933e319364 Merge pull request #5660 from meilisearch/reproduce-5650
Searchable fields aren't indexed when I add and remove them out of filterableAttributes
2025-06-12 14:46:21 +02:00
Clément Renault
596617dd31 Make sure Mistral base url is well defined 2025-06-12 13:45:05 +02:00
ManyTheFish
f3dd6834c6 Update version for the next release (v1.15.2) in Cargo.toml 2025-06-12 10:51:09 +00:00
ManyTheFish
5d191c479e Skip indexing on settings update when possible,
when removing a field from the filterable settings,
this will trigger a reindexing of the negative version of the document,
which removes the document from the searchable as well because the field was considered removed.
2025-06-12 12:37:27 +02:00
ManyTheFish
40776ed4cd add test reproducing #5650 2025-06-12 11:09:31 +02:00
Clément Renault
9275ce1503 Merge pull request #5655 from meilisearch/update-version-v1.15.1
Update version for the next release (v1.15.1) in Cargo.toml
2025-06-11 14:54:01 +00:00
Clément Renault
48d2d3a5cd Fix more tests 2025-06-11 14:53:34 +02:00
Louis Dureuil
7ec0c9aa83 Merge pull request #5556 from meilisearch/chat-route
Chat route
2025-06-11 12:09:30 +00:00
Clément Renault
484fdd9ce2 Fix the insta snapshots 2025-06-11 10:59:14 +02:00
Louis Dureuil
7533a11143 Make sure to send the tool response before the error message 2025-06-11 10:49:21 +02:00
Kerollmops
19d077a4b1 Update version for the next release (v1.15.1) in Cargo.toml 2025-06-11 08:35:24 +00:00
Clément Renault
77cc3678b5 Make sure template errors are reported to the LLM and front-end without panicking 2025-06-11 09:27:14 +02:00
Clément Renault
7f5a0c0013 Merge pull request #5646 from meilisearch/revert-5635-prompt-for-email 2025-06-09 12:03:11 +02:00
Clément Renault
f5c3dad3ed Revert "Prompt for Email" 2025-06-09 10:47:21 +02:00
Clément Renault
62e2a5a324 Merge pull request #5635 from meilisearch/prompt-for-email
Prompt for Email
2025-06-05 19:18:23 +00:00
Clément Renault
90d96ee415 Make clippy happy 2025-06-05 18:21:55 +02:00
Clément Renault
38b317857d Improve the wording again 2025-06-05 18:19:19 +02:00
Tamo
765e76857f store the email file in the global config directory instead of the local data.ms so it's shared between all instances 2025-06-05 16:01:30 +02:00
Clément Renault
204cf423b2 Fix Docker Image 2025-06-05 15:02:09 +02:00
Clément Renault
e575b5af74 Improve the contact email flag to make it friendly to disable prompt 2025-06-05 14:49:08 +02:00
Clément Renault
4fc24cb691 Improve prompting again 2025-06-05 14:45:05 +02:00
Clément Renault
8bc8484e95 Skip the prompt when the email was once provided 2025-06-05 14:43:09 +02:00
Clément Renault
7b49c30d8c Change the email prompting 2025-06-05 12:02:30 +02:00
Clément Renault
239851046d Send requests to Hubspot 2025-06-05 12:00:23 +02:00
Clément Renault
60796dfb14 Disable it by default in our Docker image 2025-06-05 11:02:30 +02:00
Clément Renault
c7cb72a77a Make sure we skip empty prompted emails 2025-06-05 10:59:06 +02:00
Clément Renault
4d819ea636 Initial working version for a prompt for email 2025-06-05 10:54:46 +02:00
27 changed files with 310 additions and 100 deletions

34
Cargo.lock generated
View File

@@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2
[[package]]
name = "benchmarks"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"anyhow",
"bumpalo",
@@ -791,7 +791,7 @@ dependencies = [
[[package]]
name = "build-info"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"anyhow",
"time",
@@ -1787,7 +1787,7 @@ dependencies = [
[[package]]
name = "dump"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"anyhow",
"big_s",
@@ -2009,7 +2009,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "file-store"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"tempfile",
"thiserror 2.0.12",
@@ -2031,7 +2031,7 @@ dependencies = [
[[package]]
name = "filter-parser"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"insta",
"nom",
@@ -2051,7 +2051,7 @@ dependencies = [
[[package]]
name = "flatten-serde-json"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"criterion",
"serde_json",
@@ -2196,7 +2196,7 @@ dependencies = [
[[package]]
name = "fuzzers"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"arbitrary",
"bumpalo",
@@ -2996,7 +2996,7 @@ dependencies = [
[[package]]
name = "index-scheduler"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"anyhow",
"big_s",
@@ -3231,7 +3231,7 @@ dependencies = [
[[package]]
name = "json-depth-checker"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"criterion",
"serde_json",
@@ -3717,7 +3717,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "meili-snap"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"insta",
"md5",
@@ -3726,7 +3726,7 @@ dependencies = [
[[package]]
name = "meilisearch"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"actix-cors",
"actix-http",
@@ -3822,7 +3822,7 @@ dependencies = [
[[package]]
name = "meilisearch-auth"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"base64 0.22.1",
"enum-iterator",
@@ -3841,7 +3841,7 @@ dependencies = [
[[package]]
name = "meilisearch-types"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"actix-web",
"anyhow",
@@ -3875,7 +3875,7 @@ dependencies = [
[[package]]
name = "meilitool"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"anyhow",
"clap",
@@ -3909,7 +3909,7 @@ dependencies = [
[[package]]
name = "milli"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"allocator-api2",
"arroy",
@@ -4414,7 +4414,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "permissive-json-pointer"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"big_s",
"serde_json",
@@ -7195,7 +7195,7 @@ dependencies = [
[[package]]
name = "xtask"
version = "1.15.0"
version = "1.15.2"
dependencies = [
"anyhow",
"build-info",

View File

@@ -22,7 +22,7 @@ members = [
]
[workspace.package]
version = "1.15.0"
version = "1.15.2"
authors = [
"Quentin de Quelen <quentin@dequelen.me>",
"Clément Renault <clement@meilisearch.com>",

View File

@@ -25,6 +25,13 @@ impl IndexScheduler {
i as u32,
indexes.len() as u32,
));
if uid.starts_with("long-") {
tracing::warn!("taking a long time to upgrade for test purposes");
std::thread::sleep(std::time::Duration::from_secs(1200));
}
if uid.starts_with("fail-") {
panic!("failing for test purposes");
}
let index = self.index(uid)?;
let mut index_wtxn = index.write_txn()?;
let regen_stats = milli::update::upgrade::upgrade(

View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} }
[timestamp] [4,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.2"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", }
2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", }
3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", }

View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
----------------------------------------------------------------------
### Status:
enqueued [0,]

View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
----------------------------------------------------------------------
### Status:

View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
----------------------------------------------------------------------
### Status:
@@ -37,7 +37,7 @@ catto [1,]
[timestamp] [0,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.2"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
----------------------------------------------------------------------
@@ -40,7 +40,7 @@ doggo [2,]
[timestamp] [0,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.2"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -43,7 +43,7 @@ doggo [2,3,]
[timestamp] [0,]
----------------------------------------------------------------------
### All Batches:
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.2"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
----------------------------------------------------------------------
### Batch to tasks mapping:
0 [0,]

View File

@@ -394,6 +394,7 @@ UnimplementedExternalFunctionCalling , InvalidRequest , NOT_IMPL
UnimplementedNonStreamingChatCompletions , InvalidRequest , NOT_IMPLEMENTED ;
UnimplementedMultiChoiceChatCompletions , InvalidRequest , NOT_IMPLEMENTED ;
ChatNotFound , InvalidRequest , NOT_FOUND ;
InvalidChatSettingDocumentTemplate , InvalidRequest , BAD_REQUEST ;
InvalidChatCompletionOrgId , InvalidRequest , BAD_REQUEST ;
InvalidChatCompletionProjectId , InvalidRequest , BAD_REQUEST ;
InvalidChatCompletionApiVersion , InvalidRequest , BAD_REQUEST ;

View File

@@ -107,7 +107,7 @@ impl ChatCompletionSettings {
}
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)]
#[serde(rename_all = "camelCase")]
pub enum ChatCompletionSource {
#[default]

View File

@@ -23,7 +23,10 @@ use futures::StreamExt;
use index_scheduler::IndexScheduler;
use meilisearch_auth::AuthController;
use meilisearch_types::error::{Code, ResponseError};
use meilisearch_types::features::{ChatCompletionPrompts as DbChatCompletionPrompts, SystemRole};
use meilisearch_types::features::{
ChatCompletionPrompts as DbChatCompletionPrompts,
ChatCompletionSource as DbChatCompletionSource, SystemRole,
};
use meilisearch_types::keys::actions;
use meilisearch_types::milli::index::ChatConfig;
use meilisearch_types::milli::{all_obkv_to_json, obkv_to_json, TimeBudget};
@@ -34,7 +37,7 @@ use tokio::runtime::Handle;
use tokio::sync::mpsc::error::SendError;
use super::config::Config;
use super::errors::StreamErrorEvent;
use super::errors::{MistralError, OpenAiOutsideError, StreamErrorEvent};
use super::utils::format_documents;
use super::{
ChatsParam, MEILI_APPEND_CONVERSATION_MESSAGE_NAME, MEILI_SEARCH_IN_INDEX_FUNCTION_NAME,
@@ -469,6 +472,7 @@ async fn streamed_chat(
&search_queue,
&auth_token,
&client,
chat_settings.source,
&mut chat_completion,
&tx,
&mut global_tool_calls,
@@ -501,6 +505,7 @@ async fn run_conversation<C: async_openai::config::Config>(
search_queue: &web::Data<SearchQueue>,
auth_token: &str,
client: &Client<C>,
source: DbChatCompletionSource,
chat_completion: &mut CreateChatCompletionRequest,
tx: &SseEventSender,
global_tool_calls: &mut HashMap<u32, Call>,
@@ -595,7 +600,13 @@ async fn run_conversation<C: async_openai::config::Config>(
}
}
Err(error) => {
let error = StreamErrorEvent::from_openai_error(error).await.unwrap();
let result = match source {
DbChatCompletionSource::Mistral => {
StreamErrorEvent::from_openai_error::<MistralError>(error).await
}
_ => StreamErrorEvent::from_openai_error::<OpenAiOutsideError>(error).await,
};
let error = result.unwrap_or_else(StreamErrorEvent::from_reqwest_error);
tx.send_error(&error).await?;
return Ok(ControlFlow::Break(None));
}
@@ -645,8 +656,10 @@ async fn handle_meili_tools(
.await?;
}
let mut error = None;
let result = match serde_json::from_str(&call.function.arguments) {
Ok(SearchInIndexParameters { index_uid, q }) => process_search_request(
Ok(SearchInIndexParameters { index_uid, q }) => match process_search_request(
index_scheduler,
auth_ctrl.clone(),
search_queue,
@@ -655,7 +668,14 @@ async fn handle_meili_tools(
q,
)
.await
.map_err(|e| e.to_string()),
{
Ok(output) => Ok(output),
Err(err) => {
let error_text = format!("the search tool call failed with {err}");
error = Some(err);
Err(error_text)
}
},
Err(err) => Err(err.to_string()),
};
@@ -664,7 +684,6 @@ async fn handle_meili_tools(
if report_sources {
tx.report_sources(resp.clone(), &call.id, &documents).await?;
}
text
}
Err(err) => err,
@@ -680,6 +699,10 @@ async fn handle_meili_tools(
}
chat_completion.messages.push(tool);
if let Some(error) = error {
tx.send_error(&StreamErrorEvent::from_response_error(error)).await?;
}
}
Ok(())

View File

@@ -24,7 +24,8 @@ impl Config {
if let Some(api_key) = chat_settings.api_key.as_ref() {
config = config.with_api_key(api_key);
}
if let Some(base_url) = chat_settings.base_url.as_ref() {
let base_url = chat_settings.base_url.as_deref();
if let Some(base_url) = chat_settings.source.base_url().or(base_url) {
config = config.with_api_base(base_url);
}
Self::OpenAiCompatible(config)

View File

@@ -1,8 +1,42 @@
use async_openai::error::{ApiError, OpenAIError};
use async_openai::reqwest_eventsource::Error as EventSourceError;
use meilisearch_types::error::ResponseError;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
/// The error type which is always `error`.
const ERROR_TYPE: &str = "error";
/// The error struct returned by the Mistral API.
///
/// ```json
/// {
/// "object": "error",
/// "message": "Service tier capacity exceeded for this model.",
/// "type": "invalid_request_error",
/// "param": null,
/// "code": null
/// }
/// ```
#[derive(Debug, Clone, Deserialize)]
pub struct MistralError {
message: String,
r#type: String,
param: Option<String>,
code: Option<String>,
}
impl From<MistralError> for StreamErrorEvent {
fn from(error: MistralError) -> Self {
let MistralError { message, r#type, param, code } = error;
StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: ERROR_TYPE.to_owned(),
error: StreamError { r#type, code, message, param, event_id: None },
}
}
}
#[derive(Debug, Clone, Deserialize)]
pub struct OpenAiOutsideError {
/// Emitted when an error occurs.
@@ -22,6 +56,17 @@ pub struct OpenAiInnerError {
r#type: String,
}
impl From<OpenAiOutsideError> for StreamErrorEvent {
fn from(error: OpenAiOutsideError) -> Self {
let OpenAiOutsideError { error: OpenAiInnerError { code, message, param, r#type } } = error;
StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: ERROR_TYPE.to_string(),
error: StreamError { r#type, code, message, param, event_id: None },
}
}
}
/// An error that occurs during the streaming process.
///
/// It directly comes from the OpenAI API and you can
@@ -53,12 +98,15 @@ pub struct StreamError {
}
impl StreamErrorEvent {
pub async fn from_openai_error(error: OpenAIError) -> Result<Self, reqwest::Error> {
let error_type = "error".to_string();
pub async fn from_openai_error<E>(error: OpenAIError) -> Result<Self, reqwest::Error>
where
E: serde::de::DeserializeOwned,
Self: From<E>,
{
match error {
OpenAIError::Reqwest(e) => Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "internal_reqwest_error".to_string(),
code: Some("internal".to_string()),
@@ -69,7 +117,7 @@ impl StreamErrorEvent {
}),
OpenAIError::ApiError(ApiError { message, r#type, param, code }) => {
Ok(StreamErrorEvent {
r#type: error_type,
r#type: ERROR_TYPE.to_string(),
event_id: Uuid::new_v4().to_string(),
error: StreamError {
r#type: r#type.unwrap_or_else(|| "unknown".to_string()),
@@ -82,7 +130,7 @@ impl StreamErrorEvent {
}
OpenAIError::JSONDeserialize(error) => Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "json_deserialize_error".to_string(),
code: Some("internal".to_string()),
@@ -94,30 +142,16 @@ impl StreamErrorEvent {
OpenAIError::FileSaveError(_) | OpenAIError::FileReadError(_) => unreachable!(),
OpenAIError::StreamError(error) => match error {
EventSourceError::InvalidStatusCode(_status_code, response) => {
let OpenAiOutsideError {
error: OpenAiInnerError { code, message, param, r#type },
} = response.json().await?;
Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
error: StreamError { r#type, code, message, param, event_id: None },
})
let error = response.json::<E>().await?;
Ok(StreamErrorEvent::from(error))
}
EventSourceError::InvalidContentType(_header_value, response) => {
let OpenAiOutsideError {
error: OpenAiInnerError { code, message, param, r#type },
} = response.json().await?;
Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
error: StreamError { r#type, code, message, param, event_id: None },
})
let error = response.json::<E>().await?;
Ok(StreamErrorEvent::from(error))
}
EventSourceError::Utf8(error) => Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "invalid_utf8_error".to_string(),
code: None,
@@ -128,7 +162,7 @@ impl StreamErrorEvent {
}),
EventSourceError::Parser(error) => Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "parser_error".to_string(),
code: None,
@@ -139,7 +173,7 @@ impl StreamErrorEvent {
}),
EventSourceError::Transport(error) => Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "transport_error".to_string(),
code: None,
@@ -150,7 +184,7 @@ impl StreamErrorEvent {
}),
EventSourceError::InvalidLastEventId(message) => Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "invalid_last_event_id".to_string(),
code: None,
@@ -161,7 +195,7 @@ impl StreamErrorEvent {
}),
EventSourceError::StreamEnded => Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "stream_ended".to_string(),
code: None,
@@ -173,7 +207,7 @@ impl StreamErrorEvent {
},
OpenAIError::InvalidArgument(message) => Ok(StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: error_type,
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "invalid_argument".to_string(),
code: None,
@@ -184,4 +218,33 @@ impl StreamErrorEvent {
}),
}
}
pub fn from_response_error(error: ResponseError) -> Self {
let ResponseError { code, message, .. } = error;
StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "response_error".to_string(),
code: Some(code.as_str().to_string()),
message,
param: None,
event_id: None,
},
}
}
pub fn from_reqwest_error(error: reqwest::Error) -> Self {
StreamErrorEvent {
event_id: Uuid::new_v4().to_string(),
r#type: ERROR_TYPE.to_string(),
error: StreamError {
r#type: "reqwest_error".to_string(),
code: None,
message: error.to_string(),
param: None,
event_id: None,
},
}
}
}

View File

@@ -9,7 +9,7 @@ use async_openai::types::{
FunctionCall, FunctionCallStream, Role,
};
use bumpalo::Bump;
use meilisearch_types::error::ResponseError;
use meilisearch_types::error::{Code, ResponseError};
use meilisearch_types::heed::RoTxn;
use meilisearch_types::milli::index::ChatConfig;
use meilisearch_types::milli::prompt::{Prompt, PromptData};
@@ -237,7 +237,15 @@ pub fn format_documents<'doc>(
Some(doc) => doc,
None => unreachable!("Document with internal ID {docid} not found"),
};
let text = prompt.render_document(&external_docid, document, &gfid_map, doc_alloc).unwrap();
let text = match prompt.render_document(&external_docid, document, &gfid_map, doc_alloc) {
Ok(text) => text,
Err(err) => {
return Err(ResponseError::from_msg(
err.to_string(),
Code::InvalidChatSettingDocumentTemplate,
))
}
};
renders.push(text);
}

View File

@@ -2086,3 +2086,76 @@ async fn test_exact_typos_terms() {
)
.await;
}
#[actix_rt::test]
async fn simple_search_changing_unrelated_settings() {
let server = Server::new_shared();
let index = server.unique_index();
let documents = DOCUMENTS.clone();
let (task, _status_code) = index.add_documents(documents, None).await;
index.wait_task(task.uid()).await.succeeded();
index
.search(json!({"q": "Dragon"}), |response, code| {
snapshot!(code, @"200 OK");
snapshot!(json_string!(response["hits"]), @r###"
[
{
"title": "How to Train Your Dragon: The Hidden World",
"id": "166428",
"color": [
"green",
"red"
]
}
]
"###);
})
.await;
let (task, _status_code) =
index.update_settings(json!({ "filterableAttributes": ["title"] })).await;
let r = index.wait_task(task.uid()).await.succeeded();
snapshot!(r["status"], @r###""succeeded""###);
index
.search(json!({"q": "Dragon"}), |response, code| {
snapshot!(code, @"200 OK");
snapshot!(json_string!(response["hits"]), @r###"
[
{
"title": "How to Train Your Dragon: The Hidden World",
"id": "166428",
"color": [
"green",
"red"
]
}
]
"###);
})
.await;
let (task, _status_code) = index.update_settings(json!({ "filterableAttributes": [] })).await;
let r = index.wait_task(task.uid()).await.succeeded();
snapshot!(r["status"], @r###""succeeded""###);
index
.search(json!({"q": "Dragon"}), |response, code| {
snapshot!(code, @"200 OK");
snapshot!(json_string!(response["hits"]), @r###"
[
{
"title": "How to Train Your Dragon: The Hidden World",
"id": "166428",
"color": [
"green",
"red"
]
}
]
"###);
})
.await;
}

View File

@@ -43,7 +43,7 @@ async fn version_too_old() {
std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap();
let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.15.0");
snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.15.2");
}
#[actix_rt::test]
@@ -58,7 +58,7 @@ async fn version_requires_downgrade() {
std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap();
let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
snapshot!(err, @"Database version 1.15.1 is higher than the Meilisearch version 1.15.0. Downgrade is not supported");
snapshot!(err, @"Database version 1.15.3 is higher than the Meilisearch version 1.15.2. Downgrade is not supported");
}
#[actix_rt::test]

View File

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"progress": null,
"details": {
"upgradeFrom": "v1.12.0",
"upgradeTo": "v1.15.0"
"upgradeTo": "v1.15.2"
},
"stats": {
"totalNbTasks": 1,

View File

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"progress": null,
"details": {
"upgradeFrom": "v1.12.0",
"upgradeTo": "v1.15.0"
"upgradeTo": "v1.15.2"
},
"stats": {
"totalNbTasks": 1,

View File

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"progress": null,
"details": {
"upgradeFrom": "v1.12.0",
"upgradeTo": "v1.15.0"
"upgradeTo": "v1.15.2"
},
"stats": {
"totalNbTasks": 1,

View File

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"canceledBy": null,
"details": {
"upgradeFrom": "v1.12.0",
"upgradeTo": "v1.15.0"
"upgradeTo": "v1.15.2"
},
"error": null,
"duration": "[duration]",

View File

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"canceledBy": null,
"details": {
"upgradeFrom": "v1.12.0",
"upgradeTo": "v1.15.0"
"upgradeTo": "v1.15.2"
},
"error": null,
"duration": "[duration]",

View File

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"canceledBy": null,
"details": {
"upgradeFrom": "v1.12.0",
"upgradeTo": "v1.15.0"
"upgradeTo": "v1.15.2"
},
"error": null,
"duration": "[duration]",

View File

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"progress": null,
"details": {
"upgradeFrom": "v1.12.0",
"upgradeTo": "v1.15.0"
"upgradeTo": "v1.15.2"
},
"stats": {
"totalNbTasks": 1,

View File

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"canceledBy": null,
"details": {
"upgradeFrom": "v1.12.0",
"upgradeTo": "v1.15.0"
"upgradeTo": "v1.15.2"
},
"error": null,
"duration": "[duration]",

View File

@@ -29,7 +29,6 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
let max_positions_per_attributes = max_positions_per_attributes
.map_or(MAX_POSITION_PER_ATTRIBUTE, |max| max.min(MAX_POSITION_PER_ATTRIBUTE));
let max_memory = indexer.max_memory_by_thread();
let force_reindexing = settings_diff.reindex_searchable();
// initialize destination values.
let mut documents_ids = RoaringBitmap::new();
@@ -43,6 +42,12 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
true,
);
let force_reindexing = settings_diff.reindex_searchable();
let skip_indexing = !force_reindexing && settings_diff.settings_update_only();
if skip_indexing {
return sorter_into_reader(docid_word_positions_sorter, indexer);
}
// initialize buffers.
let mut del_buffers = Buffers::default();
let mut add_buffers = Buffers::default();

View File

@@ -111,6 +111,8 @@ impl<'extractor> Extractor<'extractor> for EmbeddingExtractor<'_, '_> {
let prompt = chunks.prompt();
let old_vectors = old_vectors.vectors_for_key(embedder_name)?.unwrap();
// case where we have a `_vectors` field in the updated document
if let Some(new_vectors) = new_vectors.as_ref().and_then(|new_vectors| {
new_vectors.vectors_for_key(embedder_name).transpose()
}) {
@@ -130,18 +132,9 @@ impl<'extractor> Extractor<'extractor> for EmbeddingExtractor<'_, '_> {
error: error.to_string(),
})?,
)?;
// regenerate if the new `_vectors` fields is set to.
} else if new_vectors.regenerate {
let new_rendered = prompt.render_document(
update.external_document_id(),
update.current(
&context.rtxn,
context.index,
context.db_fields_ids_map,
)?,
context.new_fields_ids_map,
&context.doc_alloc,
)?;
let old_rendered = prompt.render_document(
update.external_document_id(),
update.merged(
&context.rtxn,
@@ -151,7 +144,31 @@ impl<'extractor> Extractor<'extractor> for EmbeddingExtractor<'_, '_> {
context.new_fields_ids_map,
&context.doc_alloc,
)?;
if new_rendered != old_rendered {
let must_regenerate = if !old_vectors.regenerate {
// we just enabled `regenerate`
true
} else {
let old_rendered = prompt.render_document(
update.external_document_id(),
update.current(
&context.rtxn,
context.index,
context.db_fields_ids_map,
)?,
context.new_fields_ids_map,
&context.doc_alloc,
);
if let Ok(old_rendered) = old_rendered {
// must regenerate if the rendered changed
new_rendered != old_rendered
} else {
// cannot check previous rendered, better regenerate
true
}
};
if must_regenerate {
chunks.set_autogenerated(
update.docid(),
update.external_document_id(),
@@ -160,17 +177,8 @@ impl<'extractor> Extractor<'extractor> for EmbeddingExtractor<'_, '_> {
)?;
}
}
// no `_vectors` field, so only regenerate if the document is already set to in the DB.
} else if old_vectors.regenerate {
let old_rendered = prompt.render_document(
update.external_document_id(),
update.current(
&context.rtxn,
context.index,
context.db_fields_ids_map,
)?,
context.new_fields_ids_map,
&context.doc_alloc,
)?;
let new_rendered = prompt.render_document(
update.external_document_id(),
update.merged(
@@ -181,7 +189,28 @@ impl<'extractor> Extractor<'extractor> for EmbeddingExtractor<'_, '_> {
context.new_fields_ids_map,
&context.doc_alloc,
)?;
if new_rendered != old_rendered {
let must_regenerate = {
let old_rendered = prompt.render_document(
update.external_document_id(),
update.current(
&context.rtxn,
context.index,
context.db_fields_ids_map,
)?,
context.new_fields_ids_map,
&context.doc_alloc,
);
if let Ok(old_rendered) = old_rendered {
// regenerate if the rendered version changed
new_rendered != old_rendered
} else {
// if we cannot render the previous version of the documents, let's regenerate
true
}
};
if must_regenerate {
chunks.set_autogenerated(
update.docid(),
update.external_document_id(),