diff --git a/Cargo.lock b/Cargo.lock index 41947aa5a..7ef5187f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -470,7 +470,7 @@ dependencies = [ [[package]] name = "async-openai" version = "0.28.1" -source = "git+https://github.com/meilisearch/async-openai?branch=optional-type-function#dd328d4c35ca24c30284c8aff616541ac82eb47a" +source = "git+https://github.com/meilisearch/async-openai?branch=optional-type-function#603f1d17bb4530c45fb9a6e93294ab715a7af869" dependencies = [ "async-openai-macros", "backoff", @@ -495,7 +495,7 @@ dependencies = [ [[package]] name = "async-openai-macros" version = "0.1.0" -source = "git+https://github.com/meilisearch/async-openai?branch=optional-type-function#dd328d4c35ca24c30284c8aff616541ac82eb47a" +source = "git+https://github.com/meilisearch/async-openai?branch=optional-type-function#603f1d17bb4530c45fb9a6e93294ab715a7af869" dependencies = [ "proc-macro2", "quote", diff --git a/crates/meilisearch/src/routes/chat.rs b/crates/meilisearch/src/routes/chat.rs index fe5fdd415..04de94b88 100644 --- a/crates/meilisearch/src/routes/chat.rs +++ b/crates/meilisearch/src/routes/chat.rs @@ -268,30 +268,25 @@ async fn streamed_chat( let _join_handle = Handle::current().spawn(async move { let client = Client::with_config(config.clone()); let mut global_tool_calls = HashMap::::new(); + let mut finish_reason = None; - 'main: loop { + 'main: while finish_reason.map_or(true, |fr| fr == FinishReason::ToolCalls) { let mut response = client.chat().create_stream(chat_completion.clone()).await.unwrap(); while let Some(result) = response.next().await { match result { Ok(resp) => { - let delta = &resp.choices[0].delta; + let choice = &resp.choices[0]; + finish_reason = choice.finish_reason; + #[allow(deprecated)] let ChatCompletionStreamResponseDelta { content, // Using deprecated field but keeping for compatibility function_call: _, ref tool_calls, - role, + role: _, refusal: _, - } = delta; - - if content.as_ref().map_or(true, |s| s.is_empty()) - && tool_calls.is_none() - && global_tool_calls.is_empty() - && role.is_none() - { - break 'main; - } + } = &choice.delta; if content.is_some() { tx.send(Event::Data(sse::Data::new_json(&resp).unwrap())).await.unwrap() @@ -409,6 +404,7 @@ async fn streamed_chat( Err(err) => { // writeln!(lock, "error: {err}").unwrap(); tracing::error!("{err:?}"); + break 'main; } } }