Revert mimalloc to 0.1.37

.github/workflows/flaky-tests.yml

@@ -1,6 +1,4 @@
 name: Look for flaky tests
-env:
-  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 on:
   workflow_dispatch:
   schedule:

.github/workflows/fuzzer-indexing.yml

@@ -1,6 +1,5 @@
 name: Run the indexing fuzzer
-env:
-  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 on:
   push:
     branches:

.github/workflows/publish-apt-brew-pkg.yml

@@ -15,8 +15,6 @@ jobs:
 
   debian:
     name: Publish debian packagge
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     needs: check-version
     container:

.github/workflows/publish-binaries.yml

@@ -35,8 +35,6 @@ jobs:
   publish-linux:
     name: Publish binary for Linux
     runs-on: ubuntu-latest
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     needs: check-version
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27
@@ -134,8 +132,6 @@ jobs:
     name: Publish binary for aarch64
     runs-on: ubuntu-latest
     needs: check-version
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27
       image: ubuntu:18.04

.github/workflows/test-suite.yml

@@ -21,8 +21,6 @@ jobs:
   test-linux:
     name: Tests on ubuntu-18.04
     runs-on: ubuntu-latest
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
       image: ubuntu:18.04
@@ -79,8 +77,6 @@ jobs:
   test-all-features:
     name: Tests almost all features
     runs-on: ubuntu-latest
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
       image: ubuntu:18.04
@@ -104,8 +100,6 @@ jobs:
 
   test-disabled-tokenization:
     name: Test disabled tokenization
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       image: ubuntu:18.04
@@ -133,8 +127,6 @@ jobs:
   # We run tests in debug also, to make sure that the debug_assertions are hit
   test-debug:
     name: Run tests in debug
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations

Cargo.lock

@@ -494,7 +494,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 
 [[package]]
 name = "benchmarks"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "anyhow",
  "bytes",
@@ -639,7 +639,7 @@ dependencies = [
 
 [[package]]
 name = "build-info"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "anyhow",
  "time",
@@ -1539,7 +1539,7 @@ dependencies = [
 
 [[package]]
 name = "dump"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -1787,7 +1787,7 @@ dependencies = [
 
 [[package]]
 name = "file-store"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "faux",
  "tempfile",
@@ -1810,7 +1810,7 @@ dependencies = [
 
 [[package]]
 name = "filter-parser"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "insta",
  "nom",
@@ -1830,7 +1830,7 @@ dependencies = [
 
 [[package]]
 name = "flatten-serde-json"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -1948,7 +1948,7 @@ dependencies = [
 
 [[package]]
 name = "fuzzers"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "arbitrary",
  "clap",
@@ -2442,7 +2442,7 @@ checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d"
 
 [[package]]
 name = "index-scheduler"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -2638,7 +2638,7 @@ dependencies = [
 
 [[package]]
 name = "json-depth-checker"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -3275,7 +3275,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "meili-snap"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "insta",
  "md5",
@@ -3284,7 +3284,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "actix-cors",
  "actix-http",
@@ -3377,7 +3377,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-auth"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "base64 0.21.7",
  "enum-iterator",
@@ -3396,7 +3396,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-types"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "actix-web",
  "anyhow",
@@ -3426,7 +3426,7 @@ dependencies = [
 
 [[package]]
 name = "meilitool"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -3465,7 +3465,7 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "arroy",
  "big_s",
@@ -3906,7 +3906,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
 [[package]]
 name = "permissive-json-pointer"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "big_s",
  "serde_json",
@@ -6074,7 +6074,7 @@ dependencies = [
 
 [[package]]
 name = "xtask"
-version = "1.8.4"
+version = "1.8.1"
 dependencies = [
  "anyhow",
  "build-info",

Cargo.toml

@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.8.4"
+version = "1.8.1"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

benchmarks/Cargo.toml

@@ -14,7 +14,7 @@ license.workspace = true
 anyhow = "1.0.79"
 csv = "1.3.0"
 milli = { path = "../milli" }
-mimalloc = { version = "0.1.39", default-features = false }
+mimalloc = { version = "0.1.37", default-features = false }
 serde_json = { version = "1.0.111", features = ["preserve_order"] }
 
 [dev-dependencies]

dump/src/reader/v3/settings.rs

@@ -152,7 +152,6 @@ impl Settings<Unchecked> {
 }
 
 #[derive(Debug, Clone, Deserialize)]
-#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[cfg_attr(test, derive(serde::Serialize))]
 #[serde(deny_unknown_fields)]
 #[serde(rename_all = "camelCase")]

dump/src/reader/v4/settings.rs

@@ -182,7 +182,6 @@ impl Settings<Unchecked> {
     }
 }
 
-#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[derive(Debug, Clone, Deserialize)]
 #[cfg_attr(test, derive(serde::Serialize))]
 #[serde(deny_unknown_fields)]

dump/src/reader/v5/tasks.rs

@@ -200,7 +200,6 @@ impl std::ops::Deref for IndexUid {
     }
 }
 
-#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[derive(Debug)]
 #[cfg_attr(test, derive(serde::Serialize))]
 #[cfg_attr(test, serde(rename_all = "camelCase"))]

index-scheduler/src/batch.rs

@@ -914,34 +914,8 @@ impl IndexScheduler {
                         if self.must_stop_processing.get() {
                             return Err(Error::AbortedTask);
                         }
-                        let (id, doc) = ret?;
-                        let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, doc)?;
-
-                        'inject_vectors: {
-                            let embeddings = index.embeddings(&rtxn, id)?;
-
-                            if embeddings.is_empty() {
-                                break 'inject_vectors;
-                            }
-
-                            let vectors = document
-                                .entry("_vectors".to_owned())
-                                .or_insert(serde_json::Value::Object(Default::default()));
-
-                            let serde_json::Value::Object(vectors) = vectors else {
-                                break 'inject_vectors;
-                            };
-
-                            for (embedder_name, embeddings) in embeddings {
-                                vectors.entry(embedder_name).or_insert_with(|| {
-                                    serde_json::json!({
-                                        "embeddings": embeddings,
-                                        "regenerate": true
-                                    })
-                                });
-                            }
-                        }
-
+                        let (_id, doc) = ret?;
+                        let document = milli::obkv_to_json(&all_fields, &fields_ids_map, doc)?;
                         index_dumper.push_document(&document)?;
                     }
 

meilisearch/Cargo.toml

@@ -56,7 +56,7 @@ jsonwebtoken = "9.2.0"
 lazy_static = "1.4.0"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-mimalloc = { version = "0.1.39", default-features = false }
+mimalloc = { version = "0.1.37", default-features = false }
 mime = "0.3.17"
 num_cpus = "1.16.0"
 obkv = "0.2.1"

meilisearch/src/lib.rs

@@ -419,41 +419,7 @@ fn import_dump(
         let file = tempfile::tempfile()?;
         let mut builder = DocumentsBatchBuilder::new(BufWriter::new(file));
         for document in index_reader.documents()? {
-            let mut document = document?;
-
-            'remove_injected_vectors: {
-                let Some(vectors) = document.get_mut("_vectors") else {
-                    break 'remove_injected_vectors;
-                };
-
-                let Some(vectors) = vectors.as_object_mut() else { break 'remove_injected_vectors };
-
-                vectors.retain(|_embedder, embedding_object| {
-                    // don't touch values that aren't objects
-                    let Some(embedding_object) = embedding_object.as_object() else {
-                        return true;
-                    };
-
-                    let mut has_regenerate_true = false;
-                    for (field, value) in embedding_object {
-                        match (field.as_str(), value) {
-                            // detected regenerate : true
-                            // if we don't have any superfluous field, we'll remove the entire entry
-                            ("regenerate", serde_json::Value::Bool(true)) => {
-                                has_regenerate_true = true;
-                            }
-                            // ignore embeddings
-                            ("embeddings", _) => continue,
-                            // any other field: immediately retain the entry
-                            _ => return true,
-                        }
-                    }
-                    // retain the entry unless it has regenerate: true
-                    !has_regenerate_true
-                })
-            }
-
-            builder.append_json_object(&document)?;
+            builder.append_json_object(&document?)?;
         }
 
         // This flush the content of the batch builder.

meilisearch/src/search_queue.rs

@@ -40,9 +40,8 @@ pub struct Permit {
 
 impl Drop for Permit {
     fn drop(&mut self) {
-        let sender = self.sender.clone();
         // if the channel is closed then the whole instance is down
-        std::mem::drop(tokio::spawn(async move { sender.send(()).await }));
+        let _ = futures::executor::block_on(self.sender.send(()));
     }
 }
 

milli/Cargo.toml

@@ -74,10 +74,10 @@ csv = "1.3.0"
 candle-core = { version = "0.4.1" }
 candle-transformers = { version = "0.4.1" }
 candle-nn = { version = "0.4.1" }
-tokenizers = { git = "https://github.com/huggingface/tokenizers.git", tag = "v0.15.2", version = "0.15.2", default-features = false, features = [
+tokenizers = { git = "https://github.com/huggingface/tokenizers.git", tag = "v0.15.2", version = "0.15.2", default_features = false, features = [
     "onig",
 ] }
-hf-hub = { git = "https://github.com/dureuill/hf-hub.git", branch = "rust_tls", default-features = false, features = [
+hf-hub = { git = "https://github.com/dureuill/hf-hub.git", branch = "rust_tls", default_features = false, features = [
     "online",
 ] }
 tiktoken-rs = "0.5.8"
@@ -89,7 +89,7 @@ ureq = { version = "2.9.6", features = ["json"] }
 url = "2.5.0"
 
 [dev-dependencies]
-mimalloc = { version = "0.1.39", default-features = false }
+mimalloc = { version = "0.1.37", default-features = false }
 big_s = "1.0.2"
 insta = "1.34.0"
 maplit = "1.0.2"

milli/src/index.rs

@@ -22,7 +22,7 @@ use crate::heed_codec::{
 };
 use crate::order_by_map::OrderByMap;
 use crate::proximity::ProximityPrecision;
-use crate::vector::{Embedding, EmbeddingConfig};
+use crate::vector::EmbeddingConfig;
 use crate::{
     default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds,
     FacetDistribution, FieldDistribution, FieldId, FieldIdWordCountCodec, GeoPoint, ObkvCodec,
@@ -1516,42 +1516,6 @@ impl Index {
             .unwrap_or_default())
     }
 
-    pub fn embeddings(
-        &self,
-        rtxn: &RoTxn<'_>,
-        docid: DocumentId,
-    ) -> Result<BTreeMap<String, Vec<Embedding>>> {
-        let mut res = BTreeMap::new();
-        for row in self.embedder_category_id.iter(rtxn)? {
-            let (embedder_name, embedder_id) = row?;
-            let embedder_id = (embedder_id as u16) << 8;
-            let mut embeddings = Vec::new();
-            'vectors: for i in 0..=u8::MAX {
-                let reader = arroy::Reader::open(rtxn, embedder_id | (i as u16), self.vector_arroy)
-                    .map(Some)
-                    .or_else(|e| match e {
-                        arroy::Error::MissingMetadata => Ok(None),
-                        e => Err(e),
-                    })
-                    .transpose();
-
-                let Some(reader) = reader else {
-                    break 'vectors;
-                };
-
-                let embedding = reader?.item_vector(rtxn, docid)?;
-                if let Some(embedding) = embedding {
-                    embeddings.push(embedding)
-                } else {
-                    break 'vectors;
-                }
-            }
-
-            res.insert(embedder_name.to_owned(), embeddings);
-        }
-        Ok(res)
-    }
-
     pub(crate) fn put_search_cutoff(&self, wtxn: &mut RwTxn<'_>, cutoff: u64) -> heed::Result<()> {
         self.main.remap_types::<Str, BEU64>().put(wtxn, main_key::SEARCH_CUTOFF, &cutoff)
     }

milli/src/search/new/logger/visual.rs

@@ -22,7 +22,7 @@ pub enum SearchEvents {
     RankingRuleStartIteration { ranking_rule_idx: usize, universe_len: u64 },
     RankingRuleNextBucket { ranking_rule_idx: usize, universe_len: u64, bucket_len: u64 },
     RankingRuleSkipBucket { ranking_rule_idx: usize, bucket_len: u64 },
-    RankingRuleEndIteration { ranking_rule_idx: usize },
+    RankingRuleEndIteration { ranking_rule_idx: usize, universe_len: u64 },
     ExtendResults { new: Vec<u32> },
     ProximityGraph { graph: RankingRuleGraph<ProximityGraph> },
     ProximityPaths { paths: Vec<Vec<Interned<ProximityCondition>>> },
@@ -123,9 +123,12 @@ impl SearchLogger<QueryGraph> for VisualSearchLogger {
         &mut self,
         ranking_rule_idx: usize,
         _ranking_rule: &dyn RankingRule<QueryGraph>,
-        _universe: &RoaringBitmap,
+        universe: &RoaringBitmap,
     ) {
-        self.events.push(SearchEvents::RankingRuleEndIteration { ranking_rule_idx });
+        self.events.push(SearchEvents::RankingRuleEndIteration {
+            ranking_rule_idx,
+            universe_len: universe.len(),
+        });
         self.location.pop();
     }
     fn add_to_results(&mut self, docids: &[u32]) {
@@ -323,7 +326,7 @@ impl<'ctx> DetailedLoggerFinish<'ctx> {
                 assert!(ranking_rule_idx == self.rr_action_counter.len() - 1);
                 self.write_skip_bucket(bucket_len)?;
             }
-            SearchEvents::RankingRuleEndIteration { ranking_rule_idx } => {
+            SearchEvents::RankingRuleEndIteration { ranking_rule_idx, universe_len: _ } => {
                 assert!(ranking_rule_idx == self.rr_action_counter.len() - 1);
                 self.write_end_iteration()?;
             }

milli/src/update/index_documents/extract/mod.rs

@@ -11,7 +11,7 @@ mod extract_word_position_docids;
 
 use std::fs::File;
 use std::io::BufReader;
-use std::sync::{Arc, OnceLock};
+use std::sync::Arc;
 
 use crossbeam_channel::Sender;
 use rayon::prelude::*;
@@ -31,7 +31,7 @@ use self::extract_word_position_docids::extract_word_position_docids;
 use super::helpers::{as_cloneable_grenad, CursorClonableMmap, GrenadParameters};
 use super::{helpers, TypedChunk};
 use crate::update::settings::InnerIndexSettingsDiff;
-use crate::{FieldId, Result, ThreadPoolNoAbort, ThreadPoolNoAbortBuilder};
+use crate::{FieldId, Result, ThreadPoolNoAbortBuilder};
 
 /// Extract data for each databases from obkv documents in parallel.
 /// Send data in grenad file over provided Sender.
@@ -213,18 +213,6 @@ fn run_extraction_task<FE, FS, M>(
     })
 }
 
-fn request_threads() -> &'static ThreadPoolNoAbort {
-    static REQUEST_THREADS: OnceLock<ThreadPoolNoAbort> = OnceLock::new();
-
-    REQUEST_THREADS.get_or_init(|| {
-        ThreadPoolNoAbortBuilder::new()
-            .num_threads(crate::vector::REQUEST_PARALLELISM)
-            .thread_name(|index| format!("embedding-request-{index}"))
-            .build()
-            .unwrap()
-    })
-}
-
 /// Extract chunked data and send it into lmdb_writer_sx sender:
 /// - documents
 fn send_original_documents_data(
@@ -239,6 +227,11 @@ fn send_original_documents_data(
     let documents_chunk_cloned = original_documents_chunk.clone();
     let lmdb_writer_sx_cloned = lmdb_writer_sx.clone();
 
+    let request_threads = ThreadPoolNoAbortBuilder::new()
+        .num_threads(crate::vector::REQUEST_PARALLELISM)
+        .thread_name(|index| format!("embedding-request-{index}"))
+        .build()?;
+
     if settings_diff.reindex_vectors() || !settings_diff.settings_update_only() {
         let settings_diff = settings_diff.clone();
         rayon::spawn(move || {
@@ -256,7 +249,7 @@ fn send_original_documents_data(
                             prompts,
                             indexer,
                             embedder.clone(),
-                            request_threads(),
+                            &request_threads,
                         ) {
                             Ok(results) => Some(results),
                             Err(error) => {

milli/src/update/index_documents/transform.rs

@@ -48,6 +48,7 @@ pub struct Transform<'a, 'i> {
     fields_ids_map: FieldsIdsMap,
 
     indexer_settings: &'a IndexerConfig,
+    pub autogenerate_docids: bool,
     pub index_documents_method: IndexDocumentsMethod,
     available_documents_ids: AvailableDocumentsIds,
 
@@ -101,7 +102,7 @@ impl<'a, 'i> Transform<'a, 'i> {
         index: &'i Index,
         indexer_settings: &'a IndexerConfig,
         index_documents_method: IndexDocumentsMethod,
-        _autogenerate_docids: bool,
+        autogenerate_docids: bool,
     ) -> Result<Self> {
         // We must choose the appropriate merge function for when two or more documents
         // with the same user id must be merged or fully replaced in the same batch.
@@ -135,6 +136,7 @@ impl<'a, 'i> Transform<'a, 'i> {
             index,
             fields_ids_map: index.fields_ids_map(wtxn)?,
             indexer_settings,
+            autogenerate_docids,
             available_documents_ids: AvailableDocumentsIds::from_documents_ids(&documents_ids),
             original_sorter,
             flattened_sorter,

xtask/Cargo.toml

@@ -21,7 +21,7 @@ reqwest = { version = "0.11.23", features = [
     "stream",
     "json",
     "rustls-tls",
-], default-features = false }
+], default_features = false }
 serde = { version = "1.0.195", features = ["derive"] }
 serde_json = "1.0.111"
 sha2 = "0.10.8"