Update milli version to v0.8.0

2025-10-31 16:06:31 +00:00 · 2021-07-28 10:52:47 +02:00
parent dd18319b44
commit e5dabf265a
4 changed files with 58 additions and 21 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -620,6 +620,17 @@ dependencies = [
 "vec_map",
 ]

+[[package]]
+name = "concat-arrays"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1df715824eb382e34b7afb7463b0247bf41538aeba731fba05241ecdb5dc3747"
+dependencies = [
+ "proc-macro2 1.0.27",
+ "quote 1.0.9",
+ "syn 1.0.73",
+]
+
 [[package]]
 name = "const_fn"
 version = "0.4.8"
@@ -1191,7 +1202,7 @@ dependencies = [
 [[package]]
 name = "heed"
 version = "0.12.0"
-source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
+source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
 dependencies = [
 "byteorder",
 "heed-traits",
@@ -1209,12 +1220,12 @@ dependencies = [
 [[package]]
 name = "heed-traits"
 version = "0.7.0"
-source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
+source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"

 [[package]]
 name = "heed-types"
 version = "0.7.2"
-source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
+source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
 dependencies = [
 "bincode",
 "heed-traits",
@@ -1647,7 +1658,7 @@ dependencies = [
 "log",
 "main_error",
 "meilisearch-error",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.3",
 "memmap",
 "milli",
 "mime",
@@ -1699,7 +1710,23 @@ dependencies = [
 "once_cell",
 "slice-group-by",
 "unicode-segmentation",
- "whatlang",
+ "whatlang 0.9.0",
+]
+
+[[package]]
+name = "meilisearch-tokenizer"
+version = "0.2.4"
+source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.4#135d08dce465a756abaf6a1bcad70f315bda99b9"
+dependencies = [
+ "character_converter",
+ "cow-utils",
+ "deunicode",
+ "fst",
+ "jieba-rs",
+ "once_cell",
+ "slice-group-by",
+ "unicode-segmentation",
+ "whatlang 0.12.0",
 ]

 [[package]]
@@ -1729,12 +1756,13 @@ dependencies = [

 [[package]]
 name = "milli"
-version = "0.7.2"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.7.2#007fec21fcb3accad807aa2f2cf66c168afb043f"
+version = "0.8.0"
+source = "git+https://github.com/meilisearch/milli.git?tag=v0.8.0#200e98c211510750e2ead475b8571b533ac6caa1"
 dependencies = [
 "bstr",
 "byteorder",
 "chrono",
+ "concat-arrays",
 "csv",
 "either",
 "flate2",
@@ -1748,7 +1776,7 @@ dependencies = [
 "linked-hash-map",
 "log",
 "logging_timer",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.4",
 "memmap",
 "obkv",
 "once_cell",
@@ -1910,9 +1938,9 @@ dependencies = [

 [[package]]
 name = "obkv"
-version = "0.1.1"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8"
+checksum = "f69e48cd7c8e5bb52a1da1287fdbfd877c32673176583ce664cd63b201aba385"

 [[package]]
 name = "once_cell"
@@ -3532,6 +3560,15 @@ dependencies = [
 "hashbrown 0.7.2",
 ]

+[[package]]
+name = "whatlang"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a346d2eb29c03618693ed24a29d1acd0c3f2cb08ae58b9669d7461e033cf703"
+dependencies = [
+ "hashbrown 0.7.2",
+]
+
 [[package]]
 name = "whoami"
 version = "1.1.2"
--- a/meilisearch-http/Cargo.toml
+++ b/meilisearch-http/Cargo.toml
@@ -42,7 +42,7 @@ fst = "0.4.5"
 futures = "0.3.7"
 futures-util = "0.3.8"
 grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
-heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
 http = "0.2.1"
 indexmap = { version = "1.3.2", features = ["serde-1"] }
 itertools = "0.10.0"
@@ -51,7 +51,7 @@ main_error = "0.1.0"
 meilisearch-error = { path = "../meilisearch-error" }
 meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" }
 memmap = "0.7.0"
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.2" }
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.8.0" }
 mime = "0.3.16"
 num_cpus = "1.13.0"
 once_cell = "1.5.2"
@@ -73,7 +73,7 @@ thiserror = "1.0.24"
 tokio = { version = "1", features = ["full"] }
 uuid = { version =  "0.8.2", features = ["serde"] }
 walkdir = "2.3.2"
-obkv = "0.1.1"
+obkv = "0.2.0"
 pin-project = "1.0.7"
 whoami = { version = "1.1.2", optional = true }
 reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true }
--- a/meilisearch-http/src/index/mod.rs
+++ b/meilisearch-http/src/index/mod.rs
@@ -174,7 +174,7 @@ impl Index {
        txn: &heed::RoTxn,
        attributes_to_retrieve: &Option<Vec<S>>,
        fields_ids_map: &milli::FieldsIdsMap,
-    ) -> Result<Vec<u8>> {
+    ) -> Result<Vec<u16>> {
        let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? {
            Some(ids) => ids.into_iter().collect::<Vec<_>>(),
            None => fields_ids_map.iter().map(|(id, _)| id).collect(),
--- a/meilisearch-http/src/index/search.rs
+++ b/meilisearch-http/src/index/search.rs
@@ -281,9 +281,9 @@ fn compute_formatted_options(
    attr_to_highlight: &HashSet<String>,
    attr_to_crop: &[String],
    query_crop_length: usize,
-    to_retrieve_ids: &BTreeSet<u8>,
+    to_retrieve_ids: &BTreeSet<u16>,
    fields_ids_map: &FieldsIdsMap,
-    displayed_ids: &BTreeSet<u8>,
+    displayed_ids: &BTreeSet<u16>,
 ) -> BTreeMap<FieldId, FormatOptions> {
    let mut formatted_options = BTreeMap::new();

@@ -314,7 +314,7 @@ fn add_highlight_to_formatted_options(
    formatted_options: &mut BTreeMap<FieldId, FormatOptions>,
    attr_to_highlight: &HashSet<String>,
    fields_ids_map: &FieldsIdsMap,
-    displayed_ids: &BTreeSet<u8>,
+    displayed_ids: &BTreeSet<u16>,
 ) {
    for attr in attr_to_highlight {
        let new_format = FormatOptions {
@@ -342,7 +342,7 @@ fn add_crop_to_formatted_options(
    attr_to_crop: &[String],
    crop_length: usize,
    fields_ids_map: &FieldsIdsMap,
-    displayed_ids: &BTreeSet<u8>,
+    displayed_ids: &BTreeSet<u16>,
 ) {
    for attr in attr_to_crop {
        let mut split = attr.rsplitn(2, ':');
@@ -382,7 +382,7 @@ fn add_crop_to_formatted_options(

 fn add_non_formatted_ids_to_formatted_options(
    formatted_options: &mut BTreeMap<FieldId, FormatOptions>,
-    to_retrieve_ids: &BTreeSet<u8>,
+    to_retrieve_ids: &BTreeSet<u16>,
 ) {
    for id in to_retrieve_ids {
        formatted_options.entry(*id).or_insert(FormatOptions {
@@ -395,7 +395,7 @@ fn add_non_formatted_ids_to_formatted_options(
 fn make_document(
    attributes_to_retrieve: &BTreeSet<FieldId>,
    field_ids_map: &FieldsIdsMap,
-    obkv: obkv::KvReader,
+    obkv: obkv::KvReaderU16,
 ) -> Result<Document> {
    let mut document = Document::new();

@@ -418,7 +418,7 @@ fn make_document(

 fn format_fields<A: AsRef<[u8]>>(
    field_ids_map: &FieldsIdsMap,
-    obkv: obkv::KvReader,
+    obkv: obkv::KvReaderU16,
    formatter: &Formatter<A>,
    matching_words: &impl Matcher,
    formatted_options: &BTreeMap<FieldId, FormatOptions>,