Merge pull request #1191 from meilisearch/release-v0.18.1

Release v0.18.1

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/check-updated-changelog.yml

@@ -1,16 +0,0 @@
-name: Check if the CHANGELOG.md has been updated
-
-on: [pull_request]
-
-jobs:
-  check:
-    name: Test on ${{ matrix.os }}
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'ignore-changelog') }}
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Checking the CHANGELOG.md has been updated in this PR
-      run: |
-        set -e
-        git fetch origin ${{ github.base_ref }}
-        git diff --name-only origin/${{ github.base_ref }} | grep -q CHANGELOG.md

.github/workflows/coverage.yml

@@ -0,0 +1,34 @@
+---
+on:
+  pull_request:
+    types: [review_requested, ready_for_review]
+
+name: Execute code coverage
+
+jobs:
+  nightly-coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clean
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all-features --no-fail-fast
+        env:
+          CARGO_INCREMENTAL: "0"
+          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=unwind -Zpanic_abort_tests"
+      - uses: actions-rs/grcov@v0.1
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ${{ steps.coverage.outputs.report }}
+          yml: ./codecov.yml
+          fail_ci_if_error: true

.github/workflows/test.yml

@@ -30,11 +30,6 @@ jobs:
       with:
         command: test
         args: --locked --release
-    - name: Run cargo test dump
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: dump --locked --release -- --ignored --test-threads 1
     - name: Run cargo clippy
       uses: actions-rs/cargo@v1
       with:

CHANGELOG.md

@@ -1,6 +1,34 @@
+## v0.18.1
+
+  - Fix unexpected CORS error (#1185)
+
+## v0.18.0
+
+  - Integration with the new tokenizer (#1091)
+  - Fix setting consistency bug (#1128)
+  - Fix attributes to retrieve bug (#1131)
+  - Increase default payload size (#1147)
+  - Improvements to code quality (#1167, #1165, #1126, #1151)
+
+## v0.17.0
+  - Fix corrupted data during placeholder search (#1089)
+  - Remove maintenance error from http (#1082)
+  - Disable frontend in production (#1097)
+  - Update nbHits count with filtered documents (#849)
+  - Remove update changelog ci check (#1090)
+  - Add deploy on Platform.sh option to README (#1087)
+  - Change movie gifs in README (#1077)
+  - Remove some clippy warnings (#1100)
+  - Improve script `download-latest.sh` (#1054)
+  - Bump dependencies version (#1056, #1057, #1059)
+
+## v0.16.0
+
+  - Automatically create index on document push if index doesn't exist (#914)
+  - Sort displayedAttributes and facetDistribution (#946)
+
 ## v0.15.0
 
-  - Dumps (#887)
   - Update actix-web dependency to 3.0.0 (#963)
   - Consider an empty query to be a placeholder search (#916)
 
@@ -10,6 +38,7 @@
 
 ## v0.14.0
 
+  - Sort displayedAttributes (#943)
   - Fix facet distribution case (#797)
   - Snapshotting (#839)
   - Fix bucket-sort unwrap bug (#915)

Cargo.toml

@@ -3,7 +3,6 @@ members = [
     "meilisearch-core",
     "meilisearch-http",
     "meilisearch-schema",
-    "meilisearch-tokenizer",
     "meilisearch-types",
 ]
 

README.md

@@ -6,6 +6,7 @@
 
 <h4 align="center">
   <a href="https://www.meilisearch.com">Website</a> |
+  <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
   <a href="https://blog.meilisearch.com">Blog</a> |
   <a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> |
   <a href="https://twitter.com/meilisearch">Twitter</a> |
@@ -28,7 +29,7 @@
 For more information about features go to [our documentation](https://docs.meilisearch.com/).
 
 <p align="center">
-  <img src="assets/movies-web-demo.gif" alt="Web interface gif" />
+  <img src="assets/trumen_quick_loop.gif" alt="Web interface gif" />
 </p>
 
 ## ✨ Features
@@ -60,10 +61,20 @@ meilisearch
 docker run -p 7700:7700 -v $(pwd)/data.ms:/data.ms getmeili/meilisearch
 ```
 
+#### Try MeiliSearch in our Sandbox
+
+Create a MeiliSearch instance in [MeiliSearch Sandbox](https://sandbox.meilisearch.com/). This instance is free, and will be active for 72 hours.
+
 #### Run on Digital Ocean
 
 [![DigitalOcean Marketplace](assets/do-btn-blue.svg)](https://marketplace.digitalocean.com/apps/meilisearch?action=deploy&refcode=7c67bd97e101)
 
+#### Deploy on Platform.sh
+
+<a href="https://console.platform.sh/projects/create-project?template=https://raw.githubusercontent.com/platformsh/template-builder/master/templates/meilisearch/.platform.template.yaml&utm_content=meilisearch&utm_source=github&utm_medium=button&utm_campaign=deploy_on_platform">
+    <img src="https://platform.sh/images/deploy/lg-blue.svg" alt="Deploy on Platform.sh" width="180px" />
+</a>
+
 #### APT (Debian & Ubuntu)
 
 ```bash

datasets/movies/README.md

@@ -1 +1 @@
-_datas in movies.csv are from https://www.themoviedb.org/_
+_datas in movies.json are from https://www.themoviedb.org/_

datasets/movies/settings.json

@@ -1,10 +0,0 @@
-{
-    "searchableAttributes": ["title", "overview"],
-    "displayedAttributes": [
-        "id",
-        "title",
-        "overview",
-        "release_date",
-        "poster"
-    ]
-}

download-latest.sh

@@ -71,7 +71,7 @@ semverLT() {
 # Returns the tag of the latest stable release (in terms of semver and not of release date)
 get_latest() {
     temp_file='temp_file' # temp_file needed because the grep would start before the download is over
-    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file"
+    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file" || return 1
     releases=$(cat "$temp_file" | \
         grep -E "tag_name|draft|prerelease" \
         | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
@@ -168,16 +168,17 @@ failure_usage() {
 
 # MAIN
 latest="$(get_latest)"
-get_os
-if [ "$?" -eq 1 ]; then
+
+if ! get_os; then
     failure_usage
     exit 1
 fi
-get_archi
-if [ "$?" -eq 1 ]; then
+
+if ! get_archi; then
     failure_usage
     exit 1
 fi
+
 echo "Downloading MeiliSearch binary $latest for $os, architecture $archi..."
 release_file="meilisearch-$os-$archi"
 link="https://github.com/meilisearch/MeiliSearch/releases/download/$latest/$release_file"

meilisearch-core/Cargo.toml

@@ -1,57 +1,53 @@
 [package]
 name = "meilisearch-core"
-version = "0.15.0"
+version = "0.18.1"
 license = "MIT"
 authors = ["Kerollmops <clement@meilisearch.com>"]
 edition = "2018"
 
 [dependencies]
-arc-swap = "0.4.5"
-bincode = "1.2.1"
+arc-swap = "1.2.0"
+bincode = "1.3.1"
 byteorder = "1.3.4"
-chrono = { version = "0.4.11", features = ["serde"] }
-compact_arena = "0.4.0"
+chrono = { version = "0.4.19", features = ["serde"] }
+compact_arena = "0.4.1"
 cow-utils = "0.1.2"
-crossbeam-channel = "0.4.2"
-deunicode = "1.1.0"
-either = "1.5.3"
-env_logger = "0.7.1"
-fst = "0.4.4"
-hashbrown = { version = "0.7.1", features = ["serde"] }
-heed = "0.8.0"
-indexmap = { version = "1.3.2", features = ["serde-1"] }
-intervaltree = "0.2.5"
-itertools = "0.9.0"
+crossbeam-channel = "0.5.0"
+deunicode = "1.1.1"
+either = "1.6.1"
+env_logger = "0.8.2"
+fst = "0.4.5"
+hashbrown = { version = "0.9.1", features = ["serde"] }
+heed = "0.10.6"
+indexmap = { version = "1.6.1", features = ["serde-1"] }
+intervaltree = "0.2.6"
+itertools = "0.10.0"
 levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
-log = "0.4.8"
-meilisearch-error = { path = "../meilisearch-error", version = "0.15.0" }
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.15.0" }
-meilisearch-tokenizer = { path = "../meilisearch-tokenizer", version = "0.15.0" }
-meilisearch-types = { path = "../meilisearch-types", version = "0.15.0" }
-once_cell = "1.3.1"
-ordered-float = { version = "1.0.2", features = ["serde"] }
+log = "0.4.11"
+meilisearch-error = { path = "../meilisearch-error", version = "0.18.1" }
+meilisearch-schema = { path = "../meilisearch-schema", version = "0.18.1" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.1.2" }
+meilisearch-types = { path = "../meilisearch-types", version = "0.18.1" }
+once_cell = "1.5.2"
+ordered-float = { version = "2.0.1", features = ["serde"] }
 pest = { git = "https://github.com/pest-parser/pest.git", rev = "51fd1d49f1041f7839975664ef71fe15c7dcaf67" }
-pest_derive = "2.0"
-regex = "1.3.6"
+pest_derive = "2.1.0"
+regex = "1.4.2"
 sdset = "0.4.0"
-serde = { version = "1.0.105", features = ["derive"] }
-serde_json = { version = "1.0.50", features = ["preserve_order"] }
+serde = { version = "1.0.118", features = ["derive"] }
+serde_json = { version = "1.0.61", features = ["preserve_order"] }
 slice-group-by = "0.2.6"
 unicase = "2.6.0"
 zerocopy = "0.3.0"
 
 [dev-dependencies]
-assert_matches = "1.3.0"
-criterion = "0.3.1"
-csv = "1.1.3"
-rustyline = { version = "6.0.0", default-features = false }
-structopt = "0.3.12"
+assert_matches = "1.4.0"
+criterion = "0.3.3"
+csv = "1.1.5"
+rustyline = { version = "7.1.0", default-features = false }
+structopt = "0.3.21"
 tempfile = "3.1.0"
-termcolor = "1.1.0"
+termcolor = "1.1.2"
 
 [target.'cfg(unix)'.dev-dependencies]
 jemallocator = "0.3.2"
-
-[[bench]]
-name = "search_benchmark"
-harness = false

meilisearch-core/benches/search_benchmark.rs

@@ -1,108 +0,0 @@
-#[cfg(test)]
-#[macro_use]
-extern crate assert_matches;
-
-use std::error::Error;
-use std::fs::File;
-use std::io::BufReader;
-use std::iter;
-use std::path::Path;
-use std::sync::mpsc;
-
-use meilisearch_core::{Database, DatabaseOptions};
-use meilisearch_core::{ProcessedUpdateResult, UpdateStatus};
-use meilisearch_core::settings::{Settings, SettingsUpdate};
-use meilisearch_schema::Schema;
-use serde_json::Value;
-
-use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId};
-
-fn prepare_database(path: &Path) -> Database {
-    let database = Database::open_or_create(path, DatabaseOptions::default()).unwrap();
-    let db = &database;
-
-    let (sender, receiver) = mpsc::sync_channel(100);
-    let update_fn = move |_name: &str, update: ProcessedUpdateResult| {
-        sender.send(update.update_id).unwrap()
-    };
-    let index = database.create_index("bench").unwrap();
-
-    database.set_update_callback(Box::new(update_fn));
-
-    db.main_write::<_, _, Box<dyn Error>>(|writer| {
-        index.main.put_schema(writer, &Schema::with_primary_key("id")).unwrap();
-        Ok(())
-    }).unwrap();
-
-    let settings_update: SettingsUpdate = {
-        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/settings.json");
-        let file = File::open(path).unwrap();
-        let reader = BufReader::new(file);
-        let settings: Settings = serde_json::from_reader(reader).unwrap();
-        settings.to_update().unwrap()
-    };
-
-    db.update_write::<_, _, Box<dyn Error>>(|writer| {
-        let _update_id = index.settings_update(writer, settings_update).unwrap();
-        Ok(())
-    }).unwrap();
-
-    let mut additions = index.documents_addition();
-
-    let json: Value = {
-        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../datasets/movies/movies.json");
-        let movies_file = File::open(path).expect("find movies");
-        serde_json::from_reader(movies_file).unwrap()
-    };
-
-    let documents = json.as_array().unwrap();
-
-    for document in documents {
-        additions.update_document(document);
-    }
-
-    let update_id = db.update_write::<_, _, Box<dyn Error>>(|writer| {
-        let update_id = additions.finalize(writer).unwrap();
-        Ok(update_id)
-    }).unwrap();
-
-    // block until the transaction is processed
-    let _ = receiver.into_iter().find(|id| *id == update_id);
-
-    let update_reader = db.update_read_txn().unwrap();
-    let result = index.update_status(&update_reader, update_id).unwrap();
-    assert_matches!(result, Some(UpdateStatus::Processed { content }) if content.error.is_none());
-
-    database
-}
-
-pub fn criterion_benchmark(c: &mut Criterion) {
-    let dir = tempfile::tempdir().unwrap();
-    let database = prepare_database(dir.path());
-
-    let reader = database.main_read_txn().unwrap();
-    let index = database.open_index("bench").unwrap();
-
-    let mut count = 0;
-    let query = "I love paris ";
-
-    let iter = iter::from_fn(|| {
-        count += 1;
-        query.get(0..count)
-    });
-
-    let mut group = c.benchmark_group("searching in movies (19654 docs)");
-    group.sample_size(10);
-
-    for query in iter {
-        let bench_name = BenchmarkId::from_parameter(format!("{:?}", query));
-        group.bench_with_input(bench_name, &query, |b, query| b.iter(|| {
-            let builder = index.query_builder();
-            builder.query(&reader, query, 0..20).unwrap();
-        }));
-    }
-    group.finish();
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

meilisearch-core/examples/from_file.rs

@@ -349,8 +349,8 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box<
 
                 if let Some(ref filter) = command.filter {
                     let filter = filter.as_str();
-                    let (positive, filter) = if filter.chars().next() == Some('!') {
-                        (false, &filter[1..])
+                    let (positive, filter) = if let Some(stripped) = filter.strip_prefix('!') {
+                        (false, stripped)
                     } else {
                         (true, filter)
                     };

meilisearch-core/src/automaton/mod.rs

@@ -1,15 +1,4 @@
 mod dfa;
 
-use meilisearch_tokenizer::is_cjk;
-
 pub use self::dfa::{build_dfa, build_prefix_dfa, build_exact_dfa};
 
-pub fn normalize_str(string: &str) -> String {
-    let mut string = string.to_lowercase();
-
-    if !string.contains(is_cjk) {
-        string = deunicode::deunicode_with_tofu(&string, "");
-    }
-
-    string
-}

meilisearch-core/src/bucket_sort.rs

@@ -212,6 +212,7 @@ where
     FD: Fn(DocumentId) -> Option<u64>,
 {
     let mut result = SortResult::default();
+    let mut filtered_count = 0;
 
     let words_set = index.main.words_fst(reader)?;
     let stop_words = index.main.stop_words_fst(reader)?;
@@ -322,19 +323,36 @@ where
                     let filter_accepted = match &filter {
                         Some(filter) => {
                             let entry = filter_map.entry(document.id);
-                            *entry.or_insert_with(|| (filter)(document.id))
+                            *entry.or_insert_with(|| {
+                                let accepted = (filter)(document.id);
+                                // we only want to count it out the first time we see it
+                                if !accepted {
+                                    filtered_count += 1;
+                                }
+                                accepted
+                            })
                         }
                         None => true,
                     };
 
                     if filter_accepted {
                         let entry = key_cache.entry(document.id);
-                        let key = entry.or_insert_with(|| (distinct)(document.id).map(Rc::new));
+                        let mut seen = true;
+                        let key = entry.or_insert_with(|| {
+                            seen = false;
+                            (distinct)(document.id).map(Rc::new)
+                        });
 
-                        match key.clone() {
+                        let distinct = match key.clone() {
                             Some(key) => buf_distinct.register(key),
                             None => buf_distinct.register_without_key(),
                         };
+
+                        // we only want to count the document if it is the first time we see it and
+                        // if it wasn't accepted by distinct
+                        if !seen && !distinct {
+                            filtered_count += 1;
+                        }
                     }
 
                     // the requested range end is reached: stop computing distinct
@@ -396,7 +414,7 @@ where
         }
     }
     result.documents = documents;
-    result.nb_hits = docids.len();
+    result.nb_hits = docids.len() - filtered_count;
 
     Ok(result)
 }

meilisearch-core/src/criterion/mod.rs

@@ -61,7 +61,7 @@ pub trait Criterion {
 }
 
 pub struct ContextMut<'h, 'p, 'tag, 'txn, 'q> {
-    pub reader: &'h heed::RoTxn<MainT>,
+    pub reader: &'h heed::RoTxn<'h, MainT>,
     pub postings_lists: &'p mut SmallArena<'tag, PostingsListView<'txn>>,
     pub query_mapping: &'q HashMap<QueryId, Range<usize>>,
     pub documents_fields_counts_store: store::DocumentsFieldsCounts,

meilisearch-core/src/database.rs

@@ -1,4 +1,5 @@
 use std::collections::hash_map::{Entry, HashMap};
+use std::collections::BTreeMap;
 use std::fs::File;
 use std::path::Path;
 use std::sync::{Arc, RwLock};
@@ -21,13 +22,12 @@ type ArcSwapFn = arc_swap::ArcSwapOption<BoxUpdateFn>;
 
 type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
 
-pub type MainWriter<'a> = heed::RwTxn<'a, MainT>;
-pub type MainReader = heed::RoTxn<MainT>;
+pub type MainWriter<'a, 'b> = heed::RwTxn<'a, 'b, MainT>;
+pub type MainReader<'a, 'b> = heed::RoTxn<'a, MainT>;
 
-pub type UpdateWriter<'a> = heed::RwTxn<'a, UpdateT>;
-pub type UpdateReader = heed::RoTxn<UpdateT>;
+pub type UpdateWriter<'a, 'b> = heed::RwTxn<'a, 'b, UpdateT>;
+pub type UpdateReader<'a> = heed::RoTxn<'a, UpdateT>;
 
-const UNHEALTHY_KEY: &str = "_is_unhealthy";
 const LAST_UPDATE_KEY: &str = "last-update";
 
 pub struct MainT;
@@ -193,9 +193,9 @@ fn version_guard(path: &Path, create: bool) -> MResult<(u32, u32, u32)> {
                 Err(Error::VersionMismatch(format!("{}.{}.XX", version_major, version_minor)))
             } else {
                 Ok((
-                    version_major.parse().or_else(|e| Err(Error::VersionMismatch(format!("error parsing database version: {}", e))))?, 
-                    version_minor.parse().or_else(|e| Err(Error::VersionMismatch(format!("error parsing database version: {}", e))))?, 
-                    version_patch.parse().or_else(|e| Err(Error::VersionMismatch(format!("error parsing database version: {}", e))))?
+                    version_major.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?,
+                    version_minor.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?,
+                    version_patch.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?
                 ))
             }
         }
@@ -212,9 +212,9 @@ fn version_guard(path: &Path, create: bool) -> MResult<(u32, u32, u32)> {
                                 current_version_patch).as_bytes())?;
 
                         Ok((
-                            current_version_major.parse().or_else(|e| Err(Error::VersionMismatch(format!("error parsing database version: {}", e))))?, 
-                            current_version_minor.parse().or_else(|e| Err(Error::VersionMismatch(format!("error parsing database version: {}", e))))?, 
-                            current_version_patch.parse().or_else(|e| Err(Error::VersionMismatch(format!("error parsing database version: {}", e))))?
+                            current_version_major.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?,
+                            current_version_minor.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?,
+                            current_version_patch.parse().map_err(|e| Error::VersionMismatch(format!("error parsing database version: {}", e)))?
                         ))
                     } else {
                         // when no version file is found and we were not told to create one, this
@@ -350,7 +350,7 @@ impl Database {
                 index.main.put_name(&mut writer, name)?;
                 index.main.put_created_at(&mut writer)?;
                 index.main.put_updated_at(&mut writer)?;
-                index.main.put_schema(&mut writer, &Schema::new())?;
+                index.main.put_schema(&mut writer, &Schema::default())?;
 
                 let env_clone = self.env.clone();
                 let update_env_clone = self.update_env.clone();
@@ -532,23 +532,6 @@ impl Database {
         Ok(())
     }
 
-    pub fn set_healthy(&self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
-        let common_store = self.common_store();
-        common_store.delete::<_, Str>(writer, UNHEALTHY_KEY)?;
-        Ok(())
-    }
-
-    pub fn set_unhealthy(&self, writer: &mut heed::RwTxn<MainT>) -> MResult<()> {
-        let common_store = self.common_store();
-        common_store.put::<_, Str, Unit>(writer, UNHEALTHY_KEY, &())?;
-        Ok(())
-    }
-
-    pub fn get_health(&self, reader: &heed::RoTxn<MainT>) -> MResult<Option<()>> {
-        let common_store = self.common_store();
-        Ok(common_store.get::<_, Str, Unit>(&reader, UNHEALTHY_KEY)?)
-    }
-
     pub fn compute_stats(&self, writer: &mut MainWriter, index_uid: &str) -> MResult<()> {
         let index = match self.open_index(&index_uid) {
             Some(index) => index,
@@ -577,7 +560,7 @@ impl Database {
         }
 
         // convert attributes to their names
-        let frequency: HashMap<_, _> = fields_frequency
+        let frequency: BTreeMap<_, _> = fields_frequency
             .into_iter()
             .filter_map(|(a, c)| schema.name(a).map(|name| (name.to_string(), c)))
             .collect();

meilisearch-core/src/facets.rs

@@ -245,8 +245,8 @@ mod test {
 
     #[test]
     fn test_facet_key() {
-        let mut schema = Schema::new();
-        let id = schema.insert_and_index("hello").unwrap();
+        let mut schema = Schema::default();
+        let id = schema.insert_with_position("hello").unwrap().0;
         let facet_list = [schema.id("hello").unwrap()];
         assert_eq!(
             FacetKey::from_str("hello:12", &schema, &facet_list).unwrap(),
@@ -286,8 +286,8 @@ mod test {
     #[test]
     fn test_parse_facet_array() {
         use either::Either::{Left, Right};
-        let mut schema = Schema::new();
-        let _id = schema.insert_and_index("hello").unwrap();
+        let mut schema = Schema::default();
+        let _id = schema.insert_with_position("hello").unwrap();
         let facet_list = [schema.id("hello").unwrap()];
         assert_eq!(
             FacetFilter::from_str("[[\"hello:12\"]]", &schema, &facet_list).unwrap(),

meilisearch-core/src/query_builder.rs

@@ -225,10 +225,17 @@ impl<'c, 'f, 'd, 'i> QueryBuilder<'c, 'f, 'd, 'i> {
 
     fn sort_result_from_docids(&self, docids: &[DocumentId], range: Range<usize>) -> SortResult {
         let mut sort_result = SortResult::default();
+        let mut filtered_count = 0;
         let mut result = match self.filter {
             Some(ref filter) => docids
                 .iter()
-                .filter(|item| (filter)(**item))
+                .filter(|item| {
+                    let accepted = (filter)(**item);
+                    if !accepted {
+                        filtered_count += 1;
+                    }
+                    accepted
+                })
                 .skip(range.start)
                 .take(range.end - range.start)
                 .map(|&id| Document::from_highlights(id, &[]))
@@ -248,15 +255,19 @@ impl<'c, 'f, 'd, 'i> QueryBuilder<'c, 'f, 'd, 'i> {
             result.retain(|doc| {
                 let id = doc.id;
                 let key = (distinct)(id);
-                match key {
+                let distinct_accepted = match key {
                     Some(key) => distinct_map.register(key),
                     None => distinct_map.register_without_key(),
+                };
+                if !distinct_accepted {
+                    filtered_count += 1;
                 }
+                distinct_accepted
             });
         }
 
         sort_result.documents = result;
-        sort_result.nb_hits = docids.len();
+        sort_result.nb_hits = docids.len() - filtered_count;
         sort_result
     }
 
@@ -285,7 +296,6 @@ mod tests {
     use sdset::SetBuf;
     use tempfile::TempDir;
 
-    use crate::automaton::normalize_str;
     use crate::bucket_sort::SimpleMatch;
     use crate::database::{Database, DatabaseOptions};
     use crate::store::Index;
@@ -293,6 +303,35 @@ mod tests {
     use crate::Document;
     use meilisearch_schema::Schema;
 
+    fn is_cjk(c: char) -> bool {
+        ('\u{1100}'..'\u{11ff}').contains(&c)  // Hangul Jamo
+            || ('\u{2e80}'..'\u{2eff}').contains(&c)  // CJK Radicals Supplement
+            || ('\u{2f00}'..'\u{2fdf}').contains(&c) // Kangxi radical
+            || ('\u{3000}'..'\u{303f}').contains(&c) // Japanese-style punctuation
+            || ('\u{3040}'..'\u{309f}').contains(&c) // Japanese Hiragana
+            || ('\u{30a0}'..'\u{30ff}').contains(&c) // Japanese Katakana
+            || ('\u{3100}'..'\u{312f}').contains(&c)
+            || ('\u{3130}'..'\u{318F}').contains(&c) // Hangul Compatibility Jamo
+            || ('\u{3200}'..'\u{32ff}').contains(&c) // Enclosed CJK Letters and Months
+            || ('\u{3400}'..'\u{4dbf}').contains(&c) // CJK Unified Ideographs Extension A
+            || ('\u{4e00}'..'\u{9fff}').contains(&c) // CJK Unified Ideographs
+            || ('\u{a960}'..'\u{a97f}').contains(&c) // Hangul Jamo Extended-A
+            || ('\u{ac00}'..'\u{d7a3}').contains(&c) // Hangul Syllables
+            || ('\u{d7b0}'..'\u{d7ff}').contains(&c) // Hangul Jamo Extended-B
+            || ('\u{f900}'..'\u{faff}').contains(&c) // CJK Compatibility Ideographs
+            || ('\u{ff00}'..'\u{ffef}').contains(&c) // Full-width roman characters and half-width katakana
+    }
+
+    fn normalize_str(string: &str) -> String {
+        let mut string = string.to_lowercase();
+
+        if !string.contains(is_cjk) {
+            string = deunicode::deunicode_with_tofu(&string, "");
+        }
+
+        string
+    }
+
     fn set_from_stream<'f, I, S>(stream: I) -> fst::Set<Vec<u8>>
     where
         I: for<'a> fst::IntoStreamer<'a, Into = S, Item = &'a [u8]>,
@@ -405,7 +444,7 @@ mod tests {
                 for index in indexes {
                     let name = index.attribute.to_string();
                     schema.insert(&name).unwrap();
-                    let indexed_pos = schema.set_indexed(&name).unwrap().1;
+                    let indexed_pos = schema.insert_with_position(&name).unwrap().1;
                     let index = DocIndex {
                         attribute: indexed_pos.0,
                         ..*index
@@ -1258,15 +1297,15 @@ mod tests {
         let builder = store.query_builder();
         let SortResult { documents, .. } = builder.query(&reader, Some("télephone"), 0..20).unwrap();
         let mut iter = documents.into_iter();
-
-        assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => {
+        assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
             let mut iter = matches.into_iter();
             assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. }));
+            assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. }));
             assert_matches!(iter.next(), None);
         });
-        assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => {
+        assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => {
             let mut iter = matches.into_iter();
-            assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, distance: 1, word_index: 0, is_exact: false, .. })); // iphone | telephone
+            assert_matches!(iter.next(), Some(SimpleMatch { query_index: 0, .. }));
             assert_matches!(iter.next(), None);
         });
         assert_matches!(iter.next(), None);

meilisearch-core/src/query_tree.rs

@@ -7,13 +7,13 @@ use std::{cmp, fmt, iter::once};
 
 use fst::{IntoStreamer, Streamer};
 use itertools::{EitherOrBoth, merge_join_by};
-use meilisearch_tokenizer::split_query_string;
-use sdset::{Set, SetBuf, SetOperation};
 use log::debug;
+use meilisearch_tokenizer::analyzer::{Analyzer, AnalyzerConfig};
+use sdset::{Set, SetBuf, SetOperation};
 
 use crate::database::MainT;
 use crate::{store, DocumentId, DocIndex, MResult, FstSetCow};
-use crate::automaton::{normalize_str, build_dfa, build_prefix_dfa, build_exact_dfa};
+use crate::automaton::{build_dfa, build_prefix_dfa, build_exact_dfa};
 use crate::QueryWordsMapper;
 
 #[derive(Clone, PartialEq, Eq, Hash)]
@@ -146,7 +146,7 @@ fn split_best_frequency<'a>(reader: &heed::RoTxn<MainT>, ctx: &Context, word: &'
 }
 
 fn fetch_synonyms(reader: &heed::RoTxn<MainT>, ctx: &Context, words: &[&str]) -> MResult<Vec<Vec<String>>> {
-    let words = normalize_str(&words.join(" "));
+    let words = &words.join(" ");
     let set = ctx.synonyms.synonyms_fst(reader, words.as_bytes())?;
 
     let mut strings = Vec::new();
@@ -174,15 +174,25 @@ where I: IntoIterator<Item=Operation>,
 
 const MAX_NGRAM: usize = 3;
 
+fn split_query_string<'a, A: AsRef<[u8]>>(s: &str, stop_words: &'a fst::Set<A>) -> Vec<(usize, String)> {
+    // TODO: Use global instance instead
+    Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words))
+        .analyze(s)
+        .tokens()
+        .filter(|t| t.is_word())
+        .map(|t| t.word.to_string())
+        .enumerate()
+        .collect()
+}
+
 pub fn create_query_tree(
     reader: &heed::RoTxn<MainT>,
     ctx: &Context,
     query: &str,
 ) -> MResult<(Operation, HashMap<QueryId, Range<usize>>)>
 {
-    let words = split_query_string(query).map(str::to_lowercase);
-    let words = words.filter(|w| !ctx.stop_words.contains(w));
-    let words: Vec<_> = words.enumerate().collect();
+    // TODO: use a shared analyzer instance
+    let words = split_query_string(query, &ctx.stop_words);
 
     let mut mapper = QueryWordsMapper::new(words.iter().map(|(_, w)| w));
 

meilisearch-core/src/raw_indexer.rs

@@ -2,9 +2,9 @@ use std::borrow::Cow;
 use std::collections::{BTreeMap, HashMap};
 use std::convert::TryFrom;
 
-use deunicode::deunicode_with_tofu;
 use meilisearch_schema::IndexedPos;
-use meilisearch_tokenizer::{is_cjk, SeqTokenizer, Token, Tokenizer};
+use meilisearch_tokenizer::analyzer::{Analyzer, AnalyzerConfig};
+use meilisearch_tokenizer::{Token, token::SeparatorKind, TokenKind};
 use sdset::SetBuf;
 
 use crate::{DocIndex, DocumentId};
@@ -14,11 +14,11 @@ const WORD_LENGTH_LIMIT: usize = 80;
 
 type Word = Vec<u8>; // TODO make it be a SmallVec
 
-pub struct RawIndexer<A> {
+pub struct RawIndexer<'a, A> {
     word_limit: usize, // the maximum number of indexed words
-    stop_words: fst::Set<A>,
     words_doc_indexes: BTreeMap<Word, Vec<DocIndex>>,
     docs_words: HashMap<DocumentId, Vec<Word>>,
+    analyzer: Analyzer<'a, A>,
 }
 
 pub struct Indexed<'a> {
@@ -26,32 +26,35 @@ pub struct Indexed<'a> {
     pub docs_words: HashMap<DocumentId, FstSetCow<'a>>,
 }
 
-impl<A> RawIndexer<A> {
-    pub fn new(stop_words: fst::Set<A>) -> RawIndexer<A> {
+impl<'a, A> RawIndexer<'a, A>
+where
+    A: AsRef<[u8]>
+{
+    pub fn new(stop_words: &'a fst::Set<A>) -> RawIndexer<'a, A> {
         RawIndexer::with_word_limit(stop_words, 1000)
     }
 
-    pub fn with_word_limit(stop_words: fst::Set<A>, limit: usize) -> RawIndexer<A> {
+    pub fn with_word_limit(stop_words: &'a fst::Set<A>, limit: usize) -> RawIndexer<A> {
         RawIndexer {
             word_limit: limit,
-            stop_words,
             words_doc_indexes: BTreeMap::new(),
             docs_words: HashMap::new(),
+            analyzer: Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)),
         }
     }
-}
 
-impl<A: AsRef<[u8]>> RawIndexer<A> {
     pub fn index_text(&mut self, id: DocumentId, indexed_pos: IndexedPos, text: &str) -> usize {
         let mut number_of_words = 0;
 
-        for token in Tokenizer::new(text) {
+        let analyzed_text = self.analyzer.analyze(text);
+        for (token_pos, (word_pos, token)) in process_tokens(analyzed_text.tokens()).enumerate() {
             let must_continue = index_token(
                 token,
+                word_pos,
+                token_pos,
                 id,
                 indexed_pos,
                 self.word_limit,
-                &self.stop_words,
                 &mut self.words_doc_indexes,
                 &mut self.docs_words,
             );
@@ -66,24 +69,37 @@ impl<A: AsRef<[u8]>> RawIndexer<A> {
         number_of_words
     }
 
-    pub fn index_text_seq<'s, I>(&mut self, id: DocumentId, indexed_pos: IndexedPos, iter: I)
+    pub fn index_text_seq<'s, I>(&mut self, id: DocumentId, indexed_pos: IndexedPos, text_iter: I)
     where
         I: IntoIterator<Item = &'s str>,
     {
-        let iter = iter.into_iter();
-        for token in SeqTokenizer::new(iter) {
-            let must_continue = index_token(
-                token,
-                id,
-                indexed_pos,
-                self.word_limit,
-                &self.stop_words,
-                &mut self.words_doc_indexes,
-                &mut self.docs_words,
-            );
+        let mut word_offset = 0;
 
-            if !must_continue {
-                break;
+        for text in text_iter.into_iter() {
+            let current_word_offset = word_offset;
+
+            let analyzed_text = self.analyzer.analyze(text);
+            let tokens = process_tokens(analyzed_text.tokens())
+                .map(|(i, t)| (i + current_word_offset, t))
+                .enumerate();
+
+            for (token_pos, (word_pos, token)) in tokens  {
+                word_offset = word_pos + 1;
+
+                let must_continue = index_token(
+                    token,
+                    word_pos,
+                    token_pos,
+                    id,
+                    indexed_pos,
+                    self.word_limit,
+                    &mut self.words_doc_indexes,
+                    &mut self.docs_words,
+                );
+
+                if !must_continue {
+                    break;
+                }
             }
         }
     }
@@ -113,31 +129,53 @@ impl<A: AsRef<[u8]>> RawIndexer<A> {
     }
 }
 
-fn index_token<A>(
+fn process_tokens<'a>(tokens: impl Iterator<Item = Token<'a>>) -> impl Iterator<Item = (usize, Token<'a>)> {
+    tokens
+        .skip_while(|token| !token.is_word())
+        .scan((0, None), |(offset, prev_kind), token| {
+                match token.kind {
+                    TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => {
+                        *offset += match *prev_kind {
+                            Some(TokenKind::Separator(SeparatorKind::Hard)) => 8,
+                            Some(_) => 1,
+                            None => 0,
+                        };
+                        *prev_kind = Some(token.kind)
+                    }
+                    TokenKind::Separator(SeparatorKind::Hard) => {
+                        *prev_kind = Some(token.kind);
+                    }
+                    TokenKind::Separator(SeparatorKind::Soft)
+                        if *prev_kind != Some(TokenKind::Separator(SeparatorKind::Hard)) => {
+                        *prev_kind = Some(token.kind);
+                    }
+                    _ => (),
+                }
+            Some((*offset, token))
+        })
+    .filter(|(_, t)| t.is_word())
+}
+
+#[allow(clippy::too_many_arguments)]
+fn index_token(
     token: Token,
+    word_pos: usize,
+    token_pos: usize,
     id: DocumentId,
     indexed_pos: IndexedPos,
     word_limit: usize,
-    stop_words: &fst::Set<A>,
     words_doc_indexes: &mut BTreeMap<Word, Vec<DocIndex>>,
     docs_words: &mut HashMap<DocumentId, Vec<Word>>,
 ) -> bool
-where A: AsRef<[u8]>,
 {
-    if token.index >= word_limit {
+    if token_pos >= word_limit {
         return false;
     }
 
-    let lower = token.word.to_lowercase();
-    let token = Token {
-        word: &lower,
-        ..token
-    };
-
-    if !stop_words.contains(&token.word) {
-        match token_to_docindex(id, indexed_pos, token) {
+    if !token.is_stopword() {
+        match token_to_docindex(id, indexed_pos, &token, word_pos) {
             Some(docindex) => {
-                let word = Vec::from(token.word);
+                let word = Vec::from(token.word.as_ref());
 
                 if word.len() <= WORD_LENGTH_LIMIT {
                     words_doc_indexes
@@ -145,20 +183,6 @@ where A: AsRef<[u8]>,
                         .or_insert_with(Vec::new)
                         .push(docindex);
                     docs_words.entry(id).or_insert_with(Vec::new).push(word);
-
-                    if !lower.contains(is_cjk) {
-                        let unidecoded = deunicode_with_tofu(&lower, "");
-                        if unidecoded != lower && !unidecoded.is_empty() {
-                            let word = Vec::from(unidecoded);
-                            if word.len() <= WORD_LENGTH_LIMIT {
-                                words_doc_indexes
-                                    .entry(word.clone())
-                                    .or_insert_with(Vec::new)
-                                    .push(docindex);
-                                docs_words.entry(id).or_insert_with(Vec::new).push(word);
-                            }
-                        }
-                    }
                 }
             }
             None => return false,
@@ -168,10 +192,10 @@ where A: AsRef<[u8]>,
     true
 }
 
-fn token_to_docindex(id: DocumentId, indexed_pos: IndexedPos, token: Token) -> Option<DocIndex> {
-    let word_index = u16::try_from(token.word_index).ok()?;
-    let char_index = u16::try_from(token.char_index).ok()?;
-    let char_length = u16::try_from(token.word.chars().count()).ok()?;
+fn token_to_docindex(id: DocumentId, indexed_pos: IndexedPos, token: &Token, word_index: usize) -> Option<DocIndex> {
+    let word_index = u16::try_from(word_index).ok()?;
+    let char_index = u16::try_from(token.byte_start).ok()?;
+    let char_length = u16::try_from(token.word.len()).ok()?;
 
     let docindex = DocIndex {
         document_id: id,
@@ -188,10 +212,23 @@ fn token_to_docindex(id: DocumentId, indexed_pos: IndexedPos, token: Token) -> O
 mod tests {
     use super::*;
     use meilisearch_schema::IndexedPos;
+    use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
+    use fst::Set;
+
+    #[test]
+    fn test_process_token() {
+        let text = " 為一包含一千多萬目詞的帶標記平衡語料庫";
+        let stopwords = Set::default();
+        let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(&stopwords));
+        let analyzer = analyzer.analyze(text);
+        let tokens: Vec<_> = process_tokens(analyzer.tokens()).map(|(_, t)| t.text().to_string()).collect();
+        assert_eq!(tokens, ["为", "一", "包含", "一千多万", "目词", "的", "带", "标记", "平衡", "语料库"]);
+    }
 
     #[test]
     fn strange_apostrophe() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
 
         let docid = DocumentId(0);
         let indexed_pos = IndexedPos(0);
@@ -206,14 +243,12 @@ mod tests {
         assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
         assert!(words_doc_indexes.get(&b"ai"[..]).is_some());
         assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
     }
 
     #[test]
     fn strange_apostrophe_in_sequence() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
 
         let docid = DocumentId(0);
         let indexed_pos = IndexedPos(0);
@@ -228,9 +263,6 @@ mod tests {
         assert!(words_doc_indexes.get(&b"aspirateur"[..]).is_some());
         assert!(words_doc_indexes.get(&b"ai"[..]).is_some());
         assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
     }
 
     #[test]
@@ -238,7 +270,7 @@ mod tests {
         let stop_words = sdset::SetBuf::from_dirty(vec!["l", "j", "ai", "de"]);
         let stop_words = fst::Set::from_iter(stop_words).unwrap();
 
-        let mut indexer = RawIndexer::new(stop_words);
+        let mut indexer = RawIndexer::new(&stop_words);
 
         let docid = DocumentId(0);
         let indexed_pos = IndexedPos(0);
@@ -255,14 +287,12 @@ mod tests {
         assert!(words_doc_indexes.get(&b"ai"[..]).is_none());
         assert!(words_doc_indexes.get(&b"de"[..]).is_none());
         assert!(words_doc_indexes.get(&b"eteindre"[..]).is_some());
-        assert!(words_doc_indexes
-            .get(&"éteindre".to_owned().into_bytes())
-            .is_some());
     }
 
     #[test]
     fn no_empty_unidecode() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
 
         let docid = DocumentId(0);
         let indexed_pos = IndexedPos(0);
@@ -281,7 +311,8 @@ mod tests {
     #[test]
     // test sample from 807
     fn very_long_text() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
         let indexed_pos = IndexedPos(0);
         let docid = DocumentId(0);
         let text = " The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means &ldquo;requests for example.com/&rdquo;, while '/admin' means &ldquo;requests for example.com/admin&rdquo;. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:&#39;/&#39;:# Rules for all requests that don&#39;t otherwise match....&#39;/sites/default/files&#39;:# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:&#39;/&#39;:passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:&#39;/&#39;:root:&#39;web&#39;passthru:&#39;/app.php&#39;This block will serve requests to / from the web directory in the application, and if a file doesn&rsquo;t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n  root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n  passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n  index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n  expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with &ldquo;ms&rdquo; (milliseconds), &ldquo;s&rdquo; (seconds), &ldquo;m&rdquo; (minutes), &ldquo;h&rdquo; (hours), &ldquo;d&rdquo; (days), &ldquo;w&rdquo; (weeks), &ldquo;M&rdquo; (months, 30d) or &ldquo;y&rdquo; (years, 365d).\n  scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n  allow: Whether to allow serving files which don&rsquo;t match a rule (true or false, default: true).\n  headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren&rsquo;t affected, to avoid overlap with the application&rsquo;s own ability to include custom headers in the response.\n  rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n  request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:&#39;/&#39;:passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application&rsquo;s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ";
@@ -289,12 +320,13 @@ mod tests {
         let Indexed {
             words_doc_indexes, ..
         } = indexer.build();
-        assert!(words_doc_indexes.get(&"buffering".to_owned().into_bytes()).is_some());
+        assert!(words_doc_indexes.get(&"request".to_owned().into_bytes()).is_some());
     }
 
     #[test]
     fn words_over_index_1000_not_indexed() {
-        let mut indexer = RawIndexer::new(fst::Set::default());
+        let stop_words = fst::Set::default();
+        let mut indexer = RawIndexer::new(&stop_words);
         let indexed_pos = IndexedPos(0);
         let docid = DocumentId(0);
         let mut text = String::with_capacity(5000);

meilisearch-core/src/serde/deserializer.rs

@@ -51,7 +51,7 @@ impl From<heed::Error> for DeserializerError {
 
 pub struct Deserializer<'a> {
     pub document_id: DocumentId,
-    pub reader: &'a heed::RoTxn<MainT>,
+    pub reader: &'a heed::RoTxn<'a, MainT>,
     pub documents_fields: DocumentsFields,
     pub schema: &'a Schema,
     pub fields: Option<&'a HashSet<FieldId>>,

meilisearch-core/src/settings.rs

@@ -1,4 +1,4 @@
-use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::collections::{BTreeMap, BTreeSet};
 use std::str::FromStr;
 use std::iter::IntoIterator;
 
@@ -13,7 +13,7 @@ static RANKING_RULE_REGEX: Lazy<regex::Regex> = Lazy::new(|| {
     regex::Regex::new(r"(asc|desc)\(([a-zA-Z0-9-_]*)\)").unwrap()
 });
 
-#[derive(Default, Clone, Serialize, Deserialize)]
+#[derive(Default, Clone, Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct Settings {
     #[serde(default, deserialize_with = "deserialize_some")]
@@ -23,7 +23,7 @@ pub struct Settings {
     #[serde(default, deserialize_with = "deserialize_some")]
     pub searchable_attributes: Option<Option<Vec<String>>>,
     #[serde(default, deserialize_with = "deserialize_some")]
-    pub displayed_attributes: Option<Option<HashSet<String>>>,
+    pub displayed_attributes: Option<Option<BTreeSet<String>>>,
     #[serde(default, deserialize_with = "deserialize_some")]
     pub stop_words: Option<Option<BTreeSet<String>>>,
     #[serde(default, deserialize_with = "deserialize_some")]
@@ -161,7 +161,7 @@ pub struct SettingsUpdate {
     pub distinct_attribute: UpdateState<String>,
     pub primary_key: UpdateState<String>,
     pub searchable_attributes: UpdateState<Vec<String>>,
-    pub displayed_attributes: UpdateState<HashSet<String>>,
+    pub displayed_attributes: UpdateState<BTreeSet<String>>,
     pub stop_words: UpdateState<BTreeSet<String>>,
     pub synonyms: UpdateState<BTreeMap<String, Vec<String>>>,
     pub attributes_for_faceting: UpdateState<Vec<String>>,

meilisearch-core/src/store/docs_words.rs

@@ -33,7 +33,7 @@ impl DocsWords {
         self.docs_words.clear(writer)
     }
 
-    pub fn doc_words(self, reader: &heed::RoTxn<MainT>, document_id: DocumentId) -> ZResult<FstSetCow> {
+    pub fn doc_words<'a>(self, reader: &'a heed::RoTxn<'a, MainT>, document_id: DocumentId) -> ZResult<FstSetCow> {
         let document_id = BEU32::new(document_id.0);
         match self.docs_words.get(reader, &document_id)? {
             Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),

meilisearch-core/src/store/facets.rs

@@ -2,7 +2,7 @@ use std::borrow::Cow;
 use std::collections::HashMap;
 use std::mem;
 
-use heed::{RwTxn, RoTxn, RoRange, types::Str, BytesEncode, BytesDecode};
+use heed::{RwTxn, RoTxn, RoPrefix, types::Str, BytesEncode, BytesDecode};
 use sdset::{SetBuf, Set, SetOperation};
 
 use meilisearch_types::DocumentId;
@@ -62,7 +62,7 @@ impl Facets {
         Ok(self.facets.put(writer, &facet_key, &(facet_value, doc_ids))?)
     }
 
-    pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, field_id: FieldId) -> MResult<RoRange<'txn, FacetKey, FacetData>> {
+    pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, field_id: FieldId) -> MResult<RoPrefix<'txn, FacetKey, FacetData>> {
         Ok(self.facets.prefix_iter(reader, &FacetKey::new(field_id, String::new()))?)
     }
 

meilisearch-core/src/store/main.rs

@@ -1,5 +1,5 @@
 use std::borrow::Cow;
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 
 use chrono::{DateTime, Utc};
 use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str, CowSlice};
@@ -31,7 +31,7 @@ const SYNONYMS_KEY: &str = "synonyms";
 const UPDATED_AT_KEY: &str = "updated-at";
 const WORDS_KEY: &str = "words";
 
-pub type FreqsMap = HashMap<String, usize>;
+pub type FreqsMap = BTreeMap<String, usize>;
 type SerdeFreqsMap = SerdeBincode<FreqsMap>;
 type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
 
@@ -143,7 +143,7 @@ impl Main {
         self.put_external_docids(writer, &external_docids)
     }
 
-    pub fn external_docids(self, reader: &heed::RoTxn<MainT>) -> MResult<FstMapCow> {
+    pub fn external_docids<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstMapCow> {
         match self.main.get::<_, Str, ByteSlice>(reader, EXTERNAL_DOCIDS_KEY)? {
             Some(bytes) => Ok(fst::Map::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
             None => Ok(fst::Map::default().map_data(Cow::Owned).unwrap()),
@@ -155,7 +155,7 @@ impl Main {
         Ok(external_ids.get(external_docid).map(|id| DocumentId(id as u32)))
     }
 
-    pub fn words_fst(self, reader: &heed::RoTxn<MainT>) -> MResult<FstSetCow> {
+    pub fn words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstSetCow> {
         match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
             Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
             None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
@@ -170,7 +170,7 @@ impl Main {
         Ok(self.main.put::<_, Str, CowSlice<DocumentId>>(writer, SORTED_DOCUMENT_IDS_CACHE_KEY, documents_ids)?)
     }
 
-    pub fn sorted_document_ids_cache(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<Cow<[DocumentId]>>> {
+    pub fn sorted_document_ids_cache<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<Option<Cow<[DocumentId]>>> {
         Ok(self.main.get::<_, Str, CowSlice<DocumentId>>(reader, SORTED_DOCUMENT_IDS_CACHE_KEY)?)
     }
 
@@ -199,7 +199,7 @@ impl Main {
         Ok(self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)?)
     }
 
-    pub(crate) fn synonyms_fst(self, reader: &heed::RoTxn<MainT>) -> MResult<FstSetCow> {
+    pub(crate) fn synonyms_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstSetCow> {
         match self.main.get::<_, Str, ByteSlice>(reader, SYNONYMS_KEY)? {
             Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
             None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
@@ -219,7 +219,7 @@ impl Main {
         Ok(self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)?)
     }
 
-    pub(crate) fn stop_words_fst(self, reader: &heed::RoTxn<MainT>) -> MResult<FstSetCow> {
+    pub(crate) fn stop_words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstSetCow> {
         match self.main.get::<_, Str, ByteSlice>(reader, STOP_WORDS_KEY)? {
             Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
             None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),

meilisearch-core/src/update/clear_all.rs

@@ -11,12 +11,14 @@ pub fn apply_clear_all(
     index.main.put_internal_docids(writer, &sdset::SetBuf::default())?;
     index.main.put_ranked_map(writer, &RankedMap::default())?;
     index.main.put_number_of_documents(writer, |_| 0)?;
+    index.main.put_sorted_document_ids_cache(writer, &[])?;
     index.documents_fields.clear(writer)?;
     index.documents_fields_counts.clear(writer)?;
     index.postings_lists.clear(writer)?;
     index.docs_words.clear(writer)?;
     index.prefix_documents_cache.clear(writer)?;
     index.prefix_postings_lists_cache.clear(writer)?;
+    index.facets.clear(writer)?;
 
     Ok(())
 }

meilisearch-core/src/update/documents_addition.rs

@@ -110,7 +110,7 @@ pub fn push_documents_addition<D: serde::Serialize>(
 }
 
 #[allow(clippy::too_many_arguments)]
-fn index_document<A>(
+fn index_document<A: AsRef<[u8]>>(
     writer: &mut heed::RwTxn<MainT>,
     documents_fields: DocumentsFields,
     documents_fields_counts: DocumentsFieldsCounts,
@@ -121,18 +121,17 @@ fn index_document<A>(
     document_id: DocumentId,
     value: &Value,
 ) -> MResult<()>
-where A: AsRef<[u8]>,
 {
     let serialized = serde_json::to_vec(value)?;
     documents_fields.put_document_field(writer, document_id, field_id, &serialized)?;
 
-    if let Some(indexed_pos) = schema.is_indexed(field_id) {
-        let number_of_words = index_value(indexer, document_id, *indexed_pos, value);
+    if let Some(indexed_pos) = schema.is_searchable(field_id) {
+        let number_of_words = index_value(indexer, document_id, indexed_pos, value);
         if let Some(number_of_words) = number_of_words {
             documents_fields_counts.put_document_field_count(
                 writer,
                 document_id,
-                *indexed_pos,
+                indexed_pos,
                 number_of_words as u16,
             )?;
         }
@@ -146,8 +145,8 @@ where A: AsRef<[u8]>,
     Ok(())
 }
 
-pub fn apply_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
+pub fn apply_addition(
+    writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
     new_documents: Vec<IndexMap<String, Value>>,
     partial: bool
@@ -222,13 +221,13 @@ pub fn apply_addition<'a, 'b>(
     let stop_words = index.main.stop_words_fst(writer)?.map_data(Cow::into_owned)?;
 
 
-    let mut indexer = RawIndexer::new(stop_words);
+    let mut indexer = RawIndexer::new(&stop_words);
 
     // For each document in this update
     for (document_id, document) in &documents_additions {
         // For each key-value pair in the document.
         for (attribute, value) in document {
-            let field_id = schema.insert_and_index(&attribute)?;
+            let (field_id, _) = schema.insert_with_position(&attribute)?;
             index_document(
                 writer,
                 index.documents_fields,
@@ -272,16 +271,16 @@ pub fn apply_addition<'a, 'b>(
     Ok(())
 }
 
-pub fn apply_documents_partial_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
+pub fn apply_documents_partial_addition(
+    writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
     new_documents: Vec<IndexMap<String, Value>>,
 ) -> MResult<()> {
     apply_addition(writer, index, new_documents, true)
 }
 
-pub fn apply_documents_addition<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
+pub fn apply_documents_addition(
+    writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
     new_documents: Vec<IndexMap<String, Value>>,
 ) -> MResult<()> {
@@ -317,7 +316,7 @@ pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Ind
         .unwrap();
 
     let number_of_inserted_documents = documents_ids_to_reindex.len();
-    let mut indexer = RawIndexer::new(stop_words);
+    let mut indexer = RawIndexer::new(&stop_words);
     let mut ram_store = HashMap::new();
 
     if let Some(ref attributes_for_facetting) = index.main.attributes_for_faceting(writer)? {
@@ -373,14 +372,13 @@ pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Ind
     Ok(())
 }
 
-pub fn write_documents_addition_index<A>(
+pub fn write_documents_addition_index<A: AsRef<[u8]>>(
     writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
     ranked_map: &RankedMap,
     number_of_inserted_documents: usize,
     indexer: RawIndexer<A>,
 ) -> MResult<()>
-where A: AsRef<[u8]>,
 {
     let indexed = indexer.build();
     let mut delta_words_builder = SetBuilder::memory();

meilisearch-core/src/update/helpers.rs

@@ -12,13 +12,12 @@ use crate::serde::SerializerError;
 use crate::store::DiscoverIds;
 
 /// Returns the number of words indexed or `None` if the type is unindexable.
-pub fn index_value<A>(
+pub fn index_value<A: AsRef<[u8]>>(
     indexer: &mut RawIndexer<A>,
     document_id: DocumentId,
     indexed_pos: IndexedPos,
     value: &Value,
 ) -> Option<usize>
-where A: AsRef<[u8]>,
 {
     match value {
         Value::Null => None,

meilisearch-core/src/update/mod.rs

@@ -212,8 +212,8 @@ pub fn next_update_id(
     Ok(new_update_id)
 }
 
-pub fn update_task<'a, 'b>(
-    writer: &'a mut heed::RwTxn<'b, MainT>,
+pub fn update_task(
+    writer: &mut heed::RwTxn<MainT>,
     index: &store::Index,
     update_id: u64,
     update: Update,

meilisearch-core/src/update/settings_update.rs

@@ -71,14 +71,14 @@ pub fn apply_settings_update(
     match settings.searchable_attributes.clone() {
         UpdateState::Update(v) => {
             if v.iter().any(|e| e == "*") || v.is_empty() {
-                schema.set_all_fields_as_indexed();
+                schema.set_all_searchable();
             } else {
-                schema.update_indexed(v)?;
+                schema.update_searchable(v)?;
             }
             must_reindex = true;
         },
         UpdateState::Clear => {
-            schema.set_all_fields_as_indexed();
+            schema.set_all_searchable();
             must_reindex = true;
         },
         UpdateState::Nothing => (),
@@ -86,13 +86,13 @@ pub fn apply_settings_update(
     match settings.displayed_attributes.clone() {
         UpdateState::Update(v) => {
             if v.contains("*") || v.is_empty() {
-                schema.set_all_fields_as_displayed();
+                schema.set_all_displayed();
             } else {
                 schema.update_displayed(v)?
             }
         },
         UpdateState::Clear => {
-            schema.set_all_fields_as_displayed();
+            schema.set_all_displayed();
         },
         UpdateState::Nothing => (),
     }

meilisearch-error/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "meilisearch-error"
-version = "0.15.0"
+version = "0.18.1"
 authors = ["marin <postma.marin@protonmail.com>"]
 edition = "2018"
 
 [dependencies]
-actix-http = "2"
+actix-http = "2.2.0"

meilisearch-error/src/lib.rs

@@ -69,7 +69,6 @@ pub enum Code {
     DocumentNotFound,
     Internal,
     InvalidToken,
-    Maintenance,
     MissingAuthorizationHeader,
     NotFound,
     PayloadTooLarge,
@@ -118,7 +117,6 @@ impl Code {
             DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
             Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
             InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN),
-            Maintenance =>  ErrCode::internal("maintenance", StatusCode::SERVICE_UNAVAILABLE),
             MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED),
             NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND),
             PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE),

meilisearch-http/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "meilisearch-http"
 description = "MeiliSearch HTTP server"
-version = "0.15.0"
+version = "0.18.1"
 license = "MIT"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
@@ -17,43 +17,42 @@ path = "src/main.rs"
 default = ["sentry"]
 
 [dependencies]
-actix-cors = "0.3"
-actix-http = "2"
-actix-rt = "1"
+actix-cors = "0.5.4"
+actix-http = "2.2.0"
+actix-rt = "1.1.1"
 actix-service = "1.0.6"
-actix-web = { version = "3", features = ["rustls"] }
-bytes = "0.5.4"
-chrono = { version = "0.4.11", features = ["serde"] }
-crossbeam-channel = "0.4.2"
-env_logger = "0.7.1"
-flate2 = "1.0.16"
-futures = "0.3.4"
-http = "0.1.19"
-indexmap = { version = "1.3.2", features = ["serde-1"] }
-log = "0.4.8"
-main_error = "0.1.0"
-meilisearch-core = { path = "../meilisearch-core", version = "0.15.0" }
-meilisearch-error = { path = "../meilisearch-error", version = "0.15.0" }
-meilisearch-schema = { path = "../meilisearch-schema", version = "0.15.0" }
-meilisearch-tokenizer = {path = "../meilisearch-tokenizer", version = "0.15.0"}
+actix-web = { version = "3.3.2", features = ["rustls"] }
+bytes = "1.0.0"
+chrono = { version = "0.4.19", features = ["serde"] }
+crossbeam-channel = "0.5.0"
+env_logger = "0.8.2"
+flate2 = "1.0.19"
+futures = "0.3.8"
+http = "0.2.2"
+indexmap = { version = "1.6.1", features = ["serde-1"] }
+log = "0.4.11"
+main_error = "0.1.1"
+meilisearch-core = { path = "../meilisearch-core", version = "0.18.0" }
+meilisearch-error = { path = "../meilisearch-error", version = "0.18.1" }
+meilisearch-schema = { path = "../meilisearch-schema", version = "0.18.1" }
 mime = "0.3.16"
-once_cell = "1.4.1"
-rand = "0.7.3"
-regex = "1.3.6"
-rustls = "0.18"
-serde = { version = "1.0.105", features = ["derive"] }
-serde_json = { version = "1.0.50", features = ["preserve_order"] }
-serde_qs = "0.5.2"
-sha2 = "0.8.1"
-siphasher = "0.3.2"
+once_cell = "1.5.2"
+rand = "0.8.1"
+regex = "1.4.2"
+rustls = "0.18.0"
+serde = { version = "1.0.118", features = ["derive"] }
+serde_json = { version = "1.0.61", features = ["preserve_order"] }
+serde_qs = "0.8.2"
+sha2 = "0.9.2"
+siphasher = "0.3.3"
 slice-group-by = "0.2.6"
-structopt = "0.3.12"
-tar = "0.4.29"
+structopt = "0.3.21"
+tar = "0.4.30"
 tempfile = "3.1.0"
-tokio = { version = "0.2.18", features = ["macros"] }
-ureq = { version = "0.12.0", features = ["tls"], default-features = false }
+tokio = { version = "0.2", features = ["macros"] }
+ureq = { version = "2.0.0", features = ["tls"], default-features = false }
 walkdir = "2.3.1"
-whoami = "0.8.1"
+whoami = "1.0.3"
 
 [dependencies.sentry]
 version = "0.18.1"
@@ -73,7 +72,7 @@ optional = true
 [dev-dependencies]
 serde_url_params = "0.2.0"
 tempdir = "0.3.7"
-tokio = { version = "0.2.18", features = ["macros", "time"] }
+tokio = { version = "0.2", features = ["macros", "time"] }
 
 [dev-dependencies.assert-json-diff]
 git = "https://github.com/qdequele/assert-json-diff"

meilisearch-http/public/interface.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="/bulma.min.css">
+    <link rel="stylesheet" href="bulma.min.css">
     <title>MeiliSearch</title>
     <style>
       em {
@@ -28,8 +28,6 @@
       }
 
       .document {
-        padding: 20px 20px;
-        background-color: #f5f5f5;
         border-radius: 4px;
         margin-bottom: 20px;
         display: flex;
@@ -40,121 +38,141 @@
         max-width: 75%;
         padding: 0;
         margin: 0;
+        list-style-type: none;
       }
 
+      .document ol li {
+        list-style: none;
+      }
+
+
       .document .image {
-        max-width: 25%;
-        flex: 0 0 25%;
-        padding-left: 30px;
+        max-width: 50%;
+        margin: 0 auto;
         box-sizing: border-box;
       }
 
+      @media screen and (min-width: 770px) {
+        .document .image {
+          max-width: 25%;
+          flex: 0 0 25%;
+          margin: 0;
+          padding-left: 30px;
+          box-sizing: border-box;
+        }
+      }
+
       .document .image img {
         width: 100%;
       }
 
-      .field {
-        list-style-type: none;
-        display: flex;
-        flex-wrap: wrap;
-      }
-
-      .field:not(:last-child) {
-        margin-bottom: 7px;
-      }
-
       .attribute {
-        flex: 0 0 25%;
-        max-width: 25%;
-        text-align: right;
-        padding-right: 10px;
+        text-align: center;
         box-sizing: border-box;
         text-transform: uppercase;
+        font-weight: bold;
         color: rgba(0,0,0,.7);
       }
 
+      @media screen and (min-width: 770px) {
+        .attribute {
+          flex: 0 0 25%;
+          max-width: 25%;
+          text-align: right;
+          padding-right: 10px;
+          font-weight: normal;
+          box-sizing: border-box;
+        }
+      }
+      @media screen and (max-width: 770px) {
+        .attribute {
+          padding-bottom: 0;
+        }
+      }
+
       .content {
-        max-width: 75%;
         flex: 0 0 75%;
         box-sizing: border-box;
-        padding-left: 10px;
         color: rgba(0,0,0,.9);
-        overflow-wrap: break-word;
+        overflow-wrap: anywhere;
+      }
+
+      .hero-foot {
+        padding-bottom: 3rem;
+      }
+
+      @media screen and (max-width: 770px) {
+        .align-on-mobile {
+          text-align: center;
+        }
       }
     </style>
   </head>
   <body>
 
     <section class="hero is-light">
+      
       <div class="hero-body">
         <div class="container">
-          <h1 class="title">
-            Welcome to MeiliSearch
-          </h1>
-          <h2 class="subtitle">
-            This dashboard will help you check the search results with ease.
-          </h2>
-
-          <div class="field">
-            <!-- API Key -->
-            <div class="field">
-              <div class="control">
-                <input id="apiKey" class="input is-small" type="password" placeholder="API key (optional)">
-                <div class="help">At least a private API key is required for the dashboard to access the indexes list.</div>
+          <div class="content is-medium align-on-mobile">
+            <h1 class="title is-1 is-spaced">
+              Welcome to MeiliSearch
+            </h1>
+            <p class="subtitle is-4">
+              This dashboard will help you check the search results with ease.
+            </p>
+          </div>
+          <div class="columns">
+            <div class="column is-4">
+              <div class="field">
+                  <!-- API Key -->
+                    <label class="label" for="apiKey">API Key (optional)</label>
+                    <div class="control">
+                      <input id="apiKey" class="input is-small" type="password" placeholder="Enter your API key">
+                    </div>
+                    <p class="help">At least a private API key is required for the dashboard to access the indexes list.</p>
               </div>
             </div>
           </div>
-
-        </div>
-      </div>
-    </section>
-
-    <section class="hero container">
-        <div class="notification" style="border-radius: 0 0 4px 4px;">
-
-          <nav class="level">
-            <!-- Left side -->
-            <div class="level-left">
-              <div class="level-item">
-                <div class="field has-addons has-addons-right">
-                  <p class="control">
+          <div class="columns">
+            <div class="column is-8">
+                <label class="label" for="search">Search something</label>
+                <div class="field has-addons">
+                  <div class="control">
                     <span class="select">
-                      <select id="index">
+                      <select role="listbox" id="index" aria-label="Select the index you want to search on">
                         <!-- indexes names -->
                       </select>
                     </span>
-                  </p>
-                  <p class="control">
-                    <input id="search" class="input" type="text" autofocus placeholder="e.g. George Clooney">
-                  </p>
+                  </div>
+                  <div class="control is-expanded">
+                    <input id="search" class="input" type="search" autofocus placeholder="e.g. George Clooney" aria-label="Search through your documents">
+                  </div>
                 </div>
-              </div>
             </div>
-
-            <!-- Right side -->
-            <nav class="level-right">
-              <div class="level-item has-text-centered">
-                <div>
+            <div class="column is-4">
+              <div class="columns">
+                <div class="column is-6 has-text-centered">
                   <p class="heading">Documents</p>
                   <p id="count" class="title">0</p>
                 </div>
-              </div>
-              <div class="level-item has-text-centered">
-                <div>
+                <div class="column is-6 has-text-centered">
                   <p class="heading">Time Spent</p>
                   <p id="time" class="title">N/A</p>
                 </div>
               </div>
-            </nav>
-          </nav>
-
+            </div>
+          </div>
         </div>
+      </div>
     </section>
 
     <section>
-      <ol id="results" class="content">
-        <!-- documents matching resquests -->
-      </ol>
+      <div class="container">
+        <ol id="results" class="content">
+          <!-- documents matching resquests -->
+        </ol>
+      </div>
     </section>
   </body>
 
@@ -203,7 +221,7 @@
         if (e.selectedIndex == -1) { return }
         var index = e.options[e.selectedIndex].value;
 
-        let theUrl = `${baseUrl}/indexes/${index}/search?q=${search.value}&attributesToHighlight=*`;
+        let theUrl = `${baseUrl}/indexes/${index}/search?q=${encodeURIComponent(search.value)}&attributesToHighlight=*`;
 
         if (lastRequest) { lastRequest.abort() }
         lastRequest = new XMLHttpRequest();
@@ -230,9 +248,12 @@
                     delete element._formatted;
 
                     const elem = document.createElement('li');
-                    elem.classList.add("document");
+                    elem.classList.add("document","box");
 
-                    const ol = document.createElement('ol');
+                    const div = document.createElement('div');
+                    div.classList.add("columns","is-desktop","is-tablet");
+                    const info = document.createElement('div');
+                    info.classList.add("column","align-on-mobile");
                     let image = undefined;
 
                     for (const prop in element) {
@@ -243,15 +264,16 @@
                             }
                         }
 
-                        const field = document.createElement('li');
-                        field.classList.add("field");
+                        const field = document.createElement('div');
+                        field.classList.add("columns");
 
                         const attribute = document.createElement('div');
-                        attribute.classList.add("attribute");
+                        attribute.classList.add("attribute", "column");
                         attribute.innerHTML = prop;
 
                         const content = document.createElement('div');
-                        content.classList.add("content");
+                        content.classList.add("content", "column");
+                        
                         if (typeof (element[prop]) === "object") {
                           content.innerHTML = JSON.stringify(element[prop]);
                         } else {
@@ -261,19 +283,22 @@
                         field.appendChild(attribute);
                         field.appendChild(content);
 
-                        ol.appendChild(field);
+                        info.appendChild(field);
                     }
 
-                    elem.appendChild(ol);
+                    div.appendChild(info);
+                    elem.appendChild(div);
 
                     if (image != undefined) {
-                        const div = document.createElement('div');
-                        div.classList.add("image");
-
+                        const divImage = document.createElement('div');
+                        divImage.classList.add("image","column","align-on-mobile");
+                        
                         const img = document.createElement('img');
                         img.src = image;
+                        img.setAttribute("alt","Item illustration");
 
-                        div.appendChild(img);
+                        divImage.appendChild(img);
+                        div.appendChild(divImage);
                         elem.appendChild(div);
                     }
 

meilisearch-http/src/analytics.rs

@@ -127,9 +127,14 @@ pub fn analytics_sender(data: Data, opt: Opt) {
 
         let body = qs::to_string(&request).unwrap();
         let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body);
-        if !response.ok() {
-            let body = response.into_string().unwrap();
-            error!("Unsuccessful call to Amplitude: {}", body);
+        match response {
+            Err(ureq::Error::Status(_ , response)) => {
+                error!("Unsuccessful call to Amplitude: {}", response.into_string().unwrap_or_default());
+            }
+            Err(e) => {
+                error!("Unsuccessful call to Amplitude: {}", e);
+            }
+            _ => (),
         }
 
         thread::sleep(Duration::from_secs(3600)) // one hour

meilisearch-http/src/data.rs

@@ -1,13 +1,15 @@
 use std::error::Error;
 use std::ops::Deref;
 use std::path::PathBuf;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
-use meilisearch_core::{Database, DatabaseOptions};
+use meilisearch_core::{Database, DatabaseOptions, Index};
 use sha2::Digest;
 
+use crate::error::{Error as MSError, ResponseError};
 use crate::index_update_callback;
 use crate::option::Opt;
+use crate::dump::DumpInfo;
 
 #[derive(Clone)]
 pub struct Data {
@@ -26,11 +28,12 @@ impl Deref for Data {
 pub struct DataInner {
     pub db: Arc<Database>,
     pub db_path: String,
-    pub dumps_folder: PathBuf,
+    pub dumps_dir: PathBuf,
     pub dump_batch_size: usize,
     pub api_keys: ApiKeys,
     pub server_pid: u32,
     pub http_payload_size_limit: usize,
+    pub current_dump: Arc<Mutex<Option<DumpInfo>>>,
 }
 
 #[derive(Clone)]
@@ -60,7 +63,7 @@ impl ApiKeys {
 impl Data {
     pub fn new(opt: Opt) -> Result<Data, Box<dyn Error>> {
         let db_path = opt.db_path.clone();
-        let dumps_folder = opt.dumps_folder.clone();
+        let dumps_dir = opt.dumps_dir.clone();
         let dump_batch_size = opt.dump_batch_size;
         let server_pid = std::process::id();
 
@@ -81,14 +84,17 @@ impl Data {
 
         api_keys.generate_missing_api_keys();
 
+        let current_dump = Arc::new(Mutex::new(None));
+
         let inner_data = DataInner {
             db: db.clone(),
             db_path,
-            dumps_folder,
+            dumps_dir,
             dump_batch_size,
             api_keys,
             server_pid,
             http_payload_size_limit,
+            current_dump,
         };
 
         let data = Data {
@@ -102,4 +108,68 @@ impl Data {
 
         Ok(data)
     }
+
+    fn create_index(&self, uid: &str) -> Result<Index, ResponseError> {
+        if !uid
+            .chars()
+            .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
+        {
+            return Err(MSError::InvalidIndexUid.into());
+        }
+
+        let created_index = self.db.create_index(&uid).map_err(|e| match e {
+            meilisearch_core::Error::IndexAlreadyExists => e.into(),
+            _ => ResponseError::from(MSError::create_index(e)),
+        })?;
+
+        self.db.main_write::<_, _, ResponseError>(|mut writer| {
+            created_index.main.put_name(&mut writer, uid)?;
+
+            created_index
+                .main
+                .created_at(&writer)?
+                .ok_or(MSError::internal("Impossible to read created at"))?;
+
+            created_index
+                .main
+                .updated_at(&writer)?
+                .ok_or(MSError::internal("Impossible to read updated at"))?;
+            Ok(())
+        })?;
+
+        Ok(created_index)
+    }
+
+    pub fn get_current_dump_info(&self) -> Option<DumpInfo> {
+        self.current_dump.lock().unwrap().clone()
+    }
+
+    pub fn set_current_dump_info(&self, dump_info: DumpInfo) {
+        self.current_dump.lock().unwrap().replace(dump_info);
+    }
+
+    pub fn get_or_create_index<F, R>(&self, uid: &str, f: F) -> Result<R, ResponseError>
+    where
+        F: FnOnce(&Index) -> Result<R, ResponseError>,
+    {
+        let mut index_has_been_created = false;
+
+        let index = match self.db.open_index(&uid) {
+            Some(index) => index,
+            None => {
+                index_has_been_created = true;
+                self.create_index(&uid)?
+            }
+        };
+
+        match f(&index) {
+            Ok(r) => Ok(r),
+            Err(err) => {
+                if index_has_been_created {
+                    let _ = self.db.delete_index(&uid);
+                }
+                Err(err)
+            }
+        }
+    }
 }

meilisearch-http/src/dump.rs

@@ -1,7 +1,6 @@
 use std::fs::{create_dir_all, File};
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
-use std::sync::Mutex;
 use std::thread;
 
 use actix_web::web;
@@ -11,19 +10,16 @@ use log::{error, info};
 use meilisearch_core::{MainWriter, MainReader, UpdateReader};
 use meilisearch_core::settings::Settings;
 use meilisearch_core::update::{apply_settings_update, apply_documents_addition};
-use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
+use serde_json::json;
 use tempfile::TempDir;
 
 use crate::Data;
-use crate::error::Error;
+use crate::error::{Error, ResponseError};
 use crate::helpers::compression;
 use crate::routes::index;
 use crate::routes::index::IndexResponse;
 
-// Mutex to share dump progress.
-static DUMP_INFO: Lazy<Mutex<Option<DumpInfo>>> = Lazy::new(Mutex::default);
-
 #[derive(Debug, Serialize, Deserialize, Copy, Clone)]
 enum DumpVersion {
     V1,
@@ -51,9 +47,9 @@ impl DumpMetadata {
         }
     }
 
-    /// Extract DumpMetadata from `metadata.json` file present at provided `folder_path`
-    fn from_path(folder_path: &Path) -> Result<Self, Error> {
-        let path = folder_path.join("metadata.json");
+    /// Extract DumpMetadata from `metadata.json` file present at provided `dir_path`
+    fn from_path(dir_path: &Path) -> Result<Self, Error> {
+        let path = dir_path.join("metadata.json");
         let file = File::open(path)?;
         let reader = std::io::BufReader::new(file);
         let metadata = serde_json::from_reader(reader)?;
@@ -61,9 +57,9 @@ impl DumpMetadata {
         Ok(metadata)
     }
 
-    /// Write DumpMetadata in `metadata.json` file at provided `folder_path`
-    fn to_path(&self, folder_path: &Path) -> Result<(), Error> {
-        let path = folder_path.join("metadata.json");
+    /// Write DumpMetadata in `metadata.json` file at provided `dir_path`
+    fn to_path(&self, dir_path: &Path) -> Result<(), Error> {
+        let path = dir_path.join("metadata.json");
         let file = File::create(path)?;
 
         serde_json::to_writer(file, &self)?;
@@ -72,9 +68,9 @@ impl DumpMetadata {
     }
 }
 
-/// Extract Settings from `settings.json` file present at provided `folder_path`
-fn settings_from_path(folder_path: &Path) -> Result<Settings, Error> {
-    let path = folder_path.join("settings.json");
+/// Extract Settings from `settings.json` file present at provided `dir_path`
+fn settings_from_path(dir_path: &Path) -> Result<Settings, Error> {
+    let path = dir_path.join("settings.json");
     let file = File::open(path)?;
     let reader = std::io::BufReader::new(file);
     let metadata = serde_json::from_reader(reader)?;
@@ -82,9 +78,9 @@ fn settings_from_path(folder_path: &Path) -> Result<Settings, Error> {
     Ok(metadata)
 }
 
-/// Write Settings in `settings.json` file at provided `folder_path`
-fn settings_to_path(settings: &Settings, folder_path: &Path) -> Result<(), Error> {
-    let path = folder_path.join("settings.json");
+/// Write Settings in `settings.json` file at provided `dir_path`
+fn settings_to_path(settings: &Settings, dir_path: &Path) -> Result<(), Error> {
+    let path = dir_path.join("settings.json");
     let file = File::create(path)?;
 
     serde_json::to_writer(file, settings)?;
@@ -95,7 +91,7 @@ fn settings_to_path(settings: &Settings, folder_path: &Path) -> Result<(), Error
 /// Import settings and documents of a dump with version `DumpVersion::V1` in specified index.
 fn import_index_v1(
     data: &Data,
-    dumps_folder: &Path,
+    dumps_dir: &Path,
     index_uid: &str,
     document_batch_size: usize,
     write_txn: &mut MainWriter,
@@ -107,12 +103,12 @@ fn import_index_v1(
         .open_index(index_uid)
         .ok_or(Error::index_not_found(index_uid))?;
 
-    // index folder path in  dump folder
-    let index_path = &dumps_folder.join(index_uid);
+    // index dir path in  dump dir
+    let index_path = &dumps_dir.join(index_uid);
 
     // extract `settings.json` file and import content
     let settings = settings_from_path(&index_path)?;
-    let settings = settings.to_update().or_else(|_e| Err(Error::dump_failed()))?;
+    let settings = settings.to_update().map_err(|e| Error::dump_failed(format!("importing settings for index {}; {}", index_uid, e)))?;
     apply_settings_update(write_txn, &index, settings)?;
 
     // create iterator over documents in `documents.jsonl` to make batch importation
@@ -199,17 +195,18 @@ pub fn import_dump(
 #[serde(rename_all = "snake_case")]
 pub enum DumpStatus {
     Done,
-    Processing,
-    DumpProcessFailed,
+    InProgress,
+    Failed,
 }
 
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct DumpInfo {
     pub uid: String,
     pub status: DumpStatus,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub error: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none", flatten)]
+    pub error: Option<serde_json::Value>,
+
 }
 
 impl DumpInfo {
@@ -217,23 +214,15 @@ impl DumpInfo {
         Self { uid, status, error: None }
     }
 
-    pub fn with_error(mut self, error: String) -> Self {
-        self.status = DumpStatus::DumpProcessFailed;
-        self.error = Some(error);
+    pub fn with_error(mut self, error: ResponseError) -> Self {
+        self.status = DumpStatus::Failed;
+        self.error = Some(json!(error));
 
         self
     }
 
     pub fn dump_already_in_progress(&self) -> bool {
-        self.status == DumpStatus::Processing
-    }
-
-    pub fn get_current() -> Option<Self> {
-        DUMP_INFO.lock().unwrap().clone()
-    }
-
-    pub fn set_current(&self) {
-        *DUMP_INFO.lock().unwrap() = Some(self.clone());
+        self.status == DumpStatus::InProgress
     }
 }
 
@@ -242,29 +231,29 @@ fn generate_uid() -> String {
     Utc::now().format("%Y%m%d-%H%M%S%3f").to_string()
 }
 
-/// Infer dumps_folder from dump_uid
-pub fn compressed_dumps_folder(dumps_folder: &Path, dump_uid: &str) -> PathBuf {
-    dumps_folder.join(format!("{}.tar.gz", dump_uid))
+/// Infer dumps_dir from dump_uid
+pub fn compressed_dumps_dir(dumps_dir: &Path, dump_uid: &str) -> PathBuf {
+    dumps_dir.join(format!("{}.dump", dump_uid))
 }
 
 /// Write metadata in dump
-fn dump_metadata(data: &web::Data<Data>, folder_path: &Path, indexes: Vec<IndexResponse>) -> Result<(), Error> {
+fn dump_metadata(data: &web::Data<Data>, dir_path: &Path, indexes: Vec<IndexResponse>) -> Result<(), Error> {
     let (db_major, db_minor, db_patch) = data.db.version();
     let metadata = DumpMetadata::new(indexes, format!("{}.{}.{}", db_major, db_minor, db_patch));
 
-    metadata.to_path(folder_path)
+    metadata.to_path(dir_path)
 }
 
 /// Export settings of provided index in dump
-fn dump_index_settings(data: &web::Data<Data>, reader: &MainReader, folder_path: &Path, index_uid: &str) -> Result<(), Error> {
+fn dump_index_settings(data: &web::Data<Data>, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> {
     let settings = crate::routes::setting::get_all_sync(data, reader, index_uid)?;
 
-    settings_to_path(&settings, folder_path)
+    settings_to_path(&settings, dir_path)
 }
 
 /// Export updates of provided index in dump
-fn dump_index_updates(data: &web::Data<Data>, reader: &UpdateReader, folder_path: &Path, index_uid: &str) -> Result<(), Error> {
-    let updates_path = folder_path.join("updates.jsonl");
+fn dump_index_updates(data: &web::Data<Data>, reader: &UpdateReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> {
+    let updates_path = dir_path.join("updates.jsonl");
     let updates = crate::routes::index::get_all_updates_status_sync(data, reader, index_uid)?;
 
     let file = File::create(updates_path)?;
@@ -278,15 +267,15 @@ fn dump_index_updates(data: &web::Data<Data>, reader: &UpdateReader, folder_path
 }
 
 /// Export documents of provided index in dump
-fn dump_index_documents(data: &web::Data<Data>, reader: &MainReader, folder_path: &Path, index_uid: &str) -> Result<(), Error> {
-    let documents_path = folder_path.join("documents.jsonl");
+fn dump_index_documents(data: &web::Data<Data>, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> {
+    let documents_path = dir_path.join("documents.jsonl");
     let file = File::create(documents_path)?;
     let dump_batch_size = data.dump_batch_size;
 
     let mut offset = 0;
     loop {
         let documents = crate::routes::document::get_all_documents_sync(data, reader, index_uid, offset, dump_batch_size, None)?;
-        if documents.len() == 0 { break; } else { offset += dump_batch_size; }
+        if documents.is_empty() { break; } else { offset += dump_batch_size; }
 
         for document in documents {
             serde_json::to_writer(&file, &document)?;
@@ -298,20 +287,20 @@ fn dump_index_documents(data: &web::Data<Data>, reader: &MainReader, folder_path
 }
 
 /// Write error with a context.
-fn fail_dump_process<E: std::error::Error>(dump_info: DumpInfo, context: &str, error: E) {
-        let error = format!("Something went wrong during dump process: {}; {}", context, error);
+fn fail_dump_process<E: std::error::Error>(data: &web::Data<Data>, dump_info: DumpInfo, context: &str, error: E) {
+        let error_message = format!("{}; {}", context, error);
         
-        error!("{}", &error);
-        dump_info.with_error(error).set_current();
+        error!("Something went wrong during dump process: {}", &error_message);
+        data.set_current_dump_info(dump_info.with_error(Error::dump_failed(error_message).into()))
 }
 
 /// Main function of dump.
-fn dump_process(data: web::Data<Data>, dumps_folder: PathBuf, dump_info: DumpInfo) {
+fn dump_process(data: web::Data<Data>, dumps_dir: PathBuf, dump_info: DumpInfo) {
     // open read transaction on Update
     let update_reader = match data.db.update_read_txn() {
         Ok(r) => r,
         Err(e) => {
-            fail_dump_process(dump_info, "creating RO transaction on updates", e);
+            fail_dump_process(&data, dump_info, "creating RO transaction on updates", e);
             return ;
         }
     };
@@ -320,7 +309,7 @@ fn dump_process(data: web::Data<Data>, dumps_folder: PathBuf, dump_info: DumpInf
     let main_reader = match data.db.main_read_txn() {
         Ok(r) => r,
         Err(e) => {
-            fail_dump_process(dump_info, "creating RO transaction on main", e);
+            fail_dump_process(&data, dump_info, "creating RO transaction on main", e);
             return ;
         }
     };
@@ -329,7 +318,7 @@ fn dump_process(data: web::Data<Data>, dumps_folder: PathBuf, dump_info: DumpInf
     let tmp_dir = match TempDir::new() {
         Ok(tmp_dir) => tmp_dir,
         Err(e) => {
-            fail_dump_process(dump_info, "creating temporary directory", e);
+            fail_dump_process(&data, dump_info, "creating temporary directory", e);
             return ;
         }
     };
@@ -339,14 +328,14 @@ fn dump_process(data: web::Data<Data>, dumps_folder: PathBuf, dump_info: DumpInf
     let indexes = match crate::routes::index::list_indexes_sync(&data, &main_reader) {
         Ok(indexes) => indexes,
         Err(e) => {
-            fail_dump_process(dump_info, "listing indexes", e);
+            fail_dump_process(&data, dump_info, "listing indexes", e);
             return ;
         }
     };
 
     // create metadata
     if let Err(e) = dump_metadata(&data, &tmp_dir_path, indexes.clone()) {
-        fail_dump_process(dump_info, "generating metadata", e);
+        fail_dump_process(&data, dump_info, "generating metadata", e);
         return ;
     }
 
@@ -356,32 +345,32 @@ fn dump_process(data: web::Data<Data>, dumps_folder: PathBuf, dump_info: DumpInf
 
         // create index sub-dircetory
         if let Err(e) = create_dir_all(&index_path) {
-            fail_dump_process(dump_info, &format!("creating directory for index {}", &index.uid), e);
+            fail_dump_process(&data, dump_info, &format!("creating directory for index {}", &index.uid), e);
             return ;
         }
 
         // export settings
         if let Err(e) = dump_index_settings(&data, &main_reader, &index_path, &index.uid) {
-            fail_dump_process(dump_info, &format!("generating settings for index {}", &index.uid), e);
+            fail_dump_process(&data, dump_info, &format!("generating settings for index {}", &index.uid), e);
             return ;
         }
 
         // export documents
         if let Err(e) = dump_index_documents(&data, &main_reader, &index_path, &index.uid) {
-            fail_dump_process(dump_info, &format!("generating documents for index {}", &index.uid), e);
+            fail_dump_process(&data, dump_info, &format!("generating documents for index {}", &index.uid), e);
             return ;
         }
 
         // export updates
         if let Err(e) = dump_index_updates(&data, &update_reader, &index_path, &index.uid) {
-            fail_dump_process(dump_info, &format!("generating updates for index {}", &index.uid), e);
+            fail_dump_process(&data, dump_info, &format!("generating updates for index {}", &index.uid), e);
             return ;
         }
     }
 
-    // compress dump in a file named `{dump_uid}.tar.gz` in `dumps_folder`
-    if let Err(e) = crate::helpers::compression::to_tar_gz(&tmp_dir_path, &compressed_dumps_folder(&dumps_folder, &dump_info.uid)) {
-        fail_dump_process(dump_info, "compressing dump", e);
+    // compress dump in a file named `{dump_uid}.dump` in `dumps_dir`
+    if let Err(e) = crate::helpers::compression::to_tar_gz(&tmp_dir_path, &compressed_dumps_dir(&dumps_dir, &dump_info.uid)) {
+        fail_dump_process(&data, dump_info, "compressing dump", e);
         return ;
     }
 
@@ -391,14 +380,14 @@ fn dump_process(data: web::Data<Data>, dumps_folder: PathBuf, dump_info: DumpInf
         DumpStatus::Done
     );
 
-    resume.set_current();
+    data.set_current_dump_info(resume);
 }
 
-pub fn init_dump_process(data: &web::Data<Data>, dumps_folder: &Path) -> Result<DumpInfo, Error> {
-    create_dir_all(dumps_folder).or(Err(Error::dump_failed()))?;
+pub fn init_dump_process(data: &web::Data<Data>, dumps_dir: &Path) -> Result<DumpInfo, Error> {
+    create_dir_all(dumps_dir).map_err(|e| Error::dump_failed(format!("creating temporary directory {}", e)))?;
 
     // check if a dump is already in progress
-    if let Some(resume) = DumpInfo::get_current() {
+    if let Some(resume) = data.get_current_dump_info() {
         if resume.dump_already_in_progress() {
             return Err(Error::dump_conflict())
         }
@@ -407,17 +396,17 @@ pub fn init_dump_process(data: &web::Data<Data>, dumps_folder: &Path) -> Result<
     // generate a new dump info
     let info = DumpInfo::new(
         generate_uid(),
-        DumpStatus::Processing
+        DumpStatus::InProgress
     );
 
-    info.set_current();
+    data.set_current_dump_info(info.clone());
 
     let data = data.clone();
-    let dumps_folder = dumps_folder.to_path_buf();
+    let dumps_dir = dumps_dir.to_path_buf();
     let info_cloned = info.clone();
     // run dump process in a new thread
     thread::spawn(move || 
-        dump_process(data, dumps_folder, info_cloned)
+        dump_process(data, dumps_dir, info_cloned)
     );
 
     Ok(info)

meilisearch-http/src/error.rs

@@ -5,7 +5,7 @@ use actix_http::ResponseBuilder;
 use actix_web as aweb;
 use actix_web::error::{JsonPayloadError, QueryPayloadError};
 use actix_web::http::StatusCode;
-use serde_json::json;
+use serde::ser::{Serialize, Serializer, SerializeStruct};
 
 use meilisearch_error::{ErrorCode, Code};
 
@@ -34,6 +34,51 @@ impl From<Error> for ResponseError {
     }
 }
 
+impl From<meilisearch_core::Error> for ResponseError {
+    fn from(err: meilisearch_core::Error) -> ResponseError {
+        ResponseError { inner: Box::new(err) }
+    }
+}
+
+impl From<meilisearch_schema::Error> for ResponseError {
+    fn from(err: meilisearch_schema::Error) -> ResponseError {
+        ResponseError { inner: Box::new(err) }
+    }
+}
+
+impl From<FacetCountError> for ResponseError {
+    fn from(err: FacetCountError) -> ResponseError {
+        ResponseError { inner: Box::new(err) }
+    }
+}
+
+impl Serialize for ResponseError {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let struct_name = "ResponseError";
+        let field_count = 4;
+
+        let mut state = serializer.serialize_struct(struct_name, field_count)?;
+        state.serialize_field("message", &self.to_string())?;
+        state.serialize_field("errorCode", &self.error_name())?;
+        state.serialize_field("errorType", &self.error_type())?;
+        state.serialize_field("errorLink", &self.error_url())?;
+        state.end()
+    }
+}
+
+impl aweb::error::ResponseError for ResponseError {
+    fn error_response(&self) -> aweb::HttpResponse {
+        ResponseBuilder::new(self.status_code()).json(&self)
+    }
+
+    fn status_code(&self) -> StatusCode {
+        self.http_status()
+    }
+}
+
 #[derive(Debug)]
 pub enum Error {
     BadParameter(String, String),
@@ -45,7 +90,6 @@ pub enum Error {
     Internal(String),
     InvalidIndexUid,
     InvalidToken(String),
-    Maintenance,
     MissingAuthorizationHeader,
     NotFound(String),
     OpenIndex(String),
@@ -54,7 +98,7 @@ pub enum Error {
     PayloadTooLarge,
     UnsupportedMediaType,
     DumpAlreadyInProgress,
-    DumpProcessFailed,
+    DumpProcessFailed(String),
 }
 
 impl error::Error for Error {}
@@ -72,7 +116,6 @@ impl ErrorCode for Error {
             Internal(_) => Code::Internal,
             InvalidIndexUid => Code::InvalidIndexUid,
             InvalidToken(_) => Code::InvalidToken,
-            Maintenance => Code::Maintenance,
             MissingAuthorizationHeader => Code::MissingAuthorizationHeader,
             NotFound(_) => Code::NotFound,
             OpenIndex(_) => Code::OpenIndex,
@@ -81,7 +124,7 @@ impl ErrorCode for Error {
             PayloadTooLarge => Code::PayloadTooLarge,
             UnsupportedMediaType => Code::UnsupportedMediaType,
             DumpAlreadyInProgress => Code::DumpAlreadyInProgress,
-            DumpProcessFailed => Code::DumpProcessFailed,
+            DumpProcessFailed(_) => Code::DumpProcessFailed,
         }
     }
 }
@@ -173,10 +216,6 @@ impl Error {
         Error::InvalidIndexUid
     }
 
-    pub fn maintenance() -> Error {
-        Error::Maintenance
-    }
-
     pub fn retrieve_document(doc_id: u32, err: impl fmt::Display) -> Error {
         Error::RetrieveDocument(doc_id, err.to_string())
     }
@@ -189,8 +228,8 @@ impl Error {
         Error::DumpAlreadyInProgress
     }
 
-    pub fn dump_failed() -> Error {
-        Error::DumpProcessFailed
+    pub fn dump_failed(message: String) -> Error {
+        Error::DumpProcessFailed(message)
     }
 }
 
@@ -206,7 +245,6 @@ impl fmt::Display for Error {
             Self::Internal(err) => f.write_str(err),
             Self::InvalidIndexUid => f.write_str("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."),
             Self::InvalidToken(err) => write!(f, "Invalid API key: {}", err),
-            Self::Maintenance => f.write_str("Server is in maintenance, please try again later"),
             Self::MissingAuthorizationHeader => f.write_str("You must have an authorization token"),
             Self::NotFound(err) => write!(f, "{} not found", err),
             Self::OpenIndex(err) => write!(f, "Impossible to open index; {}", err),
@@ -215,44 +253,17 @@ impl fmt::Display for Error {
             Self::PayloadTooLarge => f.write_str("Payload too large"),
             Self::UnsupportedMediaType => f.write_str("Unsupported media type"),
             Self::DumpAlreadyInProgress => f.write_str("Another dump is already in progress"),
-            Self::DumpProcessFailed => f.write_str("Dump process failed"),
+            Self::DumpProcessFailed(message) => write!(f, "Dump process failed: {}", message),
         }
     }
 }
 
-impl aweb::error::ResponseError for ResponseError {
-    fn error_response(&self) -> aweb::HttpResponse {
-        ResponseBuilder::new(self.status_code()).json(json!({
-            "message": self.to_string(),
-            "errorCode": self.error_name(),
-            "errorType": self.error_type(),
-            "errorLink": self.error_url(),
-        }))
-    }
-
-    fn status_code(&self) -> StatusCode {
-        self.http_status()
-    }
-}
-
 impl From<std::io::Error> for Error {
     fn from(err: std::io::Error) -> Error {
         Error::Internal(err.to_string())
     }
 }
 
-impl From<meilisearch_core::Error> for ResponseError {
-    fn from(err: meilisearch_core::Error) -> ResponseError {
-        ResponseError { inner: Box::new(err) }
-    }
-}
-
-impl From<meilisearch_schema::Error> for ResponseError {
-    fn from(err: meilisearch_schema::Error) -> ResponseError {
-        ResponseError { inner: Box::new(err) }
-    }
-}
-
 impl From<actix_http::Error> for Error {
     fn from(err: actix_http::Error) -> Error {
         Error::Internal(err.to_string())
@@ -271,12 +282,6 @@ impl From<serde_json::error::Error> for Error {
     }
 }
 
-impl From<FacetCountError> for ResponseError {
-    fn from(err: FacetCountError) -> ResponseError {
-        ResponseError { inner: Box::new(err) }
-    }
-}
-
 impl From<JsonPayloadError> for Error {
     fn from(err: JsonPayloadError) -> Error {
         match err {

meilisearch-http/src/helpers/authentication.rs

@@ -6,6 +6,8 @@ use std::task::{Context, Poll};
 use actix_service::{Service, Transform};
 use actix_web::{dev::ServiceRequest, dev::ServiceResponse, web};
 use futures::future::{err, ok, Future, Ready};
+use actix_web::error::ResponseError as _;
+use actix_web::dev::Body;
 
 use crate::error::{Error, ResponseError};
 use crate::Data;
@@ -17,14 +19,13 @@ pub enum Authentication {
     Admin,
 }
 
-impl<S: 'static, B> Transform<S> for Authentication
+impl<S: 'static> Transform<S> for Authentication
 where
-    S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = actix_web::Error>,
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<Body>, Error = actix_web::Error>,
     S::Future: 'static,
-    B: 'static,
 {
     type Request = ServiceRequest;
-    type Response = ServiceResponse<B>;
+    type Response = ServiceResponse<Body>;
     type Error = actix_web::Error;
     type InitError = ();
     type Transform = LoggingMiddleware<S>;
@@ -44,14 +45,13 @@ pub struct LoggingMiddleware<S> {
 }
 
 #[allow(clippy::type_complexity)]
-impl<S, B> Service for LoggingMiddleware<S>
+impl<S> Service for LoggingMiddleware<S>
 where
-    S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = actix_web::Error> + 'static,
+    S: Service<Request = ServiceRequest, Response = ServiceResponse<Body>, Error = actix_web::Error> + 'static,
     S::Future: 'static,
-    B: 'static,
 {
     type Request = ServiceRequest;
-    type Response = ServiceResponse<B>;
+    type Response = ServiceResponse<Body>;
     type Error = actix_web::Error;
     type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>>>>;
 
@@ -72,7 +72,11 @@ where
         let auth_header = match req.headers().get("X-Meili-API-Key") {
             Some(auth) => match auth.to_str() {
                 Ok(auth) => auth,
-                Err(_) => return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into())),
+                Err(_) => {
+                    let error = ResponseError::from(Error::MissingAuthorizationHeader).error_response();
+                    let (request, _) = req.into_parts();
+                    return Box::pin(ok(ServiceResponse::new(request, error)))
+                }
             },
             None => {
                 return Box::pin(err(ResponseError::from(Error::MissingAuthorizationHeader).into()));
@@ -95,9 +99,9 @@ where
         if authenticated {
             Box::pin(svc.call(req))
         } else {
-            Box::pin(err(
-                ResponseError::from(Error::InvalidToken(auth_header.to_string())).into()
-            ))
+            let error = ResponseError::from(Error::InvalidToken(auth_header.to_string())).error_response();
+            let (request, _) = req.into_parts();
+            return Box::pin(ok(ServiceResponse::new(request, error)))
         }
     }
 }

meilisearch-http/src/helpers/meilisearch.rs

@@ -11,7 +11,6 @@ use meilisearch_core::criterion::*;
 use meilisearch_core::settings::RankingRule;
 use meilisearch_core::{Highlight, Index, RankedMap};
 use meilisearch_schema::{FieldId, Schema};
-use meilisearch_tokenizer::is_cjk;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use siphasher::sip::SipHasher;
@@ -178,7 +177,7 @@ impl<'a> SearchBuilder<'a> {
                 all_attributes.extend(&all_formatted);
             },
             None => {
-                all_attributes.extend(schema.displayed_name());
+                all_attributes.extend(schema.displayed_names());
                 // If we specified at least one attribute to highlight or crop then
                 // all available attributes will be returned in the _formatted field.
                 if self.attributes_to_highlight.is_some() || self.attributes_to_crop.is_some() {
@@ -193,9 +192,7 @@ impl<'a> SearchBuilder<'a> {
                 .index
                 .document(reader, Some(&all_attributes), doc.id)
                 .map_err(|e| Error::retrieve_document(doc.id.0, e))?
-                .ok_or(Error::internal(
-                    "Impossible to retrieve the document; Corrupted data",
-                ))?;
+                .unwrap_or_default();
 
             let mut formatted = document.iter()
                 .filter(|(key, _)| all_formatted.contains(key.as_str()))
@@ -293,12 +290,18 @@ impl<'a> SearchBuilder<'a> {
     }
 }
 
-#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Serialize, Deserialize)]
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
 pub struct MatchPosition {
     pub start: usize,
     pub length: usize,
 }
 
+impl PartialOrd for MatchPosition {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 impl Ord for MatchPosition {
     fn cmp(&self, other: &Self) -> Ordering {
         match self.start.cmp(&other.start) {
@@ -340,7 +343,7 @@ pub struct SearchResult {
 
 /// returns the start index and the length on the crop.
 fn aligned_crop(text: &str, match_index: usize, context: usize) -> (usize, usize) {
-    let is_word_component = |c: &char| c.is_alphanumeric() && !is_cjk(*c);
+    let is_word_component = |c: &char| c.is_alphanumeric() && !super::is_cjk(*c);
 
     let word_end_index = |mut index| {
         if text.chars().nth(index - 1).map_or(false, |c| is_word_component(&c)) {
@@ -441,7 +444,7 @@ fn calculate_matches(
                     continue;
                 }
             }
-            if !schema.displayed_name().contains(attribute) {
+            if !schema.displayed_names().contains(&attribute) {
                 continue;
             }
             if let Some(pos) = matches_result.get_mut(attribute) {
@@ -476,7 +479,7 @@ fn calculate_highlights(
     for (attribute, matches) in matches.iter() {
         if attributes_to_highlight.contains(attribute) {
             if let Some(Value::String(value)) = document.get(attribute) {
-                let value: Vec<_> = value.chars().collect();
+                let value = value;
                 let mut highlighted_value = String::new();
                 let mut index = 0;
 
@@ -489,16 +492,16 @@ fn calculate_highlights(
                     let before = value.get(index..m.start);
                     let highlighted = value.get(m.start..(m.start + m.length));
                     if let (Some(before), Some(highlighted)) = (before, highlighted) {
-                        highlighted_value.extend(before);
+                        highlighted_value.push_str(before);
                         highlighted_value.push_str("<em>");
-                        highlighted_value.extend(highlighted);
+                        highlighted_value.push_str(highlighted);
                         highlighted_value.push_str("</em>");
                         index = m.start + m.length;
                     } else {
                         error!("value: {:?}; index: {:?}, match: {:?}", value, index, m);
                     }
                 }
-                highlighted_value.extend(value[index..].iter());
+                highlighted_value.push_str(&value[index..]);
                 highlight_result.insert(attribute.to_string(), Value::String(highlighted_value));
             };
         }
@@ -594,7 +597,7 @@ mod tests {
 
         let mut m = Vec::new();
         m.push(MatchPosition {
-            start: 510,
+            start: 529,
             length: 9,
         });
         matches.insert("description".to_string(), m);

meilisearch-http/src/helpers/mod.rs

@@ -5,3 +5,22 @@ pub mod compression;
 
 pub use authentication::Authentication;
 pub use normalize_path::NormalizePath;
+
+pub fn is_cjk(c: char) -> bool {
+    ('\u{1100}'..'\u{11ff}').contains(&c)  // Hangul Jamo
+        || ('\u{2e80}'..'\u{2eff}').contains(&c)  // CJK Radicals Supplement
+        || ('\u{2f00}'..'\u{2fdf}').contains(&c) // Kangxi radical
+        || ('\u{3000}'..'\u{303f}').contains(&c) // Japanese-style punctuation
+        || ('\u{3040}'..'\u{309f}').contains(&c) // Japanese Hiragana
+        || ('\u{30a0}'..'\u{30ff}').contains(&c) // Japanese Katakana
+        || ('\u{3100}'..'\u{312f}').contains(&c)
+        || ('\u{3130}'..'\u{318F}').contains(&c) // Hangul Compatibility Jamo
+        || ('\u{3200}'..'\u{32ff}').contains(&c) // Enclosed CJK Letters and Months
+        || ('\u{3400}'..'\u{4dbf}').contains(&c) // CJK Unified Ideographs Extension A
+        || ('\u{4e00}'..'\u{9fff}').contains(&c) // CJK Unified Ideographs
+        || ('\u{a960}'..'\u{a97f}').contains(&c) // Hangul Jamo Extended-A
+        || ('\u{ac00}'..'\u{d7a3}').contains(&c) // Hangul Syllables
+        || ('\u{d7b0}'..'\u{d7ff}').contains(&c) // Hangul Jamo Extended-B
+        || ('\u{f900}'..'\u{faff}').contains(&c) // CJK Compatibility Ideographs
+        || ('\u{ff00}'..'\u{ffef}').contains(&c) // Full-width roman characters and half-width katakana
+}

meilisearch-http/src/lib.rs

@@ -24,6 +24,7 @@ use self::error::{payload_error_handler, ResponseError};
 
 pub fn create_app(
     data: &Data,
+    enable_frontend: bool,
 ) -> App<
     impl ServiceFactory<
         Config = (),
@@ -34,7 +35,7 @@ pub fn create_app(
     >,
     actix_http::body::Body,
 > {
-    App::new()
+    let app = App::new()
         .data(data.clone())
         .app_data(
             web::JsonConfig::default()
@@ -46,8 +47,6 @@ pub fn create_app(
             web::QueryConfig::default()
             .error_handler(|err, _req| payload_error_handler(err).into())
         )
-        .service(routes::load_html)
-        .service(routes::load_css)
         .configure(routes::document::services)
         .configure(routes::index::services)
         .configure(routes::search::services)
@@ -57,7 +56,14 @@ pub fn create_app(
         .configure(routes::health::services)
         .configure(routes::stats::services)
         .configure(routes::key::services)
-        .configure(routes::dump::services)
+        .configure(routes::dump::services);
+    if enable_frontend {
+        app
+            .service(routes::load_html)
+            .service(routes::load_css)
+    } else {
+        app
+    }
 }
 
 pub fn index_update_callback_txn(index: Index, index_uid: &str, data: &Data, mut writer: &mut MainWriter) -> Result<(), String> {

meilisearch-http/src/main.rs

@@ -47,12 +47,12 @@ async fn main() -> Result<(), MainError> {
             }
         }
         "development" => {
-            env_logger::from_env(env_logger::Env::default().default_filter_or("info")).init();
+            env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
         }
         _ => unreachable!(),
     }
 
-    if let Some(path) = &opt.load_from_snapshot {
+    if let Some(path) = &opt.import_snapshot {
         snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?;
     }
 
@@ -74,21 +74,23 @@ async fn main() -> Result<(), MainError> {
         dump::import_dump(&data, path, opt.dump_batch_size)?;
     }
 
-    if let Some(path) = &opt.snapshot_path {
-        snapshot::schedule_snapshot(data.clone(), &path, opt.snapshot_interval_sec.unwrap_or(86400))?;
+    if opt.schedule_snapshot {
+        snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?;
     }
 
     print_launch_resume(&opt, &data);
 
+    let enable_frontend = opt.env != "production";
     let http_server = HttpServer::new(move || {
-        create_app(&data)
-            .wrap(
-                Cors::new()
+        let cors = Cors::default()
                     .send_wildcard()
                     .allowed_headers(vec!["content-type", "x-meili-api-key"])
-                    .max_age(86_400) // 24h
-                    .finish(),
-            )
+                    .allow_any_origin()
+                    .allow_any_method()
+                    .max_age(86_400); // 24h
+
+        create_app(&data, enable_frontend)
+            .wrap(cors)
             .wrap(middleware::Logger::default())
             .wrap(middleware::Compress::default())
             .wrap(NormalizePath)

meilisearch-http/src/option.rs

@@ -57,7 +57,7 @@ pub struct Opt {
     pub max_udb_size: usize,
 
     /// The maximum size, in bytes, of accepted JSON payloads
-    #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "10485760")] // 10MB
+    #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "104857600")] // 100MB
     pub http_payload_size_limit: usize,
 
     /// Read server certificates from CERTFILE.
@@ -97,31 +97,35 @@ pub struct Opt {
     /// Defines the path of the snapshot file to import.
     /// This option will, by default, stop the process if a database already exist or if no snapshot exists at
     /// the given path. If this option is not specified no snapshot is imported.
-    #[structopt(long, env = "MEILI_LOAD_FROM_SNAPSHOT")]
-    pub load_from_snapshot: Option<PathBuf>,
+    #[structopt(long)]
+    pub import_snapshot: Option<PathBuf>,
 
     /// The engine will ignore a missing snapshot and not return an error in such case.
-    #[structopt(long, requires = "load-from-snapshot", env = "MEILI_IGNORE_MISSING_SNAPSHOT")]
+    #[structopt(long, requires = "import-snapshot")]
     pub ignore_missing_snapshot: bool,
 
     /// The engine will skip snapshot importation and not return an error in such case.
-    #[structopt(long, requires = "load-from-snapshot", env = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS")]
+    #[structopt(long, requires = "import-snapshot")]
     pub ignore_snapshot_if_db_exists: bool,
 
     /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap.
-    #[structopt(long, env = "MEILI_SNAPSHOT_PATH")]
-    pub snapshot_path: Option<PathBuf>,
+    #[structopt(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")]
+    pub snapshot_dir: PathBuf,
+
+    /// Activate snapshot scheduling.
+    #[structopt(long, env = "MEILI_SCHEDULE_SNAPSHOT")]
+    pub schedule_snapshot: bool,
 
     /// Defines time interval, in seconds, between each snapshot creation.
-    #[structopt(long, requires = "snapshot-path", env = "MEILI_SNAPSHOT_INTERVAL_SEC")]
+    #[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC")]
     pub snapshot_interval_sec: Option<u64>,
 
     /// Folder where dumps are created when the dump route is called.
-    #[structopt(long, env = "MEILI_DUMPS_FOLDER", default_value = "dumps/")]
-    pub dumps_folder: PathBuf,
+    #[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")]
+    pub dumps_dir: PathBuf,
 
     /// Import a dump from the specified path, must be a `.tar.gz` file.
-    #[structopt(long, env = "MEILI_IMPORT_DUMP", conflicts_with = "load-from-snapshot")]
+    #[structopt(long, conflicts_with = "import-snapshot")]
     pub import_dump: Option<PathBuf>,
 
     /// The batch size used in the importation process, the bigger it is the faster the dump is created.

meilisearch-http/src/routes/document.rs

@@ -45,7 +45,8 @@ async fn get_document(
 
     let reader = data.db.main_read_txn()?;
 
-    let internal_id = index.main
+    let internal_id = index
+        .main
         .external_to_internal_docid(&reader, &path.document_id)?
         .ok_or(Error::document_not_found(&path.document_id))?;
 
@@ -166,47 +167,41 @@ async fn update_multiple_documents(
     body: web::Json<Vec<Document>>,
     is_partial: bool,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let reader = data.db.main_read_txn()?;
 
-    let reader = data.db.main_read_txn()?;
+        let mut schema = index
+            .main
+            .schema(&reader)?
+            .ok_or(meilisearch_core::Error::SchemaMissing)?;
 
-    let mut schema = index
-        .main
-        .schema(&reader)?
-        .ok_or(meilisearch_core::Error::SchemaMissing)?;
+        if schema.primary_key().is_none() {
+            let id = match &params.primary_key {
+                Some(id) => id.to_string(),
+                None => body
+                    .first()
+                    .and_then(find_primary_key)
+                    .ok_or(meilisearch_core::Error::MissingPrimaryKey)?,
+            };
 
-    if schema.primary_key().is_none() {
-        let id = match &params.primary_key {
-            Some(id) => id.to_string(),
-            None => body
-                .first()
-                .and_then(find_primary_key)
-                .ok_or(meilisearch_core::Error::MissingPrimaryKey)?
+            schema.set_primary_key(&id).map_err(Error::bad_request)?;
+
+            data.db.main_write(|w| index.main.put_schema(w, &schema))?;
+        }
+
+        let mut document_addition = if is_partial {
+            index.documents_partial_addition()
+        } else {
+            index.documents_addition()
         };
 
-        schema
-            .set_primary_key(&id)
-            .map_err(Error::bad_request)?;
+        for document in body.into_inner() {
+            document_addition.update_document(document);
+        }
 
-        data.db.main_write(|w| index.main.put_schema(w, &schema))?;
-    }
-
-    let mut document_addition = if is_partial {
-        index.documents_partial_addition()
-    } else {
-        index.documents_addition()
-    };
-
-    for document in body.into_inner() {
-        document_addition.update_document(document);
-    }
-
-    let update_id = data.db.update_write(|w| document_addition.finalize(w))?;
-
-    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
+        Ok(data.db.update_write(|w| document_addition.finalize(w))?)
+    })?;
+    return Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)));
 }
 
 #[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")]
@@ -243,7 +238,6 @@ async fn delete_documents(
         .open_index(&path.index_uid)
         .ok_or(Error::index_not_found(&path.index_uid))?;
 
-
     let mut documents_deletion = index.documents_deletion();
 
     for document_id in body.into_inner() {

meilisearch-http/src/routes/dump.rs

@@ -5,7 +5,7 @@ use actix_web::{get, post};
 use actix_web::{HttpResponse, web};
 use serde::{Deserialize, Serialize};
 
-use crate::dump::{DumpInfo, DumpStatus, compressed_dumps_folder, init_dump_process};
+use crate::dump::{DumpInfo, DumpStatus, compressed_dumps_dir, init_dump_process};
 use crate::Data;
 use crate::error::{Error, ResponseError};
 use crate::helpers::Authentication;
@@ -19,8 +19,8 @@ pub fn services(cfg: &mut web::ServiceConfig) {
 async fn trigger_dump(
     data: web::Data<Data>,
 ) -> Result<HttpResponse, ResponseError> {
-    let dumps_folder = Path::new(&data.dumps_folder);
-    match init_dump_process(&data, &dumps_folder) {
+    let dumps_dir = Path::new(&data.dumps_dir);
+    match init_dump_process(&data, &dumps_dir) {
         Ok(resume) => Ok(HttpResponse::Accepted().json(resume)),
         Err(e) => Err(e.into())
     }
@@ -42,16 +42,16 @@ async fn get_dump_status(
     data: web::Data<Data>,
     path: web::Path<DumpParam>,
 ) -> Result<HttpResponse, ResponseError> {
-    let dumps_folder = Path::new(&data.dumps_folder);
+    let dumps_dir = Path::new(&data.dumps_dir);
     let dump_uid = &path.dump_uid;
 
-    if let Some(resume) = DumpInfo::get_current() {
+    if let Some(resume) = data.get_current_dump_info() {
         if &resume.uid == dump_uid {
             return Ok(HttpResponse::Ok().json(resume));
         }
     }
 
-    if File::open(compressed_dumps_folder(Path::new(dumps_folder), dump_uid)).is_ok() {
+    if File::open(compressed_dumps_dir(Path::new(dumps_dir), dump_uid)).is_ok() {
         let resume = DumpInfo::new(
             dump_uid.into(),
             DumpStatus::Done

meilisearch-http/src/routes/health.rs

@@ -1,47 +1,13 @@
+use actix_web::get;
 use actix_web::{web, HttpResponse};
-use actix_web::{get, put};
-use serde::Deserialize;
 
-use crate::error::{Error, ResponseError};
-use crate::helpers::Authentication;
-use crate::Data;
+use crate::error::ResponseError;
 
 pub fn services(cfg: &mut web::ServiceConfig) {
-    cfg.service(get_health).service(change_healthyness);
+    cfg.service(get_health);
 }
 
 #[get("/health")]
-async fn get_health(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
-    let reader = data.db.main_read_txn()?;
-    if let Ok(Some(_)) = data.db.get_health(&reader) {
-        return Err(Error::Maintenance.into());
-    }
-    Ok(HttpResponse::Ok().finish())
-}
-
-async fn set_healthy(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
-    data.db.main_write(|w| data.db.set_healthy(w))?;
-    Ok(HttpResponse::Ok().finish())
-}
-
-async fn set_unhealthy(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
-    data.db.main_write(|w| data.db.set_unhealthy(w))?;
-    Ok(HttpResponse::Ok().finish())
-}
-
-#[derive(Deserialize, Clone)]
-struct HealthBody {
-    health: bool,
-}
-
-#[put("/health", wrap = "Authentication::Private")]
-async fn change_healthyness(
-    data: web::Data<Data>,
-    body: web::Json<HealthBody>,
-) -> Result<HttpResponse, ResponseError> {
-    if body.health {
-        set_healthy(data).await
-    } else {
-        set_unhealthy(data).await
-    }
+async fn get_health() -> Result<HttpResponse, ResponseError> {
+    Ok(HttpResponse::NoContent().finish())
 }

meilisearch-http/src/routes/search.rs

@@ -1,4 +1,4 @@
-use std::collections::{HashMap, HashSet};
+use std::collections::{HashMap, HashSet, BTreeSet};
 
 use actix_web::{get, post, web, HttpResponse};
 use log::warn;
@@ -120,8 +120,8 @@ impl SearchQuery {
             search_builder.limit(limit);
         }
 
-        let available_attributes = schema.displayed_name();
-        let mut restricted_attributes: HashSet<&str>;
+        let available_attributes = schema.displayed_names();
+        let mut restricted_attributes: BTreeSet<&str>;
         match &self.attributes_to_retrieve {
             Some(attributes_to_retrieve) => {
                 let attributes_to_retrieve: HashSet<&str> =
@@ -129,13 +129,14 @@ impl SearchQuery {
                 if attributes_to_retrieve.contains("*") {
                     restricted_attributes = available_attributes.clone();
                 } else {
-                    restricted_attributes = HashSet::new();
+                    restricted_attributes = BTreeSet::new();
+                    search_builder.attributes_to_retrieve(HashSet::new());
                     for attr in attributes_to_retrieve {
                         if available_attributes.contains(attr) {
                             restricted_attributes.insert(attr);
                             search_builder.add_retrievable_field(attr.to_string());
                         } else {
-                            warn!("The attributes {:?} present in attributesToCrop parameter doesn't exist", attr);
+                            warn!("The attributes {:?} present in attributesToRetrieve parameter doesn't exist", attr);
                         }
                     }
                 }

meilisearch-http/src/routes/setting.rs

@@ -1,4 +1,4 @@
-use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::collections::{BTreeMap, BTreeSet};
 
 use actix_web::{delete, get, post};
 use actix_web::{web, HttpResponse};
@@ -53,13 +53,12 @@ async fn update_all(
     path: web::Path<IndexParam>,
     body: web::Json<Settings>,
 ) -> Result<HttpResponse, ResponseError> {
-    let settings = body
-        .into_inner()
-        .to_update()
-        .map_err(Error::bad_request)?;
-
-    let update_id = data.db.update_write::<_, _, Error>(|writer| {
-        update_all_settings_txn(&data, settings, &path.index_uid, writer)
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        Ok(data.db.update_write::<_, _, ResponseError>(|writer| {
+            let settings = body.into_inner().to_update().map_err(Error::bad_request)?;
+            let update_id = index.settings_update(writer, settings)?;
+            Ok(update_id)
+        })?)
     })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
@@ -71,11 +70,7 @@ pub fn get_all_sync(data: &web::Data<Data>, reader: &MainReader, index_uid: &str
         .open_index(index_uid)
         .ok_or(Error::index_not_found(index_uid))?;
 
-    let stop_words: BTreeSet<String> = index
-        .main
-        .stop_words(reader)?
-        .into_iter()
-        .collect();
+    let stop_words: BTreeSet<String> = index.main.stop_words(&reader)?.into_iter().collect();
 
     let synonyms_list = index.main.synonyms(reader)?;
 
@@ -94,22 +89,19 @@ pub fn get_all_sync(data: &web::Data<Data>, reader: &MainReader, index_uid: &str
         .map(|r| r.to_string())
         .collect();
 
-
-    let schema = index.main.schema(reader)?;
+    let schema = index.main.schema(&reader)?;
 
     let distinct_attribute = match (index.main.distinct_attribute(reader)?, &schema) {
         (Some(id), Some(schema)) => schema.name(id).map(str::to_string),
         _ => None,
     };
 
-    let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(reader)?) {
-        (Some(schema), Some(attrs)) => {
-            attrs
-                .iter()
-                .filter_map(|&id| schema.name(id))
-                .map(str::to_string)
-                .collect()
-        }
+    let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(&reader)?) {
+        (Some(schema), Some(attrs)) => attrs
+            .iter()
+            .filter_map(|&id| schema.name(id))
+            .map(str::to_string)
+            .collect(),
         _ => vec![],
     };
 
@@ -159,7 +151,9 @@ async fn delete_all(
         attributes_for_faceting: UpdateState::Clear,
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -198,18 +192,17 @@ async fn update_rules(
     path: web::Path<IndexParam>,
     body: web::Json<Option<Vec<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            ranking_rules: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        ranking_rules: Some(body.into_inner()),
-        ..Settings::default()
-    };
-
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        let settings = settings.to_update().map_err(Error::bad_request)?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -232,7 +225,9 @@ async fn delete_rules(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -269,18 +264,17 @@ async fn update_distinct(
     path: web::Path<IndexParam>,
     body: web::Json<Option<String>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            distinct_attribute: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        distinct_attribute: Some(body.into_inner()),
-        ..Settings::default()
-    };
-
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        let settings = settings.to_update().map_err(Error::bad_request)?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -303,7 +297,9 @@ async fn delete_distinct(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -322,8 +318,7 @@ async fn get_searchable(
         .ok_or(Error::index_not_found(&path.index_uid))?;
     let reader = data.db.main_read_txn()?;
     let schema = index.main.schema(&reader)?;
-    let searchable_attributes: Option<Vec<String>> =
-        schema.as_ref().map(get_indexed_attributes);
+    let searchable_attributes: Option<Vec<String>> = schema.as_ref().map(get_indexed_attributes);
 
     Ok(HttpResponse::Ok().json(searchable_attributes))
 }
@@ -337,19 +332,18 @@ async fn update_searchable(
     path: web::Path<IndexParam>,
     body: web::Json<Option<Vec<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            searchable_attributes: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        searchable_attributes: Some(body.into_inner()),
-        ..Settings::default()
-    };
+        let settings = settings.to_update().map_err(Error::bad_request)?;
 
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -372,7 +366,9 @@ async fn delete_searchable(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -405,20 +401,19 @@ async fn get_displayed(
 async fn update_displayed(
     data: web::Data<Data>,
     path: web::Path<IndexParam>,
-    body: web::Json<Option<HashSet<String>>>,
+    body: web::Json<Option<BTreeSet<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            displayed_attributes: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        displayed_attributes: Some(body.into_inner()),
-        ..Settings::default()
-    };
-
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        let settings = settings.to_update().map_err(Error::bad_request)?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -441,7 +436,9 @@ async fn delete_displayed(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -459,20 +456,16 @@ async fn get_attributes_for_faceting(
         .open_index(&path.index_uid)
         .ok_or(Error::index_not_found(&path.index_uid))?;
 
-    let attributes_for_faceting = data
-        .db
-        .main_read::<_, _, ResponseError>(|reader| {
+    let attributes_for_faceting = data.db.main_read::<_, _, ResponseError>(|reader| {
         let schema = index.main.schema(reader)?;
         let attrs = index.main.attributes_for_faceting(reader)?;
         let attr_names = match (&schema, &attrs) {
-            (Some(schema), Some(attrs)) => {
-                attrs
-                    .iter()
-                    .filter_map(|&id| schema.name(id))
-                    .map(str::to_string)
-                    .collect()
-            }
-            _ => vec![]
+            (Some(schema), Some(attrs)) => attrs
+                .iter()
+                .filter_map(|&id| schema.name(id))
+                .map(str::to_string)
+                .collect(),
+            _ => vec![],
         };
         Ok(attr_names)
     })?;
@@ -489,18 +482,17 @@ async fn update_attributes_for_faceting(
     path: web::Path<IndexParam>,
     body: web::Json<Option<Vec<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = Settings {
+            attributes_for_faceting: Some(body.into_inner()),
+            ..Settings::default()
+        };
 
-    let settings = Settings {
-        attributes_for_faceting: Some(body.into_inner()),
-        ..Settings::default()
-    };
-
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        let settings = settings.to_update().map_err(Error::bad_request)?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -523,27 +515,31 @@ async fn delete_attributes_for_faceting(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
 fn get_indexed_attributes(schema: &Schema) -> Vec<String> {
-    if schema.is_indexed_all() {
-        ["*"].iter().map(|s| s.to_string()).collect()
+    if schema.is_searchable_all() {
+        vec!["*".to_string()]
     } else {
-        schema.indexed_name()
+        schema
+            .searchable_names()
             .iter()
             .map(|s| s.to_string())
             .collect()
     }
 }
 
-fn get_displayed_attributes(schema: &Schema) -> HashSet<String> {
+fn get_displayed_attributes(schema: &Schema) -> BTreeSet<String> {
     if schema.is_displayed_all() {
         ["*"].iter().map(|s| s.to_string()).collect()
     } else {
-        schema.displayed_name()
+        schema
+            .displayed_names()
             .iter()
             .map(|s| s.to_string())
             .collect()

meilisearch-http/src/routes/stats.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, BTreeMap};
 
 use actix_web::web;
 use actix_web::HttpResponse;
@@ -24,7 +24,7 @@ pub fn services(cfg: &mut web::ServiceConfig) {
 struct IndexStatsResponse {
     number_of_documents: u64,
     is_indexing: bool,
-    fields_distribution: HashMap<String, usize>,
+    fields_distribution: BTreeMap<String, usize>,
 }
 
 #[get("/indexes/{index_uid}/stats", wrap = "Authentication::Private")]

meilisearch-http/src/routes/stop_words.rs

@@ -39,17 +39,16 @@ async fn update(
     path: web::Path<IndexParam>,
     body: web::Json<BTreeSet<String>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = SettingsUpdate {
+            stop_words: UpdateState::Update(body.into_inner()),
+            ..SettingsUpdate::default()
+        };
 
-    let settings = SettingsUpdate {
-        stop_words: UpdateState::Update(body.into_inner()),
-        ..SettingsUpdate::default()
-    };
-
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -72,7 +71,9 @@ async fn delete(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }

meilisearch-http/src/routes/synonym.rs

@@ -50,17 +50,16 @@ async fn update(
     path: web::Path<IndexParam>,
     body: web::Json<BTreeMap<String, Vec<String>>>,
 ) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
+    let update_id = data.get_or_create_index(&path.index_uid, |index| {
+        let settings = SettingsUpdate {
+            synonyms: UpdateState::Update(body.into_inner()),
+            ..SettingsUpdate::default()
+        };
 
-    let settings = SettingsUpdate {
-        synonyms: UpdateState::Update(body.into_inner()),
-        ..SettingsUpdate::default()
-    };
-
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+        Ok(data
+            .db
+            .update_write(|w| index.settings_update(w, settings))?)
+    })?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
@@ -83,7 +82,9 @@ async fn delete(
         ..SettingsUpdate::default()
     };
 
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
+    let update_id = data
+        .db
+        .update_write(|w| index.settings_update(w, settings))?;
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }

meilisearch-http/src/snapshot.rs

@@ -20,9 +20,9 @@ pub fn load_snapshot(
     if !db_path.exists() && snapshot_path.exists() {
         compression::from_tar_gz(snapshot_path, db_path)
     } else if db_path.exists() && !ignore_snapshot_if_db_exists {
-        Err(Error::Internal(format!("database already exists at {:?}", db_path)))
+        Err(Error::Internal(format!("database already exists at {:?}, try to delete it or rename it", db_path.canonicalize().unwrap_or(db_path.into()))))
     } else if !snapshot_path.exists() && !ignore_missing_snapshot {
-        Err(Error::Internal(format!("snapshot doesn't exist at {:?}", snapshot_path)))
+        Err(Error::Internal(format!("snapshot doesn't exist at {:?}", snapshot_path.canonicalize().unwrap_or(snapshot_path.into()))))
     } else {
         Ok(())
     }
@@ -33,7 +33,7 @@ pub fn create_snapshot(data: &Data, snapshot_path: &Path) -> Result<(), Error> {
 
     data.db.copy_and_compact_to_path(tmp_dir.path())?;
 
-    compression::to_tar_gz(tmp_dir.path(), snapshot_path).or_else(|e| Err(Error::Internal(format!("something went wrong during snapshot compression: {}", e))))
+    compression::to_tar_gz(tmp_dir.path(), snapshot_path).map_err(|e| Error::Internal(format!("something went wrong during snapshot compression: {}", e)))
 }
 
 pub fn schedule_snapshot(data: Data, snapshot_dir: &Path, time_gap_s: u64) -> Result<(), Error> {
@@ -42,13 +42,13 @@ pub fn schedule_snapshot(data: Data, snapshot_dir: &Path, time_gap_s: u64) -> Re
     }
     let db_name = Path::new(&data.db_path).file_name().ok_or_else(|| Error::Internal("invalid database name".to_string()))?;
     create_dir_all(snapshot_dir)?;
-    let snapshot_path = snapshot_dir.join(format!("{}.tar.gz", db_name.to_str().unwrap_or("data.ms")));
+    let snapshot_path = snapshot_dir.join(format!("{}.snapshot", db_name.to_str().unwrap_or("data.ms")));
     
     thread::spawn(move || loop { 
-        thread::sleep(Duration::from_secs(time_gap_s));
         if let Err(e) = create_snapshot(&data, &snapshot_path) {
             error!("Unsuccessful snapshot creation: {}", e);
         }
+        thread::sleep(Duration::from_secs(time_gap_s));
     });
 
     Ok(())
@@ -67,13 +67,13 @@ mod tests {
         let test_dir = tempdir.path();
         let src_dir = test_dir.join("src");
         let dest_dir = test_dir.join("complex/destination/path/");
-        let archive_path = test_dir.join("archive.tar.gz");
+        let archive_path = test_dir.join("archive.snapshot");
 
         let file_1_relative = Path::new("file1.txt");
-        let subfolder_relative = Path::new("subfolder/");
-        let file_2_relative = Path::new("subfolder/file2.txt");
+        let subdir_relative = Path::new("subdir/");
+        let file_2_relative = Path::new("subdir/file2.txt");
         
-        create_dir_all(src_dir.join(subfolder_relative)).unwrap();
+        create_dir_all(src_dir.join(subdir_relative)).unwrap();
         fs::File::create(src_dir.join(file_1_relative)).unwrap().write_all(b"Hello_file_1").unwrap();
         fs::File::create(src_dir.join(file_2_relative)).unwrap().write_all(b"Hello_file_2").unwrap();
 
@@ -84,7 +84,7 @@ mod tests {
 
         assert!(dest_dir.exists());
         assert!(dest_dir.join(file_1_relative).exists());
-        assert!(dest_dir.join(subfolder_relative).exists());
+        assert!(dest_dir.join(subdir_relative).exists());
         assert!(dest_dir.join(file_2_relative).exists());
 
         let contents = fs::read_to_string(dest_dir.join(file_1_relative)).unwrap();

meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl

@@ -74,4 +74,4 @@
 {"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]}
 {"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]}
 {"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]}
-{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]}
+{"id":77,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Red","name":"孫武","gender":"male","email":"SunTzu@chorizon.com","phone":"+1 (810) 407-3258","address":"吴國","about":"孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]}

meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl

@@ -1,2 +1,3 @@
-{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}}
-{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}}
+{"status":"processed","updateId":0,"type":{"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":"Nothing","displayed_attributes":"Nothing","stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}}
+{"status":"processed","updateId":1,"type":{"name":"DocumentsAddition","number":77}}
+

meilisearch-http/tests/assets/test_set.json

@@ -1590,18 +1590,18 @@
     "tags": []
   },
   {
-    "id": 76,
+    "id": 77,
     "isActive": false,
     "balance": "$1,274.29",
     "picture": "http://placehold.it/32x32",
     "age": 25,
-    "color": "Green",
-    "name": "Clarice Gardner",
-    "gender": "female",
-    "email": "claricegardner@chorizon.com",
+    "color": "Red",
+    "name": "孫武",
+    "gender": "male",
+    "email": "SunTzu@chorizon.com",
     "phone": "+1 (810) 407-3258",
-    "address": "894 Brooklyn Road, Utting, New Hampshire, 6404",
-    "about": "Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n",
+    "address": "吴國",
+    "about": "孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n",
     "registered": "2014-10-20T10:13:32 -02:00",
     "latitude": 17.11935,
     "longitude": 65.38197,

meilisearch-http/tests/common.rs

@@ -15,15 +15,24 @@ use meilisearch_http::option::Opt;
 #[macro_export]
 macro_rules! test_post_get_search {
     ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => {
-        let post_query: meilisearch_http::routes::search::SearchQueryPost = serde_json::from_str(&$query.clone().to_string()).unwrap();
+        let post_query: meilisearch_http::routes::search::SearchQueryPost =
+            serde_json::from_str(&$query.clone().to_string()).unwrap();
         let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into();
         let get_query = ::serde_url_params::to_string(&get_query).unwrap();
         let ($response, $status_code) = $server.search_get(&get_query).await;
-        let _ =::std::panic::catch_unwind(|| $block)
-            .map_err(|e| panic!("panic in get route: {:?}", e.downcast_ref::<&str>().unwrap()));
+        let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| {
+            panic!(
+                "panic in get route: {:?}",
+                e.downcast_ref::<&str>().unwrap()
+            )
+        });
         let ($response, $status_code) = $server.search_post($query).await;
-        let _ = ::std::panic::catch_unwind(|| $block)
-            .map_err(|e| panic!("panic in post route: {:?}", e.downcast_ref::<&str>().unwrap()));
+        let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| {
+            panic!(
+                "panic in post route: {:?}",
+                e.downcast_ref::<&str>().unwrap()
+            )
+        });
     };
 }
 
@@ -40,7 +49,7 @@ impl Server {
 
         let opt = Opt {
             db_path: tmp_dir.path().join("db").to_str().unwrap().to_string(),
-            dumps_folder: tmp_dir.path().join("dump"),
+            dumps_dir: tmp_dir.path().join("dump"),
             dump_batch_size: 16,
             http_addr: "127.0.0.1:7700".to_owned(),
             master_key: None,
@@ -48,11 +57,11 @@ impl Server {
             no_analytics: true,
             max_mdb_size: default_db_options.main_map_size,
             max_udb_size: default_db_options.update_map_size,
-            http_payload_size_limit: 10000000,
+            http_payload_size_limit: 100000000,
             ..Opt::default()
         };
 
-        let data = Data::new(opt.clone()).unwrap();
+        let data = Data::new(opt).unwrap();
 
         Server {
             uid: uid.to_string(),
@@ -61,7 +70,6 @@ impl Server {
     }
 
     pub async fn test_server() -> Self {
-
         let mut server = Self::with_uid("test");
 
         let body = json!({
@@ -80,40 +88,6 @@ impl Server {
                 "wordsPosition",
                 "exactness",
             ],
-            "searchableAttributes": [
-                "balance",
-                "picture",
-                "age",
-                "color",
-                "name",
-                "gender",
-                "email",
-                "phone",
-                "address",
-                "about",
-                "registered",
-                "latitude",
-                "longitude",
-                "tags",
-            ],
-            "displayedAttributes": [
-                "id",
-                "isActive",
-                "balance",
-                "picture",
-                "age",
-                "color",
-                "name",
-                "gender",
-                "email",
-                "phone",
-                "address",
-                "about",
-                "registered",
-                "latitude",
-                "longitude",
-                "tags",
-            ],
         });
 
         server.update_all_settings(body).await;
@@ -151,11 +125,12 @@ impl Server {
     pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) {
         eprintln!("get_request: {}", url);
 
-        let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizePath)).await;
+        let mut app =
+            test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await;
 
         let req = test::TestRequest::get().uri(url).to_request();
         let res = test::call_service(&mut app, req).await;
-        let status_code = res.status().clone();
+        let status_code = res.status();
 
         let body = test::read_body(res).await;
         let response = serde_json::from_slice(&body).unwrap_or_default();
@@ -165,14 +140,15 @@ impl Server {
     pub async fn post_request(&self, url: &str, body: Value) -> (Value, StatusCode) {
         eprintln!("post_request: {}", url);
 
-        let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizePath)).await;
+        let mut app =
+            test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await;
 
         let req = test::TestRequest::post()
             .uri(url)
             .set_json(&body)
             .to_request();
         let res = test::call_service(&mut app, req).await;
-        let status_code = res.status().clone();
+        let status_code = res.status();
 
         let body = test::read_body(res).await;
         let response = serde_json::from_slice(&body).unwrap_or_default();
@@ -183,8 +159,7 @@ impl Server {
         eprintln!("post_request_async: {}", url);
 
         let (response, status_code) = self.post_request(url, body).await;
-        // eprintln!("response: {}", response);
-        assert_eq!(status_code, 202);
+        eprintln!("response: {}", response);
         assert!(response["updateId"].as_u64().is_some());
         self.wait_update_id(response["updateId"].as_u64().unwrap())
             .await;
@@ -194,14 +169,15 @@ impl Server {
     pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) {
         eprintln!("put_request: {}", url);
 
-        let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizePath)).await;
+        let mut app =
+            test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await;
 
         let req = test::TestRequest::put()
             .uri(url)
             .set_json(&body)
             .to_request();
         let res = test::call_service(&mut app, req).await;
-        let status_code = res.status().clone();
+        let status_code = res.status();
 
         let body = test::read_body(res).await;
         let response = serde_json::from_slice(&body).unwrap_or_default();
@@ -222,11 +198,12 @@ impl Server {
     pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) {
         eprintln!("delete_request: {}", url);
 
-        let mut app = test::init_service(meilisearch_http::create_app(&self.data).wrap(NormalizePath)).await;
+        let mut app =
+            test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await;
 
         let req = test::TestRequest::delete().uri(url).to_request();
         let res = test::call_service(&mut app, req).await;
-        let status_code = res.status().clone();
+        let status_code = res.status();
 
         let body = test::read_body(res).await;
         let response = serde_json::from_slice(&body).unwrap_or_default();
@@ -340,9 +317,9 @@ impl Server {
         self.delete_request_async(&url).await
     }
 
-    pub async fn delete_multiple_documents(&mut self, body: Value) {
+    pub async fn delete_multiple_documents(&mut self, body: Value) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/documents/delete-batch", self.uid);
-        self.post_request_async(&url, body).await;
+        self.post_request_async(&url, body).await
     }
 
     pub async fn get_all_settings(&mut self) -> (Value, StatusCode) {
@@ -355,6 +332,11 @@ impl Server {
         self.post_request_async(&url, body).await;
     }
 
+    pub async fn update_all_settings_sync(&mut self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", self.uid);
+        self.post_request(&url, body).await
+    }
+
     pub async fn delete_all_settings(&mut self) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/settings", self.uid);
         self.delete_request_async(&url).await
@@ -390,6 +372,11 @@ impl Server {
         self.post_request_async(&url, body).await;
     }
 
+    pub async fn update_distinct_attribute_sync(&mut self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/distinct-attribute", self.uid);
+        self.post_request(&url, body).await
+    }
+
     pub async fn delete_distinct_attribute(&mut self) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/settings/distinct-attribute", self.uid);
         self.delete_request_async(&url).await
@@ -410,6 +397,11 @@ impl Server {
         self.post_request_async(&url, body).await;
     }
 
+    pub async fn update_searchable_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/searchable-attributes", self.uid);
+        self.post_request(&url, body).await
+    }
+
     pub async fn delete_searchable_attributes(&mut self) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/settings/searchable-attributes", self.uid);
         self.delete_request_async(&url).await
@@ -425,11 +417,39 @@ impl Server {
         self.post_request_async(&url, body).await;
     }
 
+    pub async fn update_displayed_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/displayed-attributes", self.uid);
+        self.post_request(&url, body).await
+    }
+
     pub async fn delete_displayed_attributes(&mut self) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/settings/displayed-attributes", self.uid);
         self.delete_request_async(&url).await
     }
 
+    pub async fn get_attributes_for_faceting(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid);
+        self.get_request(&url).await
+    }
+
+    pub async fn update_attributes_for_faceting(&mut self, body: Value) {
+        let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid);
+        self.post_request_async(&url, body).await;
+    }
+
+    pub async fn update_attributes_for_faceting_sync(
+        &mut self,
+        body: Value,
+    ) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid);
+        self.post_request(&url, body).await
+    }
+
+    pub async fn delete_attributes_for_faceting(&mut self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid);
+        self.delete_request_async(&url).await
+    }
+
     pub async fn get_synonyms(&mut self) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/settings/synonyms", self.uid);
         self.get_request(&url).await
@@ -440,6 +460,11 @@ impl Server {
         self.post_request_async(&url, body).await;
     }
 
+    pub async fn update_synonyms_sync(&mut self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/synonyms", self.uid);
+        self.post_request(&url, body).await
+    }
+
     pub async fn delete_synonyms(&mut self) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/settings/synonyms", self.uid);
         self.delete_request_async(&url).await
@@ -455,6 +480,11 @@ impl Server {
         self.post_request_async(&url, body).await;
     }
 
+    pub async fn update_stop_words_sync(&mut self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings/stop-words", self.uid);
+        self.post_request(&url, body).await
+    }
+
     pub async fn delete_stop_words(&mut self) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/settings/stop-words", self.uid);
         self.delete_request_async(&url).await

meilisearch-http/tests/documents_add.rs

@@ -192,7 +192,9 @@ async fn add_document_with_long_field() {
         "url":"/configuration/app/web.html#locations"
     }]);
     server.add_or_replace_multiple_documents(body).await;
-    let (response, _status) = server.search_post(json!({ "q": "request_buffering" })).await;
+    let (response, _status) = server
+        .search_post(json!({ "q": "request_buffering" }))
+        .await;
     assert!(!response["hits"].as_array().unwrap().is_empty());
 }
 
@@ -213,5 +215,8 @@ async fn documents_with_same_id_are_overwritten() {
     server.add_or_replace_multiple_documents(documents).await;
     let (response, _status) = server.get_all_documents().await;
     assert_eq!(response.as_array().unwrap().len(), 1);
-    assert_eq!(response.as_array().unwrap()[0].as_object().unwrap()["content"], "test2");
+    assert_eq!(
+        response.as_array().unwrap()[0].as_object().unwrap()["content"],
+        "test2"
+    );
 }

meilisearch-http/tests/documents_delete.rs

@@ -1,5 +1,7 @@
 mod common;
 
+use serde_json::json;
+
 #[actix_rt::test]
 async fn delete() {
     let mut server = common::Server::test_server().await;
@@ -32,3 +34,34 @@ async fn delete_batch() {
         assert_eq!(status_code, 404);
     }
 }
+
+#[actix_rt::test]
+async fn text_clear_all_placeholder_search() {
+    let mut server = common::Server::with_uid("test");
+    let body = json!({
+        "uid": "test",
+    });
+
+    server.create_index(body).await;
+    let settings = json!({
+        "attributesForFaceting": ["genre"],
+    });
+
+    server.update_all_settings(settings).await;
+
+    let documents = json!([
+        { "id": 2,    "title": "Pride and Prejudice",                    "author": "Jane Austin",              "genre": "romance" },
+        { "id": 456,  "title": "Le Petit Prince",                        "author": "Antoine de Saint-Exupéry", "genre": "adventure" },
+        { "id": 1,    "title": "Alice In Wonderland",                    "author": "Lewis Carroll",            "genre": "fantasy" },
+        { "id": 1344, "title": "The Hobbit",                             "author": "J. R. R. Tolkien",         "genre": "fantasy" },
+        { "id": 4,    "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling",            "genre": "fantasy" },
+        { "id": 42,   "title": "The Hitchhiker's Guide to the Galaxy",   "author": "Douglas Adams" }
+    ]);
+
+    server.add_or_update_multiple_documents(documents).await;
+    server.clear_all_documents().await;
+    let (response, _) = server.search_post(json!({ "q": "", "facetsDistribution": ["genre"] })).await;
+    assert_eq!(response["nbHits"], 0);
+    let (response, _) = server.search_post(json!({ "q": "" })).await;
+    assert_eq!(response["nbHits"], 0);
+}

meilisearch-http/tests/dump.rs

@@ -42,14 +42,12 @@ fn current_dump_version() -> String {
 }
 
 fn read_all_jsonline<R: std::io::Read>(r: R) -> Value {
-    let deserializer = serde_json::Deserializer::from_reader(r);
-    let iterator = deserializer.into_iter::<serde_json::Value>();
+    let deserializer = serde_json::Deserializer::from_reader(r); let iterator = deserializer.into_iter::<serde_json::Value>();
 
     json!(iterator.map(|v| v.unwrap()).collect::<Vec<Value>>())
 }
 
 #[actix_rt::test]
-#[ignore]
 async fn trigger_dump_should_return_ok() {
     let server = common::Server::test_server().await;
 
@@ -59,7 +57,6 @@ async fn trigger_dump_should_return_ok() {
 }
 
 #[actix_rt::test]
-#[ignore]
 async fn trigger_dump_twice_should_return_conflict() {
     let server = common::Server::test_server().await;
 
@@ -77,12 +74,11 @@ async fn trigger_dump_twice_should_return_conflict() {
     let (value, status_code) = server.trigger_dump().await;
 
     
-    assert_json_eq!(expected.clone(), value.clone(), ordered: false);
+    assert_json_eq!(expected, value, ordered: false);
     assert_eq!(status_code, 409);
 }
 
 #[actix_rt::test]
-#[ignore]
 async fn trigger_dump_concurently_should_return_conflict() {
     let server = common::Server::test_server().await;
 
@@ -95,13 +91,12 @@ async fn trigger_dump_concurently_should_return_conflict() {
 
     let ((_value_1, _status_code_1), (value_2, status_code_2)) = futures::join!(server.trigger_dump(), server.trigger_dump());
     
-    assert_json_eq!(expected.clone(), value_2.clone(), ordered: false);
+    assert_json_eq!(expected, value_2, ordered: false);
     assert_eq!(status_code_2, 409);
 }
 
 #[actix_rt::test]
-#[ignore]
-async fn get_dump_status_early_should_return_processing() {
+async fn get_dump_status_early_should_return_in_progress() {
     let mut server = common::Server::test_server().await;
     
 
@@ -116,16 +111,15 @@ async fn get_dump_status_early_should_return_processing() {
 
     let expected = json!({
         "uid": dump_uid,
-        "status": "processing"
+        "status": "in_progress"
     });
 
     assert_eq!(status_code, 200);
 
-    assert_json_eq!(expected.clone(), value.clone(), ordered: false);
+    assert_json_eq!(expected, value, ordered: false);
 }
 
 #[actix_rt::test]
-#[ignore]
 async fn get_dump_status_should_return_done() {
     let mut server = common::Server::test_server().await;
 
@@ -147,11 +141,42 @@ async fn get_dump_status_should_return_done() {
 
     assert_eq!(status_code, 200);
 
-    assert_json_eq!(expected.clone(), value.clone(), ordered: false);
+    assert_json_eq!(expected, value, ordered: false);
+}
+
+#[actix_rt::test]
+async fn get_dump_status_should_return_error_provoking_it() {
+    let mut server = common::Server::test_server().await;
+
+
+    let (value, status_code) = server.trigger_dump().await;
+
+    // removing destination directory provoking `No such file or directory` error
+    std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap();
+
+    assert_eq!(status_code, 202);
+
+    let dump_uid = value["uid"].as_str().unwrap().to_string();
+    
+    let expected = json!({
+        "uid": dump_uid.clone(),
+        "status": "failed",
+        "message": "Dump process failed: compressing dump; No such file or directory (os error 2)",
+        "errorCode": "dump_process_failed",
+        "errorType": "internal_error",
+        "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed"
+    });
+
+    thread::sleep(Duration::from_secs(1)); // wait dump until process end
+
+    let (value, status_code) = server.get_dump_status(&dump_uid).await;
+
+    assert_eq!(status_code, 200);
+
+    assert_json_eq!(expected, value, ordered: false);
 }
 
 #[actix_rt::test]
-#[ignore]
 async fn dump_metadata_should_be_valid() {
     let mut server = common::Server::test_server().await;
     
@@ -164,11 +189,11 @@ async fn dump_metadata_should_be_valid() {
     
     let uid = trigger_and_wait_dump(&mut server).await;
 
-    let dumps_folder = Path::new(&server.data().dumps_folder);
+    let dumps_dir = Path::new(&server.data().dumps_dir);
     let tmp_dir = TempDir::new().unwrap();
     let tmp_dir_path = tmp_dir.path();
 
-    compression::from_tar_gz(&dumps_folder.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap();
+    compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap();
 
     let file = File::open(tmp_dir_path.join("metadata.json")).unwrap();
     let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap();
@@ -195,24 +220,22 @@ async fn dump_metadata_should_be_valid() {
         "dumpVersion": current_dump_version()
     });
 
-    assert_json_include!(expected: expected.clone(), actual: metadata.clone());
+    assert_json_include!(expected: expected, actual: metadata);
 }
 
 #[actix_rt::test]
-#[ignore]
 async fn dump_gzip_should_have_been_created() {
     let mut server = common::Server::test_server().await;
     
 
     let dump_uid = trigger_and_wait_dump(&mut server).await;
-    let dumps_folder = Path::new(&server.data().dumps_folder);
+    let dumps_dir = Path::new(&server.data().dumps_dir);
 
-    let compressed_path = dumps_folder.join(format!("{}.tar.gz", dump_uid));
+    let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid));
     assert!(File::open(compressed_path).is_ok());
 }
 
 #[actix_rt::test]
-#[ignore]
 async fn dump_index_settings_should_be_valid() {
     let mut server = common::Server::test_server().await;
 
@@ -279,20 +302,19 @@ async fn dump_index_settings_should_be_valid() {
 
     let uid = trigger_and_wait_dump(&mut server).await;
 
-    let dumps_folder = Path::new(&server.data().dumps_folder);
+    let dumps_dir = Path::new(&server.data().dumps_dir);
     let tmp_dir = TempDir::new().unwrap();
     let tmp_dir_path = tmp_dir.path();
 
-    compression::from_tar_gz(&dumps_folder.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap();
+    compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap();
 
     let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap();
     let settings: serde_json::Value = serde_json::from_reader(file).unwrap();
 
-    assert_json_eq!(expected.clone(), settings.clone(), ordered: false);
+    assert_json_eq!(expected, settings, ordered: false);
 }
 
 #[actix_rt::test]
-#[ignore]
 async fn dump_index_documents_should_be_valid() {
     let mut server = common::Server::test_server().await;
 
@@ -303,20 +325,19 @@ async fn dump_index_documents_should_be_valid() {
 
     let uid = trigger_and_wait_dump(&mut server).await;
 
-    let dumps_folder = Path::new(&server.data().dumps_folder);
+    let dumps_dir = Path::new(&server.data().dumps_dir);
     let tmp_dir = TempDir::new().unwrap();
     let tmp_dir_path = tmp_dir.path();
 
-    compression::from_tar_gz(&dumps_folder.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap();
+    compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap();
 
     let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap();
     let documents = read_all_jsonline(file);
 
-    assert_json_eq!(expected.clone(), documents.clone(), ordered: false);
+    assert_json_eq!(expected, documents, ordered: false);
 }
 
 #[actix_rt::test]
-#[ignore]
 async fn dump_index_updates_should_be_valid() {
     let mut server = common::Server::test_server().await;
 
@@ -327,32 +348,21 @@ async fn dump_index_updates_should_be_valid() {
 
     let uid = trigger_and_wait_dump(&mut server).await;
 
-    let dumps_folder = Path::new(&server.data().dumps_folder);
+    let dumps_dir = Path::new(&server.data().dumps_dir);
     let tmp_dir = TempDir::new().unwrap();
     let tmp_dir_path = tmp_dir.path();
 
-    compression::from_tar_gz(&dumps_folder.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap();
+    compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap();
 
     let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap();
-    let mut updates = read_all_jsonline(file);
+    let updates = read_all_jsonline(file);
 
-
-    // hotfix until #943 is fixed (https://github.com/meilisearch/MeiliSearch/issues/943)
-    updates.as_array_mut().unwrap()
-            .get_mut(0).unwrap()
-            .get_mut("type").unwrap()
-            .get_mut("settings").unwrap()
-            .get_mut("displayed_attributes").unwrap()
-            .get_mut("Update").unwrap()
-            .as_array_mut().unwrap().sort_by(|a, b| a.as_str().cmp(&b.as_str()));
-
-    eprintln!("{}\n", updates.to_string());
-    eprintln!("{}", expected.to_string());
-    assert_json_include!(expected: expected.clone(), actual: updates.clone());
+    eprintln!("{}\n", updates);
+    eprintln!("{}", expected);
+    assert_json_include!(expected: expected, actual: updates);
 }
  
 #[actix_rt::test]
-#[ignore]
 async fn get_unexisting_dump_status_should_return_not_found() {
     let mut server = common::Server::test_server().await;
 

meilisearch-http/tests/errors.rs

@@ -177,7 +177,7 @@ async fn document_not_found_error() {
 #[actix_rt::test]
 async fn payload_too_large_error() {
     let mut server = common::Server::with_uid("test");
-    let bigvec = vec![0u64; 10_000_000]; // 80mb
+    let bigvec = vec![0u64; 100_000_000]; // 800mb
     assert_error!(
         "payload_too_large",
         "invalid_request_error",

meilisearch-http/tests/health.rs

@@ -1,6 +1,3 @@
-use serde_json::json;
-use std::convert::Into;
-
 mod common;
 
 #[actix_rt::test]
@@ -10,29 +7,5 @@ async fn test_healthyness() {
     // Check that the server is healthy
 
     let (_response, status_code) = server.get_health().await;
-    assert_eq!(status_code, 200);
-
-    // Set the serve Unhealthy
-    let body = json!({
-        "health": false,
-    });
-    let (_response, status_code) = server.update_health(body).await;
-    assert_eq!(status_code, 200);
-
-    // Check that the server is unhealthy
-
-    let (_response, status_code) = server.get_health().await;
-    assert_eq!(status_code, 503);
-
-    // Set the server healthy
-    let body = json!({
-        "health": true,
-    });
-    let (_response, status_code) = server.update_health(body).await;
-    assert_eq!(status_code, 200);
-
-    // Check if the server is healthy
-
-    let (_response, status_code) = server.get_health().await;
-    assert_eq!(status_code, 200);
+    assert_eq!(status_code, 204);
 }

meilisearch-http/tests/index.rs

@@ -777,3 +777,35 @@ async fn update_existing_primary_key_is_error() {
     assert_eq!(response["errorCode"], "primary_key_already_present");
     assert_eq!(response["errorType"], "invalid_request_error");
 }
+
+#[actix_rt::test]
+async fn test_field_distribution_attribute() {
+    let mut server = common::Server::test_server().await;
+
+    let (response, _status_code) = server.get_index_stats().await;
+
+    let expected = json!({
+        "fieldsDistribution": {
+            "about": 77,
+            "address": 77,
+            "age": 77,
+            "balance": 77,
+            "color": 77,
+            "email": 77,
+            "gender": 77,
+            "id": 77,
+            "isActive": 77,
+            "latitude": 77,
+            "longitude": 77,
+            "name": 77,
+            "phone": 77,
+            "picture": 77,
+            "registered": 77,
+            "tags": 77
+        },
+        "isIndexing": false,
+        "numberOfDocuments": 77
+    });
+
+    assert_json_eq!(expected, response, ordered: true);
+}

meilisearch-http/tests/lazy_index_creation.rs

@@ -0,0 +1,446 @@
+use serde_json::json;
+
+mod common;
+
+#[actix_rt::test]
+async fn create_index_lazy_by_pushing_documents() {
+    let mut server = common::Server::with_uid("movies");
+
+    // 1 - Add documents
+
+    let body = json!([{
+      "title": "Test",
+      "comment": "comment test"
+    }]);
+
+    let url = "/indexes/movies/documents?primaryKey=title";
+    let (response, status_code) = server.post_request(&url, body).await;
+    assert_eq!(status_code, 202);
+    let update_id = response["updateId"].as_u64().unwrap();
+    server.wait_update_id(update_id).await;
+
+    // 3 - Check update success
+
+    let (response, status_code) = server.get_update_status(update_id).await;
+    assert_eq!(status_code, 200);
+    assert_eq!(response["status"], "processed");
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_pushing_documents_and_discover_pk() {
+    let mut server = common::Server::with_uid("movies");
+
+    // 1 - Add documents
+
+    let body = json!([{
+      "id": 1,
+      "title": "Test",
+      "comment": "comment test"
+    }]);
+
+    let url = "/indexes/movies/documents";
+    let (response, status_code) = server.post_request(&url, body).await;
+    assert_eq!(status_code, 202);
+    let update_id = response["updateId"].as_u64().unwrap();
+    server.wait_update_id(update_id).await;
+
+    // 3 - Check update success
+
+    let (response, status_code) = server.get_update_status(update_id).await;
+    assert_eq!(status_code, 200);
+    assert_eq!(response["status"], "processed");
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_pushing_documents_with_wrong_name() {
+    let server = common::Server::with_uid("wrong&name");
+
+    let body = json!([{
+      "title": "Test",
+      "comment": "comment test"
+    }]);
+
+    let url = "/indexes/wrong&name/documents?primaryKey=title";
+    let (response, status_code) = server.post_request(&url, body).await;
+    assert_eq!(status_code, 400);
+    assert_eq!(response["errorCode"], "invalid_index_uid");
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_add_documents_failed() {
+    let mut server = common::Server::with_uid("wrong&name");
+
+    let body = json!([{
+      "title": "Test",
+      "comment": "comment test"
+    }]);
+
+    let url = "/indexes/wrong&name/documents";
+    let (response, status_code) = server.post_request(&url, body).await;
+    assert_eq!(status_code, 400);
+    assert_eq!(response["errorCode"], "invalid_index_uid");
+
+    let (_, status_code) = server.get_index().await;
+    assert_eq!(status_code, 404);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_settings() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!({
+        "rankingRules": [
+            "typo",
+            "words",
+            "proximity",
+            "attribute",
+            "wordsPosition",
+            "exactness",
+            "desc(registered)",
+            "desc(age)",
+        ],
+        "distinctAttribute": "id",
+        "searchableAttributes": [
+            "id",
+            "name",
+            "color",
+            "gender",
+            "email",
+            "phone",
+            "address",
+            "registered",
+            "about"
+        ],
+        "displayedAttributes": [
+            "name",
+            "gender",
+            "email",
+            "registered",
+            "age",
+        ],
+        "stopWords": [
+            "ad",
+            "in",
+            "ut",
+        ],
+        "synonyms": {
+            "road": ["street", "avenue"],
+            "street": ["avenue"],
+        },
+        "attributesForFaceting": ["name"],
+    });
+
+    server.update_all_settings(body.clone()).await;
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 200);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_settings_with_error() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!({
+        "rankingRules": [
+            "other",
+            "words",
+            "proximity",
+            "attribute",
+            "wordsPosition",
+            "exactness",
+            "desc(registered)",
+            "desc(age)",
+        ],
+        "distinctAttribute": "id",
+        "searchableAttributes": [
+            "id",
+            "name",
+            "color",
+            "gender",
+            "email",
+            "phone",
+            "address",
+            "registered",
+            "about"
+        ],
+        "displayedAttributes": [
+            "name",
+            "gender",
+            "email",
+            "registered",
+            "age",
+        ],
+        "stopWords": [
+            "ad",
+            "in",
+            "ut",
+        ],
+        "synonyms": {
+            "road": ["street", "avenue"],
+            "street": ["avenue"],
+        },
+        "anotherSettings": ["name"],
+    });
+
+    let (_, status_code) = server.update_all_settings_sync(body.clone()).await;
+    assert_eq!(status_code, 400);
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 404);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_ranking_rules() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!([
+        "typo",
+        "words",
+        "proximity",
+        "attribute",
+        "wordsPosition",
+        "exactness",
+        "desc(registered)",
+        "desc(age)",
+    ]);
+
+    server.update_ranking_rules(body.clone()).await;
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 200);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_ranking_rules_with_error() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!({
+        "rankingRules": 123,
+    });
+
+    let (_, status_code) = server.update_ranking_rules_sync(body.clone()).await;
+    assert_eq!(status_code, 400);
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 404);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_distinct_attribute() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!("type");
+
+    server.update_distinct_attribute(body.clone()).await;
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 200);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_distinct_attribute_with_error() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(123);
+
+    let (resp, status_code) = server.update_distinct_attribute_sync(body.clone()).await;
+    eprintln!("resp: {:?}", resp);
+    assert_eq!(status_code, 400);
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (resp, status_code) = server.get_all_settings().await;
+    eprintln!("resp: {:?}", resp);
+    assert_eq!(status_code, 404);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_searchable_attributes() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(["title", "description"]);
+
+    server.update_searchable_attributes(body.clone()).await;
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 200);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_searchable_attributes_with_error() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(123);
+
+    let (_, status_code) = server.update_searchable_attributes_sync(body.clone()).await;
+    assert_eq!(status_code, 400);
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 404);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_displayed_attributes() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(["title", "description"]);
+
+    server.update_displayed_attributes(body.clone()).await;
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 200);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_displayed_attributes_with_error() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(123);
+
+    let (_, status_code) = server.update_displayed_attributes_sync(body.clone()).await;
+    assert_eq!(status_code, 400);
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 404);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_attributes_for_faceting() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(["title", "description"]);
+
+    server.update_attributes_for_faceting(body.clone()).await;
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 200);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_attributes_for_faceting_with_error() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(123);
+
+    let (_, status_code) = server
+        .update_attributes_for_faceting_sync(body.clone())
+        .await;
+    assert_eq!(status_code, 400);
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 404);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_synonyms() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!({
+        "road": ["street", "avenue"],
+        "street": ["avenue"],
+    });
+
+    server.update_synonyms(body.clone()).await;
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 200);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_synonyms_with_error() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(123);
+
+    let (_, status_code) = server.update_synonyms_sync(body.clone()).await;
+    assert_eq!(status_code, 400);
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 404);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_stop_words() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(["le", "la", "les"]);
+
+    server.update_stop_words(body.clone()).await;
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 200);
+}
+
+#[actix_rt::test]
+async fn create_index_lazy_by_sending_stop_words_with_error() {
+    let mut server = common::Server::with_uid("movies");
+    // 2 - Send the settings
+
+    let body = json!(123);
+
+    let (_, status_code) = server.update_stop_words_sync(body.clone()).await;
+    assert_eq!(status_code, 400);
+
+    // 3 - Get all settings and compare to the previous one
+
+    let (_, status_code) = server.get_all_settings().await;
+
+    assert_eq!(status_code, 404);
+}

meilisearch-http/tests/placeholder_search.rs

@@ -37,14 +37,8 @@ async fn placeholder_search_with_offset() {
         assert_eq!(status_code, 200);
         // take results at offset 3 as reference
         let lock = expected.lock().unwrap();
-        lock.replace(
-            response["hits"].as_array().unwrap()[3..6]
-                .iter()
-                .cloned()
-                .collect(),
-        );
+        lock.replace(response["hits"].as_array().unwrap()[3..6].to_vec());
     });
-
     let expected = expected.into_inner().unwrap().into_inner();
 
     let query = json!({
@@ -588,3 +582,48 @@ async fn placeholder_search_with_empty_query() {
         assert_eq!(response["hits"].as_array().unwrap().len(), 3);
     });
 }
+
+#[actix_rt::test]
+async fn test_filter_nb_hits_search_placeholder() {
+    let mut server = common::Server::with_uid("test");
+
+    let body = json!({
+        "uid": "test",
+        "primaryKey": "id",
+    });
+
+    server.create_index(body).await;
+    let documents = json!([
+        {
+            "id": 1,
+            "content": "a",
+            "color": "green",
+            "size": 1,
+        },
+        {
+            "id": 2,
+            "content": "a",
+            "color": "green",
+            "size": 2,
+        },
+        {
+            "id": 3,
+            "content": "a",
+            "color": "blue",
+            "size": 3,
+        },
+    ]);
+
+    server.add_or_update_multiple_documents(documents).await;
+    let (response, _) = server.search_post(json!({})).await;
+    assert_eq!(response["nbHits"], 3);
+
+    server.update_distinct_attribute(json!("color")).await;
+
+    let (response, _) = server.search_post(json!({})).await;
+    assert_eq!(response["nbHits"], 2);
+
+    let (response, _) = server.search_post(json!({"filters": "size < 3"})).await;
+    println!("result: {}", response);
+    assert_eq!(response["nbHits"], 1);
+}

meilisearch-http/tests/search.rs

@@ -6,6 +6,139 @@ use serde_json::Value;
 
 #[macro_use] mod common;
 
+#[actix_rt::test]
+async fn search() {
+    let mut server = common::Server::test_server().await;
+
+    let query = json! ({
+        "q": "exercitation"
+    });
+
+    let expected = json!([
+        {
+          "id": 1,
+          "balance": "$1,706.13",
+          "picture": "http://placehold.it/32x32",
+          "age": 27,
+          "color": "Green",
+          "name": "Cherry Orr",
+          "gender": "female",
+          "email": "cherryorr@chorizon.com",
+          "phone": "+1 (995) 479-3174",
+          "address": "442 Beverly Road, Ventress, New Mexico, 3361",
+          "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n",
+          "registered": "2020-03-18T11:12:21 -01:00",
+          "latitude": -24.356932,
+          "longitude": 27.184808,
+          "tags": [
+            "new issue",
+            "bug"
+          ],
+          "isActive": true
+        },
+        {
+          "id": 59,
+          "balance": "$1,921.58",
+          "picture": "http://placehold.it/32x32",
+          "age": 31,
+          "color": "Green",
+          "name": "Harper Carson",
+          "gender": "male",
+          "email": "harpercarson@chorizon.com",
+          "phone": "+1 (912) 430-3243",
+          "address": "883 Dennett Place, Knowlton, New Mexico, 9219",
+          "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n",
+          "registered": "2019-12-07T07:33:15 -01:00",
+          "latitude": -60.812605,
+          "longitude": -27.129016,
+          "tags": [
+            "bug",
+            "new issue"
+          ],
+          "isActive": true
+        },
+        {
+          "id": 49,
+          "balance": "$1,476.39",
+          "picture": "http://placehold.it/32x32",
+          "age": 28,
+          "color": "brown",
+          "name": "Maureen Dale",
+          "gender": "female",
+          "email": "maureendale@chorizon.com",
+          "phone": "+1 (984) 538-3684",
+          "address": "817 Newton Street, Bannock, Wyoming, 1468",
+          "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n",
+          "registered": "2018-04-26T06:04:40 -02:00",
+          "latitude": -64.196802,
+          "longitude": -117.396238,
+          "tags": [
+            "wontfix"
+          ],
+          "isActive": true
+        }
+    ]);
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        let hits = response["hits"].as_array().unwrap();
+        let hits: Vec<Value> = hits.iter().cloned().take(3).collect();
+        assert_json_eq!(expected.clone(), serde_json::to_value(hits).unwrap(), ordered: false);
+    });
+}
+
+#[actix_rt::test]
+async fn search_no_params() {
+    let mut server = common::Server::test_server().await;
+
+    let query = json! ({});
+
+    // an empty search should return the 20 first indexed document
+    let dataset: Vec<Value> = serde_json::from_slice(include_bytes!("assets/test_set.json")).unwrap();
+    let expected: Vec<Value> = dataset.into_iter().take(20).collect();
+    let expected: Value = serde_json::to_value(expected).unwrap();
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false);
+    });
+}
+
+#[actix_rt::test]
+async fn search_in_unexisting_index() {
+    let mut server = common::Server::with_uid("test");
+
+    let query = json! ({
+        "q": "exercitation"
+    });
+
+    let expected = json! ({
+        "message": "Index test not found",
+        "errorCode": "index_not_found",
+        "errorType": "invalid_request_error",
+        "errorLink": "https://docs.meilisearch.com/errors#index_not_found"
+      });
+
+    test_post_get_search!(server, query, |response, status_code| {
+        assert_eq!(404, status_code);
+        assert_json_eq!(expected.clone(), response.clone(), ordered: false);
+    });
+}
+
+#[actix_rt::test]
+async fn search_unexpected_params() {
+
+    let query = json! ({"lol": "unexpected"});
+
+    let expected = "unknown field `lol`, expected one of `q`, `offset`, `limit`, `attributesToRetrieve`, `attributesToCrop`, `cropLength`, `attributesToHighlight`, `filters`, `matches`, `facetFilters`, `facetsDistribution` at line 1 column 6";
+
+    let post_query = serde_json::from_str::<meilisearch_http::routes::search::SearchQueryPost>(&query.to_string());
+    assert!(post_query.is_err());
+    assert_eq!(expected, post_query.err().unwrap().to_string());
+
+    let get_query: Result<meilisearch_http::routes::search::SearchQuery, _> = serde_json::from_str(&query.to_string());
+    assert!(get_query.is_err());
+    assert_eq!(expected, get_query.err().unwrap().to_string());
+}
+
 #[actix_rt::test]
 async fn search_with_limit() {
     let mut server = common::Server::test_server().await;
@@ -225,6 +358,66 @@ async fn search_with_attribute_to_highlight_wildcard() {
     });
 }
 
+#[actix_rt::test]
+async fn search_with_attribute_to_highlight_wildcard_chinese() {
+    let mut server = common::Server::test_server().await;
+
+    let query = json!({
+        "q": "子孙",
+        "limit": 1,
+        "attributesToHighlight": ["*"]
+    });
+
+    let expected = json!([
+        {
+            "id": 77,
+            "isActive": false,
+            "balance": "$1,274.29",
+            "picture": "http://placehold.it/32x32",
+            "age": 25,
+            "color": "Red",
+            "name": "孫武",
+            "gender": "male",
+            "email": "SunTzu@chorizon.com",
+            "phone": "+1 (810) 407-3258",
+            "address": "吴國",
+            "about": "孫武（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《孫子兵法》的作者，後人尊稱為孫子、兵聖、東方兵聖，山東、蘇州等地尚有祀奉孫武的廟宇兵聖廟。其族人为樂安孫氏始祖，次子孙明为富春孫氏始祖。\r\n",
+            "registered": "2014-10-20T10:13:32 -02:00",
+            "latitude": 17.11935,
+            "longitude": 65.38197,
+            "tags": [
+              "new issue",
+              "wontfix"
+            ],
+          "_formatted": {
+            "id": 77,
+            "isActive": false,
+            "balance": "$1,274.29",
+            "picture": "http://placehold.it/32x32",
+            "age": 25,
+            "color": "Red",
+            "name": "<em>孫武</em>",
+            "gender": "male",
+            "email": "SunTzu@chorizon.com",
+            "phone": "+1 (810) 407-3258",
+            "address": "吴國",
+            "about": "<em>孫武</em>（前544年－前470年或前496年），字長卿，春秋時期齊國人，著名軍事家、政治家，兵家代表人物。兵書《<em>孫子</em>兵法》的作者，後人尊稱為<em>孫子</em>、兵聖、東方兵聖，山東、蘇州等地尚有祀奉<em>孫武</em>的廟宇兵聖廟。其族人为樂安<em>孫氏</em>始祖，次<em>子孙</em>明为富春孫氏始祖。\r\n",
+            "registered": "2014-10-20T10:13:32 -02:00",
+            "latitude": 17.11935,
+            "longitude": 65.38197,
+            "tags": [
+              "new issue",
+              "wontfix"
+            ]
+          }
+        }
+    ]);
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false);
+    });
+}
+
 #[actix_rt::test]
 async fn search_with_attribute_to_highlight_1() {
     let mut server = common::Server::test_server().await;
@@ -421,6 +614,16 @@ async fn search_with_attributes_to_retrieve() {
     test_post_get_search!(server, query, |response, _status_code| {
         assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false);
     });
+
+    let query = json!({
+        "q": "cherry",
+        "limit": 1,
+        "attributesToRetrieve": [],
+    });
+
+    test_post_get_search!(server, query, |response, _status_code| {
+        assert_json_eq!(json!([{}]), response["hits"].clone(), ordered: false);
+    });
 }
 
 #[actix_rt::test]
@@ -1646,8 +1849,6 @@ async fn update_documents_with_facet_distribution() {
     server.create_index(body).await;
     let settings = json!({
         "attributesForFaceting": ["genre"],
-        "displayedAttributes": ["genre"],
-        "searchableAttributes": ["genre"]
     });
     server.update_all_settings(settings).await;
     let update1 = json!([
@@ -1696,3 +1897,51 @@ async fn update_documents_with_facet_distribution() {
     let (response2, _) = server.search_post(search).await;
     assert_json_eq!(expected_facet_distribution, response2["facetsDistribution"].clone());
 }
+
+#[actix_rt::test]
+async fn test_filter_nb_hits_search_normal() {
+    let mut server = common::Server::with_uid("test");
+
+    let body = json!({
+        "uid": "test",
+        "primaryKey": "id",
+    });
+
+    server.create_index(body).await;
+    let documents = json!([
+        {
+            "id": 1,
+            "content": "a",
+            "color": "green",
+            "size": 1,
+        },
+        {
+            "id": 2,
+            "content": "a",
+            "color": "green",
+            "size": 2,
+        },
+        {
+            "id": 3,
+            "content": "a",
+            "color": "blue",
+            "size": 3,
+        },
+    ]);
+
+    server.add_or_update_multiple_documents(documents).await;
+    let (response, _) = server.search_post(json!({"q": "a"})).await;
+    assert_eq!(response["nbHits"], 3);
+
+    let (response, _) = server.search_post(json!({"q": "a", "filters": "size = 1"})).await;
+    assert_eq!(response["nbHits"], 1);
+
+    server.update_distinct_attribute(json!("color")).await;
+
+    let (response, _) = server.search_post(json!({"q": "a"})).await;
+    assert_eq!(response["nbHits"], 2);
+
+    let (response, _) = server.search_post(json!({"q": "a", "filters": "size < 3"})).await;
+    println!("result: {}", response);
+    assert_eq!(response["nbHits"], 1);
+}

meilisearch-http/tests/settings.rs

@@ -468,3 +468,56 @@ async fn settings_that_contains_wildcard_is_wildcard() {
     assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*");
     assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*");
 }
+
+#[actix_rt::test]
+async fn test_displayed_attributes_field() {
+    let mut server = common::Server::test_server().await;
+
+    let body = json!({
+        "rankingRules": [
+            "typo",
+            "words",
+            "proximity",
+            "attribute",
+            "wordsPosition",
+            "exactness",
+            "desc(registered)",
+            "desc(age)",
+        ],
+        "distinctAttribute": "id",
+        "searchableAttributes": [
+            "id",
+            "name",
+            "color",
+            "gender",
+            "email",
+            "phone",
+            "address",
+            "registered",
+            "about"
+        ],
+        "displayedAttributes": [
+            "age",
+            "email",
+            "gender",
+            "name",
+            "registered",
+        ],
+        "stopWords": [
+            "ad",
+            "in",
+            "ut",
+        ],
+        "synonyms": {
+            "road": ["avenue", "street"],
+            "street": ["avenue"],
+        },
+        "attributesForFaceting": ["name"],
+    });
+
+    server.update_all_settings(body.clone()).await;
+
+    let (response, _status_code) = server.get_all_settings().await;
+
+    assert_json_eq!(body, response, ordered: true);
+}

meilisearch-schema/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "meilisearch-schema"
-version = "0.15.0"
+version = "0.18.1"
 license = "MIT"
 authors = ["Kerollmops <renault.cle@gmail.com>"]
 edition = "2018"
 
 [dependencies]
-indexmap = { version = "1.3.2", features = ["serde-1"] }
-meilisearch-error = { path = "../meilisearch-error", version = "0.15.0" }
-serde = { version = "1.0.105", features = ["derive"] }
-serde_json = { version = "1.0.50", features = ["preserve_order"] }
+indexmap = { version = "1.6.1", features = ["serde-1"] }
+meilisearch-error = { path = "../meilisearch-error", version = "0.18.1" }
+serde = { version = "1.0.118", features = ["derive"] }
+serde_json = { version = "1.0.61", features = ["preserve_order"] }
 zerocopy = "0.3.0"

meilisearch-schema/src/fields_map.rs

@@ -6,22 +6,14 @@ use serde::{Deserialize, Serialize};
 use crate::{SResult, FieldId};
 
 #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct FieldsMap {
+pub(crate) struct FieldsMap {
     name_map: HashMap<String, FieldId>,
     id_map: HashMap<FieldId, String>,
     next_id: FieldId
 }
 
 impl FieldsMap {
-    pub fn len(&self) -> usize {
-        self.name_map.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.name_map.is_empty()
-    }
-
-    pub fn insert(&mut self, name: &str) -> SResult<FieldId> {
+    pub(crate) fn insert(&mut self, name: &str) -> SResult<FieldId> {
         if let Some(id) = self.name_map.get(name) {
             return Ok(*id)
         }
@@ -32,22 +24,15 @@ impl FieldsMap {
         Ok(id)
     }
 
-    pub fn remove(&mut self, name: &str) {
-        if let Some(id) = self.name_map.get(name) {
-            self.id_map.remove(&id);
-        }
-        self.name_map.remove(name);
-    }
-
-    pub fn id(&self, name: &str) -> Option<FieldId> {
+    pub(crate) fn id(&self, name: &str) -> Option<FieldId> {
         self.name_map.get(name).copied()
     }
 
-    pub fn name<I: Into<FieldId>>(&self, id: I) -> Option<&str> {
+    pub(crate) fn name<I: Into<FieldId>>(&self, id: I) -> Option<&str> {
         self.id_map.get(&id.into()).map(|s| s.as_str())
     }
 
-    pub fn iter(&self) -> Iter<'_, String, FieldId> {
+    pub(crate) fn iter(&self) -> Iter<'_, String, FieldId> {
         self.name_map.iter()
     }
 }
@@ -69,14 +54,10 @@ mod tests {
         assert_eq!(fields_map.id("title"), Some(1.into()));
         assert_eq!(fields_map.id("descritpion"), Some(2.into()));
         assert_eq!(fields_map.id("date"), None);
-        assert_eq!(fields_map.len(), 3);
         assert_eq!(fields_map.name(0), Some("id"));
         assert_eq!(fields_map.name(1), Some("title"));
         assert_eq!(fields_map.name(2), Some("descritpion"));
         assert_eq!(fields_map.name(4), None);
-        fields_map.remove("title");
-        assert_eq!(fields_map.id("title"), None);
-        assert_eq!(fields_map.insert("title").unwrap(), 3.into());
-        assert_eq!(fields_map.len(), 3);
+        assert_eq!(fields_map.insert("title").unwrap(), 1.into());
     }
 }

meilisearch-schema/src/lib.rs

@@ -1,9 +1,10 @@
 mod error;
 mod fields_map;
 mod schema;
+mod position_map;
 
 pub use error::{Error, SResult};
-pub use fields_map::FieldsMap;
+use fields_map::FieldsMap;
 pub use schema::Schema;
 use serde::{Deserialize, Serialize};
 use zerocopy::{AsBytes, FromBytes};

meilisearch-schema/src/position_map.rs

@@ -0,0 +1,161 @@
+use std::collections::BTreeMap;
+
+use crate::{FieldId, IndexedPos};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct PositionMap {
+    pos_to_field: Vec<FieldId>,
+    field_to_pos: BTreeMap<FieldId, IndexedPos>,
+}
+
+impl PositionMap {
+    /// insert `id` at the specified `position` updating the other position if a shift is caused by
+    /// the operation. If `id` is already present in the position map, it is moved to the requested
+    /// `position`, potentially causing shifts.
+    pub fn insert(&mut self, id: FieldId, position: IndexedPos) -> IndexedPos {
+        let mut upos = position.0 as usize;
+        let mut must_rebuild_map = false;
+
+        if let Some(old_pos) = self.field_to_pos.get(&id) {
+            let uold_pos = old_pos.0 as usize;
+            self.pos_to_field.remove(uold_pos);
+            must_rebuild_map = true;
+        }
+
+        if upos < self.pos_to_field.len() {
+            self.pos_to_field.insert(upos, id);
+            must_rebuild_map = true;
+        } else {
+            upos = self.pos_to_field.len();
+            self.pos_to_field.push(id);
+        }
+
+        // we only need to update all the positions if there have been a shift a some point. In
+        // most cases we only did a push, so we don't need to rebuild the `field_to_pos` map.
+        if must_rebuild_map {
+            self.field_to_pos.clear();
+            self.field_to_pos.extend(
+                self.pos_to_field
+                .iter()
+                .enumerate()
+                .map(|(p, f)| (*f, IndexedPos(p as u16))),
+            );
+        } else {
+            self.field_to_pos.insert(id, IndexedPos(upos as u16));
+        }
+        IndexedPos(upos as u16)
+    }
+
+    /// Pushes `id` in last position
+    pub fn push(&mut self, id: FieldId) -> IndexedPos {
+        let pos = self.len();
+        self.insert(id, IndexedPos(pos as u16))
+    }
+
+    pub fn len(&self) -> usize {
+        self.pos_to_field.len()
+    }
+
+    pub fn field_to_pos(&self, id: FieldId) -> Option<IndexedPos> {
+        self.field_to_pos.get(&id).cloned()
+    }
+
+    pub fn pos_to_field(&self, pos: IndexedPos) -> Option<FieldId> {
+        let pos = pos.0 as usize;
+        self.pos_to_field.get(pos).cloned()
+    }
+
+    pub fn field_pos(&self) -> impl Iterator<Item = (FieldId, IndexedPos)> + '_ {
+        self.pos_to_field
+            .iter()
+            .enumerate()
+            .map(|(i, f)| (*f, IndexedPos(i as u16)))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_default() {
+        assert_eq!(
+            format!("{:?}", PositionMap::default()),
+            r##"PositionMap { pos_to_field: [], field_to_pos: {} }"##
+        );
+    }
+
+    #[test]
+    fn test_insert() {
+        let mut map = PositionMap::default();
+        // changing position removes from old position
+        map.insert(0.into(), 0.into());
+        map.insert(1.into(), 1.into());
+        assert_eq!(
+            format!("{:?}", map),
+            r##"PositionMap { pos_to_field: [FieldId(0), FieldId(1)], field_to_pos: {FieldId(0): IndexedPos(0), FieldId(1): IndexedPos(1)} }"##
+        );
+        map.insert(0.into(), 1.into());
+        assert_eq!(
+            format!("{:?}", map),
+            r##"PositionMap { pos_to_field: [FieldId(1), FieldId(0)], field_to_pos: {FieldId(0): IndexedPos(1), FieldId(1): IndexedPos(0)} }"##
+        );
+        map.insert(2.into(), 1.into());
+        assert_eq!(
+            format!("{:?}", map),
+            r##"PositionMap { pos_to_field: [FieldId(1), FieldId(2), FieldId(0)], field_to_pos: {FieldId(0): IndexedPos(2), FieldId(1): IndexedPos(0), FieldId(2): IndexedPos(1)} }"##
+        );
+    }
+
+    #[test]
+    fn test_push() {
+        let mut map = PositionMap::default();
+        map.push(0.into());
+        map.push(2.into());
+        assert_eq!(map.len(), 2);
+        assert_eq!(
+            format!("{:?}", map),
+            r##"PositionMap { pos_to_field: [FieldId(0), FieldId(2)], field_to_pos: {FieldId(0): IndexedPos(0), FieldId(2): IndexedPos(1)} }"##
+        );
+    }
+
+    #[test]
+    fn test_field_to_pos() {
+        let mut map = PositionMap::default();
+        map.push(0.into());
+        map.push(2.into());
+        assert_eq!(map.field_to_pos(2.into()), Some(1.into()));
+        assert_eq!(map.field_to_pos(0.into()), Some(0.into()));
+        assert_eq!(map.field_to_pos(4.into()), None);
+    }
+
+    #[test]
+    fn test_pos_to_field() {
+        let mut map = PositionMap::default();
+        map.push(0.into());
+        map.push(2.into());
+        map.push(3.into());
+        map.push(4.into());
+        assert_eq!(
+            format!("{:?}", map),
+            r##"PositionMap { pos_to_field: [FieldId(0), FieldId(2), FieldId(3), FieldId(4)], field_to_pos: {FieldId(0): IndexedPos(0), FieldId(2): IndexedPos(1), FieldId(3): IndexedPos(2), FieldId(4): IndexedPos(3)} }"##
+        );
+        assert_eq!(map.pos_to_field(0.into()), Some(0.into()));
+        assert_eq!(map.pos_to_field(1.into()), Some(2.into()));
+        assert_eq!(map.pos_to_field(2.into()), Some(3.into()));
+        assert_eq!(map.pos_to_field(3.into()), Some(4.into()));
+        assert_eq!(map.pos_to_field(4.into()), None);
+    }
+
+    #[test]
+    fn test_field_pos() {
+        let mut map = PositionMap::default();
+        map.push(0.into());
+        map.push(2.into());
+        let mut iter = map.field_pos();
+        assert_eq!(iter.next(), Some((0.into(), 0.into())));
+        assert_eq!(iter.next(), Some((2.into(), 1.into())));
+        assert_eq!(iter.next(), None);
+    }
+}

meilisearch-schema/src/schema.rs

@@ -1,42 +1,10 @@
-use crate::{FieldsMap, FieldId, SResult, Error, IndexedPos};
-use serde::{Serialize, Deserialize};
-use std::collections::{HashMap, HashSet};
 use std::borrow::Cow;
+use std::collections::{BTreeSet, HashSet};
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-enum OptionAll<T> {
-    All,
-    Some(T),
-    None,
-}
+use serde::{Deserialize, Serialize};
 
-impl<T> OptionAll<T> {
-    // replace the value with None and return the previous value
-    fn take(&mut self) -> OptionAll<T> {
-        std::mem::replace(self, OptionAll::None)
-    }
-
-    fn map<U, F: FnOnce(T) -> U>(self, f: F) -> OptionAll<U> {
-        match self {
-            OptionAll::Some(x) => OptionAll::Some(f(x)),
-            OptionAll::All => OptionAll::All,
-            OptionAll::None => OptionAll::None,
-        }
-    }
-
-    pub fn is_all(&self) -> bool {
-        match self {
-            OptionAll::All => true,
-            _ => false,
-        }
-    }
-}
-
-impl<T> Default for OptionAll<T> {
-    fn default() -> OptionAll<T> {
-        OptionAll::All
-    }
-}
+use crate::position_map::PositionMap;
+use crate::{Error, FieldId, FieldsMap, IndexedPos, SResult};
 
 #[derive(Clone, Debug, Serialize, Deserialize, Default)]
 pub struct Schema {
@@ -44,34 +12,26 @@ pub struct Schema {
 
     primary_key: Option<FieldId>,
     ranked: HashSet<FieldId>,
-    displayed: OptionAll<HashSet<FieldId>>,
+    displayed: Option<BTreeSet<FieldId>>,
 
-    indexed: OptionAll<Vec<FieldId>>,
-    indexed_map: HashMap<FieldId, IndexedPos>,
+    searchable: Option<Vec<FieldId>>,
+    pub indexed_position: PositionMap,
 }
 
 impl Schema {
-    pub fn new() -> Schema {
-        Schema::default()
-    }
-
     pub fn with_primary_key(name: &str) -> Schema {
         let mut fields_map = FieldsMap::default();
         let field_id = fields_map.insert(name).unwrap();
-
-        let mut displayed = HashSet::new();
-        let mut indexed_map = HashMap::new();
-
-        displayed.insert(field_id);
-        indexed_map.insert(field_id, 0.into());
+        let mut indexed_position = PositionMap::default();
+        indexed_position.push(field_id);
 
         Schema {
             fields_map,
             primary_key: Some(field_id),
             ranked: HashSet::new(),
-            displayed: OptionAll::All,
-            indexed: OptionAll::All,
-            indexed_map,
+            displayed: None,
+            searchable: None,
+            indexed_position,
         }
     }
 
@@ -81,13 +41,11 @@ impl Schema {
 
     pub fn set_primary_key(&mut self, name: &str) -> SResult<FieldId> {
         if self.primary_key.is_some() {
-            return Err(Error::PrimaryKeyAlreadyPresent)
+            return Err(Error::PrimaryKeyAlreadyPresent);
         }
 
         let id = self.insert(name)?;
         self.primary_key = Some(id);
-        self.set_indexed(name)?;
-        self.set_displayed(name)?;
 
         Ok(id)
     }
@@ -104,202 +62,98 @@ impl Schema {
         self.fields_map.iter().map(|(k, _)| k.as_ref())
     }
 
-    pub fn contains(&self, name: &str) -> bool {
-        self.fields_map.id(name).is_some()
-    }
-
+    /// add `name` to the list of known fields
     pub fn insert(&mut self, name: &str) -> SResult<FieldId> {
         self.fields_map.insert(name)
     }
 
-    pub fn insert_and_index(&mut self, name: &str) -> SResult<FieldId> {
-        match self.fields_map.id(name) {
-            Some(id) => {
-                Ok(id)
-            }
-            None => {
-                self.set_indexed(name)?;
-                self.set_displayed(name)
-            }
-        }
+    /// Adds `name` to the list of known fields, and in the last position of the indexed_position map. This
+    /// field is taken into acccount when `searchableAttribute` or `displayedAttributes` is set to `"*"`
+    pub fn insert_with_position(&mut self, name: &str) -> SResult<(FieldId, IndexedPos)> {
+        let field_id = self.fields_map.insert(name)?;
+        let position = self
+            .is_searchable(field_id)
+            .unwrap_or_else(|| self.indexed_position.push(field_id));
+        Ok((field_id, position))
     }
 
     pub fn ranked(&self) -> &HashSet<FieldId> {
         &self.ranked
     }
 
-    pub fn ranked_name(&self) -> HashSet<&str> {
-        self.ranked.iter().filter_map(|a| self.name(*a)).collect()
-    }
-
-    pub fn displayed(&self) -> Cow<HashSet<FieldId>> {
-        match self.displayed {
-            OptionAll::Some(ref v) => Cow::Borrowed(v),
-            OptionAll::All => {
-                let fields = self
-                    .fields_map
-                    .iter()
-                    .map(|(_, &v)| v)
-                    .collect::<HashSet<_>>();
-                Cow::Owned(fields)
-            }
-            OptionAll::None => Cow::Owned(HashSet::new())
+    fn displayed(&self) -> Cow<BTreeSet<FieldId>> {
+        match &self.displayed {
+            Some(displayed) => Cow::Borrowed(displayed),
+            None => Cow::Owned(self.indexed_position.field_pos().map(|(f, _)| f).collect()),
         }
     }
 
     pub fn is_displayed_all(&self) -> bool {
-        self.displayed.is_all()
+        self.displayed.is_none()
     }
 
-    pub fn displayed_name(&self) -> HashSet<&str> {
-        match self.displayed {
-            OptionAll::All => self.fields_map.iter().filter_map(|(_, &v)| self.name(v)).collect(),
-            OptionAll::Some(ref v) => v.iter().filter_map(|a| self.name(*a)).collect(),
-            OptionAll::None => HashSet::new(),
+    pub fn displayed_names(&self) -> BTreeSet<&str> {
+        self.displayed()
+            .iter()
+            .filter_map(|&f| self.name(f))
+            .collect()
+    }
+
+    fn searchable(&self) -> Cow<[FieldId]> {
+        match &self.searchable {
+            Some(searchable) => Cow::Borrowed(&searchable),
+            None => Cow::Owned(self.indexed_position.field_pos().map(|(f, _)| f).collect()),
         }
     }
 
-    pub fn indexed(&self) -> Cow<[FieldId]> {
-        match self.indexed {
-            OptionAll::Some(ref v) => Cow::Borrowed(v),
-            OptionAll::All => {
-                let fields = self
-                    .fields_map
-                    .iter()
-                    .map(|(_, &f)| f)
-                    .collect();
-                Cow::Owned(fields)
-            },
-            OptionAll::None => Cow::Owned(Vec::new())
-        }
+    pub fn searchable_names(&self) -> Vec<&str> {
+        self.searchable()
+            .iter()
+            .filter_map(|a| self.name(*a))
+            .collect()
     }
 
-    pub fn indexed_name(&self) -> Vec<&str> {
-        self.indexed().iter().filter_map(|a| self.name(*a)).collect()
-    }
-
-    pub fn set_ranked(&mut self, name: &str) -> SResult<FieldId> {
+    pub(crate) fn set_ranked(&mut self, name: &str) -> SResult<FieldId> {
         let id = self.fields_map.insert(name)?;
         self.ranked.insert(id);
         Ok(id)
     }
 
-    pub fn set_displayed(&mut self, name: &str) -> SResult<FieldId> {
-        let id = self.fields_map.insert(name)?;
-        self.displayed = match self.displayed.take() {
-            OptionAll::All => OptionAll::All,
-            OptionAll::None => {
-                let mut displayed = HashSet::new();
-                displayed.insert(id);
-                OptionAll::Some(displayed)
-            },
-            OptionAll::Some(mut v) => {
-                v.insert(id);
-                OptionAll::Some(v)
-            }
-        };
-        Ok(id)
-    }
-
-    pub fn set_indexed(&mut self, name: &str) -> SResult<(FieldId, IndexedPos)> {
-        let id = self.fields_map.insert(name)?;
-
-        if let Some(indexed_pos) = self.indexed_map.get(&id) {
-            return Ok((id, *indexed_pos))
-        };
-        let pos = self.indexed_map.len() as u16;
-        self.indexed_map.insert(id, pos.into());
-        self.indexed = self.indexed.take().map(|mut v| {
-            v.push(id);
-            v
-        });
-        Ok((id, pos.into()))
-    }
-
     pub fn clear_ranked(&mut self) {
         self.ranked.clear();
     }
 
-    pub fn remove_ranked(&mut self, name: &str) {
-        if let Some(id) = self.fields_map.id(name) {
-            self.ranked.remove(&id);
-        }
-    }
-
-    /// remove field from displayed attributes. If diplayed attributes is OptionAll::All,
-    /// dipslayed attributes is turned into OptionAll::Some(v) where v is all displayed attributes
-    /// except name.
-    pub fn remove_displayed(&mut self, name: &str) {
-        if let Some(id) = self.fields_map.id(name) {
-            self.displayed = match self.displayed.take() {
-                OptionAll::Some(mut v) => {
-                    v.remove(&id);
-                    OptionAll::Some(v)
-                }
-                OptionAll::All => {
-                    let displayed = self.fields_map
-                        .iter()
-                        .filter_map(|(key, &value)| {
-                            if key != name {
-                                Some(value)
-                            } else {
-                                None
-                            }
-                        })
-                        .collect::<HashSet<_>>();
-                    OptionAll::Some(displayed)
-                }
-                OptionAll::None => OptionAll::None,
-            };
-        }
-    }
-
-    pub fn remove_indexed(&mut self, name: &str) {
-        if let Some(id) = self.fields_map.id(name) {
-            self.indexed_map.remove(&id);
-            self.indexed = match self.indexed.take() {
-                // valid because indexed is All and indexed() return the content of
-                // indexed_map that is already updated
-                OptionAll::All => OptionAll::Some(self.indexed().into_owned()),
-                OptionAll::Some(mut v) => {
-                    v.retain(|x| *x != id);
-                    OptionAll::Some(v)
-                }
-                OptionAll::None => OptionAll::None,
-            }
-        }
-    }
-
     pub fn is_ranked(&self, id: FieldId) -> bool {
         self.ranked.get(&id).is_some()
     }
 
     pub fn is_displayed(&self, id: FieldId) -> bool {
-        match self.displayed {
-            OptionAll::Some(ref v) => v.contains(&id),
-            OptionAll::All => true,
-            OptionAll::None => false,
+        match &self.displayed {
+            Some(displayed) => displayed.contains(&id),
+            None => true,
         }
     }
 
-    pub fn is_indexed(&self, id: FieldId) -> Option<&IndexedPos> {
-        self.indexed_map.get(&id)
+    pub fn is_searchable(&self, id: FieldId) -> Option<IndexedPos> {
+        match &self.searchable {
+            Some(searchable) if searchable.contains(&id) => self.indexed_position.field_to_pos(id),
+            None => self.indexed_position.field_to_pos(id),
+            _ => None,
+        }
     }
 
-    pub fn is_indexed_all(&self) -> bool {
-        self.indexed.is_all()
+    pub fn is_searchable_all(&self) -> bool {
+        self.searchable.is_none()
     }
 
     pub fn indexed_pos_to_field_id<I: Into<IndexedPos>>(&self, pos: I) -> Option<FieldId> {
-        let indexed_pos = pos.into().0;
-        self
-            .indexed_map
-            .iter()
-            .find(|(_, &v)| v.0 == indexed_pos)
-            .map(|(&k, _)| k)
+        self.indexed_position.pos_to_field(pos.into())
     }
 
-    pub fn update_ranked<S: AsRef<str>>(&mut self, data: impl IntoIterator<Item = S>) -> SResult<()> {
+    pub fn update_ranked<S: AsRef<str>>(
+        &mut self,
+        data: impl IntoIterator<Item = S>,
+    ) -> SResult<()> {
         self.ranked.clear();
         for name in data {
             self.set_ranked(name.as_ref())?;
@@ -307,46 +161,208 @@ impl Schema {
         Ok(())
     }
 
-    pub fn update_displayed<S: AsRef<str>>(&mut self, data: impl IntoIterator<Item = S>) -> SResult<()> {
-        self.displayed = match self.displayed.take() {
-            OptionAll::Some(mut v) => {
-                v.clear();
-                OptionAll::Some(v)
-            }
-            _ => OptionAll::Some(HashSet::new())
-        };
+    pub fn update_displayed<S: AsRef<str>>(
+        &mut self,
+        data: impl IntoIterator<Item = S>,
+    ) -> SResult<()> {
+        let mut displayed = BTreeSet::new();
         for name in data {
-            self.set_displayed(name.as_ref())?;
+            let id = self.fields_map.insert(name.as_ref())?;
+            displayed.insert(id);
         }
+        self.displayed.replace(displayed);
         Ok(())
     }
 
-    pub fn update_indexed<S: AsRef<str>>(&mut self, data: Vec<S>) -> SResult<()> {
-        self.indexed = match self.indexed.take() {
-            OptionAll::Some(mut v) => {
-                v.clear();
-                OptionAll::Some(v)
-            },
-            _ => OptionAll::Some(Vec::new()),
-        };
-        self.indexed_map.clear();
-        for name in data {
-            self.set_indexed(name.as_ref())?;
+    pub fn update_searchable<S: AsRef<str>>(&mut self, data: Vec<S>) -> SResult<()> {
+        let mut searchable = Vec::with_capacity(data.len());
+        for (pos, name) in data.iter().enumerate() {
+            let id = self.insert(name.as_ref())?;
+            self.indexed_position.insert(id, IndexedPos(pos as u16));
+            searchable.push(id);
         }
+        self.searchable.replace(searchable);
         Ok(())
     }
 
-    pub fn set_all_fields_as_indexed(&mut self) {
-        self.indexed = OptionAll::All;
-        self.indexed_map.clear();
-
-        for (_name, id) in self.fields_map.iter() {
-            let pos = self.indexed_map.len() as u16;
-            self.indexed_map.insert(*id, pos.into());
-        }
+    pub fn set_all_searchable(&mut self) {
+        self.searchable.take();
     }
 
-    pub fn set_all_fields_as_displayed(&mut self) {
-        self.displayed = OptionAll::All
+    pub fn set_all_displayed(&mut self) {
+        self.displayed.take();
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_with_primary_key() {
+        let schema = Schema::with_primary_key("test");
+        assert_eq!(
+            format!("{:?}", schema),
+            r##"Schema { fields_map: FieldsMap { name_map: {"test": FieldId(0)}, id_map: {FieldId(0): "test"}, next_id: FieldId(1) }, primary_key: Some(FieldId(0)), ranked: {}, displayed: None, searchable: None, indexed_position: PositionMap { pos_to_field: [FieldId(0)], field_to_pos: {FieldId(0): IndexedPos(0)} } }"##
+        );
+    }
+
+    #[test]
+    fn primary_key() {
+        let schema = Schema::with_primary_key("test");
+        assert_eq!(schema.primary_key(), Some("test"));
+    }
+
+    #[test]
+    fn test_insert_with_position_base() {
+        let mut schema = Schema::default();
+        let (id, position) = schema.insert_with_position("foo").unwrap();
+        assert!(schema.searchable.is_none());
+        assert!(schema.displayed.is_none());
+        assert_eq!(id, 0.into());
+        assert_eq!(position, 0.into());
+        let (id, position) = schema.insert_with_position("bar").unwrap();
+        assert_eq!(id, 1.into());
+        assert_eq!(position, 1.into());
+    }
+
+    #[test]
+    fn test_insert_with_position_primary_key() {
+        let mut schema = Schema::with_primary_key("test");
+        let (id, position) = schema.insert_with_position("foo").unwrap();
+        assert!(schema.searchable.is_none());
+        assert!(schema.displayed.is_none());
+        assert_eq!(id, 1.into());
+        assert_eq!(position, 1.into());
+        let (id, position) = schema.insert_with_position("test").unwrap();
+        assert_eq!(id, 0.into());
+        assert_eq!(position, 0.into());
+    }
+
+    #[test]
+    fn test_insert() {
+        let mut schema = Schema::default();
+        let field_id = schema.insert("foo").unwrap();
+        assert!(schema.fields_map.name(field_id).is_some());
+        assert!(schema.searchable.is_none());
+        assert!(schema.displayed.is_none());
+    }
+
+    #[test]
+    fn test_update_searchable() {
+        let mut schema = Schema::default();
+
+        schema.update_searchable(vec!["foo", "bar"]).unwrap();
+        assert_eq!(
+            format!("{:?}", schema.indexed_position),
+            r##"PositionMap { pos_to_field: [FieldId(0), FieldId(1)], field_to_pos: {FieldId(0): IndexedPos(0), FieldId(1): IndexedPos(1)} }"##
+        );
+        assert_eq!(
+            format!("{:?}", schema.searchable),
+            r##"Some([FieldId(0), FieldId(1)])"##
+        );
+        schema.update_searchable(vec!["bar"]).unwrap();
+        assert_eq!(
+            format!("{:?}", schema.searchable),
+            r##"Some([FieldId(1)])"##
+        );
+        assert_eq!(
+            format!("{:?}", schema.indexed_position),
+            r##"PositionMap { pos_to_field: [FieldId(1), FieldId(0)], field_to_pos: {FieldId(0): IndexedPos(1), FieldId(1): IndexedPos(0)} }"##
+        );
+    }
+
+    #[test]
+    fn test_update_displayed() {
+        let mut schema = Schema::default();
+        schema.update_displayed(vec!["foobar"]).unwrap();
+        assert_eq!(
+            format!("{:?}", schema.displayed),
+            r##"Some({FieldId(0)})"##
+        );
+        assert_eq!(
+            format!("{:?}", schema.indexed_position),
+            r##"PositionMap { pos_to_field: [], field_to_pos: {} }"##
+        );
+    }
+
+    #[test]
+    fn test_is_searchable_all() {
+        let mut schema = Schema::default();
+        assert!(schema.is_searchable_all());
+        schema.update_searchable(vec!["foo"]).unwrap();
+        assert!(!schema.is_searchable_all());
+    }
+
+    #[test]
+    fn test_is_displayed_all() {
+        let mut schema = Schema::default();
+        assert!(schema.is_displayed_all());
+        schema.update_displayed(vec!["foo"]).unwrap();
+        assert!(!schema.is_displayed_all());
+    }
+
+    #[test]
+    fn test_searchable_names() {
+        let mut schema = Schema::default();
+        assert_eq!(format!("{:?}", schema.searchable_names()), r##"[]"##);
+        schema.insert_with_position("foo").unwrap();
+        schema.insert_with_position("bar").unwrap();
+        assert_eq!(
+            format!("{:?}", schema.searchable_names()),
+            r##"["foo", "bar"]"##
+        );
+        schema.update_searchable(vec!["hello", "world"]).unwrap();
+        assert_eq!(
+            format!("{:?}", schema.searchable_names()),
+            r##"["hello", "world"]"##
+        );
+        schema.set_all_searchable();
+        assert_eq!(
+            format!("{:?}", schema.searchable_names()),
+            r##"["hello", "world", "foo", "bar"]"##
+        );
+    }
+
+    #[test]
+    fn test_displayed_names() {
+        let mut schema = Schema::default();
+        assert_eq!(format!("{:?}", schema.displayed_names()), r##"{}"##);
+        schema.insert_with_position("foo").unwrap();
+        schema.insert_with_position("bar").unwrap();
+        assert_eq!(
+            format!("{:?}", schema.displayed_names()),
+            r##"{"bar", "foo"}"##
+        );
+        schema.update_displayed(vec!["hello", "world"]).unwrap();
+        assert_eq!(
+            format!("{:?}", schema.displayed_names()),
+            r##"{"hello", "world"}"##
+        );
+        schema.set_all_displayed();
+        assert_eq!(
+            format!("{:?}", schema.displayed_names()),
+            r##"{"bar", "foo"}"##
+        );
+    }
+
+    #[test]
+    fn test_set_all_searchable() {
+        let mut schema = Schema::default();
+        assert!(schema.is_searchable_all());
+        schema.update_searchable(vec!["foobar"]).unwrap();
+        assert!(!schema.is_searchable_all());
+        schema.set_all_searchable();
+        assert!(schema.is_searchable_all());
+    }
+
+    #[test]
+    fn test_set_all_displayed() {
+        let mut schema = Schema::default();
+        assert!(schema.is_displayed_all());
+        schema.update_displayed(vec!["foobar"]).unwrap();
+        assert!(!schema.is_displayed_all());
+        schema.set_all_displayed();
+        assert!(schema.is_displayed_all());
     }
 }

meilisearch-tokenizer/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "meilisearch-tokenizer"
-version = "0.15.0"
+version = "0.18.1"
 license = "MIT"
 authors = ["Kerollmops <renault.cle@gmail.com>"]
 edition = "2018"
 
 [dependencies]
-deunicode = "1.1.0"
+deunicode = "1.1.1"
 slice-group-by = "0.2.6"

meilisearch-tokenizer/src/lib.rs

@@ -4,22 +4,22 @@ use slice_group_by::StrGroupBy;
 use std::iter::Peekable;
 
 pub fn is_cjk(c: char) -> bool {
-    (c >= '\u{1100}' && c <= '\u{11ff}')  // Hangul Jamo
-        || (c >= '\u{2e80}' && c <= '\u{2eff}')  // CJK Radicals Supplement
-        || (c >= '\u{2f00}' && c <= '\u{2fdf}') // Kangxi radical
-        || (c >= '\u{3000}' && c <= '\u{303f}') // Japanese-style punctuation
-        || (c >= '\u{3040}' && c <= '\u{309f}') // Japanese Hiragana
-        || (c >= '\u{30a0}' && c <= '\u{30ff}') // Japanese Katakana
-        || (c >= '\u{3100}' && c <= '\u{312f}')
-        || (c >= '\u{3130}' && c <= '\u{318F}') // Hangul Compatibility Jamo
-        || (c >= '\u{3200}' && c <= '\u{32ff}') // Enclosed CJK Letters and Months
-        || (c >= '\u{3400}' && c <= '\u{4dbf}') // CJK Unified Ideographs Extension A
-        || (c >= '\u{4e00}' && c <= '\u{9fff}') // CJK Unified Ideographs
-        || (c >= '\u{a960}' && c <= '\u{a97f}') // Hangul Jamo Extended-A
-        || (c >= '\u{ac00}' && c <= '\u{d7a3}') // Hangul Syllables
-        || (c >= '\u{d7b0}' && c <= '\u{d7ff}') // Hangul Jamo Extended-B
-        || (c >= '\u{f900}' && c <= '\u{faff}') // CJK Compatibility Ideographs
-        || (c >= '\u{ff00}' && c <= '\u{ffef}') // Full-width roman characters and half-width katakana
+    ('\u{1100}'..='\u{11ff}').contains(&c)
+        || ('\u{2e80}'..='\u{2eff}').contains(&c)  // CJK Radicals Supplement
+        || ('\u{2f00}'..='\u{2fdf}').contains(&c) // Kangxi radical
+        || ('\u{3000}'..='\u{303f}').contains(&c) // Japanese-style punctuation
+        || ('\u{3040}'..='\u{309f}').contains(&c) // Japanese Hiragana
+        || ('\u{30a0}'..='\u{30ff}').contains(&c) // Japanese Katakana
+        || ('\u{3100}'..='\u{312f}').contains(&c)
+        || ('\u{3130}'..='\u{318F}').contains(&c) // Hangul Compatibility Jamo
+        || ('\u{3200}'..='\u{32ff}').contains(&c) // Enclosed CJK Letters and Months
+        || ('\u{3400}'..='\u{4dbf}').contains(&c) // CJK Unified Ideographs Extension A
+        || ('\u{4e00}'..='\u{9fff}').contains(&c) // CJK Unified Ideographs
+        || ('\u{a960}'..='\u{a97f}').contains(&c) // Hangul Jamo Extended-A
+        || ('\u{ac00}'..='\u{d7a3}').contains(&c) // Hangul Syllables
+        || ('\u{d7b0}'..='\u{d7ff}').contains(&c) // Hangul Jamo Extended-B
+        || ('\u{f900}'..='\u{faff}').contains(&c) // CJK Compatibility Ideographs
+        || ('\u{ff00}'..='\u{ffef}').contains(&c) // Full-width roman characters and half-width katakana
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]

meilisearch-types/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "meilisearch-types"
-version = "0.15.0"
+version = "0.18.1"
 license = "MIT"
 authors = ["Clément Renault <renault.cle@gmail.com>"]
 edition = "2018"
@@ -10,7 +10,7 @@ version = "0.3.0"
 optional = true
 
 [dependencies.serde]
-version = "1.0.105"
+version = "1.0.118"
 features = ["derive"]
 optional = true