fix auto-synchronization with zk

add logs
fix the tests
2025-11-30 01:35:36 +00:00 · 2023-08-03 14:43:29 +02:00 · 2023-08-03 13:59:05 +02:00 · 2023-08-03 12:28:08 +02:00 · 2023-08-03 12:25:32 +02:00 · 2023-08-03 10:52:13 +02:00
35 changed files with 550 additions and 1732 deletions
--- a/.github/workflows/publish-apt-brew-pkg.yml
+++ b/.github/workflows/publish-apt-brew-pkg.yml
@@ -35,7 +35,7 @@ jobs:
    - name: Build deb package
      run: cargo deb -p meilisearch -o target/debian/meilisearch.deb
    - name: Upload debian pkg to release
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.6.1
      with:
        repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
        file: target/debian/meilisearch.deb
--- a/.github/workflows/publish-binaries.yml
+++ b/.github/workflows/publish-binaries.yml
@@ -54,7 +54,7 @@ jobs:
    # No need to upload binaries for dry run (cron)
    - name: Upload binaries to release
      if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.6.1
      with:
        repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
        file: target/release/meilisearch
@@ -87,7 +87,7 @@ jobs:
    # No need to upload binaries for dry run (cron)
    - name: Upload binaries to release
      if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.6.1
      with:
        repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
        file: target/release/${{ matrix.artifact_name }}
@@ -121,7 +121,7 @@ jobs:
      - name: Upload the binary to release
        # No need to upload binaries for dry run (cron)
        if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.6.1
        with:
          repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
          file: target/${{ matrix.target }}/release/meilisearch
@@ -183,7 +183,7 @@ jobs:
      - name: Upload the binary to release
        # No need to upload binaries for dry run (cron)
        if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.6.1
        with:
          repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
          file: target/${{ matrix.target }}/release/meilisearch
--- a/.github/workflows/test-suite.yml
+++ b/.github/workflows/test-suite.yml
@@ -43,7 +43,7 @@ jobs:
          toolchain: nightly
          override: true
      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.5.0
      - name: Run cargo check without any default features
        uses: actions-rs/cargo@v1
        with:
@@ -65,7 +65,7 @@ jobs:
    steps:
      - uses: actions/checkout@v3
      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.5.0
      - name: Run cargo check without any default features
        uses: actions-rs/cargo@v1
        with:
@@ -146,7 +146,7 @@ jobs:
          toolchain: stable
          override: true
      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.5.0
      - name: Run tests in debug
        uses: actions-rs/cargo@v1
        with:
@@ -165,7 +165,7 @@ jobs:
          override: true
          components: clippy
      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.5.0
      - name: Run cargo clippy
        uses: actions-rs/cargo@v1
        with:
@@ -184,7 +184,7 @@ jobs:
          override: true
          components: rustfmt
      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.5.0
      - name: Run cargo fmt
        # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
        # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -303,6 +303,12 @@ dependencies = [
 "alloc-no-stdlib",
 ]

+[[package]]
+name = "allocator-api2"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
+
 [[package]]
 name = "anes"
 version = "0.1.6"
@@ -664,6 +670,15 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"

+[[package]]
+name = "castaway"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -791,7 +806,7 @@ version = "4.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b"
 dependencies = [
- "heck",
+ "heck 0.4.1",
 "proc-macro2",
 "quote",
 "syn 2.0.28",
@@ -809,6 +824,17 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"

+[[package]]
+name = "compact_str"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e33b5c3ee2b4ffa00ac2b00d1645cd9229ade668139bccf95f15fadcf374127b"
+dependencies = [
+ "castaway",
+ "itoa",
+ "ryu",
+]
+
 [[package]]
 name = "concat-arrays"
 version = "0.1.2"
@@ -832,6 +858,26 @@ dependencies = [
 "windows-sys 0.45.0",
 ]

+[[package]]
+name = "const_format"
+version = "0.2.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c990efc7a285731f9a4378d81aff2f0e85a2c8781a05ef0f8baa8dac54d0ff48"
+dependencies = [
+ "const_format_proc_macros",
+]
+
+[[package]]
+name = "const_format_proc_macros"
+version = "0.2.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e026b6ce194a874cb9cf32cd5772d1ef9767cc8fcb5765948d74f37a9d8b2bf6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -1224,9 +1270,9 @@ dependencies = [

 [[package]]
 name = "either"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 dependencies = [
 "serde",
 ]
@@ -1715,6 +1761,19 @@ name = "hashbrown"
 version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+dependencies = [
+ "ahash 0.8.3",
+ "allocator-api2",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f"
+dependencies = [
+ "hashbrown 0.14.0",
+]

 [[package]]
 name = "heapless"
@@ -1729,6 +1788,15 @@ dependencies = [
 "stable_deref_trait",
 ]

+[[package]]
+name = "heck"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "heck"
 version = "0.4.1"
@@ -1892,6 +1960,12 @@ dependencies = [
 "unicode-normalization",
 ]

+[[package]]
+name = "ignore-result"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "665ff4dce8edd10d490641ccb78949832f1ddbff02c584fb1f85ab888fe0e50c"
+
 [[package]]
 name = "index-scheduler"
 version = "1.3.0"
@@ -2596,6 +2670,7 @@ dependencies = [
 "walkdir",
 "yaup",
 "zip",
+ "zookeeper-client",
 ]

 [[package]]
@@ -2605,6 +2680,7 @@ dependencies = [
 "base64 0.21.2",
 "enum-iterator",
 "hmac",
+ "log",
 "maplit",
 "meilisearch-types",
 "rand",
@@ -2614,7 +2690,9 @@ dependencies = [
 "sha2",
 "thiserror",
 "time",
+ "tokio",
 "uuid 1.3.3",
+ "zookeeper-client",
 ]

 [[package]]
@@ -2864,6 +2942,27 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "num_enum"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9"
+dependencies = [
+ "num_enum_derive",
+]
+
+[[package]]
+name = "num_enum_derive"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "object"
 version = "0.30.3"
@@ -3159,6 +3258,16 @@ version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"

+[[package]]
+name = "proc-macro-crate"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919"
+dependencies = [
+ "once_cell",
+ "toml_edit",
+]
+
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -3853,6 +3962,28 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

+[[package]]
+name = "strum"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"
+dependencies = [
+ "strum_macros",
+]
+
+[[package]]
+name = "strum_macros"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38"
+dependencies = [
+ "heck 0.3.3",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "subtle"
 version = "2.5.0"
@@ -4732,6 +4863,27 @@ dependencies = [
 "zstd 0.11.2+zstd.1.5.2",
 ]

+[[package]]
+name = "zookeeper-client"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ade019937af9b1e2160582cb8d905bcff5d4b71da82444b7bf2ccc9c93f344f2"
+dependencies = [
+ "bytes",
+ "compact_str",
+ "const_format",
+ "either",
+ "hashbrown 0.12.3",
+ "hashlink",
+ "ignore-result",
+ "log",
+ "num_enum",
+ "static_assertions",
+ "strum",
+ "thiserror",
+ "tokio",
+]
+
 [[package]]
 name = "zstd"
 version = "0.11.2+zstd.1.5.2"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -0,0 +1,37 @@
+version: "3.9"
+services:
+  zk1:
+    container_name: zk1
+    hostname: zk1
+    image: bitnami/zookeeper:3.7.1
+    ports:
+      - 21811:2181
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+      - ZOO_SERVER_ID=1
+      - ZOO_SERVERS=0.0.0.0:2888:3888,zk2:2888:3888,zk3:2888:3888
+  zk2:
+    container_name: zk2
+    hostname: zk2
+    image: bitnami/zookeeper:3.7.1
+    ports:
+      - 21812:2181
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+      - ZOO_SERVER_ID=2
+      - ZOO_SERVERS=zk1:2888:3888,0.0.0.0:2888:3888,zk3:2888:3888
+  zk3:
+    container_name: zk3
+    hostname: zk3
+    image: bitnami/zookeeper:3.7.1
+    ports:
+      - 21813:2181
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+      - ZOO_SERVER_ID=3
+      - ZOO_SERVERS=zk1:2888:3888,zk2:2888:3888,0.0.0.0:2888:3888
+  zoonavigator:
+    container_name: zoonavigator
+    image: elkozmon/zoonavigator
+    ports:
+      - 9000:9000
--- a/dump/src/lib.rs
+++ b/dump/src/lib.rs
@@ -262,9 +262,6 @@ pub(crate) mod test {
            sortable_attributes: Setting::Set(btreeset! { S("age") }),
            ranking_rules: Setting::NotSet,
            stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
            synonyms: Setting::NotSet,
            distinct_attribute: Setting::NotSet,
            typo_tolerance: Setting::NotSet,
--- a/dump/src/reader/compat/v5_to_v6.rs
+++ b/dump/src/reader/compat/v5_to_v6.rs
@@ -340,9 +340,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                }
            },
            stop_words: settings.stop_words.into(),
-            non_separator_tokens: v6::Setting::NotSet,
-            separator_tokens: v6::Setting::NotSet,
-            dictionary: v6::Setting::NotSet,
            synonyms: settings.synonyms.into(),
            distinct_attribute: settings.distinct_attribute.into(),
            typo_tolerance: match settings.typo_tolerance {
--- a/meilisearch-auth/Cargo.toml
+++ b/meilisearch-auth/Cargo.toml
@@ -14,6 +14,7 @@ license.workspace = true
 base64 = "0.21.0"
 enum-iterator = "1.4.0"
 hmac = "0.12.1"
+log = "0.4.19"
 maplit = "1.0.2"
 meilisearch-types = { path = "../meilisearch-types" }
 rand = "0.8.5"
@@ -23,4 +24,6 @@ serde_json = { version = "1.0.95", features = ["preserve_order"] }
 sha2 = "0.10.6"
 thiserror = "1.0.40"
 time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+tokio = { version = "1.27.0", features = ["full"] }
 uuid = { version = "1.3.1", features = ["serde", "v4"] }
+zookeeper-client = "0.4.0"
--- a/meilisearch-auth/src/error.rs
+++ b/meilisearch-auth/src/error.rs
@@ -2,6 +2,7 @@ use std::error::Error;

 use meilisearch_types::error::{Code, ErrorCode};
 use meilisearch_types::internal_error;
+use zookeeper_client as zk;

 pub type Result<T> = std::result::Result<T, AuthControllerError>;

@@ -19,6 +20,8 @@ internal_error!(
    AuthControllerError: meilisearch_types::milli::heed::Error,
    std::io::Error,
    serde_json::Error,
+    tokio::task::JoinError,
+    zk::Error,
    std::str::Utf8Error
 );

--- a/meilisearch-auth/src/lib.rs
+++ b/meilisearch-auth/src/lib.rs
@@ -16,22 +16,119 @@ pub use store::open_auth_store_env;
 use store::{generate_key_as_hexa, HeedAuthStore};
 use time::OffsetDateTime;
 use uuid::Uuid;
+use zookeeper_client as zk;

 #[derive(Clone)]
 pub struct AuthController {
    store: Arc<HeedAuthStore>,
    master_key: Option<String>,
+    zk: Option<zk::Client>,
 }

 impl AuthController {
-    pub fn new(db_path: impl AsRef<Path>, master_key: &Option<String>) -> Result<Self> {
+    pub async fn new(
+        db_path: impl AsRef<Path>,
+        master_key: &Option<String>,
+        zk: Option<zk::Client>,
+    ) -> Result<Self> {
        let store = HeedAuthStore::new(db_path)?;
+        let controller = Self { store: Arc::new(store), master_key: master_key.clone(), zk };

-        if store.is_empty()? {
-            generate_default_keys(&store)?;
+        match controller.zk {
+            // setup the auth zk environment, the `auth` node
+            Some(ref zk) => {
+                let options =
+                    zk::CreateOptions::new(zk::CreateMode::Persistent, zk::Acl::anyone_all());
+                // TODO: we should catch the potential unexpected errors here https://docs.rs/zookeeper-client/latest/zookeeper_client/struct.Client.html#method.create
+                // for the moment we consider that `create` only returns Error::NodeExists.
+                match zk.create("/auth", &[], &options).await {
+                    // If the store is empty, we must generate and push the default api-keys.
+                    Ok(_) => generate_default_keys(&controller).await?,
+                    // If the node exist we should clear our DB and download all the existing api-keys
+                    Err(zk::Error::NodeExists) => {
+                        log::warn!("Auth directory already exists, we need to clear our keys + import the one in zookeeper");
+
+                        let store = controller.store.clone();
+                        tokio::task::spawn_blocking(move || store.delete_all_keys()).await??;
+                        let children = zk
+                            .list_children("/auth")
+                            .await
+                            .expect("Internal, the auth directory was deleted during execution.");
+
+                        log::info!("Importing {} api-keys", children.len());
+                        for path in children {
+                            log::info!("  Importing {}", path);
+                            match zk.get_data(&format!("/auth/{}", &path)).await {
+                                Ok((key, _stat)) => {
+                                let key = serde_json::from_slice(&key).unwrap();
+                                let store = controller.store.clone();
+                                tokio::task::spawn_blocking(move || store.put_api_key(key))
+                                    .await??;
+
+                                },
+                                Err(e) => panic!("{e}")
+                            }
+                            // else the file was deleted while we were inserting the key. We ignore it.
+                            // TODO: What happens if someone updates the files before we have the time
+                            //       to setup the watcher
+                        }
+                    }
+                    e @ Err(
+                        zk::Error::NoNode
+                        | zk::Error::NoChildrenForEphemerals
+                        | zk::Error::InvalidAcl,
+                    ) => unreachable!("{e:?}"),
+                    Err(e) => {
+                        panic!("{e}")
+                    }
+                }
+                // TODO: Race condition above:
+                //       What happens if two node join exactly at the same moment:
+                //       One will create the dir
+                //       The second one will delete its DB, load nothing and install a watcher
+                //       The first one will push its keys and should wake up and update the second one.
+                //     / BUT, if the second one delete its DB and the first one push its files before the second one install the watcher we're fucked
+
+                // Zookeeper Event listener loop
+                let controller_clone = controller.clone();
+                let mut watcher = zk.watch("/auth", zk::AddWatchMode::PersistentRecursive).await?;
+                let czk = zk.clone();
+                tokio::spawn(async move {
+                    let zk = czk;
+                    loop {
+                        let zk::WatchedEvent { event_type, session_state, path } =
+                            dbg!(watcher.changed().await);
+
+                        match event_type {
+                            zk::EventType::Session => panic!("Session error {:?}", session_state),
+                            // a key is deleted from zk
+                            zk::EventType::NodeDeleted => {
+                                // TODO: ugly unwraps
+                                let uuid = path.strip_prefix("/auth/").unwrap();
+                                let uuid = Uuid::parse_str(&uuid).unwrap();
+                                log::info!("The key {} has been deleted", uuid);
+                                dbg!(controller_clone.store.delete_api_key(uuid).unwrap());
+                            }
+                            zk::EventType::NodeCreated | zk::EventType::NodeDataChanged => {
+                                let (key, stat) = zk.get_data(&path).await.unwrap();
+                                let key: Key = serde_json::from_slice(&key).unwrap();
+                                log::info!("The key {} has been deleted", key.uid);
+                                
+                                dbg!(controller_clone.store.put_api_key(key).unwrap());
+                            }
+                            zk::EventType::NodeChildrenChanged => panic!("Got the unexpected NodeChildrenChanged event, what is it used for?"),
+                        }
+                    }
+                });
+            }
+            None => {
+                if controller.store.is_empty()? {
+                    generate_default_keys(&controller).await?;
+                }
+            }
        }

-        Ok(Self { store: Arc::new(store), master_key: master_key.clone() })
+        Ok(controller)
    }

    /// Return `Ok(())` if the auth controller is able to access one of its database.
@@ -50,14 +147,27 @@ impl AuthController {
        self.store.used_size()
    }

-    pub fn create_key(&self, create_key: CreateApiKey) -> Result<Key> {
+    pub async fn create_key(&self, create_key: CreateApiKey) -> Result<Key> {
        match self.store.get_api_key(create_key.uid)? {
            Some(_) => Err(AuthControllerError::ApiKeyAlreadyExists(create_key.uid.to_string())),
-            None => self.store.put_api_key(create_key.to_key()),
+            None => self.put_key(create_key.to_key()).await,
        }
    }

-    pub fn update_key(&self, uid: Uuid, patch: PatchApiKey) -> Result<Key> {
+    pub async fn put_key(&self, key: Key) -> Result<Key> {
+        let store = self.store.clone();
+        // TODO: we may commit only after zk persisted the keys
+        let key = tokio::task::spawn_blocking(move || store.put_api_key(key)).await??;
+        if let Some(ref zk) = self.zk {
+            let options = zk::CreateOptions::new(zk::CreateMode::Persistent, zk::Acl::anyone_all());
+
+            zk.create(&format!("/auth/{}", key.uid), &serde_json::to_vec_pretty(&key)?, &options)
+                .await?;
+        }
+        Ok(key)
+    }
+
+    pub async fn update_key(&self, uid: Uuid, patch: PatchApiKey) -> Result<Key> {
        let mut key = self.get_key(uid)?;
        match patch.description {
            Setting::NotSet => (),
@@ -68,7 +178,14 @@ impl AuthController {
            name => key.name = name.set(),
        };
        key.updated_at = OffsetDateTime::now_utc();
-        self.store.put_api_key(key)
+        let store = self.store.clone();
+        // TODO: we may commit only after zk persisted the keys
+        let key = tokio::task::spawn_blocking(move || store.put_api_key(key)).await??;
+        if let Some(ref zk) = self.zk {
+            zk.set_data(&format!("/auth/{}", key.uid), &serde_json::to_vec_pretty(&key)?, None)
+                .await?;
+        }
+        Ok(key)
    }

    pub fn get_key(&self, uid: Uuid) -> Result<Key> {
@@ -109,8 +226,13 @@ impl AuthController {
        self.store.list_api_keys()
    }

-    pub fn delete_key(&self, uid: Uuid) -> Result<()> {
-        if self.store.delete_api_key(uid)? {
+    pub async fn delete_key(&self, uid: Uuid) -> Result<()> {
+        let store = self.store.clone();
+        let deleted = tokio::task::spawn_blocking(move || store.delete_api_key(uid)).await??;
+        if deleted {
+            if let Some(ref zk) = self.zk {
+                zk.delete(&format!("/auth/{}", uid), None).await?;
+            }
            Ok(())
        } else {
            Err(AuthControllerError::ApiKeyNotFound(uid.to_string()))
@@ -159,7 +281,7 @@ impl AuthController {
        self.store.delete_all_keys()
    }

-    /// Delete all the keys in the DB.
+    /// Insert a key in the DB without any check on its validity
    pub fn raw_insert_key(&mut self, key: Key) -> Result<()> {
        self.store.put_api_key(key)?;
        Ok(())
@@ -304,9 +426,9 @@ pub struct IndexSearchRules {
    pub filter: Option<serde_json::Value>,
 }

-fn generate_default_keys(store: &HeedAuthStore) -> Result<()> {
-    store.put_api_key(Key::default_admin())?;
-    store.put_api_key(Key::default_search())?;
+async fn generate_default_keys(controller: &AuthController) -> Result<()> {
+    controller.put_key(Key::default_admin()).await?;
+    controller.put_key(Key::default_search()).await?;

    Ok(())
 }
--- a/meilisearch-types/src/error.rs
+++ b/meilisearch-types/src/error.rs
@@ -259,9 +259,6 @@ InvalidSettingsRankingRules           , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSearchableAttributes   , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSortableAttributes     , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsStopWords              , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsNonSeparatorTokens     , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSeparatorTokens        , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsDictionary             , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSynonyms               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsTypoTolerance          , InvalidRequest       , BAD_REQUEST ;
 InvalidState                          , Internal             , INTERNAL_SERVER_ERROR ;
--- a/meilisearch-types/src/settings.rs
+++ b/meilisearch-types/src/settings.rs
@@ -171,15 +171,6 @@ pub struct Settings<T> {
    #[deserr(default, error = DeserrJsonError<InvalidSettingsStopWords>)]
    pub stop_words: Setting<BTreeSet<String>>,
    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsNonSeparatorTokens>)]
-    pub non_separator_tokens: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsSeparatorTokens>)]
-    pub separator_tokens: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsDictionary>)]
-    pub dictionary: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
    #[deserr(default, error = DeserrJsonError<InvalidSettingsSynonyms>)]
    pub synonyms: Setting<BTreeMap<String, Vec<String>>>,
    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
@@ -210,9 +201,6 @@ impl Settings<Checked> {
            ranking_rules: Setting::Reset,
            stop_words: Setting::Reset,
            synonyms: Setting::Reset,
-            non_separator_tokens: Setting::Reset,
-            separator_tokens: Setting::Reset,
-            dictionary: Setting::Reset,
            distinct_attribute: Setting::Reset,
            typo_tolerance: Setting::Reset,
            faceting: Setting::Reset,
@@ -229,9 +217,6 @@ impl Settings<Checked> {
            sortable_attributes,
            ranking_rules,
            stop_words,
-            non_separator_tokens,
-            separator_tokens,
-            dictionary,
            synonyms,
            distinct_attribute,
            typo_tolerance,
@@ -247,9 +232,6 @@ impl Settings<Checked> {
            sortable_attributes,
            ranking_rules,
            stop_words,
-            non_separator_tokens,
-            separator_tokens,
-            dictionary,
            synonyms,
            distinct_attribute,
            typo_tolerance,
@@ -292,9 +274,6 @@ impl Settings<Unchecked> {
            ranking_rules: self.ranking_rules,
            stop_words: self.stop_words,
            synonyms: self.synonyms,
-            non_separator_tokens: self.non_separator_tokens,
-            separator_tokens: self.separator_tokens,
-            dictionary: self.dictionary,
            distinct_attribute: self.distinct_attribute,
            typo_tolerance: self.typo_tolerance,
            faceting: self.faceting,
@@ -356,28 +335,6 @@ pub fn apply_settings_to_builder(
        Setting::NotSet => (),
    }

-    match settings.non_separator_tokens {
-        Setting::Set(ref non_separator_tokens) => {
-            builder.set_non_separator_tokens(non_separator_tokens.clone())
-        }
-        Setting::Reset => builder.reset_non_separator_tokens(),
-        Setting::NotSet => (),
-    }
-
-    match settings.separator_tokens {
-        Setting::Set(ref separator_tokens) => {
-            builder.set_separator_tokens(separator_tokens.clone())
-        }
-        Setting::Reset => builder.reset_separator_tokens(),
-        Setting::NotSet => (),
-    }
-
-    match settings.dictionary {
-        Setting::Set(ref dictionary) => builder.set_dictionary(dictionary.clone()),
-        Setting::Reset => builder.reset_dictionary(),
-        Setting::NotSet => (),
-    }
-
    match settings.synonyms {
        Setting::Set(ref synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()),
        Setting::Reset => builder.reset_synonyms(),
@@ -502,14 +459,15 @@ pub fn settings(
        })
        .transpose()?
        .unwrap_or_default();
-
-    let non_separator_tokens = index.non_separator_tokens(rtxn)?.unwrap_or_default();
-    let separator_tokens = index.separator_tokens(rtxn)?.unwrap_or_default();
-    let dictionary = index.dictionary(rtxn)?.unwrap_or_default();
-
    let distinct_field = index.distinct_field(rtxn)?.map(String::from);

-    let synonyms = index.user_defined_synonyms(rtxn)?;
+    // in milli each word in the synonyms map were split on their separator. Since we lost
+    // this information we are going to put space between words.
+    let synonyms = index
+        .synonyms(rtxn)?
+        .iter()
+        .map(|(key, values)| (key.join(" "), values.iter().map(|value| value.join(" ")).collect()))
+        .collect();

    let min_typo_word_len = MinWordSizeTyposSetting {
        one_typo: Setting::Set(index.min_word_len_one_typo(rtxn)?),
@@ -562,9 +520,6 @@ pub fn settings(
        sortable_attributes: Setting::Set(sortable_attributes),
        ranking_rules: Setting::Set(criteria.iter().map(|c| c.clone().into()).collect()),
        stop_words: Setting::Set(stop_words),
-        non_separator_tokens: Setting::Set(non_separator_tokens),
-        separator_tokens: Setting::Set(separator_tokens),
-        dictionary: Setting::Set(dictionary),
        distinct_attribute: match distinct_field {
            Some(field) => Setting::Set(field),
            None => Setting::Reset,
@@ -687,9 +642,6 @@ pub(crate) mod test {
            sortable_attributes: Setting::NotSet,
            ranking_rules: Setting::NotSet,
            stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
            synonyms: Setting::NotSet,
            distinct_attribute: Setting::NotSet,
            typo_tolerance: Setting::NotSet,
@@ -711,9 +663,6 @@ pub(crate) mod test {
            sortable_attributes: Setting::NotSet,
            ranking_rules: Setting::NotSet,
            stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
            synonyms: Setting::NotSet,
            distinct_attribute: Setting::NotSet,
            typo_tolerance: Setting::NotSet,
--- a/meilisearch/Cargo.toml
+++ b/meilisearch/Cargo.toml
@@ -105,6 +105,7 @@ walkdir = "2.3.3"
 yaup = "0.2.1"
 serde_urlencoded = "0.7.1"
 termcolor = "1.2.0"
+zookeeper-client = "0.4.0"

 [dev-dependencies]
 actix-rt = "2.8.0"
--- a/meilisearch/src/analytics/segment_analytics.rs
+++ b/meilisearch/src/analytics/segment_analytics.rs
@@ -312,6 +312,7 @@ impl From<Opt> for Infos {
            config_file_path,
            #[cfg(all(not(debug_assertions), feature = "analytics"))]
                no_analytics: _,
+            zk_url: _,
        } = options;

        let schedule_snapshot = match schedule_snapshot {
--- a/meilisearch/src/lib.rs
+++ b/meilisearch/src/lib.rs
@@ -39,6 +39,7 @@ use meilisearch_types::versioning::{check_version_file, create_version_file};
 use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
 pub use option::Opt;
 use option::ScheduleSnapshot;
+use zookeeper_client as zk;

 use crate::error::MeilisearchHttpError;

@@ -136,14 +137,17 @@ enum OnFailure {
    KeepDb,
 }

-pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<AuthController>)> {
+pub async fn setup_meilisearch(
+    opt: &Opt,
+    zk: Option<zk::Client>,
+) -> anyhow::Result<(Arc<IndexScheduler>, Arc<AuthController>)> {
    let empty_db = is_empty_db(&opt.db_path);
    let (index_scheduler, auth_controller) = if let Some(ref snapshot_path) = opt.import_snapshot {
        let snapshot_path_exists = snapshot_path.exists();
        // the db is empty and the snapshot exists, import it
        if empty_db && snapshot_path_exists {
            match compression::from_tar_gz(snapshot_path, &opt.db_path) {
-                Ok(()) => open_or_create_database_unchecked(opt, OnFailure::RemoveDb)?,
+                Ok(()) => open_or_create_database_unchecked(opt, OnFailure::RemoveDb, zk).await?,
                Err(e) => {
                    std::fs::remove_dir_all(&opt.db_path)?;
                    return Err(e);
@@ -160,14 +164,14 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<
            bail!("snapshot doesn't exist at {}", snapshot_path.display())
        // the snapshot and the db exist, and we can ignore the snapshot because of the ignore_snapshot_if_db_exists flag
        } else {
-            open_or_create_database(opt, empty_db)?
+            open_or_create_database(opt, empty_db, zk).await?
        }
    } else if let Some(ref path) = opt.import_dump {
        let src_path_exists = path.exists();
        // the db is empty and the dump exists, import it
        if empty_db && src_path_exists {
            let (mut index_scheduler, mut auth_controller) =
-                open_or_create_database_unchecked(opt, OnFailure::RemoveDb)?;
+                open_or_create_database_unchecked(opt, OnFailure::RemoveDb, zk).await?;
            match import_dump(&opt.db_path, path, &mut index_scheduler, &mut auth_controller) {
                Ok(()) => (index_scheduler, auth_controller),
                Err(e) => {
@@ -187,10 +191,10 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<
        // the dump and the db exist and we can ignore the dump because of the ignore_dump_if_db_exists flag
        // or, the dump is missing but we can ignore that because of the ignore_missing_dump flag
        } else {
-            open_or_create_database(opt, empty_db)?
+            open_or_create_database(opt, empty_db, zk).await?
        }
    } else {
-        open_or_create_database(opt, empty_db)?
+        open_or_create_database(opt, empty_db, zk).await?
    };

    // We create a loop in a thread that registers snapshotCreation tasks
@@ -214,13 +218,14 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<
 }

 /// Try to start the IndexScheduler and AuthController without checking the VERSION file or anything.
-fn open_or_create_database_unchecked(
+async fn open_or_create_database_unchecked(
    opt: &Opt,
    on_failure: OnFailure,
+    zk: Option<zk::Client>,
 ) -> anyhow::Result<(IndexScheduler, AuthController)> {
    // we don't want to create anything in the data.ms yet, thus we
    // wrap our two builders in a closure that'll be executed later.
-    let auth_controller = AuthController::new(&opt.db_path, &opt.master_key);
+    let auth_controller = AuthController::new(&opt.db_path, &opt.master_key, zk);
    let instance_features = opt.to_instance_features();
    let index_scheduler_builder = || -> anyhow::Result<_> {
        Ok(IndexScheduler::new(IndexSchedulerOptions {
@@ -245,7 +250,7 @@ fn open_or_create_database_unchecked(

    match (
        index_scheduler_builder(),
-        auth_controller.map_err(anyhow::Error::from),
+        auth_controller.await.map_err(anyhow::Error::from),
        create_version_file(&opt.db_path).map_err(anyhow::Error::from),
    ) {
        (Ok(i), Ok(a), Ok(())) => Ok((i, a)),
@@ -259,15 +264,16 @@ fn open_or_create_database_unchecked(
 }

 /// Ensure you're in a valid state and open the IndexScheduler + AuthController for you.
-fn open_or_create_database(
+async fn open_or_create_database(
    opt: &Opt,
    empty_db: bool,
+    zk: Option<zk::Client>,
 ) -> anyhow::Result<(IndexScheduler, AuthController)> {
    if !empty_db {
        check_version_file(&opt.db_path)?;
    }

-    open_or_create_database_unchecked(opt, OnFailure::KeepDb)
+    open_or_create_database_unchecked(opt, OnFailure::KeepDb, zk).await
 }

 fn import_dump(
--- a/meilisearch/src/main.rs
+++ b/meilisearch/src/main.rs
@@ -12,6 +12,7 @@ use meilisearch::analytics::Analytics;
 use meilisearch::{analytics, create_app, prototype_name, setup_meilisearch, Opt};
 use meilisearch_auth::{generate_master_key, AuthController, MASTER_KEY_MIN_SIZE};
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
+use zookeeper_client as zk;

 #[global_allocator]
 static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
@@ -63,7 +64,11 @@ async fn main() -> anyhow::Result<()> {
        _ => (),
    }

-    let (index_scheduler, auth_controller) = setup_meilisearch(&opt)?;
+    let zk = match opt.zk_url {
+        Some(ref url) => Some(zk::Client::connect(url).await.unwrap()),
+        None => None,
+    };
+    let (index_scheduler, auth_controller) = setup_meilisearch(&opt, zk).await?;

    #[cfg(all(not(debug_assertions), feature = "analytics"))]
    let analytics = if !opt.no_analytics {
--- a/meilisearch/src/option.rs
+++ b/meilisearch/src/option.rs
@@ -28,6 +28,7 @@ const MEILI_DB_PATH: &str = "MEILI_DB_PATH";
 const MEILI_HTTP_ADDR: &str = "MEILI_HTTP_ADDR";
 const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY";
 const MEILI_ENV: &str = "MEILI_ENV";
+const MEILI_ZK_URL: &str = "MEILI_ZK_URL";
 #[cfg(all(not(debug_assertions), feature = "analytics"))]
 const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS";
 const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT";
@@ -154,6 +155,11 @@ pub struct Opt {
    #[serde(default = "default_env")]
    pub env: String,

+    /// Sets the HTTP address and port used to communicate with the zookeeper cluster.
+    /// If ran locally, the default url is `http://localhost:2181/`.
+    #[clap(long, env = MEILI_ZK_URL)]
+    pub zk_url: Option<String>,
+
    /// Deactivates Meilisearch's built-in telemetry when provided.
    ///
    /// Meilisearch automatically collects data from all instances that do not opt out using this flag.
@@ -368,6 +374,7 @@ impl Opt {
            http_addr,
            master_key,
            env,
+            zk_url,
            max_index_size: _,
            max_task_db_size: _,
            http_payload_size_limit,
@@ -401,6 +408,9 @@ impl Opt {
            export_to_env_if_not_present(MEILI_MASTER_KEY, master_key);
        }
        export_to_env_if_not_present(MEILI_ENV, env);
+        if let Some(zk_url) = zk_url {
+            export_to_env_if_not_present(MEILI_ZK_URL, zk_url);
+        }
        #[cfg(all(not(debug_assertions), feature = "analytics"))]
        {
            export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string());
@@ -715,6 +725,10 @@ fn default_env() -> String {
    DEFAULT_ENV.to_string()
 }

+pub fn default_zk_url() -> String {
+    DEFAULT_HTTP_ADDR.to_string()
+}
+
 fn default_max_index_size() -> Byte {
    Byte::from_bytes(INDEX_SIZE)
 }
--- a/meilisearch/src/routes/api_key.rs
+++ b/meilisearch/src/routes/api_key.rs
@@ -41,14 +41,10 @@ pub async fn create_api_key(
    _req: HttpRequest,
 ) -> Result<HttpResponse, ResponseError> {
    let v = body.into_inner();
-    let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> {
-        let key = auth_controller.create_key(v)?;
-        Ok(KeyView::from_key(key, &auth_controller))
-    })
-    .await
-    .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??;
+    let key = auth_controller.create_key(v).await?;
+    let key = KeyView::from_key(key, &auth_controller);

-    Ok(HttpResponse::Created().json(res))
+    Ok(HttpResponse::Created().json(key))
 }

 #[derive(Deserr, Debug, Clone, Copy)]
@@ -110,17 +106,11 @@ pub async fn patch_api_key(
 ) -> Result<HttpResponse, ResponseError> {
    let key = path.into_inner().key;
    let patch_api_key = body.into_inner();
-    let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> {
-        let uid =
-            Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?;
-        let key = auth_controller.update_key(uid, patch_api_key)?;
+    let uid = Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?;
+    let key = auth_controller.update_key(uid, patch_api_key).await?;
+    let key = KeyView::from_key(key, &auth_controller);

-        Ok(KeyView::from_key(key, &auth_controller))
-    })
-    .await
-    .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??;
-
-    Ok(HttpResponse::Ok().json(res))
+    Ok(HttpResponse::Ok().json(key))
 }

 pub async fn delete_api_key(
@@ -128,13 +118,8 @@ pub async fn delete_api_key(
    path: web::Path<AuthParam>,
 ) -> Result<HttpResponse, ResponseError> {
    let key = path.into_inner().key;
-    tokio::task::spawn_blocking(move || {
-        let uid =
-            Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?;
-        auth_controller.delete_key(uid)
-    })
-    .await
-    .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??;
+    let uid = Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?;
+    auth_controller.delete_key(uid).await?;

    Ok(HttpResponse::NoContent().finish())
 }
--- a/meilisearch/src/routes/indexes/settings.rs
+++ b/meilisearch/src/routes/indexes/settings.rs
@@ -310,81 +310,6 @@ make_setting_route!(
    }
 );

-make_setting_route!(
-    "/non-separator-tokens",
-    put,
-    std::collections::BTreeSet<String>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsNonSeparatorTokens,
-    >,
-    non_separator_tokens,
-    "nonSeparatorTokens",
-    analytics,
-    |non_separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
-        use serde_json::json;
-
-        analytics.publish(
-            "nonSeparatorTokens Updated".to_string(),
-            json!({
-                "non_separator_tokens": {
-                    "total": non_separator_tokens.as_ref().map(|non_separator_tokens| non_separator_tokens.len()),
-                },
-            }),
-            Some(req),
-        );
-    }
-);
-
-make_setting_route!(
-    "/separator-tokens",
-    put,
-    std::collections::BTreeSet<String>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsSeparatorTokens,
-    >,
-    separator_tokens,
-    "separatorTokens",
-    analytics,
-    |separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
-        use serde_json::json;
-
-        analytics.publish(
-            "separatorTokens Updated".to_string(),
-            json!({
-                "separator_tokens": {
-                    "total": separator_tokens.as_ref().map(|separator_tokens| separator_tokens.len()),
-                },
-            }),
-            Some(req),
-        );
-    }
-);
-
-make_setting_route!(
-    "/dictionary",
-    put,
-    std::collections::BTreeSet<String>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsDictionary,
-    >,
-    dictionary,
-    "dictionary",
-    analytics,
-    |dictionary: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
-        use serde_json::json;
-
-        analytics.publish(
-            "dictionary Updated".to_string(),
-            json!({
-                "dictionary": {
-                    "total": dictionary.as_ref().map(|dictionary| dictionary.len()),
-                },
-            }),
-            Some(req),
-        );
-    }
-);
-
 make_setting_route!(
    "/synonyms",
    put,
@@ -541,9 +466,6 @@ generate_configure!(
    searchable_attributes,
    distinct_attribute,
    stop_words,
-    separator_tokens,
-    non_separator_tokens,
-    dictionary,
    synonyms,
    ranking_rules,
    typo_tolerance,
--- a/meilisearch/src/search.rs
+++ b/meilisearch/src/search.rs
@@ -491,20 +491,6 @@ pub fn perform_search(
        tokenizer_builder.allow_list(&script_lang_map);
    }

-    let separators = index.allowed_separators(&rtxn)?;
-    let separators: Option<Vec<_>> =
-        separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref separators) = separators {
-        tokenizer_builder.separators(separators);
-    }
-
-    let dictionary = index.dictionary(&rtxn)?;
-    let dictionary: Option<Vec<_>> =
-        dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref dictionary) = dictionary {
-        tokenizer_builder.words_dict(dictionary);
-    }
-
    let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_builder.build());
    formatter_builder.crop_marker(query.crop_marker);
    formatter_builder.highlight_prefix(query.highlight_pre_tag);
--- a/meilisearch/tests/common/server.rs
+++ b/meilisearch/tests/common/server.rs
@@ -39,7 +39,7 @@ impl Server {

        let options = default_settings(dir.path());

-        let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
+        let (index_scheduler, auth) = setup_meilisearch(&options, None).await.unwrap();
        let service = Service { index_scheduler, auth, options, api_key: None };

        Server { service, _dir: Some(dir) }
@@ -54,7 +54,7 @@ impl Server {

        options.master_key = Some("MASTER_KEY".to_string());

-        let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
+        let (index_scheduler, auth) = setup_meilisearch(&options, None).await.unwrap();
        let service = Service { index_scheduler, auth, options, api_key: None };

        Server { service, _dir: Some(dir) }
@@ -67,7 +67,7 @@ impl Server {
    }

    pub async fn new_with_options(options: Opt) -> Result<Self, anyhow::Error> {
-        let (index_scheduler, auth) = setup_meilisearch(&options)?;
+        let (index_scheduler, auth) = setup_meilisearch(&options, None).await?;
        let service = Service { index_scheduler, auth, options, api_key: None };

        Ok(Server { service, _dir: None })
--- a/meilisearch/tests/dumps/mod.rs
+++ b/meilisearch/tests/dumps/mod.rs
--- a/meilisearch/tests/settings/get_settings.rs
+++ b/meilisearch/tests/settings/get_settings.rs
@@ -16,9 +16,6 @@ static DEFAULT_SETTINGS_VALUES: Lazy<HashMap<&'static str, Value>> = Lazy::new(|
        json!(["words", "typo", "proximity", "attribute", "sort", "exactness"]),
    );
    map.insert("stop_words", json!([]));
-    map.insert("non_separator_tokens", json!([]));
-    map.insert("separator_tokens", json!([]));
-    map.insert("dictionary", json!([]));
    map.insert("synonyms", json!({}));
    map.insert(
        "faceting",
@@ -54,7 +51,7 @@ async fn get_settings() {
    let (response, code) = index.settings().await;
    assert_eq!(code, 200);
    let settings = response.as_object().unwrap();
-    assert_eq!(settings.keys().len(), 14);
+    assert_eq!(settings.keys().len(), 11);
    assert_eq!(settings["displayedAttributes"], json!(["*"]));
    assert_eq!(settings["searchableAttributes"], json!(["*"]));
    assert_eq!(settings["filterableAttributes"], json!([]));
@@ -65,9 +62,6 @@ async fn get_settings() {
        json!(["words", "typo", "proximity", "attribute", "sort", "exactness"])
    );
    assert_eq!(settings["stopWords"], json!([]));
-    assert_eq!(settings["nonSeparatorTokens"], json!([]));
-    assert_eq!(settings["separatorTokens"], json!([]));
-    assert_eq!(settings["dictionary"], json!([]));
    assert_eq!(
        settings["faceting"],
        json!({
@@ -278,9 +272,6 @@ test_setting_routes!(
    searchable_attributes put,
    distinct_attribute put,
    stop_words put,
-    separator_tokens put,
-    non_separator_tokens put,
-    dictionary put,
    ranking_rules put,
    synonyms put,
    pagination patch,
--- a/meilisearch/tests/settings/mod.rs
+++ b/meilisearch/tests/settings/mod.rs
@@ -1,4 +1,3 @@
 mod distinct;
 mod errors;
 mod get_settings;
-mod tokenizer_customization;
--- a/meilisearch/tests/settings/tokenizer_customization.rs
+++ b/meilisearch/tests/settings/tokenizer_customization.rs
@@ -1,467 +0,0 @@
-use meili_snap::{json_string, snapshot};
-use serde_json::json;
-
-use crate::common::Server;
-
-#[actix_rt::test]
-async fn set_and_reset() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (_response, _code) = index
-        .update_settings(json!({
-            "nonSeparatorTokens": ["#", "&"],
-            "separatorTokens": ["&sep", "<br/>"],
-            "dictionary": ["J.R.R.", "J. R. R."],
-        }))
-        .await;
-    index.wait_task(0).await;
-
-    let (response, _) = index.settings().await;
-    snapshot!(json_string!(response["nonSeparatorTokens"]), @r###"
-    [
-      "#",
-      "&"
-    ]
-    "###);
-    snapshot!(json_string!(response["separatorTokens"]), @r###"
-    [
-      "&sep",
-      "<br/>"
-    ]
-    "###);
-    snapshot!(json_string!(response["dictionary"]), @r###"
-    [
-      "J. R. R.",
-      "J.R.R."
-    ]
-    "###);
-
-    index
-        .update_settings(json!({
-            "nonSeparatorTokens": null,
-            "separatorTokens": null,
-            "dictionary": null,
-        }))
-        .await;
-
-    index.wait_task(1).await;
-
-    let (response, _) = index.settings().await;
-    snapshot!(json_string!(response["nonSeparatorTokens"]), @"[]");
-    snapshot!(json_string!(response["separatorTokens"]), @"[]");
-    snapshot!(json_string!(response["dictionary"]), @"[]");
-}
-
-#[actix_rt::test]
-async fn set_and_search() {
-    let documents = json!([
-        {
-            "id": 1,
-            "content": "Mac & cheese",
-        },
-        {
-            "id": 2,
-            "content": "G#D#G#D#G#C#D#G#C#",
-        },
-        {
-            "id": 3,
-            "content": "Mac&sep&&sepcheese",
-        },
-    ]);
-
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
-    let (_response, _code) = index
-        .update_settings(json!({
-            "nonSeparatorTokens": ["#", "&"],
-            "separatorTokens": ["<br/>", "&sep"],
-            "dictionary": ["#", "A#", "B#", "C#", "D#", "E#", "F#", "G#"],
-        }))
-        .await;
-    index.wait_task(1).await;
-
-    index
-        .search(json!({"q": "&", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 1,
-                "content": "Mac & cheese",
-                "_formatted": {
-                  "id": "1",
-                  "content": "Mac <em>&</em> cheese"
-                }
-              },
-              {
-                "id": 3,
-                "content": "Mac&sep&&sepcheese",
-                "_formatted": {
-                  "id": "3",
-                  "content": "Mac&sep<em>&</em>&sepcheese"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(
-            json!({"q": "Mac & cheese", "attributesToHighlight": ["content"]}),
-            |response, code| {
-                snapshot!(code, @"200 OK");
-                snapshot!(json_string!(response["hits"]), @r###"
-                [
-                  {
-                    "id": 1,
-                    "content": "Mac & cheese",
-                    "_formatted": {
-                      "id": "1",
-                      "content": "<em>Mac</em> <em>&</em> <em>cheese</em>"
-                    }
-                  },
-                  {
-                    "id": 3,
-                    "content": "Mac&sep&&sepcheese",
-                    "_formatted": {
-                      "id": "3",
-                      "content": "<em>Mac</em>&sep<em>&</em>&sep<em>cheese</em>"
-                    }
-                  }
-                ]
-                "###);
-            },
-        )
-        .await;
-
-    index
-        .search(
-            json!({"q": "Mac&sep&&sepcheese", "attributesToHighlight": ["content"]}),
-            |response, code| {
-                snapshot!(code, @"200 OK");
-                snapshot!(json_string!(response["hits"]), @r###"
-                [
-                  {
-                    "id": 1,
-                    "content": "Mac & cheese",
-                    "_formatted": {
-                      "id": "1",
-                      "content": "<em>Mac</em> <em>&</em> <em>cheese</em>"
-                    }
-                  },
-                  {
-                    "id": 3,
-                    "content": "Mac&sep&&sepcheese",
-                    "_formatted": {
-                      "id": "3",
-                      "content": "<em>Mac</em>&sep<em>&</em>&sep<em>cheese</em>"
-                    }
-                  }
-                ]
-                "###);
-            },
-        )
-        .await;
-
-    index
-        .search(json!({"q": "C#D#G", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 2,
-                "content": "G#D#G#D#G#C#D#G#C#",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>G</em>#<em>D#</em><em>G</em>#<em>D#</em><em>G</em>#<em>C#</em><em>D#</em><em>G</em>#<em>C#</em>"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "#", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @"[]");
-        })
-        .await;
-}
-
-#[actix_rt::test]
-async fn advanced_synergies() {
-    let documents = json!([
-        {
-            "id": 1,
-            "content": "J.R.R. Tolkien",
-        },
-        {
-            "id": 2,
-            "content": "J. R. R. Tolkien",
-        },
-        {
-            "id": 3,
-            "content": "jrr Tolkien",
-        },
-        {
-            "id": 4,
-            "content": "J.K. Rowlings",
-        },
-        {
-            "id": 5,
-            "content": "J. K. Rowlings",
-        },
-        {
-            "id": 6,
-            "content": "jk Rowlings",
-        },
-    ]);
-
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
-    let (_response, _code) = index
-        .update_settings(json!({
-            "dictionary": ["J.R.R.", "J. R. R."],
-            "synonyms": {
-                "J.R.R.": ["jrr", "J. R. R."],
-                "J. R. R.": ["jrr", "J.R.R."],
-                "jrr": ["J.R.R.", "J. R. R."],
-                "J.K.": ["jk", "J. K."],
-                "J. K.": ["jk", "J.K."],
-                "jk": ["J.K.", "J. K."],
-            }
-        }))
-        .await;
-    index.wait_task(1).await;
-
-    index
-        .search(json!({"q": "J.R.R.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 1,
-                "content": "J.R.R. Tolkien",
-                "_formatted": {
-                  "id": "1",
-                  "content": "<em>J.R.R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R. R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 3,
-                "content": "jrr Tolkien",
-                "_formatted": {
-                  "id": "3",
-                  "content": "<em>jrr</em> Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "jrr", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 3,
-                "content": "jrr Tolkien",
-                "_formatted": {
-                  "id": "3",
-                  "content": "<em>jrr</em> Tolkien"
-                }
-              },
-              {
-                "id": 1,
-                "content": "J.R.R. Tolkien",
-                "_formatted": {
-                  "id": "1",
-                  "content": "<em>J.R.R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R. R.</em> Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "J. R. R.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R. R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 1,
-                "content": "J.R.R. Tolkien",
-                "_formatted": {
-                  "id": "1",
-                  "content": "<em>J.R.R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 3,
-                "content": "jrr Tolkien",
-                "_formatted": {
-                  "id": "3",
-                  "content": "<em>jrr</em> Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    // Only update dictionary, the synonyms should be recomputed.
-    let (_response, _code) = index
-        .update_settings(json!({
-            "dictionary": ["J.R.R.", "J. R. R.", "J.K.", "J. K."],
-        }))
-        .await;
-    index.wait_task(2).await;
-
-    index
-        .search(json!({"q": "jk", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 6,
-                "content": "jk Rowlings",
-                "_formatted": {
-                  "id": "6",
-                  "content": "<em>jk</em> Rowlings"
-                }
-              },
-              {
-                "id": 4,
-                "content": "J.K. Rowlings",
-                "_formatted": {
-                  "id": "4",
-                  "content": "<em>J.K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 5,
-                "content": "J. K. Rowlings",
-                "_formatted": {
-                  "id": "5",
-                  "content": "<em>J. K.</em> Rowlings"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "J.K.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 4,
-                "content": "J.K. Rowlings",
-                "_formatted": {
-                  "id": "4",
-                  "content": "<em>J.K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 5,
-                "content": "J. K. Rowlings",
-                "_formatted": {
-                  "id": "5",
-                  "content": "<em>J. K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 6,
-                "content": "jk Rowlings",
-                "_formatted": {
-                  "id": "6",
-                  "content": "<em>jk</em> Rowlings"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "J. K.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 5,
-                "content": "J. K. Rowlings",
-                "_formatted": {
-                  "id": "5",
-                  "content": "<em>J. K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 4,
-                "content": "J.K. Rowlings",
-                "_formatted": {
-                  "id": "4",
-                  "content": "<em>J.K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 6,
-                "content": "jk Rowlings",
-                "_formatted": {
-                  "id": "6",
-                  "content": "<em>jk</em> Rowlings"
-                }
-              },
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R.</em> R. Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-}
--- a/milli/src/index.rs
+++ b/milli/src/index.rs
@@ -1,5 +1,5 @@
 use std::borrow::Cow;
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeSet, HashMap, HashSet};
 use std::fs::File;
 use std::mem::size_of;
 use std::path::Path;
@@ -61,12 +61,8 @@ pub mod main_key {
    pub const USER_DEFINED_SEARCHABLE_FIELDS_KEY: &str = "user-defined-searchable-fields";
    pub const SOFT_EXTERNAL_DOCUMENTS_IDS_KEY: &str = "soft-external-documents-ids";
    pub const STOP_WORDS_KEY: &str = "stop-words";
-    pub const NON_SEPARATOR_TOKENS_KEY: &str = "non-separator-tokens";
-    pub const SEPARATOR_TOKENS_KEY: &str = "separator-tokens";
-    pub const DICTIONARY_KEY: &str = "dictionary";
    pub const STRING_FACETED_DOCUMENTS_IDS_PREFIX: &str = "string-faceted-documents-ids";
    pub const SYNONYMS_KEY: &str = "synonyms";
-    pub const USER_DEFINED_SYNONYMS_KEY: &str = "user-defined-synonyms";
    pub const WORDS_FST_KEY: &str = "words-fst";
    pub const WORDS_PREFIXES_FST_KEY: &str = "words-prefixes-fst";
    pub const CREATED_AT_KEY: &str = "created-at";
@@ -1059,116 +1055,18 @@ impl Index {
        }
    }

-    /* non separator tokens */
-
-    pub(crate) fn put_non_separator_tokens(
-        &self,
-        wtxn: &mut RwTxn,
-        set: &BTreeSet<String>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::NON_SEPARATOR_TOKENS_KEY, set)
-    }
-
-    pub(crate) fn delete_non_separator_tokens(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::NON_SEPARATOR_TOKENS_KEY)
-    }
-
-    pub fn non_separator_tokens(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        Ok(self.main.get::<_, Str, SerdeBincode<BTreeSet<String>>>(
-            rtxn,
-            main_key::NON_SEPARATOR_TOKENS_KEY,
-        )?)
-    }
-
-    /* separator tokens */
-
-    pub(crate) fn put_separator_tokens(
-        &self,
-        wtxn: &mut RwTxn,
-        set: &BTreeSet<String>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::SEPARATOR_TOKENS_KEY, set)
-    }
-
-    pub(crate) fn delete_separator_tokens(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::SEPARATOR_TOKENS_KEY)
-    }
-
-    pub fn separator_tokens(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        Ok(self
-            .main
-            .get::<_, Str, SerdeBincode<BTreeSet<String>>>(rtxn, main_key::SEPARATOR_TOKENS_KEY)?)
-    }
-
-    /* separators easing method */
-
-    pub fn allowed_separators(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        let default_separators =
-            charabia::separators::DEFAULT_SEPARATORS.iter().map(|s| s.to_string());
-        let mut separators: Option<BTreeSet<_>> = None;
-        if let Some(mut separator_tokens) = self.separator_tokens(rtxn)? {
-            separator_tokens.extend(default_separators.clone());
-            separators = Some(separator_tokens);
-        }
-
-        if let Some(non_separator_tokens) = self.non_separator_tokens(rtxn)? {
-            separators = separators
-                .or_else(|| Some(default_separators.collect()))
-                .map(|separators| &separators - &non_separator_tokens);
-        }
-
-        Ok(separators)
-    }
-
-    /* dictionary */
-
-    pub(crate) fn put_dictionary(
-        &self,
-        wtxn: &mut RwTxn,
-        set: &BTreeSet<String>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::DICTIONARY_KEY, set)
-    }
-
-    pub(crate) fn delete_dictionary(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::DICTIONARY_KEY)
-    }
-
-    pub fn dictionary(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        Ok(self
-            .main
-            .get::<_, Str, SerdeBincode<BTreeSet<String>>>(rtxn, main_key::DICTIONARY_KEY)?)
-    }
-
    /* synonyms */

    pub(crate) fn put_synonyms(
        &self,
        wtxn: &mut RwTxn,
        synonyms: &HashMap<Vec<String>, Vec<Vec<String>>>,
-        user_defined_synonyms: &BTreeMap<String, Vec<String>>,
    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::SYNONYMS_KEY, synonyms)?;
-        self.main.put::<_, Str, SerdeBincode<_>>(
-            wtxn,
-            main_key::USER_DEFINED_SYNONYMS_KEY,
-            user_defined_synonyms,
-        )
+        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::SYNONYMS_KEY, synonyms)
    }

    pub(crate) fn delete_synonyms(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::SYNONYMS_KEY)?;
-        self.main.delete::<_, Str>(wtxn, main_key::USER_DEFINED_SYNONYMS_KEY)
-    }
-
-    pub fn user_defined_synonyms(
-        &self,
-        rtxn: &RoTxn,
-    ) -> heed::Result<BTreeMap<String, Vec<String>>> {
-        Ok(self
-            .main
-            .get::<_, Str, SerdeBincode<_>>(rtxn, main_key::USER_DEFINED_SYNONYMS_KEY)?
-            .unwrap_or_default())
+        self.main.delete::<_, Str>(wtxn, main_key::SYNONYMS_KEY)
    }

    pub fn synonyms(&self, rtxn: &RoTxn) -> heed::Result<HashMap<Vec<String>, Vec<Vec<String>>>> {
--- a/milli/src/search/new/mod.rs
+++ b/milli/src/search/new/mod.rs
@@ -488,20 +488,6 @@ pub fn execute_search(
            tokbuilder.stop_words(stop_words);
        }

-        let separators = ctx.index.allowed_separators(ctx.txn)?;
-        let separators: Option<Vec<_>> =
-            separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
-        if let Some(ref separators) = separators {
-            tokbuilder.separators(separators);
-        }
-
-        let dictionary = ctx.index.dictionary(ctx.txn)?;
-        let dictionary: Option<Vec<_>> =
-            dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
-        if let Some(ref dictionary) = dictionary {
-            tokbuilder.words_dict(dictionary);
-        }
-
        let script_lang_map = ctx.index.script_language(ctx.txn)?;
        if !script_lang_map.is_empty() {
            tokbuilder.allow_list(&script_lang_map);
--- a/milli/src/search/new/tests/integration.rs
+++ b/milli/src/search/new/tests/integration.rs
@@ -2,7 +2,7 @@ use std::io::Cursor;

 use big_s::S;
 use heed::EnvOpenOptions;
-use maplit::{btreemap, hashset};
+use maplit::{hashmap, hashset};

 use crate::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
 use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
@@ -33,7 +33,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
        S("tag"),
        S("asc_desc_rank"),
    });
-    builder.set_synonyms(btreemap! {
+    builder.set_synonyms(hashmap! {
        S("hello") => vec![S("good morning")],
        S("world") => vec![S("earth")],
        S("america") => vec![S("the united states")],
--- a/milli/src/search/new/tests/proximity.rs
+++ b/milli/src/search/new/tests/proximity.rs
@@ -15,7 +15,7 @@ they store fewer sprximities than the regular word sprximity DB.

 */

-use std::collections::BTreeMap;
+use std::collections::HashMap;

 use crate::index::tests::TempIndex;
 use crate::search::new::tests::collect_field_values;
@@ -336,7 +336,7 @@ fn test_proximity_split_word() {

    index
        .update_settings(|s| {
-            let mut syns = BTreeMap::new();
+            let mut syns = HashMap::new();
            syns.insert("xyz".to_owned(), vec!["sun flower".to_owned()]);
            s.set_synonyms(syns);
        })
--- a/milli/src/search/new/tests/typo.rs
+++ b/milli/src/search/new/tests/typo.rs
@@ -18,7 +18,7 @@ if `words` doesn't exist before it.
 14. Synonyms cost nothing according to the typo ranking rule
 */

-use std::collections::BTreeMap;
+use std::collections::HashMap;

 use crate::index::tests::TempIndex;
 use crate::search::new::tests::collect_field_values;
@@ -591,7 +591,7 @@ fn test_typo_synonyms() {
        .update_settings(|s| {
            s.set_criteria(vec![Criterion::Typo]);

-            let mut synonyms = BTreeMap::new();
+            let mut synonyms = HashMap::new();
            synonyms.insert("lackadaisical".to_owned(), vec!["lazy".to_owned()]);
            synonyms.insert("fast brownish".to_owned(), vec!["quick brown".to_owned()]);

--- a/milli/src/update/index_documents/extract/extract_docid_word_positions.rs
+++ b/milli/src/update/index_documents/extract/extract_docid_word_positions.rs
@@ -28,8 +28,6 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
    indexer: GrenadParameters,
    searchable_fields: &Option<HashSet<FieldId>>,
    stop_words: Option<&fst::Set<&[u8]>>,
-    allowed_separators: Option<&Vec<&str>>,
-    dictionary: Option<&Vec<&str>>,
    max_positions_per_attributes: Option<u32>,
 ) -> Result<(RoaringBitmap, grenad::Reader<File>, ScriptLanguageDocidsMap)> {
    puffin::profile_function!();
@@ -54,14 +52,6 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
    if let Some(stop_words) = stop_words {
        tokenizer_builder.stop_words(stop_words);
    }
-    if let Some(dictionary) = dictionary {
-        // let dictionary: Vec<_> = dictionary.iter().map(String::as_str).collect();
-        tokenizer_builder.words_dict(dictionary.as_slice());
-    }
-    if let Some(separators) = allowed_separators {
-        // let separators: Vec<_> = separators.iter().map(String::as_str).collect();
-        tokenizer_builder.separators(separators.as_slice());
-    }
    let tokenizer = tokenizer_builder.build();

    let mut cursor = obkv_documents.into_cursor()?;
--- a/milli/src/update/index_documents/extract/mod.rs
+++ b/milli/src/update/index_documents/extract/mod.rs
@@ -49,8 +49,6 @@ pub(crate) fn data_from_obkv_documents(
    geo_fields_ids: Option<(FieldId, FieldId)>,
    vectors_field_id: Option<FieldId>,
    stop_words: Option<fst::Set<&[u8]>>,
-    allowed_separators: Option<Vec<&str>>,
-    dictionary: Option<Vec<&str>>,
    max_positions_per_attributes: Option<u32>,
    exact_attributes: HashSet<FieldId>,
 ) -> Result<()> {
@@ -78,8 +76,6 @@ pub(crate) fn data_from_obkv_documents(
                    geo_fields_ids,
                    vectors_field_id,
                    &stop_words,
-                    &allowed_separators,
-                    &dictionary,
                    max_positions_per_attributes,
                )
            })
@@ -293,8 +289,6 @@ fn send_and_extract_flattened_documents_data(
    geo_fields_ids: Option<(FieldId, FieldId)>,
    vectors_field_id: Option<FieldId>,
    stop_words: &Option<fst::Set<&[u8]>>,
-    allowed_separators: &Option<Vec<&str>>,
-    dictionary: &Option<Vec<&str>>,
    max_positions_per_attributes: Option<u32>,
 ) -> Result<(
    grenad::Reader<CursorClonableMmap>,
@@ -350,8 +344,6 @@ fn send_and_extract_flattened_documents_data(
                        indexer,
                        searchable_fields,
                        stop_words.as_ref(),
-                        allowed_separators.as_ref(),
-                        dictionary.as_ref(),
                        max_positions_per_attributes,
                    )?;

--- a/milli/src/update/index_documents/mod.rs
+++ b/milli/src/update/index_documents/mod.rs
@@ -316,12 +316,6 @@ where
        let vectors_field_id = self.index.fields_ids_map(self.wtxn)?.id("_vectors");

        let stop_words = self.index.stop_words(self.wtxn)?;
-        let separators = self.index.allowed_separators(self.wtxn)?;
-        let separators: Option<Vec<_>> =
-            separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
-        let dictionary = self.index.dictionary(self.wtxn)?;
-        let dictionary: Option<Vec<_>> =
-            dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
        let exact_attributes = self.index.exact_attributes_ids(self.wtxn)?;

        let pool_params = GrenadParameters {
@@ -359,8 +353,6 @@ where
                    geo_fields_ids,
                    vectors_field_id,
                    stop_words,
-                    separators,
-                    dictionary,
                    max_positions_per_attributes,
                    exact_attributes,
                )
--- a/milli/src/update/settings.rs
+++ b/milli/src/update/settings.rs
@@ -1,4 +1,4 @@
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeSet, HashMap, HashSet};
 use std::result::Result as StdResult;

 use charabia::{Normalize, Tokenizer, TokenizerBuilder};
@@ -112,11 +112,8 @@ pub struct Settings<'a, 't, 'u, 'i> {
    sortable_fields: Setting<HashSet<String>>,
    criteria: Setting<Vec<Criterion>>,
    stop_words: Setting<BTreeSet<String>>,
-    non_separator_tokens: Setting<BTreeSet<String>>,
-    separator_tokens: Setting<BTreeSet<String>>,
-    dictionary: Setting<BTreeSet<String>>,
    distinct_field: Setting<String>,
-    synonyms: Setting<BTreeMap<String, Vec<String>>>,
+    synonyms: Setting<HashMap<String, Vec<String>>>,
    primary_key: Setting<String>,
    authorize_typos: Setting<bool>,
    min_word_len_two_typos: Setting<u8>,
@@ -144,9 +141,6 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
            sortable_fields: Setting::NotSet,
            criteria: Setting::NotSet,
            stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
            distinct_field: Setting::NotSet,
            synonyms: Setting::NotSet,
            primary_key: Setting::NotSet,
@@ -211,39 +205,6 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
            if stop_words.is_empty() { Setting::Reset } else { Setting::Set(stop_words) }
    }

-    pub fn reset_non_separator_tokens(&mut self) {
-        self.non_separator_tokens = Setting::Reset;
-    }
-
-    pub fn set_non_separator_tokens(&mut self, non_separator_tokens: BTreeSet<String>) {
-        self.non_separator_tokens = if non_separator_tokens.is_empty() {
-            Setting::Reset
-        } else {
-            Setting::Set(non_separator_tokens)
-        }
-    }
-
-    pub fn reset_separator_tokens(&mut self) {
-        self.separator_tokens = Setting::Reset;
-    }
-
-    pub fn set_separator_tokens(&mut self, separator_tokens: BTreeSet<String>) {
-        self.separator_tokens = if separator_tokens.is_empty() {
-            Setting::Reset
-        } else {
-            Setting::Set(separator_tokens)
-        }
-    }
-
-    pub fn reset_dictionary(&mut self) {
-        self.dictionary = Setting::Reset;
-    }
-
-    pub fn set_dictionary(&mut self, dictionary: BTreeSet<String>) {
-        self.dictionary =
-            if dictionary.is_empty() { Setting::Reset } else { Setting::Set(dictionary) }
-    }
-
    pub fn reset_distinct_field(&mut self) {
        self.distinct_field = Setting::Reset;
    }
@@ -256,7 +217,7 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
        self.synonyms = Setting::Reset;
    }

-    pub fn set_synonyms(&mut self, synonyms: BTreeMap<String, Vec<String>>) {
+    pub fn set_synonyms(&mut self, synonyms: HashMap<String, Vec<String>>) {
        self.synonyms = if synonyms.is_empty() { Setting::Reset } else { Setting::Set(synonyms) }
    }

@@ -491,84 +452,9 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
        }
    }

-    fn update_non_separator_tokens(&mut self) -> Result<bool> {
-        let changes = match self.non_separator_tokens {
-            Setting::Set(ref non_separator_tokens) => {
-                let current = self.index.non_separator_tokens(self.wtxn)?;
-
-                // Does the new list differ from the previous one?
-                if current.map_or(true, |current| &current != non_separator_tokens) {
-                    self.index.put_non_separator_tokens(self.wtxn, non_separator_tokens)?;
-                    true
-                } else {
-                    false
-                }
-            }
-            Setting::Reset => self.index.delete_non_separator_tokens(self.wtxn)?,
-            Setting::NotSet => false,
-        };
-
-        // the synonyms must be updated if non separator tokens have been updated.
-        if changes && self.synonyms == Setting::NotSet {
-            self.synonyms = Setting::Set(self.index.user_defined_synonyms(self.wtxn)?);
-        }
-
-        Ok(changes)
-    }
-
-    fn update_separator_tokens(&mut self) -> Result<bool> {
-        let changes = match self.separator_tokens {
-            Setting::Set(ref separator_tokens) => {
-                let current = self.index.separator_tokens(self.wtxn)?;
-
-                // Does the new list differ from the previous one?
-                if current.map_or(true, |current| &current != separator_tokens) {
-                    self.index.put_separator_tokens(self.wtxn, separator_tokens)?;
-                    true
-                } else {
-                    false
-                }
-            }
-            Setting::Reset => self.index.delete_separator_tokens(self.wtxn)?,
-            Setting::NotSet => false,
-        };
-
-        // the synonyms must be updated if separator tokens have been updated.
-        if changes && self.synonyms == Setting::NotSet {
-            self.synonyms = Setting::Set(self.index.user_defined_synonyms(self.wtxn)?);
-        }
-
-        Ok(changes)
-    }
-
-    fn update_dictionary(&mut self) -> Result<bool> {
-        let changes = match self.dictionary {
-            Setting::Set(ref dictionary) => {
-                let current = self.index.dictionary(self.wtxn)?;
-
-                // Does the new list differ from the previous one?
-                if current.map_or(true, |current| &current != dictionary) {
-                    self.index.put_dictionary(self.wtxn, dictionary)?;
-                    true
-                } else {
-                    false
-                }
-            }
-            Setting::Reset => self.index.delete_dictionary(self.wtxn)?,
-            Setting::NotSet => false,
-        };
-
-        // the synonyms must be updated if dictionary has been updated.
-        if changes && self.synonyms == Setting::NotSet {
-            self.synonyms = Setting::Set(self.index.user_defined_synonyms(self.wtxn)?);
-        }
-
-        Ok(changes)
-    }
-
    fn update_synonyms(&mut self) -> Result<bool> {
        match self.synonyms {
-            Setting::Set(ref user_synonyms) => {
+            Setting::Set(ref synonyms) => {
                fn normalize(tokenizer: &Tokenizer, text: &str) -> Vec<String> {
                    tokenizer
                        .tokenize(text)
@@ -587,25 +473,10 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
                if let Some(ref stop_words) = stop_words {
                    builder.stop_words(stop_words);
                }
-
-                let separators = self.index.allowed_separators(self.wtxn)?;
-                let separators: Option<Vec<_>> =
-                    separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
-                if let Some(ref separators) = separators {
-                    builder.separators(separators);
-                }
-
-                let dictionary = self.index.dictionary(self.wtxn)?;
-                let dictionary: Option<Vec<_>> =
-                    dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
-                if let Some(ref dictionary) = dictionary {
-                    builder.words_dict(dictionary);
-                }
-
                let tokenizer = builder.build();

                let mut new_synonyms = HashMap::new();
-                for (word, synonyms) in user_synonyms {
+                for (word, synonyms) in synonyms {
                    // Normalize both the word and associated synonyms.
                    let normalized_word = normalize(&tokenizer, word);
                    let normalized_synonyms =
@@ -626,7 +497,7 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
                let old_synonyms = self.index.synonyms(self.wtxn)?;

                if new_synonyms != old_synonyms {
-                    self.index.put_synonyms(self.wtxn, &new_synonyms, user_synonyms)?;
+                    self.index.put_synonyms(self.wtxn, &new_synonyms)?;
                    Ok(true)
                } else {
                    Ok(false)
@@ -886,17 +757,11 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
        let faceted_updated = old_faceted_fields != new_faceted_fields;

        let stop_words_updated = self.update_stop_words()?;
-        let non_separator_tokens_updated = self.update_non_separator_tokens()?;
-        let separator_tokens_updated = self.update_separator_tokens()?;
-        let dictionary_updated = self.update_dictionary()?;
        let synonyms_updated = self.update_synonyms()?;
        let searchable_updated = self.update_searchable()?;
        let exact_attributes_updated = self.update_exact_attributes()?;

        if stop_words_updated
-            || non_separator_tokens_updated
-            || separator_tokens_updated
-            || dictionary_updated
            || faceted_updated
            || synonyms_updated
            || searchable_updated
@@ -913,7 +778,7 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
 mod tests {
    use big_s::S;
    use heed::types::ByteSlice;
-    use maplit::{btreemap, btreeset, hashset};
+    use maplit::{btreeset, hashmap, hashset};

    use super::*;
    use crate::error::Error;
@@ -1379,7 +1244,7 @@ mod tests {
        // In the same transaction provide some synonyms
        index
            .update_settings_using_wtxn(&mut wtxn, |settings| {
-                settings.set_synonyms(btreemap! {
+                settings.set_synonyms(hashmap! {
                    "blini".to_string() => vec!["crepes".to_string()],
                    "super like".to_string() => vec!["love".to_string()],
                    "puppies".to_string() => vec!["dogs".to_string(), "doggos".to_string()]
@@ -1675,9 +1540,6 @@ mod tests {
                    sortable_fields,
                    criteria,
                    stop_words,
-                    non_separator_tokens,
-                    separator_tokens,
-                    dictionary,
                    distinct_field,
                    synonyms,
                    primary_key,
@@ -1696,9 +1558,6 @@ mod tests {
                assert!(matches!(sortable_fields, Setting::NotSet));
                assert!(matches!(criteria, Setting::NotSet));
                assert!(matches!(stop_words, Setting::NotSet));
-                assert!(matches!(non_separator_tokens, Setting::NotSet));
-                assert!(matches!(separator_tokens, Setting::NotSet));
-                assert!(matches!(dictionary, Setting::NotSet));
                assert!(matches!(distinct_field, Setting::NotSet));
                assert!(matches!(synonyms, Setting::NotSet));
                assert!(matches!(primary_key, Setting::NotSet));
--- a/milli/tests/search/mod.rs
+++ b/milli/tests/search/mod.rs
@@ -5,7 +5,7 @@ use std::io::Cursor;
 use big_s::S;
 use either::{Either, Left, Right};
 use heed::EnvOpenOptions;
-use maplit::{btreemap, hashset};
+use maplit::{hashmap, hashset};
 use milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
 use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
 use milli::{AscDesc, Criterion, DocumentId, Index, Member, Object, TermsMatchingStrategy};
@@ -51,7 +51,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
        S("tag"),
        S("asc_desc_rank"),
    });
-    builder.set_synonyms(btreemap! {
+    builder.set_synonyms(hashmap! {
        S("hello") => vec![S("good morning")],
        S("world") => vec![S("earth")],
        S("america") => vec![S("the united states")],
Author	SHA1	Message	Date
ManyTheFish	ad7f8edff8	fix auto-synchronization with zk	2023-08-03 14:43:29 +02:00
Tamo	5ce01bcb53	add logs	2023-08-03 13:59:05 +02:00
Tamo	d5523cc6ac	fix the tests	2023-08-03 12:28:08 +02:00
Tamo	fe7a312ec6	Import the already existing api keys on startup	2023-08-03 12:25:32 +02:00
Tamo	57dc4b148c	implement the watcher for all kind of operations	2023-08-03 10:52:13 +02:00
Tamo	a325ddfe6a	Forward the key deletions to zookeeper	2023-08-03 10:36:49 +02:00
Tamo	0cd81573b4	Forward the keys update to zookeeper	2023-08-03 10:22:34 +02:00
ManyTheFish	b0ff595f60	Event Listener: delete local key if deleted on ZK	2023-08-02 18:36:36 +02:00
ManyTheFish	3eb6f4b56f	Create api keys	2023-08-02 16:52:45 +02:00
ManyTheFish	49f976c8d8	fix analitics compilation	2023-08-02 14:17:03 +02:00
Tamo	84d56f3320	send the creation of api-key to zookeeper	2023-08-02 13:57:30 +02:00
Tamo	97e3dfd99d	makes zk available inside the auth-controller with config coming from the cli, it compiles	2023-08-02 13:17:40 +02:00
ManyTheFish	dc38da95c4	WIP	2023-08-02 12:00:02 +02:00
ManyTheFish	2ce8b42757	REMOVE: add docker compôse for tests	2023-08-02 11:59:54 +02:00