start distributing meilisearch

cluster/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "cluster"
+publish = false
+
+version.workspace = true
+authors.workspace = true
+description.workspace = true
+homepage.workspace = true
+readme.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+ductile = "0.3.0"
+serde = { version = "1.0.155", features = ["derive"] }
+serde_json = "1.0.94"
+thiserror = "1.0.39"
+meilisearch-types = { path = "../meilisearch-types" }
+roaring = "0.10.1"
+log = "0.4.17"

cluster/src/leader.rs

@@ -0,0 +1,111 @@
+use std::net::ToSocketAddrs;
+use std::time::Duration;
+
+use ductile::{ChannelReceiver, ChannelSender, ChannelServer};
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+
+use crate::{Consistency, Error, FollowerMsg, LeaderMsg};
+
+pub struct Leader {
+    listener: ChannelServer<LeaderMsg, FollowerMsg>,
+    active_followers: Vec<Follower>,
+    new_followers: Vec<Follower>,
+    dead_followers: Vec<Follower>,
+
+    batch_id: u32,
+    tick: Duration,
+}
+
+struct Follower {
+    sender: ChannelSender<LeaderMsg>,
+    receiver: ChannelReceiver<FollowerMsg>,
+}
+
+impl Leader {
+    pub fn new(listen_on: impl ToSocketAddrs) -> Leader {
+        let listener = ChannelServer::bind(listen_on).unwrap();
+
+        Leader {
+            listener,
+            active_followers: Vec::new(),
+            new_followers: Vec::new(),
+            dead_followers: Vec::new(),
+            batch_id: 0,
+            tick: Duration::new(1, 0),
+        }
+    }
+
+    pub fn starts_batch(&mut self, batch: Vec<u32>) -> Result<(), Error> {
+        let mut dead_nodes = Vec::new();
+
+        for (idx, follower) in self.active_followers.iter_mut().enumerate() {
+            match follower
+                .sender
+                .send(LeaderMsg::StartBatch { id: self.batch_id, batch: batch.clone() })
+            {
+                Ok(_) => (),
+                // if a node can't be joined we consider it as dead
+                Err(_) => dead_nodes.push(idx),
+            }
+        }
+
+        // we do it from the end so the indices stays correct while removing elements
+        for dead_node in dead_nodes.into_iter().rev() {
+            let dead = self.active_followers.swap_remove(dead_node);
+            self.dead_followers.push(dead);
+        }
+
+        Ok(())
+    }
+
+    pub fn commit(&mut self, consistency_level: Consistency) -> Result<(), Error> {
+        let mut dead_nodes = Vec::new();
+        let mut ready_to_commit = 0;
+        // get the size of the cluster to compute what a quorum means
+        // it's mutable because if followers die we must remove them
+        // from the quorum
+        let mut cluster_size = self.active_followers.len();
+
+        // wait till enough nodes are ready to commit
+        for (idx, follower) in self.active_followers.iter_mut().enumerate() {
+            match consistency_level {
+                Consistency::Zero => break,
+                Consistency::One if ready_to_commit >= 1 => break,
+                Consistency::Two if ready_to_commit >= 2 => break,
+                Consistency::Quorum if ready_to_commit >= (cluster_size / 2) => break,
+                _ => (),
+            }
+            match follower.receiver.recv() {
+                Ok(FollowerMsg::ReadyToCommit(id)) if id == self.batch_id => ready_to_commit += 1,
+                Ok(FollowerMsg::RegisterNewTask(_)) => log::warn!("Missed a task"),
+                Ok(_) => (),
+                // if a node can't be joined we consider it as dead
+                Err(_) => {
+                    dead_nodes.push(idx);
+                    cluster_size -= 1
+                }
+            }
+        }
+
+        let dn = dead_nodes.clone();
+        for (idx, follower) in
+            self.active_followers.iter_mut().enumerate().filter(|(i, _)| !dn.contains(i))
+        {
+            match follower.sender.send(LeaderMsg::Commit(self.batch_id)) {
+                Ok(_) => (),
+                Err(_) => dead_nodes.push(idx),
+            }
+        }
+
+        // we do it from the end so the indices stays correct while removing elements
+        for dead_node in dead_nodes.into_iter().rev() {
+            let dead = self.active_followers.swap_remove(dead_node);
+            self.dead_followers.push(dead);
+        }
+
+        self.batch_id += 1;
+
+        Ok(())
+    }
+}

cluster/src/lib.rs

@@ -0,0 +1,79 @@
+use std::net::ToSocketAddrs;
+
+use ductile::{connect_channel, ChannelReceiver, ChannelSender};
+use meilisearch_types::tasks::KindWithContent;
+use serde::{Deserialize, Serialize};
+
+mod leader;
+
+pub use leader::Leader;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("Network issue occured")]
+    NetworkIssue,
+    #[error("Internal error: {0}")]
+    SerdeJson(#[from] serde_json::Error),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum LeaderMsg {
+    // Starts a new batch
+    StartBatch { id: u32, batch: Vec<u32> },
+    // Tell the follower to commit the update asap
+    Commit(u32),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum FollowerMsg {
+    // Let the leader knows you're ready to commit
+    ReadyToCommit(u32),
+    RegisterNewTask(KindWithContent),
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Consistency {
+    Zero,
+    One,
+    Two,
+    Quorum,
+    All,
+}
+
+pub struct Follower {
+    sender: ChannelSender<FollowerMsg>,
+    receiver: ChannelReceiver<LeaderMsg>,
+    batch_id: u32,
+}
+
+impl Follower {
+    pub fn join(leader: impl ToSocketAddrs) -> Follower {
+        let (sender, receiver) = connect_channel(leader).unwrap();
+        Follower { sender, receiver, batch_id: 0 }
+    }
+
+    pub fn get_new_batch(&mut self) -> Vec<u32> {
+        loop {
+            match self.receiver.recv() {
+                Ok(LeaderMsg::StartBatch { id, batch }) if id == self.batch_id => {
+                    self.batch_id = id;
+                    break batch;
+                }
+                Err(_) => log::error!("lost connection to the leader"),
+                _ => (),
+            }
+        }
+    }
+
+    pub fn ready_to_commit(&mut self) {
+        self.sender.send(FollowerMsg::ReadyToCommit(self.batch_id)).unwrap();
+
+        loop {
+            match self.receiver.recv() {
+                Ok(LeaderMsg::Commit(id)) if id == self.batch_id => break,
+                Err(_) => panic!("lost connection to the leader"),
+                _ => (),
+            }
+        }
+    }
+}