feat: Introduce a basic http service

raptor-search/src/main.rs

@@ -1,14 +1,33 @@
 use std::env;
+use std::fs::File;
+use std::path::Path;
+use std::collections::HashSet;
 use std::str::from_utf8_unchecked;
-use std::io::{self, Write};
+use std::io::{self, BufReader, BufRead, Write};
 use elapsed::measure_time;
 use fst::Streamer;
 use rocksdb::{DB, DBOptions, IngestExternalFileOptions};
 use raptor::{automaton, Metadata, RankedStream};
 
-fn search(metadata: &Metadata, database: &DB, query: &str) {
+type CommonWords = HashSet<String>;
+
+fn common_words<P>(path: P) -> io::Result<CommonWords>
+where P: AsRef<Path>,
+{
+    let file = File::open(path)?;
+    let file = BufReader::new(file);
+    let mut set = HashSet::new();
+    for line in file.lines().filter_map(|l| l.ok()) {
+        for word in line.split_whitespace() {
+            set.insert(word.to_owned());
+        }
+    }
+    Ok(set)
+}
+
+fn search(metadata: &Metadata, database: &DB, common_words: &CommonWords, query: &str) {
     let mut automatons = Vec::new();
-    for query in query.split_whitespace() {
+    for query in query.split_whitespace().filter(|q| !common_words.contains(*q)) {
         let lev = automaton::build(query);
         automatons.push(lev);
     }
@@ -47,6 +66,12 @@ fn main() {
     });
     println!("{} to load the SST file in RocksDB and reopen it for read-only", elapsed);
 
+    let common_path = "fr.stopwords.txt";
+    let common_words = common_words(common_path).unwrap_or_else(|e| {
+        println!("{:?}: {:?}", common_path, e);
+        HashSet::new()
+    });
+
     loop {
         print!("Searching for: ");
         io::stdout().flush().unwrap();
@@ -57,7 +82,7 @@ fn main() {
 
         if query.is_empty() { break }
 
-        let (elapsed, _) = measure_time(|| search(&meta, &db, &query));
+        let (elapsed, _) = measure_time(|| search(&meta, &db, &common_words, &query));
         println!("Finished in {}", elapsed);
     }
 }