snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-09-27 07:16:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Introduce RocksDB in this project

Browse Source 
in order to save fields of the products
This commit is contained in:
Clément Renault
2018-07-06 21:26:07 +02:00
parent b9a4be10c6
commit d6e113c683
6 changed files with 354 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
raptor-search/src/main.rs
View File

@ -1,15 +1,18 @@
extern crate env_logger;
extern crate rocksdb;
extern crate fst;
extern crate raptor;
extern crate elapsed;
use std::env;
use std::str::from_utf8_unchecked;
use std::io::{self, Write};
use elapsed::measure_time;
use fst::Streamer;
use rocksdb::{DB, DBOptions};
use raptor::{load_map, DocIndexMap, RankedStream, LevBuilder};
fn search(map: &DocIndexMap, lev_builder: &LevBuilder, query: &str) {
fn search(map: &DocIndexMap, lev_builder: &LevBuilder, db: &DB, query: &str) {
let mut automatons = Vec::new();
for query in query.split_whitespace() {
let lev = lev_builder.get_automaton(query);
@ -18,26 +21,12 @@ fn search(map: &DocIndexMap, lev_builder: &LevBuilder, query: &str) {
let mut stream = RankedStream::new(&map, map.values(), automatons, 20);
while let Some(document_id) = stream.next() {
print!("{:?}", document_id);
print!("{:?} ", document_id);
// /* only here to debug !
use std::{fs, process::Command};
if let Ok(_) = fs::File::open("products.json_lines") {
let output = Command::new("rg")
.arg(document_id.to_string())
.arg("products.json_lines")
.output();
if let Ok(Ok(output)) = output.map(|o| String::from_utf8(o.stdout)) {
if let Some(line) = output.lines().next() {
let pattern = "\"title\":";
if let Some(index) = line.find(pattern) {
let line: String = line[index..].chars().skip(pattern.len()).take(100).collect();
print!(" => {}", line);
}
}
}
}
// */
let title_key = format!("{}-title", document_id);
let title = db.get(title_key.as_bytes()).unwrap().unwrap();
let title = unsafe { from_utf8_unchecked(&title) };
print!("{:?}", title);
println!();
}
@ -52,11 +41,18 @@ fn main() {
let (elapsed, lev_builder) = measure_time(|| LevBuilder::new());
println!("{} to load the levenshtein automaton", elapsed);
let (elapsed, db) = measure_time(|| {
let opts = DBOptions::new();
let error_if_log_file_exist = false;
DB::open_for_read_only(opts, "rocksdb/storage", error_if_log_file_exist).unwrap()
});
println!("{} to load the rocksdb DB", elapsed);
match env::args().nth(1) {
Some(query) => {
println!("Searching for: {:?}", query);
let query = query.to_lowercase();
let (elapsed, _) = measure_time(|| search(&map, &lev_builder, &query));
let (elapsed, _) = measure_time(|| search(&map, &lev_builder, &db, &query));
println!("Finished in {}", elapsed);
},
None => loop {
@ -69,7 +65,7 @@ fn main() {
if query.is_empty() { break }
let (elapsed, _) = measure_time(|| search(&map, &lev_builder, &query));
let (elapsed, _) = measure_time(|| search(&map, &lev_builder, &db, &query));
println!("Finished in {}", elapsed);
},
}