snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-27 08:41:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

highlight with new tokenizer

Browse Source
This commit is contained in:
mpostma
2020-12-23 20:04:19 +01:00
parent 1ae761311e
commit 4f7f7538f7
6 changed files with 49 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/update/index_documents/store.rs
View File

@ -8,21 +8,21 @@ use std::{cmp, iter};
use anyhow::{bail, Context};
use bstr::ByteSlice as _;
use fst::Set;
use grenad::{Reader, FileFuse, Writer, Sorter, CompressionType};
use heed::BytesEncode;
use linked_hash_map::LinkedHashMap;
use log::{debug, info};
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
use ordered_float::OrderedFloat;
use roaring::RoaringBitmap;
use serde_json::Value;
use tempfile::tempfile;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, TokenKind};
use fst::Set;
use crate::facet::FacetType;
use crate::heed_codec::{BoRoaringBitmapCodec, CboRoaringBitmapCodec};
use crate::heed_codec::facet::{FacetValueStringCodec, FacetLevelValueF64Codec, FacetLevelValueI64Codec};
use crate::heed_codec::facet::{FieldDocIdFacetStringCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetI64Codec};
use crate::heed_codec::{BoRoaringBitmapCodec, CboRoaringBitmapCodec};
use crate::update::UpdateIndexingStep;
use crate::{json_to_string, SmallVec8, SmallVec32, SmallString32, Position, DocumentId, FieldId};
@ -167,7 +167,7 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
// MTBL writers
docid_word_positions_writer,
documents_writer,
//tokenizer
// tokenizer
analyzer,
})
}