snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-26 16:21:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use the zstd library directly to be able to define the compression level

Browse Source
This commit is contained in:
Clément Renault
2024-07-02 17:34:02 +02:00
parent b15e8aacb6
commit e18b06ddda
13 changed files with 503 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
index-scheduler/src/batch.rs
View File

@ -37,6 +37,7 @@ use meilisearch_types::milli::vector::parsed_vectors::{
use meilisearch_types::milli::{self, Filter};
use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};
use meilisearch_types::zstd::dict::DecoderDictionary;
use meilisearch_types::{compression, Index, VERSION_FILE_NAME};
use roaring::RoaringBitmap;
use time::macros::format_description;
@ -908,7 +909,8 @@ impl IndexScheduler {
let mut index_dumper = dump.create_index(uid, &metadata)?;
let fields_ids_map = index.fields_ids_map(&rtxn)?;
let dictionary = index.document_compression_dictionary(&rtxn)?;
let dictionary =
index.document_compression_dictionary(&rtxn)?.map(DecoderDictionary::copy);
let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
let embedding_configs = index.embedding_configs(&rtxn)?;
let mut buffer = Vec::new();
@ -920,9 +922,9 @@ impl IndexScheduler {
}
let (id, compressed) = ret?;
let doc = match dictionary {
let doc = match dictionary.as_ref() {
// TODO manage this unwrap correctly
Some(dict) => compressed.decompress_with(&mut buffer, dict).unwrap(),
Some(dict) => compressed.decompress_with(&mut buffer, dict)?,
None => compressed.as_non_compressed(),
};