snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-09-05 20:26:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Expose Hannoy progress when upgrading

Browse Source
This commit is contained in:
Clément Renault
2025-08-12 14:24:50 +02:00
committed by Louis Dureuil
parent de90455809
commit e4b28464fd
3 changed files with 6 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
crates/milli/src/progress.rs
View File

@ -269,47 +269,6 @@ impl<U: Send + Sync + 'static> Step for VariableNameStep<U> {
}
}
// impl Step for hannoy::MainStep {
// fn name(&self) -> Cow<'static, str> {
// match self {
// hannoy::MainStep::PreProcessingTheItems => "pre processing the items",
// hannoy::MainStep::WritingTheDescendantsAndMetadata => {
// "writing the descendants and metadata"
// }
// hannoy::MainStep::RetrieveTheUpdatedItems => "retrieve the updated items",
// hannoy::MainStep::RetrievingTheTreeAndItemNodes => "retrieving the tree and item nodes",
// hannoy::MainStep::UpdatingTheTrees => "updating the trees",
// hannoy::MainStep::CreateNewTrees => "create new trees",
// hannoy::MainStep::WritingNodesToDatabase => "writing nodes to database",
// hannoy::MainStep::DeleteExtraneousTrees => "delete extraneous trees",
// hannoy::MainStep::WriteTheMetadata => "write the metadata",
// }
// .into()
// }
// fn current(&self) -> u32 {
// *self as u32
// }
// fn total(&self) -> u32 {
// Self::CARDINALITY as u32
// }
// }
// impl Step for hannoy::SubStep {
// fn name(&self) -> Cow<'static, str> {
// self.unit.into()
// }
// fn current(&self) -> u32 {
// self.current.load(Ordering::Relaxed)
// }
// fn total(&self) -> u32 {
// self.max
// }
// }
// Integration with steppe
impl steppe::Progress for Progress {
@ -322,7 +281,7 @@ struct Compat<T: steppe::Step>(T);
impl<T: steppe::Step> Step for Compat<T> {
fn name(&self) -> Cow<'static, str> {
self.0.name().into()
self.0.name()
}
fn current(&self) -> u32 {

4
crates/milli/src/update/upgrade/v1_18.rs
View File

@ -14,7 +14,7 @@ impl UpgradeIndex for Latest_V1_17_To_V1_18_0 {
wtxn: &mut RwTxn,
index: &Index,
_original: (u32, u32, u32),
_progress: Progress,
progress: Progress,
) -> Result<bool> {
let embedding_configs = index.embedding_configs();
for config in embedding_configs.embedding_configs(wtxn)? {
@ -22,7 +22,7 @@ impl UpgradeIndex for Latest_V1_17_To_V1_18_0 {
let quantized = config.config.quantized();
let embedder_id = embedding_configs.embedder_id(wtxn, &config.name)?.unwrap();
let vector_store = VectorStore::new(index.vector_store, embedder_id, quantized);
vector_store.convert_from_arroy(wtxn)?;
vector_store.convert_from_arroy(wtxn, progress.clone())?;
}
Ok(false)

6
crates/milli/src/vector/mod.rs
View File

@ -155,7 +155,7 @@ impl VectorStore {
}
}
pub fn convert_from_arroy(&self, wtxn: &mut RwTxn) -> crate::Result<()> {
pub fn convert_from_arroy(&self, wtxn: &mut RwTxn, progress: Progress) -> crate::Result<()> {
if self.quantized {
let dimensions = self
.arroy_readers(wtxn, self.arroy_quantized_db())
@ -168,7 +168,7 @@ impl VectorStore {
for index in vector_store_range_for_embedder(self.embedder_index) {
let mut rng = rand::rngs::StdRng::from_entropy();
let writer = hannoy::Writer::new(self.quantized_db(), index, dimensions);
let mut builder = writer.builder(&mut rng);
let mut builder = writer.builder(&mut rng).progress(progress.clone());
builder.prepare_arroy_conversion(wtxn)?;
builder.build::<HANNOY_M, HANNOY_M0>(wtxn)?;
}
@ -186,7 +186,7 @@ impl VectorStore {
for index in vector_store_range_for_embedder(self.embedder_index) {
let mut rng = rand::rngs::StdRng::from_entropy();
let writer = hannoy::Writer::new(self.angular_db(), index, dimensions);
let mut builder = writer.builder(&mut rng);
let mut builder = writer.builder(&mut rng).progress(progress.clone());
builder.prepare_arroy_conversion(wtxn)?;
builder.build::<HANNOY_M, HANNOY_M0>(wtxn)?;
}