mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-10-10 13:46:28 +00:00
WIP: Add metadata field with queryUid, indexUid, primaryKey, and remote
- Add SearchMetadata struct with queryUid, indexUid, primaryKey, and remote fields - Update SearchResult to include metadata field - Update FederatedSearchResult to include metadata array - Refactor federated search metadata building to maintain query order - Support primary key extraction from both local and remote results - Add remote field to identify remote instance for federated queries - Ensure metadata array matches query order in federated search Features: - queryUid: UUID v7 for each query - indexUid: Index identifier - primaryKey: Primary key field name (null if not available) - remote: Remote instance name (null for local queries) This provides complete traceability for search operations across local and remote instances.
This commit is contained in:
@@ -60,18 +60,21 @@ pub async fn perform_federated_search(
|
|||||||
|
|
||||||
// 1. partition queries by host and index
|
// 1. partition queries by host and index
|
||||||
let mut partitioned_queries = PartitionedQueries::new();
|
let mut partitioned_queries = PartitionedQueries::new();
|
||||||
let mut query_metadata = Vec::new();
|
|
||||||
|
// Store the original queries order for later metadata building
|
||||||
|
let original_queries = queries.clone();
|
||||||
|
|
||||||
for (query_index, federated_query) in queries.into_iter().enumerate() {
|
for (query_index, federated_query) in queries.into_iter().enumerate() {
|
||||||
let query_uid = Uuid::now_v7();
|
|
||||||
query_metadata
|
|
||||||
.push(SearchMetadata { query_uid, index_uid: federated_query.index_uid.to_string() });
|
|
||||||
partitioned_queries.partition(federated_query, query_index, &network, features)?
|
partitioned_queries.partition(federated_query, query_index, &network, features)?
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. perform queries, merge and make hits index by index
|
// 2. perform queries, merge and make hits index by index
|
||||||
// 2.1. start remote queries
|
// 2.1. start remote queries
|
||||||
let remote_search =
|
let remote_search = RemoteSearch::start(
|
||||||
RemoteSearch::start(partitioned_queries.remote_queries_by_host, &federation, deadline);
|
partitioned_queries.remote_queries_by_host.clone(),
|
||||||
|
&federation,
|
||||||
|
deadline,
|
||||||
|
);
|
||||||
|
|
||||||
// 2.2. concurrently execute local queries
|
// 2.2. concurrently execute local queries
|
||||||
let params = SearchByIndexParams {
|
let params = SearchByIndexParams {
|
||||||
@@ -117,6 +120,50 @@ pub async fn perform_federated_search(
|
|||||||
let (estimated_total_hits, degraded, used_negative_operator, facets, max_remote_duration) =
|
let (estimated_total_hits, degraded, used_negative_operator, facets, max_remote_duration) =
|
||||||
merge_metadata(&mut results_by_index, &remote_results);
|
merge_metadata(&mut results_by_index, &remote_results);
|
||||||
|
|
||||||
|
// 3.1.1. Build metadata in the same order as the original queries
|
||||||
|
let mut query_metadata = Vec::new();
|
||||||
|
|
||||||
|
// Create a map of remote results by index_uid for quick lookup
|
||||||
|
let mut remote_results_by_index = std::collections::BTreeMap::new();
|
||||||
|
for remote_result in &remote_results {
|
||||||
|
if let Some(remote_metadata) = &remote_result.metadata {
|
||||||
|
for remote_meta in remote_metadata {
|
||||||
|
remote_results_by_index.insert(remote_meta.index_uid.clone(), remote_meta.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build metadata in the same order as the original queries
|
||||||
|
for original_query in original_queries {
|
||||||
|
let query_uid = Uuid::now_v7();
|
||||||
|
let index_uid = original_query.index_uid.to_string();
|
||||||
|
|
||||||
|
// Determine if this is a remote query
|
||||||
|
let (_, _, federation_options) = original_query.into_index_query_federation();
|
||||||
|
let remote = federation_options.and_then(|options| options.remote);
|
||||||
|
|
||||||
|
// Get primary key for this index
|
||||||
|
let mut primary_key = None;
|
||||||
|
|
||||||
|
if remote.is_some() {
|
||||||
|
// For remote queries, try to get primary key from remote results
|
||||||
|
if let Some(remote_meta) = remote_results_by_index.get(&index_uid) {
|
||||||
|
primary_key = remote_meta.primary_key.clone();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// For local queries, get primary key from local index
|
||||||
|
primary_key = index_scheduler.index(&index_uid).ok().and_then(|index| {
|
||||||
|
index.read_txn().ok().and_then(|rtxn| {
|
||||||
|
let pk = index.primary_key(&rtxn).ok().flatten().map(|pk| pk.to_string());
|
||||||
|
drop(rtxn);
|
||||||
|
pk
|
||||||
|
})
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
query_metadata.push(SearchMetadata { query_uid, index_uid, primary_key, remote });
|
||||||
|
}
|
||||||
|
|
||||||
// 3.2. merge hits
|
// 3.2. merge hits
|
||||||
let merged_hits: Vec<_> = merge_index_global_results(results_by_index, &mut remote_results)
|
let merged_hits: Vec<_> = merge_index_global_results(results_by_index, &mut remote_results)
|
||||||
.skip(federation.offset)
|
.skip(federation.offset)
|
||||||
|
@@ -842,6 +842,10 @@ pub struct SearchHit {
|
|||||||
pub struct SearchMetadata {
|
pub struct SearchMetadata {
|
||||||
pub query_uid: Uuid,
|
pub query_uid: Uuid,
|
||||||
pub index_uid: String,
|
pub index_uid: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub primary_key: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub remote: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Clone, PartialEq, ToSchema)]
|
#[derive(Serialize, Clone, PartialEq, ToSchema)]
|
||||||
@@ -1167,6 +1171,9 @@ pub fn perform_search(
|
|||||||
semantic_hit_count,
|
semantic_hit_count,
|
||||||
) = search_from_kind(index_uid.clone(), search_kind, search)?;
|
) = search_from_kind(index_uid.clone(), search_kind, search)?;
|
||||||
|
|
||||||
|
let query_uid = Uuid::now_v7();
|
||||||
|
let primary_key = index.primary_key(&rtxn)?.map(|pk| pk.to_string());
|
||||||
|
|
||||||
let SearchQuery {
|
let SearchQuery {
|
||||||
q,
|
q,
|
||||||
limit,
|
limit,
|
||||||
@@ -1248,8 +1255,6 @@ pub fn perform_search(
|
|||||||
.transpose()?
|
.transpose()?
|
||||||
.map(|ComputedFacets { distribution, stats }| (distribution, stats))
|
.map(|ComputedFacets { distribution, stats }| (distribution, stats))
|
||||||
.unzip();
|
.unzip();
|
||||||
|
|
||||||
let query_uid = Uuid::now_v7();
|
|
||||||
let result = SearchResult {
|
let result = SearchResult {
|
||||||
hits: documents,
|
hits: documents,
|
||||||
hits_info,
|
hits_info,
|
||||||
@@ -1262,7 +1267,12 @@ pub fn perform_search(
|
|||||||
used_negative_operator,
|
used_negative_operator,
|
||||||
semantic_hit_count,
|
semantic_hit_count,
|
||||||
request_uid: Some(request_uid),
|
request_uid: Some(request_uid),
|
||||||
metadata: Some(SearchMetadata { query_uid, index_uid: index_uid_for_metadata }),
|
metadata: Some(SearchMetadata {
|
||||||
|
query_uid,
|
||||||
|
index_uid: index_uid_for_metadata,
|
||||||
|
primary_key,
|
||||||
|
remote: None, // Local searches don't have a remote
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user