Compare commits

..

2 Commits

Author SHA1 Message Date
Tamo
0a34f70566 update the CI 2023-10-26 13:39:52 +02:00
Tamo
75d8d4f3a8 remove the use unstable in rustfmt 2023-10-26 13:38:11 +02:00
22 changed files with 35 additions and 476 deletions

View File

@@ -90,8 +90,7 @@ jobs:
set -x set -x
export base_ref=$(git merge-base origin/main ${{ steps.comment-branch.outputs.head_ref }} | head -c8) export base_ref=$(git merge-base origin/main ${{ steps.comment-branch.outputs.head_ref }} | head -c8)
export base_filename=$(echo ${{ steps.command.outputs.command-arguments }}_main_${base_ref}.json) export base_filename=$(echo ${{ steps.command.outputs.command-arguments }}_main_${base_ref}.json)
export bench_name=$(echo ${{ steps.command.outputs.command-arguments }}) echo 'Here are your benchmarks diff 👊' >> body.txt
echo "Here are your $bench_name benchmarks diff 👊" >> body.txt
echo '```' >> body.txt echo '```' >> body.txt
./benchmarks/scripts/compare.sh $base_filename ${{ steps.file.outputs.basename }}.json >> body.txt ./benchmarks/scripts/compare.sh $base_filename ${{ steps.file.outputs.basename }}.json >> body.txt
echo '```' >> body.txt echo '```' >> body.txt

View File

@@ -50,7 +50,7 @@ jobs:
needs: check-version needs: check-version
steps: steps:
- name: Create PR to Homebrew - name: Create PR to Homebrew
uses: mislav/bump-homebrew-formula-action@v3 uses: mislav/bump-homebrew-formula-action@v2
with: with:
formula-name: meilisearch formula-name: meilisearch
formula-path: Formula/m/meilisearch.rb formula-path: Formula/m/meilisearch.rb

View File

@@ -63,7 +63,7 @@ jobs:
uses: docker/setup-buildx-action@v3 uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@v3 uses: docker/login-action@v2
with: with:
username: ${{ secrets.DOCKERHUB_USERNAME }} username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }} password: ${{ secrets.DOCKERHUB_TOKEN }}

View File

@@ -160,7 +160,7 @@ jobs:
with: with:
repository: meilisearch/meilisearch-js repository: meilisearch/meilisearch-js
- name: Setup node - name: Setup node
uses: actions/setup-node@v4 uses: actions/setup-node@v3
with: with:
cache: 'yarn' cache: 'yarn'
- name: Install dependencies - name: Install dependencies
@@ -318,7 +318,7 @@ jobs:
with: with:
repository: meilisearch/meilisearch-js-plugins repository: meilisearch/meilisearch-js-plugins
- name: Setup node - name: Setup node
uses: actions/setup-node@v4 uses: actions/setup-node@v3
with: with:
cache: yarn cache: yarn
- name: Install dependencies - name: Install dependencies

View File

@@ -43,7 +43,7 @@ jobs:
toolchain: nightly toolchain: nightly
override: true override: true
- name: Cache dependencies - name: Cache dependencies
uses: Swatinem/rust-cache@v2.7.1 uses: Swatinem/rust-cache@v2.6.2
- name: Run cargo check without any default features - name: Run cargo check without any default features
uses: actions-rs/cargo@v1 uses: actions-rs/cargo@v1
with: with:
@@ -65,7 +65,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: Cache dependencies - name: Cache dependencies
uses: Swatinem/rust-cache@v2.7.1 uses: Swatinem/rust-cache@v2.6.2
- name: Run cargo check without any default features - name: Run cargo check without any default features
uses: actions-rs/cargo@v1 uses: actions-rs/cargo@v1
with: with:
@@ -149,7 +149,7 @@ jobs:
toolchain: stable toolchain: stable
override: true override: true
- name: Cache dependencies - name: Cache dependencies
uses: Swatinem/rust-cache@v2.7.1 uses: Swatinem/rust-cache@v2.6.2
- name: Run tests in debug - name: Run tests in debug
uses: actions-rs/cargo@v1 uses: actions-rs/cargo@v1
with: with:
@@ -168,7 +168,7 @@ jobs:
override: true override: true
components: clippy components: clippy
- name: Cache dependencies - name: Cache dependencies
uses: Swatinem/rust-cache@v2.7.1 uses: Swatinem/rust-cache@v2.6.2
- name: Run cargo clippy - name: Run cargo clippy
uses: actions-rs/cargo@v1 uses: actions-rs/cargo@v1
with: with:
@@ -183,11 +183,11 @@ jobs:
- uses: actions-rs/toolchain@v1 - uses: actions-rs/toolchain@v1
with: with:
profile: minimal profile: minimal
toolchain: nightly toolchain: 1.71.1
override: true override: true
components: rustfmt components: rustfmt
- name: Cache dependencies - name: Cache dependencies
uses: Swatinem/rust-cache@v2.7.1 uses: Swatinem/rust-cache@v2.6.2
- name: Run cargo fmt - name: Run cargo fmt
# Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file. # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
# Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate

View File

@@ -1,5 +1,3 @@
unstable_features = true
use_small_heuristics = "max" use_small_heuristics = "max"
imports_granularity = "Module" imports_granularity = "Module"
group_imports = "StdExternalCrate" group_imports = "StdExternalCrate"

21
Cargo.lock generated
View File

@@ -231,9 +231,9 @@ dependencies = [
[[package]] [[package]]
name = "addr2line" name = "addr2line"
version = "0.21.0" version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
dependencies = [ dependencies = [
"gimli", "gimli",
] ]
@@ -435,9 +435,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]] [[package]]
name = "backtrace" name = "backtrace"
version = "0.3.69" version = "0.3.68"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
dependencies = [ dependencies = [
"addr2line", "addr2line",
"cc", "cc",
@@ -1638,9 +1638,9 @@ dependencies = [
[[package]] [[package]]
name = "gimli" name = "gimli"
version = "0.28.0" version = "0.27.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
[[package]] [[package]]
name = "git2" name = "git2"
@@ -1894,7 +1894,6 @@ name = "index-scheduler"
version = "1.4.1" version = "1.4.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"backtrace",
"big_s", "big_s",
"bincode", "bincode",
"crossbeam", "crossbeam",
@@ -2857,9 +2856,9 @@ dependencies = [
[[package]] [[package]]
name = "object" name = "object"
version = "0.32.1" version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@@ -3641,9 +3640,9 @@ dependencies = [
[[package]] [[package]]
name = "serde_json" name = "serde_json"
version = "1.0.108" version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
dependencies = [ dependencies = [
"indexmap 2.0.0", "indexmap 2.0.0",
"itoa", "itoa",

View File

@@ -28,7 +28,6 @@ license = "MIT"
[profile.release] [profile.release]
codegen-units = 1 codegen-units = 1
debug = true
[profile.dev.package.flate2] [profile.dev.package.flate2]
opt-level = 3 opt-level = 3

View File

@@ -25,6 +25,12 @@
<p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p> <p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
---
### 🔥 On November 2nd, we are hosting our first-ever live demo and product updates for [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). Make sure to [register here](https://us06web.zoom.us/meeting/register/tZMlc-mqrjIsH912-HTRe-AaT-pp41bDe81a#/registration) and bring your questions for live Q&A!
---
Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow. Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
<p align="center" name="demo"> <p align="center" name="demo">

View File

@@ -12,7 +12,6 @@ license.workspace = true
[dependencies] [dependencies]
anyhow = "1.0.70" anyhow = "1.0.70"
backtrace = "0.3.69"
bincode = "1.3.3" bincode = "1.3.3"
csv = "1.2.1" csv = "1.2.1"
derive_builder = "0.12.0" derive_builder = "0.12.0"

View File

@@ -825,10 +825,6 @@ impl IndexScheduler {
// 2. dump the tasks // 2. dump the tasks
let mut dump_tasks = dump.create_tasks_queue()?; let mut dump_tasks = dump.create_tasks_queue()?;
for ret in self.all_tasks.iter(&rtxn)? { for ret in self.all_tasks.iter(&rtxn)? {
if self.must_stop_processing.get() {
return Err(Error::AbortedTask);
}
let (_, mut t) = ret?; let (_, mut t) = ret?;
let status = t.status; let status = t.status;
let content_file = t.content_uuid(); let content_file = t.content_uuid();
@@ -849,9 +845,6 @@ impl IndexScheduler {
// 2.1. Dump the `content_file` associated with the task if there is one and the task is not finished yet. // 2.1. Dump the `content_file` associated with the task if there is one and the task is not finished yet.
if let Some(content_file) = content_file { if let Some(content_file) = content_file {
if self.must_stop_processing.get() {
return Err(Error::AbortedTask);
}
if status == Status::Enqueued { if status == Status::Enqueued {
let content_file = self.file_store.get_update(content_file)?; let content_file = self.file_store.get_update(content_file)?;
@@ -891,9 +884,6 @@ impl IndexScheduler {
// 3.1. Dump the documents // 3.1. Dump the documents
for ret in index.all_documents(&rtxn)? { for ret in index.all_documents(&rtxn)? {
if self.must_stop_processing.get() {
return Err(Error::AbortedTask);
}
let (_id, doc) = ret?; let (_id, doc) = ret?;
let document = milli::obkv_to_json(&all_fields, &fields_ids_map, doc)?; let document = milli::obkv_to_json(&all_fields, &fields_ids_map, doc)?;
index_dumper.push_document(&document)?; index_dumper.push_document(&document)?;
@@ -913,9 +903,6 @@ impl IndexScheduler {
"[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]"
)).unwrap(); )).unwrap();
if self.must_stop_processing.get() {
return Err(Error::AbortedTask);
}
let path = self.dumps_path.join(format!("{}.dump", dump_uid)); let path = self.dumps_path.join(format!("{}.dump", dump_uid));
let file = File::create(path)?; let file = File::create(path)?;
dump.persist_to(BufWriter::new(file))?; dump.persist_to(BufWriter::new(file))?;

View File

@@ -108,8 +108,6 @@ pub enum Error {
TaskDeletionWithEmptyQuery, TaskDeletionWithEmptyQuery,
#[error("Query parameters to filter the tasks to cancel are missing. Available query parameters are: `uids`, `indexUids`, `statuses`, `types`, `canceledBy`, `beforeEnqueuedAt`, `afterEnqueuedAt`, `beforeStartedAt`, `afterStartedAt`, `beforeFinishedAt`, `afterFinishedAt`.")] #[error("Query parameters to filter the tasks to cancel are missing. Available query parameters are: `uids`, `indexUids`, `statuses`, `types`, `canceledBy`, `beforeEnqueuedAt`, `afterEnqueuedAt`, `beforeStartedAt`, `afterStartedAt`, `beforeFinishedAt`, `afterFinishedAt`.")]
TaskCancelationWithEmptyQuery, TaskCancelationWithEmptyQuery,
#[error("Aborted task")]
AbortedTask,
#[error(transparent)] #[error(transparent)]
Dump(#[from] dump::Error), Dump(#[from] dump::Error),
@@ -117,13 +115,8 @@ pub enum Error {
Heed(#[from] heed::Error), Heed(#[from] heed::Error),
#[error(transparent)] #[error(transparent)]
Milli(#[from] milli::Error), Milli(#[from] milli::Error),
#[error("An unexpected crash occurred when processing the task. {}", { #[error("An unexpected crash occurred when processing the task.")]
match .0 { ProcessBatchPanicked,
Some(report) => format!("Get /reports/{}", report),
None => "No report was saved.".into(),
}
})]
ProcessBatchPanicked(Option<uuid::Uuid>),
#[error(transparent)] #[error(transparent)]
FileStore(#[from] file_store::Error), FileStore(#[from] file_store::Error),
#[error(transparent)] #[error(transparent)]
@@ -182,11 +175,10 @@ impl Error {
| Error::TaskNotFound(_) | Error::TaskNotFound(_)
| Error::TaskDeletionWithEmptyQuery | Error::TaskDeletionWithEmptyQuery
| Error::TaskCancelationWithEmptyQuery | Error::TaskCancelationWithEmptyQuery
| Error::AbortedTask
| Error::Dump(_) | Error::Dump(_)
| Error::Heed(_) | Error::Heed(_)
| Error::Milli(_) | Error::Milli(_)
| Error::ProcessBatchPanicked(_) | Error::ProcessBatchPanicked
| Error::FileStore(_) | Error::FileStore(_)
| Error::IoError(_) | Error::IoError(_)
| Error::Persist(_) | Error::Persist(_)
@@ -229,7 +221,7 @@ impl ErrorCode for Error {
Error::NoSpaceLeftInTaskQueue => Code::NoSpaceLeftOnDevice, Error::NoSpaceLeftInTaskQueue => Code::NoSpaceLeftOnDevice,
Error::Dump(e) => e.error_code(), Error::Dump(e) => e.error_code(),
Error::Milli(e) => e.error_code(), Error::Milli(e) => e.error_code(),
Error::ProcessBatchPanicked(_) => Code::Internal, Error::ProcessBatchPanicked => Code::Internal,
Error::Heed(e) => e.error_code(), Error::Heed(e) => e.error_code(),
Error::HeedTransaction(e) => e.error_code(), Error::HeedTransaction(e) => e.error_code(),
Error::FileStore(e) => e.error_code(), Error::FileStore(e) => e.error_code(),
@@ -244,9 +236,6 @@ impl ErrorCode for Error {
Error::TaskDatabaseUpdate(_) => Code::Internal, Error::TaskDatabaseUpdate(_) => Code::Internal,
Error::CreateBatch(_) => Code::Internal, Error::CreateBatch(_) => Code::Internal,
// This one should never be seen by the end user
Error::AbortedTask => Code::Internal,
#[cfg(test)] #[cfg(test)]
Error::PlannedFailure => Code::Internal, Error::PlannedFailure => Code::Internal,
} }

View File

@@ -39,7 +39,6 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
test_breakpoint_sdr: _, test_breakpoint_sdr: _,
planned_failures: _, planned_failures: _,
run_loop_iteration: _, run_loop_iteration: _,
panic_reader: _,
} = scheduler; } = scheduler;
let rtxn = env.read_txn().unwrap(); let rtxn = env.read_txn().unwrap();

View File

@@ -26,7 +26,6 @@ mod index_mapper;
#[cfg(test)] #[cfg(test)]
mod insta_snapshot; mod insta_snapshot;
mod lru; mod lru;
mod panic_hook;
mod utils; mod utils;
mod uuid_codec; mod uuid_codec;
@@ -54,8 +53,6 @@ use meilisearch_types::milli::documents::DocumentsBatchBuilder;
use meilisearch_types::milli::update::IndexerConfig; use meilisearch_types::milli::update::IndexerConfig;
use meilisearch_types::milli::{self, CboRoaringBitmapCodec, Index, RoaringBitmapCodec, BEU32}; use meilisearch_types::milli::{self, CboRoaringBitmapCodec, Index, RoaringBitmapCodec, BEU32};
use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task}; use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
use panic_hook::ReportReader;
pub use panic_hook::{Panic, Report, ReportRegistry};
use puffin::FrameView; use puffin::FrameView;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use synchronoise::SignalEvent; use synchronoise::SignalEvent;
@@ -334,8 +331,6 @@ pub struct IndexScheduler {
/// The path to the version file of Meilisearch. /// The path to the version file of Meilisearch.
pub(crate) version_file_path: PathBuf, pub(crate) version_file_path: PathBuf,
pub(crate) panic_reader: ReportReader,
// ================= test // ================= test
// The next entry is dedicated to the tests. // The next entry is dedicated to the tests.
/// Provide a way to set a breakpoint in multiple part of the scheduler. /// Provide a way to set a breakpoint in multiple part of the scheduler.
@@ -386,7 +381,6 @@ impl IndexScheduler {
#[cfg(test)] #[cfg(test)]
run_loop_iteration: self.run_loop_iteration.clone(), run_loop_iteration: self.run_loop_iteration.clone(),
features: self.features.clone(), features: self.features.clone(),
panic_reader: self.panic_reader.clone(),
} }
} }
} }
@@ -444,12 +438,6 @@ impl IndexScheduler {
let finished_at = env.create_database(&mut wtxn, Some(db_name::FINISHED_AT))?; let finished_at = env.create_database(&mut wtxn, Some(db_name::FINISHED_AT))?;
wtxn.commit()?; wtxn.commit()?;
const MAX_REPORT_COUNT: usize = 20;
let panic_reader = panic_hook::ReportReader::install_panic_hook(
std::num::NonZeroUsize::new(MAX_REPORT_COUNT).unwrap(),
);
// allow unreachable_code to get rids of the warning in the case of a test build. // allow unreachable_code to get rids of the warning in the case of a test build.
let this = Self { let this = Self {
must_stop_processing: MustStopProcessing::default(), must_stop_processing: MustStopProcessing::default(),
@@ -490,7 +478,6 @@ impl IndexScheduler {
#[cfg(test)] #[cfg(test)]
run_loop_iteration: Arc::new(RwLock::new(0)), run_loop_iteration: Arc::new(RwLock::new(0)),
features, features,
panic_reader,
}; };
this.run(); this.run();
@@ -1143,10 +1130,7 @@ impl IndexScheduler {
.name(String::from("batch-operation")) .name(String::from("batch-operation"))
.spawn(move || cloned_index_scheduler.process_batch(batch)) .spawn(move || cloned_index_scheduler.process_batch(batch))
.unwrap(); .unwrap();
handle.join().unwrap_or(Err(Error::ProcessBatchPanicked))
self.panic_reader
.join_thread(handle)
.unwrap_or_else(|maybe_report| Err(Error::ProcessBatchPanicked(maybe_report)))
}; };
#[cfg(test)] #[cfg(test)]
@@ -1183,8 +1167,7 @@ impl IndexScheduler {
// If we have an abortion error we must stop the tick here and re-schedule tasks. // If we have an abortion error we must stop the tick here and re-schedule tasks.
Err(Error::Milli(milli::Error::InternalError( Err(Error::Milli(milli::Error::InternalError(
milli::InternalError::AbortedIndexation, milli::InternalError::AbortedIndexation,
))) ))) => {
| Err(Error::AbortedTask) => {
#[cfg(test)] #[cfg(test)]
self.breakpoint(Breakpoint::AbortedIndexation); self.breakpoint(Breakpoint::AbortedIndexation);
wtxn.abort().map_err(Error::HeedTransaction)?; wtxn.abort().map_err(Error::HeedTransaction)?;
@@ -1327,10 +1310,6 @@ impl IndexScheduler {
} }
} }
pub fn reports(&self) -> Arc<RwLock<ReportRegistry>> {
self.panic_reader.registry()
}
/// Blocks the thread until the test handle asks to progress to/through this breakpoint. /// Blocks the thread until the test handle asks to progress to/through this breakpoint.
/// ///
/// Two messages are sent through the channel for each breakpoint. /// Two messages are sent through the channel for each breakpoint.
@@ -4344,26 +4323,4 @@ mod tests {
} }
"###); "###);
} }
#[test]
fn cancel_processing_dump() {
let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
let dump_creation = KindWithContent::DumpCreation { keys: Vec::new(), instance_uid: None };
let dump_cancellation = KindWithContent::TaskCancelation {
query: "cancel dump".to_owned(),
tasks: RoaringBitmap::from_iter([0]),
};
let _ = index_scheduler.register(dump_creation).unwrap();
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_dump_register");
handle.advance_till([Start, BatchCreated, InsideProcessBatch]);
let _ = index_scheduler.register(dump_cancellation).unwrap();
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "cancel_registered");
snapshot!(format!("{:?}", handle.advance()), @"AbortedIndexation");
handle.advance_one_successful_batch();
snapshot!(snapshot_index_scheduler(&index_scheduler), name: "cancel_processed");
}
} }

View File

@@ -1,211 +0,0 @@
//! Panic hook designed to fetch a panic from a subthread and recover it on join.
use std::collections::VecDeque;
use std::num::NonZeroUsize;
use std::panic::PanicInfo;
use std::sync::{Arc, RwLock};
use std::thread::{JoinHandle, ThreadId};
use backtrace::Backtrace;
// Represents a panic in a shallowy structured fashion
pub struct Panic {
pub payload: Option<String>,
pub location: Option<String>,
pub thread_name: Option<String>,
pub thread_id: ThreadId,
pub backtrace: Backtrace,
}
/// A panic enriched with a unique id
#[derive(serde::Serialize)]
pub struct Report {
pub id: uuid::Uuid,
#[serde(serialize_with = "serialize_panic")]
pub panic: Panic,
}
fn serialize_panic<S>(panic: &Panic, s: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
use serde::Serialize;
panic.to_json().serialize(s)
}
impl Report {
pub fn new(panic: Panic) -> Self {
Self { id: uuid::Uuid::new_v4(), panic }
}
}
impl Panic {
pub fn to_json(&self) -> serde_json::Value {
json::panic_to_json(self)
}
}
mod json {
use backtrace::{Backtrace, BacktraceFrame, BacktraceSymbol};
use serde_json::{json, Value};
use super::Panic;
fn symbol_to_json(symbol: &BacktraceSymbol) -> Value {
let address = symbol.addr().map(|addr| format!("{:p}", addr));
let column = symbol.colno();
let line = symbol.lineno();
let function = symbol.name().map(|name| name.to_string());
let filename = symbol.filename();
json!({
"function": function,
"filename": filename,
"line": line,
"column": column,
"address": address,
})
}
fn frame_to_json(frame: &BacktraceFrame) -> Value {
let symbols: Vec<_> = frame.symbols().iter().map(symbol_to_json).collect();
match symbols.as_slice() {
[] => {
let address = format!("{:p}", frame.ip());
json!({"address": address})
}
[symbol] => json!(symbol),
symbols => json!(symbols),
}
}
fn backtrace_to_json(backtrace: &Backtrace) -> Value {
let frames: Vec<_> = backtrace.frames().iter().map(frame_to_json).collect();
json!(frames)
}
pub fn panic_to_json(panic: &Panic) -> Value {
let thread_id = format!("{:?}", panic.thread_id);
serde_json::json!({
"payload": panic.payload,
"location": panic.location,
"thread": {
"id": thread_id,
"name": panic.thread_name,
},
"backtrace": backtrace_to_json(&panic.backtrace),
})
}
}
struct ReportWriter(Arc<RwLock<ReportRegistry>>);
/// A FIFO queue of reports.
pub struct ReportRegistry {
reports: std::collections::VecDeque<Report>,
}
impl ReportRegistry {
pub fn new(capacity: NonZeroUsize) -> Self {
Self { reports: VecDeque::with_capacity(capacity.get()) }
}
pub fn push(&mut self, report: Report) -> Option<Report> {
let popped = if self.reports.len() == self.reports.capacity() {
self.reports.pop_back()
} else {
None
};
self.reports.push_front(report);
popped
}
pub fn iter(&self) -> impl Iterator<Item = &Report> {
self.reports.iter()
}
pub fn find(&self, report_id: uuid::Uuid) -> Option<&Report> {
self.iter().find(|report| report.id == report_id)
}
}
impl ReportWriter {
#[track_caller]
fn write_panic(&self, panic_info: &PanicInfo<'_>) {
let payload = panic_info
.payload()
.downcast_ref::<&str>()
.map(ToString::to_string)
.or_else(|| panic_info.payload().downcast_ref::<String>().cloned());
let location = panic_info.location().map(|loc| {
format!(
"{file}:{line}:{column}",
file = loc.file(),
line = loc.line(),
column = loc.column()
)
});
let thread_name = std::thread::current().name().map(ToString::to_string);
let thread_id = std::thread::current().id();
let backtrace = backtrace::Backtrace::new();
let panic = Panic { payload, location, thread_name, thread_id, backtrace };
let report = Report::new(panic);
log::error!(
"An unexpected panic occurred on thread {name} at {location}: {payload}. See report '{report}' for details.",
payload = report.panic.payload.as_deref().unwrap_or("Box<dyn Any>"),
name = report.panic.thread_name.as_deref().unwrap_or("<unnamed>"),
location = report.panic.location.as_deref().unwrap_or("<unknown>"),
report = report.id,
);
if let Ok(mut registry) = self.0.write() {
if let Some(old_report) = registry.push(report) {
log::trace!("Forgetting report {} to make space for new report.", old_report.id)
}
}
}
}
/// Reads the reports written in case of a panic.
#[derive(Clone)]
pub struct ReportReader(Arc<RwLock<ReportRegistry>>);
impl ReportReader {
/// Installs a new global panic hook, overriding any existing hook.
///
/// The hook writes any incoming panic in reports.
/// The reports can then be read by the returned [`ReportReader`].
pub fn install_panic_hook(capacity: NonZeroUsize) -> Self {
let registry = Arc::new(RwLock::new(ReportRegistry::new(capacity)));
let reader = ReportReader(registry.clone());
let writer = ReportWriter(registry.clone());
std::panic::set_hook(Box::new(move |panic_info| writer.write_panic(panic_info)));
reader
}
/// Join the thread corresponding to the passed handle, recovering either its value
/// or, in case the thread panicked, the id of the report corresponding to the panic.
///
/// The id can be used to read the report from the [`self.registry()`].
pub fn join_thread<T>(&self, thread: JoinHandle<T>) -> Result<T, Option<uuid::Uuid>> {
let thread_id = thread.thread().id();
thread.join().map_err(|_e| {
self.0
.read()
.unwrap()
.iter()
.find(|report| report.panic.thread_id == thread_id)
.map(|report| report.id)
})
}
/// Returns a registry that can be used to read the reports written during a panic.
pub fn registry(&self) -> Arc<RwLock<ReportRegistry>> {
self.0.clone()
}
}

View File

@@ -1,35 +0,0 @@
---
source: index-scheduler/src/lib.rs
---
### Autobatching Enabled = true
### Processing Tasks:
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, status: enqueued, details: { dump_uid: None }, kind: DumpCreation { keys: [], instance_uid: None }}
----------------------------------------------------------------------
### Status:
enqueued [0,]
----------------------------------------------------------------------
### Kind:
"dumpCreation" [0,]
----------------------------------------------------------------------
### Index Tasks:
----------------------------------------------------------------------
### Index Mapper:
----------------------------------------------------------------------
### Canceled By:
----------------------------------------------------------------------
### Enqueued At:
[timestamp] [0,]
----------------------------------------------------------------------
### Started At:
----------------------------------------------------------------------
### Finished At:
----------------------------------------------------------------------
### File Store:
----------------------------------------------------------------------

View File

@@ -1,45 +0,0 @@
---
source: index-scheduler/src/lib.rs
---
### Autobatching Enabled = true
### Processing Tasks:
[]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, status: canceled, canceled_by: 1, details: { dump_uid: None }, kind: DumpCreation { keys: [], instance_uid: None }}
1 {uid: 1, status: succeeded, details: { matched_tasks: 1, canceled_tasks: Some(0), original_filter: "cancel dump" }, kind: TaskCancelation { query: "cancel dump", tasks: RoaringBitmap<[0]> }}
----------------------------------------------------------------------
### Status:
enqueued []
succeeded [1,]
canceled [0,]
----------------------------------------------------------------------
### Kind:
"taskCancelation" [1,]
"dumpCreation" [0,]
----------------------------------------------------------------------
### Index Tasks:
----------------------------------------------------------------------
### Index Mapper:
----------------------------------------------------------------------
### Canceled By:
1 [0,]
----------------------------------------------------------------------
### Enqueued At:
[timestamp] [0,]
[timestamp] [1,]
----------------------------------------------------------------------
### Started At:
[timestamp] [0,]
[timestamp] [1,]
----------------------------------------------------------------------
### Finished At:
[timestamp] [0,]
[timestamp] [1,]
----------------------------------------------------------------------
### File Store:
----------------------------------------------------------------------

View File

@@ -1,38 +0,0 @@
---
source: index-scheduler/src/lib.rs
---
### Autobatching Enabled = true
### Processing Tasks:
[0,]
----------------------------------------------------------------------
### All Tasks:
0 {uid: 0, status: enqueued, details: { dump_uid: None }, kind: DumpCreation { keys: [], instance_uid: None }}
1 {uid: 1, status: enqueued, details: { matched_tasks: 1, canceled_tasks: None, original_filter: "cancel dump" }, kind: TaskCancelation { query: "cancel dump", tasks: RoaringBitmap<[0]> }}
----------------------------------------------------------------------
### Status:
enqueued [0,1,]
----------------------------------------------------------------------
### Kind:
"taskCancelation" [1,]
"dumpCreation" [0,]
----------------------------------------------------------------------
### Index Tasks:
----------------------------------------------------------------------
### Index Mapper:
----------------------------------------------------------------------
### Canceled By:
----------------------------------------------------------------------
### Enqueued At:
[timestamp] [0,]
[timestamp] [1,]
----------------------------------------------------------------------
### Started At:
----------------------------------------------------------------------
### Finished At:
----------------------------------------------------------------------
### File Store:
----------------------------------------------------------------------

View File

@@ -88,6 +88,7 @@ pub trait ErrorCode {
} }
} }
#[allow(clippy::enum_variant_names)]
enum ErrorType { enum ErrorType {
Internal, Internal,
InvalidRequest, InvalidRequest,
@@ -297,7 +298,6 @@ MissingSwapIndexes , InvalidRequest , BAD_REQUEST ;
MissingTaskFilters , InvalidRequest , BAD_REQUEST ; MissingTaskFilters , InvalidRequest , BAD_REQUEST ;
NoSpaceLeftOnDevice , System , UNPROCESSABLE_ENTITY; NoSpaceLeftOnDevice , System , UNPROCESSABLE_ENTITY;
PayloadTooLarge , InvalidRequest , PAYLOAD_TOO_LARGE ; PayloadTooLarge , InvalidRequest , PAYLOAD_TOO_LARGE ;
ReportNotFound , InvalidRequest , NOT_FOUND ;
TaskNotFound , InvalidRequest , NOT_FOUND ; TaskNotFound , InvalidRequest , NOT_FOUND ;
TooManyOpenFiles , System , UNPROCESSABLE_ENTITY ; TooManyOpenFiles , System , UNPROCESSABLE_ENTITY ;
UnretrievableDocument , Internal , BAD_REQUEST ; UnretrievableDocument , Internal , BAD_REQUEST ;

View File

@@ -51,8 +51,6 @@ pub enum MeilisearchHttpError {
DocumentFormat(#[from] DocumentFormatError), DocumentFormat(#[from] DocumentFormatError),
#[error(transparent)] #[error(transparent)]
Join(#[from] JoinError), Join(#[from] JoinError),
#[error("Report `{0}` not found. Either its id is incorrect, or it was deleted. To save on memory, only a limited amount of reports are kept.")]
ReportNotFound(uuid::Uuid),
} }
impl ErrorCode for MeilisearchHttpError { impl ErrorCode for MeilisearchHttpError {
@@ -76,7 +74,6 @@ impl ErrorCode for MeilisearchHttpError {
MeilisearchHttpError::FileStore(_) => Code::Internal, MeilisearchHttpError::FileStore(_) => Code::Internal,
MeilisearchHttpError::DocumentFormat(e) => e.error_code(), MeilisearchHttpError::DocumentFormat(e) => e.error_code(),
MeilisearchHttpError::Join(_) => Code::Internal, MeilisearchHttpError::Join(_) => Code::Internal,
MeilisearchHttpError::ReportNotFound(_) => Code::ReportNotFound,
} }
} }
} }

View File

@@ -24,7 +24,6 @@ pub mod features;
pub mod indexes; pub mod indexes;
mod metrics; mod metrics;
mod multi_search; mod multi_search;
mod reports;
mod snapshot; mod snapshot;
mod swap_indexes; mod swap_indexes;
pub mod tasks; pub mod tasks;
@@ -41,8 +40,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
.service(web::scope("/multi-search").configure(multi_search::configure)) .service(web::scope("/multi-search").configure(multi_search::configure))
.service(web::scope("/swap-indexes").configure(swap_indexes::configure)) .service(web::scope("/swap-indexes").configure(swap_indexes::configure))
.service(web::scope("/metrics").configure(metrics::configure)) .service(web::scope("/metrics").configure(metrics::configure))
.service(web::scope("/experimental-features").configure(features::configure)) .service(web::scope("/experimental-features").configure(features::configure));
.service(web::scope("/reports").configure(reports::configure));
} }
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]

View File

@@ -1,39 +0,0 @@
use actix_web::web::{self, Data};
use actix_web::HttpResponse;
use index_scheduler::{IndexScheduler, Report};
use meilisearch_types::error::ResponseError;
use meilisearch_types::keys::actions;
use crate::extractors::authentication::policies::ActionPolicy;
use crate::extractors::authentication::GuardedData;
use crate::extractors::sequential_extractor::SeqHandler;
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("").route(web::get().to(list_reports))).service(
web::scope("/{report_uid}")
.service(web::resource("").route(web::get().to(SeqHandler(get_report)))),
);
}
pub async fn list_reports(
index_scheduler: GuardedData<ActionPolicy<{ actions::SETTINGS_ALL }>, Data<IndexScheduler>>,
) -> Result<HttpResponse, ResponseError> {
let reports = &index_scheduler.reports();
let reports = &reports.read().unwrap();
let reports: Vec<&Report> = reports.iter().collect();
Ok(HttpResponse::Ok().json(reports))
}
pub async fn get_report(
index_scheduler: GuardedData<ActionPolicy<{ actions::SETTINGS_ALL }>, Data<IndexScheduler>>,
report_id: web::Path<uuid::Uuid>,
) -> Result<HttpResponse, ResponseError> {
let reports = &index_scheduler.reports();
let reports = &reports.read().unwrap();
let report = reports
.find(*report_id)
.ok_or(crate::error::MeilisearchHttpError::ReportNotFound(*report_id))?;
Ok(HttpResponse::Ok().json(report))
}